Update develop-ref after #2701

.github/jobs/docker_utils.py

@@ -39,10 +39,13 @@ def get_dockerhub_url(branch_name):
 def docker_get_volumes_last_updated(current_branch):
     import requests
     dockerhub_url = get_dockerhub_url(current_branch)
-    dockerhub_request = requests.get(dockerhub_url, timeout=60)
+    dockerhub_request = requests.get(dockerhub_url, timeout=90)
     if dockerhub_request.status_code != 200:
-        print(f"Could not find DockerHub URL: {dockerhub_url}")
-        return None
+        print(f"Retrying DockerHub request: {dockerhub_url}")
+        dockerhub_request = requests.get(dockerhub_url, timeout=60)
+        if dockerhub_request.status_code != 200:
+            print(f"Could not query DockerHub URL: {dockerhub_url}")
+            return None
 
     # get version number to search for if main_vX.Y branch
     if current_branch.startswith('main_v'):

.github/jobs/get_data_volumes.py

@@ -7,8 +7,6 @@
 
 import sys
 import os
-import subprocess
-import shlex
 
 from docker_utils import docker_get_volumes_last_updated, get_branch_name
 from docker_utils import get_data_repo, DOCKERHUB_METPLUS_DATA_DEV
@@ -54,7 +52,12 @@ def main(args):
         branch_name = branch_name[5:]
 
     # get all docker data volumes associated with current branch
-    available_volumes = docker_get_volumes_last_updated(branch_name).keys()
+    try:
+        available_volumes = docker_get_volumes_last_updated(branch_name).keys()
+    except AttributeError:
+        print("ERROR: Could not get available Docker data volumes "
+              f"for {branch_name}. Try rerunning failed jobs in GitHub Actions")
+        return None
 
     # loop through arguments and get data volume for each category
     for model_app_name in args:
@@ -78,7 +81,7 @@ def main(args):
             volume_name = f'{prefix}-{model_app_name}'
 
             # add output- to model app name
-            model_app_name=f'output-{model_app_name}'
+            model_app_name = f'output-{model_app_name}'
 
             # set DockerHub repo to dev version because all output data
             # should be in dev repository
@@ -108,11 +111,11 @@ def main(args):
 
 if __name__ == "__main__":
     # split up command line args that have commas before passing into main
-    args = []
+    arg_list = []
 
     for arg in sys.argv[1:]:
-        args.extend(arg.split(','))
-    out = main(args)
+        arg_list.extend(arg.split(','))
+    out = main(arg_list)
     if out is None:
         print("ERROR: Something went wrong")
         sys.exit(1)

.github/jobs/setup_and_run_use_cases.py

@@ -63,7 +63,7 @@ def main():
     # use BuildKit to build image
     os.environ['DOCKER_BUILDKIT'] = '1'
 
-    isOK = True
+    is_ok = True
     failed_use_cases = []
     for setup_commands, use_case_commands, requirements in all_commands:
         # get environment image tag
@@ -79,45 +79,38 @@ def main():
             f"-f {DOCKERFILE_DIR}/{dockerfile_name} ."
         )
 
-        print(f'Building Docker environment/branch image...')
+        print('Building Docker environment/branch image...')
         if not run_commands(docker_build_cmd):
-            isOK = False
+            is_ok = False
             continue
 
-        commands = []
-
-        # print list of existing docker images
-        commands.append('docker images')
-
+        # print list of existing docker images,
         # remove docker image after creating run env or prune untagged images
-        commands.append(f'docker image rm dtcenter/metplus-dev:{branch_name} -f')
-        commands.append('docker image prune -f')
-        run_commands(commands)
-
-        commands = []
-
-        # list docker images again after removal
-        commands.append('docker images')
-
-        # start interactive container in the background
-        commands.append(
-            f"docker run -d --rm -it -e GITHUB_WORKSPACE "
-            f"--name {RUN_TAG} "
-            f"{os.environ.get('NETWORK_ARG', '')} "
-            f"{' '.join(VOLUME_MOUNTS)} "
-            f"{volumes_from} --workdir {GITHUB_WORKSPACE} "
-            f'{RUN_TAG} bash'
-        )
-
-        # list running containers
-        commands.append('docker ps -a')
-
-        # execute setup commands in running docker container
-        commands.append(_format_docker_exec_command(setup_commands))
+        run_commands([
+            'docker images',
+            f'docker image rm dtcenter/metplus-dev:{branch_name} -f',
+            'docker image prune -f',
+        ])
+
+        # list docker images again after removal,
+        # start interactive container in the background,
+        # list running containers,
+        # then execute setup commands in running docker container
+        commands = [
+            'docker images',
+            (f"docker run -d --rm -it -e GITHUB_WORKSPACE "
+             f"--name {RUN_TAG} "
+             f"{os.environ.get('NETWORK_ARG', '')} "
+             f"{' '.join(VOLUME_MOUNTS)} "
+             f"{volumes_from} --workdir {GITHUB_WORKSPACE} "
+             f'{RUN_TAG} bash'),
+            'docker ps -a',
+            _format_docker_exec_command(setup_commands),
+        ]
 
         # run docker commands and skip running cases if something went wrong
         if not run_commands(commands):
-            isOK = False
+            is_ok = False
 
             # force remove container if setup step fails
             run_commands(f'docker rm -f {RUN_TAG}')
@@ -132,23 +125,28 @@ def main():
         for use_case_command in use_case_commands:
             if not run_commands(_format_docker_exec_command(use_case_command)):
                 failed_use_cases.append(use_case_command)
-                isOK = False
+                is_ok = False
 
         # print bashrc file to see what was added by setup commands
         # then force remove container to stop and remove it
         if not run_commands([
             _format_docker_exec_command('cat /root/.bashrc'),
             f'docker rm -f {RUN_TAG}',
         ]):
-            isOK = False
+            is_ok = False
 
     # print summary of use cases that failed
-    for failed_use_case in failed_use_cases:
-        print(f'ERROR: Use case failed: {failed_use_case}')
+    _print_failed_use_cases(failed_use_cases)
 
-    if not isOK:
+    if not is_ok:
         print("ERROR: Some commands failed.")
-        sys.exit(1)
+
+    return is_ok
+
+
+def _print_failed_use_cases(failed_use_cases):
+    for failed_use_case in failed_use_cases:
+        print(f'ERROR: Use case failed: {failed_use_case}')
 
 
 def _format_docker_exec_command(command):
@@ -201,4 +199,7 @@ def _get_dockerfile_name(requirements):
 
 
 if __name__ == '__main__':
-    main()
+    ret = main()
+    # exit non-zero if anything went wrong
+    if not ret:
+        sys.exit(1)

.github/parm/use_case_groups.json

@@ -9,6 +9,11 @@
     "index_list": "30-58",
     "run": false
   },
+  {
+    "category": "met_tool_wrapper",
+    "index_list": "65",
+    "run": false
+  },
   {
     "category": "air_quality_and_comp",
     "index_list": "0",
@@ -198,12 +203,12 @@
   {
     "category": "s2s_mid_lat",
     "index_list": "0-2",
-    "run": true
+    "run": false
   },
   {
     "category": "s2s_mid_lat",
     "index_list": "3",
-    "run": true
+    "run": false
   },
   {
     "category": "s2s_mjo",

.github/update_truth_change_log.txt

@@ -6,3 +6,4 @@
 [20240830_21:30:39 develop] dtcenter/MET#2951 - Modifies Series-Analysis output variable names for PSTD statistics by including the observation threshold string, as is done for PCT and PRC outputs.
 [20240905_16:45:56 develop] dtcenter/MET#2959 - Correct bugs in SEDI and BAGSS contingency table statistics found in the CTS line type
 [20240910_19:03:38 develop] dtcenter/MET#2967 - Fixes integer overflow bug in the computation of the CTS and NBRCTS ORSS, ORSS_NCL, and ORSS_NCU columns.
+[20240926_16:50:38 develop] #2701 - #2701 added a new use case (SeriesAnalysis_aggr) that produces new output

docs/Contributors_Guide/user_support.rst

@@ -13,12 +13,12 @@ discussions and begin conversations in the right place.
 Support Responsibilities
 ========================
 
-Five staff members take turns monitoring Discussions each day of the week.
+Staff members take turns monitoring Discussions throughout the week.
 
-Each day, all support members should follow up on existing Discussions in
+All support members should follow up on existing Discussions in
 which they are assigned (i.e. tagged).
 
-The responsibilities for each daily assignee are described below.
+The responsibilities for each assignee are described below.
 
 
 Review the Tips/Tricks Page
@@ -62,19 +62,22 @@ Review the `unanswered discussions <https://github.com/dtcenter/METplus/discussi
     required.
 
     * If an obvious answer exists, consider marking it as the answer and
-      locking the discussion to prevent future posts. See
+      locking the discussion to prevent future posts, and then close it. See
       :ref:`lock_discussions`.
 
     * If the user has not responded in 7 or more days, consider following up to
-      see if the user's issue is resolved or if they need more assistance.
+      see if the user's issue is resolved or if they need more assistance. If
+      the user has not responded in two weeks, close the discussion with a comment
+      indicating that is our policy to close discussions after two weeks of
+      inactivity.
 
     * If unsure how to proceed, post in the User Support Slack channel of
       dtc-metplus.slack.com tagging the person who has been assisting the user.
 
 Review Answered Discussions that are Unlocked
 ---------------------------------------------
 
-Review `anwered discussions that are unlocked <https://github.com/dtcenter/METplus/discussions?discussions_q=is%3Aanswered+is%3Aunlocked>`_.
+Review `answered discussions that are unlocked <https://github.com/dtcenter/METplus/discussions?discussions_q=is%3Aanswered+is%3Aunlocked>`_.
 
 If there is any recent activity on an **answered** question that indicates the
 questioner is satisified with the response, click on the button to **lock** the
@@ -95,6 +98,9 @@ Actions for Handling New Discussions
 
   * If you can answer it yourself, do so.
 
+  * If the question is due to unclear or insufficient documentation, please fix
+    the documentation and point the user to the updated version.
+
   * If you can answer it with help from someone, post to the User Support Slack
     channel of dtc-metplus.slack.com, asking for advice.
 
@@ -120,23 +126,49 @@ If an obvious answer exists and it is apparent the questioner is satisified
 with the response, consider posting something like the following before locking
 the discussion:
 
-*At this point I'm going to LOCK this conversation to prevent future posts. Our team has decided that once discussions have been answered, and the answer has been confirmed, we'll lock them. We want to encourage users to ask new questions in new discussions rather than posting to old ones. Hopefully that'll make the questions/answers easier for other users to follow. So if/when more issues/questions arise, please feel free to start a new discussion.*
+*I'm going to lock and close this conversation to prevent future posts. Our
+team has decided that once discussions have been answered, we'll lock them to
+encourage users to ask new questions in new discussions rather than posting
+to old ones. Hopefully that'll make the questions/answers easier for other
+users to follow. So if/when more issues/questions arise, please feel free
+to start a new discussion.*
 
 If there is no answer selected and no indication that the questioner is
 satisified with the response, follow up, requesting that the user mark
-the best answer so we can lock the discussion. The following text may need
-to be modified to match a specific situation, but this is a good starting
-point as an example:
-
-*I'd like to mark this discussion as being answered and "lock it" to prevent future posts. That's how we encourage users to create new discussions for new questions. But I wanted to give you an opportunity to comment on it before doing so. Please feel free to select one of the responses as being the best answer to your original question. That'll help future users with similar questions find answers.*
-
-If following up on discussions where the user hasn't responded back in a while,
-consider posting something like the following before locking the discussion,
-filling in the MONTH and DAY below:
-
-*I wanted to check in with you about the initial discussion you submitted. We were able to provide some guidance for you on <MONTH DAY>, but haven't heard back if the suggested actions helped. If it did, or if you found a different way of accomplishing your goal, we'd appreciate hearing from you. If you have additional questions on this, we can try to find a solution; otherwise we'll choose an answer and lock this discussion due to inactivity.*
+the best answer so we can lock and close the discussion. The following
+text may need to be modified to match a specific situation, but this is
+a good starting point as an example:
+
+*I'd like to mark this discussion as being answered, and lock and close it,
+to prevent future posts. That's how we encourage users to create new
+discussions for new questions. But I wanted to give you an opportunity to
+comment on it before doing so. Please feel free to select one of the
+responses as being the best answer to your original question. That'll help
+future users with similar questions find answers.*
+
+or
+
+*I wanted to check in with you about the initial discussion you submitted.
+We were able to provide some guidance for you on <MONTH DAY>, but haven't
+heard back if the suggested actions helped. If it did, or if you found a
+different way of accomplishing your goal, we'd appreciate hearing from you.
+If you have additional questions on this, we can try to find a solution;
+otherwise we'll choose an answer, and lock and close this discussion due
+to inactivity.*
+
+If following up on discussions where the user hasn't responded back in two
+weeks, consider posting something like the following before locking and closing
+the discussion, filling in the MONTH and DAY below:
+
+*We were able to provide some guidance for you on <MONTH DAY>, regarding the
+initial discussion you submitted, but we haven't heard back if the suggested
+actions helped. It is our
+`policy <https://github.com/dtcenter/METplus/discussions/939>`_ to lock and
+close discussions after two weeks of inactivity.*
 
 GitHub users with sufficient permissions in the METplus repository should see a
-"Lock conversation" link on the right side navigation of each Discussions page.
+"Lock conversation" link on the right side navigation of each Discussions page
+and a "Close discussion" button at the bottom of the page under the comment
+box.
 
 

docs/Release_Guide/met_official.rst

@@ -21,6 +21,7 @@ Create a new vX.Y.Z official release from the develop branch.
 .. include:: release_steps/create_release_extra.rst
 .. include:: release_steps/met/update_dtc_website.rst
 .. include:: release_steps/finalize_release_on_github_official.rst
+.. include:: release_steps/met/update_tar_files_link_official.rst
 .. include:: release_steps/update_docs_official.rst
 .. include:: release_steps/met/update_zenodo.rst
 .. include:: release_steps/set_beta_deletion_reminder_official.rst

docs/Release_Guide/release_steps/met/update_tar_files_link_official.rst

@@ -0,0 +1,8 @@
+Update tar_files Link
+^^^^^^^^^^^^^^^^^^^^^
+
+On the DTC web server machine (i.e. mohawk), in /d2/www/dtcenter/dfiles/code/METplus/MET/installation/, update
+the link for tar_files.latest.tgz, to point to the latest version of the
+tar_files.met-base-vX.Y.tgz file (viewable at https://dtcenter.ucar.edu/dfiles/code/METplus/MET/installation/).
+
+

docs/Release_Guide/release_steps/met/update_version_official.rst

@@ -8,6 +8,8 @@ Update Version Number
   * If necessary, update the *met_version* variable in *src/basic/vx_util/util_constants.h* which defines the version number written to the MET output files.
 
   * In *docs/conf.py*, update the *version*, *release_year*, and *release_date* variables for the documentation.
+
+  * In *docs/Users_Guide/installation.rst*, search for the X.Y version, replacing the current X.Y version with the official X.Y version, if necessary. Pay particular attention to the "Note" about the C++ standard and modify if necessary.  The X.Y version number in the "Note" box should NOT change unless the default C++ standard changes.
 
   * If necessary, update the version number listed in the MET configuration files: