From 1c9c4052ae04ae0e15fadc069e955cc5651dab60 Mon Sep 17 00:00:00 2001
From: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Date: Wed, 1 Nov 2023 16:49:44 -0600
Subject: [PATCH 1/7] download conda-forge's conda installer that is configured
 to only use the conda-forge channel to avoid licensing issues with Anaconda
 -- using an explicit version instead of 'latest' for reproducibility and so
 we can easily track which version was used

---
 internal/scripts/docker_env/Dockerfile.conda | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/internal/scripts/docker_env/Dockerfile.conda b/internal/scripts/docker_env/Dockerfile.conda
index f1a62fa780..e2679c05a9 100644
--- a/internal/scripts/docker_env/Dockerfile.conda
+++ b/internal/scripts/docker_env/Dockerfile.conda
@@ -5,6 +5,6 @@ ARG DEBIAN_VERSION=12
 FROM debian:${DEBIAN_VERSION}-slim
 
 RUN apt update && apt install -y curl \
- && curl https://repo.anaconda.com/miniconda/Miniconda3-py311_23.5.2-0-Linux-x86_64.sh > /miniconda.sh \
- && bash /miniconda.sh -b -p /usr/local/conda \
+ && curl -L https://github.com/conda-forge/miniforge/releases/download/23.3.1-1/Mambaforge-23.3.1-1-$(uname)-$(uname -m).sh -o /miniforge.sh \
+ && bash /miniforge.sh -b -p /usr/local/conda \
  && /usr/local/conda/bin/conda update -y -n base -c conda-forge conda

From b786e948f63c9a3677bb00f395627e7f56e3d6e9 Mon Sep 17 00:00:00 2001
From: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Date: Thu, 2 Nov 2023 12:46:53 -0600
Subject: [PATCH 2/7] added timeout to URL request to prevent script from
 hanging

---
 .github/jobs/docker_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/jobs/docker_utils.py b/.github/jobs/docker_utils.py
index a40d6b2e56..edf0b00167 100644
--- a/.github/jobs/docker_utils.py
+++ b/.github/jobs/docker_utils.py
@@ -39,7 +39,7 @@ def get_dockerhub_url(branch_name):
 def docker_get_volumes_last_updated(current_branch):
     import requests
     dockerhub_url = get_dockerhub_url(current_branch)
-    dockerhub_request = requests.get(dockerhub_url)
+    dockerhub_request = requests.get(dockerhub_url, timeout=60)
     if dockerhub_request.status_code != 200:
         print(f"Could not find DockerHub URL: {dockerhub_url}")
         return None
@@ -61,7 +61,7 @@ def docker_get_volumes_last_updated(current_branch):
                 volumes_last_updated[repo_name] = repo['last_updated']
         if not page['next']:
             break
-        page = requests.get(page['next']).json()
+        page = requests.get(page['next'], timeout=60).json()
         attempts += 1
 
     return volumes_last_updated

From db72d85fc1daa97bd7ef13cda4c73cead734c39b Mon Sep 17 00:00:00 2001
From: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Date: Thu, 2 Nov 2023 12:48:17 -0600
Subject: [PATCH 3/7] fail if data required to run use case or diff test cannot
 be downloaded so GHA error logs are more clear -- previously use cases or
 diff tests would fail and it was not clear why

---
 .github/jobs/get_data_volumes.py        | 11 ++++++++---
 .github/jobs/setup_and_run_diff.py      |  3 +++
 .github/jobs/setup_and_run_use_cases.py |  4 ++++
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/.github/jobs/get_data_volumes.py b/.github/jobs/get_data_volumes.py
index 5c568577a1..e492711710 100755
--- a/.github/jobs/get_data_volumes.py
+++ b/.github/jobs/get_data_volumes.py
@@ -40,7 +40,7 @@ def main(args):
     branch_name = get_branch_name()
     if not branch_name:
         print("Could not get current branch. Exiting.")
-        sys.exit(1)
+        return None
 
     # remove -ref from end of branch name if found
     if branch_name.endswith('-ref'):
@@ -88,7 +88,7 @@ def main(args):
         # use it, otherwise use develop version of data volume
         elif (metplus_version == 'develop' and
               f'{branch_name}-{model_app_name}' in available_volumes):
-                volume_name = f'{branch_name}-{model_app_name}'
+            volume_name = f'{branch_name}-{model_app_name}'
         else:
             volume_name = f'{metplus_version}-{model_app_name}'
 
@@ -97,13 +97,15 @@ def main(args):
         cmd = (f'docker create --name {model_app_name} '
                f'{full_volume_name}')
         if not run_commands(cmd):
-            continue
+            print(f'ERROR: Could not create data volume for {full_volume_name}')
+            return None
 
         # add name to volumes from list to pass to docker build
         volume_list.append(f'--volumes-from {model_app_name}')
 
     return ' '.join(volume_list)
 
+
 if __name__ == "__main__":
     # split up command line args that have commas before passing into main
     args = []
@@ -111,4 +113,7 @@ def main(args):
     for arg in sys.argv[1:]:
         args.extend(arg.split(','))
     out = main(args)
+    if out is None:
+        print("ERROR: Something went wrong")
+        sys.exit(1)
     print(out)
diff --git a/.github/jobs/setup_and_run_diff.py b/.github/jobs/setup_and_run_diff.py
index 1a4c908b14..59c5be37ed 100755
--- a/.github/jobs/setup_and_run_diff.py
+++ b/.github/jobs/setup_and_run_diff.py
@@ -43,6 +43,9 @@
 output_category = f"output-{output_data_branch}-{artifact_name}"
 
 VOLUMES_FROM = get_data_volumes.main([output_category])
+if VOLUMES_FROM is None:
+    print("ERROR: Could not get truth data to run diff")
+    sys.exit(1)
 
 print(f"Output Volumes: {VOLUMES_FROM}")
 
diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py
index 931a79d76d..d65cf247c5 100755
--- a/.github/jobs/setup_and_run_use_cases.py
+++ b/.github/jobs/setup_and_run_use_cases.py
@@ -49,6 +49,10 @@ def main():
     )
     # get input data volumes
     volumes_from = get_data_volumes.main(categories_list)
+    if volumes_from is None:
+        print('ERROR: Could not get input data to run use cases')
+        sys.exit(1)
+
     print(f"Input Volumes: {volumes_from}")
 
     # build Docker image with conda environment and METplus branch image

From f082c54aecb069f274af89cd755f813b8c659953 Mon Sep 17 00:00:00 2001
From: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Date: Thu, 2 Nov 2023 14:59:49 -0600
Subject: [PATCH 4/7] run diff for -ref branch run and only save output data
 artifacts if there are differences -- this should only update the truth data
 for use case groups that need to be updated

---
 .github/jobs/set_job_controls.sh | 1 +
 .github/workflows/testing.yml    | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh
index 032197ff46..8e4e91e341 100755
--- a/.github/jobs/set_job_controls.sh
+++ b/.github/jobs/set_job_controls.sh
@@ -33,6 +33,7 @@ elif [ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]; then
 elif [ "${GITHUB_REF: -4}" == -ref ]; then
   run_all_use_cases=true
   run_save_truth_data=true
+  run_diff=true
 # if not pull request or -ref branch, apply commit messages overrides
 else
 
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index e08aac81e4..65b56a1114 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -205,12 +205,12 @@ jobs:
       # copy output data to save as artifact
       - name: Save output data
         id: save-output
-        if: ${{ always() && steps.run_tests.conclusion != 'skipped' }}
+        if: ${{ always() && steps.run_tests.conclusion != 'skipped' && (!endsWith(needs.job_control.outputs.branch_name, '-ref') || steps.run-diff.conclusion == 'failure') }}
         run: .github/jobs/copy_output_to_artifact.sh ${{ steps.get-artifact-name.outputs.artifact_name }}
 
       - name: Upload output data artifact
         uses: actions/upload-artifact@v3
-        if: ${{ always() && steps.run_tests.conclusion != 'skipped' }}
+        if: ${{ always() && steps.run_tests.conclusion != 'skipped' && (!endsWith(needs.job_control.outputs.branch_name, '-ref') || steps.run-diff.conclusion == 'failure') }}
         with:
           name: ${{ steps.get-artifact-name.outputs.artifact_name }}
           path: artifact/${{ steps.get-artifact-name.outputs.artifact_name }}

From fd984789c3d2ec588c820df2fc68b4a95ffbca80 Mon Sep 17 00:00:00 2001
From: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Date: Thu, 2 Nov 2023 15:01:41 -0600
Subject: [PATCH 5/7] turn on use case to test

---
 .github/parm/use_case_groups.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json
index 1752f76df0..f361d41f0a 100644
--- a/.github/parm/use_case_groups.json
+++ b/.github/parm/use_case_groups.json
@@ -12,7 +12,7 @@
   {
     "category": "air_quality_and_comp",
     "index_list": "0",
-    "run": false
+    "run": true
   },
   {
     "category": "climate",

From 9efb6b4f6e570b9697457c1300f5558ac5049d71 Mon Sep 17 00:00:00 2001
From: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Date: Thu, 2 Nov 2023 15:49:32 -0600
Subject: [PATCH 6/7] turn off use cases

---
 .github/parm/use_case_groups.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json
index f361d41f0a..1752f76df0 100644
--- a/.github/parm/use_case_groups.json
+++ b/.github/parm/use_case_groups.json
@@ -12,7 +12,7 @@
   {
     "category": "air_quality_and_comp",
     "index_list": "0",
-    "run": true
+    "run": false
   },
   {
     "category": "climate",

From 39d578616e73f560be1e972f60dbb65042422fc5 Mon Sep 17 00:00:00 2001
From: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Date: Thu, 2 Nov 2023 17:51:44 -0600
Subject: [PATCH 7/7] Revert "run diff for -ref branch run and only save output
 data artifacts if there are differences -- this should only update the truth
 data for use case groups that need to be updated"

This reverts commit f082c54aecb069f274af89cd755f813b8c659953.
---
 .github/jobs/set_job_controls.sh | 1 -
 .github/workflows/testing.yml    | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh
index 8e4e91e341..032197ff46 100755
--- a/.github/jobs/set_job_controls.sh
+++ b/.github/jobs/set_job_controls.sh
@@ -33,7 +33,6 @@ elif [ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]; then
 elif [ "${GITHUB_REF: -4}" == -ref ]; then
   run_all_use_cases=true
   run_save_truth_data=true
-  run_diff=true
 # if not pull request or -ref branch, apply commit messages overrides
 else
 
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 65b56a1114..e08aac81e4 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -205,12 +205,12 @@ jobs:
       # copy output data to save as artifact
       - name: Save output data
         id: save-output
-        if: ${{ always() && steps.run_tests.conclusion != 'skipped' && (!endsWith(needs.job_control.outputs.branch_name, '-ref') || steps.run-diff.conclusion == 'failure') }}
+        if: ${{ always() && steps.run_tests.conclusion != 'skipped' }}
         run: .github/jobs/copy_output_to_artifact.sh ${{ steps.get-artifact-name.outputs.artifact_name }}
 
       - name: Upload output data artifact
         uses: actions/upload-artifact@v3
-        if: ${{ always() && steps.run_tests.conclusion != 'skipped' && (!endsWith(needs.job_control.outputs.branch_name, '-ref') || steps.run-diff.conclusion == 'failure') }}
+        if: ${{ always() && steps.run_tests.conclusion != 'skipped' }}
         with:
           name: ${{ steps.get-artifact-name.outputs.artifact_name }}
           path: artifact/${{ steps.get-artifact-name.outputs.artifact_name }}