Update develop-ref after #2488, #2480, and dtcenter/MET#2812

.github/jobs/free_disk_space.sh

@@ -10,6 +10,41 @@ sudo rm -rf /opt/ghc
 sudo rm -rf "/usr/local/share/boost"
 sudo rm -rf "$AGENT_TOOLSDIRECTORY"
 
+###
+# more cleanup taken from https://github.com/apache/flink/blob/master/tools/azure-pipelines/free_disk_space.sh
+###
+
+echo "=============================================================================="
+echo "Freeing up disk space on CI system"
+echo "=============================================================================="
+
+echo "Listing 100 largest packages"
+dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100
+df -h
+echo "Removing large packages"
+sudo apt-get remove -y '^dotnet-.*'
+sudo apt-get remove -y '^llvm-.*'
+sudo apt-get remove -y 'php.*'
+sudo apt-get remove -y '^mongodb-.*'
+sudo apt-get remove -y '^mysql-.*'
+sudo apt-get remove -y azure-cli google-cloud-sdk hhvm google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri
+sudo apt-get autoremove -y
+sudo apt-get clean
+df -h
+echo "Removing large directories"
+
+sudo rm -rf /usr/share/dotnet/
+sudo rm -rf /usr/local/graalvm/
+sudo rm -rf /usr/local/.ghcup/
+sudo rm -rf /usr/local/share/powershell
+sudo rm -rf /usr/local/share/chromium
+sudo rm -rf /usr/local/lib/android
+sudo rm -rf /usr/local/lib/node_modules
+
+###
+# end cleanup from azure script
+###
+
 printf "\nChecking disk usage after cleanup"
 
 df -h

.github/jobs/setup_and_run_use_cases.py

@@ -89,6 +89,16 @@ def main():
         # print list of existing docker images
         commands.append('docker images')
 
+        # remove docker image after creating run env or prune untagged images
+        commands.append(f'docker image rm dtcenter/metplus-dev:{branch_name} -f')
+        commands.append('docker image prune -f')
+        run_commands(commands)
+
+        commands = []
+
+        # list docker images again after removal
+        commands.append('docker images')
+
         # start interactive container in the background
         commands.append(
             f"docker run -d --rm -it -e GITHUB_WORKSPACE "

.github/parm/use_case_groups.json

@@ -89,6 +89,11 @@
     "index_list": "9",
     "run": false
   },
+  {
+    "category": "marine_and_cryosphere",
+    "index_list": "10",
+    "run": false
+  },
   {
     "category": "medium_range",
     "index_list": "0",
@@ -259,6 +264,11 @@
     "index_list": "13",
     "run": false
   },
+  {
+    "category": "short_range",
+    "index_list": "14",
+    "run": false
+  },
   {
     "category": "space_weather",
     "index_list": "0-1",

.github/workflows/documentation.yml

@@ -16,8 +16,8 @@ jobs:
     name: Documentation
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
       - name: Install dependencies
@@ -27,12 +27,12 @@ jobs:
           python -m pip install -r docs/requirements.txt
       - name: Build Documentation
         run: ./.github/jobs/build_documentation.sh
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: METplus_documentation
           path: artifact/documentation
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: documentation_warnings.log

.github/workflows/testing.yml

@@ -36,19 +36,18 @@ on:
 
   workflow_dispatch:
     inputs:
+      force_met_image:
+        description: 'MET DockerHub repo to force run to use, e.g. met:11.1.0 or met-dev:feature_XYZ_name-PR. Leave this blank to determine repo automatically.'
       repository:
-        description: 'Repository that triggered workflow'
-        required: true
+        description: 'Repository that triggered workflow (used by external repo triggering)'
       sha:
-        description: 'Commit hash that triggered the event'
-        required: true
+        description: 'Commit hash that triggered the event (used by external repo triggering)'
       ref:
-        description: 'Branch that triggered event'
-        required: true
+        description: 'Branch that triggered event (used by external repo triggering)'
       actor:
-        description: 'User that triggered the event'
+        description: 'User that triggered the event (used by external repo triggering)'
       pusher_email:
-        description: 'Email address of user who triggered push event'
+        description: 'Email address of user who triggered push event (used by external repo triggering)'
 
 jobs:
 
@@ -69,7 +68,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set job controls
         id: job_status
         run: .github/jobs/set_job_controls.sh
@@ -93,17 +92,19 @@ jobs:
     needs: job_control
     if: ${{ needs.job_control.outputs.run_get_image == 'true' && needs.job_control.outputs.run_some_tests == 'true' }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
+      - name: Free disk space
+        run: .github/jobs/free_disk_space.sh
       - name: Get METplus Image
         run: .github/jobs/docker_setup.sh
         env:
           DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
           EXTERNAL_TRIGGER: ${{ needs.job_control.outputs.external_trigger }}
-          #SET_MET_IMAGE: met:11.0.0
+          SET_MET_IMAGE: ${{ github.event.inputs.force_met_image }}
 
   update_data_volumes:
     name: Docker Setup - Update Data Volumes
@@ -131,8 +132,8 @@ jobs:
     needs: [job_control]
     if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
           cache: 'pip'
@@ -166,7 +167,7 @@ jobs:
       matrix: ${{fromJson(needs.job_control.outputs.matrix)}}
     steps:
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Free disk space
         run: .github/jobs/free_disk_space.sh
@@ -209,36 +210,63 @@ jobs:
         run: .github/jobs/copy_output_to_artifact.sh ${{ steps.get-artifact-name.outputs.artifact_name }}
 
       - name: Upload output data artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ always() && steps.run_tests.conclusion != 'skipped' }}
         with:
           name: ${{ steps.get-artifact-name.outputs.artifact_name }}
           path: artifact/${{ steps.get-artifact-name.outputs.artifact_name }}
 
       - name: Upload error logs artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ always() && steps.save-errors.outputs.upload_error_logs }}
         with:
-          name: error_logs
+          name: error_logs-${{ steps.get-artifact-name.outputs.artifact_name }}
           path: artifact/error_logs
           if-no-files-found: ignore
 
       - name: Upload difference data artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ always() && steps.run-diff.outputs.upload_diff == 'true' }}
         with:
           name: diff-${{ steps.get-artifact-name.outputs.artifact_name }}
           path: artifact/diff-${{ steps.get-artifact-name.outputs.artifact_name }}
           if-no-files-found: ignore
 
+  merge_error_logs:
+    name: Merge Error Logs
+    runs-on: ubuntu-latest
+    needs: use_case_tests
+    if: ${{ always() && needs.use_case_tests.result == 'failure' }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check for error logs
+        id: check-for-error-logs
+        run: |
+          json_data=$(curl -s -H "Authorization: Bearer ${{ github.token }}" \
+                 "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs")
+          error_log_jobs=($(echo "$json_data" | jq -r '.jobs[] | select(.name | startswith("Use Case Tests")) | .steps[] | select(.name | startswith("Save error logs")) | select(.conclusion | startswith("success"))'))
+          # save output variable to merge error logs if any error logs were created
+          if [ ! -z "${error_log_jobs}" ]; then
+              echo "has_error_logs=true" >> $GITHUB_OUTPUT
+          else
+              echo "has_error_logs=false" >> $GITHUB_OUTPUT
+          fi
+      - name: Merge Artifacts
+        if: ${{ always() && steps.check-for-error-logs.outputs.has_error_logs == 'true' }}
+        uses: actions/upload-artifact/merge@v4
+        with:
+          name: error_logs
+          pattern: error_logs-*
+          delete-merged: true
+
   create_output_data_volumes:
     name: Create Output Docker Data Volumes
     runs-on: ubuntu-latest
     needs: [use_case_tests]
     if: ${{ needs.job_control.outputs.run_save_truth_data == 'true' }}
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
       - run: .github/jobs/create_output_data_volumes.sh
         env:
           DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}

.github/workflows/update_truth.yml

@@ -26,7 +26,7 @@ jobs:
           fi
           echo ERROR: Branch is $branch_name - must be develop or match main_vX.Y
           exit 1
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         name: Checkout repository
         with:
           fetch-depth: 0