From 0436e699a41af485a1820b0bcee2f92e91b92513 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 25 Sep 2024 21:36:19 +0000
Subject: [PATCH 01/95] added GitHub pipeline for runner on pw

---
 .github/workflows/pw-ci-workflow.yaml | 33 +++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 .github/workflows/pw-ci-workflow.yaml

diff --git a/.github/workflows/pw-ci-workflow.yaml b/.github/workflows/pw-ci-workflow.yaml
new file mode 100644
index 0000000000..89e407b989
--- /dev/null
+++ b/.github/workflows/pw-ci-workflow.yaml
@@ -0,0 +1,33 @@
+name: pw-ci-hera
+
+on: [workflow_dispatch]
+
+# TEST_DIR contains 2 directories;
+# 1. HOMEgfs: clone of the global-workflow
+# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
+# e.g. $> tree ./TEST_DIR
+# ./TEST_DIR
+# ├── HOMEgfs
+# └── RUNTESTS
+#     ├── COMROT
+#     │   └── ${pslot}
+#     └── EXPDIR
+#         └── ${pslot}
+env:
+  TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
+  MACHINE_ID: hera 
+
+jobs:
+  checkout-build-link:
+    runs-on: [self-hosted, parallel-works, pw]
+    timeout-minutes: 600
+
+    steps:
+    - name: Checkout global-workflow
+      uses: actions/checkout@v3
+      with:
+        repository: owner/repo
+        ssh-key: ${{ secrets.SSH_PW_PRIVATE_KEY }}
+        path: ${{ github.run_id }}/HOMEgfs  # This path needs to be relative
+        submodules: true
+        submodules_recursive: true

From 7829f7d929aa79229d8f99b263ced31e2feeca6b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 25 Sep 2024 21:42:36 +0000
Subject: [PATCH 02/95] added repos name to chekcout

---
 .github/workflows/pw-ci-workflow.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pw-ci-workflow.yaml b/.github/workflows/pw-ci-workflow.yaml
index 89e407b989..1a12fbc97f 100644
--- a/.github/workflows/pw-ci-workflow.yaml
+++ b/.github/workflows/pw-ci-workflow.yaml
@@ -26,7 +26,7 @@ jobs:
     - name: Checkout global-workflow
       uses: actions/checkout@v3
       with:
-        repository: owner/repo
+        repository: TerrenceMcGuinness-NOAA/global-workflow
         ssh-key: ${{ secrets.SSH_PW_PRIVATE_KEY }}
         path: ${{ github.run_id }}/HOMEgfs  # This path needs to be relative
         submodules: true

From fc4bf1f15e589dbe0e683f05767e1c9583cd9b53 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 25 Sep 2024 21:57:06 +0000
Subject: [PATCH 03/95] added ssh to hera from pw before actions/checkout@v3

---
 .github/workflows/pw-ci-workflow.yaml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pw-ci-workflow.yaml b/.github/workflows/pw-ci-workflow.yaml
index 1a12fbc97f..4266dc40e6 100644
--- a/.github/workflows/pw-ci-workflow.yaml
+++ b/.github/workflows/pw-ci-workflow.yaml
@@ -23,11 +23,17 @@ jobs:
     timeout-minutes: 600
 
     steps:
+
+    - name: SSH into remote host and prepare environment
+      run: |
+        ssh hfe01.fairmont.rdhpcs.noaa.gov
+          cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}
+        EOF
+
     - name: Checkout global-workflow
       uses: actions/checkout@v3
       with:
         repository: TerrenceMcGuinness-NOAA/global-workflow
-        ssh-key: ${{ secrets.SSH_PW_PRIVATE_KEY }}
         path: ${{ github.run_id }}/HOMEgfs  # This path needs to be relative
         submodules: true
         submodules_recursive: true

From a4126a33110ff4529bcf9bda5ba2c71ce847c323 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 25 Sep 2024 21:59:01 +0000
Subject: [PATCH 04/95] had to create dir on hera first

---
 .github/workflows/pw-ci-workflow.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pw-ci-workflow.yaml b/.github/workflows/pw-ci-workflow.yaml
index 4266dc40e6..b627af523e 100644
--- a/.github/workflows/pw-ci-workflow.yaml
+++ b/.github/workflows/pw-ci-workflow.yaml
@@ -27,6 +27,7 @@ jobs:
     - name: SSH into remote host and prepare environment
       run: |
         ssh hfe01.fairmont.rdhpcs.noaa.gov
+          mkdir -p /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}
           cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}
         EOF
 

From 82f4b996b4963425983981272a5f9e895177331c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 25 Sep 2024 22:08:32 +0000
Subject: [PATCH 05/95] ssh then two commands

---
 .github/workflows/pw-ci-workflow.yaml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pw-ci-workflow.yaml b/.github/workflows/pw-ci-workflow.yaml
index b627af523e..f99791e210 100644
--- a/.github/workflows/pw-ci-workflow.yaml
+++ b/.github/workflows/pw-ci-workflow.yaml
@@ -27,9 +27,8 @@ jobs:
     - name: SSH into remote host and prepare environment
       run: |
         ssh hfe01.fairmont.rdhpcs.noaa.gov
-          mkdir -p /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}
-          cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}
-        EOF
+        mkdir -p /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}
+        cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}
 
     - name: Checkout global-workflow
       uses: actions/checkout@v3

From 581c31e0d93c656f854d627ba6b9b3cdf8abb00a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 25 Sep 2024 22:11:19 +0000
Subject: [PATCH 06/95] cd to target dir firsts the mk

---
 .github/workflows/pw-ci-workflow.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pw-ci-workflow.yaml b/.github/workflows/pw-ci-workflow.yaml
index f99791e210..4030e54b0c 100644
--- a/.github/workflows/pw-ci-workflow.yaml
+++ b/.github/workflows/pw-ci-workflow.yaml
@@ -27,8 +27,9 @@ jobs:
     - name: SSH into remote host and prepare environment
       run: |
         ssh hfe01.fairmont.rdhpcs.noaa.gov
-        mkdir -p /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}
-        cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}
+        cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR
+        mkdir -p ${{ github.run_id }}
+        cd ${{ github.run_id }}
 
     - name: Checkout global-workflow
       uses: actions/checkout@v3

From 9d7970585183e9a7e47ad5389670e66983b97947 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 25 Sep 2024 22:14:51 +0000
Subject: [PATCH 07/95] added -t and EOF for getting ssh to run in one shot

---
 .github/workflows/pw-ci-workflow.yaml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pw-ci-workflow.yaml b/.github/workflows/pw-ci-workflow.yaml
index 4030e54b0c..cd6554a580 100644
--- a/.github/workflows/pw-ci-workflow.yaml
+++ b/.github/workflows/pw-ci-workflow.yaml
@@ -26,10 +26,11 @@ jobs:
 
     - name: SSH into remote host and prepare environment
       run: |
-        ssh hfe01.fairmont.rdhpcs.noaa.gov
-        cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR
-        mkdir -p ${{ github.run_id }}
-        cd ${{ github.run_id }}
+        ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
+         cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR
+         mkdir -p ${{ github.run_id }}
+         cd ${{ github.run_id }}
+        EOF
 
     - name: Checkout global-workflow
       uses: actions/checkout@v3

From 3d1552100af9911bf10a7be27495dae9bbef7436 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 25 Sep 2024 22:27:10 +0000
Subject: [PATCH 08/95] skipping checkout@v3 and pushing forwrad

---
 .github/workflows/pw-ci-hera-ssh.yaml | 97 +++++++++++++++++++++++++++
 1 file changed, 97 insertions(+)
 create mode 100644 .github/workflows/pw-ci-hera-ssh.yaml

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
new file mode 100644
index 0000000000..545e4210f2
--- /dev/null
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -0,0 +1,97 @@
+name: pw-ci-hera
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Branch to checkout'
+        required: true
+        default: 'main'
+
+# TEST_DIR contains 2 directories;
+# 1. HOMEgfs: clone of the global-workflow
+# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
+# e.g. $> tree ./TEST_DIR
+# ./TEST_DIR
+# ├── HOMEgfs
+# └── RUNTESTS
+#     ├── COMROT
+#     │   └── ${pslot}
+#     └── EXPDIR
+#         └── ${pslot}
+env:
+  TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
+  MACHINE_ID: hera 
+
+jobs:
+  checkout-build-link:
+    runs-on: [self-hosted, parallel-works, pw]
+    timeout-minutes: 600
+
+    steps:
+    - name: SSH into remote host and prepare environment
+      run: |
+        ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
+         cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR
+         mkdir -p ${{ github.run_id }}
+         cd ${{ github.run_id }}
+        EOF
+
+    - name: Checkout global-workflow
+      uses: actions/checkout@v3
+      with:
+        repository: TerrenceMcGuinness-NOAA/global-workflow
+        ref: ${{ github.event.inputs.branch }}
+        path: ${{ github.run_id }}/HOMEgfs  # This path needs to be relative
+        submodules: true
+        submodules_recursive: true
+
+    - name: Build components
+      run: |
+        ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
+          cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}/HOMEgfs/sorc
+          ./build_all.sh
+        EOF
+
+    - name: Link artifacts
+      run: |
+        ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
+          cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}/HOMEgfs/sorc
+          ./link_workflow.sh
+        EOF
+
+  create-experiments:
+    needs: checkout-build-link
+    runs-on: [self-hosted, orion-ready]
+    strategy:
+      matrix:
+        case: ["C48_S2S", "C96_atm3DVar"]
+
+    steps:
+      - name: Create Experiments ${{ matrix.case }}
+        env:
+          HOMEgfs_PR: ${{ env.TEST_DIR }}/HOMEgfs
+          RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS
+          pslot: ${{ matrix.case }}.${{ github.run_id }}
+        run: |
+          ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
+            cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}/HOMEgfs
+            source workflow/gw_setup.sh
+            source ci/platforms/orion.sh
+            ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml --dir ${{ env.HOMEgfs_PR }}
+          EOF
+
+  run-experiments:
+    needs: create-experiments
+    runs-on: [self-hosted, orion-ready]
+    strategy:
+      max-parallel: 2
+      matrix:
+        case: ["C48_S2S", "C96_atm3DVar"]
+    steps:
+      - name: Run Experiment ${{ matrix.case }}
+        run: |
+          ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
+            cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}/HOMEgfs
+            # Add commands to run the experiment
+          EOF
\ No newline at end of file

From 25bc12bac5301f8101e1e6c84d21f9627328e66c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 26 Sep 2024 16:08:32 +0000
Subject: [PATCH 09/95] ready to test CI pw pipeline using self hosted runner
 on pw

---
 .github/workflows/pw-ci-hera-ssh.yaml | 30 +++++++-------------
 .github/workflows/pw-ci-workflow.yaml | 41 ---------------------------
 2 files changed, 11 insertions(+), 60 deletions(-)
 delete mode 100644 .github/workflows/pw-ci-workflow.yaml

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 545e4210f2..a0e3e304f2 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -6,7 +6,7 @@ on:
       branch:
         description: 'Branch to checkout'
         required: true
-        default: 'main'
+        default: ${{ github.ref_name }}
 
 # TEST_DIR contains 2 directories;
 # 1. HOMEgfs: clone of the global-workflow
@@ -20,7 +20,7 @@ on:
 #     └── EXPDIR
 #         └── ${pslot}
 env:
-  TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
+  TEST_DIR: /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ env.BRANCH_NAME }}/${{ env.PR_NUMBER }}/${{ github.run_id }}
   MACHINE_ID: hera 
 
 jobs:
@@ -29,34 +29,26 @@ jobs:
     timeout-minutes: 600
 
     steps:
-    - name: SSH into remote host and prepare environment
+    - name: Checkout global-workflow
       run: |
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-         cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR
-         mkdir -p ${{ github.run_id }}
-         cd ${{ github.run_id }}
+         mkdir -p ${TEST_DIR}/HOMEgfs
+         cd ${TEST_DIR}/HOMEgfs
+         git clone --branch ${{ github.event.inputs.branch }} https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}.git .
+         git submodule update --init --recursive
         EOF
 
-    - name: Checkout global-workflow
-      uses: actions/checkout@v3
-      with:
-        repository: TerrenceMcGuinness-NOAA/global-workflow
-        ref: ${{ github.event.inputs.branch }}
-        path: ${{ github.run_id }}/HOMEgfs  # This path needs to be relative
-        submodules: true
-        submodules_recursive: true
-
     - name: Build components
       run: |
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-          cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}/HOMEgfs/sorc
+          cd ${TESTDIR}/HOMEgfs/sorc
           ./build_all.sh
         EOF
 
     - name: Link artifacts
       run: |
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-          cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}/HOMEgfs/sorc
+          cd ${TESTDIR}/HOMEgfs/sorc
           ./link_workflow.sh
         EOF
 
@@ -75,10 +67,10 @@ jobs:
           pslot: ${{ matrix.case }}.${{ github.run_id }}
         run: |
           ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-            cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}/HOMEgfs
+            cd ${HOMEgfs_PR}
             source workflow/gw_setup.sh
             source ci/platforms/orion.sh
-            ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml --dir ${{ env.HOMEgfs_PR }}
+            ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
           EOF
 
   run-experiments:
diff --git a/.github/workflows/pw-ci-workflow.yaml b/.github/workflows/pw-ci-workflow.yaml
deleted file mode 100644
index cd6554a580..0000000000
--- a/.github/workflows/pw-ci-workflow.yaml
+++ /dev/null
@@ -1,41 +0,0 @@
-name: pw-ci-hera
-
-on: [workflow_dispatch]
-
-# TEST_DIR contains 2 directories;
-# 1. HOMEgfs: clone of the global-workflow
-# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
-# e.g. $> tree ./TEST_DIR
-# ./TEST_DIR
-# ├── HOMEgfs
-# └── RUNTESTS
-#     ├── COMROT
-#     │   └── ${pslot}
-#     └── EXPDIR
-#         └── ${pslot}
-env:
-  TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
-  MACHINE_ID: hera 
-
-jobs:
-  checkout-build-link:
-    runs-on: [self-hosted, parallel-works, pw]
-    timeout-minutes: 600
-
-    steps:
-
-    - name: SSH into remote host and prepare environment
-      run: |
-        ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-         cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR
-         mkdir -p ${{ github.run_id }}
-         cd ${{ github.run_id }}
-        EOF
-
-    - name: Checkout global-workflow
-      uses: actions/checkout@v3
-      with:
-        repository: TerrenceMcGuinness-NOAA/global-workflow
-        path: ${{ github.run_id }}/HOMEgfs  # This path needs to be relative
-        submodules: true
-        submodules_recursive: true

From cb7fa7a29695802cc6b50a87b24878dc15f37be2 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 26 Sep 2024 16:20:23 +0000
Subject: [PATCH 10/95] removed default branch in dispatch

---
 .github/workflows/pw-ci-hera-ssh.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index a0e3e304f2..68d29b5032 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -6,7 +6,6 @@ on:
       branch:
         description: 'Branch to checkout'
         required: true
-        default: ${{ github.ref_name }}
 
 # TEST_DIR contains 2 directories;
 # 1. HOMEgfs: clone of the global-workflow

From d56f313d4ab83e3e2a099414cb343c1976e8b1da Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 26 Sep 2024 16:21:53 +0000
Subject: [PATCH 11/95] removed branch input as it not needed

---
 .github/workflows/pw-ci-hera-ssh.yaml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 68d29b5032..05645b2f8a 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -2,10 +2,6 @@ name: pw-ci-hera
 
 on:
   workflow_dispatch:
-    inputs:
-      branch:
-        description: 'Branch to checkout'
-        required: true
 
 # TEST_DIR contains 2 directories;
 # 1. HOMEgfs: clone of the global-workflow

From 658e745bafe2c0e2b71f0d40c6f1308f43b9e4c0 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 26 Sep 2024 16:28:01 +0000
Subject: [PATCH 12/95] set BRANCH_NAME and PR_NUMBER in advance

---
 .github/workflows/pw-ci-hera-ssh.yaml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 05645b2f8a..fe69ae0ce9 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -14,9 +14,11 @@ on:
 #     │   └── ${pslot}
 #     └── EXPDIR
 #         └── ${pslot}
-env:
-  TEST_DIR: /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ env.BRANCH_NAME }}/${{ env.PR_NUMBER }}/${{ github.run_id }}
-  MACHINE_ID: hera 
+    env:
+      BRANCH_NAME: ${{ github.event.inputs.branch || github.ref_name }}
+      PR_NUMBER: ${{ github.event.pull_request.number || '' }}
+      TEST_DIR: /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ env.BRANCH_NAME }}${{ env.PR_NUMBER && '-' + env.PR_NUMBER }}
+      MACHINE_ID: hera 
 
 jobs:
   checkout-build-link:
@@ -29,7 +31,7 @@ jobs:
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
          mkdir -p ${TEST_DIR}/HOMEgfs
          cd ${TEST_DIR}/HOMEgfs
-         git clone --branch ${{ github.event.inputs.branch }} https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}.git .
+         git clone --branch ${{ env.BRANCH_NAME }} https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}.git .
          git submodule update --init --recursive
         EOF
 

From 08bc4b81cf31e6aa12995b4b561388d1d566d9e9 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 26 Sep 2024 16:32:11 +0000
Subject: [PATCH 13/95] set BRANCH_NAME and PR_NUMBER in advance

---
 .github/workflows/pw-ci-hera-ssh.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index fe69ae0ce9..a44375f3dc 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -15,9 +15,9 @@ on:
 #     └── EXPDIR
 #         └── ${pslot}
     env:
-      BRANCH_NAME: ${{ github.event.inputs.branch || github.ref_name }}
+      BRANCH_NAME: ${{ github.ref_name }}
       PR_NUMBER: ${{ github.event.pull_request.number || '' }}
-      TEST_DIR: /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ env.BRANCH_NAME }}${{ env.PR_NUMBER && '-' + env.PR_NUMBER }}
+      TEST_DIR: /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ env.BRANCH_NAME }}/${{ env.PR_NUMBER }}
       MACHINE_ID: hera 
 
 jobs:

From 9f910f761a2c6864e37f12afdfa014b267822ead Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 26 Sep 2024 16:44:11 +0000
Subject: [PATCH 14/95] only add TEST_DIR in env and use githug when needed

---
 .github/workflows/pw-ci-hera-ssh.yaml | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index a44375f3dc..fc9b7d8a58 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -14,24 +14,28 @@ on:
 #     │   └── ${pslot}
 #     └── EXPDIR
 #         └── ${pslot}
-    env:
-      BRANCH_NAME: ${{ github.ref_name }}
-      PR_NUMBER: ${{ github.event.pull_request.number || '' }}
-      TEST_DIR: /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ env.BRANCH_NAME }}/${{ env.PR_NUMBER }}
-      MACHINE_ID: hera 
 
 jobs:
   checkout-build-link:
     runs-on: [self-hosted, parallel-works, pw]
     timeout-minutes: 600
 
+    env:
+      MACHINE_ID: hera
+
+        steps:
+    - name: Set environment variables
+      id: set-env
+      run: |
+        echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+
     steps:
     - name: Checkout global-workflow
       run: |
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
          mkdir -p ${TEST_DIR}/HOMEgfs
          cd ${TEST_DIR}/HOMEgfs
-         git clone --branch ${{ env.BRANCH_NAME }} https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}.git .
+         git clone --branch ${{ githug.ref_name }} https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}.git .
          git submodule update --init --recursive
         EOF
 

From 8d2fcfde1e86ad31bc19d9d27c937352537f7ad1 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 26 Sep 2024 18:34:42 +0000
Subject: [PATCH 15/95] had extra steps section now removed

---
 .github/workflows/pw-ci-hera-ssh.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index fc9b7d8a58..ef8b8adc51 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -16,6 +16,7 @@ on:
 #         └── ${pslot}
 
 jobs:
+
   checkout-build-link:
     runs-on: [self-hosted, parallel-works, pw]
     timeout-minutes: 600
@@ -23,19 +24,19 @@ jobs:
     env:
       MACHINE_ID: hera
 
-        steps:
+    steps:
+
     - name: Set environment variables
       id: set-env
       run: |
         echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
 
-    steps:
     - name: Checkout global-workflow
       run: |
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
          mkdir -p ${TEST_DIR}/HOMEgfs
          cd ${TEST_DIR}/HOMEgfs
-         git clone --branch ${{ githug.ref_name }} https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}.git .
+         git clone --branch ${{ }} https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}.git .
          git submodule update --init --recursive
         EOF
 

From 3c207b4993960dea4af12903ce5e88cce796aa66 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 26 Sep 2024 18:40:31 +0000
Subject: [PATCH 16/95] forgot to put in expression for branch in checkout

---
 .github/workflows/pw-ci-hera-ssh.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index ef8b8adc51..2522b4858f 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -36,7 +36,7 @@ jobs:
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
          mkdir -p ${TEST_DIR}/HOMEgfs
          cd ${TEST_DIR}/HOMEgfs
-         git clone --branch ${{ }} https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}.git .
+         git clone --branch ${{ github.ref_name }} https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}.git .
          git submodule update --init --recursive
         EOF
 

From 0c8b9fa6760775fc66bdfc8a6a620ea78a22e832 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 26 Sep 2024 18:44:12 +0000
Subject: [PATCH 17/95] needed env for TEST_DIR on subsquent sections

---
 .github/workflows/pw-ci-hera-ssh.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 2522b4858f..14fd37d83a 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -34,8 +34,8 @@ jobs:
     - name: Checkout global-workflow
       run: |
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-         mkdir -p ${TEST_DIR}/HOMEgfs
-         cd ${TEST_DIR}/HOMEgfs
+         mkdir -p ${{ env.TEST_DIR }}/HOMEgfs
+         cd ${{ env.TEST_DIR }}/HOMEgfs
          git clone --branch ${{ github.ref_name }} https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}.git .
          git submodule update --init --recursive
         EOF
@@ -43,14 +43,14 @@ jobs:
     - name: Build components
       run: |
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-          cd ${TESTDIR}/HOMEgfs/sorc
+          cd ${{ env.TESTDIR }}/HOMEgfs/sorc
           ./build_all.sh
         EOF
 
     - name: Link artifacts
       run: |
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-          cd ${TESTDIR}/HOMEgfs/sorc
+          cd ${{ env.TESTDIR }}/HOMEgfs/sorc
           ./link_workflow.sh
         EOF
 

From 00ff9f9eb335a89686eee05576dd7129b4ed6927 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 26 Sep 2024 19:10:20 +0000
Subject: [PATCH 18/95] misspelled TEST_DIR in a few places

---
 .github/workflows/pw-ci-hera-ssh.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 14fd37d83a..5b108f3fa6 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -43,14 +43,14 @@ jobs:
     - name: Build components
       run: |
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-          cd ${{ env.TESTDIR }}/HOMEgfs/sorc
+          cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
           ./build_all.sh
         EOF
 
     - name: Link artifacts
       run: |
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-          cd ${{ env.TESTDIR }}/HOMEgfs/sorc
+          cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
           ./link_workflow.sh
         EOF
 

From 57b8cd6739d9c2f7695e8b88ff5d99c90c3fc102 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 26 Sep 2024 19:45:52 +0000
Subject: [PATCH 19/95] Had tags wrong for running on Runner in subsquent jobs

---
 .github/workflows/pw-ci-hera-ssh.yaml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 5b108f3fa6..dbac20834b 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -56,7 +56,7 @@ jobs:
 
   create-experiments:
     needs: checkout-build-link
-    runs-on: [self-hosted, orion-ready]
+    runs-on: [self-hosted, parallel-works, pw]
     strategy:
       matrix:
         case: ["C48_S2S", "C96_atm3DVar"]
@@ -69,15 +69,15 @@ jobs:
           pslot: ${{ matrix.case }}.${{ github.run_id }}
         run: |
           ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-            cd ${HOMEgfs_PR}
+            cd ${{ env.TEST_DIR }}/HOMEgfs
             source workflow/gw_setup.sh
-            source ci/platforms/orion.sh
+            source ci/platforms/${{ env.MACHINE_ID }}.sh
             ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
           EOF
 
   run-experiments:
     needs: create-experiments
-    runs-on: [self-hosted, orion-ready]
+    runs-on: [self-hosted, parallel-works, pw]
     strategy:
       max-parallel: 2
       matrix:
@@ -86,6 +86,6 @@ jobs:
       - name: Run Experiment ${{ matrix.case }}
         run: |
           ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-            cd /scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}/HOMEgfs
+            cd 
             # Add commands to run the experiment
           EOF
\ No newline at end of file

From f78d573138c9a7b60e790514ec31653de06834fc Mon Sep 17 00:00:00 2001
From: Terrence McGuinness <tmcguinness@Terrences-Mac-Pro.local>
Date: Thu, 26 Sep 2024 16:50:11 -0400
Subject: [PATCH 20/95] added shared env between jobs using ouputs for TEST_DIR

---
 .github/workflows/pw-ci-hera-ssh.yaml | 31 ++++++++-------------------
 1 file changed, 9 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index dbac20834b..5aee69f784 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -1,20 +1,3 @@
-name: pw-ci-hera
-
-on:
-  workflow_dispatch:
-
-# TEST_DIR contains 2 directories;
-# 1. HOMEgfs: clone of the global-workflow
-# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
-# e.g. $> tree ./TEST_DIR
-# ./TEST_DIR
-# ├── HOMEgfs
-# └── RUNTESTS
-#     ├── COMROT
-#     │   └── ${pslot}
-#     └── EXPDIR
-#         └── ${pslot}
-
 jobs:
 
   checkout-build-link:
@@ -24,12 +7,16 @@ jobs:
     env:
       MACHINE_ID: hera
 
+    outputs:
+      TEST_DIR: ${{ steps.set-env.outputs.TEST_DIR }}
+
     steps:
 
     - name: Set environment variables
       id: set-env
       run: |
         echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+        echo "::set-output name=TEST_DIR::/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}"
 
     - name: Checkout global-workflow
       run: |
@@ -64,12 +51,12 @@ jobs:
     steps:
       - name: Create Experiments ${{ matrix.case }}
         env:
-          HOMEgfs_PR: ${{ env.TEST_DIR }}/HOMEgfs
-          RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS
+          HOMEgfs_PR: ${{ needs.checkout-build-link.outputs.TEST_DIR }}/HOMEgfs
+          RUNTESTS: ${{ needs.checkout-build-link.outputs.TEST_DIR }}/RUNTESTS
           pslot: ${{ matrix.case }}.${{ github.run_id }}
         run: |
           ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-            cd ${{ env.TEST_DIR }}/HOMEgfs
+            cd ${{ needs.checkout-build-link.outputs.TEST_DIR }}/HOMEgfs
             source workflow/gw_setup.sh
             source ci/platforms/${{ env.MACHINE_ID }}.sh
             ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
@@ -86,6 +73,6 @@ jobs:
       - name: Run Experiment ${{ matrix.case }}
         run: |
           ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-            cd 
-            # Add commands to run the experiment
+            cd ${{ needs.checkout-build-link.outputs.TEST_DIR }}/HOMEgfs
+            ./ci/scripts/run_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
           EOF
\ No newline at end of file

From 08ab7ec2ccdf83ab3c12ce276f508df856288dd0 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Thu, 26 Sep 2024 16:55:04 -0400
Subject: [PATCH 21/95] missing header and on: dispatch

---
 .github/workflows/pw-ci-hera-ssh.yaml | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 5aee69f784..51f5e58595 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -1,3 +1,16 @@
+name: pw-ci-hera
+
+on:
+  workflow_dispatch:
+
+# TEST_DIR contains 2 directories;
+# 1. HOMEgfs: clone of the global-workflow
+# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
+# e.g. $> tree ./TEST_DIR
+# ./TEST_DIR
+# ├── HOMEgfs
+# └── RUNTESTS
+
 jobs:
 
   checkout-build-link:

From 00f2d9b12024485cd4f32c89d03425158ba4ce34 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 27 Sep 2024 18:09:49 +0000
Subject: [PATCH 22/95] fix new location of create_experment and added local
 env def into next job

---
 .github/workflows/pw-ci-hera-ssh.yaml | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index dbac20834b..cd7c156548 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -28,6 +28,7 @@ jobs:
 
     - name: Set environment variables
       id: set-env
+
       run: |
         echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
 
@@ -61,6 +62,10 @@ jobs:
       matrix:
         case: ["C48_S2S", "C96_atm3DVar"]
 
+    env:
+      TEST_DIR: ${{ steps.set-env.outputs.TEST_DIR }}
+      MACHINE_ID: hera
+
     steps:
       - name: Create Experiments ${{ matrix.case }}
         env:
@@ -72,7 +77,7 @@ jobs:
             cd ${{ env.TEST_DIR }}/HOMEgfs
             source workflow/gw_setup.sh
             source ci/platforms/${{ env.MACHINE_ID }}.sh
-            ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
+            workflow/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
           EOF
 
   run-experiments:
@@ -82,10 +87,15 @@ jobs:
       max-parallel: 2
       matrix:
         case: ["C48_S2S", "C96_atm3DVar"]
+
+      env:
+        TEST_DIR: ${{ steps.set-env.outputs.TEST_DIR }}
+        MACHINE_ID: hera
+
     steps:
       - name: Run Experiment ${{ matrix.case }}
         run: |
           ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-            cd 
-            # Add commands to run the experiment
+            cd ${{ env.TEST_DIR }}/HOMEgfs
+            echo "$PWD: Skipping Run Experiment ${{ matrix.case }} for now"
           EOF
\ No newline at end of file

From c0734a91491dc44ee3b8384c540552d2060c0362 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 27 Sep 2024 18:15:20 +0000
Subject: [PATCH 23/95] moved env section in create-experiments

---
 .github/workflows/pw-ci-hera-ssh.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 56c520a82d..40a76c9b23 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -58,14 +58,15 @@ jobs:
   create-experiments:
     needs: checkout-build-link
     runs-on: [self-hosted, parallel-works, pw]
-    strategy:
-      matrix:
-        case: ["C48_S2S", "C96_atm3DVar"]
 
     env:
       TEST_DIR: ${{ steps.set-env.outputs.TEST_DIR }}
       MACHINE_ID: hera
 
+    strategy:
+      matrix:
+        case: ["C48_S2S", "C96_atm3DVar"]
+
     steps:
       - name: Create Experiments ${{ matrix.case }}
         env:

From 146aa62c9194efc69d40d783aaba173f67e2ed80 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 27 Sep 2024 18:22:50 +0000
Subject: [PATCH 24/95] corrected path to getting TEST_DIR address in
 create-experiments

---
 .github/workflows/pw-ci-hera-ssh.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 40a76c9b23..66de544069 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -60,7 +60,7 @@ jobs:
     runs-on: [self-hosted, parallel-works, pw]
 
     env:
-      TEST_DIR: ${{ steps.set-env.outputs.TEST_DIR }}
+      TEST_DIR: ${{ needs.checkout-build-link.outputs.TEST_DIR }}
       MACHINE_ID: hera
 
     strategy:

From 09f80759ab2b37e284e57737bc991702b176f98c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 27 Sep 2024 18:29:45 +0000
Subject: [PATCH 25/95] moved env for HOMEgfs into steps on create

---
 .github/workflows/pw-ci-hera-ssh.yaml | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 66de544069..07867b1aa9 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -59,10 +59,6 @@ jobs:
     needs: checkout-build-link
     runs-on: [self-hosted, parallel-works, pw]
 
-    env:
-      TEST_DIR: ${{ needs.checkout-build-link.outputs.TEST_DIR }}
-      MACHINE_ID: hera
-
     strategy:
       matrix:
         case: ["C48_S2S", "C96_atm3DVar"]
@@ -73,9 +69,10 @@ jobs:
           HOMEgfs_PR: ${{ needs.checkout-build-link.outputs.TEST_DIR }}/HOMEgfs
           RUNTESTS: ${{ needs.checkout-build-link.outputs.TEST_DIR }}/RUNTESTS
           pslot: ${{ matrix.case }}.${{ github.run_id }}
+          MACHINE_ID: hera
         run: |
           ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-            cd ${{ needs.checkout-build-link.outputs.TEST_DIR }}/HOMEgfs
+            cd ${{ env.HOMEgfs_PR }}
             source workflow/gw_setup.sh
             source ci/platforms/${{ env.MACHINE_ID }}.sh
             workflow/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml

From daddf59e1625c20056e9768b907f7e4709f04f27 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 27 Sep 2024 18:36:20 +0000
Subject: [PATCH 26/95] updaed env for run experiments too

---
 .github/workflows/pw-ci-hera-ssh.yaml | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 07867b1aa9..ee37c3c840 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -17,9 +17,6 @@ jobs:
     runs-on: [self-hosted, parallel-works, pw]
     timeout-minutes: 600
 
-    env:
-      MACHINE_ID: hera
-
     outputs:
       TEST_DIR: ${{ steps.set-env.outputs.TEST_DIR }}
 
@@ -66,13 +63,13 @@ jobs:
     steps:
       - name: Create Experiments ${{ matrix.case }}
         env:
-          HOMEgfs_PR: ${{ needs.checkout-build-link.outputs.TEST_DIR }}/HOMEgfs
+          HOMEgfs: ${{ needs.checkout-build-link.outputs.TEST_DIR }}/HOMEgfs
           RUNTESTS: ${{ needs.checkout-build-link.outputs.TEST_DIR }}/RUNTESTS
           pslot: ${{ matrix.case }}.${{ github.run_id }}
           MACHINE_ID: hera
         run: |
           ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-            cd ${{ env.HOMEgfs_PR }}
+            cd ${{ env.HOMEgfs }}
             source workflow/gw_setup.sh
             source ci/platforms/${{ env.MACHINE_ID }}.sh
             workflow/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
@@ -86,12 +83,11 @@ jobs:
       matrix:
         case: ["C48_S2S", "C96_atm3DVar"]
 
+    steps:
+      - name: Run Experiment ${{ matrix.case }}
       env:
         TEST_DIR: ${{ steps.set-env.outputs.TEST_DIR }}
         MACHINE_ID: hera
-
-    steps:
-      - name: Run Experiment ${{ matrix.case }}
         run: |
           ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
             cd ${{ env.TEST_DIR }}/HOMEgfs

From d8b9b1310aa79c4c4e3d14c2243d841a8ed7e4da Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 27 Sep 2024 18:41:12 +0000
Subject: [PATCH 27/95] update env section to define HOMEgfs in run

---
 .github/workflows/pw-ci-hera-ssh.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index ee37c3c840..6e74355a74 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -85,11 +85,11 @@ jobs:
 
     steps:
       - name: Run Experiment ${{ matrix.case }}
-      env:
-        TEST_DIR: ${{ steps.set-env.outputs.TEST_DIR }}
-        MACHINE_ID: hera
+        env:
+          HOMEgfs: ${{ needs.checkout-build-link.outputs.TEST_DIR }}/HOMEgfs
+          MACHINE_ID: hera
         run: |
           ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-            cd ${{ env.TEST_DIR }}/HOMEgfs
+            cd ${{ env.HOMEgfs }}
             echo "$PWD: Skipping Run Experiment ${{ matrix.case }} for now"
           EOF
\ No newline at end of file

From 6fa30fb50f4d906ad364cb9a1879a4d00d47dedf Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 27 Sep 2024 19:51:17 +0000
Subject: [PATCH 28/95] need pr in path for cases

---
 .github/workflows/pw-ci-hera-ssh.yaml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 6e74355a74..35e4ee28b6 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -34,15 +34,15 @@ jobs:
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
          mkdir -p ${{ env.TEST_DIR }}/HOMEgfs
          cd ${{ env.TEST_DIR }}/HOMEgfs
-         git clone --branch ${{ github.ref_name }} https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}.git .
-         git submodule update --init --recursive
+         #git clone --branch ${{ github.ref_name }} https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}.git .
+         #git submodule update --init --recursive
         EOF
 
     - name: Build components
       run: |
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
           cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-          ./build_all.sh
+          #./build_all.sh
         EOF
 
     - name: Link artifacts
@@ -58,7 +58,7 @@ jobs:
 
     strategy:
       matrix:
-        case: ["C48_S2S", "C96_atm3DVar"]
+        case: ["C48_ATM", "C96_atm3DVar"]
 
     steps:
       - name: Create Experiments ${{ matrix.case }}
@@ -71,8 +71,8 @@ jobs:
           ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
             cd ${{ env.HOMEgfs }}
             source workflow/gw_setup.sh
-            source ci/platforms/${{ env.MACHINE_ID }}.sh
-            workflow/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
+            source ci/platforms/config.${{ env.MACHINE_ID }}
+            workflow/create_experiment.py --yaml ci/cases/pr/${{ matrix.case }}.yaml
           EOF
 
   run-experiments:
@@ -81,7 +81,7 @@ jobs:
     strategy:
       max-parallel: 2
       matrix:
-        case: ["C48_S2S", "C96_atm3DVar"]
+        case: ["C48_ATM", "C96_atm3DVar"]
 
     steps:
       - name: Run Experiment ${{ matrix.case }}

From c4c99dfa42366961b541f9973fa25232154fb507 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 27 Sep 2024 19:55:22 +0000
Subject: [PATCH 29/95] hard coded id so not to have to rebuild

---
 .github/workflows/pw-ci-hera-ssh.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 35e4ee28b6..cef862210b 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -26,8 +26,10 @@ jobs:
       id: set-env
 
       run: |
-        echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
-        echo "::set-output name=TEST_DIR::/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}"
+        #echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+        #echo "::set-output name=TEST_DIR::/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}"
+        echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/11076014062-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+        echo "::set-output name=TEST_DIR::/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/11076014062-${{ github.ref_name }}/${{ github.event.pull_request.number }}"
 
     - name: Checkout global-workflow
       run: |

From 71a8d87a7614d0f02f5bc922498b734f9a8ff091 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 27 Sep 2024 19:59:21 +0000
Subject: [PATCH 30/95] added overwrite to create experment

---
 .github/workflows/pw-ci-hera-ssh.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index cef862210b..5908da9fda 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -74,7 +74,7 @@ jobs:
             cd ${{ env.HOMEgfs }}
             source workflow/gw_setup.sh
             source ci/platforms/config.${{ env.MACHINE_ID }}
-            workflow/create_experiment.py --yaml ci/cases/pr/${{ matrix.case }}.yaml
+            workflow/create_experiment.py --overwrite --yaml ci/cases/pr/${{ matrix.case }}.yaml
           EOF
 
   run-experiments:

From 70e249a6d1c9a7440c838db012c562a72af69924 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 27 Sep 2024 20:05:45 +0000
Subject: [PATCH 31/95] removed skips

---
 .github/workflows/pw-ci-hera-ssh.yaml | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 5908da9fda..7b81ecf84e 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -26,25 +26,23 @@ jobs:
       id: set-env
 
       run: |
-        #echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
-        #echo "::set-output name=TEST_DIR::/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}"
-        echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/11076014062-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
-        echo "::set-output name=TEST_DIR::/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/11076014062-${{ github.ref_name }}/${{ github.event.pull_request.number }}"
+        echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+        echo "::set-output name=TEST_DIR::/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}"
 
     - name: Checkout global-workflow
       run: |
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
          mkdir -p ${{ env.TEST_DIR }}/HOMEgfs
          cd ${{ env.TEST_DIR }}/HOMEgfs
-         #git clone --branch ${{ github.ref_name }} https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}.git .
-         #git submodule update --init --recursive
+         git clone --branch ${{ github.ref_name }} https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}.git .
+         git submodule update --init --recursive
         EOF
 
     - name: Build components
       run: |
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
           cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-          #./build_all.sh
+          ./build_all.sh
         EOF
 
     - name: Link artifacts

From 19c2c86146f4b732ce1cd59236ce2a48ad2015e7 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 30 Sep 2024 16:13:37 +0000
Subject: [PATCH 32/95] removed direct git clone and used cross mounted path to
 RHHPCS directly using pw env inorder to use github action checkout@v3

---
 .github/workflows/pw-ci-hera-ssh.yaml | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 7b81ecf84e..4ec1c5d4a1 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -26,17 +26,16 @@ jobs:
       id: set-env
 
       run: |
-        echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+        echo "CHECKOUT_DIR_PW=$HOME/pw/clusters/noaaheracluster/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
         echo "::set-output name=TEST_DIR::/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}"
 
     - name: Checkout global-workflow
-      run: |
-        ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-         mkdir -p ${{ env.TEST_DIR }}/HOMEgfs
-         cd ${{ env.TEST_DIR }}/HOMEgfs
-         git clone --branch ${{ github.ref_name }} https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}.git .
-         git submodule update --init --recursive
-        EOF
+      uses: actions/checkout@v3
+      with:
+        path: ${{ steps.set-env.outputs.CHKOUT_DIR_PW }}/HOMEgfs
+        submodules: true
+        submodules-fetch-depth: 1
+        submodules-recurse: true
 
     - name: Build components
       run: |

From 751054dc840c7e68e5c3f3b604ad51a26d647aa9 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 30 Sep 2024 17:14:51 +0000
Subject: [PATCH 33/95] made distinction between runners path and RDPHCS path
 with CHECKOUT_DIR_PW and TEST_DIR respectivily

---
 .github/workflows/pw-ci-hera-ssh.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 4ec1c5d4a1..9522628f16 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -26,8 +26,8 @@ jobs:
       id: set-env
 
       run: |
-        echo "CHECKOUT_DIR_PW=$HOME/pw/clusters/noaaheracluster/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
-        echo "::set-output name=TEST_DIR::/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}"
+        echo "CHECKOUT_DIR_PW=${{ github.workspace }}/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+        echo "::set-output name=TEST_DIR::/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}"
 
     - name: Checkout global-workflow
       uses: actions/checkout@v3

From 9b163e94a453ceb7529522be3d3b2e06702cbfad Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 30 Sep 2024 17:34:18 +0000
Subject: [PATCH 34/95] fixed reference to CHECKOUT_DIR and updated submodule
 arguments to checkout@v3

---
 .github/workflows/pw-ci-hera-ssh.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 9522628f16..25cc86dde6 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -27,15 +27,16 @@ jobs:
 
       run: |
         echo "CHECKOUT_DIR_PW=${{ github.workspace }}/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+        echo "TEST_DIR::/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
         echo "::set-output name=TEST_DIR::/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}"
 
     - name: Checkout global-workflow
       uses: actions/checkout@v3
       with:
-        path: ${{ steps.set-env.outputs.CHKOUT_DIR_PW }}/HOMEgfs
+        path: ${{ env.CHECKOUT_DIR_PW }}/HOMEgfs
         submodules: true
-        submodules-fetch-depth: 1
-        submodules-recurse: true
+        fetch-depth: 1
+        lsf: true
 
     - name: Build components
       run: |

From bd8da4eae612258f85cad487bddee8e73a6c1602 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 30 Sep 2024 17:48:25 +0000
Subject: [PATCH 35/95] updated GITHUB ENV and OUTPUT syntax

---
 .github/workflows/pw-ci-hera-ssh.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 25cc86dde6..e19d2eb2f6 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -27,8 +27,8 @@ jobs:
 
       run: |
         echo "CHECKOUT_DIR_PW=${{ github.workspace }}/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
-        echo "TEST_DIR::/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
-        echo "::set-output name=TEST_DIR::/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}"
+        echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+        echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
 
     - name: Checkout global-workflow
       uses: actions/checkout@v3

From fd878f98825e51f5a50bff66a9c98615110d1926 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 30 Sep 2024 18:35:14 +0000
Subject: [PATCH 36/95] added lfs and shallow depth for submodules to get
 checkout to work

---
 .github/workflows/pw-ci-hera-ssh.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index e19d2eb2f6..7add3b07e4 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -35,8 +35,9 @@ jobs:
       with:
         path: ${{ env.CHECKOUT_DIR_PW }}/HOMEgfs
         submodules: true
+        submodules-depth: 1
         fetch-depth: 1
-        lsf: true
+        lfs: true
 
     - name: Build components
       run: |

From 15dbf304851d35418baa30b6e172595b9354c56c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 30 Sep 2024 18:49:09 +0000
Subject: [PATCH 37/95] turn off git-lfs becasue it is not in the pw env

---
 .github/workflows/pw-ci-hera-ssh.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 7add3b07e4..8aec09c868 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -37,7 +37,7 @@ jobs:
         submodules: true
         submodules-depth: 1
         fetch-depth: 1
-        lfs: true
+        lfs: false
 
     - name: Build components
       run: |

From 4a85a180eba95fff380dcd632440b1df996e6f73 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 30 Sep 2024 19:52:54 +0000
Subject: [PATCH 38/95] gdas checkout submodule failed again because it did not
 complete, removed lfs: false in case that was causing something to missed

---
 .github/workflows/pw-ci-hera-ssh.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 8aec09c868..0f4493b9b6 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -37,7 +37,6 @@ jobs:
         submodules: true
         submodules-depth: 1
         fetch-depth: 1
-        lfs: false
 
     - name: Build components
       run: |

From ec4e85051607c722e99acd7576534e42f9c2ab4c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 30 Sep 2024 20:39:04 +0000
Subject: [PATCH 39/95] added extra git configure to try to help with the
 checkout issues with the gdas.cd submodule clone issues

---
 .github/workflows/pw-ci-hera-ssh.yaml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 0f4493b9b6..5546e5fc69 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -30,12 +30,19 @@ jobs:
         echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
         echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
 
+    - name: Set Git postBuffer size
+      run: |
+        git config --global http.postBuffer 524288000
+        git config --global http.maxRequestBuffer 100M
+        git config --global core.compression 0
+        git config --global submodule.fetchJobs 1
+        git config --global submodule.'gdas.cd'.shallow true
+
     - name: Checkout global-workflow
       uses: actions/checkout@v3
       with:
         path: ${{ env.CHECKOUT_DIR_PW }}/HOMEgfs
         submodules: true
-        submodules-depth: 1
         fetch-depth: 1
 
     - name: Build components

From 7c11c96fb2fe16fb976c857fc4e9feebcb7891b0 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 30 Sep 2024 22:21:50 +0000
Subject: [PATCH 40/95] upgrade checkout@v3 to checckout@v4.2.0

---
 .github/workflows/pw-ci-hera-ssh.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 5546e5fc69..1595b81f5d 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -39,7 +39,7 @@ jobs:
         git config --global submodule.'gdas.cd'.shallow true
 
     - name: Checkout global-workflow
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4.2.0
       with:
         path: ${{ env.CHECKOUT_DIR_PW }}/HOMEgfs
         submodules: true

From bf42c6a87af417e62db0a480029d27b149a175c6 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 30 Sep 2024 22:24:12 +0000
Subject: [PATCH 41/95] removed redundent specification of TEST_DIR when
 writing to GITHUB_OUTPUT

---
 .github/workflows/pw-ci-hera-ssh.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 1595b81f5d..1083eb1ea9 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -28,7 +28,7 @@ jobs:
       run: |
         echo "CHECKOUT_DIR_PW=${{ github.workspace }}/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
         echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
-        echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
+        echo "TEST_DIR=${TEST_DIR}" >> $GITHUB_OUTPUT
 
     - name: Set Git postBuffer size
       run: |

From 0e91eba374fbf426acc4af021e2a855a39ad400c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 30 Sep 2024 22:49:33 +0000
Subject: [PATCH 42/95] also added verif-global.fd as a shallow clone

---
 .github/workflows/pw-ci-hera-ssh.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 1083eb1ea9..920b0a5d6e 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -37,6 +37,7 @@ jobs:
         git config --global core.compression 0
         git config --global submodule.fetchJobs 1
         git config --global submodule.'gdas.cd'.shallow true
+        git config --global submodule.'verif-global.fd'.shallow true
 
     - name: Checkout global-workflow
       uses: actions/checkout@v4.2.0

From 9c00b4ee872382c2830c747e585a8e3cf2e56ebe Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 1 Oct 2024 00:25:11 +0000
Subject: [PATCH 43/95] rsync check out dir after it is cloned on pw onto
 RDPHCS cross mounted link in HOME dir

---
 .github/workflows/pw-ci-hera-ssh.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 920b0a5d6e..df133c602b 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -48,6 +48,8 @@ jobs:
 
     - name: Build components
       run: |
+        mkdir -p $HOME/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}
+        rsync -av ${{ env.CHECKOUT_DIR_PW }}/HOMEgfs $HOME/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
           cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
           ./build_all.sh

From a5db8bd24f82463459b4286e44b1f75471f2ee75 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 1 Oct 2024 00:41:30 +0000
Subject: [PATCH 44/95] worked on correct path for rsync to RDHPCS

---
 .github/workflows/pw-ci-hera-ssh.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index df133c602b..275c3c5984 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -48,8 +48,9 @@ jobs:
 
     - name: Build components
       run: |
-        mkdir -p $HOME/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}
-        rsync -av ${{ env.CHECKOUT_DIR_PW }}/HOMEgfs $HOME/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}
+        TARGET_DIR=$HOME/pw/clusters/noaaheracluster/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}
+        mkdir -p $TARGET_DIR
+        rsync -av ${{ env.CHECKOUT_DIR_PW }}/HOMEgfs/ $TARGET_DIR/HOMEgfs
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
           cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
           ./build_all.sh

From 6454f416289e08463cc9ddae589ebaa47c7fb924 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 1 Oct 2024 00:55:05 +0000
Subject: [PATCH 45/95] remore optons to chmod on dest in rysnc

---
 .github/workflows/pw-ci-hera-ssh.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
index 275c3c5984..26fc852393 100644
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ b/.github/workflows/pw-ci-hera-ssh.yaml
@@ -50,7 +50,7 @@ jobs:
       run: |
         TARGET_DIR=$HOME/pw/clusters/noaaheracluster/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}
         mkdir -p $TARGET_DIR
-        rsync -av ${{ env.CHECKOUT_DIR_PW }}/HOMEgfs/ $TARGET_DIR/HOMEgfs
+        rsync -a --no-o --no-g --no-perms ${{ env.CHECKOUT_DIR_PW }}/HOMEgfs/ $TARGET_DIR/HOMEgfs
         ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
           cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
           ./build_all.sh

From 9cfab8ed2823eec8c903b35ee9de5be47be6cf5d Mon Sep 17 00:00:00 2001
From: Parallel Works app-run user
 <Terry.McGuinness@mgmt-terrymcguinness-globalworkflowciplatform-00001.pw-noaa-us-east-1.pw.local>
Date: Wed, 2 Oct 2024 23:10:12 +0000
Subject: [PATCH 46/95] update fms version to match spack stack on pw

---
 versions/spack.ver | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/versions/spack.ver b/versions/spack.ver
index 1f553134f0..410f509eb6 100644
--- a/versions/spack.ver
+++ b/versions/spack.ver
@@ -6,7 +6,7 @@ export jasper_ver=2.0.32
 export libpng_ver=1.6.37
 export zlib_ver=1.2.13
 export esmf_ver=8.5.0
-export fms_ver=2023.02.01
+export fms_ver=2023.04
 export cdo_ver=2.2.0
 export nco_ver=5.0.6
 

From 249bd9fac6ed60f03e0b0592aa28b504d4815652 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 2 Oct 2024 23:38:08 +0000
Subject: [PATCH 47/95] updated run.sh to use MACHINE_ID in load fv4gfs modules
 script

---
 ush/load_fv3gfs_modules.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh
index ff6f64cece..7a7c8ba16c 100755
--- a/ush/load_fv3gfs_modules.sh
+++ b/ush/load_fv3gfs_modules.sh
@@ -13,9 +13,6 @@ ulimit_s=$( ulimit -S -s )
 source "${HOMEgfs}/ush/detect_machine.sh"
 source "${HOMEgfs}/ush/module-setup.sh"
 
-# Source versions file for runtime
-source "${HOMEgfs}/versions/run.ver"
-
 # Load our modules:
 module use "${HOMEgfs}/modulefiles"
 
@@ -28,6 +25,9 @@ case "${MACHINE_ID}" in
     ;;
 esac
 
+# Source versions file for runtime
+source "${HOMEgfs}/versions/run.${MACHINE_ID}.ver"
+
 module list
 
 # Add wxflow to PYTHONPATH

From 6267f58ba1482640d246eecf39e1e5c05da7c143 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Thu, 3 Oct 2024 01:39:47 +0000
Subject: [PATCH 48/95] updated version of g2tmpl and change to module file
 noaacloud.intl for ufs_modle.fd

---
 sorc/ufs_noaacloud.intel.lua | 24 ++++++++++++++++++++++++
 versions/spack.ver           |  2 +-
 2 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100644 sorc/ufs_noaacloud.intel.lua

diff --git a/sorc/ufs_noaacloud.intel.lua b/sorc/ufs_noaacloud.intel.lua
new file mode 100644
index 0000000000..3c2b44c6ee
--- /dev/null
+++ b/sorc/ufs_noaacloud.intel.lua
@@ -0,0 +1,24 @@
+help([[
+loads UFS Model prerequisites for NOAA Parallelworks/Intel
+]])
+
+prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core")
+
+stack_intel_ver=os.getenv("stack_intel_ver") or "2021.3.0"
+load(pathJoin("stack-intel", stack_intel_ver))
+
+stack_impi_ver=os.getenv("stack_impi_ver") or "2021.3.0"
+load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
+
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
+load(pathJoin("cmake", cmake_ver))
+
+load("ufs_common")
+
+setenv("CC", "mpiicc")
+setenv("CXX", "mpiicpc")
+setenv("FC", "mpiifort")
+setenv("CMAKE_Platform", "noaacloud.intel")
+
+whatis("Description: UFS build environment")
diff --git a/versions/spack.ver b/versions/spack.ver
index 410f509eb6..dbe304a5fe 100644
--- a/versions/spack.ver
+++ b/versions/spack.ver
@@ -23,7 +23,7 @@ export g2_ver=3.4.5
 export sp_ver=2.5.0
 export ip_ver=4.3.0
 export gsi_ncdiag_ver=1.1.2
-export g2tmpl_ver=1.10.2
+export g2tmpl_ver=1.13.0
 export crtm_ver=2.4.0.1
 export wgrib2_ver=2.0.8
 export grib_util_ver=1.3.0

From 12fc5c04b86ab1d33c1bd62ab5b374af2b24bcf2 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Thu, 3 Oct 2024 17:07:35 +0000
Subject: [PATCH 49/95] add patch for updating ufs_noaacloud.intel module file
 from global-workflow repo (temp)

---
 sorc/build_ufs.sh             |  5 +++++
 sorc/ufs_noaacloud.intel.diff | 10 ++++++++++
 sorc/ufs_noaacloud.intel.lua  | 24 ------------------------
 3 files changed, 15 insertions(+), 24 deletions(-)
 create mode 100644 sorc/ufs_noaacloud.intel.diff
 delete mode 100644 sorc/ufs_noaacloud.intel.lua

diff --git a/sorc/build_ufs.sh b/sorc/build_ufs.sh
index 44c8c7a2ad..a69ce6b2d7 100755
--- a/sorc/build_ufs.sh
+++ b/sorc/build_ufs.sh
@@ -41,6 +41,11 @@ COMPILE_NR=0
 CLEAN_BEFORE=YES
 CLEAN_AFTER=NO
 
+#TODO temp patch for build update for noaacload in advance of updating ufs_module.fd repo for global-workflow building
+if [[ "${MACHINE_ID}" == "noaacloud" ]] ; then
+    patch -R ufs_model.fd/modulefiles/ufs_noaacloud.intel.lua ufs_noaacloud.intel.diff
+fi
+
 BUILD_JOBS=${BUILD_JOBS:-8} ./tests/compile.sh "${MACHINE_ID}" "${MAKE_OPT}" "${COMPILE_NR}" "intel" "${CLEAN_BEFORE}" "${CLEAN_AFTER}"
 mv "./tests/fv3_${COMPILE_NR}.exe" ./tests/ufs_model.x
 mv "./tests/modules.fv3_${COMPILE_NR}.lua" ./tests/modules.ufs_model.lua
diff --git a/sorc/ufs_noaacloud.intel.diff b/sorc/ufs_noaacloud.intel.diff
new file mode 100644
index 0000000000..5080781641
--- /dev/null
+++ b/sorc/ufs_noaacloud.intel.diff
@@ -0,0 +1,10 @@
+--- ufs_noaacloud.intel.lua	2024-10-03 15:54:33.334583588 +0000
++++ ufs_model.fd/modulefiles/ufs_noaacloud.intel.lua	2024-10-03 16:11:28.534275972 +0000
+@@ -3,7 +3,6 @@
+ ]])
+ 
+ prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
+-prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core")
+ 
+ stack_intel_ver=os.getenv("stack_intel_ver") or "2021.3.0"
+ load(pathJoin("stack-intel", stack_intel_ver))
diff --git a/sorc/ufs_noaacloud.intel.lua b/sorc/ufs_noaacloud.intel.lua
deleted file mode 100644
index 3c2b44c6ee..0000000000
--- a/sorc/ufs_noaacloud.intel.lua
+++ /dev/null
@@ -1,24 +0,0 @@
-help([[
-loads UFS Model prerequisites for NOAA Parallelworks/Intel
-]])
-
-prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
-prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core")
-
-stack_intel_ver=os.getenv("stack_intel_ver") or "2021.3.0"
-load(pathJoin("stack-intel", stack_intel_ver))
-
-stack_impi_ver=os.getenv("stack_impi_ver") or "2021.3.0"
-load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
-
-cmake_ver=os.getenv("cmake_ver") or "3.23.1"
-load(pathJoin("cmake", cmake_ver))
-
-load("ufs_common")
-
-setenv("CC", "mpiicc")
-setenv("CXX", "mpiicpc")
-setenv("FC", "mpiifort")
-setenv("CMAKE_Platform", "noaacloud.intel")
-
-whatis("Description: UFS build environment")

From 109b67c161baf85199c8aeb9f4a729f397d7a8cc Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Thu, 3 Oct 2024 18:44:41 +0000
Subject: [PATCH 50/95] add versioning for noaacloud and temp patch to
 ufs_module module file for noaacloud

---
 sorc/build_all.sh | 3 +++
 sorc/build_ufs.sh | 5 ++++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/sorc/build_all.sh b/sorc/build_all.sh
index b2f4e6ce0e..33b440f242 100755
--- a/sorc/build_all.sh
+++ b/sorc/build_all.sh
@@ -108,6 +108,9 @@ fi
 #------------------------------------
 # TODO: Commented out until components aligned for build
 #source ../versions/build.ver
+if [[ "${MACHINE_ID}" == "noaacloud" ]] ; then
+   source "../versions/build.${MACHINE_ID}.ver"
+fi
 
 #------------------------------------
 # Exception Handling Init
diff --git a/sorc/build_ufs.sh b/sorc/build_ufs.sh
index a69ce6b2d7..bacbf9fd85 100755
--- a/sorc/build_ufs.sh
+++ b/sorc/build_ufs.sh
@@ -43,7 +43,10 @@ CLEAN_AFTER=NO
 
 #TODO temp patch for build update for noaacload in advance of updating ufs_module.fd repo for global-workflow building
 if [[ "${MACHINE_ID}" == "noaacloud" ]] ; then
-    patch -R ufs_model.fd/modulefiles/ufs_noaacloud.intel.lua ufs_noaacloud.intel.diff
+    patched=$(grep upp-addon-env modulefiles/ufs_noaacloud.intel.lua; echo $?)
+    if [[ ${patched} == "1" ]] ; then
+       patch -R modulefiles/ufs_noaacloud.intel.lua ../ufs_noaacloud.intel.diff
+    fi   
 fi
 
 BUILD_JOBS=${BUILD_JOBS:-8} ./tests/compile.sh "${MACHINE_ID}" "${MAKE_OPT}" "${COMPILE_NR}" "intel" "${CLEAN_BEFORE}" "${CLEAN_AFTER}"

From 557a13fc4f0113ce4ac2b7e2b9e88abbac81e889 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Thu, 3 Oct 2024 19:01:55 +0000
Subject: [PATCH 51/95] removed githug action pipeline for pw to make this a
 build pr

---
 .github/workflows/pw-ci-hera-ssh.yaml | 106 --------------------------
 1 file changed, 106 deletions(-)
 delete mode 100644 .github/workflows/pw-ci-hera-ssh.yaml

diff --git a/.github/workflows/pw-ci-hera-ssh.yaml b/.github/workflows/pw-ci-hera-ssh.yaml
deleted file mode 100644
index 26fc852393..0000000000
--- a/.github/workflows/pw-ci-hera-ssh.yaml
+++ /dev/null
@@ -1,106 +0,0 @@
-name: pw-ci-hera
-
-on:
-  workflow_dispatch:
-
-# TEST_DIR contains 2 directories;
-# 1. HOMEgfs: clone of the global-workflow
-# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
-# e.g. $> tree ./TEST_DIR
-# ./TEST_DIR
-# ├── HOMEgfs
-# └── RUNTESTS
-
-jobs:
-
-  checkout-build-link:
-    runs-on: [self-hosted, parallel-works, pw]
-    timeout-minutes: 600
-
-    outputs:
-      TEST_DIR: ${{ steps.set-env.outputs.TEST_DIR }}
-
-    steps:
-
-    - name: Set environment variables
-      id: set-env
-
-      run: |
-        echo "CHECKOUT_DIR_PW=${{ github.workspace }}/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
-        echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
-        echo "TEST_DIR=${TEST_DIR}" >> $GITHUB_OUTPUT
-
-    - name: Set Git postBuffer size
-      run: |
-        git config --global http.postBuffer 524288000
-        git config --global http.maxRequestBuffer 100M
-        git config --global core.compression 0
-        git config --global submodule.fetchJobs 1
-        git config --global submodule.'gdas.cd'.shallow true
-        git config --global submodule.'verif-global.fd'.shallow true
-
-    - name: Checkout global-workflow
-      uses: actions/checkout@v4.2.0
-      with:
-        path: ${{ env.CHECKOUT_DIR_PW }}/HOMEgfs
-        submodules: true
-        fetch-depth: 1
-
-    - name: Build components
-      run: |
-        TARGET_DIR=$HOME/pw/clusters/noaaheracluster/GFS_CI_ROOT/pw/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}
-        mkdir -p $TARGET_DIR
-        rsync -a --no-o --no-g --no-perms ${{ env.CHECKOUT_DIR_PW }}/HOMEgfs/ $TARGET_DIR/HOMEgfs
-        ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-          cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-          ./build_all.sh
-        EOF
-
-    - name: Link artifacts
-      run: |
-        ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-          cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-          ./link_workflow.sh
-        EOF
-
-  create-experiments:
-    needs: checkout-build-link
-    runs-on: [self-hosted, parallel-works, pw]
-
-    strategy:
-      matrix:
-        case: ["C48_ATM", "C96_atm3DVar"]
-
-    steps:
-      - name: Create Experiments ${{ matrix.case }}
-        env:
-          HOMEgfs: ${{ needs.checkout-build-link.outputs.TEST_DIR }}/HOMEgfs
-          RUNTESTS: ${{ needs.checkout-build-link.outputs.TEST_DIR }}/RUNTESTS
-          pslot: ${{ matrix.case }}.${{ github.run_id }}
-          MACHINE_ID: hera
-        run: |
-          ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-            cd ${{ env.HOMEgfs }}
-            source workflow/gw_setup.sh
-            source ci/platforms/config.${{ env.MACHINE_ID }}
-            workflow/create_experiment.py --overwrite --yaml ci/cases/pr/${{ matrix.case }}.yaml
-          EOF
-
-  run-experiments:
-    needs: create-experiments
-    runs-on: [self-hosted, parallel-works, pw]
-    strategy:
-      max-parallel: 2
-      matrix:
-        case: ["C48_ATM", "C96_atm3DVar"]
-
-    steps:
-      - name: Run Experiment ${{ matrix.case }}
-        env:
-          HOMEgfs: ${{ needs.checkout-build-link.outputs.TEST_DIR }}/HOMEgfs
-          MACHINE_ID: hera
-        run: |
-          ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-            cd ${{ env.HOMEgfs }}
-            echo "$PWD: Skipping Run Experiment ${{ matrix.case }} for now"
-          EOF
\ No newline at end of file

From ca69f8aee6cc03ef026190ef1302ed2553a318e4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 3 Oct 2024 21:40:28 +0000
Subject: [PATCH 52/95] moved source of ver for runtime in load fv3gfs modules
 back to before the module is loaded

---
 ush/load_fv3gfs_modules.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh
index 7a7c8ba16c..a26f36a5bb 100755
--- a/ush/load_fv3gfs_modules.sh
+++ b/ush/load_fv3gfs_modules.sh
@@ -16,6 +16,9 @@ source "${HOMEgfs}/ush/module-setup.sh"
 # Load our modules:
 module use "${HOMEgfs}/modulefiles"
 
+# Source versions file for runtime
+source "${HOMEgfs}/versions/run.${MACHINE_ID}.ver"
+
 case "${MACHINE_ID}" in
   "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4" | "noaacloud")
     module load "module_base.${MACHINE_ID}"
@@ -25,9 +28,6 @@ case "${MACHINE_ID}" in
     ;;
 esac
 
-# Source versions file for runtime
-source "${HOMEgfs}/versions/run.${MACHINE_ID}.ver"
-
 module list
 
 # Add wxflow to PYTHONPATH

From 2366a9b7222b88eb2f36100120c6aba69541a008 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Thu, 3 Oct 2024 21:51:53 +0000
Subject: [PATCH 53/95] changed the runtime version of g2tmp down

---
 versions/run.noaacloud.ver | 1 +
 1 file changed, 1 insertion(+)

diff --git a/versions/run.noaacloud.ver b/versions/run.noaacloud.ver
index 1fc3779b2e..2f312b310e 100644
--- a/versions/run.noaacloud.ver
+++ b/versions/run.noaacloud.ver
@@ -6,3 +6,4 @@ source "${HOMEgfs:-}/versions/spack.ver"
 export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core"
 
 export cdo_ver=2.2.0
+export g2tmpl_ver=1.10.2

From 068ada615f8c88cc49d7ae7fa49296ddc6b67127 Mon Sep 17 00:00:00 2001
From: Parallel Works app-run user
 <Terry.McGuinness@mgmt-terrymcguinness-globalworkflowciplatform-00003.pw-noaa-us-east-1.pw.local>
Date: Fri, 4 Oct 2024 17:42:09 +0000
Subject: [PATCH 54/95] update bucket to contrib in awspw hosts yaml file for
 ICs

---
 workflow/hosts/awspw.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml
index ef17d8f2f4..a4e834a485 100644
--- a/workflow/hosts/awspw.yaml
+++ b/workflow/hosts/awspw.yaml
@@ -18,8 +18,8 @@ CHGRP_RSTPROD: 'YES'
 CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported.
 HPSSARCH: 'NO'
 HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below.
-BASE_DATA: '/bucket/global-workflow-shared-data'
-BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR'
+BASE_DATA: '/contrib/global-workflow-shared-data'
+BASE_IC: '/contrib/global-workflow-shared-data/ICSDIR'
 LOCALARCH: 'NO'
 ATARDIR: '' # TODO: This will not yet work from AWS.
 MAKE_NSSTBUFR: 'NO'

From e6da9e812e38fd4dbfa7a0ef7e6d87f9a09a7920 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Fri, 4 Oct 2024 19:58:26 +0000
Subject: [PATCH 55/95] updated location of ICs (no bucker dir in /contrib)

---
 ci/cases/pr/C48_ATM.yaml  | 4 ++--
 workflow/hosts/awspw.yaml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ci/cases/pr/C48_ATM.yaml b/ci/cases/pr/C48_ATM.yaml
index 79706556e6..4aafbed0af 100644
--- a/ci/cases/pr/C48_ATM.yaml
+++ b/ci/cases/pr/C48_ATM.yaml
@@ -8,6 +8,6 @@ arguments:
   resdetatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  idate: 2021032312
-  edate: 2021032312
+  idate: 2024010112
+  edate: 2024010112
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml
diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml
index a4e834a485..1672a85229 100644
--- a/workflow/hosts/awspw.yaml
+++ b/workflow/hosts/awspw.yaml
@@ -19,7 +19,7 @@ CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported.
 HPSSARCH: 'NO'
 HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below.
 BASE_DATA: '/contrib/global-workflow-shared-data'
-BASE_IC: '/contrib/global-workflow-shared-data/ICSDIR'
+BASE_IC: '/contrib/global-workflow-shared-data/ICs'
 LOCALARCH: 'NO'
 ATARDIR: '' # TODO: This will not yet work from AWS.
 MAKE_NSSTBUFR: 'NO'

From 5b077032581c4610ad9af9a37cfad1cd4a7ac095 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Fri, 4 Oct 2024 20:22:35 +0000
Subject: [PATCH 56/95] update start hour to 00 for C48_ATM

---
 ci/cases/pr/C48_ATM.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/cases/pr/C48_ATM.yaml b/ci/cases/pr/C48_ATM.yaml
index 4aafbed0af..354ea0736e 100644
--- a/ci/cases/pr/C48_ATM.yaml
+++ b/ci/cases/pr/C48_ATM.yaml
@@ -8,6 +8,6 @@ arguments:
   resdetatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  idate: 2024010112
-  edate: 2024010112
+  idate: 2024010100
+  edate: 2024010100
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

From 40a8592dfe29d290ab67597c323d4a3d897451d8 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@nooa.gov>
Date: Tue, 8 Oct 2024 19:09:10 +0000
Subject: [PATCH 57/95] added first pass at GitHub Action Pipelines for CI on
 pw

---
 .../workflows/globalworkflow-ci-pw-ssh.yaml   | 91 +++++++++++++++++++
 .github/workflows/globalworkflow-ci-pw.yaml   | 80 ++++++++++++++++
 2 files changed, 171 insertions(+)
 create mode 100644 .github/workflows/globalworkflow-ci-pw-ssh.yaml
 create mode 100644 .github/workflows/globalworkflow-ci-pw.yaml

diff --git a/.github/workflows/globalworkflow-ci-pw-ssh.yaml b/.github/workflows/globalworkflow-ci-pw-ssh.yaml
new file mode 100644
index 0000000000..51f5e58595
--- /dev/null
+++ b/.github/workflows/globalworkflow-ci-pw-ssh.yaml
@@ -0,0 +1,91 @@
+name: pw-ci-hera
+
+on:
+  workflow_dispatch:
+
+# TEST_DIR contains 2 directories;
+# 1. HOMEgfs: clone of the global-workflow
+# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
+# e.g. $> tree ./TEST_DIR
+# ./TEST_DIR
+# ├── HOMEgfs
+# └── RUNTESTS
+
+jobs:
+
+  checkout-build-link:
+    runs-on: [self-hosted, parallel-works, pw]
+    timeout-minutes: 600
+
+    env:
+      MACHINE_ID: hera
+
+    outputs:
+      TEST_DIR: ${{ steps.set-env.outputs.TEST_DIR }}
+
+    steps:
+
+    - name: Set environment variables
+      id: set-env
+      run: |
+        echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+        echo "::set-output name=TEST_DIR::/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}"
+
+    - name: Checkout global-workflow
+      run: |
+        ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
+         mkdir -p ${{ env.TEST_DIR }}/HOMEgfs
+         cd ${{ env.TEST_DIR }}/HOMEgfs
+         git clone --branch ${{ github.ref_name }} https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}.git .
+         git submodule update --init --recursive
+        EOF
+
+    - name: Build components
+      run: |
+        ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
+          cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
+          ./build_all.sh
+        EOF
+
+    - name: Link artifacts
+      run: |
+        ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
+          cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
+          ./link_workflow.sh
+        EOF
+
+  create-experiments:
+    needs: checkout-build-link
+    runs-on: [self-hosted, parallel-works, pw]
+    strategy:
+      matrix:
+        case: ["C48_S2S", "C96_atm3DVar"]
+
+    steps:
+      - name: Create Experiments ${{ matrix.case }}
+        env:
+          HOMEgfs_PR: ${{ needs.checkout-build-link.outputs.TEST_DIR }}/HOMEgfs
+          RUNTESTS: ${{ needs.checkout-build-link.outputs.TEST_DIR }}/RUNTESTS
+          pslot: ${{ matrix.case }}.${{ github.run_id }}
+        run: |
+          ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
+            cd ${{ needs.checkout-build-link.outputs.TEST_DIR }}/HOMEgfs
+            source workflow/gw_setup.sh
+            source ci/platforms/${{ env.MACHINE_ID }}.sh
+            ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
+          EOF
+
+  run-experiments:
+    needs: create-experiments
+    runs-on: [self-hosted, parallel-works, pw]
+    strategy:
+      max-parallel: 2
+      matrix:
+        case: ["C48_S2S", "C96_atm3DVar"]
+    steps:
+      - name: Run Experiment ${{ matrix.case }}
+        run: |
+          ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
+            cd ${{ needs.checkout-build-link.outputs.TEST_DIR }}/HOMEgfs
+            ./ci/scripts/run_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
+          EOF
\ No newline at end of file
diff --git a/.github/workflows/globalworkflow-ci-pw.yaml b/.github/workflows/globalworkflow-ci-pw.yaml
new file mode 100644
index 0000000000..22e99ecb2b
--- /dev/null
+++ b/.github/workflows/globalworkflow-ci-pw.yaml
@@ -0,0 +1,80 @@
+name: gw-ci-aws
+
+on: [workflow_dispatch]
+
+# TEST_DIR contains 2 directories;
+# 1. HOMEgfs: clone of the global-workflow
+# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
+# e.g. $> tree ./TEST_DIR
+# ./TEST_DIR
+# ├── HOMEgfs
+# └── RUNTESTS
+#     ├── COMROT
+#     │   └── ${pslot}
+#     └── EXPDIR
+#         └── ${pslot}
+env:
+  TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
+  MACHINE_ID: noaacloud
+
+jobs:
+  checkout-build-link:
+    runs-on: [self-hosted, aws, pw]
+    timeout-minutes: 600
+
+    steps:
+    - name: Checkout global-workflow
+      uses: actions/checkout@v4
+      with:
+        path: ${{ github.run_id }}/HOMEgfs  # This path needs to be relative
+
+    - name: Build components
+      run: |
+        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
+        ./build_all.sh -j 8
+
+    - name: Link artifacts
+      run: |
+        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
+        ./link_workflow.sh
+
+  create-experiments:
+    needs: checkout-build-link
+    runs-on: [self-hosted, aws, pw]
+    strategy:
+      matrix:
+        case: ["C48_ATM"]
+
+    steps:
+      - name: Create Experiments ${{ matrix.case }}
+        env:
+          HOMEgfs_PR: ${{ env.TEST_DIR }}/HOMEgfs
+          RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS
+          pslot: ${{ matrix.case }}.${{ github.run_id }}
+        run: |
+          cd ${{ env.TEST_DIR }}/HOMEgfs
+          source workflow/gw_setup.sh
+          source ci/platforms/config.noaacloud
+          ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
+
+  run-experiments:
+    needs: create-experiments
+    runs-on: [self-hosted, aws, pw]
+    strategy:
+      max-parallel: 2
+      matrix:
+        case: ["C48_ATM"]
+    steps:
+      - name: Run Experiment ${{ matrix.case }}
+        run: |
+          cd ${{ env.TEST_DIR }}/HOMEgfs
+          ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }}
+
+  clean-up:
+    needs: run-experiments
+    runs-on: [self-hosted, aws, pw]
+    steps:
+      - name: Clean-up
+        run: |
+          cd ${{ github.workspace }}
+          rm -rf ${{ github.run_id }}

From 3616743ec3e6c51e785f647f6d1a51fd0425e319 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Tue, 8 Oct 2024 16:24:13 -0400
Subject: [PATCH 58/95] updated pw to paralleworks in lables to runners

---
 .github/workflows/globalworkflow-ci-pw.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/globalworkflow-ci-pw.yaml b/.github/workflows/globalworkflow-ci-pw.yaml
index 22e99ecb2b..a63153901a 100644
--- a/.github/workflows/globalworkflow-ci-pw.yaml
+++ b/.github/workflows/globalworkflow-ci-pw.yaml
@@ -19,7 +19,7 @@ env:
 
 jobs:
   checkout-build-link:
-    runs-on: [self-hosted, aws, pw]
+    runs-on: [self-hosted, aws, parallelworks]
     timeout-minutes: 600
 
     steps:
@@ -40,7 +40,7 @@ jobs:
 
   create-experiments:
     needs: checkout-build-link
-    runs-on: [self-hosted, aws, pw]
+    runs-on: [self-hosted, aws, parallelworks]
     strategy:
       matrix:
         case: ["C48_ATM"]
@@ -59,7 +59,7 @@ jobs:
 
   run-experiments:
     needs: create-experiments
-    runs-on: [self-hosted, aws, pw]
+    runs-on: [self-hosted, aws, parallelworks]
     strategy:
       max-parallel: 2
       matrix:
@@ -72,7 +72,7 @@ jobs:
 
   clean-up:
     needs: run-experiments
-    runs-on: [self-hosted, aws, pw]
+    runs-on: [self-hosted, aws, parallelworks]
     steps:
       - name: Clean-up
         run: |

From 267e39d611b9f5c8c0e90ac88b01c9b02fda306f Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Tue, 8 Oct 2024 17:14:54 -0400
Subject: [PATCH 59/95] trying checkout@v3 because Centos GLIB not supported

---
 .github/workflows/globalworkflow-ci-pw.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/globalworkflow-ci-pw.yaml b/.github/workflows/globalworkflow-ci-pw.yaml
index a63153901a..8a94998128 100644
--- a/.github/workflows/globalworkflow-ci-pw.yaml
+++ b/.github/workflows/globalworkflow-ci-pw.yaml
@@ -24,7 +24,7 @@ jobs:
 
     steps:
     - name: Checkout global-workflow
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
       with:
         path: ${{ github.run_id }}/HOMEgfs  # This path needs to be relative
 

From 176780332b8563e9779e3a8204d9e86371cbe229 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 9 Oct 2024 16:02:53 +0000
Subject: [PATCH 60/95] beefed up checkout action to v4 and added a bunch of
 helpers so run recurslivly on submodules as fast as possible

---
 .github/workflows/globalworkflow-ci-pw.yaml | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/globalworkflow-ci-pw.yaml b/.github/workflows/globalworkflow-ci-pw.yaml
index 8a94998128..2f1b41535d 100644
--- a/.github/workflows/globalworkflow-ci-pw.yaml
+++ b/.github/workflows/globalworkflow-ci-pw.yaml
@@ -22,11 +22,17 @@ jobs:
     runs-on: [self-hosted, aws, parallelworks]
     timeout-minutes: 600
 
-    steps:
+  steps:
     - name: Checkout global-workflow
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
-        path: ${{ github.run_id }}/HOMEgfs  # This path needs to be relative
+        path: ${{ github.run_id }}/HOMEgfs
+        submodules: true
+        submodules-depth: 1
+        fetch-depth: 1
+        persist-credentials: false
+        submodules-recursive: true
+        submodules-fetch-jobs: 8
 
     - name: Build components
       run: |

From 2cd9d225fd67a638f67abe0ce36c28b9c8cf46f3 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 9 Oct 2024 16:08:21 +0000
Subject: [PATCH 61/95] needed to indent steps on first stage

---
 .github/workflows/globalworkflow-ci-pw.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/globalworkflow-ci-pw.yaml b/.github/workflows/globalworkflow-ci-pw.yaml
index 2f1b41535d..e04fe61a17 100644
--- a/.github/workflows/globalworkflow-ci-pw.yaml
+++ b/.github/workflows/globalworkflow-ci-pw.yaml
@@ -22,7 +22,8 @@ jobs:
     runs-on: [self-hosted, aws, parallelworks]
     timeout-minutes: 600
 
-  steps:
+    steps:
+
     - name: Checkout global-workflow
       uses: actions/checkout@v4
       with:

From a4de837758b3abe20e4c3e599aadf71ede426a28 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 9 Oct 2024 16:20:03 +0000
Subject: [PATCH 62/95] made sure suboudles where fully recursive

---
 .github/workflows/globalworkflow-ci-pw.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/globalworkflow-ci-pw.yaml b/.github/workflows/globalworkflow-ci-pw.yaml
index e04fe61a17..fe309606b8 100644
--- a/.github/workflows/globalworkflow-ci-pw.yaml
+++ b/.github/workflows/globalworkflow-ci-pw.yaml
@@ -29,8 +29,7 @@ jobs:
       with:
         path: ${{ github.run_id }}/HOMEgfs
         submodules: true
-        submodules-depth: 1
-        fetch-depth: 1
+        fetch-depth: 0
         persist-credentials: false
         submodules-recursive: true
         submodules-fetch-jobs: 8

From 2c877078f7821bab347287bb3945ded6f72981dd Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 9 Oct 2024 16:29:38 +0000
Subject: [PATCH 63/95] updated submodule command for v4

---
 .github/workflows/globalworkflow-ci-pw.yaml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.github/workflows/globalworkflow-ci-pw.yaml b/.github/workflows/globalworkflow-ci-pw.yaml
index fe309606b8..b65267a130 100644
--- a/.github/workflows/globalworkflow-ci-pw.yaml
+++ b/.github/workflows/globalworkflow-ci-pw.yaml
@@ -28,11 +28,7 @@ jobs:
       uses: actions/checkout@v4
       with:
         path: ${{ github.run_id }}/HOMEgfs
-        submodules: true
-        fetch-depth: 0
-        persist-credentials: false
-        submodules-recursive: true
-        submodules-fetch-jobs: 8
+        submodules: 'recursive'
 
     - name: Build components
       run: |

From 16cde87145592064e955923f37e629019293a2b2 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 9 Oct 2024 17:00:12 +0000
Subject: [PATCH 64/95] add noaacloud config file for create experment bash
 script

---
 ci/platforms/config.noaacloud | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 ci/platforms/config.noaacloud

diff --git a/ci/platforms/config.noaacloud b/ci/platforms/config.noaacloud
new file mode 100644
index 0000000000..cce109d494
--- /dev/null
+++ b/ci/platforms/config.noaacloud
@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+
+export GFS_CI_ROOT=/gpfs/f5/epic/proj-shared/global/GFS_CI_ROOT
+export ICSDIR_ROOT=/gpfs/f5/epic/proj-shared/global/glopara/data/ICSDIR
+export STMP="/gpfs/f5/epic/scratch/${USER}"
+export SLURM_ACCOUNT=ufs-ard
+export max_concurrent_cases=5
+export max_concurrent_pr=4

From d6f383ab610286e66f5146e3acacf02a29100da4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 9 Oct 2024 17:05:51 +0000
Subject: [PATCH 65/95] added mkdir for RUNTESTS and added confg file under
 ci/platforms for noaacloud

---
 .github/workflows/globalworkflow-ci-pw.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/globalworkflow-ci-pw.yaml b/.github/workflows/globalworkflow-ci-pw.yaml
index b65267a130..1312dc91bb 100644
--- a/.github/workflows/globalworkflow-ci-pw.yaml
+++ b/.github/workflows/globalworkflow-ci-pw.yaml
@@ -50,10 +50,11 @@ jobs:
     steps:
       - name: Create Experiments ${{ matrix.case }}
         env:
-          HOMEgfs_PR: ${{ env.TEST_DIR }}/HOMEgfs
+          HOMEgfs: ${{ env.TEST_DIR }}/HOMEgfs
           RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS
           pslot: ${{ matrix.case }}.${{ github.run_id }}
         run: |
+          mkdir -p ${{ env.RUNTESTS }}
           cd ${{ env.TEST_DIR }}/HOMEgfs
           source workflow/gw_setup.sh
           source ci/platforms/config.noaacloud

From 1d5d291bc0ed88466d4ba5fa0ee509fd0ba0ed19 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 9 Oct 2024 17:28:50 +0000
Subject: [PATCH 66/95] path to create experment was wrong

---
 .github/workflows/globalworkflow-ci-pw.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/globalworkflow-ci-pw.yaml b/.github/workflows/globalworkflow-ci-pw.yaml
index 1312dc91bb..c6862bcac3 100644
--- a/.github/workflows/globalworkflow-ci-pw.yaml
+++ b/.github/workflows/globalworkflow-ci-pw.yaml
@@ -58,7 +58,7 @@ jobs:
           cd ${{ env.TEST_DIR }}/HOMEgfs
           source workflow/gw_setup.sh
           source ci/platforms/config.noaacloud
-          ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
+          ./workflow/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
 
   run-experiments:
     needs: create-experiments

From 75514d29239c024f1fefc6c767c8f0b906c994c4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 9 Oct 2024 18:02:56 +0000
Subject: [PATCH 67/95] path to pr cases was also stale and wrong

---
 .github/workflows/globalworkflow-ci-pw.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/globalworkflow-ci-pw.yaml b/.github/workflows/globalworkflow-ci-pw.yaml
index c6862bcac3..8326fc1854 100644
--- a/.github/workflows/globalworkflow-ci-pw.yaml
+++ b/.github/workflows/globalworkflow-ci-pw.yaml
@@ -58,7 +58,7 @@ jobs:
           cd ${{ env.TEST_DIR }}/HOMEgfs
           source workflow/gw_setup.sh
           source ci/platforms/config.noaacloud
-          ./workflow/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
+          ./workflow/create_experiment.py --yaml ci/cases/pr/${{ matrix.case }}.yaml
 
   run-experiments:
     needs: create-experiments

From 577e8df89df42af01dea717019b7cc9516c06160 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 9 Oct 2024 18:30:06 +0000
Subject: [PATCH 68/95] removed bug feature HOMEgfs and updated runtime ver
 numbers for fv3gfs runtime module versions

---
 .github/workflows/globalworkflow-ci-pw.yaml | 1 -
 versions/run.noaacloud.ver                  | 5 ++++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/globalworkflow-ci-pw.yaml b/.github/workflows/globalworkflow-ci-pw.yaml
index 8326fc1854..55155920d2 100644
--- a/.github/workflows/globalworkflow-ci-pw.yaml
+++ b/.github/workflows/globalworkflow-ci-pw.yaml
@@ -50,7 +50,6 @@ jobs:
     steps:
       - name: Create Experiments ${{ matrix.case }}
         env:
-          HOMEgfs: ${{ env.TEST_DIR }}/HOMEgfs
           RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS
           pslot: ${{ matrix.case }}.${{ github.run_id }}
         run: |
diff --git a/versions/run.noaacloud.ver b/versions/run.noaacloud.ver
index 2f312b310e..a628c78b8c 100644
--- a/versions/run.noaacloud.ver
+++ b/versions/run.noaacloud.ver
@@ -5,5 +5,8 @@ export spack_env=gsi-addon-env
 source "${HOMEgfs:-}/versions/spack.ver"
 export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core"
 
-export cdo_ver=2.2.0
+export cdo_ver=2.3.0
 export g2tmpl_ver=1.10.2
+export wgrib2_ver=3.1.2_wmo
+export nco_ver=5.1.6
+export jasper_ver=4.2.0 

From 7a4dedf5cb1269f405266cc53b5a77d7c4fe9305 Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 9 Oct 2024 18:07:47 -0400
Subject: [PATCH 69/95] needed third arg to run-check_ci.sh for getting the
 TESTDIR path with HOMEgfs overwriting global-workflow

---
 .github/workflows/globalworkflow-ci-pw.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/globalworkflow-ci-pw.yaml b/.github/workflows/globalworkflow-ci-pw.yaml
index 55155920d2..8ec6d21200 100644
--- a/.github/workflows/globalworkflow-ci-pw.yaml
+++ b/.github/workflows/globalworkflow-ci-pw.yaml
@@ -70,7 +70,7 @@ jobs:
       - name: Run Experiment ${{ matrix.case }}
         run: |
           cd ${{ env.TEST_DIR }}/HOMEgfs
-          ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }}
+          ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs
 
   clean-up:
     needs: run-experiments

From 6c763cbc814fa4ed10fecfa950a59844eb79e77c Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 10 Oct 2024 12:55:04 +0000
Subject: [PATCH 70/95] removed cron bash ci values from platforms config for
 noaacload

---
 ci/platforms/config.noaacloud | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/ci/platforms/config.noaacloud b/ci/platforms/config.noaacloud
index cce109d494..7b129a19d7 100644
--- a/ci/platforms/config.noaacloud
+++ b/ci/platforms/config.noaacloud
@@ -1,8 +1,3 @@
 #!/usr/bin/bash
 
-export GFS_CI_ROOT=/gpfs/f5/epic/proj-shared/global/GFS_CI_ROOT
-export ICSDIR_ROOT=/gpfs/f5/epic/proj-shared/global/glopara/data/ICSDIR
-export STMP="/gpfs/f5/epic/scratch/${USER}"
-export SLURM_ACCOUNT=ufs-ard
-export max_concurrent_cases=5
-export max_concurrent_pr=4
+export HPC_ACCOUNT=${USER}
\ No newline at end of file

From 508ffc6144d764ec11b8fe1f1fd96a7a97d89c0d Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Thu, 10 Oct 2024 08:56:08 -0400
Subject: [PATCH 71/95] Update config.noaacloud

add last line
---
 ci/platforms/config.noaacloud | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/platforms/config.noaacloud b/ci/platforms/config.noaacloud
index 7b129a19d7..5303f24860 100644
--- a/ci/platforms/config.noaacloud
+++ b/ci/platforms/config.noaacloud
@@ -1,3 +1,3 @@
 #!/usr/bin/bash
 
-export HPC_ACCOUNT=${USER}
\ No newline at end of file
+export HPC_ACCOUNT=${USER}

From e1448a49b08554c17bdd5f27bedc94be7f28d3a4 Mon Sep 17 00:00:00 2001
From: TerrenceMcGuinness-NOAA <terrence.mcguinness@cox.net>
Date: Thu, 10 Oct 2024 08:56:55 -0400
Subject: [PATCH 72/95] Update globalworkflow-ci-pw-ssh.yaml

---
 .github/workflows/globalworkflow-ci-pw-ssh.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/globalworkflow-ci-pw-ssh.yaml b/.github/workflows/globalworkflow-ci-pw-ssh.yaml
index 51f5e58595..3a67897b6e 100644
--- a/.github/workflows/globalworkflow-ci-pw-ssh.yaml
+++ b/.github/workflows/globalworkflow-ci-pw-ssh.yaml
@@ -88,4 +88,4 @@ jobs:
           ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
             cd ${{ needs.checkout-build-link.outputs.TEST_DIR }}/HOMEgfs
             ./ci/scripts/run_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
-          EOF
\ No newline at end of file
+          EOF

From 99e3cab3a13c902308c5c707a945b4fd5b51b307 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 10 Oct 2024 13:00:29 +0000
Subject: [PATCH 73/95] added the switch overwirte to create experment in
 GitHub Pipeline CI scripts

---
 .github/workflows/globalworkflow-ci-pw.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/globalworkflow-ci-pw.yaml b/.github/workflows/globalworkflow-ci-pw.yaml
index 8ec6d21200..29739bad25 100644
--- a/.github/workflows/globalworkflow-ci-pw.yaml
+++ b/.github/workflows/globalworkflow-ci-pw.yaml
@@ -57,7 +57,7 @@ jobs:
           cd ${{ env.TEST_DIR }}/HOMEgfs
           source workflow/gw_setup.sh
           source ci/platforms/config.noaacloud
-          ./workflow/create_experiment.py --yaml ci/cases/pr/${{ matrix.case }}.yaml
+          ./workflow/create_experiment.py --yaml ci/cases/pr/${{ matrix.case }}.yaml --overwrite
 
   run-experiments:
     needs: create-experiments

From cf284fcfc3607fcaf49378256e71bc9396380e2b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 10 Oct 2024 17:12:08 +0000
Subject: [PATCH 74/95] using checkout v3 for centos 7

---
 .github/workflows/globalworkflow-ci-pw.yaml | 2 +-
 versions/spack.ver                          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/globalworkflow-ci-pw.yaml b/.github/workflows/globalworkflow-ci-pw.yaml
index 29739bad25..9efaf28f03 100644
--- a/.github/workflows/globalworkflow-ci-pw.yaml
+++ b/.github/workflows/globalworkflow-ci-pw.yaml
@@ -25,7 +25,7 @@ jobs:
     steps:
 
     - name: Checkout global-workflow
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
       with:
         path: ${{ github.run_id }}/HOMEgfs
         submodules: 'recursive'
diff --git a/versions/spack.ver b/versions/spack.ver
index dbe304a5fe..410f509eb6 100644
--- a/versions/spack.ver
+++ b/versions/spack.ver
@@ -23,7 +23,7 @@ export g2_ver=3.4.5
 export sp_ver=2.5.0
 export ip_ver=4.3.0
 export gsi_ncdiag_ver=1.1.2
-export g2tmpl_ver=1.13.0
+export g2tmpl_ver=1.10.2
 export crtm_ver=2.4.0.1
 export wgrib2_ver=2.0.8
 export grib_util_ver=1.3.0

From cf41b80c9a3ed533ffd116519df3c44a3787e112 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 10 Oct 2024 17:24:34 +0000
Subject: [PATCH 75/95] revered to checkout v4 for centos 7 because found fix
 and reverted to /bucket as is now mounted

---
 .github/workflows/globalworkflow-ci-pw.yaml | 2 +-
 ci/cases/pr/C48_ATM.yaml                    | 4 ++--
 workflow/hosts/awspw.yaml                   | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/globalworkflow-ci-pw.yaml b/.github/workflows/globalworkflow-ci-pw.yaml
index 9efaf28f03..29739bad25 100644
--- a/.github/workflows/globalworkflow-ci-pw.yaml
+++ b/.github/workflows/globalworkflow-ci-pw.yaml
@@ -25,7 +25,7 @@ jobs:
     steps:
 
     - name: Checkout global-workflow
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         path: ${{ github.run_id }}/HOMEgfs
         submodules: 'recursive'
diff --git a/ci/cases/pr/C48_ATM.yaml b/ci/cases/pr/C48_ATM.yaml
index 354ea0736e..79706556e6 100644
--- a/ci/cases/pr/C48_ATM.yaml
+++ b/ci/cases/pr/C48_ATM.yaml
@@ -8,6 +8,6 @@ arguments:
   resdetatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  idate: 2024010100
-  edate: 2024010100
+  idate: 2021032312
+  edate: 2021032312
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml
diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml
index 1672a85229..df5a3f2b4e 100644
--- a/workflow/hosts/awspw.yaml
+++ b/workflow/hosts/awspw.yaml
@@ -18,8 +18,8 @@ CHGRP_RSTPROD: 'YES'
 CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported.
 HPSSARCH: 'NO'
 HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below.
-BASE_DATA: '/contrib/global-workflow-shared-data'
-BASE_IC: '/contrib/global-workflow-shared-data/ICs'
+BASE_DATA: '/bucket/global-workflow-shared-data'
+BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR
 LOCALARCH: 'NO'
 ATARDIR: '' # TODO: This will not yet work from AWS.
 MAKE_NSSTBUFR: 'NO'

From 3eea38d8385655b7ef5b832050871f7eddf61513 Mon Sep 17 00:00:00 2001
From: Terry McGuinness
 <Terry.McGuinness@mgmt-terrymcguinness-awsemctmcgc7i48xlargeciplatform-00001.pw-noaa-us-east-1.pw.local>
Date: Thu, 10 Oct 2024 19:38:17 +0000
Subject: [PATCH 76/95] putting g2tmpl_ver back to 1.13.0

---
 versions/spack.ver | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/versions/spack.ver b/versions/spack.ver
index 410f509eb6..dbe304a5fe 100644
--- a/versions/spack.ver
+++ b/versions/spack.ver
@@ -23,7 +23,7 @@ export g2_ver=3.4.5
 export sp_ver=2.5.0
 export ip_ver=4.3.0
 export gsi_ncdiag_ver=1.1.2
-export g2tmpl_ver=1.10.2
+export g2tmpl_ver=1.13.0
 export crtm_ver=2.4.0.1
 export wgrib2_ver=2.0.8
 export grib_util_ver=1.3.0

From 25944e7ec494a640b0a844e6791d33a7bc69ef39 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 10 Oct 2024 20:11:17 +0000
Subject: [PATCH 77/95] dropped a quote on the end of BACI_IC string

---
 workflow/hosts/awspw.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml
index df5a3f2b4e..ef17d8f2f4 100644
--- a/workflow/hosts/awspw.yaml
+++ b/workflow/hosts/awspw.yaml
@@ -19,7 +19,7 @@ CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported.
 HPSSARCH: 'NO'
 HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below.
 BASE_DATA: '/bucket/global-workflow-shared-data'
-BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR
+BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR'
 LOCALARCH: 'NO'
 ATARDIR: '' # TODO: This will not yet work from AWS.
 MAKE_NSSTBUFR: 'NO'

From bfa65ce27d6321ab1c5037150e108390eec4986b Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 10 Oct 2024 21:02:21 +0000
Subject: [PATCH 78/95] two runners, one for rocky the other for centos

---
 ....yaml => globalworkflow-ci-pw-centos.yaml} |  0
 .../workflows/globalworkflow-ci-pw-rocky.yaml | 82 +++++++++++++++++++
 2 files changed, 82 insertions(+)
 rename .github/workflows/{globalworkflow-ci-pw.yaml => globalworkflow-ci-pw-centos.yaml} (100%)
 create mode 100644 .github/workflows/globalworkflow-ci-pw-rocky.yaml

diff --git a/.github/workflows/globalworkflow-ci-pw.yaml b/.github/workflows/globalworkflow-ci-pw-centos.yaml
similarity index 100%
rename from .github/workflows/globalworkflow-ci-pw.yaml
rename to .github/workflows/globalworkflow-ci-pw-centos.yaml
diff --git a/.github/workflows/globalworkflow-ci-pw-rocky.yaml b/.github/workflows/globalworkflow-ci-pw-rocky.yaml
new file mode 100644
index 0000000000..51cd20f050
--- /dev/null
+++ b/.github/workflows/globalworkflow-ci-pw-rocky.yaml
@@ -0,0 +1,82 @@
+name: gw-ci-aws
+
+on: [workflow_dispatch]
+
+# TEST_DIR contains 2 directories;
+# 1. HOMEgfs: clone of the global-workflow
+# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
+# e.g. $> tree ./TEST_DIR
+# ./TEST_DIR
+# ├── HOMEgfs
+# └── RUNTESTS
+#     ├── COMROT
+#     │   └── ${pslot}
+#     └── EXPDIR
+#         └── ${pslot}
+env:
+  TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
+  MACHINE_ID: noaacloud
+
+jobs:
+  checkout-build-link:
+    runs-on: [self-hosted, aws, parallelworks, rocky]
+    timeout-minutes: 600
+
+    steps:
+
+    - name: Checkout global-workflow
+      uses: actions/checkout@v4
+      with:
+        path: ${{ github.run_id }}/HOMEgfs
+        submodules: 'recursive'
+
+    - name: Build components
+      run: |
+        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
+        ./build_all.sh -j 8
+
+    - name: Link artifacts
+      run: |
+        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
+        ./link_workflow.sh
+
+  create-experiments:
+    needs: checkout-build-link
+    runs-on: [self-hosted, aws, parallelworks, rocky]
+    strategy:
+      matrix:
+        case: ["C48_ATM"]
+
+    steps:
+      - name: Create Experiments ${{ matrix.case }}
+        env:
+          RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS
+          pslot: ${{ matrix.case }}.${{ github.run_id }}
+        run: |
+          mkdir -p ${{ env.RUNTESTS }}
+          cd ${{ env.TEST_DIR }}/HOMEgfs
+          source workflow/gw_setup.sh
+          source ci/platforms/config.noaacloud
+          ./workflow/create_experiment.py --yaml ci/cases/pr/${{ matrix.case }}.yaml --overwrite
+
+  run-experiments:
+    needs: create-experiments
+    runs-on: [self-hosted, aws, parallelworks, rocky]
+    strategy:
+      max-parallel: 2
+      matrix:
+        case: ["C48_ATM"]
+    steps:
+      - name: Run Experiment ${{ matrix.case }}
+        run: |
+          cd ${{ env.TEST_DIR }}/HOMEgfs
+          ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs
+
+  clean-up:
+    needs: run-experiments
+    runs-on: [self-hosted, aws, parallelworks, rocky]
+    steps:
+      - name: Clean-up
+        run: |
+          cd ${{ github.workspace }}
+          rm -rf ${{ github.run_id }}

From 7b20639ede35d8e467810da9d42bad9e11f647d4 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Thu, 10 Oct 2024 21:04:59 +0000
Subject: [PATCH 79/95] renamed actons for ssh, rocky, and centos

---
 .github/workflows/globalworkflow-ci-pw-centos.yaml | 10 +++++-----
 .github/workflows/globalworkflow-ci-pw-rocky.yaml  |  2 +-
 .github/workflows/globalworkflow-ci-pw-ssh.yaml    |  2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/globalworkflow-ci-pw-centos.yaml b/.github/workflows/globalworkflow-ci-pw-centos.yaml
index 29739bad25..0d6401e083 100644
--- a/.github/workflows/globalworkflow-ci-pw-centos.yaml
+++ b/.github/workflows/globalworkflow-ci-pw-centos.yaml
@@ -1,4 +1,4 @@
-name: gw-ci-aws
+name: gw-ci-aws-centos
 
 on: [workflow_dispatch]
 
@@ -19,7 +19,7 @@ env:
 
 jobs:
   checkout-build-link:
-    runs-on: [self-hosted, aws, parallelworks]
+    runs-on: [self-hosted, aws, parallelworks, centos]
     timeout-minutes: 600
 
     steps:
@@ -42,7 +42,7 @@ jobs:
 
   create-experiments:
     needs: checkout-build-link
-    runs-on: [self-hosted, aws, parallelworks]
+    runs-on: [self-hosted, aws, parallelworks, centos]
     strategy:
       matrix:
         case: ["C48_ATM"]
@@ -61,7 +61,7 @@ jobs:
 
   run-experiments:
     needs: create-experiments
-    runs-on: [self-hosted, aws, parallelworks]
+    runs-on: [self-hosted, aws, parallelworks, centos]
     strategy:
       max-parallel: 2
       matrix:
@@ -74,7 +74,7 @@ jobs:
 
   clean-up:
     needs: run-experiments
-    runs-on: [self-hosted, aws, parallelworks]
+    runs-on: [self-hosted, aws, parallelworks, centos]
     steps:
       - name: Clean-up
         run: |
diff --git a/.github/workflows/globalworkflow-ci-pw-rocky.yaml b/.github/workflows/globalworkflow-ci-pw-rocky.yaml
index 51cd20f050..6ff232f097 100644
--- a/.github/workflows/globalworkflow-ci-pw-rocky.yaml
+++ b/.github/workflows/globalworkflow-ci-pw-rocky.yaml
@@ -1,4 +1,4 @@
-name: gw-ci-aws
+name: gw-ci-aws-rocky
 
 on: [workflow_dispatch]
 
diff --git a/.github/workflows/globalworkflow-ci-pw-ssh.yaml b/.github/workflows/globalworkflow-ci-pw-ssh.yaml
index 3a67897b6e..f9edebca56 100644
--- a/.github/workflows/globalworkflow-ci-pw-ssh.yaml
+++ b/.github/workflows/globalworkflow-ci-pw-ssh.yaml
@@ -1,4 +1,4 @@
-name: pw-ci-hera
+name: pw-ci-hera-ssh
 
 on:
   workflow_dispatch:

From 6a29eb6442581b6645d85dd724379f677a3790aa Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 11 Oct 2024 13:09:23 +0000
Subject: [PATCH 80/95] move source of run time module versions up a few lines
 to its original location in the script

---
 ush/load_fv3gfs_modules.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh
index a26f36a5bb..b7e0a0f04f 100755
--- a/ush/load_fv3gfs_modules.sh
+++ b/ush/load_fv3gfs_modules.sh
@@ -13,12 +13,12 @@ ulimit_s=$( ulimit -S -s )
 source "${HOMEgfs}/ush/detect_machine.sh"
 source "${HOMEgfs}/ush/module-setup.sh"
 
-# Load our modules:
-module use "${HOMEgfs}/modulefiles"
-
 # Source versions file for runtime
 source "${HOMEgfs}/versions/run.${MACHINE_ID}.ver"
 
+# Load our modules:
+module use "${HOMEgfs}/modulefiles"
+
 case "${MACHINE_ID}" in
   "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4" | "noaacloud")
     module load "module_base.${MACHINE_ID}"

From 17d061e1d97cf13f9720e57dbfc21003d8129b2a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 15 Oct 2024 18:54:11 +0000
Subject: [PATCH 81/95] remove runtime ver conflics in run noaacloud env
 settings

---
 ci/scripts/utils/provision_runner.sh | 27 +++++++++++++++++++++++++++
 versions/run.noaacloud.ver           |  4 ----
 2 files changed, 27 insertions(+), 4 deletions(-)
 create mode 100644 ci/scripts/utils/provision_runner.sh

diff --git a/ci/scripts/utils/provision_runner.sh b/ci/scripts/utils/provision_runner.sh
new file mode 100644
index 0000000000..b009432dc8
--- /dev/null
+++ b/ci/scripts/utils/provision_runner.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+CI_USER="Terry.McGuinness"
+
+# Get the Operating System name from /etc/os-release
+OS_NAME=$(grep -E '^ID=' /etc/os-release | sed -E 's/ID="?([^"]*)"?/\1/')
+
+# Check if the OS is Rocky or CentOS
+if [[ "$OS_NAME" == "rocky" || "$OS_NAME" == "centos" ]]; then
+  echo "Operating System is $OS_NAME"
+else
+  echo "Unsupported Operating System: $OS_NAME"
+  exit 1
+fi
+
+running=$(ps aux | grep actions-runner | grep -v color | wc -l)
+if [[ "${running}" -gt 0 ]]; then
+   echo "actions-runner is already running"
+   exit
+fi
+
+cp /contrib/${CI_USER}/SETUP/actions-runner_${OS_NAME}.tar.gz $HOME
+cd $HOME
+tar -xf actions-runner_${OS_NAME}.tar.gz
+cd actions-runner
+d=$(date +%Y-%m-%d-%H:%M)
+nohup ./run.sh >& run_noup${d}.log &
\ No newline at end of file
diff --git a/versions/run.noaacloud.ver b/versions/run.noaacloud.ver
index a628c78b8c..1b5482a67e 100644
--- a/versions/run.noaacloud.ver
+++ b/versions/run.noaacloud.ver
@@ -5,8 +5,4 @@ export spack_env=gsi-addon-env
 source "${HOMEgfs:-}/versions/spack.ver"
 export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core"
 
-export cdo_ver=2.3.0
 export g2tmpl_ver=1.10.2
-export wgrib2_ver=3.1.2_wmo
-export nco_ver=5.1.6
-export jasper_ver=4.2.0 

From 0f38f0d0d7d6577be663190f73dcaf7bfa502659 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 15 Oct 2024 19:25:24 +0000
Subject: [PATCH 82/95] improved provision runner script from bash sheel check

---
 ci/scripts/utils/provision_runner.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/ci/scripts/utils/provision_runner.sh b/ci/scripts/utils/provision_runner.sh
index b009432dc8..d21d4e5dcf 100644
--- a/ci/scripts/utils/provision_runner.sh
+++ b/ci/scripts/utils/provision_runner.sh
@@ -3,25 +3,25 @@
 CI_USER="Terry.McGuinness"
 
 # Get the Operating System name from /etc/os-release
-OS_NAME=$(grep -E '^ID=' /etc/os-release | sed -E 's/ID="?([^"]*)"?/\1/')
+OS_NAME=$(grep -E '^ID=' /etc/os-release | sed -E 's/ID="?([^"]*)"?/\1/') || true
 
 # Check if the OS is Rocky or CentOS
-if [[ "$OS_NAME" == "rocky" || "$OS_NAME" == "centos" ]]; then
-  echo "Operating System is $OS_NAME"
+if [[ "${OS_NAME}" == "rocky" || "${OS_NAME}" == "centos" ]]; then
+  echo "Operating System is ${OS_NAME}"
 else
-  echo "Unsupported Operating System: $OS_NAME"
+  echo "Unsupported Operating System: ${OS_NAME}"
   exit 1
 fi
 
-running=$(ps aux | grep actions-runner | grep -v color | wc -l)
+running=$(pgrep -u $USER run-helper -c) || true
 if [[ "${running}" -gt 0 ]]; then
    echo "actions-runner is already running"
    exit
 fi
 
-cp /contrib/${CI_USER}/SETUP/actions-runner_${OS_NAME}.tar.gz $HOME
+cp /contrib/${CI_USER}/SETUP/actions-runner_${OS_NAME}.tar.gz ${HOME}
 cd $HOME
 tar -xf actions-runner_${OS_NAME}.tar.gz
-cd actions-runner
+cd actions-runner || exit
 d=$(date +%Y-%m-%d-%H:%M)
 nohup ./run.sh >& run_noup${d}.log &
\ No newline at end of file

From 1e297a334eb8c48e8f7b21dc63e702a7ba9fd584 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 15 Oct 2024 19:26:58 +0000
Subject: [PATCH 83/95] couple of more shell check updates to provision runner

---
 ci/scripts/utils/provision_runner.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/scripts/utils/provision_runner.sh b/ci/scripts/utils/provision_runner.sh
index d21d4e5dcf..1ef7c450e8 100644
--- a/ci/scripts/utils/provision_runner.sh
+++ b/ci/scripts/utils/provision_runner.sh
@@ -20,8 +20,8 @@ if [[ "${running}" -gt 0 ]]; then
 fi
 
 cp /contrib/${CI_USER}/SETUP/actions-runner_${OS_NAME}.tar.gz ${HOME}
-cd $HOME
-tar -xf actions-runner_${OS_NAME}.tar.gz
+cd $HOME || exit
+tar -xf "actions-runner_${OS_NAME}.tar.gz"
 cd actions-runner || exit
 d=$(date +%Y-%m-%d-%H:%M)
 nohup ./run.sh >& run_noup${d}.log &
\ No newline at end of file

From b01d0b0012eea72bd1c180bd1e114f0fa75afd62 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 15 Oct 2024 19:28:13 +0000
Subject: [PATCH 84/95] couple of more shell check updates to provision runner

---
 ci/scripts/utils/provision_runner.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/scripts/utils/provision_runner.sh b/ci/scripts/utils/provision_runner.sh
index 1ef7c450e8..7727cec176 100644
--- a/ci/scripts/utils/provision_runner.sh
+++ b/ci/scripts/utils/provision_runner.sh
@@ -13,7 +13,7 @@ else
   exit 1
 fi
 
-running=$(pgrep -u $USER run-helper -c) || true
+running=$(pgrep -u ${USER} run-helper -c) || true
 if [[ "${running}" -gt 0 ]]; then
    echo "actions-runner is already running"
    exit
@@ -24,4 +24,4 @@ cd $HOME || exit
 tar -xf "actions-runner_${OS_NAME}.tar.gz"
 cd actions-runner || exit
 d=$(date +%Y-%m-%d-%H:%M)
-nohup ./run.sh >& run_noup${d}.log &
\ No newline at end of file
+nohup ./run.sh >& "run_noup${d}.log" &
\ No newline at end of file

From bb5658d08accc6f8a06cdca74c343708bdf8eecb Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 15 Oct 2024 19:29:34 +0000
Subject: [PATCH 85/95] added last line to provision runner script

---
 ci/scripts/utils/provision_runner.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/scripts/utils/provision_runner.sh b/ci/scripts/utils/provision_runner.sh
index 7727cec176..9f3c22cafd 100644
--- a/ci/scripts/utils/provision_runner.sh
+++ b/ci/scripts/utils/provision_runner.sh
@@ -24,4 +24,4 @@ cd $HOME || exit
 tar -xf "actions-runner_${OS_NAME}.tar.gz"
 cd actions-runner || exit
 d=$(date +%Y-%m-%d-%H:%M)
-nohup ./run.sh >& "run_noup${d}.log" &
\ No newline at end of file
+nohup ./run.sh >& "run_noup${d}.log" &

From 1715e7d490533db3b4d27c74664540dd26132c8e Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Tue, 15 Oct 2024 22:25:32 +0000
Subject: [PATCH 86/95] chmod x on provision runner scripts and add
 UserBootstrap code for Centos

---
 ci/scripts/utils/UserBootstrap_centos7.txt | 5 +++++
 ci/scripts/utils/provision_runner.sh       | 0
 2 files changed, 5 insertions(+)
 create mode 100644 ci/scripts/utils/UserBootstrap_centos7.txt
 mode change 100644 => 100755 ci/scripts/utils/provision_runner.sh

diff --git a/ci/scripts/utils/UserBootstrap_centos7.txt b/ci/scripts/utils/UserBootstrap_centos7.txt
new file mode 100644
index 0000000000..ddc6b05706
--- /dev/null
+++ b/ci/scripts/utils/UserBootstrap_centos7.txt
@@ -0,0 +1,5 @@
+sudo yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
+sudo yum -y install git
+/contrib/Terry.McGuinness/SETUP/provision_runner.sh
+ALLNODES
+/contrib/Terry.McGuinness/SETUP/mount-epic-contrib.sh
\ No newline at end of file
diff --git a/ci/scripts/utils/provision_runner.sh b/ci/scripts/utils/provision_runner.sh
old mode 100644
new mode 100755

From 881f917efcb984d7c6d455c109c7bbce52654859 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Wed, 16 Oct 2024 16:52:32 +0000
Subject: [PATCH 87/95] reduced this branch to only build run changes seperate
 to ci

---
 .../globalworkflow-ci-pw-centos.yaml          | 82 -----------------
 .../workflows/globalworkflow-ci-pw-rocky.yaml | 82 -----------------
 .../workflows/globalworkflow-ci-pw-ssh.yaml   | 91 -------------------
 ci/scripts/utils/UserBootstrap_centos7.txt    |  5 -
 ci/scripts/utils/provision_runner.sh          | 27 ------
 5 files changed, 287 deletions(-)
 delete mode 100644 .github/workflows/globalworkflow-ci-pw-centos.yaml
 delete mode 100644 .github/workflows/globalworkflow-ci-pw-rocky.yaml
 delete mode 100644 .github/workflows/globalworkflow-ci-pw-ssh.yaml
 delete mode 100644 ci/scripts/utils/UserBootstrap_centos7.txt
 delete mode 100755 ci/scripts/utils/provision_runner.sh

diff --git a/.github/workflows/globalworkflow-ci-pw-centos.yaml b/.github/workflows/globalworkflow-ci-pw-centos.yaml
deleted file mode 100644
index 0d6401e083..0000000000
--- a/.github/workflows/globalworkflow-ci-pw-centos.yaml
+++ /dev/null
@@ -1,82 +0,0 @@
-name: gw-ci-aws-centos
-
-on: [workflow_dispatch]
-
-# TEST_DIR contains 2 directories;
-# 1. HOMEgfs: clone of the global-workflow
-# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
-# e.g. $> tree ./TEST_DIR
-# ./TEST_DIR
-# ├── HOMEgfs
-# └── RUNTESTS
-#     ├── COMROT
-#     │   └── ${pslot}
-#     └── EXPDIR
-#         └── ${pslot}
-env:
-  TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
-  MACHINE_ID: noaacloud
-
-jobs:
-  checkout-build-link:
-    runs-on: [self-hosted, aws, parallelworks, centos]
-    timeout-minutes: 600
-
-    steps:
-
-    - name: Checkout global-workflow
-      uses: actions/checkout@v4
-      with:
-        path: ${{ github.run_id }}/HOMEgfs
-        submodules: 'recursive'
-
-    - name: Build components
-      run: |
-        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-        ./build_all.sh -j 8
-
-    - name: Link artifacts
-      run: |
-        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-        ./link_workflow.sh
-
-  create-experiments:
-    needs: checkout-build-link
-    runs-on: [self-hosted, aws, parallelworks, centos]
-    strategy:
-      matrix:
-        case: ["C48_ATM"]
-
-    steps:
-      - name: Create Experiments ${{ matrix.case }}
-        env:
-          RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS
-          pslot: ${{ matrix.case }}.${{ github.run_id }}
-        run: |
-          mkdir -p ${{ env.RUNTESTS }}
-          cd ${{ env.TEST_DIR }}/HOMEgfs
-          source workflow/gw_setup.sh
-          source ci/platforms/config.noaacloud
-          ./workflow/create_experiment.py --yaml ci/cases/pr/${{ matrix.case }}.yaml --overwrite
-
-  run-experiments:
-    needs: create-experiments
-    runs-on: [self-hosted, aws, parallelworks, centos]
-    strategy:
-      max-parallel: 2
-      matrix:
-        case: ["C48_ATM"]
-    steps:
-      - name: Run Experiment ${{ matrix.case }}
-        run: |
-          cd ${{ env.TEST_DIR }}/HOMEgfs
-          ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs
-
-  clean-up:
-    needs: run-experiments
-    runs-on: [self-hosted, aws, parallelworks, centos]
-    steps:
-      - name: Clean-up
-        run: |
-          cd ${{ github.workspace }}
-          rm -rf ${{ github.run_id }}
diff --git a/.github/workflows/globalworkflow-ci-pw-rocky.yaml b/.github/workflows/globalworkflow-ci-pw-rocky.yaml
deleted file mode 100644
index 6ff232f097..0000000000
--- a/.github/workflows/globalworkflow-ci-pw-rocky.yaml
+++ /dev/null
@@ -1,82 +0,0 @@
-name: gw-ci-aws-rocky
-
-on: [workflow_dispatch]
-
-# TEST_DIR contains 2 directories;
-# 1. HOMEgfs: clone of the global-workflow
-# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
-# e.g. $> tree ./TEST_DIR
-# ./TEST_DIR
-# ├── HOMEgfs
-# └── RUNTESTS
-#     ├── COMROT
-#     │   └── ${pslot}
-#     └── EXPDIR
-#         └── ${pslot}
-env:
-  TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
-  MACHINE_ID: noaacloud
-
-jobs:
-  checkout-build-link:
-    runs-on: [self-hosted, aws, parallelworks, rocky]
-    timeout-minutes: 600
-
-    steps:
-
-    - name: Checkout global-workflow
-      uses: actions/checkout@v4
-      with:
-        path: ${{ github.run_id }}/HOMEgfs
-        submodules: 'recursive'
-
-    - name: Build components
-      run: |
-        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-        ./build_all.sh -j 8
-
-    - name: Link artifacts
-      run: |
-        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-        ./link_workflow.sh
-
-  create-experiments:
-    needs: checkout-build-link
-    runs-on: [self-hosted, aws, parallelworks, rocky]
-    strategy:
-      matrix:
-        case: ["C48_ATM"]
-
-    steps:
-      - name: Create Experiments ${{ matrix.case }}
-        env:
-          RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS
-          pslot: ${{ matrix.case }}.${{ github.run_id }}
-        run: |
-          mkdir -p ${{ env.RUNTESTS }}
-          cd ${{ env.TEST_DIR }}/HOMEgfs
-          source workflow/gw_setup.sh
-          source ci/platforms/config.noaacloud
-          ./workflow/create_experiment.py --yaml ci/cases/pr/${{ matrix.case }}.yaml --overwrite
-
-  run-experiments:
-    needs: create-experiments
-    runs-on: [self-hosted, aws, parallelworks, rocky]
-    strategy:
-      max-parallel: 2
-      matrix:
-        case: ["C48_ATM"]
-    steps:
-      - name: Run Experiment ${{ matrix.case }}
-        run: |
-          cd ${{ env.TEST_DIR }}/HOMEgfs
-          ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs
-
-  clean-up:
-    needs: run-experiments
-    runs-on: [self-hosted, aws, parallelworks, rocky]
-    steps:
-      - name: Clean-up
-        run: |
-          cd ${{ github.workspace }}
-          rm -rf ${{ github.run_id }}
diff --git a/.github/workflows/globalworkflow-ci-pw-ssh.yaml b/.github/workflows/globalworkflow-ci-pw-ssh.yaml
deleted file mode 100644
index f9edebca56..0000000000
--- a/.github/workflows/globalworkflow-ci-pw-ssh.yaml
+++ /dev/null
@@ -1,91 +0,0 @@
-name: pw-ci-hera-ssh
-
-on:
-  workflow_dispatch:
-
-# TEST_DIR contains 2 directories;
-# 1. HOMEgfs: clone of the global-workflow
-# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
-# e.g. $> tree ./TEST_DIR
-# ./TEST_DIR
-# ├── HOMEgfs
-# └── RUNTESTS
-
-jobs:
-
-  checkout-build-link:
-    runs-on: [self-hosted, parallel-works, pw]
-    timeout-minutes: 600
-
-    env:
-      MACHINE_ID: hera
-
-    outputs:
-      TEST_DIR: ${{ steps.set-env.outputs.TEST_DIR }}
-
-    steps:
-
-    - name: Set environment variables
-      id: set-env
-      run: |
-        echo "TEST_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}" >> $GITHUB_ENV
-        echo "::set-output name=TEST_DIR::/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT/PR/${{ github.run_id }}-${{ github.ref_name }}/${{ github.event.pull_request.number }}"
-
-    - name: Checkout global-workflow
-      run: |
-        ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-         mkdir -p ${{ env.TEST_DIR }}/HOMEgfs
-         cd ${{ env.TEST_DIR }}/HOMEgfs
-         git clone --branch ${{ github.ref_name }} https://github.com/${{ github.repository_owner }}/${{ github.event.repository.name }}.git .
-         git submodule update --init --recursive
-        EOF
-
-    - name: Build components
-      run: |
-        ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-          cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-          ./build_all.sh
-        EOF
-
-    - name: Link artifacts
-      run: |
-        ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-          cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-          ./link_workflow.sh
-        EOF
-
-  create-experiments:
-    needs: checkout-build-link
-    runs-on: [self-hosted, parallel-works, pw]
-    strategy:
-      matrix:
-        case: ["C48_S2S", "C96_atm3DVar"]
-
-    steps:
-      - name: Create Experiments ${{ matrix.case }}
-        env:
-          HOMEgfs_PR: ${{ needs.checkout-build-link.outputs.TEST_DIR }}/HOMEgfs
-          RUNTESTS: ${{ needs.checkout-build-link.outputs.TEST_DIR }}/RUNTESTS
-          pslot: ${{ matrix.case }}.${{ github.run_id }}
-        run: |
-          ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-            cd ${{ needs.checkout-build-link.outputs.TEST_DIR }}/HOMEgfs
-            source workflow/gw_setup.sh
-            source ci/platforms/${{ env.MACHINE_ID }}.sh
-            ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
-          EOF
-
-  run-experiments:
-    needs: create-experiments
-    runs-on: [self-hosted, parallel-works, pw]
-    strategy:
-      max-parallel: 2
-      matrix:
-        case: ["C48_S2S", "C96_atm3DVar"]
-    steps:
-      - name: Run Experiment ${{ matrix.case }}
-        run: |
-          ssh -t hfe01.fairmont.rdhpcs.noaa.gov << 'EOF'
-            cd ${{ needs.checkout-build-link.outputs.TEST_DIR }}/HOMEgfs
-            ./ci/scripts/run_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml
-          EOF
diff --git a/ci/scripts/utils/UserBootstrap_centos7.txt b/ci/scripts/utils/UserBootstrap_centos7.txt
deleted file mode 100644
index ddc6b05706..0000000000
--- a/ci/scripts/utils/UserBootstrap_centos7.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-sudo yum -y install https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
-sudo yum -y install git
-/contrib/Terry.McGuinness/SETUP/provision_runner.sh
-ALLNODES
-/contrib/Terry.McGuinness/SETUP/mount-epic-contrib.sh
\ No newline at end of file
diff --git a/ci/scripts/utils/provision_runner.sh b/ci/scripts/utils/provision_runner.sh
deleted file mode 100755
index 9f3c22cafd..0000000000
--- a/ci/scripts/utils/provision_runner.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-
-CI_USER="Terry.McGuinness"
-
-# Get the Operating System name from /etc/os-release
-OS_NAME=$(grep -E '^ID=' /etc/os-release | sed -E 's/ID="?([^"]*)"?/\1/') || true
-
-# Check if the OS is Rocky or CentOS
-if [[ "${OS_NAME}" == "rocky" || "${OS_NAME}" == "centos" ]]; then
-  echo "Operating System is ${OS_NAME}"
-else
-  echo "Unsupported Operating System: ${OS_NAME}"
-  exit 1
-fi
-
-running=$(pgrep -u ${USER} run-helper -c) || true
-if [[ "${running}" -gt 0 ]]; then
-   echo "actions-runner is already running"
-   exit
-fi
-
-cp /contrib/${CI_USER}/SETUP/actions-runner_${OS_NAME}.tar.gz ${HOME}
-cd $HOME || exit
-tar -xf "actions-runner_${OS_NAME}.tar.gz"
-cd actions-runner || exit
-d=$(date +%Y-%m-%d-%H:%M)
-nohup ./run.sh >& "run_noup${d}.log" &

From a5f46142d4e3b18c3764faf8c5d8c228fdaa502c Mon Sep 17 00:00:00 2001
From: Terry McGuinness <terry.mcguinness@noaa.gov>
Date: Wed, 16 Oct 2024 16:21:28 -0400
Subject: [PATCH 88/95] added correct pipline for ci

---
 ...balworkflow-ci.yaml => pw_aws_cenots.yaml} | 47 ++++++++++---------
 1 file changed, 25 insertions(+), 22 deletions(-)
 rename .github/workflows/{globalworkflow-ci.yaml => pw_aws_cenots.yaml} (63%)

diff --git a/.github/workflows/globalworkflow-ci.yaml b/.github/workflows/pw_aws_cenots.yaml
similarity index 63%
rename from .github/workflows/globalworkflow-ci.yaml
rename to .github/workflows/pw_aws_cenots.yaml
index 1474c79a48..76258c3044 100644
--- a/.github/workflows/globalworkflow-ci.yaml
+++ b/.github/workflows/pw_aws_cenots.yaml
@@ -1,4 +1,4 @@
-name: gw-ci-orion
+name: gw-ci-aws-centos
 
 on: [workflow_dispatch]
 
@@ -15,28 +15,32 @@ on: [workflow_dispatch]
 #         └── ${pslot}
 env:
   TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
-  MACHINE_ID: orion
+  MACHINE_ID: noaacloud
 
 jobs:
-  checkout-build-link:
-    runs-on: [self-hosted, orion-ready]
+  checkout:
+    runs-on: [self-hosted, aws, parallelworks, centos]
     timeout-minutes: 600
 
     steps:
+
     - name: Checkout global-workflow
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
-        path: ${{ github.run_id }}/HOMEgfs  # This path needs to be relative
+        path: ${{ github.run_id }}/HOMEgfs
+        submodules: 'recursive'
+        ref: ${{ github.event.pull_request.head.ref }}
 
-    - name: Checkout components
-      run: |
-        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-        ./checkout.sh -c -g  # Options e.g. -u can be added late
+  build-link:      
+    runs-on: [self-hosted, aws, parallelworks, centos]
+    needs: checkout
+
+    steps:
 
     - name: Build components
       run: |
         cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-        ./build_all.sh
+        ./build_all.sh -j 8
 
     - name: Link artifacts
       run: |
@@ -44,43 +48,42 @@ jobs:
         ./link_workflow.sh
 
   create-experiments:
-    needs: checkout-build-link
-    runs-on: [self-hosted, orion-ready]
+    needs: checkout
+    runs-on: [self-hosted, aws, parallelworks, centos]
     strategy:
       matrix:
-        case: ["C48_S2S", "C96_atm3DVar"]
+        case: ["C48_ATM"]
 
     steps:
       - name: Create Experiments ${{ matrix.case }}
         env:
-          HOMEgfs_PR: ${{ env.TEST_DIR }}/HOMEgfs
           RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS
           pslot: ${{ matrix.case }}.${{ github.run_id }}
         run: |
+          mkdir -p ${{ env.RUNTESTS }}
           cd ${{ env.TEST_DIR }}/HOMEgfs
           source workflow/gw_setup.sh
-          source ci/platforms/orion.sh
-          ./ci/scripts/create_experiment.py --yaml ci/cases/${{ matrix.case }}.yaml --dir ${{ env.HOMEgfs_PR }}
+          source ci/platforms/config.noaacloud
+          ./workflow/create_experiment.py --yaml ci/cases/pr/${{ matrix.case }}.yaml --overwrite
 
   run-experiments:
     needs: create-experiments
-    runs-on: [self-hosted, orion-ready]
+    runs-on: [self-hosted, aws, parallelworks, centos]
     strategy:
       max-parallel: 2
       matrix:
-        case: ["C48_S2S", "C96_atm3DVar"]
+        case: ["C48_ATM"]
     steps:
       - name: Run Experiment ${{ matrix.case }}
         run: |
           cd ${{ env.TEST_DIR }}/HOMEgfs
-          ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }}
+          ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs
 
   clean-up:
     needs: run-experiments
-    runs-on: [self-hosted, orion-ready]
+    runs-on: [self-hosted, aws, parallelworks, centos]
     steps:
       - name: Clean-up
         run: |
           cd ${{ github.workspace }}
           rm -rf ${{ github.run_id }}
-

From ba8f0ff4f4dfa63e747a6a7a1cf89ededb866f35 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Fri, 18 Oct 2024 16:43:27 +0000
Subject: [PATCH 89/95] added rocky workflow for testing development

---
 .github/workflows/pw_aws_rocky.yaml | 82 +++++++++++++++++++++++++++++
 1 file changed, 82 insertions(+)
 create mode 100644 .github/workflows/pw_aws_rocky.yaml

diff --git a/.github/workflows/pw_aws_rocky.yaml b/.github/workflows/pw_aws_rocky.yaml
new file mode 100644
index 0000000000..d72e93962f
--- /dev/null
+++ b/.github/workflows/pw_aws_rocky.yaml
@@ -0,0 +1,82 @@
+name: gw-ci-aws-rocky
+
+on: [workflow_dispatch]
+
+# TEST_DIR contains 2 directories;
+# 1. HOMEgfs: clone of the global-workflow
+# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
+# e.g. $> tree ./TEST_DIR
+# ./TEST_DIR
+# ├── HOMEgfs
+# └── RUNTESTS
+#     ├── COMROT
+#     │   └── ${pslot}
+#     └── EXPDIR
+#         └── ${pslot}
+env:
+  TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
+  MACHINE_ID: noaacloud
+
+jobs:
+  checkout-build-link:
+    runs-on: [self-hosted, aws, parallelworks, rocky]
+    timeout-minutes: 600
+
+    steps:
+
+    - name: Checkout global-workflow
+      uses: actions/checkout@v4
+      with:
+        path: ${{ github.run_id }}/HOMEgfs
+        submodules: 'recursive'
+
+    - name: Build components
+      run: |
+        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
+        ./build_all.sh -j 8
+
+    - name: Link artifacts
+      run: |
+        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
+        ./link_workflow.sh
+
+  create-experiments:
+    needs: checkout-build-link
+    runs-on: [self-hosted, aws, parallelworks, rocky]
+    strategy:
+      matrix:
+        case: ["C48_ATM"]
+
+    steps:
+      - name: Create Experiments ${{ matrix.case }}
+        env:
+          RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS
+          pslot: ${{ matrix.case }}.${{ github.run_id }}
+        run: |
+          mkdir -p ${{ env.RUNTESTS }}
+          cd ${{ env.TEST_DIR }}/HOMEgfs
+          source workflow/gw_setup.sh
+          source ci/platforms/config.noaacloud
+          ./workflow/create_experiment.py --yaml ci/cases/pr/${{ matrix.case }}.yaml --overwrite
+
+  run-experiments:
+    needs: create-experiments
+    runs-on: [self-hosted, aws, parallelworks, rocky]
+    strategy:
+      max-parallel: 2
+      matrix:
+        case: ["C48_ATM"]
+    steps:
+      - name: Run Experiment ${{ matrix.case }}
+        run: |
+          cd ${{ env.TEST_DIR }}/HOMEgfs
+          ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs
+
+  clean-up:
+    needs: run-experiments
+    runs-on: [self-hosted, aws, parallelworks, rocky]
+    steps:
+      - name: Clean-up
+        run: |
+          cd ${{ github.workspace }}
+          rm -rf ${{ github.run_id }}
\ No newline at end of file

From 05ffb6d94a8a9600667f7851269e1058d9a2f89d Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 21 Oct 2024 18:39:02 +0000
Subject: [PATCH 90/95] added a kludge fix to mound epic contrib disk for
 access on the compute notes

---
 ush/load_fv3gfs_modules.sh | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh
index b7e0a0f04f..0e5d44d6f9 100755
--- a/ush/load_fv3gfs_modules.sh
+++ b/ush/load_fv3gfs_modules.sh
@@ -20,6 +20,14 @@ source "${HOMEgfs}/versions/run.${MACHINE_ID}.ver"
 module use "${HOMEgfs}/modulefiles"
 
 case "${MACHINE_ID}" in
+  "noaacloud")
+    #TODO this is a total kludge to get epic mount point for compute nodes
+    #    to be the same as the login node.  This should be workng from in the
+    #    ALLNODES section of the User Bootstrap of Parllel Works but it doen't
+    #    on the Rokcky Clusters (works fine in the Centos 7 cluster)
+    /contrib/Terry.McGuinness/SETUP/mount-epic-contrib.sh
+    sudo systemctl daemon-reload
+    ;;
   "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4" | "noaacloud")
     module load "module_base.${MACHINE_ID}"
     ;;

From 79269890b14a4bc0d104b72aca36ac6c48511b84 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 21 Oct 2024 18:42:37 +0000
Subject: [PATCH 91/95] added runtime ver for noaa cload rocky 8

---
 versions/run.noaacloud.ver | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/versions/run.noaacloud.ver b/versions/run.noaacloud.ver
index 1b5482a67e..0f2a815dec 100644
--- a/versions/run.noaacloud.ver
+++ b/versions/run.noaacloud.ver
@@ -6,3 +6,7 @@ source "${HOMEgfs:-}/versions/spack.ver"
 export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core"
 
 export g2tmpl_ver=1.10.2
+export wgrib2_ver=3.1.2_wmo
+export cdo_ver=2.3.0
+export jasper_ver=4.2.0
+export nco_ver=5.1.6

From fd3ab64d92c8710c224924a6c9b7e65a1ca3430f Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 21 Oct 2024 20:54:56 +0000
Subject: [PATCH 92/95] added test if all ready mounted

---
 ush/load_fv3gfs_modules.sh | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh
index 0e5d44d6f9..0a8b18872f 100755
--- a/ush/load_fv3gfs_modules.sh
+++ b/ush/load_fv3gfs_modules.sh
@@ -25,10 +25,13 @@ case "${MACHINE_ID}" in
     #    to be the same as the login node.  This should be workng from in the
     #    ALLNODES section of the User Bootstrap of Parllel Works but it doen't
     #    on the Rokcky Clusters (works fine in the Centos 7 cluster)
-    /contrib/Terry.McGuinness/SETUP/mount-epic-contrib.sh
-    sudo systemctl daemon-reload
+    if [[ ! -d  /contrib-epic/EPIC ]]; then
+      /contrib/Terry.McGuinness/SETUP/mount-epic-contrib.sh
+      sudo systemctl daemon-reload
+    fi
+    module load "module_base.${MACHINE_ID}"
     ;;
-  "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4" | "noaacloud")
+  "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4")
     module load "module_base.${MACHINE_ID}"
     ;;
   *)

From ca92d5469ad2ccfa7457584b4c296bf664bbe695 Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 21 Oct 2024 21:30:41 +0000
Subject: [PATCH 93/95] added to distinct runtime ver vor noaa cload for centos
 and rocky

---
 ush/load_fv3gfs_modules.sh        |  9 ++++++---
 versions/run.noaacloud.centos.ver |  8 ++++++++
 versions/run.noaacloud.rocky.ver  | 12 ++++++++++++
 3 files changed, 26 insertions(+), 3 deletions(-)
 create mode 100644 versions/run.noaacloud.centos.ver
 create mode 100644 versions/run.noaacloud.rocky.ver

diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh
index 0a8b18872f..8c8cec0648 100755
--- a/ush/load_fv3gfs_modules.sh
+++ b/ush/load_fv3gfs_modules.sh
@@ -13,9 +13,6 @@ ulimit_s=$( ulimit -S -s )
 source "${HOMEgfs}/ush/detect_machine.sh"
 source "${HOMEgfs}/ush/module-setup.sh"
 
-# Source versions file for runtime
-source "${HOMEgfs}/versions/run.${MACHINE_ID}.ver"
-
 # Load our modules:
 module use "${HOMEgfs}/modulefiles"
 
@@ -29,9 +26,15 @@ case "${MACHINE_ID}" in
       /contrib/Terry.McGuinness/SETUP/mount-epic-contrib.sh
       sudo systemctl daemon-reload
     fi
+    # Check if the OS is Rocky or CentOS
+    OS_NAME=$(grep -E '^ID=' /etc/os-release | sed -E 's/ID="?([^"]*)"?/\1/') || true
+    # Source versions file for runtime
+    source "${HOMEgfs}/versions/run.${MACHINE_ID}.${OS_NAME}.ver"
     module load "module_base.${MACHINE_ID}"
     ;;
   "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4")
+    # Source versions file for runtime
+    source "${HOMEgfs}/versions/run.${MACHINE_ID}.ver"
     module load "module_base.${MACHINE_ID}"
     ;;
   *)
diff --git a/versions/run.noaacloud.centos.ver b/versions/run.noaacloud.centos.ver
new file mode 100644
index 0000000000..1b5482a67e
--- /dev/null
+++ b/versions/run.noaacloud.centos.ver
@@ -0,0 +1,8 @@
+export stack_intel_ver=2021.3.0
+export stack_impi_ver=2021.3.0
+export spack_env=gsi-addon-env
+
+source "${HOMEgfs:-}/versions/spack.ver"
+export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core"
+
+export g2tmpl_ver=1.10.2
diff --git a/versions/run.noaacloud.rocky.ver b/versions/run.noaacloud.rocky.ver
new file mode 100644
index 0000000000..0f2a815dec
--- /dev/null
+++ b/versions/run.noaacloud.rocky.ver
@@ -0,0 +1,12 @@
+export stack_intel_ver=2021.3.0
+export stack_impi_ver=2021.3.0
+export spack_env=gsi-addon-env
+
+source "${HOMEgfs:-}/versions/spack.ver"
+export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core"
+
+export g2tmpl_ver=1.10.2
+export wgrib2_ver=3.1.2_wmo
+export cdo_ver=2.3.0
+export jasper_ver=4.2.0
+export nco_ver=5.1.6

From 0fb55a28e95d065ecf7f1671d4df5e8ac64914cd Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 21 Oct 2024 22:16:14 +0000
Subject: [PATCH 94/95] updated runtime versions for centos

---
 versions/run.noaacloud.centos.ver | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/versions/run.noaacloud.centos.ver b/versions/run.noaacloud.centos.ver
index 1b5482a67e..5a9b2b6fc5 100644
--- a/versions/run.noaacloud.centos.ver
+++ b/versions/run.noaacloud.centos.ver
@@ -6,3 +6,7 @@ source "${HOMEgfs:-}/versions/spack.ver"
 export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core"
 
 export g2tmpl_ver=1.10.2
+export jasper_ver=2.0.32
+export wgrib2_ver=2.0.8
+export cdo_ver=1.9.5
+export nco_ver=4.9.3

From 24d6388c7e3bc8ab6b751979fbc3e9324d02800a Mon Sep 17 00:00:00 2001
From: tmcguinness <terry.mcguinness@noaa.gov>
Date: Mon, 21 Oct 2024 23:16:59 +0000
Subject: [PATCH 95/95] remove github pipeline codes

---
 .github/workflows/pw_aws_cenots.yaml | 89 ----------------------------
 .github/workflows/pw_aws_rocky.yaml  | 82 -------------------------
 2 files changed, 171 deletions(-)
 delete mode 100644 .github/workflows/pw_aws_cenots.yaml
 delete mode 100644 .github/workflows/pw_aws_rocky.yaml

diff --git a/.github/workflows/pw_aws_cenots.yaml b/.github/workflows/pw_aws_cenots.yaml
deleted file mode 100644
index 76258c3044..0000000000
--- a/.github/workflows/pw_aws_cenots.yaml
+++ /dev/null
@@ -1,89 +0,0 @@
-name: gw-ci-aws-centos
-
-on: [workflow_dispatch]
-
-# TEST_DIR contains 2 directories;
-# 1. HOMEgfs: clone of the global-workflow
-# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
-# e.g. $> tree ./TEST_DIR
-# ./TEST_DIR
-# ├── HOMEgfs
-# └── RUNTESTS
-#     ├── COMROT
-#     │   └── ${pslot}
-#     └── EXPDIR
-#         └── ${pslot}
-env:
-  TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
-  MACHINE_ID: noaacloud
-
-jobs:
-  checkout:
-    runs-on: [self-hosted, aws, parallelworks, centos]
-    timeout-minutes: 600
-
-    steps:
-
-    - name: Checkout global-workflow
-      uses: actions/checkout@v4
-      with:
-        path: ${{ github.run_id }}/HOMEgfs
-        submodules: 'recursive'
-        ref: ${{ github.event.pull_request.head.ref }}
-
-  build-link:      
-    runs-on: [self-hosted, aws, parallelworks, centos]
-    needs: checkout
-
-    steps:
-
-    - name: Build components
-      run: |
-        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-        ./build_all.sh -j 8
-
-    - name: Link artifacts
-      run: |
-        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-        ./link_workflow.sh
-
-  create-experiments:
-    needs: checkout
-    runs-on: [self-hosted, aws, parallelworks, centos]
-    strategy:
-      matrix:
-        case: ["C48_ATM"]
-
-    steps:
-      - name: Create Experiments ${{ matrix.case }}
-        env:
-          RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS
-          pslot: ${{ matrix.case }}.${{ github.run_id }}
-        run: |
-          mkdir -p ${{ env.RUNTESTS }}
-          cd ${{ env.TEST_DIR }}/HOMEgfs
-          source workflow/gw_setup.sh
-          source ci/platforms/config.noaacloud
-          ./workflow/create_experiment.py --yaml ci/cases/pr/${{ matrix.case }}.yaml --overwrite
-
-  run-experiments:
-    needs: create-experiments
-    runs-on: [self-hosted, aws, parallelworks, centos]
-    strategy:
-      max-parallel: 2
-      matrix:
-        case: ["C48_ATM"]
-    steps:
-      - name: Run Experiment ${{ matrix.case }}
-        run: |
-          cd ${{ env.TEST_DIR }}/HOMEgfs
-          ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs
-
-  clean-up:
-    needs: run-experiments
-    runs-on: [self-hosted, aws, parallelworks, centos]
-    steps:
-      - name: Clean-up
-        run: |
-          cd ${{ github.workspace }}
-          rm -rf ${{ github.run_id }}
diff --git a/.github/workflows/pw_aws_rocky.yaml b/.github/workflows/pw_aws_rocky.yaml
deleted file mode 100644
index d72e93962f..0000000000
--- a/.github/workflows/pw_aws_rocky.yaml
+++ /dev/null
@@ -1,82 +0,0 @@
-name: gw-ci-aws-rocky
-
-on: [workflow_dispatch]
-
-# TEST_DIR contains 2 directories;
-# 1. HOMEgfs: clone of the global-workflow
-# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
-# e.g. $> tree ./TEST_DIR
-# ./TEST_DIR
-# ├── HOMEgfs
-# └── RUNTESTS
-#     ├── COMROT
-#     │   └── ${pslot}
-#     └── EXPDIR
-#         └── ${pslot}
-env:
-  TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
-  MACHINE_ID: noaacloud
-
-jobs:
-  checkout-build-link:
-    runs-on: [self-hosted, aws, parallelworks, rocky]
-    timeout-minutes: 600
-
-    steps:
-
-    - name: Checkout global-workflow
-      uses: actions/checkout@v4
-      with:
-        path: ${{ github.run_id }}/HOMEgfs
-        submodules: 'recursive'
-
-    - name: Build components
-      run: |
-        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-        ./build_all.sh -j 8
-
-    - name: Link artifacts
-      run: |
-        cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
-        ./link_workflow.sh
-
-  create-experiments:
-    needs: checkout-build-link
-    runs-on: [self-hosted, aws, parallelworks, rocky]
-    strategy:
-      matrix:
-        case: ["C48_ATM"]
-
-    steps:
-      - name: Create Experiments ${{ matrix.case }}
-        env:
-          RUNTESTS: ${{ env.TEST_DIR }}/RUNTESTS
-          pslot: ${{ matrix.case }}.${{ github.run_id }}
-        run: |
-          mkdir -p ${{ env.RUNTESTS }}
-          cd ${{ env.TEST_DIR }}/HOMEgfs
-          source workflow/gw_setup.sh
-          source ci/platforms/config.noaacloud
-          ./workflow/create_experiment.py --yaml ci/cases/pr/${{ matrix.case }}.yaml --overwrite
-
-  run-experiments:
-    needs: create-experiments
-    runs-on: [self-hosted, aws, parallelworks, rocky]
-    strategy:
-      max-parallel: 2
-      matrix:
-        case: ["C48_ATM"]
-    steps:
-      - name: Run Experiment ${{ matrix.case }}
-        run: |
-          cd ${{ env.TEST_DIR }}/HOMEgfs
-          ./ci/scripts/run-check_ci.sh ${{ env.TEST_DIR }} ${{ matrix.case }}.${{ github.run_id }} HOMEgfs
-
-  clean-up:
-    needs: run-experiments
-    runs-on: [self-hosted, aws, parallelworks, rocky]
-    steps:
-      - name: Clean-up
-        run: |
-          cd ${{ github.workspace }}
-          rm -rf ${{ github.run_id }}
\ No newline at end of file