Compilation Tests for ARW (#2115)

TYPE: enhancement KEYWORDS: testing, compilation SOURCE: internal DESCRIPTION OF CHANGES: Problem: The testing frameworks (github actions nor Jenkins) don't run a wide array of compilation tests per compiler family commonly available. Solution: Use the github actions / hpc-workflows testing framework to run serial, sm, dm, and dm+sm for GNU, Intel Classic & OneAPI, and PGI/nvhpc on Derecho. Tests will run collectively in one node in parallel to maximize core hour allocations. TESTS CONDUCTED: 1. Tests can be run locally on Derecho by using the .ci/hpc-workflows runner.py to launch all tests as separate jobs, but this is inefficient and not exactly as the tests run. To emulate the tests as done in the github actions workflow use the additional `args` set in the respective testSet of .github/workflows/.ci.yml and provide alternate directories from which to compile. This can be facilitated via : * clone WRF branch that features these changes into `<dir>` * list names of tests you want space-delimited in variable, e.g. `export tests="make-gnu-serial make-gnu-dm"` * create copy test dirs with something like `printf "%s\n" $tests | xargs -i -P 4 cp -Rp <dir> {}` * Create alt dirs run locations from test definitions. In this case one directory up from each, this can be done with `export altDirs=$( printf "../%s/.ci " $tests )` (note the extra space!) * launch tests using runner and join args along with `-alt $altDirs` * Final command should look like ``` <dir>/.ci/hpc-workflows/.ci/runner.py <dir>/.ci/wrf_compilation_tests-make.json -t $tests -a <account> -p <num parallel> -tp 1 -j='{"node_select":{"-l ":{"select":1}}}' -jn <job name> -alt $altDirs ```
wrf-model · Oct 16, 2024 · 788c882 · 788c882
1 parent 915f93d
commit 788c882
Show file tree

Hide file tree

Showing 4 changed files with 86 additions and 46 deletions.
diff --git a/.ci/hpc-workflows b/.ci/hpc-workflows
diff --git a/.ci/wrf_compilation_tests-make.json b/.ci/wrf_compilation_tests-make.json
@@ -17,53 +17,75 @@
       "queue"      : "main",
       "hpc_arguments"  : 
       {
-        "node_select" : { "-l " : { "select"       : 1, "ncpus" : 16 } },
+        "node_select" : { "-l " : { "select"       : 1, "ncpus" : 8 } },
         "priority"    : { "-l " : { "job_priority" : "economy"       } }
       },
       "arguments"  : 
       {
-        "base_env_numprocs"      : [ "-e", "NUM_PROCS=16" ],
+        "base_env_numprocs"      : [ "-e", "NUM_PROCS=8" ],
         "very_last_modules"       : [ "netcdf" ],
         ".*gnu.*::test_modules"   : [ "gcc" ],
-        ".*intel(?!-llvm).*::test_modules" : [ "intel-classic" ],
+        ".*intel-classic.*::test_modules" : [ "intel-classic" ],
         ".*intel-llvm.*::test_modules"     : [ "intel-oneapi" ],
         ".*pgi.*::test_modules"   : [ "nvhpc" ],
         ".*dm.*::test_mpi_module" : [ "cray-mpich" ]
       }
     }
   },
-  "make-gnu" :
-  {
-    "steps" :
-    {
-      "serial" :
-      {
-        "command"      : ".ci/tests/build.sh",
-        "arguments"    : [ "-c", "32" ]
-      },
-      "sm" :
-      {
-        "command"      : ".ci/tests/build.sh",
-        "arguments"    : [ "-c", "33" ],
-        "dependencies" : { "serial" : "afterany" }
-      }
-    }
+
+  "make-gnu-serial" : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "32" ] } } },
+  "make-gnu-sm"     : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "33" ] } } },
+  "make-gnu-dm"     : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "34" ] } } },
+  "make-gnu-dm+sm"  : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "35" ] } } },
+
+  "make-intel-classic-serial" : {
+    "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
+    "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "13" ] } }
   },
-  "make-gnu-mpi" :
-  {
-    "steps" :
-    {
-      "dm" :
-      {
-        "command"      : ".ci/tests/build.sh",
-        "arguments"    : [ "-c", "34" ]
-      },
-      "dm+sm" :
-      {
-        "command"      : ".ci/tests/build.sh",
-        "arguments"    : [ "-c", "35" ],
-        "dependencies" : { "dm" : "afterany" }
-      }
-    }
+  "make-intel-classic-sm" : {
+    "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
+    "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "14" ] } }
+  },
+  "make-intel-classic-dm" : {
+    "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
+    "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "15" ] } }
+  },
+  "make-intel-classic-dm+sm" : {
+    "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
+    "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "16" ] } }
+  },
+
+  "make-intel-llvm-serial" : {
+    "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
+    "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "76" ] } }
+  },
+  "make-intel-llvm-sm" : {
+    "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
+    "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "77" ] } }
+  },
+  "make-intel-llvm-dm" : {
+    "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
+    "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "78" ] } }
+  },
+  "make-intel-llvm-dm+sm" : {
+    "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
+    "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "79" ] } }
+  },
+
+  "make-pgi-serial" : {
+    "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
+    "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "1" ] } }
+  },
+  "make-pgi-sm"     : {
+    "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
+    "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "2" ] } }
+  },
+  "make-pgi-dm"     : {
+    "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
+    "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "3" ] } }
+  },
+  "make-pgi-dm+sm"  : {
+    "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } },
+    "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "4" ] } }
   }
 }
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -49,8 +49,22 @@ jobs:
             tpool : 1
             mkdirs : true
             tests :
-              - make-gnu
-              - make-gnu-mpi
+              - make-gnu-serial
+              - make-gnu-sm
+              - make-gnu-dm
+              - make-gnu-dm+sm
+              - make-intel-classic-serial
+              - make-intel-classic-sm
+              - make-intel-classic-dm
+              - make-intel-classic-dm+sm
+              - make-intel-llvm-serial
+              - make-intel-llvm-sm
+              - make-intel-llvm-dm
+              - make-intel-llvm-dm+sm
+              - make-pgi-serial
+              - make-pgi-sm
+              - make-pgi-dm
+              - make-pgi-dm+sm
               # add new compilation tests here
 
     uses : ./.github/workflows/test_workflow.yml

diff --git a/.github/workflows/test_workflow.yml b/.github/workflows/test_workflow.yml
@@ -69,12 +69,15 @@ jobs:
       if  : ${{ inputs.mkdirs }}
       id  : cpTestDirs
       run : |
-        for testDir in ${{ join( fromJson( inputs.tests ), ' ' ) }}; do
-          echo "Creating duplicate directory for $testDir"
-          # Remove if it exists to get a fresh start
-          rm -rf $testDir
-          cp -Rp main/ $testDir
-        done
+        # Limit parallel ops to 4 at a time to not thrash login nodes
+
+        # Remove if it exists to get a fresh start
+        echo "Cleaning old test directories..."
+        tests="${{ join( fromJson( inputs.tests ), ' ' ) }}"
+        time printf "%s\n" $tests | xargs -i -P 4 rm -rf {}
+
+        echo "Creating duplicate directory for $tests"
+        time printf "%s\n" $tests | xargs -i -P 4 cp -Rp main/ {}
 
     - name: Test ${{ inputs.name }}
       id  : runTest
@@ -87,6 +90,7 @@ jobs:
           -t   ${{ join( fromJson( inputs.tests ), ' ' ) }}  \
           -a "${{ inputs.account }}"                         \
           -p ${{ inputs.pool}} -tp ${{ inputs.tpool }}       \
+          -jn ${{ github.event_name == 'push' && github.ref_name || github.event.number }}-${{ inputs.id }} \
           ${{ inputs.args }} $ALT_DIRS
 
 
@@ -129,8 +133,8 @@ jobs:
         # *documented* functionality doesn't work as expected. Wow, bravo
         # can't use ${{ env. }} as somehow this combination of matrix->reusable workflow->call step is too complex
         # and expands to nothing
-        name: ${{ github.event_name == 'push' && 'master' || github.event.number }}-${{ inputs.id }}_logfiles
-        path: ${{ inputs.archive }}/${{ github.event_name == 'push' && 'master' || github.event.number }}/${{ inputs.id }}/
+        name: ${{ github.event_name == 'push' && github.ref_name || github.event.number }}-${{ inputs.id }}_logfiles
+        path: ${{ inputs.archive }}/${{ github.event_name == 'push' && github.ref_name || github.event.number }}/${{ inputs.id }}/
 
     # As noted in ci.yml, this will need to be moved to a separate workflow with pull_request_target
     # and strictly controlled usage of the GH token
+8 −0		.ci/Step.py
+26 −8		.ci/SubmitAction.py
+10 −0		.ci/SubmitCommon.py
+57 −48		.ci/SubmitOptions.py
+4 −0		.ci/Test.py
+0 −6		.ci/example.json
+16 −1		.ci/runner.py
+4 −2		tutorials/AdvancedRunOption_join_hpc.ipynb
+30 −21		tutorials/AdvancedRunOption_join_hpc.md
+11 −8		tutorials/AdvancedTestConfig_host_specific.md
+50 −50		tutorials/AdvancedTestConfig_hpc_argpacks.md
+17 −17		tutorials/AdvancedTestConfig_regex_argpacks.md
+13 −13		tutorials/BasicTestConfig.md