Per #1546, rearrange use case group dependencies, save output as arti…

…fact to download in dependent step, download all output artifacts and run diff at end of workflow
dtcenter · Jan 25, 2022 · 8d74b48 · 8d74b48
1 parent b79bf7e
commit 8d74b48
Show file tree

Hide file tree

Showing 4 changed files with 107 additions and 89 deletions.
diff --git a/.github/jobs/copy_diff_files.py b/.github/jobs/copy_diff_files.py
@@ -71,12 +71,9 @@ def copy_files_to_diff_dir(files_to_copy):
         shutil.copyfile(filename, output_path)
 
 def main():
-    all_files_to_copy = set()
-    log_files = glob.glob(os.path.join(LOG_DIR, 'comp_dir*.log'))
-    for log_file in log_files:
-        print(f"Parsing {log_file}")
-        new_files = get_files_with_diffs(log_file)
-        all_files_to_copy.update(new_files)
+    log_file = glob.glob(os.path.join(LOG_DIR, 'comp_dir.log'))
+    print(f"Parsing {log_file}")
+    all_files_to_copy = get_files_with_diffs(log_file)
 
     copy_files_to_diff_dir(all_files_to_copy)
 

diff --git a/.github/jobs/run_diff_docker.sh b/.github/jobs/run_diff_docker.sh
@@ -19,11 +19,14 @@ mount_args="-v $LOCAL_OUTPUT_DIR:$DOCKER_OUTPUT_DIR -v $LOCAL_DIFF_DIR:$DOCKER_D
 # Set up data volumes
 volumes_from="--volumes-from met_truth"
 
-export TESTS_TO_RUN=$TESTS
-
 # run unit test script inside Docker, mount MET input and truth data
 cmd="\${MET_REPO_DIR}/.github/jobs/run_diff_tests.sh"
-time_command docker run -e TESTS_TO_RUN ${volumes_from} ${mount_args} ${DOCKERHUB_TAG} bash -c \"${cmd}\"
+time_command docker run ${volumes_from} ${mount_args} ${DOCKERHUB_TAG} bash -c \"${cmd}\"
 if [ $? != 0 ]; then
   exit 1
 fi
+
+if [ "$(ls -A ${LOCAL_DIFF_DIR})" ]; then
+  echo "ERROR: Differences exist in the output"
+  exit 1
+fi
diff --git a/.github/jobs/run_diff_tests.sh b/.github/jobs/run_diff_tests.sh
@@ -13,20 +13,15 @@ source ${MET_REPO_DIR}/.github/jobs/test_env_vars.sh
 ###
 
 echo "Running comparison on test output"
-for testname in $TESTS_TO_RUN; do
-  CMD_LOGFILE=/met/logs/comp_dir_${testname}.log
-  time_command ${MET_TEST_BASE}/bin/comp_dir.sh ${MET_TEST_TRUTH}/${testname} ${MET_TEST_OUTPUT}/${testname}
-  if [ $? != 0 ]; then
-      echo "ERROR: Test ${testname} output comparison failed"
-      exit 1
-  fi
-done
-
-test_list=($TESTS_TO_RUN)
-first_test=${test_list[0]}
+CMD_LOGFILE=/met/logs/comp_dir.log
+time_command ${MET_TEST_BASE}/bin/comp_dir.sh ${MET_TEST_TRUTH} ${MET_TEST_OUTPUT}
+if [ $? != 0 ]; then
+    echo "ERROR: Test output comparison failed"
+    exit 1
+fi
 
 echo "Running copy_diff_files.py"
-CMD_LOGFILE=/met/logs/copy_diff_files_${first_test}_group.log
+CMD_LOGFILE=/met/logs/copy_diff_files.log
 time_command ${MET_REPO_DIR}/.github/jobs/copy_diff_files.py
 if [ $? != 0 ]; then
     echo "ERROR: Copy diff files script failed"

diff --git a/.github/workflows/pr_unit_tests.yml b/.github/workflows/pr_unit_tests.yml
@@ -73,17 +73,18 @@ jobs:
           path: ${{ runner.workspace }}/logs
           if-no-files-found: ignore
 
-  unit_tests_1:
+  unit_tests_1a:
     name: Run MET Unit Tests
     runs-on: ubuntu-latest
     needs: [job_control, compile]
     if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }}
     strategy:
       matrix:
         tests:
-          - 'ref_config'
-          - 'tc_dland tc_pairs tc_stat plot_tc tc_rmw rmw_analysis tc_gen'
-          - 'met_test_scripts mode_graphics mtd regrid airnow gsi_tools netcdf modis series_analysis gen_ens_prod wwmca_regrid gen_vx_mask grid_weight interp_shape grid_diag test_grib_tables lidar2nc shift_data_plane trmm2nc aeronet wwmca_plot ioda2nc gaussian'
+          - 'pcp_combine'
+          - 'pb2nc'
+          - 'ascii2nc'
+          - 'madis2nc'
       fail-fast: false
     steps:
       - uses: actions/checkout@v2
@@ -96,21 +97,11 @@ jobs:
           TESTS: ${{ matrix.tests }}
           INPUT_DATA_VERSION: develop
 
-      - name: Run Diff Tests in Docker
-        run: .github/jobs/run_diff_docker.sh
-        env:
-          DOCKERHUB_REPO: ${{ needs.job_control.outputs.dockerhub_repo }}
-          SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }}
-          TESTS: ${{ matrix.tests }}
-          TRUTH_DATA_VERSION: develop
-
-      - name: Upload diff files as artifact
-        if: always()
+      - name: Upload output as artifact
         uses: actions/upload-artifact@v2
         with:
-          name: diff
-          path: ${{ runner.workspace }}/diff
-          if-no-files-found: ignore
+          name: unit_tests_1a
+          path: ${{ runner.workspace }}/output
 
       - name: Upload logs as artifact
         if: always()
@@ -120,18 +111,17 @@ jobs:
           path: ${{ runner.workspace }}/logs
           if-no-files-found: ignore
 
-  unit_tests_2a:
+  unit_tests_1b:
     name: Run MET Unit Tests
     runs-on: ubuntu-latest
     needs: [job_control, compile]
     if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }}
     strategy:
       matrix:
         tests:
-          - 'pcp_combine'
-          - 'pb2nc'
-          - 'ascii2nc'
-          - 'madis2nc'
+          - 'ref_config'
+          - 'tc_dland tc_pairs tc_stat plot_tc tc_rmw rmw_analysis tc_gen'
+          - 'met_test_scripts mode_graphics mtd regrid airnow gsi_tools netcdf modis series_analysis gen_ens_prod wwmca_regrid gen_vx_mask grid_weight interp_shape grid_diag test_grib_tables lidar2nc shift_data_plane trmm2nc aeronet wwmca_plot ioda2nc gaussian'
       fail-fast: false
     steps:
       - uses: actions/checkout@v2
@@ -144,21 +134,55 @@ jobs:
           TESTS: ${{ matrix.tests }}
           INPUT_DATA_VERSION: develop
 
-      - name: Run Diff Tests in Docker
-        run: .github/jobs/run_diff_docker.sh
+      - name: Upload output as artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: unit_tests_1b
+          path: ${{ runner.workspace }}/output
+
+      - name: Upload logs as artifact
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: logs
+          path: ${{ runner.workspace }}/logs
+          if-no-files-found: ignore
+
+  unit_tests_2a:
+    name: Run MET Unit Tests
+    runs-on: ubuntu-latest
+    needs: unit_tests_1a
+    if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }}
+    strategy:
+      matrix:
+        tests:
+          - 'point_stat'
+          - 'grid_stat'
+          - 'wavelet_stat'
+          - 'ensemble_stat'
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Download 1a output from artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: unit_tests_1a
+          path: ${{ runner.workspace }}/output
+
+      - name: Run Unit Tests in Docker
+        run: .github/jobs/run_unit_docker.sh
         env:
           DOCKERHUB_REPO: ${{ needs.job_control.outputs.dockerhub_repo }}
           SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }}
           TESTS: ${{ matrix.tests }}
-          TRUTH_DATA_VERSION: develop
+          INPUT_DATA_VERSION: develop
 
-      - name: Upload diff files as artifact
-        if: always()
+      - name: Upload output as artifact
         uses: actions/upload-artifact@v2
         with:
-          name: diff
-          path: ${{ runner.workspace }}/diff
-          if-no-files-found: ignore
+          name: unit_tests_2a
+          path: ${{ runner.workspace }}/output
 
       - name: Upload logs as artifact
         if: always()
@@ -171,18 +195,23 @@ jobs:
   unit_tests_2b:
     name: Run MET Unit Tests
     runs-on: ubuntu-latest
-    needs: [job_control, compile, unit_tests_2a]
+    needs: unit_tests_1a
     if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }}
     strategy:
       matrix:
         tests:
           - 'climatology'
-          - 'point_stat grid_stat wavelet_stat ensemble_stat'
           - 'python point2grid plot_data_plane mode mode_analysis perc_thresh hira plot_point_obs quality_filter obs_summary duplicate_flag'
       fail-fast: false
     steps:
       - uses: actions/checkout@v2
 
+      - name: Download 1a output from artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: unit_tests_1a
+          path: ${{ runner.workspace }}/output
+
       - name: Run Unit Tests in Docker
         run: .github/jobs/run_unit_docker.sh
         env:
@@ -191,21 +220,11 @@ jobs:
           TESTS: ${{ matrix.tests }}
           INPUT_DATA_VERSION: develop
 
-      - name: Run Diff Tests in Docker
-        run: .github/jobs/run_diff_docker.sh
-        env:
-          DOCKERHUB_REPO: ${{ needs.job_control.outputs.dockerhub_repo }}
-          SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }}
-          TESTS: ${{ matrix.tests }}
-          TRUTH_DATA_VERSION: develop
-
-      - name: Upload diff files as artifact
-        if: always()
+      - name: Upload output as artifact
         uses: actions/upload-artifact@v2
         with:
-          name: diff
-          path: ${{ runner.workspace }}/diff
-          if-no-files-found: ignore
+          name: unit_tests_2b
+          path: ${{ runner.workspace }}/output
 
       - name: Upload logs as artifact
         if: always()
@@ -215,10 +234,11 @@ jobs:
           path: ${{ runner.workspace }}/logs
           if-no-files-found: ignore
 
-  unit_tests_2c:
+
+  unit_tests_3a:
     name: Run MET Unit Tests
     runs-on: ubuntu-latest
-    needs: [job_control, compile, unit_tests_2b]
+    needs: unit_tests_2a
     if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }}
     strategy:
       matrix:
@@ -228,6 +248,12 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - name: Download 2a output from artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: unit_tests_2a
+          path: ${{ runner.workspace }}/output
+
       - name: Run Unit Tests in Docker
         run: .github/jobs/run_unit_docker.sh
         env:
@@ -236,21 +262,11 @@ jobs:
           TESTS: ${{ matrix.tests }}
           INPUT_DATA_VERSION: develop
 
-      - name: Run Diff Tests in Docker
-        run: .github/jobs/run_diff_docker.sh
-        env:
-          DOCKERHUB_REPO: ${{ needs.job_control.outputs.dockerhub_repo }}
-          SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }}
-          TESTS: ${{ matrix.tests }}
-          TRUTH_DATA_VERSION: develop
-
-      - name: Upload diff files as artifact
-        if: always()
+      - name: Upload output as artifact
         uses: actions/upload-artifact@v2
         with:
-          name: diff
-          path: ${{ runner.workspace }}/diff
-          if-no-files-found: ignore
+          name: unit_tests_3a
+          path: ${{ runner.workspace }}/output
 
       - name: Upload logs as artifact
         if: always()
@@ -260,17 +276,24 @@ jobs:
           path: ${{ runner.workspace }}/logs
           if-no-files-found: ignore
 
-  check_for_diffs:
+  run_diffs:
     name: Check if there were any differences
     runs-on: ubuntu-latest
-    needs: [unit_tests_1, unit_tests_2c]
+    needs: [unit_tests_1b, unit_tests_2b, unit_tests_3a]
     steps:
       - uses: actions/download-artifact@v2
+
+      - name: Run Diff Tests in Docker
+        run: .github/jobs/run_diff_docker.sh
+        env:
+          DOCKERHUB_REPO: ${{ needs.job_control.outputs.dockerhub_repo }}
+          SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }}
+          TRUTH_DATA_VERSION: develop
+
+      - name: Upload diff files as artifact
+        if: always()
+        uses: actions/upload-artifact@v2
         with:
           name: diff
-          path: diff
-      - run: |
-          if [ "$(ls -A diff)" ]; then
-            echo "ERROR: Differences exist in the output"
-            exit 1
-          fi
+          path: ${{ runner.workspace }}/diff
+          if-no-files-found: ignore