From df8fa78ce0373615937684f3d79768e1fae0d80f Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 25 Jan 2024 16:31:28 -0700 Subject: [PATCH] Feature #2796 main_v11.1 gha node20 (#2798) * Per #2796, update versions of actions to fix the node 16 to 20 warning message. * Per #2796, port fixes for artifact name handling over from the develop branch to the main_v11.1 testing workflow. Also add the compilation_options.yml workflow since the workflows are being updated. --- .../build_docker_and_trigger_metplus.yml | 4 +- .github/workflows/compilation_options.yml | 90 +++++++++ .github/workflows/documentation.yml | 8 +- .github/workflows/testing.yml | 184 ++++++++++-------- .github/workflows/update_truth.yml | 2 +- 5 files changed, 205 insertions(+), 83 deletions(-) create mode 100644 .github/workflows/compilation_options.yml diff --git a/.github/workflows/build_docker_and_trigger_metplus.yml b/.github/workflows/build_docker_and_trigger_metplus.yml index 89e5e14f67..73d64f1d95 100644 --- a/.github/workflows/build_docker_and_trigger_metplus.yml +++ b/.github/workflows/build_docker_and_trigger_metplus.yml @@ -18,7 +18,7 @@ jobs: name: Handle Docker Image runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Get branch name id: get_branch_name @@ -47,7 +47,7 @@ jobs: env: GITHUB_CONTEXT: ${{ toJson(github) }} run: echo "$GITHUB_CONTEXT" - - uses: actions/github-script@v6 + - uses: actions/github-script@v7 with: github-token: ${{ secrets.METPLUS_BOT_TOKEN }} script: | diff --git a/.github/workflows/compilation_options.yml b/.github/workflows/compilation_options.yml new file mode 100644 index 0000000000..83118dcb5e --- /dev/null +++ b/.github/workflows/compilation_options.yml @@ -0,0 +1,90 @@ +name: Compilation Options + +# Test matrix of MET configuration/compilation options + +on: + + # Note that scheduled cron events are run on the default branch. + # Enable this schedule when main_v12.0 becomes the default branch. + + # schedule: + # - cron: '0 7 * * 0'' + + push: + + tags: + - '**' + + workflow_dispatch: + +env: + DOCKERHUB_REPO: dtcenter/met-dev + +jobs: + + job_control: + name: Determine which jobs to run + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set job controls + id: job_status + run: .github/jobs/set_job_controls.sh + env: + commit_msg: ${{ github.event.head_commit.message }} + force_tests: ${{ github.event.inputs.force_tests }} + + outputs: + met_base_repo: ${{ steps.job_status.outputs.met_base_repo }} + met_base_tag: ${{ steps.job_status.outputs.met_base_tag }} + branch_name: ${{ steps.job_status.outputs.branch_name }} + + compile: + name: Config Opts + runs-on: ubuntu-latest + needs: job_control + strategy: + matrix: + include: + - jobid: 'job1' + config: '' + - jobid: 'job2' + config: '--enable-grib2 --enable-python --enable-lidar2nc --enable-mode_graphics --enable-modis' + - jobid: 'job3' + config: '--enable-grib2' + - jobid: 'job4' + config: '--enable-python' + - jobid: 'job5' + config: '--enable-lidar2nc' + - jobid: 'job6' + config: '--enable-mode_graphics' + - jobid: 'job7' + config: '--enable-modis' + fail-fast: false + steps: + - uses: actions/checkout@v4 + + - name: Create directories to store output + run: mkdir -p ${RUNNER_WORKSPACE}/logs + + - name: Compile MET in Docker + run: .github/jobs/build_docker_image.sh + env: + SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }} + MET_BASE_REPO: ${{ needs.job_control.outputs.met_base_repo }} + MET_BASE_TAG: ${{ needs.job_control.outputs.met_base_tag }} + MET_CONFIG_OPTS: ${{ matrix.config }} + + - name: Copy all build log files into logs directory + if: always() + run: cp ${GITHUB_WORKSPACE}/*.log ${RUNNER_WORKSPACE}/logs/ + + - name: Upload logs as artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: logs_${{ matrix.jobid }} + path: ${{ runner.workspace }}/logs + if-no-files-found: ignore + diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index b039177ff7..9f061df17b 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -18,8 +18,8 @@ jobs: name: Build Documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: '3.8' - name: Install dependencies @@ -29,12 +29,12 @@ jobs: python -m pip install -r docs/requirements.txt - name: Build docs run: ./.github/jobs/build_documentation.sh - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: documentation path: artifact/documentation - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: failure() with: name: documentation_warnings.log diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 40c22d80e9..cc8c414fe7 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -50,7 +50,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set job controls id: job_status run: .github/jobs/set_job_controls.sh @@ -76,7 +76,7 @@ jobs: needs: job_control if: ${{ needs.job_control.outputs.run_compile == 'true' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Create directories to store output run: mkdir -p ${RUNNER_WORKSPACE}/logs @@ -102,9 +102,9 @@ jobs: - name: Upload logs as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: logs + name: logs_compile path: ${{ runner.workspace }}/logs if-no-files-found: ignore @@ -131,12 +131,14 @@ jobs: if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} strategy: matrix: - tests: - - 'ascii2nc' - - 'pb2nc madis2nc pcp_combine' + include: + - jobid: 'job1' + tests: 'ascii2nc' + - jobid: 'job2' + tests: 'pb2nc madis2nc pcp_combine' fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Free disk space run: .github/jobs/free_disk_space.sh @@ -149,16 +151,16 @@ jobs: INPUT_DATA_VERSION: ${{ needs.job_control.outputs.input_data_version }}-all - name: Upload output as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: unit_1a + name: unit_1a_${{ matrix.jobid }} path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: logs + name: logs_unit_1a_${{ matrix.jobid }} path: ${{ runner.workspace }}/logs if-no-files-found: ignore @@ -169,12 +171,14 @@ jobs: if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} strategy: matrix: - tests: - - 'ascii2nc_indy pb2nc_indy tc_dland tc_pairs tc_stat plot_tc tc_rmw rmw_analysis tc_diag tc_gen' - - 'met_test_scripts mode_multivar mode_graphics mtd regrid airnow gsi_tools netcdf modis series_analysis gen_ens_prod wwmca_regrid gen_vx_mask grid_weight interp_shape grid_diag grib_tables lidar2nc shift_data_plane trmm2nc aeronet wwmca_plot ioda2nc gaussian' + include: + - jobid: 'job1' + tests: 'ascii2nc_indy pb2nc_indy tc_dland tc_pairs tc_stat plot_tc tc_rmw rmw_analysis tc_diag tc_gen' + - jobid: 'job2' + tests: 'met_test_scripts mode_multivar mode_graphics mtd regrid airnow gsi_tools netcdf modis series_analysis gen_ens_prod wwmca_regrid gen_vx_mask grid_weight interp_shape grid_diag grib_tables lidar2nc shift_data_plane trmm2nc aeronet wwmca_plot ioda2nc gaussian' fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Free disk space run: .github/jobs/free_disk_space.sh @@ -187,33 +191,36 @@ jobs: INPUT_DATA_VERSION: ${{ needs.job_control.outputs.input_data_version }}-all - name: Upload output as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: unit_1b + name: unit_1b_${{ matrix.jobid }} path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: logs + name: logs_unit_1b_${{ matrix.jobid }} path: ${{ runner.workspace }}/logs if-no-files-found: ignore - unit_rc_leads: - name: Unit RC leads + unit_1c: + name: Unit 1c runs-on: ubuntu-latest needs: [job_control, update_input_data, compile] if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} strategy: matrix: - tests: - - 'ref_config_lead_00 ref_config_lead_12' - - 'ref_config_lead_24 ref_config_lead_48' - - 'ref_config_lead_36' + include: + - jobid: 'job1' + tests: 'ref_config_lead_00 ref_config_lead_12' + - jobid: 'job2' + tests: 'ref_config_lead_24 ref_config_lead_48' + - jobid: 'job3' + tests: 'ref_config_lead_36' fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Free disk space run: .github/jobs/free_disk_space.sh @@ -226,40 +233,42 @@ jobs: INPUT_DATA_VERSION: ${{ needs.job_control.outputs.input_data_version }}-all - name: Upload output as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: unit_rc_leads + name: unit_1c_${{ matrix.jobid }} path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: logs + name: logs_rc_leads_${{ matrix.jobid }} path: ${{ runner.workspace }}/logs if-no-files-found: ignore - unit_rc: - name: Unit RC + unit_2c: + name: Unit 2c runs-on: ubuntu-latest - needs: [job_control, unit_rc_leads] + needs: [job_control, unit_1c] if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} strategy: matrix: - tests: - - 'ref_config' + include: + - jobid: 'job1' + tests: 'ref_config' fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Free disk space run: .github/jobs/free_disk_space.sh - name: Download ref_config_leads output from artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: unit_rc_leads path: ${{ runner.workspace }}/output + pattern: unit_1c_job* + merge-multiple: true - name: Run Unit Tests in Docker run: .github/jobs/run_unit_docker.sh @@ -269,16 +278,16 @@ jobs: INPUT_DATA_VERSION: 'none' - name: Upload output as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: unit_rc + name: unit_2c_${{ matrix.jobid }} path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: logs + name: logs_unit_2c_${{ matrix.jobid }} path: ${{ runner.workspace }}/logs if-no-files-found: ignore @@ -289,23 +298,28 @@ jobs: if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} strategy: matrix: - tests: - - 'point_stat stat_analysis_ps' - - 'grid_stat stat_analysis_gs' - - 'wavelet_stat stat_analysis_ws' - - 'ensemble_stat stat_analysis_es' + include: + - jobid: 'job1' + tests: 'point_stat stat_analysis_ps' + - jobid: 'job2' + tests: 'grid_stat stat_analysis_gs' + - jobid: 'job3' + tests: 'wavelet_stat stat_analysis_ws' + - jobid: 'job4' + tests: 'ensemble_stat stat_analysis_es' fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Free disk space run: .github/jobs/free_disk_space.sh - name: Download 1a output from artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: unit_1a path: ${{ runner.workspace }}/output + pattern: unit_1a_job* + merge-multiple: true - name: Run Unit Tests in Docker run: .github/jobs/run_unit_docker.sh @@ -315,16 +329,16 @@ jobs: INPUT_DATA_VERSION: ${{ needs.job_control.outputs.input_data_version }}-all - name: Upload output as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: unit_2a + name: unit_2a_${{ matrix.jobid }} path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: logs + name: logs_unit_2a_${{ matrix.jobid }} path: ${{ runner.workspace }}/logs if-no-files-found: ignore @@ -335,23 +349,28 @@ jobs: if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} strategy: matrix: - tests: - - 'climatology_1.0deg' - - 'climatology_1.5deg' - - 'climatology_2.5deg' - - 'python point2grid plot_data_plane mode mode_analysis perc_thresh hira plot_point_obs quality_filter obs_summary duplicate_flag' + include: + - jobid: 'job1' + tests: 'climatology_1.0deg' + - jobid: 'job2' + tests: 'climatology_1.5deg' + - jobid: 'job3' + tests: 'climatology_2.5deg' + - jobid: 'job4' + tests: 'python point2grid plot_data_plane mode mode_analysis perc_thresh hira plot_point_obs quality_filter obs_summary duplicate_flag' fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Free disk space run: .github/jobs/free_disk_space.sh - name: Download 1a output from artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: unit_1a path: ${{ runner.workspace }}/output + pattern: unit_1a_job* + merge-multiple: true - name: Run Unit Tests in Docker run: .github/jobs/run_unit_docker.sh @@ -361,34 +380,34 @@ jobs: INPUT_DATA_VERSION: ${{ needs.job_control.outputs.input_data_version }}-all - name: Upload output as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: unit_2b + name: unit_2b_${{ matrix.jobid }} path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: logs + name: logs_unit_2b_${{ matrix.jobid }} path: ${{ runner.workspace }}/logs if-no-files-found: ignore run_diffs: name: Check for Differences runs-on: ubuntu-latest - needs: [job_control, unit_1b, unit_2a, unit_2b, unit_rc] + needs: [job_control, unit_1b, unit_2a, unit_2b, unit_2c] if: ${{ needs.job_control.outputs.run_diff == 'true' }} steps: - name: Download data from previous jobs - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Copy test output into single directory run: | mkdir ${RUNNER_WORKSPACE}/output cp -r unit_*/* ${RUNNER_WORKSPACE}/output/ - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Run Diff Tests in Docker run: .github/jobs/run_diff_docker.sh @@ -399,7 +418,7 @@ jobs: - name: Upload diff files as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: diff path: ${{ runner.workspace }}/diff @@ -407,25 +426,38 @@ jobs: - name: Upload logs as artifact if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: logs + name: logs_diff path: ${{ runner.workspace }}/logs if-no-files-found: ignore + merge_logs: + name: Merge Log Artifacts + runs-on: ubuntu-latest + needs: [run_diffs] + if: ${{ always() }} + steps: + - name: Upload merged logs as artifact + uses: actions/upload-artifact/merge@v4 + with: + name: logs + pattern: logs_* + delete-merged: true + update_truth: name: Update Truth Data runs-on: ubuntu-latest - needs: [job_control, unit_1b, unit_2a, unit_2b, unit_rc] + needs: [job_control, unit_1b, unit_2a, unit_2b, unit_2c] if: ${{ needs.job_control.outputs.run_update_truth == 'true' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Free disk space run: .github/jobs/free_disk_space.sh - name: Download data from previous jobs - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 - name: Copy test output into single directory run: | diff --git a/.github/workflows/update_truth.yml b/.github/workflows/update_truth.yml index cf9734e449..4b8c310dd7 100644 --- a/.github/workflows/update_truth.yml +++ b/.github/workflows/update_truth.yml @@ -26,7 +26,7 @@ jobs: fi echo ERROR: Branch is $branch_name - must be develop or match main_vX.Y exit 1 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 name: Checkout repository with: fetch-depth: 0