From 339722fe34897b0988f0fb727f1992a56b04a928 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 Nov 2023 18:24:39 -0600 Subject: [PATCH] Update develop-ref after #2402 (#2403) Co-authored-by: Dan Adriaansen Co-authored-by: johnhg Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: John Halley Gotway Co-authored-by: jprestop Co-authored-by: Tracy Hertneky Co-authored-by: Giovanni Rosa Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: Mrinal Biswas Co-authored-by: j-opatz Co-authored-by: Daniel Adriaansen Co-authored-by: Jonathan Vigh Co-authored-by: root Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> Co-authored-by: Julie Prestopnik Co-authored-by: Tracy Co-authored-by: Hank Fisher Co-authored-by: bikegeek <3753118+bikegeek@users.noreply.github.com> Co-authored-by: metplus-bot <97135045+metplus-bot@users.noreply.github.com> Co-authored-by: Lisa Goodrich Co-authored-by: Tracy Hertneky <39317287+hertneky@users.noreply.github.com> Co-authored-by: Giovanni Rosa Co-authored-by: mrinalbiswas Co-authored-by: Christina Kalb Co-authored-by: jason-english <73247785+jason-english@users.noreply.github.com> Co-authored-by: John Sharples <41682323+John-Sharples@users.noreply.github.com> Co-authored-by: reza-armuei <144857501+reza-armuei@users.noreply.github.com> fix #1939 develop - failure reading obs when zipped file also exists (#1941) Closes https://github.com/dtcenter/METplus/issues/1986 fix develop Fix broken documentation links (#2004) fix #2026 develop StatAnalysis looping (#2028) fix priority of obs_window config variables so that wrapper-specific version is preferred over generic OBS_WINDOW_BEGIN/END (#2062) fix #2070 var list numeric order (#2072) fix #2087 develop docs_pdf (#2091) fix #2096/#2098 develop - fix skip if output exists and do not error if no commands were run (#2099) Fix for Dockerfile smell DL4000 (#2112) fix #2082 develop regrid.convert/censor_thresh/censor_val (#2140) fix #2082 main_v5.0 regrid.convert/censor_thresh/censor_val (#2101) fix #2137 develop PointStat -obs_valid_beg/end (#2141) fix failured introduced by urllib3 (see https://github.com/urllib3/urllib3/issues/2168) fix #2161 develop PCPCombine additional field arguments in -subtract mode (#2162) fix #2168 develop - StatAnalysis time shift (#2169) fix releases. (#2183) fix #2189 develop - spaces in complex thresholds (#2191) fix #2179 develop TCPairs fix -diag argument (#2187) fixes (#2200) fix diff tests (#2217) fix automated tests (#2237) fix #2235 rename multivar_itensity to multivar_intensity_flag (#2236) fix #2241 Create directory containing -out_stat file (#2242) fix #2245 use unique run ID to name logger instance (#2247) fix #2244 develop fix diff tests (#2254) fixture to set pytest tmpdir (#2261) fix #1853 develop - PointStat don't require mask variables to be set (#2262) fix #2279 develop - buoy station file from 2022 (#2280) fix (#2313) fix ReadTheDocs requirements to include pillow which is a dependency of sphinx-gallery: see https://blog.readthedocs.com/defaulting-latest-build-tools/ for more info on why this was necessary --- .coveragerc | 2 + .github/jobs/docker_utils.py | 4 +- .github/jobs/free_disk_space.sh | 22 + .github/jobs/get_data_volumes.py | 11 +- .github/jobs/setup_and_run_diff.py | 3 + .github/jobs/setup_and_run_use_cases.py | 4 + .github/parm/use_case_groups.json | 2 +- .github/workflows/testing.yml | 4 +- docs/Contributors_Guide/.DS_Store | Bin 0 -> 8196 bytes docs/Contributors_Guide/conda_env.rst | 23 +- docs/Contributors_Guide/create_wrapper.rst | 31 +- docs/Contributors_Guide/deprecation.rst | 9 +- docs/Contributors_Guide/documentation.rst | 1158 ++++++- .../figure/doc_image_example.png | Bin 0 -> 344271 bytes .../figure/example_table_1.png | Bin 0 -> 106505 bytes .../figure/example_table_2.png | Bin 0 -> 149707 bytes .../figure/example_table_3.png | Bin 0 -> 155029 bytes .../figure/example_table_4.png | Bin 0 -> 180483 bytes .../figure/practice_table.png | Bin 0 -> 461109 bytes docs/Contributors_Guide/github_workflow.rst | 59 +- docs/Contributors_Guide/index.rst | 5 +- docs/Contributors_Guide/testing.rst | 19 +- docs/Release_Guide/coordinated.rst | 4 +- docs/Release_Guide/index.rst | 27 +- docs/Release_Guide/met.rst | 3 +- docs/Release_Guide/met_bugfix.rst | 2 +- docs/Release_Guide/met_development.rst | 3 +- docs/Release_Guide/met_official.rst | 2 +- docs/Release_Guide/metcalcpy.rst | 3 +- docs/Release_Guide/metcalcpy_bugfix.rst | 5 +- docs/Release_Guide/metcalcpy_development.rst | 3 +- docs/Release_Guide/metcalcpy_official.rst | 2 +- docs/Release_Guide/metdataio.rst | 3 +- docs/Release_Guide/metdataio_bugfix.rst | 2 +- docs/Release_Guide/metdataio_development.rst | 3 +- docs/Release_Guide/metdataio_official.rst | 2 +- docs/Release_Guide/metexpress.rst | 3 +- docs/Release_Guide/metexpress_development.rst | 3 +- docs/Release_Guide/metexpress_official.rst | 2 +- docs/Release_Guide/metplotpy.rst | 3 +- docs/Release_Guide/metplotpy_bugfix.rst | 2 +- docs/Release_Guide/metplotpy_development.rst | 3 +- docs/Release_Guide/metplotpy_official.rst | 2 +- docs/Release_Guide/metplus.rst | 3 +- docs/Release_Guide/metplus_bugfix.rst | 2 +- docs/Release_Guide/metplus_development.rst | 3 +- docs/Release_Guide/metplus_official.rst | 4 +- docs/Release_Guide/metviewer.rst | 3 +- docs/Release_Guide/metviewer_bugfix.rst | 2 +- docs/Release_Guide/metviewer_development.rst | 3 +- docs/Release_Guide/metviewer_official.rst | 2 +- docs/Release_Guide/recreate_release.rst | 2 +- .../release_steps/checkout_develop_branch.rst | 2 +- .../release_steps/checkout_main_branch.rst | 2 +- .../clone_project_repository.rst | 2 +- .../common/update_dtc_website.rst | 96 +- .../release_acceptance_testing.rst | 19 + .../coordinated/update_dtc_website.rst | 81 +- .../release_steps/create_release_branch.rst | 2 +- .../release_steps/create_release_extra.rst | 2 +- .../create_release_feature_branch.rst | 2 +- .../create_release_on_github.rst | 2 +- .../finalize_release_on_github_bugfix.rst | 20 +- ...finalize_release_on_github_development.rst | 23 +- .../finalize_release_on_github_official.rst | 49 +- .../release_steps/merge_release_issue.rst | 2 +- .../met/create_release_reference_branch.rst | 6 +- .../met/update_version_bugfix.rst | 2 +- .../met/update_version_development.rst | 2 +- .../met/update_version_official.rst | 5 +- .../met/update_web_server_data.rst | 4 +- .../release_steps/met/update_zenodo.rst | 2 +- .../metcalcpy/update_version.rst | 2 +- .../metcalcpy/update_version_bugfix.rst | 2 +- .../metcalcpy/update_version_official.rst | 2 +- .../metcalcpy/update_version_on_develop.rst | 2 +- .../metdataio/update_version.rst | 4 +- .../metdataio/update_version_bugfix.rst | 4 +- .../metdataio/update_version_official.rst | 2 +- .../metdataio/update_version_on_develop.rst | 4 +- .../metexpress/announce_release.rst | 2 +- .../checkout_development_branch.rst | 2 +- ...checkout_development_branch_submodules.rst | 2 +- .../metexpress/checkout_main_branch.rst | 2 +- .../metexpress/clone_project_repository.rst | 2 +- .../create_development_feature_branch.rst | 2 +- .../metexpress/deploy_AWS_apps.rst | 2 +- .../metexpress/deploy_beta_apps.rst | 2 +- .../metexpress/deploy_production_apps.rst | 2 +- ...finalize_release_on_github_development.rst | 11 +- .../metexpress/merge_development_to_main.rst | 2 +- .../metexpress/merge_feature_branch.rst | 2 +- .../pull_changes_and_merge_to_dev.rst | 2 +- .../metexpress/tag_app_version_and_build.rst | 2 +- .../metexpress/update_dates_and_versions.rst | 2 +- .../release_steps/metplotpy/update_readme.rst | 6 - .../metplotpy/update_version.rst | 2 +- .../metplotpy/update_version_bugfix.rst | 2 +- .../metplotpy/update_version_official.rst | 2 +- .../metplotpy/update_version_on_develop.rst | 2 +- .../metplus/create_release_extra.rst | 2 +- .../create_release_reference_branch.rst | 10 +- .../metplus/update_manage_externals.rst | 8 +- .../metplus/update_release_date.rst | 2 +- .../release_steps/metplus/update_version.rst | 2 +- .../metplus/update_version_bugfix.rst | 2 +- .../metplus/update_version_on_develop.rst | 2 +- .../metplus/update_web_server_data.rst | 4 +- .../metviewer/update_version.rst | 2 +- .../metviewer/update_version_bugfix.rst | 2 +- .../metviewer/update_version_official.rst | 2 +- .../metviewer/update_version_on_develop.rst | 2 +- .../release_steps/open_release_issue.rst | 2 +- .../release_steps/push_release_branch.rst | 2 +- .../release_acceptance_testing.rst | 54 + .../release_steps/rotate_authorship.rst | 2 +- .../set_beta_deletion_reminder_official.rst | 2 +- .../release_steps/update_docs_official.rst | 2 +- .../update_release_notes_bugfix.rst | 2 +- .../update_release_notes_development.rst | 2 +- .../update_release_notes_official.rst | 2 +- .../update_upgrade_instructions.rst | 2 +- docs/Users_Guide/glossary.rst | 28 +- docs/Users_Guide/installation.rst | 21 +- docs/Users_Guide/release-notes.rst | 367 +- docs/Users_Guide/statistics_list.rst | 30 +- docs/Users_Guide/systemconfiguration.rst | 39 +- docs/Users_Guide/wrappers.rst | 3004 ++++++++++------- docs/Verification_Datasets/datasets.rst | 5 +- .../datasets/aeronet.rst | 2 +- .../datasets/gdas_ice.rst | 6 +- .../datasets/gdas_prepbufr.rst | 2 +- .../datasets/goes_level_1b.rst | 2 +- .../datasets/goes_level_2_aerosol.rst | 2 +- .../datasets/gpm_imerg.rst | 2 +- .../datasets/ims_ice.rst | 2 +- docs/Verification_Datasets/datasets/madis.rst | 2 +- .../datasets/metar_isu.rst | 2 +- docs/Verification_Datasets/datasets/mping.rst | 2 +- .../datasets/mrms_composite_reflectivity.rst | 2 +- .../datasets/nexrad_level_2.rst | 2 +- .../datasets/nexrad_level_3.rst | 2 +- .../datasets/opera_eumetnet.rst | 2 +- .../datasets/template.rst | 2 +- docs/Verification_Datasets/datasets/viirs.rst | 2 +- docs/Verification_Datasets/index.rst | 5 +- docs/Verification_Datasets/overview.rst | 5 +- docs/Verification_Datasets/quicksearch.rst | 19 +- docs/index.rst | 9 +- docs/requirements.txt | 5 +- .../UserScript_fcstGEFS_Difficulty_Index.py | 19 +- .../dev_tools/add_met_config_helper.py | 4 +- internal/scripts/docker_env/Dockerfile.conda | 4 +- .../metplus_components_v5.1_py3.10.sh | 1 + .../{5.1.0.lua_wcoss2 => 6.0.0.lua_wcoss2} | 2 +- .../modulefiles/{5.1.0_acorn => 6.0.0_acorn} | 12 +- .../installation/modulefiles/6.0.0_gaea | 20 + .../modulefiles/{5.1.0_hera => 6.0.0_hera} | 8 +- .../modulefiles/{5.1.0_jet => 6.0.0_jet} | 10 +- internal/tests/pytests/conftest.py | 12 +- .../config_metplus/test_config_metplus.py | 27 +- .../pytests/util/run_util/test_run_util.py | 57 +- .../string_manip/test_util_string_manip.py | 56 +- .../command_builder/test_command_builder.py | 243 +- .../extract_tiles/test_extract_tiles.py | 117 +- .../gfdl_tracker/test_gfdl_tracker.py | 247 ++ .../grid_stat/test_grid_stat_wrapper.py | 2 + .../wrappers/met_db_load/test_met_db_load.py | 260 +- .../wrappers/pb2nc/test_pb2nc_wrapper.py | 2 +- .../py_embed_ingest/test_py_embed_ingest.py | 200 ++ .../series_analysis/test_series_analysis.py | 76 +- .../wrappers/tc_diag/test_tc_diag_wrapper.py | 3 +- .../tc_pairs/test_tc_pairs_wrapper.py | 176 +- metplus/util/config_metplus.py | 49 +- metplus/util/config_validate.py | 59 +- metplus/util/run_util.py | 113 +- metplus/util/string_manip.py | 29 + metplus/util/string_template_substitution.py | 2 +- metplus/wrappers/command_builder.py | 43 +- metplus/wrappers/command_runner.py | 181 - metplus/wrappers/grid_stat_wrapper.py | 37 +- metplus/wrappers/series_analysis_wrapper.py | 2 +- metplus/wrappers/tc_diag_wrapper.py | 3 + metplus/wrappers/tc_pairs_wrapper.py | 12 +- parm/met_config/GridStatConfig_wrapped | 3 +- parm/met_config/TCDiagConfig_wrapped | 3 + parm/metplus_config/defaults.conf | 1 + .../met_tool_wrapper/GridStat/GridStat.conf | 1 + .../met_tool_wrapper/TCDiag/TCDiag.conf | 1 + 189 files changed, 5650 insertions(+), 1994 deletions(-) create mode 100644 docs/Contributors_Guide/.DS_Store create mode 100644 docs/Contributors_Guide/figure/doc_image_example.png create mode 100644 docs/Contributors_Guide/figure/example_table_1.png create mode 100644 docs/Contributors_Guide/figure/example_table_2.png create mode 100644 docs/Contributors_Guide/figure/example_table_3.png create mode 100644 docs/Contributors_Guide/figure/example_table_4.png create mode 100644 docs/Contributors_Guide/figure/practice_table.png create mode 100644 docs/Release_Guide/release_steps/coordinated/release_acceptance_testing.rst delete mode 100644 docs/Release_Guide/release_steps/metplotpy/update_readme.rst create mode 100644 docs/Release_Guide/release_steps/release_acceptance_testing.rst rename internal/scripts/installation/modulefiles/{5.1.0.lua_wcoss2 => 6.0.0.lua_wcoss2} (90%) rename internal/scripts/installation/modulefiles/{5.1.0_acorn => 6.0.0_acorn} (66%) create mode 100644 internal/scripts/installation/modulefiles/6.0.0_gaea rename internal/scripts/installation/modulefiles/{5.1.0_hera => 6.0.0_hera} (76%) rename internal/scripts/installation/modulefiles/{5.1.0_jet => 6.0.0_jet} (58%) create mode 100644 internal/tests/pytests/wrappers/gfdl_tracker/test_gfdl_tracker.py create mode 100644 internal/tests/pytests/wrappers/py_embed_ingest/test_py_embed_ingest.py delete mode 100755 metplus/wrappers/command_runner.py diff --git a/.coveragerc b/.coveragerc index d561722b1f..4c62180950 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,3 +1,5 @@ [run] relative_files = True source = metplus +omit = + metplus/wrappers/cyclone_plotter_wrapper.py diff --git a/.github/jobs/docker_utils.py b/.github/jobs/docker_utils.py index a40d6b2e56..edf0b00167 100644 --- a/.github/jobs/docker_utils.py +++ b/.github/jobs/docker_utils.py @@ -39,7 +39,7 @@ def get_dockerhub_url(branch_name): def docker_get_volumes_last_updated(current_branch): import requests dockerhub_url = get_dockerhub_url(current_branch) - dockerhub_request = requests.get(dockerhub_url) + dockerhub_request = requests.get(dockerhub_url, timeout=60) if dockerhub_request.status_code != 200: print(f"Could not find DockerHub URL: {dockerhub_url}") return None @@ -61,7 +61,7 @@ def docker_get_volumes_last_updated(current_branch): volumes_last_updated[repo_name] = repo['last_updated'] if not page['next']: break - page = requests.get(page['next']).json() + page = requests.get(page['next'], timeout=60).json() attempts += 1 return volumes_last_updated diff --git a/.github/jobs/free_disk_space.sh b/.github/jobs/free_disk_space.sh index cfe30cf06d..9c9902a602 100755 --- a/.github/jobs/free_disk_space.sh +++ b/.github/jobs/free_disk_space.sh @@ -13,3 +13,25 @@ sudo rm -rf "$AGENT_TOOLSDIRECTORY" printf "\nChecking disk usage after cleanup" df -h + +echo Pruning docker files + +cmd="docker images" +printf "\nBEFORE CLEANUP: $cmd" +$cmd + +cmd="docker image prune -af" +printf "\nRunning $cmd" +$cmd + +cmd=docker system prune -af +printf "\nRunning $cmd" +$cmd + +cmd="docker images" +printf "\nAFTER CLEANUP: $cmd" +$cmd + +printf "\nChecking disk usage after Docker cleanup" + +df -h diff --git a/.github/jobs/get_data_volumes.py b/.github/jobs/get_data_volumes.py index 5c568577a1..e492711710 100755 --- a/.github/jobs/get_data_volumes.py +++ b/.github/jobs/get_data_volumes.py @@ -40,7 +40,7 @@ def main(args): branch_name = get_branch_name() if not branch_name: print("Could not get current branch. Exiting.") - sys.exit(1) + return None # remove -ref from end of branch name if found if branch_name.endswith('-ref'): @@ -88,7 +88,7 @@ def main(args): # use it, otherwise use develop version of data volume elif (metplus_version == 'develop' and f'{branch_name}-{model_app_name}' in available_volumes): - volume_name = f'{branch_name}-{model_app_name}' + volume_name = f'{branch_name}-{model_app_name}' else: volume_name = f'{metplus_version}-{model_app_name}' @@ -97,13 +97,15 @@ def main(args): cmd = (f'docker create --name {model_app_name} ' f'{full_volume_name}') if not run_commands(cmd): - continue + print(f'ERROR: Could not create data volume for {full_volume_name}') + return None # add name to volumes from list to pass to docker build volume_list.append(f'--volumes-from {model_app_name}') return ' '.join(volume_list) + if __name__ == "__main__": # split up command line args that have commas before passing into main args = [] @@ -111,4 +113,7 @@ def main(args): for arg in sys.argv[1:]: args.extend(arg.split(',')) out = main(args) + if out is None: + print("ERROR: Something went wrong") + sys.exit(1) print(out) diff --git a/.github/jobs/setup_and_run_diff.py b/.github/jobs/setup_and_run_diff.py index 1a4c908b14..59c5be37ed 100755 --- a/.github/jobs/setup_and_run_diff.py +++ b/.github/jobs/setup_and_run_diff.py @@ -43,6 +43,9 @@ output_category = f"output-{output_data_branch}-{artifact_name}" VOLUMES_FROM = get_data_volumes.main([output_category]) +if VOLUMES_FROM is None: + print("ERROR: Could not get truth data to run diff") + sys.exit(1) print(f"Output Volumes: {VOLUMES_FROM}") diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 931a79d76d..d65cf247c5 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -49,6 +49,10 @@ def main(): ) # get input data volumes volumes_from = get_data_volumes.main(categories_list) + if volumes_from is None: + print('ERROR: Could not get input data to run use cases') + sys.exit(1) + print(f"Input Volumes: {volumes_from}") # build Docker image with conda environment and METplus branch image diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 041b000b2c..1752f76df0 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -277,6 +277,6 @@ { "category": "unstructured_grids", "index_list": "0", - "run": true + "run": false } ] diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 239189fb1c..e08aac81e4 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -145,8 +145,10 @@ jobs: env: METPLUS_TEST_OUTPUT_BASE: ${{ runner.workspace }}/pytest_output - name: Generate coverage report - run: coverage report -m + run: | + coverage report -m --fail-under=90 || echo "::error file=coverage,line=1,col=1::Code coverage is below 90%" if: always() + continue-on-error: true - name: Run Coveralls uses: AndreMiras/coveralls-python-action@8799c9f4443ac4201d2e2f2c725d577174683b99 if: always() diff --git a/docs/Contributors_Guide/.DS_Store b/docs/Contributors_Guide/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..fd1c14a87a0e359e10489a81d6d589664996dc93 GIT binary patch literal 8196 zcmeHML2uJA6n<{UmTH<-V2B9`DH7Ldvt`xxmP)L^2Ma$x& z;4dIf{1W&toZx%UW?I{HhznxEuI!iC{=Phaw(Gu>h*Z0Xn?##LpdocBpnVGGdbCbzS`w}i2METuwq@7mL=VbBl?^eJfur5$?yxOuC&n5$ zDFY{EpR8>cb53hDGM$LjiC2jYc4MIhcCzBZOCpjIMN4>iZ1m6>tuKWgNd7 z83{)p3UzT+%2DafaI9nWT_Bi;->sbMLFMZV`ZCZOxB~NHX5=>v)!i9}WqN}#_rL%T zrdLDt^Lc%v(Xek<&X=g4Cd0hZ_@*k=rAx~z-io*Cz3Lq3Q6~@cV%QD`FZk}M)@k(X zI*gtr-BD2A*wtAcCRukNfF$ms%gg6U7VA-47g;Dy=|YmNaTZmV z1