From 34d7da88fbdfc118f5941c1496c68836a55c9c78 Mon Sep 17 00:00:00 2001 From: George McCabe <23407799+georgemccabe@users.noreply.github.com> Date: Thu, 16 Feb 2023 14:59:25 -0700 Subject: [PATCH] Feature #2022 Use Debian 10 / Python 3.10.4 in automated tests (#2050) --- .github/actions/run_tests/Dockerfile.run | 5 ++-- .../actions/run_tests/Dockerfile.run_cartopy | 5 ++-- .../actions/run_tests/Dockerfile.run_gempak | 6 ++-- .github/actions/run_tests/Dockerfile.run_gfdl | 1 + .github/actions/run_tests/entrypoint.sh | 4 +-- .github/jobs/copy_error_logs.py | 13 ++++----- .github/jobs/docker_utils.py | 2 +- .github/jobs/get_use_case_commands.py | 15 ++++------ .github/jobs/setup_and_run_diff.py | 2 +- .github/jobs/setup_and_run_use_cases.py | 2 +- .github/workflows/testing.yml | 2 +- internal/scripts/docker/Dockerfile | 2 +- .../docker_env/Dockerfile.gfdl-tracker | 4 +++ .../tests/use_cases/metplus_use_case_suite.py | 28 +++++++++++++++---- manage_externals/checkout_externals | 2 +- 15 files changed, 55 insertions(+), 38 deletions(-) diff --git a/.github/actions/run_tests/Dockerfile.run b/.github/actions/run_tests/Dockerfile.run index a780294208..2f44493a2c 100644 --- a/.github/actions/run_tests/Dockerfile.run +++ b/.github/actions/run_tests/Dockerfile.run @@ -6,5 +6,6 @@ FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env ARG METPLUS_IMG_TAG=develop FROM dtcenter/metplus-dev:${METPLUS_IMG_TAG} -COPY --from=env /usr/local/envs /usr/local/envs/ -COPY --from=env /usr/local/bin/conda /usr/local/bin/conda +RUN mkdir -p /usr/local/conda/envs && mkdir -p /usr/local/conda/bin +COPY --from=env /usr/local/conda/envs /usr/local/conda/envs/ +COPY --from=env /usr/local/conda/bin/conda /usr/local/conda/bin/conda diff --git a/.github/actions/run_tests/Dockerfile.run_cartopy b/.github/actions/run_tests/Dockerfile.run_cartopy index 196c4bd505..5d3ffb7eca 100644 --- a/.github/actions/run_tests/Dockerfile.run_cartopy +++ b/.github/actions/run_tests/Dockerfile.run_cartopy @@ -6,7 +6,8 @@ FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env ARG METPLUS_IMG_TAG=develop FROM dtcenter/metplus-dev:${METPLUS_IMG_TAG} -COPY --from=env /usr/local/envs /usr/local/envs/ -COPY --from=env /usr/local/bin/conda /usr/local/bin/conda +RUN mkdir -p /usr/local/conda/envs && mkdir -p /usr/local/conda/bin +COPY --from=env /usr/local/conda/envs /usr/local/conda/envs/ +COPY --from=env /usr/local/conda/bin/conda /usr/local/conda/bin/conda COPY --from=env /root/.local/share/cartopy /root/.local/share/cartopy/ \ No newline at end of file diff --git a/.github/actions/run_tests/Dockerfile.run_gempak b/.github/actions/run_tests/Dockerfile.run_gempak index ac649ca382..2d06a2e07a 100644 --- a/.github/actions/run_tests/Dockerfile.run_gempak +++ b/.github/actions/run_tests/Dockerfile.run_gempak @@ -6,6 +6,8 @@ FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env ARG METPLUS_IMG_TAG=develop FROM dtcenter/metplus-dev:${METPLUS_IMG_TAG} -COPY --from=env /usr/lib/jvm/jre /usr/lib/jvm/jre/ -COPY --from=env /usr/share/javazi-1.8/tzdb.dat /usr/share/javazi-1.8/ +COPY --from=env /usr/lib/jvm/adoptopenjdk-8-hotspot-amd64 /usr/lib/jvm/adoptopenjdk-8-hotspot-amd64/ COPY --from=env /data/input/GempakToCF.jar /data/input/GempakToCF.jar + +# add java executable to path +RUN echo "export PATH=/usr/lib/jvm/adoptopenjdk-8-hotspot-amd64/bin:\$PATH;" >> /root/.bashrc diff --git a/.github/actions/run_tests/Dockerfile.run_gfdl b/.github/actions/run_tests/Dockerfile.run_gfdl index 8876e567e0..7743a354a0 100644 --- a/.github/actions/run_tests/Dockerfile.run_gfdl +++ b/.github/actions/run_tests/Dockerfile.run_gfdl @@ -6,6 +6,7 @@ FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env ARG METPLUS_IMG_TAG=develop FROM dtcenter/metplus-dev:${METPLUS_IMG_TAG} +COPY --from=env /usr/local/lib/libhdf5* /usr/local/lib/ COPY --from=env /usr/local/lib/libnetcdf.so.19* /usr/local/lib/ COPY --from=env /usr/local/lib/libnetcdff* /usr/local/lib/ COPY --from=env /standalone_gfdl-vortextracker_v3.9a_gcc/trk_exec/* /usr/local/bin/ diff --git a/.github/actions/run_tests/entrypoint.sh b/.github/actions/run_tests/entrypoint.sh index 3bfad88343..6c5da65ea2 100644 --- a/.github/actions/run_tests/entrypoint.sh +++ b/.github/actions/run_tests/entrypoint.sh @@ -34,7 +34,7 @@ fi # running unit tests (pytests) if [[ "$INPUT_CATEGORIES" == pytests* ]]; then - export METPLUS_ENV_TAG="pytest" + export METPLUS_ENV_TAG="pytest.v5.1" export METPLUS_IMG_TAG=${branch_name} echo METPLUS_ENV_TAG=${METPLUS_ENV_TAG} echo METPLUS_IMG_TAG=${METPLUS_IMG_TAG} @@ -61,7 +61,7 @@ if [[ "$INPUT_CATEGORIES" == pytests* ]]; then for x in `cat $PYTESTS_GROUPS_FILEPATH`; do marker="${x//_or_/ or }" marker="${marker//not_/not }" - command+="/usr/local/envs/pytest/bin/pytest -vv --cov=../../../metplus -m \"$marker\"" + command+="/usr/local/conda/envs/${METPLUS_ENV_TAG}/bin/pytest -vv --cov=../../../metplus -m \"$marker\"" command+=";if [ \$? != 0 ]; then status=1; fi;" done command+="if [ \$status != 0 ]; then echo ERROR: Some pytests failed. Search for FAILED to review; false; fi" diff --git a/.github/jobs/copy_error_logs.py b/.github/jobs/copy_error_logs.py index 9f3cb45c7f..61c5ef2c3f 100755 --- a/.github/jobs/copy_error_logs.py +++ b/.github/jobs/copy_error_logs.py @@ -10,12 +10,11 @@ def main(output_data_dir, error_logs_dir): """! Copy log output to error log directory if any use case failed """ - for use_case_dir in os.listdir(output_data_dir): - log_dir = os.path.join(output_data_dir, - use_case_dir, - 'logs') - if not os.path.isdir(log_dir): + for log_dir, _, log_files in os.walk(output_data_dir): + # skip non logs directories + if not log_dir.endswith('/logs'): continue + use_case_dir = log_dir.replace(f'{output_data_dir}/', '')[0:-5] # check if there are errors in the metplus.log file and # only copy directory if there are any errors @@ -28,9 +27,7 @@ def main(output_data_dir, error_logs_dir): if not found_errors: continue - output_dir = os.path.join(error_logs_dir, - use_case_dir) - log_files = os.listdir(log_dir) + output_dir = os.path.join(error_logs_dir, use_case_dir) for log_file in log_files: log_path = os.path.join(log_dir, log_file) output_path = os.path.join(output_dir, log_file) diff --git a/.github/jobs/docker_utils.py b/.github/jobs/docker_utils.py index 0e5c365f3f..bb5301dff8 100644 --- a/.github/jobs/docker_utils.py +++ b/.github/jobs/docker_utils.py @@ -16,7 +16,7 @@ DOCKERHUB_METPLUS_DATA_DEV = 'dtcenter/metplus-data-dev' # extension to add to conda environments -VERSION_EXT = '.v5' +VERSION_EXT = '.v5.1' def get_data_repo(branch_name): diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index f1cb2046fb..248f59674e 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -64,7 +64,7 @@ def handle_automation_env(host_name, reqs, work_dir): setup_env.append(_add_to_bashrc('# BELOW WAS ADDED BY TEST SCRIPT')) # add conda bin to beginning of PATH - python_dir = os.path.join('/usr', 'local', 'envs', + python_dir = os.path.join('/usr', 'local', 'conda', 'envs', conda_env_w_ext, 'bin') python_path = os.path.join(python_dir, 'python3') setup_env.append(_add_to_bashrc(f'export PATH={python_dir}:$PATH')) @@ -106,12 +106,6 @@ def handle_automation_env(host_name, reqs, work_dir): 'cd -', )) - # if gempak is in requirements list, add JRE bin to path for java - if 'gempak' in str(reqs).lower(): - setup_env.append(_add_to_bashrc( - 'export PATH=$PATH:/usr/lib/jvm/jre/bin' - )) - # if metplus is in requirements list, # add top of METplus repo to PYTHONPATH so metplus can be imported if 'metplus' in str(reqs).lower(): @@ -123,9 +117,9 @@ def handle_automation_env(host_name, reqs, work_dir): if conda_env not in NOT_PYTHON_ENVS: setup_env.extend(( f'echo Using environment: dtcenter/metplus-envs:{conda_env_w_ext}', - f'echo cat /usr/local/envs/{conda_env_w_ext}/environments.yml', + f'echo cat /usr/local/conda/envs/{conda_env_w_ext}/environments.yml', f'echo ----------------------------------------', - f'cat /usr/local/envs/{conda_env_w_ext}/environments.yml', + f'cat /usr/local/conda/envs/{conda_env_w_ext}/environments.yml', 'echo ----------------------------------------', )) @@ -133,7 +127,7 @@ def handle_automation_env(host_name, reqs, work_dir): def _add_to_bashrc(command): - return f"echo '{command};' >> /etc/bashrc" + return f"echo '{command};' >> /root/.bashrc" def main(categories, subset_list, work_dir=None, @@ -178,6 +172,7 @@ def main(categories, subset_list, work_dir=None, config_args.append(config_arg) + #output_base = os.path.join(output_top_dir, group_name, use_case.name) output_base = os.path.join(output_top_dir, use_case.name) use_case_cmd = (f"run_metplus.py" f" {' '.join(config_args)}" diff --git a/.github/jobs/setup_and_run_diff.py b/.github/jobs/setup_and_run_diff.py index b1318bcd5f..35c0c27ca2 100755 --- a/.github/jobs/setup_and_run_diff.py +++ b/.github/jobs/setup_and_run_diff.py @@ -49,7 +49,7 @@ MOUNT_ARGS = ' '.join(VOLUME_MOUNTS) # command to run inside Docker -diff_command = (f'/usr/local/envs/diff{VERSION_EXT}/bin/python3 ' +diff_command = (f'/usr/local/conda/envs/diff{VERSION_EXT}/bin/python3 ' f'{GITHUB_WORKSPACE}/{CI_JOBS_DIR}/run_diff_docker.py') # start detached interactive diff env container diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 4330a6a653..8ed341ec4a 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -123,7 +123,7 @@ def main(): # print bashrc file to see what was added by setup commands # then force remove container to stop and remove it if not run_commands([ - _format_docker_exec_command('cat /etc/bashrc'), + _format_docker_exec_command('cat /root/.bashrc'), f'docker rm -f {RUN_TAG}', ]): isOK = False diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 944c81335f..e28120d768 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -85,7 +85,7 @@ jobs: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} EXTERNAL_TRIGGER: ${{ needs.job_control.outputs.external_trigger }} - #MET_FORCE_TAG: 10.1.0 + #MET_FORCE_TAG: 11.0.0 update_data_volumes: name: Docker Setup - Update Data Volumes diff --git a/internal/scripts/docker/Dockerfile b/internal/scripts/docker/Dockerfile index 20788861a8..4f2414e622 100644 --- a/internal/scripts/docker/Dockerfile +++ b/internal/scripts/docker/Dockerfile @@ -47,7 +47,7 @@ ONBUILD WORKDIR /metplus FROM build_${OBTAIN_SOURCE_CODE} MAINTAINER George McCabe -RUN echo export PATH=$PATH:`pwd`/METplus/ush >> /etc/bashrc \ +RUN echo export PATH=$PATH:`pwd`/METplus/ush >> /root/.bashrc \ && echo setenv PATH $PATH:`pwd`/METplus/ush >> /etc/csh.cshrc # if source code was retrieved, set default config variables and install package diff --git a/internal/scripts/docker_env/Dockerfile.gfdl-tracker b/internal/scripts/docker_env/Dockerfile.gfdl-tracker index 1abe9e6fdc..22be890081 100644 --- a/internal/scripts/docker_env/Dockerfile.gfdl-tracker +++ b/internal/scripts/docker_env/Dockerfile.gfdl-tracker @@ -84,4 +84,8 @@ RUN cd / \ && unset FC && unset CC \ && ./configure \ && cp configure.trk-docker-gnu configure.trk \ + && echo "\nLDFLAGS=-L/usr/local/lib -Wl,-rpath,/usr/local/lib -L/usr/lib/x86_64-linux-gnu -Wl,-rpath,/usr/lib/x86_64-linux-gnu" >> configure.trk \ && ./compile > tracker.log 2>&1 + +# check that all exes were created properly +RUN [ $(ls /standalone_gfdl-vortextracker_v3.9a_gcc/trk_exec -1 | wc -l) -ge 5 ] diff --git a/internal/tests/use_cases/metplus_use_case_suite.py b/internal/tests/use_cases/metplus_use_case_suite.py index b26accc597..d4be419de7 100644 --- a/internal/tests/use_cases/metplus_use_case_suite.py +++ b/internal/tests/use_cases/metplus_use_case_suite.py @@ -395,10 +395,12 @@ def parse_all_use_cases_file(): set_output = f'config.USE_CASE_NAME={name}' config_args.append(set_output) - use_case_dict = {'index': index, - 'name': name, - 'config_args': config_args, - 'requirements': requirements} + use_case_dict = { + 'index': index, + 'name': name, + 'config_args': config_args, + 'requirements': requirements, + } all_cases[category].append(use_case_dict) num_cases += 1 @@ -418,10 +420,14 @@ def parse_all_use_cases_file(): all_use_cases.add_use_case_groups('climate') - all_use_cases.add_use_case_groups('cryosphere') + all_use_cases.add_use_case_groups('air_quality_and_comp') + + all_use_cases.add_use_case_groups('climate') all_use_cases.add_use_case_groups('data_assimilation') + all_use_cases.add_use_case_groups('marine_and_cryosphere') + # slice(5) == medium range 0-4 all_use_cases.add_use_case_groups('medium_range', case_slice=range(5)) @@ -434,9 +440,19 @@ def parse_all_use_cases_file(): all_use_cases.add_use_case_groups('precipitation') + all_use_cases.add_use_case_groups('s2s') + + all_use_cases.add_use_case_groups('s2s_mid_lat') + + all_use_cases.add_use_case_groups('s2s_mjo') + all_use_cases.add_use_case_groups('short_range') - all_use_cases.add_use_case_groups('s2s&space_weather&tc_and_extra_tc') + all_use_cases.add_use_case_groups('space_weather') + + all_use_cases.add_use_case_groups('tc_and_extra_tc') + + all_use_cases.add_use_case_groups('unstructured_grids') all_use_cases.print() diff --git a/manage_externals/checkout_externals b/manage_externals/checkout_externals index a0698baef0..48bce24010 100755 --- a/manage_externals/checkout_externals +++ b/manage_externals/checkout_externals @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Main driver wrapper around the manic/checkout utility.