Feature #2022 Use Debian 10 / Python 3.10.4 in automated tests (#2050)

.github/actions/run_tests/Dockerfile.run

@@ -6,5 +6,6 @@ FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env
 ARG METPLUS_IMG_TAG=develop
 FROM dtcenter/metplus-dev:${METPLUS_IMG_TAG}
 
-COPY --from=env /usr/local/envs /usr/local/envs/
-COPY --from=env /usr/local/bin/conda /usr/local/bin/conda
+RUN mkdir -p /usr/local/conda/envs && mkdir -p /usr/local/conda/bin
+COPY --from=env /usr/local/conda/envs /usr/local/conda/envs/
+COPY --from=env /usr/local/conda/bin/conda /usr/local/conda/bin/conda

.github/actions/run_tests/Dockerfile.run_cartopy

@@ -6,7 +6,8 @@ FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env
 ARG METPLUS_IMG_TAG=develop
 FROM dtcenter/metplus-dev:${METPLUS_IMG_TAG}
 
-COPY --from=env /usr/local/envs /usr/local/envs/
-COPY --from=env /usr/local/bin/conda /usr/local/bin/conda
+RUN mkdir -p /usr/local/conda/envs && mkdir -p /usr/local/conda/bin
+COPY --from=env /usr/local/conda/envs /usr/local/conda/envs/
+COPY --from=env /usr/local/conda/bin/conda /usr/local/conda/bin/conda
 
 COPY --from=env /root/.local/share/cartopy /root/.local/share/cartopy/

.github/actions/run_tests/Dockerfile.run_gempak

@@ -6,6 +6,8 @@ FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env
 ARG METPLUS_IMG_TAG=develop
 FROM dtcenter/metplus-dev:${METPLUS_IMG_TAG}
 
-COPY --from=env /usr/lib/jvm/jre /usr/lib/jvm/jre/
-COPY --from=env /usr/share/javazi-1.8/tzdb.dat /usr/share/javazi-1.8/
+COPY --from=env /usr/lib/jvm/adoptopenjdk-8-hotspot-amd64 /usr/lib/jvm/adoptopenjdk-8-hotspot-amd64/
 COPY --from=env /data/input/GempakToCF.jar /data/input/GempakToCF.jar
+
+# add java executable to path
+RUN echo "export PATH=/usr/lib/jvm/adoptopenjdk-8-hotspot-amd64/bin:\$PATH;" >> /root/.bashrc

.github/actions/run_tests/Dockerfile.run_gfdl

@@ -6,6 +6,7 @@ FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env
 ARG METPLUS_IMG_TAG=develop
 FROM dtcenter/metplus-dev:${METPLUS_IMG_TAG}
 
+COPY --from=env /usr/local/lib/libhdf5* /usr/local/lib/
 COPY --from=env /usr/local/lib/libnetcdf.so.19* /usr/local/lib/
 COPY --from=env /usr/local/lib/libnetcdff* /usr/local/lib/
 COPY --from=env /standalone_gfdl-vortextracker_v3.9a_gcc/trk_exec/* /usr/local/bin/

.github/actions/run_tests/entrypoint.sh

@@ -34,7 +34,7 @@ fi
 
 # running unit tests (pytests)
 if [[ "$INPUT_CATEGORIES" == pytests* ]]; then
-  export METPLUS_ENV_TAG="pytest"
+  export METPLUS_ENV_TAG="pytest.v5.1"
   export METPLUS_IMG_TAG=${branch_name}
   echo METPLUS_ENV_TAG=${METPLUS_ENV_TAG}
   echo METPLUS_IMG_TAG=${METPLUS_IMG_TAG}
@@ -61,7 +61,7 @@ if [[ "$INPUT_CATEGORIES" == pytests* ]]; then
   for x in `cat $PYTESTS_GROUPS_FILEPATH`; do
     marker="${x//_or_/ or }"
     marker="${marker//not_/not }"
-    command+="/usr/local/envs/pytest/bin/pytest -vv --cov=../../../metplus -m \"$marker\""
+    command+="/usr/local/conda/envs/${METPLUS_ENV_TAG}/bin/pytest -vv --cov=../../../metplus -m \"$marker\""
     command+=";if [ \$? != 0 ]; then status=1; fi;"
   done
   command+="if [ \$status != 0 ]; then echo ERROR: Some pytests failed. Search for FAILED to review; false; fi"

.github/jobs/copy_error_logs.py

@@ -10,12 +10,11 @@
 
 def main(output_data_dir, error_logs_dir):
     """! Copy log output to error log directory if any use case failed """
-    for use_case_dir in os.listdir(output_data_dir):
-        log_dir = os.path.join(output_data_dir,
-                               use_case_dir,
-                               'logs')
-        if not os.path.isdir(log_dir):
+    for log_dir, _, log_files in os.walk(output_data_dir):
+        # skip non logs directories
+        if not log_dir.endswith('/logs'):
             continue
+        use_case_dir = log_dir.replace(f'{output_data_dir}/', '')[0:-5]
 
         # check if there are errors in the metplus.log file and
         # only copy directory if there are any errors
@@ -28,9 +27,7 @@ def main(output_data_dir, error_logs_dir):
         if not found_errors:
             continue
 
-        output_dir = os.path.join(error_logs_dir,
-                                  use_case_dir)
-        log_files = os.listdir(log_dir)
+        output_dir = os.path.join(error_logs_dir, use_case_dir)
         for log_file in log_files:
             log_path = os.path.join(log_dir, log_file)
             output_path = os.path.join(output_dir, log_file)

.github/jobs/docker_utils.py

@@ -16,7 +16,7 @@
 DOCKERHUB_METPLUS_DATA_DEV = 'dtcenter/metplus-data-dev'
 
 # extension to add to conda environments
-VERSION_EXT = '.v5'
+VERSION_EXT = '.v5.1'
 
 
 def get_data_repo(branch_name):

.github/jobs/get_use_case_commands.py

@@ -64,7 +64,7 @@ def handle_automation_env(host_name, reqs, work_dir):
     setup_env.append(_add_to_bashrc('# BELOW WAS ADDED BY TEST SCRIPT'))
 
     # add conda bin to beginning of PATH
-    python_dir = os.path.join('/usr', 'local', 'envs',
+    python_dir = os.path.join('/usr', 'local', 'conda', 'envs',
                               conda_env_w_ext, 'bin')
     python_path = os.path.join(python_dir, 'python3')
     setup_env.append(_add_to_bashrc(f'export PATH={python_dir}:$PATH'))
@@ -106,12 +106,6 @@ def handle_automation_env(host_name, reqs, work_dir):
             'cd -',
         ))
 
-    # if gempak is in requirements list, add JRE bin to path for java
-    if 'gempak' in str(reqs).lower():
-        setup_env.append(_add_to_bashrc(
-            'export PATH=$PATH:/usr/lib/jvm/jre/bin'
-        ))
-
     # if metplus is in requirements list,
     # add top of METplus repo to PYTHONPATH so metplus can be imported
     if 'metplus' in str(reqs).lower():
@@ -123,17 +117,17 @@ def handle_automation_env(host_name, reqs, work_dir):
     if conda_env not in NOT_PYTHON_ENVS:
         setup_env.extend((
             f'echo Using environment: dtcenter/metplus-envs:{conda_env_w_ext}',
-            f'echo cat /usr/local/envs/{conda_env_w_ext}/environments.yml',
+            f'echo cat /usr/local/conda/envs/{conda_env_w_ext}/environments.yml',
             f'echo ----------------------------------------',
-            f'cat /usr/local/envs/{conda_env_w_ext}/environments.yml',
+            f'cat /usr/local/conda/envs/{conda_env_w_ext}/environments.yml',
             'echo ----------------------------------------',
         ))
 
     return ';'.join(setup_env), py_embed_arg
 
 
 def _add_to_bashrc(command):
-    return f"echo '{command};' >> /etc/bashrc"
+    return f"echo '{command};' >> /root/.bashrc"
 
 
 def main(categories, subset_list, work_dir=None,
@@ -178,6 +172,7 @@ def main(categories, subset_list, work_dir=None,
 
                     config_args.append(config_arg)
 
+                #output_base = os.path.join(output_top_dir, group_name, use_case.name)
                 output_base = os.path.join(output_top_dir, use_case.name)
                 use_case_cmd = (f"run_metplus.py"
                                 f" {' '.join(config_args)}"

.github/jobs/setup_and_run_diff.py

@@ -49,7 +49,7 @@
 MOUNT_ARGS = ' '.join(VOLUME_MOUNTS)
 
 # command to run inside Docker
-diff_command = (f'/usr/local/envs/diff{VERSION_EXT}/bin/python3 '
+diff_command = (f'/usr/local/conda/envs/diff{VERSION_EXT}/bin/python3 '
                 f'{GITHUB_WORKSPACE}/{CI_JOBS_DIR}/run_diff_docker.py')
 
 # start detached interactive diff env container

.github/jobs/setup_and_run_use_cases.py

@@ -123,7 +123,7 @@ def main():
         # print bashrc file to see what was added by setup commands
         # then force remove container to stop and remove it
         if not run_commands([
-            _format_docker_exec_command('cat /etc/bashrc'),
+            _format_docker_exec_command('cat /root/.bashrc'),
             f'docker rm -f {RUN_TAG}',
         ]):
             isOK = False

.github/workflows/testing.yml

@@ -85,7 +85,7 @@ jobs:
           DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
           EXTERNAL_TRIGGER: ${{ needs.job_control.outputs.external_trigger }}
-          #MET_FORCE_TAG: 10.1.0
+          #MET_FORCE_TAG: 11.0.0
 
   update_data_volumes:
     name: Docker Setup - Update Data Volumes

internal/scripts/docker/Dockerfile

@@ -47,7 +47,7 @@ ONBUILD WORKDIR /metplus
 FROM build_${OBTAIN_SOURCE_CODE}
 MAINTAINER George McCabe <mccabe@ucar.edu>
 
-RUN echo export PATH=$PATH:`pwd`/METplus/ush >> /etc/bashrc \
+RUN echo export PATH=$PATH:`pwd`/METplus/ush >> /root/.bashrc \
  && echo setenv PATH $PATH:`pwd`/METplus/ush >> /etc/csh.cshrc
 
 # if source code was retrieved, set default config variables and install package

internal/scripts/docker_env/Dockerfile.gfdl-tracker

@@ -84,4 +84,8 @@ RUN cd / \
  && unset FC && unset CC \
  && ./configure \
  && cp configure.trk-docker-gnu configure.trk \
+ && echo "\nLDFLAGS=-L/usr/local/lib -Wl,-rpath,/usr/local/lib -L/usr/lib/x86_64-linux-gnu -Wl,-rpath,/usr/lib/x86_64-linux-gnu" >> configure.trk \
  && ./compile > tracker.log 2>&1
+
+# check that all exes were created properly
+RUN [ $(ls /standalone_gfdl-vortextracker_v3.9a_gcc/trk_exec -1 | wc -l) -ge 5 ]

internal/tests/use_cases/metplus_use_case_suite.py

@@ -395,10 +395,12 @@ def parse_all_use_cases_file():
             set_output = f'config.USE_CASE_NAME={name}'
             config_args.append(set_output)
 
-            use_case_dict = {'index': index,
-                             'name': name,
-                             'config_args': config_args,
-                             'requirements': requirements}
+            use_case_dict = {
+                'index': index,
+                'name': name,
+                'config_args': config_args,
+                'requirements': requirements,
+            }
             all_cases[category].append(use_case_dict)
             num_cases += 1
 
@@ -418,10 +420,14 @@ def parse_all_use_cases_file():
 
     all_use_cases.add_use_case_groups('climate')
 
-    all_use_cases.add_use_case_groups('cryosphere')
+    all_use_cases.add_use_case_groups('air_quality_and_comp')
+
+    all_use_cases.add_use_case_groups('climate')
 
     all_use_cases.add_use_case_groups('data_assimilation')
 
+    all_use_cases.add_use_case_groups('marine_and_cryosphere')
+
     # slice(5) == medium range 0-4
     all_use_cases.add_use_case_groups('medium_range',
                                       case_slice=range(5))
@@ -434,9 +440,19 @@ def parse_all_use_cases_file():
 
     all_use_cases.add_use_case_groups('precipitation')
 
+    all_use_cases.add_use_case_groups('s2s')
+
+    all_use_cases.add_use_case_groups('s2s_mid_lat')
+
+    all_use_cases.add_use_case_groups('s2s_mjo')
+
     all_use_cases.add_use_case_groups('short_range')
 
-    all_use_cases.add_use_case_groups('s2s&space_weather&tc_and_extra_tc')
+    all_use_cases.add_use_case_groups('space_weather')
+
+    all_use_cases.add_use_case_groups('tc_and_extra_tc')
+
+    all_use_cases.add_use_case_groups('unstructured_grids')
 
     all_use_cases.print()
 

manage_externals/checkout_externals

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """Main driver wrapper around the manic/checkout utility.