feature #1566 Use Python 3.8.6 for automated tests (#1710)

* changes needed to create v5.0 version of metplus_base docker/conda environment

* update metplotpy conda env to use python 3.8 and new package list, then test to see if use cases that use that env still work using the .v5 version

* turn on use cases that use metplotpy_env to test new conda env

* add .v5 to metplotpy conda env to test that environment

* removed cartopy from requirements because cartopy may no longer be needed

* changed conda environment used for use case

* update conda before building environments

* updated instructions and scripts to build python environments using Python 3.8.6 for METplus v5

* updated conda env to include .v5

* updated automated test logic to use new .v5 Docker images that use Conda environments built from Python 3.8.6

* rebuilt gempak and gfdl-tracker docker images to use .v5 extension to be consistent in the automation logic and avoid special behavior for those environments

* added note to update OS packages to prevent wget certificate errors

* renamed variables to be more clear which path they are actually storing

* Updated logic to no longer handle gempak and gfdl-tracker envs differently by excluding the version number. Cleaned up logic to add version number extension to conda environment names as needed

* updated scripts to create Docker image containing Conda environment for METdataio (renamed from METdatadb) to use in automated tests for v5

* turn off all use cases

* turn on use cases that previously failed

* turn off use cases tests that passed

* turn off all use cases

* per #1566, added explicit version numbers for each Python package that is installed in the Docker Conda environments used for the automated tests to ensure that the same environment will be generated even if the default version of the package differs from the env used to create them

.github/actions/run_tests/Dockerfile.run

@@ -1,4 +1,4 @@
-ARG METPLUS_ENV_TAG=metplus_base
+ARG METPLUS_ENV_TAG=metplus_base.v5
 ARG METPLUS_IMG_TAG=develop
 
 FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env

.github/actions/run_tests/Dockerfile.run_cartopy

@@ -1,4 +1,4 @@
-ARG METPLUS_ENV_TAG=cycloneplotter
+ARG METPLUS_ENV_TAG=cycloneplotter.v5
 ARG METPLUS_IMG_TAG=develop
 
 FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env

.github/jobs/docker_utils.py

@@ -12,6 +12,9 @@
 # repository used for storing input data for development branches
 DOCKERHUB_METPLUS_DATA_DEV = 'dtcenter/metplus-data-dev'
 
+# extension to add to conda environments
+VERSION_EXT = '.v5'
+
 def get_data_repo(branch_name):
     """! Branch names that start with main_v or contain only
        digits and dots with out without a prefix 'v' will return

.github/jobs/get_requirements.py

@@ -8,11 +8,11 @@
 
 import get_use_case_commands
 
-# add internal_tests/use_cases directory to path so the test suite can be found
-USE_CASES_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
-                                             os.pardir,
-                                             os.pardir))
-sys.path.insert(0, USE_CASES_DIR)
+# add METplus directory to path so the test suite can be found
+METPLUS_TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                               os.pardir,
+                                               os.pardir))
+sys.path.insert(0, METPLUS_TOP_DIR)
 
 from internal_tests.use_cases.metplus_use_case_suite import METplusUseCaseSuite
 

.github/jobs/get_use_case_commands.py

@@ -8,17 +8,22 @@
 import os
 
 # add METplus directory to sys path so the test suite can be found
-USE_CASES_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
-                                             os.pardir,
-                                             os.pardir))
-sys.path.insert(0, USE_CASES_DIR)
+METPLUS_TOP_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                               os.pardir,
+                                               os.pardir))
+sys.path.insert(0, METPLUS_TOP_DIR)
 
 from internal_tests.use_cases.metplus_use_case_suite import METplusUseCaseSuite
 from metplus.util.met_util import expand_int_string_to_list
+from docker_utils import VERSION_EXT
 
-METPLUS_BASE_ENV = 'metplus_base'
+
+# path to METplus install location in Docker
 METPLUS_DOCKER_LOC = '/metplus/METplus'
 
+# name of conda environment used for cases that don't need special env
+METPLUS_BASE_ENV = 'metplus_base'
+
 # keywords in requirements list that trigger obtaining METcalcpy and METplotpy
 PLOTCALC_KEYWORDS = [
     'metplotpy',
@@ -27,6 +32,12 @@
     'weatherregime',
 ]
 
+# Docker envs that do not use Python so they do not need print conda list
+NOT_PYTHON_ENVS = [
+    'gfdl-tracker',
+    'gempak',
+]
+
 def handle_automation_env(host_name, reqs, work_dir):
     # if no env is specified, use metplus base environment
     conda_env = METPLUS_BASE_ENV
@@ -44,12 +55,15 @@ def handle_automation_env(host_name, reqs, work_dir):
             return '', 'user_env_vars.MET_PYTHON_EXE=python3'
         return '', ''
 
+    # add version extension to conda environment name
+    conda_env_w_ext = f'{conda_env}{VERSION_EXT}'
+
     # start building commands to run before run_metplus.py in Docker
     setup_env = 'source /etc/bashrc;'
 
     # add conda bin to beginning of PATH
     python_dir = os.path.join('/usr', 'local', 'envs',
-                              conda_env, 'bin')
+                              conda_env_w_ext, 'bin')
     python_path = os.path.join(python_dir, 'python3')
     setup_env += f' export PATH={python_dir}:$PATH;'
 
@@ -92,12 +106,12 @@ def handle_automation_env(host_name, reqs, work_dir):
         setup_env += f'export PYTHONPATH={METPLUS_DOCKER_LOC}:$PYTHONPATH;'
 
     # list packages in python environment that will be used
-    if conda_env != 'gempak':
+    if conda_env not in NOT_PYTHON_ENVS:
         setup_env += (
-            f'echo Using environment: dtcenter/metplus-envs:{conda_env};'
-            f'echo cat /usr/local/envs/{conda_env}/environments.yml;'
+            f'echo Using environment: dtcenter/metplus-envs:{conda_env_w_ext};'
+            f'echo cat /usr/local/envs/{conda_env_w_ext}/environments.yml;'
             f'echo ----------------------------------------;'
-            f'cat /usr/local/envs/{conda_env}/environments.yml;'
+            f'cat /usr/local/envs/{conda_env_w_ext}/environments.yml;'
             'echo ----------------------------------------;'
         )
 
@@ -108,7 +122,7 @@ def main(categories, subset_list, work_dir=None,
     all_commands = []
 
     if work_dir is None:
-        work_dir = USE_CASES_DIR
+        work_dir = METPLUS_TOP_DIR
 
     test_suite = METplusUseCaseSuite()
     test_suite.add_use_case_groups(categories, subset_list)

.github/jobs/setup_and_run_diff.py

@@ -5,6 +5,8 @@
 import subprocess
 import shlex
 
+from docker_utils import VERSION_EXT
+
 ci_dir = os.path.join(os.environ.get('GITHUB_WORKSPACE'), '.github')
 sys.path.insert(0, ci_dir)
 
@@ -46,12 +48,12 @@
 mount_args = ' '.join(volume_mounts)
 
 # command to run inside Docker
-cmd = ('/usr/local/envs/diff/bin/python3 '
+cmd = (f'/usr/local/envs/diff{VERSION_EXT}/bin/python3 '
        f'{GITHUB_WORKSPACE}/{CI_JOBS_DIR}/run_diff_docker.py')
 
 # run inside diff env: mount METplus code and output dir, volumes from output volumes
 docker_cmd = (f'docker run -e GITHUB_WORKSPACE {VOLUMES_FROM} '
-              f'{mount_args} dtcenter/metplus-envs:diff '
+              f'{mount_args} dtcenter/metplus-envs:diff{VERSION_EXT} '
               f'bash -c "{cmd}"')
 print(f'RUNNING: {docker_cmd}')
 try:

.github/jobs/setup_and_run_use_cases.py

@@ -16,7 +16,7 @@
 
 import get_use_case_commands
 import get_data_volumes
-from docker_utils import get_branch_name
+from docker_utils import get_branch_name, VERSION_EXT
 
 runner_workspace = os.environ.get('RUNNER_WORKSPACE')
 github_workspace = os.environ.get('GITHUB_WORKSPACE')
@@ -70,6 +70,8 @@ def main():
         else:
             env_tag = 'metplus_base'
 
+        env_tag = f'{env_tag}{VERSION_EXT}'
+
         # get Dockerfile to use
         dockerfile_name = 'Dockerfile.run'
         if 'gempak' in str(requirements).lower():

internal_tests/use_cases/all_use_cases.txt

@@ -139,6 +139,7 @@ Category: s2s_mid_lat
 2::UserScript_obsERA_obsOnly_WeatherRegime:: model_applications/s2s_mid_lat/UserScript_obsERA_obsOnly_WeatherRegime.conf:: weatherregime_env,cartopy,metplus
 3:: UserScript_fcstGFS_obsERA_WeatherRegime:: model_applications/s2s_mid_lat/UserScript_fcstGFS_obsERA_WeatherRegime.conf:: weatherregime_env,cartopy,metplus
 
+
 Category: s2s_mjo
 0:: UserScript_obsERA_obsOnly_PhaseDiagram:: model_applications/s2s_mjo/UserScript_obsERA_obsOnly_PhaseDiagram.conf:: spacetime_env
 1:: UserScript_fcstGFS_obsERA_OMI:: model_applications/s2s_mjo/UserScript_fcstGFS_obsERA_OMI.conf:: spacetime_env, metdataio

scripts/docker/docker_env/Dockerfile

@@ -1,13 +1,14 @@
 # Dockerfile to create conda environments used for use case tests
 
-ARG BASE_ENV=metplus_base
+ARG BASE_ENV=metplus_base.v5
 FROM dtcenter/metplus-envs:${BASE_ENV}
 
 ARG ENV_NAME
 WORKDIR /scripts
 COPY scripts/${ENV_NAME}_env.sh .
 
-ARG BASE_ENV=metplus_base
-RUN ./${ENV_NAME}_env.sh ${BASE_ENV}
+ARG BASE_ENV=metplus_base.v5
+RUN conda update -y -n base -c defaults conda \
+  && ./${ENV_NAME}_env.sh ${BASE_ENV}
 
-RUN conda list --name ${ENV_NAME} > /usr/local/envs/${ENV_NAME}/environments.yml
+RUN conda list --name ${ENV_NAME}.v5 > /usr/local/envs/${ENV_NAME}.v5/environments.yml

scripts/docker/docker_env/Dockerfile.metplus_base

@@ -9,4 +9,4 @@ COPY scripts/metplus_base_env.sh .
 RUN conda update -y -n base -c defaults conda \
  && ./metplus_base_env.sh
 
-RUN conda list --name metplus_base > /usr/local/envs/metplus_base/environments.yml
+RUN conda list --name metplus_base.v5 > /usr/local/envs/metplus_base.v5/environments.yml

scripts/docker/docker_env/Dockerfile.py_embed_base

@@ -10,4 +10,4 @@ COPY scripts/py_embed_base_env.sh .
 RUN conda update -y -n base -c defaults conda \
  && ./py_embed_base_env.sh
 
-RUN conda list --name py_embed_base > /usr/local/envs/py_embed_base/environments.yml
+RUN conda list --name py_embed_base.v5 > /usr/local/envs/py_embed_base.v5/environments.yml