From c3b65b1620bd125b8a062cefcc3d2fc4aea656aa Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Thu, 2 May 2024 19:25:46 +0000 Subject: [PATCH 1/6] Updates to remove requirements that suite files start with "suite_" --- scripts/ccpp_prebuild.py | 20 +++++++++++++++++--- scripts/mkstatic.py | 15 ++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/scripts/ccpp_prebuild.py b/scripts/ccpp_prebuild.py index b67de540..6d737a86 100755 --- a/scripts/ccpp_prebuild.py +++ b/scripts/ccpp_prebuild.py @@ -55,7 +55,7 @@ def parse_arguments(): verbose = args.verbose debug = args.debug if args.suites: - sdfs = ['suite_{0}.xml'.format(x) for x in args.suites.split(',')] + sdfs = ['{0}.xml'.format(x) for x in args.suites.split(',')] else: sdfs = None builddir = args.builddir @@ -181,8 +181,22 @@ def parse_suites(suites_dir, sdfs): logging.info('Parsing suite definition files ...') suites = [] for sdf in sdfs: - logging.info('Parsing suite definition file {0} ...'.format(os.path.join(suites_dir, sdf))) - suite = Suite(sdf_name=os.path.join(suites_dir, sdf)) + sdf_file=os.path.join(suites_dir, sdf) + if not os.path.exists(sdf_file): + sdf_file_legacy=os.path.join(suites_dir, f"suite_{sdf}") + if not os.path.exists(sdf_file_legacy): + logging.critical(f"Suite definition file {sdf_file} not found.") + success = False + return success + else: + logging.info("Parsing suite definition file using legacy naming convention") + logging.info(f"Filename {os.path.basename(sdf_file_legacy)}") + logging.info(f"Suite name {sdf}") + sdf_file=sdf_file_legacy + else: + logging.info(f'Parsing suite definition file {sdf_file} ...') + + suite = Suite(sdf_name=sdf_file) success = suite.parse() if not success: logging.error('Parsing suite definition file {0} failed.'.format(sdf)) diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index 4f00b9b9..889b2c1d 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -691,11 +691,16 @@ def parse(self, make_call_tree=False): suite_xml = tree.getroot() self._name = suite_xml.get('name') # Validate name of suite in XML tag against filename; could be moved to common.py - if not (os.path.basename(self._sdf_name) == 'suite_{}.xml'.format(self._name)): - logging.critical("Invalid suite name {0} in suite definition file {1}.".format( - self._name, self._sdf_name)) - success = False - return success + if not (os.path.basename(self._sdf_name) == '{}.xml'.format(self._name)): + if (os.path.basename(self._sdf_name) == 'suite_{}.xml'.format(self._name)): + logging.debug("Parsing suite using legacy naming convention") + logging.debug(f"Filename {os.path.basename(self._sdf_name)}") + logging.debug(f"Suite name {format(self._name)}") + else: + logging.critical("Invalid suite name {0} in suite definition file {1}.".format( + self._name, self._sdf_name)) + success = False + return success # Check if suite name is too long if len(self._name) > SUITE_NAME_MAX_CHARS: From adf3cc0fe95c9b5d15d08a9fe3fdae9545c3bfcd Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Mon, 20 May 2024 10:16:00 -0600 Subject: [PATCH 2/6] Update SUITE_DEFINITION_FILENAME_PATTERN regex --- scripts/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/common.py b/scripts/common.py index c29ccda2..f5ba7f1d 100755 --- a/scripts/common.py +++ b/scripts/common.py @@ -73,7 +73,7 @@ CCPP_STATIC_SUBROUTINE_NAME = 'ccpp_physics_{stage}' # Filename pattern for suite definition files -SUITE_DEFINITION_FILENAME_PATTERN = re.compile('^suite_(.*)\.xml$') +SUITE_DEFINITION_FILENAME_PATTERN = re.compile('^(.*)\.xml$') # Maximum number of concurrent CCPP instances per MPI task CCPP_NUM_INSTANCES = 200 From 552b1dc969323d1ca178976fb85e28e86e67fd84 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Mon, 20 May 2024 13:29:42 -0600 Subject: [PATCH 3/6] Add logic to read an "alias" file (alias.json) that maps old suite names to new. Also fix logic that handles missing suite files. --- scripts/ccpp_prebuild.py | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/scripts/ccpp_prebuild.py b/scripts/ccpp_prebuild.py index 6d737a86..73a8413b 100755 --- a/scripts/ccpp_prebuild.py +++ b/scripts/ccpp_prebuild.py @@ -7,6 +7,7 @@ import filecmp import importlib import itertools +import json import logging import os import re @@ -178,24 +179,45 @@ def get_all_suites(suites_dir): def parse_suites(suites_dir, sdfs): """Parse suite definition files for prebuild""" + alias_file=os.path.join(suites_dir,"alias.json") + aliases = {} + if os.path.exists(alias_file): + logging.debug(f'Reading alias file {alias_file}') + with open(alias_file) as json_file: + aliases = json.load(json_file) + else: + logging.debug(f'No alias file {alias_file} found') logging.info('Parsing suite definition files ...') suites = [] for sdf in sdfs: sdf_file=os.path.join(suites_dir, sdf) if not os.path.exists(sdf_file): + # If suite file not found, check alias file if available + # If alias found, check old filename convention (suite_[suitename].xml) + sdf_file_alias='' + if aliases: + print(aliases) + if aliases.get(os.path.basename(sdf_file)): + logging.info(f"Found alias for SDF {sdf} ==> {aliases[sdf]}") + sdf_file_alias=os.path.join(suites_dir,aliases[sdf]) + else: + logging.debug(f"No alias found in alias file {alias_file}") + logging.debug(f"for SDF {sdf}") + sdf_file_legacy=os.path.join(suites_dir, f"suite_{sdf}") - if not os.path.exists(sdf_file_legacy): - logging.critical(f"Suite definition file {sdf_file} not found.") - success = False - return success - else: + if os.path.exists(sdf_file_alias): + sdf_file=sdf_file_alias + elif os.path.exists(sdf_file_legacy): logging.info("Parsing suite definition file using legacy naming convention") logging.info(f"Filename {os.path.basename(sdf_file_legacy)}") logging.info(f"Suite name {sdf}") sdf_file=sdf_file_legacy - else: - logging.info(f'Parsing suite definition file {sdf_file} ...') + else: + logging.critical(f"Suite definition file {sdf_file} not found.") + success = False + return (success, suites) + logging.info(f'Parsing suite definition file {sdf_file} ...') suite = Suite(sdf_name=sdf_file) success = suite.parse() if not success: From f8fd26e734f5f34d2ee6e259a5ba791c4c057c60 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Tue, 21 May 2024 14:19:36 -0600 Subject: [PATCH 4/6] Backtracking out "suite aliasing", we are just going to force people to rename --- scripts/ccpp_prebuild.py | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/scripts/ccpp_prebuild.py b/scripts/ccpp_prebuild.py index 73a8413b..0887fa9f 100755 --- a/scripts/ccpp_prebuild.py +++ b/scripts/ccpp_prebuild.py @@ -7,7 +7,6 @@ import filecmp import importlib import itertools -import json import logging import os import re @@ -179,35 +178,14 @@ def get_all_suites(suites_dir): def parse_suites(suites_dir, sdfs): """Parse suite definition files for prebuild""" - alias_file=os.path.join(suites_dir,"alias.json") - aliases = {} - if os.path.exists(alias_file): - logging.debug(f'Reading alias file {alias_file}') - with open(alias_file) as json_file: - aliases = json.load(json_file) - else: - logging.debug(f'No alias file {alias_file} found') logging.info('Parsing suite definition files ...') suites = [] for sdf in sdfs: sdf_file=os.path.join(suites_dir, sdf) if not os.path.exists(sdf_file): - # If suite file not found, check alias file if available - # If alias found, check old filename convention (suite_[suitename].xml) - sdf_file_alias='' - if aliases: - print(aliases) - if aliases.get(os.path.basename(sdf_file)): - logging.info(f"Found alias for SDF {sdf} ==> {aliases[sdf]}") - sdf_file_alias=os.path.join(suites_dir,aliases[sdf]) - else: - logging.debug(f"No alias found in alias file {alias_file}") - logging.debug(f"for SDF {sdf}") - + # If suite file not found, check old filename convention (suite_[suitename].xml) sdf_file_legacy=os.path.join(suites_dir, f"suite_{sdf}") - if os.path.exists(sdf_file_alias): - sdf_file=sdf_file_alias - elif os.path.exists(sdf_file_legacy): + if os.path.exists(sdf_file_legacy): logging.info("Parsing suite definition file using legacy naming convention") logging.info(f"Filename {os.path.basename(sdf_file_legacy)}") logging.info(f"Suite name {sdf}") From 3a9c061d0b8ed5702ae9b236971536dd7ff08632 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 28 May 2024 10:19:01 -0600 Subject: [PATCH 5/6] Move suite_blocked_data_suite.xml -> blocked_data_suite.xml, so the "blocked data" test now pulls double-duty to confirm that new suite name convention works --- .../{suite_blocked_data_suite.xml => blocked_data_suite.xml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test_prebuild/test_blocked_data/{suite_blocked_data_suite.xml => blocked_data_suite.xml} (100%) diff --git a/test_prebuild/test_blocked_data/suite_blocked_data_suite.xml b/test_prebuild/test_blocked_data/blocked_data_suite.xml similarity index 100% rename from test_prebuild/test_blocked_data/suite_blocked_data_suite.xml rename to test_prebuild/test_blocked_data/blocked_data_suite.xml From 8821d641482c9ddb9730eab11c664a9ede49044a Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Mon, 3 Jun 2024 15:45:46 +0000 Subject: [PATCH 6/6] Upgrade message about legacy naming to warning-level --- scripts/ccpp_prebuild.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/ccpp_prebuild.py b/scripts/ccpp_prebuild.py index 0887fa9f..3cb4ee19 100755 --- a/scripts/ccpp_prebuild.py +++ b/scripts/ccpp_prebuild.py @@ -186,9 +186,9 @@ def parse_suites(suites_dir, sdfs): # If suite file not found, check old filename convention (suite_[suitename].xml) sdf_file_legacy=os.path.join(suites_dir, f"suite_{sdf}") if os.path.exists(sdf_file_legacy): - logging.info("Parsing suite definition file using legacy naming convention") - logging.info(f"Filename {os.path.basename(sdf_file_legacy)}") - logging.info(f"Suite name {sdf}") + logging.warning("Parsing suite definition file using legacy naming convention") + logging.warning(f"Filename {os.path.basename(sdf_file_legacy)}") + logging.warning(f"Suite name {sdf}") sdf_file=sdf_file_legacy else: logging.critical(f"Suite definition file {sdf_file} not found.")