From 7682ec103687c516603e604b57bf7597eec6cd01 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 8 Nov 2023 09:50:38 -0700 Subject: [PATCH 01/23] Add capability to parse Fortran pointer variables to fortran_tools/parse_fortran.py --- scripts/fortran_tools/parse_fortran.py | 3 +++ 1 file changed, 3 insertions(+) mode change 100644 => 100755 scripts/fortran_tools/parse_fortran.py diff --git a/scripts/fortran_tools/parse_fortran.py b/scripts/fortran_tools/parse_fortran.py old mode 100644 new mode 100755 index 9aa4de7a..a3a05cc5 --- a/scripts/fortran_tools/parse_fortran.py +++ b/scripts/fortran_tools/parse_fortran.py @@ -800,6 +800,9 @@ def parse_fortran_var_decl(line, source, run_env): if 'allocatable' in varprops: prop_dict['allocatable'] = 'True' # end if + if 'pointer' in varprops: + prop_dict['pointer'] = 'True' + # end if if intent is not None: prop_dict['intent'] = intent # end if From a660e2fac9e2cc94546a31f13921fa9e781cb823 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 8 Nov 2023 09:51:24 -0700 Subject: [PATCH 02/23] Add --debug option to capgen (framework_env.py) --- scripts/framework_env.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) mode change 100644 => 100755 scripts/framework_env.py diff --git a/scripts/framework_env.py b/scripts/framework_env.py old mode 100644 new mode 100755 index 6456134e..c1890928 --- a/scripts/framework_env.py +++ b/scripts/framework_env.py @@ -24,7 +24,8 @@ def __init__(self, logger, ndict=None, verbose=0, clean=False, host_files=None, scheme_files=None, suites=None, preproc_directives=[], generate_docfiles=False, host_name='', kind_types=[], use_error_obj=False, force_overwrite=False, - output_root=os.getcwd(), ccpp_datafile="datatable.xml"): + output_root=os.getcwd(), ccpp_datafile="datatable.xml", + debug = False): """Initialize a new CCPPFrameworkEnv object from the input arguments. is a dict with the parsed command-line arguments (or a dictionary created with the necessary arguments). @@ -197,6 +198,13 @@ def __init__(self, logger, ndict=None, verbose=0, clean=False, self.__datatable_file = os.path.join(self.output_dir, self.datatable_file) # end if + # Enable or disable variable allocation checks + if ndict and ('debug' in ndict): + self.__debug = ndict['debug'] + del ndict['debug'] + else: + self.__debug = debug + # end if self.__logger = logger ## Check to see if anything is left in dictionary if ndict: @@ -304,6 +312,12 @@ def datatable_file(self): CCPPFrameworkEnv object.""" return self.__datatable_file + @property + def debug(self): + """Return the property for this + CCPPFrameworkEnv object.""" + return self.__debug + @property def logger(self): """Return the property for this CCPPFrameworkEnv object.""" @@ -379,7 +393,11 @@ def parse_command_line(args, description, logger=None): help="""Overwrite all CCPP-generated files, even if unmodified""") + parser.add_argument("--debug", action='store_true', default=False, + help="Add variable allocation checks to assist debugging") + parser.add_argument("--verbose", action='count', default=0, help="Log more activity, repeat for increased output") + pargs = parser.parse_args(args) return CCPPFrameworkEnv(logger, vars(pargs)) From a67b0bca4552af7c0f80b051f618cfc43848b938 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 8 Nov 2023 09:52:34 -0700 Subject: [PATCH 03/23] Add pointer attribute to Var class, add code to convert active attribute into Fortran conditional (metavar.py) --- scripts/metavar.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/scripts/metavar.py b/scripts/metavar.py index 8c600546..ed80cd70 100755 --- a/scripts/metavar.py +++ b/scripts/metavar.py @@ -85,6 +85,12 @@ 'horizontal_loop_begin', 'horizontal_loop_end', 'vertical_layer_index', 'vertical_interface_index'] +# DH* Is there a better place for these definitions? +FORTRAN_CONDITIONAL_REGEX_WORDS = [' ', '(', ')', '==', '/=', '<=', '>=', '<', '>', '.eqv.', '.neqv.', + '.true.', '.false.', '.lt.', '.le.', '.eq.', '.ge.', '.gt.', '.ne.', + '.not.', '.and.', '.or.', '.xor.'] +FORTRAN_CONDITIONAL_REGEX = re.compile(r"[\w']+|" + "|".join([word.replace('(','\(').replace(')', '\)') for word in FORTRAN_CONDITIONAL_REGEX_WORDS])) + ############################################################################### # Used for creating template variables _MVAR_DUMMY_RUN_ENV = CCPPFrameworkEnv(None, ndict={'host_files':'', @@ -187,6 +193,8 @@ class Var: optional_in=True, default_in=False), VariableProperty('allocatable', bool, optional_in=True, default_in=False), + VariableProperty('pointer', bool, + optional_in=True, default_in=False), VariableProperty('diagnostic_name', str, optional_in=True, default_in='', check_fn_in=check_diagnostic_id), @@ -294,6 +302,7 @@ def __init__(self, prop_dict, source, run_env, context=None, if 'units' not in prop_dict: prop_dict['units'] = "" # end if + # DH* Why is the DDT type copied into the kind attribute? prop_dict['kind'] = prop_dict['ddt_type'] del prop_dict['ddt_type'] self.__intrinsic = False @@ -937,6 +946,39 @@ def has_vertical_dimension(self, dims=None): # end if return find_vertical_dimension(vdims)[0] + def conditional(self, vdicts): + """Convert conditional expression from active attribute + (i.e. in standard name format) to local names based on vdict. + Return conditional and a list of variables needed to evaluate + the conditional.""" + + active = self.get_prop_value('active') + conditional = '' + vars_needed = [] + + # Find all words in the conditional, for each of them look + # for a matching standard name in the list of known variables + items = FORTRAN_CONDITIONAL_REGEX.findall(active) + for item in items: + item = item.lower() + if item in FORTRAN_CONDITIONAL_REGEX_WORDS: + conditional += item + else: + # Keep integers + try: + int(item) + conditional += item + except ValueError: + for vdict in vdicts: + dvar = vdict.find_variable(standard_name=item, any_scope=True) # or any_scope=False ? + if dvar: + break + if not dvar: + raise Exception(f"Cannot find variable '{item}' for generating conditional for '{active}") + conditional += dvar.get_prop_value('local_name') + vars_needed.append(dvar) + return (conditional, vars_needed) + def write_def(self, outfile, indent, wdict, allocatable=False, dummy=False, add_intent=None, extra_space=0): """Write the definition line for the variable to . From e53277416a2303ab556db0e5cc26dbd76ac66bd1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 8 Nov 2023 09:54:11 -0700 Subject: [PATCH 04/23] Add debug checks for variables (scalars, arrays) before and after scheme calls (suite_objects.py) --- scripts/suite_objects.py | 270 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 259 insertions(+), 11 deletions(-) mode change 100644 => 100755 scripts/suite_objects.py diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py old mode 100644 new mode 100755 index cc34a750..ba1ffcd3 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -878,7 +878,7 @@ def match_variable(self, var, vstdname=None, vdims=None): # end if # end if # end if - return found_var, var_vdim, new_vdims, missing_vert + return found_var, dict_var, var_vdim, new_vdims, missing_vert def in_process_split(self): """Find out if we are in a process-split region""" @@ -1061,6 +1061,7 @@ def __init__(self, scheme_xml, context, parent, run_env): self.__lib = scheme_xml.get('lib', None) self.__has_vertical_dimension = False self.__group = None + self.__var_debug_checks = list() super().__init__(name, context, parent, run_env, active_call_list=True) def update_group_call_list_variable(self, var): @@ -1129,8 +1130,15 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): vdims = var.get_dimensions() vintent = var.get_prop_value('intent') args = self.match_variable(var, vstdname=vstdname, vdims=vdims) - found, vert_dim, new_dims, missing_vert = args + found, dict_var, vert_dim, new_dims, missing_vert = args if found: + + if self.__group.run_env.debug: + # Add variable allocation checks for host model variables only + gvar = self.__group.find_variable(standard_name=vstdname, any_scope=False) + if dict_var and not gvar: + self.add_var_debug_check(dict_var, new_dims) + # end if if not self.has_vertical_dim: self.__has_vertical_dimension = vert_dim is not None # end if @@ -1195,7 +1203,238 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): # end if return scheme_mods - def write(self, outfile, errcode, indent): + def add_var_debug_check(self, var, new_dims): + # Get the basic attributes that decide whether we need + # to check the variable when we write the group + standard_name = var.get_prop_value('standard_name') + # We need to use the new dimensions as determined + # by the scheme's analyze logic, not the original + # variable dimension of the host variable var. + dimensions = new_dims + active = var.get_prop_value('active') + # The order is important to get the correct local name - DH* not sure ... + var_dicts = [ self.__group.call_list ] + self.__group.suite_dicts() + + # If the variable isn't active, skip it + if active.lower() =='.false.': + return + # Also, if the variable is one of the CCPP error handling messages, skip it + # since it is defined as intent(out) and we can't do meaningful checks on it + elif standard_name == 'ccpp_error_code' or standard_name == 'ccpp_error_message': + return + # To perform allocation checks, we need to know all variables + # that are part of the 'active' attribute conditional and add + # it to the group's call list. + else: + (_, vars_needed) = var.conditional(var_dicts) + for var_needed in vars_needed: + self.update_group_call_list_variable(var_needed) + + # For scalars and arrays, need a dummy variable (same kind and type) + # that we can assign the scalar or the lbound/ubound of the array to. + # We need to treat DDTs and variables with kind attributes slightly + # differently, and make sure there are no duplicate variables. We + # also need to assign a bogus standard name to these local variables. # DH* do we? + vtype = var.get_prop_value('type') + if var.is_ddt(): + vkind = '' + units = '' + else: + vkind = var.get_prop_value('kind') + units = var.get_prop_value('units') + if vkind: + dummy_lname = f'dummy_{vtype.replace("=","_")}_{vkind.replace("=","_")}' + else: + dummy_lname = f'dummy_{vtype.replace("=","_")}' + if var.is_ddt(): + dummy = Var({'local_name':dummy_lname, 'standard_name':f'{dummy_lname}_local', + 'ddt_type':vtype, 'kind':vkind, 'units':units, 'dimensions':'()'}, + _API_LOCAL, self.run_env) + else: + dummy = Var({'local_name':dummy_lname, 'standard_name':f'{dummy_lname}_local', + 'type':vtype, 'kind':vkind, 'units':units, 'dimensions':'()'}, + _API_LOCAL, self.run_env) + found = self.__group.find_variable(source_var=dummy) + if not found: + self.__group.manage_variable(dummy) + + # For arrays, we need to get information on the dimensions and add it to + # the group's call list so that we can test for the correct size later on + if dimensions: + for dim in dimensions: + if not ':' in dim: + dim_var = self.find_variable(standard_name=dim) + if not dim_var: + raise Exception(f"No dimension with standard name '{dim}'") + self.update_group_call_list_variable(dim_var) + else: + (ldim, udim) = dim.split(":") + ldim_var = self.find_variable(standard_name=ldim) + if not ldim_var: + raise Exception(f"No dimension with standard name '{ldim}'") + self.update_group_call_list_variable(ldim_var) + udim_var = self.find_variable(standard_name=udim) + if not udim_var: + raise Exception(f"No dimension with standard name '{udim}'") + self.update_group_call_list_variable(udim_var) + + # Add the modified host model variable (with new dimension) + # to the list of variables to check. Record which dummy to use. + subst_dict = {'dimensions':new_dims} + clone = var.clone(subst_dict) + if var.get_prop_value('local_name') == 'cld_liq_array': + print(f"adding var debug check for {var} with dimensions old/new {var.get_dimensions()} / {new_dims}, run phase? {self.run_phase()}") + self.__var_debug_checks.append([clone, dummy]) + + def write_var_debug_check(self, var, dummy, cldicts, outfile, errcode, errmsg, indent): + # Get the basic attributes for writing the check + standard_name = var.get_prop_value('standard_name') + dimensions = var.get_dimensions() + active = var.get_prop_value('active') + pointer = var.get_prop_value('pointer') + allocatable = var.get_prop_value('allocatable') + # DH* TEST + # The order is important to get the correct local name - DH* not sure ... + #var_dicts = [ self.__group.call_list ] + self.__group.suite_dicts() + var_dicts = cldicts + # *DH + + # Need the local name as it comes from the group call list + # or from the suite, not how it is called in the scheme (var) + dvar = self.__group.call_list.find_variable(standard_name=standard_name, any_scope=False) + if dvar: + var_in_call_list = True + else: + var_in_call_list = False + for var_dict in self.__group.suite_dicts(): + dvar = var_dict.find_variable(standard_name=standard_name) + if dvar: + break + if not dvar: + raise Exception(f"No variable with standard name '{standard_name}' in var_dicts") + local_name = dvar.get_prop_value('local_name') + + # If the variable is allocatable or a pointer and the intent for the + # scheme is 'out', then we can't test anything because the scheme is + # going to allocate the variable or associate the pointer. We don't have + # this information earlier in add_var_debug_check, therefore need to back + # out here, using the information from the scheme variable (call list). + svar = self.call_list.find_variable(standard_name=standard_name) + intent = svar.get_prop_value('intent') + if intent == 'out' and (allocatable or pointer): + return + + # Get the condition on which the variable is active + (conditional, _) = var.conditional(var_dicts) + + # For scalars, assign to dummy variable if the variable intent is in/inout + if not dimensions: + if not intent == 'out': + dummy_lname = dummy.get_prop_value('local_name') + outfile.write(f"if ({conditional}) then", indent) + outfile.write(f"! Assign value of {local_name} to dummy", indent+1) + outfile.write(f"{dummy_lname} = {local_name}", indent+1) + outfile.write(f"end if", indent) + # For arrays, check size of array against dimensions in metadata, then assign + # the lower and upper bounds to the dummy variable if the intent is in/inout + else: + array_size = 1 + dim_strings = [] + lbound_strings = [] + ubound_strings = [] + for dim in dimensions: + if not ':' in dim: + # In capgen, any true dimension (that is not a single index) does + # have a colon (:) in the dimension, therefore this is an index + for var_dict in var_dicts: + dvar = var_dict.find_variable(standard_name=dim, any_scope=False) + if dvar is not None: + break + if not dvar: + raise Exception(f"No variable with standard name '{dim}' in var_dicts") + dim_lname = dvar.get_prop_value('local_name') + dim_length = 1 + dim_strings.append(dim_lname) + lbound_strings.append(dim_lname) + ubound_strings.append(dim_lname) + else: + # I don't know how to do this better. Schemes can rely on the host cap + # passing arrays such that the horizontal dimension of the variable + # seen by the scheme runs from 1:ncol (horizontal_loop_extent) + if is_horizontal_dimension(dim): + if self.run_phase(): + if self.find_variable(standard_name="horizontal_loop_extent"): + ldim = "ccpp_constant_one" + udim = "horizontal_loop_extent" + else: + ldim = "horizontal_loop_begin" + udim = "horizontal_loop_end" + else: + ldim = "ccpp_constant_one" + udim = "horizontal_dimension" + else: + (ldim, udim) = dim.split(":") + # Get dimension for lower bound + for var_dict in var_dicts: + dvar = var_dict.find_variable(standard_name=ldim, any_scope=False) + if dvar is not None: + break + if not dvar: + raise Exception(f"No variable with standard name '{ldim}' in var_dicts") + ldim_lname = dvar.get_prop_value('local_name') + # Get dimension for upper bound + for var_dict in var_dicts: + dvar = var_dict.find_variable(standard_name=udim, any_scope=False) + if dvar is not None: + break + if not dvar: + raise Exception(f"No variable with standard name '{udim}' in var_dicts") + udim_lname = dvar.get_prop_value('local_name') + # Assemble dimensions and bounds for size checking + dim_length = f'{udim_lname}-{ldim_lname}+1' + if is_horizontal_dimension(dim): + # Is var in the call list or a module variable of this group? + if not var_in_call_list: + dim_strings.append(f"{ldim_lname}:{udim_lname}") + lbound_strings.append(ldim_lname) + ubound_strings.append(udim_lname) + else: + dim_strings.append(":") + lbound_strings.append('1') + ubound_strings.append(f'{udim_lname}-{ldim_lname}+1') + else: + dim_strings.append(":") + lbound_strings.append(ldim_lname) + ubound_strings.append(udim_lname) + array_size = f'{array_size}*({dim_length})' + + # Various strings needed to get the right size + # and lower/upper bound of the array + dim_string = '(' + ','.join(dim_strings) + ')' + lbound_string = '(' + ','.join(lbound_strings) + ')' + ubound_string = '(' + ','.join(ubound_strings) + ')' + + # Write size check + outfile.write(f"if ({conditional}) then", indent) + outfile.write(f"! Check size of array {local_name}", indent+1) + outfile.write(f"if (size({local_name}{dim_string}) /= {array_size}) then", indent+1) + outfile.write(f"write({errmsg}, '(a)') 'In group {self.__group.name} before {self.__subroutine_name}:'", indent+2) + outfile.write(f"write({errmsg}, '(2(a,i8))') 'for array {local_name}, expected size ', {array_size}, ' but got ', size({local_name})", indent+2) + outfile.write(f"{errcode} = 1", indent+2) + outfile.write(f"return", indent+2) + outfile.write(f"end if", indent+1) + outfile.write(f"end if", indent) + + # Assign lower/upper bounds to dummy (scalar) if intent is not out + if not intent == 'out': + dummy_lname = dummy.get_prop_value('local_name') + outfile.write(f"if ({conditional}) then", indent) + outfile.write(f"! Assign lower/upper bounds of {local_name} to dummy", indent+1) + outfile.write(f"{dummy_lname} = {local_name}{lbound_string}", indent+1) + outfile.write(f"{dummy_lname} = {local_name}{ubound_string}", indent+1) + outfile.write(f"end if", indent) + + def write(self, outfile, errcode, errmsg, indent): # Unused arguments are for consistent write interface # pylint: disable=unused-argument """Write code to call this Scheme to """ @@ -1206,6 +1445,15 @@ def write(self, outfile, errcode, indent): my_args = self.call_list.call_string(cldicts=cldicts, is_func_call=True, subname=self.subroutine_name) + + # Write debug checks (operating on variables + # coming from the group's call list) + for (var, dummy) in self.__var_debug_checks: + stmt = self.write_var_debug_check(var, dummy, cldicts, outfile, errcode, errmsg, indent) + + # Write variable transformations (to be developed) + + # Write call to routine stmt = 'call {}({})' outfile.write('if ({} == 0) then'.format(errcode), indent) outfile.write(stmt.format(self.subroutine_name, my_args), indent+1) @@ -1324,13 +1572,13 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): # end for return scheme_mods - def write(self, outfile, errcode, indent): + def write(self, outfile, errcode, errmsg, indent): """Write code for the vertical loop, including contents, to """ outfile.write('do {} = 1, {}'.format(self.name, self.dimension_name), indent) # Note that 'scheme' may be a sybcycle or other construct for item in self.parts: - item.write(outfile, errcode, indent+1) + item.write(outfile, errcode, errmsg, indent+1) # end for outfile.write('end do', 2) @@ -1389,12 +1637,12 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): # end for return scheme_mods - def write(self, outfile, errcode, indent): + def write(self, outfile, errcode, errmsg, indent): """Write code for the subcycle loop, including contents, to """ outfile.write('do {} = 1, {}'.format(self.name, self.loop), indent) # Note that 'scheme' may be a sybcycle or other construct for item in self.parts: - item.write(outfile, errcode, indent+1) + item.write(outfile, errcode, errmsg, indent+1) # end for outfile.write('end do', 2) @@ -1439,11 +1687,11 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): # end for return scheme_mods - def write(self, outfile, errcode, indent): + def write(self, outfile, errcode, errmsg, indent): """Write code for this TimeSplit section, including contents, to """ for item in self.parts: - item.write(outfile, errcode, indent) + item.write(outfile, errcode, errmsg, indent) # end for ############################################################################### @@ -1468,7 +1716,7 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): # Handle all the suite objects inside of this group raise CCPPError('ProcessSplit not yet implemented') - def write(self, outfile, errcode, indent): + def write(self, outfile, errcode, errmsg, indent): """Write code for this ProcessSplit section, including contents, to """ raise CCPPError('ProcessSplit not yet implemented') @@ -1867,7 +2115,7 @@ def write(self, outfile, host_arglist, indent, const_mod, # end for # Write the scheme and subcycle calls for item in self.parts: - item.write(outfile, errcode, indent + 1) + item.write(outfile, errcode, errmsg, indent + 1) # end for # Deallocate local arrays for lname in allocatable_var_set: From ddae076db670d62466330dfb8d4df67b31a8c000 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 8 Nov 2023 09:54:37 -0700 Subject: [PATCH 05/23] Fix typo in var_props.py --- scripts/var_props.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/var_props.py b/scripts/var_props.py index 15613567..d565d024 100755 --- a/scripts/var_props.py +++ b/scripts/var_props.py @@ -150,7 +150,7 @@ def standard_name_to_long_name(prop_dict, context=None): ... parse_source.CCPPError: No standard name to convert to long name, at foo.F90:4 """ - # We assume that standar_name has been checked for validity + # We assume that standard_name has been checked for validity # Make the first char uppercase and replace each underscore with a space if 'standard_name' in prop_dict: standard_name = prop_dict['standard_name'] From 29ee9ae3bf3fb6fcf6153526ea7efda78a85ce01 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 8 Nov 2023 09:55:32 -0700 Subject: [PATCH 06/23] For advection tests, add --debug flag to capgen and update test reports --- test/advection_test/CMakeLists.txt | 1 + test/advection_test/run_test | 7 ++++++- test/advection_test/test_reports.py | 10 ++++++++-- 3 files changed, 15 insertions(+), 3 deletions(-) mode change 100644 => 100755 test/advection_test/test_reports.py diff --git a/test/advection_test/CMakeLists.txt b/test/advection_test/CMakeLists.txt index 10ada283..4aacdc18 100644 --- a/test/advection_test/CMakeLists.txt +++ b/test/advection_test/CMakeLists.txt @@ -149,6 +149,7 @@ while (VERBOSITY GREATER 0) list(APPEND CAPGEN_CMD "--verbose") MATH(EXPR VERBOSITY "${VERBOSITY} - 1") endwhile () +list(APPEND CAPGEN_CMD "--debug") string(REPLACE ";" " " CAPGEN_STRING "${CAPGEN_CMD}") MESSAGE(STATUS "Running: ${CAPGEN_STRING}") EXECUTE_PROCESS(COMMAND ${CAPGEN_CMD} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} diff --git a/test/advection_test/run_test b/test/advection_test/run_test index b014470a..9c75aa08 100755 --- a/test/advection_test/run_test +++ b/test/advection_test/run_test @@ -131,18 +131,23 @@ suite_list="cld_suite" required_vars="ccpp_error_code,ccpp_error_message" required_vars="${required_vars},cloud_ice_dry_mixing_ratio" required_vars="${required_vars},cloud_liquid_dry_mixing_ratio" +required_vars="${required_vars},horizontal_dimension" required_vars="${required_vars},horizontal_loop_begin" required_vars="${required_vars},horizontal_loop_end" required_vars="${required_vars},surface_air_pressure" required_vars="${required_vars},temperature" required_vars="${required_vars},time_step_for_physics" +required_vars="${required_vars},vertical_layer_dimension" required_vars="${required_vars},water_temperature_at_freezing" required_vars="${required_vars},water_vapor_specific_humidity" input_vars="cloud_ice_dry_mixing_ratio,cloud_liquid_dry_mixing_ratio" +input_vars="${input_vars},horizontal_dimension" input_vars="${input_vars},horizontal_loop_begin" input_vars="${input_vars},horizontal_loop_end" input_vars="${input_vars},surface_air_pressure,temperature" -input_vars="${input_vars},time_step_for_physics,water_temperature_at_freezing" +input_vars="${input_vars},time_step_for_physics" +input_vars="${input_vars},vertical_layer_dimension" +input_vars="${input_vars},water_temperature_at_freezing" input_vars="${input_vars},water_vapor_specific_humidity" output_vars="ccpp_error_code,ccpp_error_message" output_vars="${output_vars},cloud_ice_dry_mixing_ratio" diff --git a/test/advection_test/test_reports.py b/test/advection_test/test_reports.py old mode 100644 new mode 100755 index c28fe38a..6de70ed6 --- a/test/advection_test/test_reports.py +++ b/test/advection_test/test_reports.py @@ -72,13 +72,19 @@ def usage(errmsg=None): "time_step_for_physics", "water_temperature_at_freezing", "water_vapor_specific_humidity", "cloud_ice_dry_mixing_ratio", - "cloud_liquid_dry_mixing_ratio"] + "cloud_liquid_dry_mixing_ratio", + # Added by --debug option + "horizontal_dimension", + "vertical_layer_dimension"] _INPUT_VARS_CLD = ["surface_air_pressure", "temperature", "horizontal_loop_begin", "horizontal_loop_end", "time_step_for_physics", "water_temperature_at_freezing", "water_vapor_specific_humidity", "cloud_ice_dry_mixing_ratio", - "cloud_liquid_dry_mixing_ratio"] + "cloud_liquid_dry_mixing_ratio", + # Added by --debug option + "horizontal_dimension", + "vertical_layer_dimension"] _OUTPUT_VARS_CLD = ["ccpp_error_code", "ccpp_error_message", "water_vapor_specific_humidity", "temperature", "cloud_ice_dry_mixing_ratio", From de0891fe2d6049d7d038c2c33d29b7cdea585b9c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 8 Nov 2023 09:56:33 -0700 Subject: [PATCH 07/23] Add active attribute to variable water_vapor_specific_humidity in capgen_test, add --debug flag to capgen and update test reports --- test/capgen_test/CMakeLists.txt | 1 + test/capgen_test/run_test | 4 ++++ test/capgen_test/test_host_data.meta | 1 + test/capgen_test/test_reports.py | 10 +++++++++- 4 files changed, 15 insertions(+), 1 deletion(-) mode change 100644 => 100755 test/capgen_test/test_reports.py diff --git a/test/capgen_test/CMakeLists.txt b/test/capgen_test/CMakeLists.txt index f02213a0..ccae4f08 100644 --- a/test/capgen_test/CMakeLists.txt +++ b/test/capgen_test/CMakeLists.txt @@ -149,6 +149,7 @@ while (VERBOSITY GREATER 0) list(APPEND CAPGEN_CMD "--verbose") MATH(EXPR VERBOSITY "${VERBOSITY} - 1") endwhile () +list(APPEND CAPGEN_CMD "--debug") string(REPLACE ";" " " CAPGEN_STRING "${CAPGEN_CMD}") MESSAGE(STATUS "Running: ${CAPGEN_STRING}") EXECUTE_PROCESS(COMMAND ${CAPGEN_CMD} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} diff --git a/test/capgen_test/run_test b/test/capgen_test/run_test index 4dfd490f..0d5d44f7 100755 --- a/test/capgen_test/run_test +++ b/test/capgen_test/run_test @@ -148,20 +148,24 @@ output_vars_ddt="${output_vars_ddt},model_times,number_of_model_times" required_vars_temp="ccpp_error_code,ccpp_error_message,horizontal_dimension" required_vars_temp="${required_vars_temp},horizontal_loop_begin" required_vars_temp="${required_vars_temp},horizontal_loop_end" +required_vars_temp="${required_vars_temp},index_of_water_vapor_specific_humidity" required_vars_temp="${required_vars_temp},potential_temperature" required_vars_temp="${required_vars_temp},potential_temperature_at_interface" required_vars_temp="${required_vars_temp},potential_temperature_increment" required_vars_temp="${required_vars_temp},surface_air_pressure" required_vars_temp="${required_vars_temp},time_step_for_physics" +required_vars_temp="${required_vars_temp},vertical_interface_dimension" required_vars_temp="${required_vars_temp},vertical_layer_dimension" required_vars_temp="${required_vars_temp},water_vapor_specific_humidity" input_vars_temp="horizontal_dimension" input_vars_temp="${input_vars_temp},horizontal_loop_begin" input_vars_temp="${input_vars_temp},horizontal_loop_end" +input_vars_temp="${input_vars_temp},index_of_water_vapor_specific_humidity" input_vars_temp="${input_vars_temp},potential_temperature" input_vars_temp="${input_vars_temp},potential_temperature_at_interface" input_vars_temp="${input_vars_temp},potential_temperature_increment" input_vars_temp="${input_vars_temp},surface_air_pressure,time_step_for_physics" +input_vars_temp="${input_vars_temp},vertical_interface_dimension" input_vars_temp="${input_vars_temp},vertical_layer_dimension" input_vars_temp="${input_vars_temp},water_vapor_specific_humidity" output_vars_temp="ccpp_error_code,ccpp_error_message,potential_temperature" diff --git a/test/capgen_test/test_host_data.meta b/test/capgen_test/test_host_data.meta index 8a54a3ed..71a1a88f 100644 --- a/test/capgen_test/test_host_data.meta +++ b/test/capgen_test/test_host_data.meta @@ -49,3 +49,4 @@ kind = kind_phys units = kg kg-1 dimensions = (horizontal_dimension, vertical_layer_dimension) + active = (index_of_water_vapor_specific_humidity > 0) \ No newline at end of file diff --git a/test/capgen_test/test_reports.py b/test/capgen_test/test_reports.py old mode 100644 new mode 100755 index 3749e8ac..de6ebb2d --- a/test/capgen_test/test_reports.py +++ b/test/capgen_test/test_reports.py @@ -78,7 +78,10 @@ def usage(errmsg=None): "number_of_model_times"] _REQUIRED_VARS_DDT = _INPUT_VARS_DDT + _OUTPUT_VARS_DDT _PROT_VARS_TEMP = ["horizontal_loop_begin", "horizontal_loop_end", - "horizontal_dimension", "vertical_layer_dimension"] + "horizontal_dimension", "vertical_layer_dimension", + # Added for --debug + "index_of_water_vapor_specific_humidity", + "vertical_interface_dimension"] _REQUIRED_VARS_TEMP = ["ccpp_error_code", "ccpp_error_message", "potential_temperature", "potential_temperature_at_interface", @@ -171,18 +174,23 @@ def check_datatable(database, report_type, check_list, value="ddt_suite"), _OUTPUT_VARS_DDT) print("\nChecking variables for temp suite from python") +print("HERE 0") NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("required_variables", value="temp_suite"), _REQUIRED_VARS_TEMP + _PROT_VARS_TEMP) +print("HERE A") NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("required_variables", value="temp_suite"), _REQUIRED_VARS_TEMP, excl_prot=True) +print("HERE B") NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("input_variables", value="temp_suite"), _INPUT_VARS_TEMP + _PROT_VARS_TEMP) +print("HERE C") NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("input_variables", value="temp_suite"), _INPUT_VARS_TEMP, excl_prot=True) +print("HERE Z") NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("output_variables", value="temp_suite"), _OUTPUT_VARS_TEMP) From b1b215df7887bde017312badaa2fc03af95e0d2b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 8 Nov 2023 09:57:11 -0700 Subject: [PATCH 08/23] For var_action tests, add --debug flag to capgen and update test reports --- test/var_action_test/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test/var_action_test/CMakeLists.txt b/test/var_action_test/CMakeLists.txt index 4b9daab8..304fa24d 100644 --- a/test/var_action_test/CMakeLists.txt +++ b/test/var_action_test/CMakeLists.txt @@ -149,6 +149,7 @@ while (VERBOSITY GREATER 0) list(APPEND CAPGEN_CMD "--verbose") MATH(EXPR VERBOSITY "${VERBOSITY} - 1") endwhile () +list(APPEND CAPGEN_CMD "--debug") string(REPLACE ";" " " CAPGEN_STRING "${CAPGEN_CMD}") MESSAGE(STATUS "Running: ${CAPGEN_STRING}") EXECUTE_PROCESS(COMMAND ${CAPGEN_CMD} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} From adeef17a6e6441ca145e545f3de87f445205f8e9 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 8 Nov 2023 10:22:03 -0700 Subject: [PATCH 09/23] Clean up based on self-review --- scripts/suite_objects.py | 11 +++++------ test/advection_test/test_reports.py | 0 test/capgen_test/test_host_data.meta | 2 +- test/capgen_test/test_reports.py | 5 ----- 4 files changed, 6 insertions(+), 12 deletions(-) mode change 100755 => 100644 test/advection_test/test_reports.py mode change 100755 => 100644 test/capgen_test/test_reports.py diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index ba1ffcd3..15b06be7 100755 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1282,8 +1282,6 @@ def add_var_debug_check(self, var, new_dims): # to the list of variables to check. Record which dummy to use. subst_dict = {'dimensions':new_dims} clone = var.clone(subst_dict) - if var.get_prop_value('local_name') == 'cld_liq_array': - print(f"adding var debug check for {var} with dimensions old/new {var.get_dimensions()} / {new_dims}, run phase? {self.run_phase()}") self.__var_debug_checks.append([clone, dummy]) def write_var_debug_check(self, var, dummy, cldicts, outfile, errcode, errmsg, indent): @@ -1293,11 +1291,9 @@ def write_var_debug_check(self, var, dummy, cldicts, outfile, errcode, errmsg, i active = var.get_prop_value('active') pointer = var.get_prop_value('pointer') allocatable = var.get_prop_value('allocatable') - # DH* TEST # The order is important to get the correct local name - DH* not sure ... #var_dicts = [ self.__group.call_list ] + self.__group.suite_dicts() var_dicts = cldicts - # *DH # Need the local name as it comes from the group call list # or from the suite, not how it is called in the scheme (var) @@ -1360,7 +1356,10 @@ def write_var_debug_check(self, var, dummy, cldicts, outfile, errcode, errmsg, i else: # I don't know how to do this better. Schemes can rely on the host cap # passing arrays such that the horizontal dimension of the variable - # seen by the scheme runs from 1:ncol (horizontal_loop_extent) + # seen by the scheme runs from 1:ncol (horizontal_loop_extent). But + # module variables for this group are passed to the schemes with the + # horizontal dimensions in the call dimstring. And it all depends + # on the phase, too. if is_horizontal_dimension(dim): if self.run_phase(): if self.find_variable(standard_name="horizontal_loop_extent"): @@ -1393,7 +1392,7 @@ def write_var_debug_check(self, var, dummy, cldicts, outfile, errcode, errmsg, i # Assemble dimensions and bounds for size checking dim_length = f'{udim_lname}-{ldim_lname}+1' if is_horizontal_dimension(dim): - # Is var in the call list or a module variable of this group? + # See comment above on call list variables vs module variables if not var_in_call_list: dim_strings.append(f"{ldim_lname}:{udim_lname}") lbound_strings.append(ldim_lname) diff --git a/test/advection_test/test_reports.py b/test/advection_test/test_reports.py old mode 100755 new mode 100644 diff --git a/test/capgen_test/test_host_data.meta b/test/capgen_test/test_host_data.meta index 71a1a88f..df4b92b4 100644 --- a/test/capgen_test/test_host_data.meta +++ b/test/capgen_test/test_host_data.meta @@ -49,4 +49,4 @@ kind = kind_phys units = kg kg-1 dimensions = (horizontal_dimension, vertical_layer_dimension) - active = (index_of_water_vapor_specific_humidity > 0) \ No newline at end of file + active = (index_of_water_vapor_specific_humidity > 0) diff --git a/test/capgen_test/test_reports.py b/test/capgen_test/test_reports.py old mode 100755 new mode 100644 index de6ebb2d..b136a95e --- a/test/capgen_test/test_reports.py +++ b/test/capgen_test/test_reports.py @@ -174,23 +174,18 @@ def check_datatable(database, report_type, check_list, value="ddt_suite"), _OUTPUT_VARS_DDT) print("\nChecking variables for temp suite from python") -print("HERE 0") NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("required_variables", value="temp_suite"), _REQUIRED_VARS_TEMP + _PROT_VARS_TEMP) -print("HERE A") NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("required_variables", value="temp_suite"), _REQUIRED_VARS_TEMP, excl_prot=True) -print("HERE B") NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("input_variables", value="temp_suite"), _INPUT_VARS_TEMP + _PROT_VARS_TEMP) -print("HERE C") NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("input_variables", value="temp_suite"), _INPUT_VARS_TEMP, excl_prot=True) -print("HERE Z") NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("output_variables", value="temp_suite"), _OUTPUT_VARS_TEMP) From 7256311935a09ec7d3d2bb0406f213e6b4ae4c02 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 22 Nov 2023 09:45:18 -0700 Subject: [PATCH 10/23] Add scripts/parse_tools/fortran_conditional.py --- scripts/parse_tools/fortran_conditional.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100755 scripts/parse_tools/fortran_conditional.py diff --git a/scripts/parse_tools/fortran_conditional.py b/scripts/parse_tools/fortran_conditional.py new file mode 100755 index 00000000..b1ec7eeb --- /dev/null +++ b/scripts/parse_tools/fortran_conditional.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# + +"""Definitions to convert a conditional statement in the metadata, expressed in standard names, +into a Fortran conditional (used in an if statement), expressed in local names. +""" + +import re + +FORTRAN_CONDITIONAL_REGEX_WORDS = [' ', '(', ')', '==', '/=', '<=', '>=', '<', '>', '.eqv.', '.neqv.', + '.true.', '.false.', '.lt.', '.le.', '.eq.', '.ge.', '.gt.', '.ne.', + '.not.', '.and.', '.or.', '.xor.'] +FORTRAN_CONDITIONAL_REGEX = re.compile(r"[\w']+|" + "|".join([word.replace('(','\(').replace(')', '\)') for word in FORTRAN_CONDITIONAL_REGEX_WORDS])) From af169355f72c1a431a2b6dcd2c21381717b92ae9 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 22 Nov 2023 09:47:48 -0700 Subject: [PATCH 11/23] Cleanup: replace var_dicts with cldicts in write_debug_checks in suite_objects.py; remove pointer attribute in metavar.py and fortran_tools/parse_fortran.py; import Fortran conditional regex statements from parse_tools --- scripts/fortran_tools/parse_fortran.py | 3 --- scripts/framework_env.py | 0 scripts/metavar.py | 9 +------- scripts/parse_tools/__init__.py | 5 +++- scripts/suite_objects.py | 32 +++++++++++--------------- 5 files changed, 19 insertions(+), 30 deletions(-) mode change 100755 => 100644 scripts/fortran_tools/parse_fortran.py mode change 100755 => 100644 scripts/framework_env.py mode change 100755 => 100644 scripts/suite_objects.py diff --git a/scripts/fortran_tools/parse_fortran.py b/scripts/fortran_tools/parse_fortran.py old mode 100755 new mode 100644 index a3a05cc5..9aa4de7a --- a/scripts/fortran_tools/parse_fortran.py +++ b/scripts/fortran_tools/parse_fortran.py @@ -800,9 +800,6 @@ def parse_fortran_var_decl(line, source, run_env): if 'allocatable' in varprops: prop_dict['allocatable'] = 'True' # end if - if 'pointer' in varprops: - prop_dict['pointer'] = 'True' - # end if if intent is not None: prop_dict['intent'] = intent # end if diff --git a/scripts/framework_env.py b/scripts/framework_env.py old mode 100755 new mode 100644 diff --git a/scripts/metavar.py b/scripts/metavar.py index ed80cd70..16b175a9 100755 --- a/scripts/metavar.py +++ b/scripts/metavar.py @@ -22,6 +22,7 @@ from parse_tools import check_default_value, check_valid_values from parse_tools import ParseContext, ParseSource, type_name from parse_tools import ParseInternalError, ParseSyntaxError, CCPPError +from parse_tools import FORTRAN_CONDITIONAL_REGEX_WORDS, FORTRAN_CONDITIONAL_REGEX from var_props import CCPP_LOOP_DIM_SUBSTS, VariableProperty, VarCompatObj from var_props import find_horizontal_dimension, find_vertical_dimension from var_props import standard_name_to_long_name, default_kind_val @@ -85,12 +86,6 @@ 'horizontal_loop_begin', 'horizontal_loop_end', 'vertical_layer_index', 'vertical_interface_index'] -# DH* Is there a better place for these definitions? -FORTRAN_CONDITIONAL_REGEX_WORDS = [' ', '(', ')', '==', '/=', '<=', '>=', '<', '>', '.eqv.', '.neqv.', - '.true.', '.false.', '.lt.', '.le.', '.eq.', '.ge.', '.gt.', '.ne.', - '.not.', '.and.', '.or.', '.xor.'] -FORTRAN_CONDITIONAL_REGEX = re.compile(r"[\w']+|" + "|".join([word.replace('(','\(').replace(')', '\)') for word in FORTRAN_CONDITIONAL_REGEX_WORDS])) - ############################################################################### # Used for creating template variables _MVAR_DUMMY_RUN_ENV = CCPPFrameworkEnv(None, ndict={'host_files':'', @@ -193,8 +188,6 @@ class Var: optional_in=True, default_in=False), VariableProperty('allocatable', bool, optional_in=True, default_in=False), - VariableProperty('pointer', bool, - optional_in=True, default_in=False), VariableProperty('diagnostic_name', str, optional_in=True, default_in='', check_fn_in=check_diagnostic_id), diff --git a/scripts/parse_tools/__init__.py b/scripts/parse_tools/__init__.py index 4f888fb1..3ce9d344 100644 --- a/scripts/parse_tools/__init__.py +++ b/scripts/parse_tools/__init__.py @@ -30,6 +30,7 @@ from xml_tools import find_schema_file, find_schema_version from xml_tools import read_xml_file, validate_xml_file from xml_tools import PrettyElementTree +from fortran_conditional import FORTRAN_CONDITIONAL_REGEX_WORDS, FORTRAN_CONDITIONAL_REGEX # pylint: enable=wrong-import-position __all__ = [ @@ -73,5 +74,7 @@ 'set_log_to_stdout', 'type_name', 'unique_standard_name', - 'validate_xml_file' + 'validate_xml_file', + 'FORTRAN_CONDITIONAL_REGEX_WORDS', + 'FORTRAN_CONDITIONAL_REGEX' ] diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py old mode 100755 new mode 100644 index 15b06be7..2ba55802 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1289,11 +1289,7 @@ def write_var_debug_check(self, var, dummy, cldicts, outfile, errcode, errmsg, i standard_name = var.get_prop_value('standard_name') dimensions = var.get_dimensions() active = var.get_prop_value('active') - pointer = var.get_prop_value('pointer') allocatable = var.get_prop_value('allocatable') - # The order is important to get the correct local name - DH* not sure ... - #var_dicts = [ self.__group.call_list ] + self.__group.suite_dicts() - var_dicts = cldicts # Need the local name as it comes from the group call list # or from the suite, not how it is called in the scheme (var) @@ -1307,21 +1303,21 @@ def write_var_debug_check(self, var, dummy, cldicts, outfile, errcode, errmsg, i if dvar: break if not dvar: - raise Exception(f"No variable with standard name '{standard_name}' in var_dicts") + raise Exception(f"No variable with standard name '{standard_name}' in cldicts") local_name = dvar.get_prop_value('local_name') - # If the variable is allocatable or a pointer and the intent for the - # scheme is 'out', then we can't test anything because the scheme is - # going to allocate the variable or associate the pointer. We don't have - # this information earlier in add_var_debug_check, therefore need to back - # out here, using the information from the scheme variable (call list). + # If the variable is allocatable and the intent for the scheme is 'out', + # then we can't test anything because the scheme is going to allocate + # the variable. We don't have this information earlier in + # add_var_debug_check, therefore need to back out here, + # using the information from the scheme variable (call list). svar = self.call_list.find_variable(standard_name=standard_name) intent = svar.get_prop_value('intent') - if intent == 'out' and (allocatable or pointer): + if intent == 'out' and allocatable: return # Get the condition on which the variable is active - (conditional, _) = var.conditional(var_dicts) + (conditional, _) = var.conditional(cldicts) # For scalars, assign to dummy variable if the variable intent is in/inout if not dimensions: @@ -1342,12 +1338,12 @@ def write_var_debug_check(self, var, dummy, cldicts, outfile, errcode, errmsg, i if not ':' in dim: # In capgen, any true dimension (that is not a single index) does # have a colon (:) in the dimension, therefore this is an index - for var_dict in var_dicts: + for var_dict in cldicts: dvar = var_dict.find_variable(standard_name=dim, any_scope=False) if dvar is not None: break if not dvar: - raise Exception(f"No variable with standard name '{dim}' in var_dicts") + raise Exception(f"No variable with standard name '{dim}' in cldicts") dim_lname = dvar.get_prop_value('local_name') dim_length = 1 dim_strings.append(dim_lname) @@ -1374,20 +1370,20 @@ def write_var_debug_check(self, var, dummy, cldicts, outfile, errcode, errmsg, i else: (ldim, udim) = dim.split(":") # Get dimension for lower bound - for var_dict in var_dicts: + for var_dict in cldicts: dvar = var_dict.find_variable(standard_name=ldim, any_scope=False) if dvar is not None: break if not dvar: - raise Exception(f"No variable with standard name '{ldim}' in var_dicts") + raise Exception(f"No variable with standard name '{ldim}' in cldicts") ldim_lname = dvar.get_prop_value('local_name') # Get dimension for upper bound - for var_dict in var_dicts: + for var_dict in cldicts: dvar = var_dict.find_variable(standard_name=udim, any_scope=False) if dvar is not None: break if not dvar: - raise Exception(f"No variable with standard name '{udim}' in var_dicts") + raise Exception(f"No variable with standard name '{udim}' in cldicts") udim_lname = dvar.get_prop_value('local_name') # Assemble dimensions and bounds for size checking dim_length = f'{udim_lname}-{ldim_lname}+1' From edb1bd5656c45a559f61cdf14704c7277898d678 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 22 Nov 2023 10:17:29 -0700 Subject: [PATCH 12/23] Update comment on ddt_type property in metavar.py --- scripts/metavar.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/metavar.py b/scripts/metavar.py index 16b175a9..cfe08743 100755 --- a/scripts/metavar.py +++ b/scripts/metavar.py @@ -295,7 +295,8 @@ def __init__(self, prop_dict, source, run_env, context=None, if 'units' not in prop_dict: prop_dict['units'] = "" # end if - # DH* Why is the DDT type copied into the kind attribute? + # DH* To investigate later: Why is the DDT type + # copied into the kind attribute? Can we remove this? prop_dict['kind'] = prop_dict['ddt_type'] del prop_dict['ddt_type'] self.__intrinsic = False From 13b6363f54788d3ba7c94f3dff4f23c9e302af3a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 22 Nov 2023 11:41:27 -0700 Subject: [PATCH 13/23] Add doctests for conditional function of Var class --- scripts/metavar.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/scripts/metavar.py b/scripts/metavar.py index cfe08743..ddb6affe 100755 --- a/scripts/metavar.py +++ b/scripts/metavar.py @@ -944,7 +944,20 @@ def conditional(self, vdicts): """Convert conditional expression from active attribute (i.e. in standard name format) to local names based on vdict. Return conditional and a list of variables needed to evaluate - the conditional.""" + the conditional. + >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).conditional([{}]) + ('.true.', []) + >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'active' : 'False'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).conditional([VarDictionary('bar', _MVAR_DUMMY_RUN_ENV, variables={})]) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + Exception: Cannot find variable 'false' for generating conditional for 'False' + >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'active' : 'mom_gone'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).conditional([ VarDictionary('bar', _MVAR_DUMMY_RUN_ENV, variables=[Var({'local_name' : 'bar', 'standard_name' : 'mom_home', 'units' : '', 'dimensions' : '()', 'type' : 'logical'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV)]) ]) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + Exception: Cannot find variable 'mom_gone' for generating conditional for 'mom_gone' + >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'active' : 'mom_home'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).conditional([ VarDictionary('bar', _MVAR_DUMMY_RUN_ENV, variables=[Var({'local_name' : 'bar', 'standard_name' : 'mom_home', 'units' : '', 'dimensions' : '()', 'type' : 'logical'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV)]) ])[0] + 'bar' + >>> len(Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'active' : 'mom_home'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).conditional([ VarDictionary('bar', _MVAR_DUMMY_RUN_ENV, variables=[Var({'local_name' : 'bar', 'standard_name' : 'mom_home', 'units' : '', 'dimensions' : '()', 'type' : 'logical'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV)]) ])[1]) + 1 + """ active = self.get_prop_value('active') conditional = '' @@ -963,12 +976,13 @@ def conditional(self, vdicts): int(item) conditional += item except ValueError: + dvar = None for vdict in vdicts: dvar = vdict.find_variable(standard_name=item, any_scope=True) # or any_scope=False ? if dvar: break if not dvar: - raise Exception(f"Cannot find variable '{item}' for generating conditional for '{active}") + raise Exception(f"Cannot find variable '{item}' for generating conditional for '{active}'") conditional += dvar.get_prop_value('local_name') vars_needed.append(dvar) return (conditional, vars_needed) From fddde47e9c6b99392bd98ad99a9da65ff173cf3b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 27 Nov 2023 20:59:24 -0700 Subject: [PATCH 14/23] Add docstring documentation for add_var_debug_check and write_var_debug_check in scripts/suite_objects.py --- scripts/suite_objects.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) mode change 100644 => 100755 scripts/suite_objects.py diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py old mode 100644 new mode 100755 index 2ba55802..6690cc9f --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1204,6 +1204,13 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): return scheme_mods def add_var_debug_check(self, var, new_dims): + """ Add a debug check for a given variable var (host model variable, + suite variable or group module variable) with dimensions new_dims + for this scheme. Return a clone of the variable with these dimensions + and a dummy variable that managed by the group subroutine that calls the + scheme, which is used to assign the scalar or the lower and upper bounds + of the array to if the intent is 'inout' or 'out'. + """ # Get the basic attributes that decide whether we need # to check the variable when we write the group standard_name = var.get_prop_value('standard_name') @@ -1234,7 +1241,7 @@ def add_var_debug_check(self, var, new_dims): # that we can assign the scalar or the lbound/ubound of the array to. # We need to treat DDTs and variables with kind attributes slightly # differently, and make sure there are no duplicate variables. We - # also need to assign a bogus standard name to these local variables. # DH* do we? + # also need to assign a bogus standard name to these local variables. vtype = var.get_prop_value('type') if var.is_ddt(): vkind = '' @@ -1285,6 +1292,11 @@ def add_var_debug_check(self, var, new_dims): self.__var_debug_checks.append([clone, dummy]) def write_var_debug_check(self, var, dummy, cldicts, outfile, errcode, errmsg, indent): + """Write the variable debug check for the given variable, as determined + in a previous step (add_var_debug_check). Assign the scalar or lower and + upper bounds of the array to the dummy variable, and for arrays also check + that the size of the array matches the dimensions from the metadata. + """ # Get the basic attributes for writing the check standard_name = var.get_prop_value('standard_name') dimensions = var.get_dimensions() From b4f108b0f933abde128dac84e74f9458abedd264 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 29 Nov 2023 16:35:04 -0700 Subject: [PATCH 15/23] Simplify logic for variable debug checks in suite_objects.py --- scripts/suite_objects.py | 156 ++++++++++++++++++++++----------------- 1 file changed, 89 insertions(+), 67 deletions(-) diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index a55761a9..92e55e57 100755 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1148,12 +1148,11 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): args = self.match_variable(var, vstdname=vstdname, vdims=vdims) found, dict_var, vert_dim, new_dims, missing_vert = args if found: - if self.__group.run_env.debug: - # Add variable allocation checks for host model variables only + # Add variable allocation checks for group, suite, and host model variables gvar = self.__group.find_variable(standard_name=vstdname, any_scope=False) if dict_var and not gvar: - self.add_var_debug_check(dict_var, new_dims) + self.add_var_debug_check(dict_var) # end if if not self.has_vertical_dim: self.__has_vertical_dimension = vert_dim is not None @@ -1219,23 +1218,19 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): # end if return scheme_mods - def add_var_debug_check(self, var, new_dims): - """ Add a debug check for a given variable var (host model variable, - suite variable or group module variable) with dimensions new_dims - for this scheme. Return a clone of the variable with these dimensions - and a dummy variable that managed by the group subroutine that calls the - scheme, which is used to assign the scalar or the lower and upper bounds + def add_var_debug_check(self, var): + """Add a debug check for a given variable var (host model variable, + suite variable or group module variable) for this scheme. + Return the variable and an associated dummy variable that is + managed by the group subroutine that calls the scheme, and + which is used to assign the scalar or the lower and upper bounds of the array to if the intent is 'inout' or 'out'. """ # Get the basic attributes that decide whether we need # to check the variable when we write the group standard_name = var.get_prop_value('standard_name') - # We need to use the new dimensions as determined - # by the scheme's analyze logic, not the original - # variable dimension of the host variable var. - dimensions = new_dims + dimensions = var.get_dimensions() active = var.get_prop_value('active') - # The order is important to get the correct local name - DH* not sure ... var_dicts = [ self.__group.call_list ] + self.__group.suite_dicts() # If the variable isn't active, skip it @@ -1301,11 +1296,56 @@ def add_var_debug_check(self, var, new_dims): raise Exception(f"No dimension with standard name '{udim}'") self.update_group_call_list_variable(udim_var) - # Add the modified host model variable (with new dimension) - # to the list of variables to check. Record which dummy to use. - subst_dict = {'dimensions':new_dims} - clone = var.clone(subst_dict) - self.__var_debug_checks.append([clone, dummy]) + # Add the variable to the list of variables to check. Record which dummy to use. + self.__var_debug_checks.append([var, dummy]) + + def replace_horiz_dim_debug_check(self, dim, cldicts, var_in_call_list): + """Determine the correct horizontal dimension to use for a given variable, + depending on the CCPP phase and origin of the variable (from the host/suite + or defined as a module variable for the parent group. Return the dimension + length and other properties needed for the variable debug checks.""" + if not is_horizontal_dimension(dim): + raise Exception(f"Dimension {dim} is not a horizontal dimension") + if self.run_phase(): + if self.find_variable(standard_name="horizontal_loop_extent"): + ldim = "ccpp_constant_one" + udim = "horizontal_loop_extent" + else: + ldim = "horizontal_loop_begin" + udim = "horizontal_loop_end" + else: + ldim = "ccpp_constant_one" + udim = "horizontal_dimension" + # Get dimension for lower bound + for var_dict in cldicts: + dvar = var_dict.find_variable(standard_name=ldim, any_scope=False) + if dvar is not None: + break + if not dvar: + raise Exception(f"No variable with standard name '{ldim}' in cldicts") + ldim_lname = dvar.get_prop_value('local_name') + # Get dimension for upper bound + for var_dict in cldicts: + dvar = var_dict.find_variable(standard_name=udim, any_scope=False) + if dvar is not None: + break + if not dvar: + raise Exception(f"No variable with standard name '{udim}' in cldicts") + udim_lname = dvar.get_prop_value('local_name') + # Assemble dimensions and bounds for size checking + dim_length = f'{udim_lname}-{ldim_lname}+1' + # If the variable that uses these dimensions is not in the group's call + # list, then it is defined as a module variable for this group and the + # dimensions run from ldim to udim, otherwise from 1:dim_length. + if not var_in_call_list: + dim_string = f"{ldim_lname}:{udim_lname}" + lbound_string = ldim_lname + ubound_string = udim_lname + else: + dim_string = ":" + lbound_string = '1' + ubound_string = f'{udim_lname}-{ldim_lname}+1' + return (dim_length, dim_string, lbound_string, ubound_string) def write_var_debug_check(self, var, dummy, cldicts, outfile, errcode, errmsg, indent): """Write the variable debug check for the given variable, as determined @@ -1378,57 +1418,39 @@ def write_var_debug_check(self, var, dummy, cldicts, outfile, errcode, errmsg, i lbound_strings.append(dim_lname) ubound_strings.append(dim_lname) else: - # I don't know how to do this better. Schemes can rely on the host cap - # passing arrays such that the horizontal dimension of the variable - # seen by the scheme runs from 1:ncol (horizontal_loop_extent). But - # module variables for this group are passed to the schemes with the - # horizontal dimensions in the call dimstring. And it all depends - # on the phase, too. + # Horizontal dimension needs to be dealt with separately, because it + # depends on the CCPP phase, whether the variable is a host/suite + # variable or locally defined on the group level. if is_horizontal_dimension(dim): - if self.run_phase(): - if self.find_variable(standard_name="horizontal_loop_extent"): - ldim = "ccpp_constant_one" - udim = "horizontal_loop_extent" - else: - ldim = "horizontal_loop_begin" - udim = "horizontal_loop_end" - else: - ldim = "ccpp_constant_one" - udim = "horizontal_dimension" + (dim_length, dim_string, lbound_string, ubound_string) = \ + self.replace_horiz_dim_debug_check(dim, cldicts, var_in_call_list) else: (ldim, udim) = dim.split(":") - # Get dimension for lower bound - for var_dict in cldicts: - dvar = var_dict.find_variable(standard_name=ldim, any_scope=False) - if dvar is not None: - break - if not dvar: - raise Exception(f"No variable with standard name '{ldim}' in cldicts") - ldim_lname = dvar.get_prop_value('local_name') - # Get dimension for upper bound - for var_dict in cldicts: - dvar = var_dict.find_variable(standard_name=udim, any_scope=False) - if dvar is not None: - break - if not dvar: - raise Exception(f"No variable with standard name '{udim}' in cldicts") - udim_lname = dvar.get_prop_value('local_name') - # Assemble dimensions and bounds for size checking - dim_length = f'{udim_lname}-{ldim_lname}+1' - if is_horizontal_dimension(dim): - # See comment above on call list variables vs module variables - if not var_in_call_list: - dim_strings.append(f"{ldim_lname}:{udim_lname}") - lbound_strings.append(ldim_lname) - ubound_strings.append(udim_lname) - else: - dim_strings.append(":") - lbound_strings.append('1') - ubound_strings.append(f'{udim_lname}-{ldim_lname}+1') - else: - dim_strings.append(":") - lbound_strings.append(ldim_lname) - ubound_strings.append(udim_lname) + # Get dimension for lower bound + for var_dict in cldicts: + dvar = var_dict.find_variable(standard_name=ldim, any_scope=False) + if dvar is not None: + break + if not dvar: + raise Exception(f"No variable with standard name '{ldim}' in cldicts") + ldim_lname = dvar.get_prop_value('local_name') + # Get dimension for upper bound + for var_dict in cldicts: + dvar = var_dict.find_variable(standard_name=udim, any_scope=False) + if dvar is not None: + break + if not dvar: + raise Exception(f"No variable with standard name '{udim}' in cldicts") + udim_lname = dvar.get_prop_value('local_name') + # Assemble dimensions and bounds for size checking + dim_length = f'{udim_lname}-{ldim_lname}+1' + dim_string = ":" + lbound_string = ldim_lname + ubound_string = udim_lname + # end if + dim_strings.append(dim_string) + lbound_strings.append(lbound_string) + ubound_strings.append(ubound_string) array_size = f'{array_size}*({dim_length})' # Various strings needed to get the right size From 2e72cc8809c10f0adac19b4677974bee0fdaee05 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 29 Nov 2023 16:36:36 -0700 Subject: [PATCH 16/23] Update variable lists for var_action_test --- test/var_action_test/run_test | 2 ++ test/var_action_test/test_reports.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/var_action_test/run_test b/test/var_action_test/run_test index 2b4db0ac..dbfe9a7b 100755 --- a/test/var_action_test/run_test +++ b/test/var_action_test/run_test @@ -135,12 +135,14 @@ required_vars_var_action="${required_vars_var_action},effective_radius_of_strati required_vars_var_action="${required_vars_var_action},effective_radius_of_stratiform_cloud_liquid_water_particle" required_vars_var_action="${required_vars_var_action},effective_radius_of_stratiform_cloud_rain_particle" required_vars_var_action="${required_vars_var_action},effective_radius_of_stratiform_cloud_snow_particle" +required_vars_var_action="${required_vars_var_action},horizontal_dimension" required_vars_var_action="${required_vars_var_action},horizontal_loop_begin" required_vars_var_action="${required_vars_var_action},horizontal_loop_end" required_vars_var_action="${required_vars_var_action},vertical_layer_dimension" input_vars_var_action="effective_radius_of_stratiform_cloud_liquid_water_particle" input_vars_var_action="${input_vars_var_action},effective_radius_of_stratiform_cloud_rain_particle" input_vars_var_action="${input_vars_var_action},effective_radius_of_stratiform_cloud_snow_particle" +input_vars_var_action="${input_vars_var_action},horizontal_dimension" input_vars_var_action="${input_vars_var_action},horizontal_loop_begin" input_vars_var_action="${input_vars_var_action},horizontal_loop_end" input_vars_var_action="${input_vars_var_action},vertical_layer_dimension" diff --git a/test/var_action_test/test_reports.py b/test/var_action_test/test_reports.py index fff8603d..152fda97 100755 --- a/test/var_action_test/test_reports.py +++ b/test/var_action_test/test_reports.py @@ -67,7 +67,7 @@ def usage(errmsg=None): os.path.join(_BUILD_DIR, "ccpp", "ccpp_var_action_suite_cap.F90")] _MODULE_LIST = ["effr_calc"] _SUITE_LIST = ["var_action_suite"] -_INPUT_VARS_VAR_ACTION = ["horizontal_loop_begin", "horizontal_loop_end", "vertical_layer_dimension", +_INPUT_VARS_VAR_ACTION = ["horizontal_loop_begin", "horizontal_loop_end", "horizontal_dimension", "vertical_layer_dimension", "effective_radius_of_stratiform_cloud_liquid_water_particle", "effective_radius_of_stratiform_cloud_rain_particle", "effective_radius_of_stratiform_cloud_snow_particle"] From 3a4c05646c38b8dc0a4ae6bc8eefec512091e7e3 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Dec 2023 08:19:18 -0700 Subject: [PATCH 17/23] Update scripts/framework_env.py Formatting Co-authored-by: mwaxmonsky <137746677+mwaxmonsky@users.noreply.github.com> --- scripts/framework_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/framework_env.py b/scripts/framework_env.py index 6d0b98cb..88c9c204 100644 --- a/scripts/framework_env.py +++ b/scripts/framework_env.py @@ -26,7 +26,7 @@ def __init__(self, logger, ndict=None, verbose=0, clean=False, preproc_directives=[], generate_docfiles=False, host_name='', kind_types=[], use_error_obj=False, force_overwrite=False, output_root=os.getcwd(), ccpp_datafile="datatable.xml", - debug = False): + debug=False): """Initialize a new CCPPFrameworkEnv object from the input arguments. is a dict with the parsed command-line arguments (or a dictionary created with the necessary arguments). From 464f62a698f00d369cf84386c4fb99ba58a221f4 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Dec 2023 09:36:20 -0700 Subject: [PATCH 18/23] Fix wrong comment about group variables in scripts/suite_objects.py --- scripts/suite_objects.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index 92e55e57..898d4a0a 100755 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1149,7 +1149,7 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): found, dict_var, vert_dim, new_dims, missing_vert = args if found: if self.__group.run_env.debug: - # Add variable allocation checks for group, suite, and host model variables + # Add variable allocation checks for suite and host model variables only gvar = self.__group.find_variable(standard_name=vstdname, any_scope=False) if dict_var and not gvar: self.add_var_debug_check(dict_var) From dfa59eb2b40acb0882ab8614735af10b50933d75 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Dec 2023 09:58:08 -0700 Subject: [PATCH 19/23] In scripts/suite_objects.py: rename 'dummy' variables for var_debug_check to 'internal_var' variables, and avoid promoting these variables to the module (suite) level --- scripts/suite_objects.py | 46 ++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index 898d4a0a..313654c7 100755 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1248,7 +1248,7 @@ def add_var_debug_check(self, var): for var_needed in vars_needed: self.update_group_call_list_variable(var_needed) - # For scalars and arrays, need a dummy variable (same kind and type) + # For scalars and arrays, need an internal_var variable (same kind and type) # that we can assign the scalar or the lbound/ubound of the array to. # We need to treat DDTs and variables with kind attributes slightly # differently, and make sure there are no duplicate variables. We @@ -1261,20 +1261,20 @@ def add_var_debug_check(self, var): vkind = var.get_prop_value('kind') units = var.get_prop_value('units') if vkind: - dummy_lname = f'dummy_{vtype.replace("=","_")}_{vkind.replace("=","_")}' + internal_var_lname = f'internal_var_{vtype.replace("=","_")}_{vkind.replace("=","_")}' else: - dummy_lname = f'dummy_{vtype.replace("=","_")}' + internal_var_lname = f'internal_var_{vtype.replace("=","_")}' if var.is_ddt(): - dummy = Var({'local_name':dummy_lname, 'standard_name':f'{dummy_lname}_local', + internal_var = Var({'local_name':internal_var_lname, 'standard_name':f'{internal_var_lname}_local', 'ddt_type':vtype, 'kind':vkind, 'units':units, 'dimensions':'()'}, _API_LOCAL, self.run_env) else: - dummy = Var({'local_name':dummy_lname, 'standard_name':f'{dummy_lname}_local', + internal_var = Var({'local_name':internal_var_lname, 'standard_name':f'{internal_var_lname}_local', 'type':vtype, 'kind':vkind, 'units':units, 'dimensions':'()'}, _API_LOCAL, self.run_env) - found = self.__group.find_variable(source_var=dummy) + found = self.__group.find_variable(source_var=internal_var, any_scope=False) if not found: - self.__group.manage_variable(dummy) + self.__group.manage_variable(internal_var) # For arrays, we need to get information on the dimensions and add it to # the group's call list so that we can test for the correct size later on @@ -1296,8 +1296,8 @@ def add_var_debug_check(self, var): raise Exception(f"No dimension with standard name '{udim}'") self.update_group_call_list_variable(udim_var) - # Add the variable to the list of variables to check. Record which dummy to use. - self.__var_debug_checks.append([var, dummy]) + # Add the variable to the list of variables to check. Record which internal_var to use. + self.__var_debug_checks.append([var, internal_var]) def replace_horiz_dim_debug_check(self, dim, cldicts, var_in_call_list): """Determine the correct horizontal dimension to use for a given variable, @@ -1347,10 +1347,10 @@ def replace_horiz_dim_debug_check(self, dim, cldicts, var_in_call_list): ubound_string = f'{udim_lname}-{ldim_lname}+1' return (dim_length, dim_string, lbound_string, ubound_string) - def write_var_debug_check(self, var, dummy, cldicts, outfile, errcode, errmsg, indent): + def write_var_debug_check(self, var, internal_var, cldicts, outfile, errcode, errmsg, indent): """Write the variable debug check for the given variable, as determined in a previous step (add_var_debug_check). Assign the scalar or lower and - upper bounds of the array to the dummy variable, and for arrays also check + upper bounds of the array to the internal_var variable, and for arrays also check that the size of the array matches the dimensions from the metadata. """ # Get the basic attributes for writing the check @@ -1387,16 +1387,16 @@ def write_var_debug_check(self, var, dummy, cldicts, outfile, errcode, errmsg, i # Get the condition on which the variable is active (conditional, _) = var.conditional(cldicts) - # For scalars, assign to dummy variable if the variable intent is in/inout + # For scalars, assign to internal_var variable if the variable intent is in/inout if not dimensions: if not intent == 'out': - dummy_lname = dummy.get_prop_value('local_name') + internal_var_lname = internal_var.get_prop_value('local_name') outfile.write(f"if ({conditional}) then", indent) - outfile.write(f"! Assign value of {local_name} to dummy", indent+1) - outfile.write(f"{dummy_lname} = {local_name}", indent+1) + outfile.write(f"! Assign value of {local_name} to internal_var", indent+1) + outfile.write(f"{internal_var_lname} = {local_name}", indent+1) outfile.write(f"end if", indent) # For arrays, check size of array against dimensions in metadata, then assign - # the lower and upper bounds to the dummy variable if the intent is in/inout + # the lower and upper bounds to the internal_var variable if the intent is in/inout else: array_size = 1 dim_strings = [] @@ -1470,13 +1470,13 @@ def write_var_debug_check(self, var, dummy, cldicts, outfile, errcode, errmsg, i outfile.write(f"end if", indent+1) outfile.write(f"end if", indent) - # Assign lower/upper bounds to dummy (scalar) if intent is not out + # Assign lower/upper bounds to internal_var (scalar) if intent is not out if not intent == 'out': - dummy_lname = dummy.get_prop_value('local_name') + internal_var_lname = internal_var.get_prop_value('local_name') outfile.write(f"if ({conditional}) then", indent) - outfile.write(f"! Assign lower/upper bounds of {local_name} to dummy", indent+1) - outfile.write(f"{dummy_lname} = {local_name}{lbound_string}", indent+1) - outfile.write(f"{dummy_lname} = {local_name}{ubound_string}", indent+1) + outfile.write(f"! Assign lower/upper bounds of {local_name} to internal_var", indent+1) + outfile.write(f"{internal_var_lname} = {local_name}{lbound_string}", indent+1) + outfile.write(f"{internal_var_lname} = {local_name}{ubound_string}", indent+1) outfile.write(f"end if", indent) def write(self, outfile, errcode, errmsg, indent): @@ -1493,8 +1493,8 @@ def write(self, outfile, errcode, errmsg, indent): # Write debug checks (operating on variables # coming from the group's call list) - for (var, dummy) in self.__var_debug_checks: - stmt = self.write_var_debug_check(var, dummy, cldicts, outfile, errcode, errmsg, indent) + for (var, internal_var) in self.__var_debug_checks: + stmt = self.write_var_debug_check(var, internal_var, cldicts, outfile, errcode, errmsg, indent) # Write variable transformations (to be developed) From 7367f985e54692d352aa28cb7df9f9fc0c303d21 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 14 Dec 2023 20:48:04 -0700 Subject: [PATCH 20/23] Bug fix in scripts/suite_objects.py: also check variable allocations for variables local to the group and assign correct dimensions for local (group) and module (suite) variables --- scripts/suite_objects.py | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) mode change 100755 => 100644 scripts/suite_objects.py diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py old mode 100755 new mode 100644 index 313654c7..aa0a5519 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1149,9 +1149,8 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): found, dict_var, vert_dim, new_dims, missing_vert = args if found: if self.__group.run_env.debug: - # Add variable allocation checks for suite and host model variables only - gvar = self.__group.find_variable(standard_name=vstdname, any_scope=False) - if dict_var and not gvar: + # Add variable allocation checks for group, suite and host variables + if dict_var: self.add_var_debug_check(dict_var) # end if if not self.has_vertical_dim: @@ -1302,12 +1301,13 @@ def add_var_debug_check(self, var): def replace_horiz_dim_debug_check(self, dim, cldicts, var_in_call_list): """Determine the correct horizontal dimension to use for a given variable, depending on the CCPP phase and origin of the variable (from the host/suite - or defined as a module variable for the parent group. Return the dimension - length and other properties needed for the variable debug checks.""" + or defined as a module variable for the parent group, or local to the group. + Return the dimension length and other properties needed for debug checks.""" if not is_horizontal_dimension(dim): raise Exception(f"Dimension {dim} is not a horizontal dimension") if self.run_phase(): - if self.find_variable(standard_name="horizontal_loop_extent"): + if var_in_call_list and \ + self.find_variable(standard_name="horizontal_loop_extent"): ldim = "ccpp_constant_one" udim = "horizontal_loop_extent" else: @@ -1359,17 +1359,27 @@ def write_var_debug_check(self, var, internal_var, cldicts, outfile, errcode, er active = var.get_prop_value('active') allocatable = var.get_prop_value('allocatable') - # Need the local name as it comes from the group call list - # or from the suite, not how it is called in the scheme (var) + # Need the local name from the group call list, + # from the locally-defined variables of the group, + # or from the suite, not how it is called in the scheme (var) + # First, check if the variable is in the call list. dvar = self.__group.call_list.find_variable(standard_name=standard_name, any_scope=False) if dvar: var_in_call_list = True else: var_in_call_list = False - for var_dict in self.__group.suite_dicts(): - dvar = var_dict.find_variable(standard_name=standard_name) - if dvar: - break + # If it is not in the call list, try to find it + # in the local variables of this group subroutine. + dvar = self.__group.find_variable(standard_name=standard_name, any_scope=False) + if dvar: + print(f"DOM DEBUG - found the variable in the LOCAL GROUP!") + else: + # This variable is handled by the group + # and is declared as a module variable + for var_dict in self.__group.suite_dicts(): + dvar = var_dict.find_variable(standard_name=standard_name, any_scope=False) + if dvar: + break if not dvar: raise Exception(f"No variable with standard name '{standard_name}' in cldicts") local_name = dvar.get_prop_value('local_name') @@ -1379,7 +1389,7 @@ def write_var_debug_check(self, var, internal_var, cldicts, outfile, errcode, er # the variable. We don't have this information earlier in # add_var_debug_check, therefore need to back out here, # using the information from the scheme variable (call list). - svar = self.call_list.find_variable(standard_name=standard_name) + svar = self.call_list.find_variable(standard_name=standard_name, any_scope=False) intent = svar.get_prop_value('intent') if intent == 'out' and allocatable: return From fc6fec3374870b95647e8ed2e85c7b73a45dfee5 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 14 Dec 2023 20:55:43 -0700 Subject: [PATCH 21/23] Remove stray debugging statement in scripts/suite_objects.py --- scripts/suite_objects.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) mode change 100644 => 100755 scripts/suite_objects.py diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py old mode 100644 new mode 100755 index aa0a5519..26690498 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1371,9 +1371,7 @@ def write_var_debug_check(self, var, internal_var, cldicts, outfile, errcode, er # If it is not in the call list, try to find it # in the local variables of this group subroutine. dvar = self.__group.find_variable(standard_name=standard_name, any_scope=False) - if dvar: - print(f"DOM DEBUG - found the variable in the LOCAL GROUP!") - else: + if not dvar: # This variable is handled by the group # and is declared as a module variable for var_dict in self.__group.suite_dicts(): From 70ae2b0568a3353c81cee3b72b721d353ad0c27c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 18 Jan 2024 15:48:40 -0700 Subject: [PATCH 22/23] Fix spelling: compatability --> compatibility --- test/run_fortran_tests.sh | 6 +++--- .../.gitignore | 0 .../CMakeLists.txt | 4 ++-- .../README.md | 4 ++-- .../effr_calc.F90 | 0 .../effr_calc.meta | 0 .../run_test | 14 +++++++------- .../test_host.F90 | 2 +- .../test_host.meta | 0 .../test_host_data.F90 | 0 .../test_host_data.meta | 0 .../test_host_mod.F90 | 0 .../test_host_mod.meta | 0 .../test_reports.py | 14 +++++++------- .../var_compatibility_files.txt} | 0 .../var_compatibility_suite.xml} | 2 +- 16 files changed, 23 insertions(+), 23 deletions(-) rename test/{var_compatability_test => var_compatibility_test}/.gitignore (100%) rename test/{var_compatability_test => var_compatibility_test}/CMakeLists.txt (98%) rename test/{var_compatability_test => var_compatibility_test}/README.md (58%) rename test/{var_compatability_test => var_compatibility_test}/effr_calc.F90 (100%) rename test/{var_compatability_test => var_compatibility_test}/effr_calc.meta (100%) rename test/{var_compatability_test => var_compatibility_test}/run_test (95%) rename test/{var_compatability_test => var_compatibility_test}/test_host.F90 (99%) rename test/{var_compatability_test => var_compatibility_test}/test_host.meta (100%) rename test/{var_compatability_test => var_compatibility_test}/test_host_data.F90 (100%) rename test/{var_compatability_test => var_compatibility_test}/test_host_data.meta (100%) rename test/{var_compatability_test => var_compatibility_test}/test_host_mod.F90 (100%) rename test/{var_compatability_test => var_compatibility_test}/test_host_mod.meta (100%) rename test/{var_compatability_test => var_compatibility_test}/test_reports.py (96%) rename test/{var_compatability_test/var_compatability_files.txt => var_compatibility_test/var_compatibility_files.txt} (100%) rename test/{var_compatability_test/var_compatability_suite.xml => var_compatibility_test/var_compatibility_suite.xml} (69%) diff --git a/test/run_fortran_tests.sh b/test/run_fortran_tests.sh index cd6436fb..ccfc85f9 100755 --- a/test/run_fortran_tests.sh +++ b/test/run_fortran_tests.sh @@ -37,12 +37,12 @@ if [ $res -ne 0 ]; then echo "Failure running advection test" fi -# Run var_compatability test - ./var_compatability_test/run_test +# Run var_compatibility test + ./var_compatibility_test/run_test res=$? errcnt=$((errcnt + res)) if [ $res -ne 0 ]; then - echo "Failure running var_compatability test" + echo "Failure running var_compatibility test" fi if [ $errcnt -eq 0 ]; then diff --git a/test/var_compatability_test/.gitignore b/test/var_compatibility_test/.gitignore similarity index 100% rename from test/var_compatability_test/.gitignore rename to test/var_compatibility_test/.gitignore diff --git a/test/var_compatability_test/CMakeLists.txt b/test/var_compatibility_test/CMakeLists.txt similarity index 98% rename from test/var_compatability_test/CMakeLists.txt rename to test/var_compatibility_test/CMakeLists.txt index 11899446..8cbd7e44 100644 --- a/test/var_compatability_test/CMakeLists.txt +++ b/test/var_compatibility_test/CMakeLists.txt @@ -19,9 +19,9 @@ get_filename_component(CCPP_ROOT "${TEST_ROOT}" DIRECTORY) # Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) # #------------------------------------------------------------------------------ -LIST(APPEND SCHEME_FILES "var_compatability_files.txt") +LIST(APPEND SCHEME_FILES "var_compatibility_files.txt") LIST(APPEND HOST_FILES "test_host_data" "test_host_mod") -LIST(APPEND SUITE_FILES "var_compatability_suite.xml") +LIST(APPEND SUITE_FILES "var_compatibility_suite.xml") # HOST is the name of the executable we will build. # We assume there are files ${HOST}.meta and ${HOST}.F90 in CMAKE_SOURCE_DIR SET(HOST "${CMAKE_PROJECT_NAME}") diff --git a/test/var_compatability_test/README.md b/test/var_compatibility_test/README.md similarity index 58% rename from test/var_compatability_test/README.md rename to test/var_compatibility_test/README.md index 066cf771..9b56ec9c 100644 --- a/test/var_compatability_test/README.md +++ b/test/var_compatibility_test/README.md @@ -1,6 +1,6 @@ -var_compatability test +var_compatibility test ================ -To build and run the var_compatability test, run ./run_test +To build and run the var_compatibility test, run ./run_test This script will build and run the test. The exit code is zero (0) on PASS and non-zero on FAIL. diff --git a/test/var_compatability_test/effr_calc.F90 b/test/var_compatibility_test/effr_calc.F90 similarity index 100% rename from test/var_compatability_test/effr_calc.F90 rename to test/var_compatibility_test/effr_calc.F90 diff --git a/test/var_compatability_test/effr_calc.meta b/test/var_compatibility_test/effr_calc.meta similarity index 100% rename from test/var_compatability_test/effr_calc.meta rename to test/var_compatibility_test/effr_calc.meta diff --git a/test/var_compatability_test/run_test b/test/var_compatibility_test/run_test similarity index 95% rename from test/var_compatability_test/run_test rename to test/var_compatibility_test/run_test index 548a34b1..ac86bf6f 100755 --- a/test/var_compatability_test/run_test +++ b/test/var_compatibility_test/run_test @@ -118,18 +118,18 @@ frame_src="${framework}/src" ## NB: This has to be after build_dir is finalized ## host_files="${build_dir}/ccpp/test_host_ccpp_cap.F90" -suite_files="${build_dir}/ccpp/ccpp_var_compatability_suite_cap.F90" +suite_files="${build_dir}/ccpp/ccpp_var_compatibility_suite_cap.F90" utility_files="${build_dir}/ccpp/ccpp_kinds.F90" utility_files="${utility_files},${frame_src}/ccpp_constituent_prop_mod.F90" utility_files="${utility_files},${frame_src}/ccpp_hashable.F90" utility_files="${utility_files},${frame_src}/ccpp_hash_table.F90" ccpp_files="${utility_files}" ccpp_files="${ccpp_files},${build_dir}/ccpp/test_host_ccpp_cap.F90" -ccpp_files="${ccpp_files},${build_dir}/ccpp/ccpp_var_compatability_suite_cap.F90" +ccpp_files="${ccpp_files},${build_dir}/ccpp/ccpp_var_compatibility_suite_cap.F90" #process_list="" module_list="effr_calc" #dependencies="" -suite_list="var_compatability_suite" +suite_list="var_compatibility_suite" required_vars_var_compatibility="ccpp_error_code,ccpp_error_message" required_vars_var_compatibility="${required_vars_var_compatibility},effective_radius_of_stratiform_cloud_ice_particle" required_vars_var_compatibility="${required_vars_var_compatibility},effective_radius_of_stratiform_cloud_liquid_water_particle" @@ -217,13 +217,13 @@ check_datatable ${report_prog} ${datafile} "--module-list" ${module_list} #check_datatable ${report_prog} ${datafile} "--dependencies" ${dependencies} check_datatable ${report_prog} ${datafile} "--suite-list" ${suite_list} \ --sep ";" -echo -e "\nChecking variables for var_compatability suite from command line" +echo -e "\nChecking variables for var_compatibility suite from command line" check_datatable ${report_prog} ${datafile} "--required-variables" \ - ${required_vars_var_compatability} "var_compatability_suite" + ${required_vars_var_compatibility} "var_compatibility_suite" check_datatable ${report_prog} ${datafile} "--input-variables" \ - ${input_vars_var_compatability} "var_compatability_suite" + ${input_vars_var_compatibility} "var_compatibility_suite" check_datatable ${report_prog} ${datafile} "--output-variables" \ - ${output_vars_var_compatability} "var_compatability_suite" + ${output_vars_var_compatibility} "var_compatibility_suite" # Run make make res=$? diff --git a/test/var_compatability_test/test_host.F90 b/test/var_compatibility_test/test_host.F90 similarity index 99% rename from test/var_compatability_test/test_host.F90 rename to test/var_compatibility_test/test_host.F90 index 6de9597c..25129405 100644 --- a/test/var_compatability_test/test_host.F90 +++ b/test/var_compatibility_test/test_host.F90 @@ -375,7 +375,7 @@ program test logical :: run_okay ! Setup expected test suite info - test_suites(1)%suite_name = 'var_compatability_suite' + test_suites(1)%suite_name = 'var_compatibility_suite' test_suites(1)%suite_parts => test_parts1 test_suites(1)%suite_input_vars => test_invars1 test_suites(1)%suite_output_vars => test_outvars1 diff --git a/test/var_compatability_test/test_host.meta b/test/var_compatibility_test/test_host.meta similarity index 100% rename from test/var_compatability_test/test_host.meta rename to test/var_compatibility_test/test_host.meta diff --git a/test/var_compatability_test/test_host_data.F90 b/test/var_compatibility_test/test_host_data.F90 similarity index 100% rename from test/var_compatability_test/test_host_data.F90 rename to test/var_compatibility_test/test_host_data.F90 diff --git a/test/var_compatability_test/test_host_data.meta b/test/var_compatibility_test/test_host_data.meta similarity index 100% rename from test/var_compatability_test/test_host_data.meta rename to test/var_compatibility_test/test_host_data.meta diff --git a/test/var_compatability_test/test_host_mod.F90 b/test/var_compatibility_test/test_host_mod.F90 similarity index 100% rename from test/var_compatability_test/test_host_mod.F90 rename to test/var_compatibility_test/test_host_mod.F90 diff --git a/test/var_compatability_test/test_host_mod.meta b/test/var_compatibility_test/test_host_mod.meta similarity index 100% rename from test/var_compatability_test/test_host_mod.meta rename to test/var_compatibility_test/test_host_mod.meta diff --git a/test/var_compatability_test/test_reports.py b/test/var_compatibility_test/test_reports.py similarity index 96% rename from test/var_compatability_test/test_reports.py rename to test/var_compatibility_test/test_reports.py index 33134574..4c8f9da6 100755 --- a/test/var_compatability_test/test_reports.py +++ b/test/var_compatibility_test/test_reports.py @@ -57,16 +57,16 @@ def usage(errmsg=None): # Check data _HOST_FILES = [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90")] -_SUITE_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_var_compatability_suite_cap.F90")] +_SUITE_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_var_compatibility_suite_cap.F90")] _UTILITY_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_kinds.F90"), os.path.join(_SRC_DIR, "ccpp_constituent_prop_mod.F90"), os.path.join(_SRC_DIR, "ccpp_hashable.F90"), os.path.join(_SRC_DIR, "ccpp_hash_table.F90")] _CCPP_FILES = _UTILITY_FILES + \ [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90"), - os.path.join(_BUILD_DIR, "ccpp", "ccpp_var_compatability_suite_cap.F90")] + os.path.join(_BUILD_DIR, "ccpp", "ccpp_var_compatibility_suite_cap.F90")] _MODULE_LIST = ["effr_calc"] -_SUITE_LIST = ["var_compatability_suite"] +_SUITE_LIST = ["var_compatibility_suite"] _INPUT_VARS_VAR_ACTION = ["horizontal_loop_begin", "horizontal_loop_end", "horizontal_dimension", "vertical_layer_dimension", "effective_radius_of_stratiform_cloud_liquid_water_particle", "effective_radius_of_stratiform_cloud_rain_particle", @@ -140,15 +140,15 @@ def check_datatable(database, report_type, check_list, _MODULE_LIST) NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("suite_list"), _SUITE_LIST) -print("\nChecking variables for var_compatability suite from python") +print("\nChecking variables for var_compatibility suite from python") NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("required_variables", - value="var_compatability_suite"), + value="var_compatibility_suite"), _REQUIRED_VARS_VAR_ACTION) NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("input_variables", - value="var_compatability_suite"), + value="var_compatibility_suite"), _INPUT_VARS_VAR_ACTION) NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("output_variables", - value="var_compatability_suite"), + value="var_compatibility_suite"), _OUTPUT_VARS_VAR_ACTION) sys.exit(NUM_ERRORS) diff --git a/test/var_compatability_test/var_compatability_files.txt b/test/var_compatibility_test/var_compatibility_files.txt similarity index 100% rename from test/var_compatability_test/var_compatability_files.txt rename to test/var_compatibility_test/var_compatibility_files.txt diff --git a/test/var_compatability_test/var_compatability_suite.xml b/test/var_compatibility_test/var_compatibility_suite.xml similarity index 69% rename from test/var_compatability_test/var_compatability_suite.xml rename to test/var_compatibility_test/var_compatibility_suite.xml index ae75d9f7..4b70fe48 100644 --- a/test/var_compatability_test/var_compatability_suite.xml +++ b/test/var_compatibility_test/var_compatibility_suite.xml @@ -1,6 +1,6 @@ - + effr_calc From 63aee18fb8ce9c3da7d24210bf62ade9ed0075ac Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 18 Jan 2024 15:53:00 -0700 Subject: [PATCH 23/23] Add missing instruction in test_prebuild/test_blocked_data/README.md --- test_prebuild/test_blocked_data/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/test_prebuild/test_blocked_data/README.md b/test_prebuild/test_blocked_data/README.md index ad977913..8802e812 100644 --- a/test_prebuild/test_blocked_data/README.md +++ b/test_prebuild/test_blocked_data/README.md @@ -10,4 +10,5 @@ mkdir build cd build cmake .. 2>&1 | tee log.cmake make 2>&1 | tee log.make +./test_blocked_data.x ```