From c2b1f9b3118081c4d889552dc01174a85c758305 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Fri, 22 Sep 2023 16:05:42 -0600 Subject: [PATCH 01/38] Initial commit. Work in progress --- scripts/metavar.py | 34 ++++++++++++++++++++++++++++++++++ scripts/suite_objects.py | 35 ++++++++++++++++++++++++++++++++++- 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/scripts/metavar.py b/scripts/metavar.py index 71609580..f500f6e1 100755 --- a/scripts/metavar.py +++ b/scripts/metavar.py @@ -25,6 +25,7 @@ from var_props import CCPP_LOOP_DIM_SUBSTS, VariableProperty, VarCompatObj from var_props import find_horizontal_dimension, find_vertical_dimension from var_props import standard_name_to_long_name, default_kind_val +from conversion_tools import unit_conversion ############################################################################## @@ -1204,6 +1205,15 @@ def add_to_list(self, vlist): ############################################################################### +class VarUnitConv(VarAction): + """A class to perform unit conversions""" + + def __init__(self, set_action): + """Initialize unit conversion""" + self._set_action = set_action + +############################################################################### + class VarLoopSubst(VarAction): """A class to handle required loop substitutions where the host model (or a suite part) does not provide a loop-like variable used by a @@ -1823,6 +1833,30 @@ def __eq__(self, other): list equality""" return self is other + @classmethod + def unit_cnv_from(cls, hunits, var): + """Return VarUnitConv if and require unit + conversion, otherwise, return None""" + function_name = '{0}__to__{1}'.format(hunits,var.get_prop_value('units')) + try: + function = getattr(unit_conversion, function_name) + ucmatch = VarUnitConv(function()) + except: + ucmatch = None + return ucmatch + + @classmethod + def unit_cnv_to(cls, hunits, var): + """Return VarUnitConv if and require unit + conversion, otherwise, return None""" + function_name = '{1}__to__{0}'.format(hunits,var.get_prop_value('units')) + try: + function = getattr(unit_conversion, function_name) + ucmatch = VarUnitConv(function()) + except: + ucmatch = None + return ucmatch + @classmethod def loop_var_match(cls, standard_name): """Return a VarLoopSubst if is a loop variable, diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index 472556cb..fb36c122 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -13,7 +13,7 @@ from code_block import CodeBlock from constituents import ConstituentVarDict from framework_env import CCPPFrameworkEnv -from metavar import Var, VarDictionary, VarLoopSubst +from metavar import Var, VarDictionary, VarLoopSubst, VarUnitConv from metavar import CCPP_CONSTANT_VARS, CCPP_LOOP_VAR_STDNAMES from parse_tools import ParseContext, ParseSource, context_string from parse_tools import ParseInternalError, CCPPError @@ -800,6 +800,30 @@ def find_variable(self, standard_name=None, source_var=None, # end if return found_var + def match_units(self, var, title): + """Compare units for in this SuiteObject's dictionary tree. + If there are differences, apply unit conversions""" + + dict_var = self.find_variable(source_var=var, any_scope=True) + hunits = dict_var.get_prop_value('units') + found_var = False + if (hunits != var.get_prop_value('units')): + found_var = True + if self.run_phase(): + if (var.get_prop_value('intent') == 'out'): + match = VarDictionary.unit_cnv_to(hunits, var) + if (var.get_prop_value('intent') == 'in'): + match = VarDictionary.unit_cnv_from(hunits, var) + if (var.get_prop_value('intent') == 'inout'): + match = VarDictionary.unit_cnv_from(hunits, var) + match = VarDictionary.unit_cnv_to(hunits, var) + print(match._set_action) + else: + match = None + self.register_action(match) + + return found_var + def match_variable(self, var, vstdname=None, vdims=None): """Try to find a source for in this SuiteObject's dictionary tree. Several items are returned: @@ -820,6 +844,7 @@ def match_variable(self, var, vstdname=None, vdims=None): else: vmatch = None # end if + found_var = False missing_vert = None new_vdims = list() @@ -1184,6 +1209,8 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): vstdname)) # end if # end if + found = self.match_units(var,my_header.title) + # end for if self.needs_vertical is not None: self.parent.add_part(self, replace=True) # Should add a vloop @@ -1546,6 +1573,7 @@ def __init__(self, group_xml, transition, parent, context, run_env): self._host_ddts = None self._loop_var_matches = list() self._phase_check_stmts = list() + self._unit_cnv_matches = list() self._set_state = None self._ddt_library = None @@ -1582,6 +1610,11 @@ def register_action(self, vaction): # Add the missing dim vaction.add_local(self, _API_LOCAL, self.run_env) return True + + if isinstance(vaction, VarUnitConv): + self._unit_cnv_matches = vaction.add_to_list(self._unit_cnv_matches) + return True + # end if return False From a3353c76c26b6960dffd06b2516004fcfc46ee5a Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 10 Oct 2023 22:03:07 -0600 Subject: [PATCH 02/38] Working. --- scripts/metavar.py | 71 ++++++++++++++++++++------ scripts/suite_objects.py | 64 ++++++++++++----------- test/var_action_test/test_host_mod.F90 | 2 +- 3 files changed, 90 insertions(+), 47 deletions(-) diff --git a/scripts/metavar.py b/scripts/metavar.py index f500f6e1..24cbfc24 100755 --- a/scripts/metavar.py +++ b/scripts/metavar.py @@ -1185,7 +1185,7 @@ def equiv(self, vmatch): of the objects. equiv should be overridden with a method that first calls this method and then tests class-specific object data.""" - return vmatch.__class__ == self.__class__ + return vmatch.__class__ != self.__class__ def add_to_list(self, vlist): """Add to unless or its equivalent is @@ -1208,10 +1208,42 @@ def add_to_list(self, vlist): class VarUnitConv(VarAction): """A class to perform unit conversions""" - def __init__(self, set_action): + def __init__(self, set_action, local_name, kind, **kwargs): """Initialize unit conversion""" + self._local_name = local_name + self._local_name_temp = local_name+'_local' + self._kind = kind + for key, value in kwargs.items(): + setattr(self, "_"+key, value) self._set_action = set_action + def add_local(self, var): + """Add a local variable for unit conversion""" + tmp_var = var.clone(self._local_name_temp) + self.__group.manage_variable(tmp_var) + + def write_action(self, vadict): + """Return unit conversion transformation""" + if self._set_action: + if self._key == 'from': + lhs = var=self._local_name_temp + rhs = self._set_action.format(var=self._local_name, kind='_'+self._kind) + if self._key == 'to': + lhs = var=self._local_name + rhs = self._set_action.format(var=self._local_name_temp, kind='_'+self._kind) + + return f"{lhs} = {rhs}" + + @property + def required_stdnames(self): + """Return the _required_stdnames for this object""" + return self._required_stdnames + + @property + def missing_stdname(self): + """Return the _missing_stdname for this object""" + return self._missing_stdname + ############################################################################### class VarLoopSubst(VarAction): @@ -1267,6 +1299,7 @@ def add_local(self, vadict, source, run_env): 'local_name':local_name, 'type':'integer', 'units':'count', 'dimensions':'()'} var = Var(prop_dict, source, run_env) + print(' add_local():') vadict.add_variable(var, run_env, exists_ok=True, gen_unique=True) # end if @@ -1620,6 +1653,7 @@ def add_variable_dimensions(self, var, ignore_sources, to_dict=None, dvar = self.find_variable(standard_name=dimname, any_scope=True) if dvar and (dvar.source.type not in ignore_sources): if to_dict: + print(' add_variable_dimensions():') to_dict.add_variable(dvar, self.__run_env, exists_ok=True, adjust_intent=adjust_intent) @@ -1788,6 +1822,7 @@ def declare_variables(self, outfile, indent, dummy=False, def merge(self, other_dict, run_env): """Add new entries from """ for ovar in other_dict.variable_list(): + print(' merge():') self.add_variable(ovar, run_env) # end for @@ -1834,27 +1869,29 @@ def __eq__(self, other): return self is other @classmethod - def unit_cnv_from(cls, hunits, var): + def unit_cnv_match(cls, hunits, var): """Return VarUnitConv if and require unit conversion, otherwise, return None""" - function_name = '{0}__to__{1}'.format(hunits,var.get_prop_value('units')) + ucmatch = {"from": '' ,"to": ''} + function_name_from = '{0}__to__{1}'.format(hunits, var.get_prop_value('units')) + function_name_to = '{0}__to__{1}'.format(var.get_prop_value('units'), hunits) + try: - function = getattr(unit_conversion, function_name) - ucmatch = VarUnitConv(function()) + function_from = getattr(unit_conversion, function_name_from) + ucmatch["from"] = VarUnitConv(function_from(),key="from", + local_name=var.get_prop_value('local_name'), + kind=var.get_prop_value('kind')) except: - ucmatch = None - return ucmatch - - @classmethod - def unit_cnv_to(cls, hunits, var): - """Return VarUnitConv if and require unit - conversion, otherwise, return None""" - function_name = '{1}__to__{0}'.format(hunits,var.get_prop_value('units')) + ucmatch["from"] = None try: - function = getattr(unit_conversion, function_name) - ucmatch = VarUnitConv(function()) + if (var.get_prop_value('intent') != 'in'): + function_to = getattr(unit_conversion, function_name_to) + ucmatch["to"] = VarUnitConv(function_to(),key="to", + local_name=var.get_prop_value('local_name'), + kind=var.get_prop_value('kind')) except: - ucmatch = None + ucmatch["to"] = None + return ucmatch @classmethod diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index fb36c122..7a15fa12 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -800,30 +800,6 @@ def find_variable(self, standard_name=None, source_var=None, # end if return found_var - def match_units(self, var, title): - """Compare units for in this SuiteObject's dictionary tree. - If there are differences, apply unit conversions""" - - dict_var = self.find_variable(source_var=var, any_scope=True) - hunits = dict_var.get_prop_value('units') - found_var = False - if (hunits != var.get_prop_value('units')): - found_var = True - if self.run_phase(): - if (var.get_prop_value('intent') == 'out'): - match = VarDictionary.unit_cnv_to(hunits, var) - if (var.get_prop_value('intent') == 'in'): - match = VarDictionary.unit_cnv_from(hunits, var) - if (var.get_prop_value('intent') == 'inout'): - match = VarDictionary.unit_cnv_from(hunits, var) - match = VarDictionary.unit_cnv_to(hunits, var) - print(match._set_action) - else: - match = None - self.register_action(match) - - return found_var - def match_variable(self, var, vstdname=None, vdims=None): """Try to find a source for in this SuiteObject's dictionary tree. Several items are returned: @@ -1209,7 +1185,11 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): vstdname)) # end if # end if - found = self.match_units(var,my_header.title) + # Are there any unit conversions? + found = self.__group.match_units(var) + if found: + tmp_var = var.clone(var.get_prop_value('local_name')+'_local') + self.__group.manage_variable(tmp_var) # end for if self.needs_vertical is not None: @@ -1235,7 +1215,15 @@ def write(self, outfile, errcode, indent): subname=self.subroutine_name) stmt = 'call {}({})' outfile.write('if ({} == 0) then'.format(errcode), indent) + # Unit conversions? Add forward transform + for umatch in self.__group._unit_cnv_matches_from: + action = umatch.write_action(vadict=self.call_list) + if action: outfile.write(action, indent+1) outfile.write(stmt.format(self.subroutine_name, my_args), indent+1) + # Unit conversions? Add reverse transform + for umatch in self.__group._unit_cnv_matches_to: + action = umatch.write_action(vadict=self.call_list) + if action: outfile.write(action, indent+1) outfile.write('end if', indent) def schemes(self): @@ -1572,8 +1560,9 @@ def __init__(self, group_xml, transition, parent, context, run_env): self._host_vars = None self._host_ddts = None self._loop_var_matches = list() + self._unit_cnv_matches_from = list() + self._unit_cnv_matches_to = list() self._phase_check_stmts = list() - self._unit_cnv_matches = list() self._set_state = None self._ddt_library = None @@ -1611,13 +1600,30 @@ def register_action(self, vaction): vaction.add_local(self, _API_LOCAL, self.run_env) return True - if isinstance(vaction, VarUnitConv): - self._unit_cnv_matches = vaction.add_to_list(self._unit_cnv_matches) + if isinstance(vaction["from"], VarUnitConv): + if (vaction["from"]): + self._unit_cnv_matches_from = vaction["from"].add_to_list(self._unit_cnv_matches_from) + if (vaction["to"]): + self._unit_cnv_matches_to = vaction["to"].add_to_list(self._unit_cnv_matches_to) return True - # end if return False + def match_units(self, var): + dict_var = self.find_variable(source_var=var, any_scope=True) + found_var = False + if dict_var: + hunits = dict_var.get_prop_value('units') + if (hunits != var.get_prop_value('units')): + found_var = True + if self.run_phase(): + match = VarDictionary.unit_cnv_match(hunits, var) + self.register_action(match) + else: + match = None + return found_var + + def manage_variable(self, newvar): """Add to our local dictionary making necessary modifications to the variable properties so that it is diff --git a/test/var_action_test/test_host_mod.F90 b/test/var_action_test/test_host_mod.F90 index d6191814..b0501c0c 100644 --- a/test/var_action_test/test_host_mod.F90 +++ b/test/var_action_test/test_host_mod.F90 @@ -36,7 +36,7 @@ logical function compare_data() real(kind_phys), parameter :: effrl_expected = 5.0E-5 ! 50 microns, in meter real(kind_phys), parameter :: effri_expected = 7.5E-5 ! 75 microns, in meter real(kind_phys), parameter :: effrs_expected = 5.1E-4 ! 510 microns, in meter - real(kind_phys), parameter :: tolerance = 1.0E-10 ! used as scaling factor for expected value + real(kind_phys), parameter :: tolerance = 1.0E-6 ! used as scaling factor for expected value compare_data = .true. From bb86f8b5339f0b325cc64156065574d5c8fea990 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 10 Oct 2023 22:10:48 -0600 Subject: [PATCH 03/38] Housekeeping --- scripts/metavar.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/metavar.py b/scripts/metavar.py index 24cbfc24..1c106863 100755 --- a/scripts/metavar.py +++ b/scripts/metavar.py @@ -1299,7 +1299,6 @@ def add_local(self, vadict, source, run_env): 'local_name':local_name, 'type':'integer', 'units':'count', 'dimensions':'()'} var = Var(prop_dict, source, run_env) - print(' add_local():') vadict.add_variable(var, run_env, exists_ok=True, gen_unique=True) # end if @@ -1653,7 +1652,6 @@ def add_variable_dimensions(self, var, ignore_sources, to_dict=None, dvar = self.find_variable(standard_name=dimname, any_scope=True) if dvar and (dvar.source.type not in ignore_sources): if to_dict: - print(' add_variable_dimensions():') to_dict.add_variable(dvar, self.__run_env, exists_ok=True, adjust_intent=adjust_intent) @@ -1822,7 +1820,6 @@ def declare_variables(self, outfile, indent, dummy=False, def merge(self, other_dict, run_env): """Add new entries from """ for ovar in other_dict.variable_list(): - print(' merge():') self.add_variable(ovar, run_env) # end for From 92d00d8affb9a1f6452458ee441096770e5cd12c Mon Sep 17 00:00:00 2001 From: dustinswales Date: Fri, 20 Oct 2023 15:36:34 -0600 Subject: [PATCH 04/38] Working with compatability object --- scripts/suite_objects.py | 86 +++++++++++++++++++++++++++++++++++----- 1 file changed, 76 insertions(+), 10 deletions(-) diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index 472556cb..23ee6093 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -20,6 +20,7 @@ from parse_tools import init_log, set_log_to_null from var_props import is_horizontal_dimension, find_horizontal_dimension from var_props import find_vertical_dimension +from var_props import VarCompatObj # pylint: disable=too-many-lines @@ -800,7 +801,7 @@ def find_variable(self, standard_name=None, source_var=None, # end if return found_var - def match_variable(self, var, vstdname=None, vdims=None): + def match_variable(self, var, run_env): """Try to find a source for in this SuiteObject's dictionary tree. Several items are returned: found_var: True if a match was found @@ -809,12 +810,8 @@ def match_variable(self, var, vstdname=None, vdims=None): missing_vert: Vertical dim in parent but not in perm: Permutation (XXgoldyXX: Not yet implemented) """ - if vstdname is None: - vstdname = var.get_prop_value('standard_name') - # end if - if vdims is None: - vdims = var.get_dimensions() - # end if + vstdname = var.get_prop_value('standard_name') + vdims = var.get_dimensions() if (not vdims) and self.run_phase(): vmatch = VarDictionary.loop_var_match(vstdname) else: @@ -824,6 +821,7 @@ def match_variable(self, var, vstdname=None, vdims=None): missing_vert = None new_vdims = list() var_vdim = var.has_vertical_dimension(dims=vdims) + compat_obj = None # Does this variable exist in the calling tree? dict_var = self.find_variable(source_var=var, any_scope=True) if dict_var is None: @@ -856,6 +854,23 @@ def match_variable(self, var, vstdname=None, vdims=None): new_vdims = list() new_dict_dims = dict_dims match = True + # Create compatability object, containing any necessary forward/reverse + # transforms from and + compat_obj = VarCompatObj( + vstdname, + var.get_prop_value('type'), + var.get_prop_value('kind'), + var.get_prop_value('units'), + var.get_dimensions(), + var.get_prop_value('local_name'), + dict_var.get_prop_value('standard_name'), + dict_var.get_prop_value('type'), + dict_var.get_prop_value('kind'), + dict_var.get_prop_value('units'), + dict_var.get_dimensions(), + dict_var.get_prop_value('local_name'), + run_env) + # end if # Add the variable to the parent call tree if dict_dims == new_dict_dims: @@ -878,7 +893,7 @@ def match_variable(self, var, vstdname=None, vdims=None): # end if # end if # end if - return found_var, var_vdim, new_vdims, missing_vert + return found_var, var_vdim, new_vdims, missing_vert, compat_obj, dict_var def in_process_split(self): """Find out if we are in a process-split region""" @@ -1061,6 +1076,8 @@ def __init__(self, scheme_xml, context, parent, run_env): self.__lib = scheme_xml.get('lib', None) self.__has_vertical_dimension = False self.__group = None + self.__forward_transforms = list() + self.__reverse_transforms = list() super().__init__(name, context, parent, run_env, active_call_list=True) def update_group_call_list_variable(self, var): @@ -1128,8 +1145,8 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): def_val = var.get_prop_value('default_value') vdims = var.get_dimensions() vintent = var.get_prop_value('intent') - args = self.match_variable(var, vstdname=vstdname, vdims=vdims) - found, vert_dim, new_dims, missing_vert = args + args = self.match_variable(var, self.run_env) + found, vert_dim, new_dims, missing_vert, compat_obj, suite_var = args if found: if not self.has_vertical_dim: self.__has_vertical_dimension = vert_dim is not None @@ -1184,6 +1201,21 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): vstdname)) # end if # end if + # Are there any forward/reverse transforms for this variable? + if compat_obj is not None and (compat_obj.has_dim_transforms or compat_obj.has_unit_transforms): + tmp_var = var.clone(var.get_prop_value('local_name')+'_local') + self.__group.manage_variable(tmp_var) + if (var.get_prop_value('intent') != 'in'): + self.__forward_transforms.append( + compat_obj.forward_transform(lvar_lname=var.get_prop_value('local_name'), + rvar_lname=tmp_var.get_prop_value('local_name'), + indices=[':']*var.get_rank())) + #indices=self.transform_dim_str(var.get_dimensions(), var.context))) + self.__reverse_transforms.append( + compat_obj.reverse_transform(lvar_lname=tmp_var.get_prop_value('local_name'), + rvar_lname=var.get_prop_value('local_name'), + indices=[':']*var.get_rank())) + #indices=self.transform_dim_str(var.get_dimensions(), var.context))) # end for if self.needs_vertical is not None: self.parent.add_part(self, replace=True) # Should add a vloop @@ -1195,6 +1227,35 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): # end if return scheme_mods + def transform_dim_str(self, dims, context): + """Create the dimension string for a transform statement""" + rdims = list() + for dim in dims: + rdparts = list() + dparts = dim.split(':') + for dpart in dparts: + dvar = self.find_variable(standard_name=dpart, any_scope=True) + if dvar is None: + dvar = self.call_list.find_variable(standard_name=dpart, + any_scope=False) + if dvar is None: + emsg = "Dimension variable, '{}', not found{}" + lvar = self.find_local_name(dpart, any_scope=True) + if lvar is not None: + emsg += "\nBe sure to use standard names!" + + ctx = context_string(context) + raise CCPPError(emsg.format(dpart, ctx)) + + lname = dvar.get_prop_value('local_name') + rdparts.append(lname) + # end if + rdims.append(':'.join(rdparts)) + # end for + # end for + + return rdims + def write(self, outfile, errcode, indent): # Unused arguments are for consistent write interface # pylint: disable=unused-argument @@ -1207,9 +1268,14 @@ def write(self, outfile, errcode, indent): is_func_call=True, subname=self.subroutine_name) stmt = 'call {}({})' + # Write any reverse transforms. + for reverse_transform in self.__reverse_transforms: outfile.write(reverse_transform, indent) + # Write the scheme call. outfile.write('if ({} == 0) then'.format(errcode), indent) outfile.write(stmt.format(self.subroutine_name, my_args), indent+1) outfile.write('end if', indent) + # Write any forward transforms. + for forward_transform in self.__forward_transforms: outfile.write(forward_transform, indent) def schemes(self): """Return self as a list for consistency with subcycle""" From eecc620ba807733fc66d182d6f408699971848c0 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Fri, 20 Oct 2023 15:39:36 -0600 Subject: [PATCH 05/38] Revert change --- scripts/metavar.py | 70 +--------------------------------------------- 1 file changed, 1 insertion(+), 69 deletions(-) diff --git a/scripts/metavar.py b/scripts/metavar.py index 1c106863..71609580 100755 --- a/scripts/metavar.py +++ b/scripts/metavar.py @@ -25,7 +25,6 @@ from var_props import CCPP_LOOP_DIM_SUBSTS, VariableProperty, VarCompatObj from var_props import find_horizontal_dimension, find_vertical_dimension from var_props import standard_name_to_long_name, default_kind_val -from conversion_tools import unit_conversion ############################################################################## @@ -1185,7 +1184,7 @@ def equiv(self, vmatch): of the objects. equiv should be overridden with a method that first calls this method and then tests class-specific object data.""" - return vmatch.__class__ != self.__class__ + return vmatch.__class__ == self.__class__ def add_to_list(self, vlist): """Add to unless or its equivalent is @@ -1205,47 +1204,6 @@ def add_to_list(self, vlist): ############################################################################### -class VarUnitConv(VarAction): - """A class to perform unit conversions""" - - def __init__(self, set_action, local_name, kind, **kwargs): - """Initialize unit conversion""" - self._local_name = local_name - self._local_name_temp = local_name+'_local' - self._kind = kind - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - self._set_action = set_action - - def add_local(self, var): - """Add a local variable for unit conversion""" - tmp_var = var.clone(self._local_name_temp) - self.__group.manage_variable(tmp_var) - - def write_action(self, vadict): - """Return unit conversion transformation""" - if self._set_action: - if self._key == 'from': - lhs = var=self._local_name_temp - rhs = self._set_action.format(var=self._local_name, kind='_'+self._kind) - if self._key == 'to': - lhs = var=self._local_name - rhs = self._set_action.format(var=self._local_name_temp, kind='_'+self._kind) - - return f"{lhs} = {rhs}" - - @property - def required_stdnames(self): - """Return the _required_stdnames for this object""" - return self._required_stdnames - - @property - def missing_stdname(self): - """Return the _missing_stdname for this object""" - return self._missing_stdname - -############################################################################### - class VarLoopSubst(VarAction): """A class to handle required loop substitutions where the host model (or a suite part) does not provide a loop-like variable used by a @@ -1865,32 +1823,6 @@ def __eq__(self, other): list equality""" return self is other - @classmethod - def unit_cnv_match(cls, hunits, var): - """Return VarUnitConv if and require unit - conversion, otherwise, return None""" - ucmatch = {"from": '' ,"to": ''} - function_name_from = '{0}__to__{1}'.format(hunits, var.get_prop_value('units')) - function_name_to = '{0}__to__{1}'.format(var.get_prop_value('units'), hunits) - - try: - function_from = getattr(unit_conversion, function_name_from) - ucmatch["from"] = VarUnitConv(function_from(),key="from", - local_name=var.get_prop_value('local_name'), - kind=var.get_prop_value('kind')) - except: - ucmatch["from"] = None - try: - if (var.get_prop_value('intent') != 'in'): - function_to = getattr(unit_conversion, function_name_to) - ucmatch["to"] = VarUnitConv(function_to(),key="to", - local_name=var.get_prop_value('local_name'), - kind=var.get_prop_value('kind')) - except: - ucmatch["to"] = None - - return ucmatch - @classmethod def loop_var_match(cls, standard_name): """Return a VarLoopSubst if is a loop variable, From 592e15f2c4cadc53a7d0b8760922c68660629c8c Mon Sep 17 00:00:00 2001 From: dustinswales Date: Fri, 20 Oct 2023 15:41:44 -0600 Subject: [PATCH 06/38] Omission from previous commit --- scripts/suite_objects.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index 0f2f4f13..85f09a12 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -13,7 +13,7 @@ from code_block import CodeBlock from constituents import ConstituentVarDict from framework_env import CCPPFrameworkEnv -from metavar import Var, VarDictionary, VarLoopSubst, VarUnitConv +from metavar import Var, VarDictionary, VarLoopSubst from metavar import CCPP_CONSTANT_VARS, CCPP_LOOP_VAR_STDNAMES from parse_tools import ParseContext, ParseSource, context_string from parse_tools import ParseInternalError, CCPPError @@ -1660,13 +1660,6 @@ def register_action(self, vaction): vaction.add_local(self, _API_LOCAL, self.run_env) return True - if isinstance(vaction["from"], VarUnitConv): - if (vaction["from"]): - self._unit_cnv_matches_from = vaction["from"].add_to_list(self._unit_cnv_matches_from) - if (vaction["to"]): - self._unit_cnv_matches_to = vaction["to"].add_to_list(self._unit_cnv_matches_to) - return True - return False def match_units(self, var): From a4b1c99ad1c470f614126658407763d7f8a0a0cc Mon Sep 17 00:00:00 2001 From: dustinswales Date: Fri, 20 Oct 2023 15:43:02 -0600 Subject: [PATCH 07/38] Omission from previous commit. again... --- scripts/suite_objects.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index 85f09a12..13162c84 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1620,8 +1620,6 @@ def __init__(self, group_xml, transition, parent, context, run_env): self._host_vars = None self._host_ddts = None self._loop_var_matches = list() - self._unit_cnv_matches_from = list() - self._unit_cnv_matches_to = list() self._phase_check_stmts = list() self._set_state = None self._ddt_library = None @@ -1662,21 +1660,6 @@ def register_action(self, vaction): return False - def match_units(self, var): - dict_var = self.find_variable(source_var=var, any_scope=True) - found_var = False - if dict_var: - hunits = dict_var.get_prop_value('units') - if (hunits != var.get_prop_value('units')): - found_var = True - if self.run_phase(): - match = VarDictionary.unit_cnv_match(hunits, var) - self.register_action(match) - else: - match = None - return found_var - - def manage_variable(self, newvar): """Add to our local dictionary making necessary modifications to the variable properties so that it is From 62a9d1293a6cc9a1196b179419fe4f017b437506 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Fri, 20 Oct 2023 15:45:02 -0600 Subject: [PATCH 08/38] Omission from previous commit. again... --- scripts/suite_objects.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index 13162c84..203add13 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1273,15 +1273,7 @@ def write(self, outfile, errcode, indent): for reverse_transform in self.__reverse_transforms: outfile.write(reverse_transform, indent) # Write the scheme call. outfile.write('if ({} == 0) then'.format(errcode), indent) - # Unit conversions? Add forward transform - for umatch in self.__group._unit_cnv_matches_from: - action = umatch.write_action(vadict=self.call_list) - if action: outfile.write(action, indent+1) outfile.write(stmt.format(self.subroutine_name, my_args), indent+1) - # Unit conversions? Add reverse transform - for umatch in self.__group._unit_cnv_matches_to: - action = umatch.write_action(vadict=self.call_list) - if action: outfile.write(action, indent+1) outfile.write('end if', indent) # Write any forward transforms. for forward_transform in self.__forward_transforms: outfile.write(forward_transform, indent) From 8d8589ebfd4cf04b749fc0f515fd8a6b52e71ff1 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Mon, 23 Oct 2023 15:22:31 +0000 Subject: [PATCH 09/38] Cleanup VarCompatObj creation --- scripts/suite_objects.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index f3185a87..9baef6e8 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -857,20 +857,7 @@ def match_variable(self, var, run_env): match = True # Create compatability object, containing any necessary forward/reverse # transforms from and - compat_obj = VarCompatObj( - vstdname, - var.get_prop_value('type'), - var.get_prop_value('kind'), - var.get_prop_value('units'), - var.get_dimensions(), - var.get_prop_value('local_name'), - dict_var.get_prop_value('standard_name'), - dict_var.get_prop_value('type'), - dict_var.get_prop_value('kind'), - dict_var.get_prop_value('units'), - dict_var.get_dimensions(), - dict_var.get_prop_value('local_name'), - run_env) + compat_obj = var.compatible(dict_var, run_env) # end if # Add the variable to the parent call tree From 219f2e9c88b7b774becac2bd1453696e105af1c4 Mon Sep 17 00:00:00 2001 From: Grant Firl Date: Mon, 23 Oct 2023 16:40:11 -0400 Subject: [PATCH 10/38] Bug fix for initialized_set_block (#503) --- scripts/mkstatic.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index 576abdf8..f86f738f 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -1611,6 +1611,14 @@ def write(self, metadata_request, metadata_define, arguments, debug): if subcycle_body: body += subcycle_body_prefix + subcycle_body + subcycle_body_suffix + #For the init stage, for the case when the suite doesn't have any schemes with init phases, + #we still need to add the host-supplied ccpp_t variable to the init group caps so that it is + #available for setting the initialized flag for the particular instance being called. Otherwise, + #the initialized_set_block for the init phase tries to reference the unavailable ccpp_t variable. + if (ccpp_stage == 'init' and not self.parents[ccpp_stage]): + ccpp_var.intent = 'in' + self.parents[ccpp_stage].update({ccpp_var.local_name:ccpp_var}) + # Get list of arguments, module use statement and variable definitions for this subroutine (=stage for the group) (self.arguments[ccpp_stage], sub_module_use, sub_var_defs) = create_arguments_module_use_var_defs( self.parents[ccpp_stage], metadata_define, tmpvars.values()) From 3e14fa8c0070bc3f75f8699e95d86aaa02d07d20 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Tue, 24 Oct 2023 21:58:25 +0000 Subject: [PATCH 11/38] Vertical flipping working w/ new metadata attribute. Work in progress --- scripts/ccpp_datafile.py | 2 +- scripts/metavar.py | 8 +++- scripts/suite_objects.py | 50 ++++++++-------------- scripts/var_props.py | 66 ++++++++++++++--------------- test/var_action_test/effr_calc.meta | 2 + 5 files changed, 59 insertions(+), 69 deletions(-) diff --git a/scripts/ccpp_datafile.py b/scripts/ccpp_datafile.py index 562a8f4b..39880aec 100755 --- a/scripts/ccpp_datafile.py +++ b/scripts/ccpp_datafile.py @@ -653,7 +653,7 @@ def _new_var_entry(parent, var, full_entry=True): prop_list.extend(["allocatable", "active", "default_value", "diagnostic_name", "diagnostic_name_fixed", "kind", "persistence", "polymorphic", "protected", - "state_variable", "type", "units"]) + "state_variable", "type", "units", "top_at_one"]) prop_list.extend(Var.constituent_property_names()) # end if ventry = ET.SubElement(parent, "var") diff --git a/scripts/metavar.py b/scripts/metavar.py index dc088fdc..119e4e46 100755 --- a/scripts/metavar.py +++ b/scripts/metavar.py @@ -196,6 +196,8 @@ class Var: VariableProperty('active', str, optional_in=True, default_in='.true.'), VariableProperty('polymorphic', bool, optional_in=True, + default_in='.false.'), + VariableProperty('top_at_one', bool, optional_in=True, default_in='.false.')] # XXgoldyXX: v debug only @@ -364,15 +366,17 @@ def compatible(self, other, run_env): sunits = self.get_prop_value('units') sstd_name = self.get_prop_value('standard_name') sloc_name = self.get_prop_value('local_name') + stopp = self.get_prop_value('top_at_one') sdims = self.get_dimensions() otype = other.get_prop_value('type') okind = other.get_prop_value('kind') ounits = other.get_prop_value('units') ostd_name = other.get_prop_value('standard_name') oloc_name = other.get_prop_value('local_name') + otopp = other.get_prop_value('top_at_one') odims = other.get_dimensions() - compat = VarCompatObj(sstd_name, stype, skind, sunits, sdims, sloc_name, - ostd_name, otype, okind, ounits, odims, oloc_name, + compat = VarCompatObj(sstd_name, stype, skind, sunits, sdims, sloc_name, stopp, + ostd_name, otype, okind, ounits, odims, oloc_name, otopp, run_env, v1_context=self.context, v2_context=other.context) if (not compat) and (run_env.logger is not None): diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index 9baef6e8..a257cd5f 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1128,7 +1128,10 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): # end if scheme_mods = set() scheme_mods.add((my_header.module, self.subroutine_name)) + var_local = {"local_name":[], "std_name":[]} for var in my_header.variable_list(): + var_local["local_name"].append(var.get_prop_value('local_name')) + var_local["std_name"].append(var.get_prop_value('standard_name')) vstdname = var.get_prop_value('standard_name') def_val = var.get_prop_value('default_value') vdims = var.get_dimensions() @@ -1193,17 +1196,27 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): if compat_obj is not None and (compat_obj.has_dim_transforms or compat_obj.has_unit_transforms): tmp_var = var.clone(var.get_prop_value('local_name')+'_local') self.__group.manage_variable(tmp_var) + + # Move this piece somewhere! + indices = [':']*tmp_var.get_rank() + dim = find_vertical_dimension(var.get_dimensions())[0] + for dpart in dim.split(':'): + if (dpart in var_local["std_name"]): + vli = 1 + if (compat_obj.has_dim_transforms): + indices[find_vertical_dimension(var.get_dimensions())[1]] = var_local["local_name"][vli] + ':1:-1' + else: + indices[find_vertical_dimension(var.get_dimensions())[1]] = '1:' + var_local["local_name"][vli] + if (var.get_prop_value('intent') != 'in'): self.__forward_transforms.append( compat_obj.forward_transform(lvar_lname=var.get_prop_value('local_name'), rvar_lname=tmp_var.get_prop_value('local_name'), - indices=[':']*var.get_rank())) - #indices=self.transform_dim_str(var.get_dimensions(), var.context))) + indices=indices)) self.__reverse_transforms.append( compat_obj.reverse_transform(lvar_lname=tmp_var.get_prop_value('local_name'), rvar_lname=var.get_prop_value('local_name'), - indices=[':']*var.get_rank())) - #indices=self.transform_dim_str(var.get_dimensions(), var.context))) + indices=indices)) # end for if self.needs_vertical is not None: self.parent.add_part(self, replace=True) # Should add a vloop @@ -1215,35 +1228,6 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): # end if return scheme_mods - def transform_dim_str(self, dims, context): - """Create the dimension string for a transform statement""" - rdims = list() - for dim in dims: - rdparts = list() - dparts = dim.split(':') - for dpart in dparts: - dvar = self.find_variable(standard_name=dpart, any_scope=True) - if dvar is None: - dvar = self.call_list.find_variable(standard_name=dpart, - any_scope=False) - if dvar is None: - emsg = "Dimension variable, '{}', not found{}" - lvar = self.find_local_name(dpart, any_scope=True) - if lvar is not None: - emsg += "\nBe sure to use standard names!" - - ctx = context_string(context) - raise CCPPError(emsg.format(dpart, ctx)) - - lname = dvar.get_prop_value('local_name') - rdparts.append(lname) - # end if - rdims.append(':'.join(rdparts)) - # end for - # end for - - return rdims - def write(self, outfile, errcode, indent): # Unused arguments are for consistent write interface # pylint: disable=unused-argument diff --git a/scripts/var_props.py b/scripts/var_props.py index 15613567..53adba30 100755 --- a/scripts/var_props.py +++ b/scripts/var_props.py @@ -820,8 +820,8 @@ class VarCompatObj: """ def __init__(self, var1_stdname, var1_type, var1_kind, var1_units, - var1_dims, var1_lname, var2_stdname, var2_type, var2_kind, - var2_units, var2_dims, var2_lname, run_env, v1_context=None, + var1_dims, var1_lname, var1_top, var2_stdname, var2_type, var2_kind, + var2_units, var2_dims, var2_lname, var2_top, run_env, v1_context=None, v2_context=None): """Initialize this object with information on the equivalence and/or conformability of two variables. @@ -843,6 +843,9 @@ def __init__(self, var1_stdname, var1_type, var1_kind, var1_units, self.__dim_transforms = None self.__kind_transforms = None self.__unit_transforms = None + self.__has_dim_transforms = False + self.__has_kind_transforms = False + self.__has_unit_transforms = False incompat_reason = list() # First, check for fatal incompatibilities if var1_stdname != var2_stdname: @@ -910,15 +913,9 @@ def __init__(self, var1_stdname, var1_type, var1_kind, var1_units, # end if if self.__compat: # Check dimensions - ##XXgoldyXX: For now, we always have to create a dimension - ## transform because we do not know if the vertical - ## dimension is flipped. - if var1_dims or var2_dims: - _, vdim_ind = find_vertical_dimension(var1_dims) - if (var1_dims != var2_dims) or (vdim_ind >= 0): - self.__dim_transforms = self._get_dim_transforms(var1_dims, - var2_dims) - self.__compat = self.__dim_transforms is not None + if var1_top or var2_top: + self.__compat = True + self.__has_dim_transforms = True # end if # end if if not self.__compat: @@ -945,22 +942,19 @@ def forward_transform(self, lvar_lname, rvar_lname, indices, "var2" (i.e., "vertical_layer_dimension" or "vertical_interface_dimension"). """ - # Grab any dimension transform - if self.has_dim_transforms: - dtrans = self.__dim_transforms - lhs_term = dtrans.forward_transform(lvar_lname, indices, - adjust_hdim=adjust_hdim, - flip_vdim=flip_vdim) - else: - lhs_term = f"{lvar_lname}({','.join(indices)})" - # end if + # Dimension transform (Indices handled externally) rhs_term = f"{rvar_lname}({','.join(indices)})" + lhs_term = f"{lvar_lname}" + + # Kind transforms if self.has_kind_transforms: kind = self.__kind_transforms[1] rhs_term = f"real({rhs_term}, {kind})" else: kind = '' # end if + + # Unit transforms if self.has_unit_transforms: if kind: kind = "_" + kind @@ -989,22 +983,19 @@ def reverse_transform(self, lvar_lname, rvar_lname, indices, "var2" (i.e., "vertical_layer_dimension" or "vertical_interface_dimension"). """ - # Grab any dimension transform - if self.has_dim_transforms: - dtrans = self.__dim_transforms - lhs_term = dtrans.reverse_transform(lvar_lname, indices, - adjust_hdim=adjust_hdim, - flip_vdim=flip_vdim) - else: - lhs_term = f"{lvar_lname}({','.join(indices)})" - # end if - rhs_term = f"{rvar_lname}({','.join(indices)})" + # Dimension transforms (Indices handled exrernally) + lhs_term = f"{lvar_lname}({','.join(indices)})" + rhs_term = f"{rvar_lname}" + + # Kind transforms if self.has_kind_transforms: kind = self.__kind_transforms[0] rhs_term = f"real({rhs_term}, {kind})" else: kind = '' # end if + + # Unit transforms if self.has_unit_transforms: if kind: kind = "_" + kind @@ -1357,7 +1348,10 @@ def has_dim_transforms(self): The reverse dimension transformation is a permutation of the indices of the second variable to the first. """ - return self.__dim_transforms is not None + result = False + if (self.__dim_transforms is not None): result = True + if (self.__has_dim_transforms): result = True + return result @property def has_kind_transforms(self): @@ -1369,7 +1363,10 @@ def has_kind_transforms(self): The reverse kind transformation is a string representation of the kind of the first variable. """ - return self.__kind_transforms is not None + result = False + if (self.__kind_transforms is not None): result = True + if (self.__has_kind_transforms): result = True + return result @property def has_unit_transforms(self): @@ -1384,7 +1381,10 @@ def has_unit_transforms(self): and arguments to produce code to transform one variable into the correct units of the other. """ - return self.__unit_transforms is not None + result = False + if (self.__unit_transforms is not None): result = True + if (self.__has_unit_transforms): result = True + return result def __bool__(self): """Return True if this object describes two Var objects which are diff --git a/test/var_action_test/effr_calc.meta b/test/var_action_test/effr_calc.meta index 12e3dac6..365aa506 100644 --- a/test/var_action_test/effr_calc.meta +++ b/test/var_action_test/effr_calc.meta @@ -25,6 +25,7 @@ type = real kind = kind_phys intent = in + top_at_one = .true. [effrl_inout] standard_name = effective_radius_of_stratiform_cloud_liquid_water_particle long_name = effective radius of cloud liquid water particle in micrometer @@ -49,6 +50,7 @@ type = real kind = kind_phys intent = inout + top_at_one = .true. [ errmsg ] standard_name = ccpp_error_message long_name = Error message for error handling in CCPP From c29b0c0c1fa550e48ecf00e80e733292e318dfa3 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 25 Oct 2023 03:29:00 +0000 Subject: [PATCH 12/38] Some more changes. Vertical flipping working --- scripts/suite_objects.py | 8 +++--- scripts/var_props.py | 58 +++++++++++++++++++++++++++------------- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index a257cd5f..6596345c 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1193,17 +1193,17 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): # end if # end if # Are there any forward/reverse transforms for this variable? - if compat_obj is not None and (compat_obj.has_dim_transforms or compat_obj.has_unit_transforms): + if compat_obj is not None and (compat_obj.has_vert_transforms or compat_obj.has_unit_transforms): tmp_var = var.clone(var.get_prop_value('local_name')+'_local') self.__group.manage_variable(tmp_var) # Move this piece somewhere! - indices = [':']*tmp_var.get_rank() + indices = [':']*var.get_rank() dim = find_vertical_dimension(var.get_dimensions())[0] for dpart in dim.split(':'): if (dpart in var_local["std_name"]): - vli = 1 - if (compat_obj.has_dim_transforms): + vli = var_local["std_name"].index(dpart) + if (compat_obj.has_vert_transforms): indices[find_vertical_dimension(var.get_dimensions())[1]] = var_local["local_name"][vli] + ':1:-1' else: indices[find_vertical_dimension(var.get_dimensions())[1]] = '1:' + var_local["local_name"][vli] diff --git a/scripts/var_props.py b/scripts/var_props.py index 53adba30..951fecd8 100755 --- a/scripts/var_props.py +++ b/scripts/var_props.py @@ -843,9 +843,7 @@ def __init__(self, var1_stdname, var1_type, var1_kind, var1_units, self.__dim_transforms = None self.__kind_transforms = None self.__unit_transforms = None - self.__has_dim_transforms = False - self.__has_kind_transforms = False - self.__has_unit_transforms = False + self.has_vert_transforms = False incompat_reason = list() # First, check for fatal incompatibilities if var1_stdname != var2_stdname: @@ -912,10 +910,23 @@ def __init__(self, var1_stdname, var1_type, var1_kind, var1_units, # end if # end if if self.__compat: - # Check dimensions + # Check for vertical array flipping (do later) if var1_top or var2_top: - self.__compat = True - self.__has_dim_transforms = True + self.__compat = True + self.has_vert_transforms = True + # end if + # end if + if self.__compat: + # Check dimensions + ##XXgoldyXX: For now, we always have to create a dimension + ## transform because we do not know if the vertical + ## dimension is flipped. + if var1_dims or var2_dims: + _, vdim_ind = find_vertical_dimension(var1_dims) + if (var1_dims != var2_dims) or (vdim_ind >= 0): + self.__dim_transforms = self._get_dim_transforms(var1_dims, + var2_dims) + self.__compat = self.__dim_transforms is not None # end if # end if if not self.__compat: @@ -1129,6 +1140,26 @@ def _get_unit_convstrs(self, var1_units, var2_units): # end if return (forward_transform, reverse_transform) + def _get_vert_transforms(self, var1_dims, var2_dims): + transforms = None + dim1 = find_vertical_dimension(var1_dims)[0] + dim2 = find_vertical_dimension(var2_dims)[0] + print("_get_vert_transforms dim1: ",dim1) + print("_get_vert_transforms dim2: ",dim2) + print(self.find_variable(standard_name='vertical_layer_dimension', any_scope=False)) + + return transforms + #indices = [':']*tmp_var.get_rank() + #dim = find_vertical_dimension(var.get_dimensions())[0] + #for dpart in dim.split(':'): + # if (dpart in var_local["std_name"]): + # vli = 1 + # if (compat_obj.has_dim_transforms): + # indices[find_vertical_dimension(var.get_dimensions())[1]] = var_local["local_name"][vli] + ':1:-1' + # else: + # indices[find_vertical_dimension(var.get_dimensions())[1]] = '1:' + var_local["local_name"][vli] + + def _get_dim_transforms(self, var1_dims, var2_dims): """Attempt to find forward and reverse permutations for transforming a variable with shape, , to / from a variable with shape, @@ -1348,10 +1379,7 @@ def has_dim_transforms(self): The reverse dimension transformation is a permutation of the indices of the second variable to the first. """ - result = False - if (self.__dim_transforms is not None): result = True - if (self.__has_dim_transforms): result = True - return result + return self.__dim_transforms is not None @property def has_kind_transforms(self): @@ -1363,10 +1391,7 @@ def has_kind_transforms(self): The reverse kind transformation is a string representation of the kind of the first variable. """ - result = False - if (self.__kind_transforms is not None): result = True - if (self.__has_kind_transforms): result = True - return result + return self.__kind_transforms is not None @property def has_unit_transforms(self): @@ -1381,10 +1406,7 @@ def has_unit_transforms(self): and arguments to produce code to transform one variable into the correct units of the other. """ - result = False - if (self.__unit_transforms is not None): result = True - if (self.__has_unit_transforms): result = True - return result + return self.__unit_transforms is not None def __bool__(self): """Return True if this object describes two Var objects which are From 19f6a30859bfab1d2426b5ea53bda1fd2ffcd36a Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 25 Oct 2023 03:38:43 +0000 Subject: [PATCH 13/38] Housekeeping --- scripts/suite_objects.py | 4 +++- scripts/var_props.py | 26 -------------------------- 2 files changed, 3 insertions(+), 27 deletions(-) diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index 6596345c..86379356 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1197,7 +1197,7 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): tmp_var = var.clone(var.get_prop_value('local_name')+'_local') self.__group.manage_variable(tmp_var) - # Move this piece somewhere! + # Create indices for vertical flipping (if needed) indices = [':']*var.get_rank() dim = find_vertical_dimension(var.get_dimensions())[0] for dpart in dim.split(':'): @@ -1208,11 +1208,13 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): else: indices[find_vertical_dimension(var.get_dimensions())[1]] = '1:' + var_local["local_name"][vli] + # Add any forward transforms. if (var.get_prop_value('intent') != 'in'): self.__forward_transforms.append( compat_obj.forward_transform(lvar_lname=var.get_prop_value('local_name'), rvar_lname=tmp_var.get_prop_value('local_name'), indices=indices)) + # Add any reverse transforms. self.__reverse_transforms.append( compat_obj.reverse_transform(lvar_lname=tmp_var.get_prop_value('local_name'), rvar_lname=var.get_prop_value('local_name'), diff --git a/scripts/var_props.py b/scripts/var_props.py index 951fecd8..e01138f0 100755 --- a/scripts/var_props.py +++ b/scripts/var_props.py @@ -957,15 +957,12 @@ def forward_transform(self, lvar_lname, rvar_lname, indices, rhs_term = f"{rvar_lname}({','.join(indices)})" lhs_term = f"{lvar_lname}" - # Kind transforms if self.has_kind_transforms: kind = self.__kind_transforms[1] rhs_term = f"real({rhs_term}, {kind})" else: kind = '' # end if - - # Unit transforms if self.has_unit_transforms: if kind: kind = "_" + kind @@ -998,15 +995,12 @@ def reverse_transform(self, lvar_lname, rvar_lname, indices, lhs_term = f"{lvar_lname}({','.join(indices)})" rhs_term = f"{rvar_lname}" - # Kind transforms if self.has_kind_transforms: kind = self.__kind_transforms[0] rhs_term = f"real({rhs_term}, {kind})" else: kind = '' # end if - - # Unit transforms if self.has_unit_transforms: if kind: kind = "_" + kind @@ -1140,26 +1134,6 @@ def _get_unit_convstrs(self, var1_units, var2_units): # end if return (forward_transform, reverse_transform) - def _get_vert_transforms(self, var1_dims, var2_dims): - transforms = None - dim1 = find_vertical_dimension(var1_dims)[0] - dim2 = find_vertical_dimension(var2_dims)[0] - print("_get_vert_transforms dim1: ",dim1) - print("_get_vert_transforms dim2: ",dim2) - print(self.find_variable(standard_name='vertical_layer_dimension', any_scope=False)) - - return transforms - #indices = [':']*tmp_var.get_rank() - #dim = find_vertical_dimension(var.get_dimensions())[0] - #for dpart in dim.split(':'): - # if (dpart in var_local["std_name"]): - # vli = 1 - # if (compat_obj.has_dim_transforms): - # indices[find_vertical_dimension(var.get_dimensions())[1]] = var_local["local_name"][vli] + ':1:-1' - # else: - # indices[find_vertical_dimension(var.get_dimensions())[1]] = '1:' + var_local["local_name"][vli] - - def _get_dim_transforms(self, var1_dims, var2_dims): """Attempt to find forward and reverse permutations for transforming a variable with shape, , to / from a variable with shape, From aae4b439f5ea4370488cd1fbac4c8d9d52e68b9e Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 25 Oct 2023 04:42:53 +0000 Subject: [PATCH 14/38] Add kind transform to transform test --- scripts/framework_env.py | 4 ++++ test/var_action_test/effr_calc.F90 | 4 ++-- test/var_action_test/effr_calc.meta | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/framework_env.py b/scripts/framework_env.py index 6456134e..49a4b7db 100644 --- a/scripts/framework_env.py +++ b/scripts/framework_env.py @@ -165,6 +165,10 @@ def __init__(self, logger, ndict=None, verbose=0, clean=False, # We always need a kind_phys so add a default if necessary if "kind_phys" not in self.__kind_dict: self.__kind_dict["kind_phys"] = "REAL64" + if "kind_dbl_prec" not in self.__kind_dict: + self.__kind_dict["kind_dbl_prec"] = "REAL64" + if "kind_sngl_prec" not in self.__kind_dict: + self.__kind_dict["kind_sngl_prec"] = "REAL32" # end if if ndict and ('use_error_obj' in ndict): self.__use_error_obj = ndict['use_error_obj'] diff --git a/test/var_action_test/effr_calc.F90 b/test/var_action_test/effr_calc.F90 index 50804a24..4282971f 100644 --- a/test/var_action_test/effr_calc.F90 +++ b/test/var_action_test/effr_calc.F90 @@ -3,7 +3,7 @@ module effr_calc - use ccpp_kinds, only: kind_phys + use ccpp_kinds, only: kind_phys, kind_dbl_prec implicit none private @@ -23,7 +23,7 @@ subroutine effr_calc_run(ncol, nlev, effrr_in, effrl_inout, & real(kind_phys), intent(in) :: effrr_in(:,:) real(kind_phys), intent(inout) :: effrl_inout(:,:) real(kind_phys), intent(out) :: effri_out(:,:) - real(kind_phys), intent(inout) :: effrs_inout(:,:) + real(kind_dbl_prec),intent(inout) :: effrs_inout(:,:) character(len=512), intent(out) :: errmsg integer, intent(out) :: errflg !---------------------------------------------------------------- diff --git a/test/var_action_test/effr_calc.meta b/test/var_action_test/effr_calc.meta index 365aa506..78bd0c39 100644 --- a/test/var_action_test/effr_calc.meta +++ b/test/var_action_test/effr_calc.meta @@ -48,7 +48,7 @@ units = um dimensions = (horizontal_loop_extent,vertical_layer_dimension) type = real - kind = kind_phys + kind = kind_dbl_prec intent = inout top_at_one = .true. [ errmsg ] From c3d2b3aa8eda1a52f29d9949c9ab7a19559b0587 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 26 Oct 2023 20:55:30 +0000 Subject: [PATCH 15/38] Add local declaration of vertical extent, needed for vertical flipping --- scripts/suite_objects.py | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index 86379356..dc6216ee 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1064,6 +1064,8 @@ def __init__(self, scheme_xml, context, parent, run_env): self.__lib = scheme_xml.get('lib', None) self.__has_vertical_dimension = False self.__group = None + self.__alloc_transforms = list() + self.__depend_transforms = list() self.__forward_transforms = list() self.__reverse_transforms = list() super().__init__(name, context, parent, run_env, active_call_list=True) @@ -1194,19 +1196,27 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): # end if # Are there any forward/reverse transforms for this variable? if compat_obj is not None and (compat_obj.has_vert_transforms or compat_obj.has_unit_transforms): + # Add local variable (_local) needed for transformation. tmp_var = var.clone(var.get_prop_value('local_name')+'_local') + alloc_stmt = "allocate({}({}))" + self.__alloc_transforms.append(alloc_stmt.format(var.get_prop_value('local_name')+'_local','')) self.__group.manage_variable(tmp_var) - # Create indices for vertical flipping (if needed) + # Add local variable (_nlay) needed for vertical flipping. indices = [':']*var.get_rank() - dim = find_vertical_dimension(var.get_dimensions())[0] - for dpart in dim.split(':'): - if (dpart in var_local["std_name"]): - vli = var_local["std_name"].index(dpart) - if (compat_obj.has_vert_transforms): - indices[find_vertical_dimension(var.get_dimensions())[1]] = var_local["local_name"][vli] + ':1:-1' - else: - indices[find_vertical_dimension(var.get_dimensions())[1]] = '1:' + var_local["local_name"][vli] + if compat_obj.has_vert_transforms: + verti_var = Var({'local_name':var.get_prop_value('local_name')+'_nlay', + 'standard_name':var.get_prop_value('local_name')+'_nlay', + 'type':'integer', 'units':'count', + 'dimensions':'()'}, _API_LOCAL, self.run_env) + self.__group.manage_variable(verti_var) + # Set indices for vertical flipping. + dim = find_vertical_dimension(var.get_dimensions()) + for dpart in dim[0].split(':'): + indices[dim[1]] = var.get_prop_value('local_name')+'_nlay:1:-1' + # Create statement for use in write stage. + write_stmt = var.get_prop_value('local_name')+"_nlay = size({},{})" + self.__depend_transforms.append(write_stmt.format(var.get_prop_value('local_name'),dim[1]+1)) # Add any forward transforms. if (var.get_prop_value('intent') != 'in'): @@ -1242,14 +1252,18 @@ def write(self, outfile, errcode, indent): is_func_call=True, subname=self.subroutine_name) stmt = 'call {}({})' - # Write any reverse transforms. - for reverse_transform in self.__reverse_transforms: outfile.write(reverse_transform, indent) # Write the scheme call. outfile.write('if ({} == 0) then'.format(errcode), indent) + # Write any allocate statements needed for transforms. + #for alloc_transform in self.__alloc_transforms: outfile.write(alloc_transform, indent+1) + # Write any dependencies needed for transforms. + for depend_transform in self.__depend_transforms: outfile.write(depend_transform, indent+1) + # Write any reverse transforms. + for reverse_transform in self.__reverse_transforms: outfile.write(reverse_transform, indent+1) outfile.write(stmt.format(self.subroutine_name, my_args), indent+1) - outfile.write('end if', indent) # Write any forward transforms. - for forward_transform in self.__forward_transforms: outfile.write(forward_transform, indent) + for forward_transform in self.__forward_transforms: outfile.write(forward_transform, indent+1) + outfile.write('end if', indent) def schemes(self): """Return self as a list for consistency with subcycle""" From ea01a4794ba4b4a719cd35dad32e143e6aa579fe Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Mon, 30 Oct 2023 17:36:11 +0000 Subject: [PATCH 16/38] Everything but getting the correct local variable name for the vertical dimension name --- scripts/suite_objects.py | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index dc6216ee..9e8c5730 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1064,8 +1064,6 @@ def __init__(self, scheme_xml, context, parent, run_env): self.__lib = scheme_xml.get('lib', None) self.__has_vertical_dimension = False self.__group = None - self.__alloc_transforms = list() - self.__depend_transforms = list() self.__forward_transforms = list() self.__reverse_transforms = list() super().__init__(name, context, parent, run_env, active_call_list=True) @@ -1198,25 +1196,14 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): if compat_obj is not None and (compat_obj.has_vert_transforms or compat_obj.has_unit_transforms): # Add local variable (_local) needed for transformation. tmp_var = var.clone(var.get_prop_value('local_name')+'_local') - alloc_stmt = "allocate({}({}))" - self.__alloc_transforms.append(alloc_stmt.format(var.get_prop_value('local_name')+'_local','')) self.__group.manage_variable(tmp_var) - # Add local variable (_nlay) needed for vertical flipping. + # Create indices, flipping if necessary. indices = [':']*var.get_rank() if compat_obj.has_vert_transforms: - verti_var = Var({'local_name':var.get_prop_value('local_name')+'_nlay', - 'standard_name':var.get_prop_value('local_name')+'_nlay', - 'type':'integer', 'units':'count', - 'dimensions':'()'}, _API_LOCAL, self.run_env) - self.__group.manage_variable(verti_var) - # Set indices for vertical flipping. dim = find_vertical_dimension(var.get_dimensions()) - for dpart in dim[0].split(':'): - indices[dim[1]] = var.get_prop_value('local_name')+'_nlay:1:-1' - # Create statement for use in write stage. - write_stmt = var.get_prop_value('local_name')+"_nlay = size({},{})" - self.__depend_transforms.append(write_stmt.format(var.get_prop_value('local_name'),dim[1]+1)) + lvar = 'nlev' + indices[dim[1]] = lvar+':1:-1' # Add any forward transforms. if (var.get_prop_value('intent') != 'in'): @@ -1225,10 +1212,11 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): rvar_lname=tmp_var.get_prop_value('local_name'), indices=indices)) # Add any reverse transforms. - self.__reverse_transforms.append( - compat_obj.reverse_transform(lvar_lname=tmp_var.get_prop_value('local_name'), - rvar_lname=var.get_prop_value('local_name'), - indices=indices)) + if (var.get_prop_value('intent') != 'out'): + self.__reverse_transforms.append( + compat_obj.reverse_transform(lvar_lname=tmp_var.get_prop_value('local_name'), + rvar_lname=var.get_prop_value('local_name'), + indices=indices)) # end for if self.needs_vertical is not None: self.parent.add_part(self, replace=True) # Should add a vloop @@ -1254,10 +1242,6 @@ def write(self, outfile, errcode, indent): stmt = 'call {}({})' # Write the scheme call. outfile.write('if ({} == 0) then'.format(errcode), indent) - # Write any allocate statements needed for transforms. - #for alloc_transform in self.__alloc_transforms: outfile.write(alloc_transform, indent+1) - # Write any dependencies needed for transforms. - for depend_transform in self.__depend_transforms: outfile.write(depend_transform, indent+1) # Write any reverse transforms. for reverse_transform in self.__reverse_transforms: outfile.write(reverse_transform, indent+1) outfile.write(stmt.format(self.subroutine_name, my_args), indent+1) From e47bf8f4b66c3e18aabdf3b9a0e171240d249cc8 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Tue, 31 Oct 2023 19:47:51 +0000 Subject: [PATCH 17/38] Add snippet from Steve G. to get correct local_name for vertical_layer_dimension. --- scripts/suite_objects.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index 9e8c5730..80232386 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1202,8 +1202,10 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): indices = [':']*var.get_rank() if compat_obj.has_vert_transforms: dim = find_vertical_dimension(var.get_dimensions()) - lvar = 'nlev' - indices[dim[1]] = lvar+':1:-1' + vdim_name = vert_dim.split(':')[-1] + group_vvar = self.__group.call_list.find_variable(vdim_name) + vname = group_vvar.get_prop_value('local_name') + indices[dim[1]] = vname+':1:-1' # Add any forward transforms. if (var.get_prop_value('intent') != 'in'): From 253ad3a62738ac402f08b6201d922614ff31d7f1 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Mon, 13 Nov 2023 14:33:08 -0700 Subject: [PATCH 18/38] Address reviewers comments --- scripts/framework_env.py | 4 ---- test/var_action_test/effr_calc.F90 | 4 ++-- test/var_action_test/effr_calc.meta | 2 +- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/scripts/framework_env.py b/scripts/framework_env.py index 49a4b7db..6456134e 100644 --- a/scripts/framework_env.py +++ b/scripts/framework_env.py @@ -165,10 +165,6 @@ def __init__(self, logger, ndict=None, verbose=0, clean=False, # We always need a kind_phys so add a default if necessary if "kind_phys" not in self.__kind_dict: self.__kind_dict["kind_phys"] = "REAL64" - if "kind_dbl_prec" not in self.__kind_dict: - self.__kind_dict["kind_dbl_prec"] = "REAL64" - if "kind_sngl_prec" not in self.__kind_dict: - self.__kind_dict["kind_sngl_prec"] = "REAL32" # end if if ndict and ('use_error_obj' in ndict): self.__use_error_obj = ndict['use_error_obj'] diff --git a/test/var_action_test/effr_calc.F90 b/test/var_action_test/effr_calc.F90 index 4282971f..d01408c6 100644 --- a/test/var_action_test/effr_calc.F90 +++ b/test/var_action_test/effr_calc.F90 @@ -3,7 +3,7 @@ module effr_calc - use ccpp_kinds, only: kind_phys, kind_dbl_prec + use ccpp_kinds, only: kind_phys implicit none private @@ -23,7 +23,7 @@ subroutine effr_calc_run(ncol, nlev, effrr_in, effrl_inout, & real(kind_phys), intent(in) :: effrr_in(:,:) real(kind_phys), intent(inout) :: effrl_inout(:,:) real(kind_phys), intent(out) :: effri_out(:,:) - real(kind_dbl_prec),intent(inout) :: effrs_inout(:,:) + real(8),intent(inout) :: effrs_inout(:,:) character(len=512), intent(out) :: errmsg integer, intent(out) :: errflg !---------------------------------------------------------------- diff --git a/test/var_action_test/effr_calc.meta b/test/var_action_test/effr_calc.meta index 78bd0c39..78243ce9 100644 --- a/test/var_action_test/effr_calc.meta +++ b/test/var_action_test/effr_calc.meta @@ -48,7 +48,7 @@ units = um dimensions = (horizontal_loop_extent,vertical_layer_dimension) type = real - kind = kind_dbl_prec + kind = 8 intent = inout top_at_one = .true. [ errmsg ] From fe5f2134de113bd5f96efdb80cf870557bf4f5e9 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Mon, 13 Nov 2023 14:36:00 -0700 Subject: [PATCH 19/38] Address reviewers comments again --- scripts/metavar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/metavar.py b/scripts/metavar.py index 119e4e46..0b186d7e 100755 --- a/scripts/metavar.py +++ b/scripts/metavar.py @@ -196,9 +196,9 @@ class Var: VariableProperty('active', str, optional_in=True, default_in='.true.'), VariableProperty('polymorphic', bool, optional_in=True, - default_in='.false.'), + default_in=False), VariableProperty('top_at_one', bool, optional_in=True, - default_in='.false.')] + default_in=False)] # XXgoldyXX: v debug only __to_add = VariableProperty('valid_values', str, From fc337a915c138337f8cbf66cea2d7a6088361586 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Mon, 13 Nov 2023 14:44:53 -0700 Subject: [PATCH 20/38] Address reviewers comments again again --- scripts/suite_objects.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index 80232386..e0f228f4 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -881,7 +881,7 @@ def match_variable(self, var, run_env): # end if # end if # end if - return found_var, var_vdim, new_vdims, missing_vert, compat_obj, dict_var + return found_var, var_vdim, new_vdims, missing_vert, compat_obj def in_process_split(self): """Find out if we are in a process-split region""" @@ -1137,7 +1137,7 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): vdims = var.get_dimensions() vintent = var.get_prop_value('intent') args = self.match_variable(var, self.run_env) - found, vert_dim, new_dims, missing_vert, compat_obj, suite_var = args + found, vert_dim, new_dims, missing_vert, compat_obj = args if found: if not self.has_vertical_dim: self.__has_vertical_dimension = vert_dim is not None @@ -1254,6 +1254,7 @@ def write(self, outfile, errcode, indent): def schemes(self): """Return self as a list for consistency with subcycle""" return [self] + # end if def variable_list(self, recursive=False, std_vars=True, loop_vars=True, consts=True): From 1db044750a7bc4546d8d25b988b71fa42f926e45 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 30 Nov 2023 08:50:58 -0700 Subject: [PATCH 21/38] Rename tests directory to test_prebuild --- {tests => test_prebuild}/test_metadata_parser.py | 0 {tests => test_prebuild}/test_mkstatic.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {tests => test_prebuild}/test_metadata_parser.py (100%) rename {tests => test_prebuild}/test_mkstatic.py (100%) diff --git a/tests/test_metadata_parser.py b/test_prebuild/test_metadata_parser.py similarity index 100% rename from tests/test_metadata_parser.py rename to test_prebuild/test_metadata_parser.py diff --git a/tests/test_mkstatic.py b/test_prebuild/test_mkstatic.py similarity index 100% rename from tests/test_mkstatic.py rename to test_prebuild/test_mkstatic.py From f08fd6344c730e5e33e617a9be6451018a2fb95c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 1 Dec 2023 11:27:32 -0700 Subject: [PATCH 22/38] Add test for ccpp_prebuild for blocked data structures --- .../test_blocked_data/CMakeLists.txt | 91 +++++++++++ test_prebuild/test_blocked_data/README.md | 13 ++ .../test_blocked_data/blocked_data_scheme.F90 | 116 ++++++++++++++ .../blocked_data_scheme.meta | 147 ++++++++++++++++++ .../test_blocked_data/ccpp_prebuild_config.py | 81 ++++++++++ test_prebuild/test_blocked_data/data.F90 | 41 +++++ test_prebuild/test_blocked_data/data.meta | 69 ++++++++ test_prebuild/test_blocked_data/main.F90 | 105 +++++++++++++ .../suite_blocked_data_suite.xml | 9 ++ 9 files changed, 672 insertions(+) create mode 100644 test_prebuild/test_blocked_data/CMakeLists.txt create mode 100644 test_prebuild/test_blocked_data/README.md create mode 100644 test_prebuild/test_blocked_data/blocked_data_scheme.F90 create mode 100644 test_prebuild/test_blocked_data/blocked_data_scheme.meta create mode 100644 test_prebuild/test_blocked_data/ccpp_prebuild_config.py create mode 100644 test_prebuild/test_blocked_data/data.F90 create mode 100644 test_prebuild/test_blocked_data/data.meta create mode 100644 test_prebuild/test_blocked_data/main.F90 create mode 100644 test_prebuild/test_blocked_data/suite_blocked_data_suite.xml diff --git a/test_prebuild/test_blocked_data/CMakeLists.txt b/test_prebuild/test_blocked_data/CMakeLists.txt new file mode 100644 index 00000000..1dc0f3db --- /dev/null +++ b/test_prebuild/test_blocked_data/CMakeLists.txt @@ -0,0 +1,91 @@ +#------------------------------------------------------------------------------ +cmake_minimum_required(VERSION 3.0) + +project(ccpp_blocked_data + VERSION 1.0.0 + LANGUAGES Fortran) + +#------------------------------------------------------------------------------ +# Request a static build +option(BUILD_SHARED_LIBS "Build a shared library" OFF) + +#------------------------------------------------------------------------------ +# Set the sources: physics type definitions +set(TYPEDEFS $ENV{CCPP_TYPEDEFS}) +if(TYPEDEFS) + message(STATUS "Got CCPP TYPEDEFS from environment variable: ${TYPEDEFS}") +else(TYPEDEFS) + include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_TYPEDEFS.cmake) + message(STATUS "Got CCPP TYPEDEFS from cmakefile include file: ${TYPEDEFS}") +endif(TYPEDEFS) + +# Generate list of Fortran modules from the CCPP type +# definitions that need need to be installed +foreach(typedef_module ${TYPEDEFS}) + list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${typedef_module}) +endforeach() + +#------------------------------------------------------------------------------ +# Set the sources: physics schemes +set(SCHEMES $ENV{CCPP_SCHEMES}) +if(SCHEMES) + message(STATUS "Got CCPP SCHEMES from environment variable: ${SCHEMES}") +else(SCHEMES) + include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_SCHEMES.cmake) + message(STATUS "Got CCPP SCHEMES from cmakefile include file: ${SCHEMES}") +endif(SCHEMES) + +# Set the sources: physics scheme caps +set(CAPS $ENV{CCPP_CAPS}) +if(CAPS) + message(STATUS "Got CCPP CAPS from environment variable: ${CAPS}") +else(CAPS) + include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_CAPS.cmake) + message(STATUS "Got CCPP CAPS from cmakefile include file: ${CAPS}") +endif(CAPS) + +# Set the sources: physics scheme caps +set(API $ENV{CCPP_API}) +if(API) + message(STATUS "Got CCPP API from environment variable: ${API}") +else(API) + include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_API.cmake) + message(STATUS "Got CCPP API from cmakefile include file: ${API}") +endif(API) + +set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check -ggdb -fbacktrace -ffree-line-length-none") #" -cpp -fno-range-check") + +#------------------------------------------------------------------------------ +add_library(ccpp_blocked_data STATIC ${SCHEMES} ${CAPS} ${API}) +# Generate list of Fortran modules from defined sources +foreach(source_f90 ${CAPS} ${API}) + get_filename_component(tmp_source_f90 ${source_f90} NAME) + string(REGEX REPLACE ".F90" ".mod" tmp_module_f90 ${tmp_source_f90}) + string(TOLOWER ${tmp_module_f90} module_f90) + list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${module_f90}) +endforeach() + +set_target_properties(ccpp_blocked_data PROPERTIES VERSION ${PROJECT_VERSION} + SOVERSION ${PROJECT_VERSION_MAJOR}) + +add_executable(test_blocked_data.x main.F90) +add_dependencies(test_blocked_data.x ccpp_blocked_data) +target_link_libraries(test_blocked_data.x ccpp_blocked_data) +set_target_properties(test_blocked_data.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Define where to install the library +install(TARGETS ccpp_blocked_data + EXPORT ccpp_blocked_data-targets + ARCHIVE DESTINATION lib + LIBRARY DESTINATION lib + RUNTIME DESTINATION lib +) +# Export our configuration +install(EXPORT ccpp_blocked_data-targets + FILE ccpp_blocked_data-config.cmake + DESTINATION lib/cmake +) +# Define where to install the C headers and Fortran modules +#install(FILES ${HEADERS_C} DESTINATION include) +install(FILES ${MODULES_F90} DESTINATION include) + diff --git a/test_prebuild/test_blocked_data/README.md b/test_prebuild/test_blocked_data/README.md new file mode 100644 index 00000000..ad977913 --- /dev/null +++ b/test_prebuild/test_blocked_data/README.md @@ -0,0 +1,13 @@ +# How to build the blocked data test + +1. Set compiler environment as appropriate for your system +2. Run the following commands: +``` +cd test_prebuild/test_blocked_data/ +rm -fr build +mkdir build +../../scripts/ccpp_prebuild.py --config=ccpp_prebuild_config.py --builddir=build +cd build +cmake .. 2>&1 | tee log.cmake +make 2>&1 | tee log.make +``` diff --git a/test_prebuild/test_blocked_data/blocked_data_scheme.F90 b/test_prebuild/test_blocked_data/blocked_data_scheme.F90 new file mode 100644 index 00000000..12a610d2 --- /dev/null +++ b/test_prebuild/test_blocked_data/blocked_data_scheme.F90 @@ -0,0 +1,116 @@ +!>\file blocked_data_scheme.F90 +!! This file contains a blocked_data_scheme CCPP scheme that does nothing +!! except requesting the minimum, mandatory variables. + +module blocked_data_scheme + + implicit none + private + public :: blocked_data_scheme_init, & + blocked_data_scheme_timestep_init, & + blocked_data_scheme_run, & + blocked_data_scheme_timestep_finalize, & + blocked_data_scheme_finalize + + ! This is for unit testing only + integer, parameter, dimension(4) :: data_array_sizes = (/6,6,6,3/) + + contains + +!! \section arg_table_blocked_data_scheme_init Argument Table +!! \htmlinclude blocked_data_scheme_init.html +!! + subroutine blocked_data_scheme_init(data_array, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: data_array(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Check size of data array + write(0,'(a,i3)') 'In blocked_data_scheme_init: checking size of data array to be', sum(data_array_sizes) + if (size(data_array)/=sum(data_array_sizes)) then + write(errmsg,'(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(data_array) + errflg = 1 + return + end if + end subroutine blocked_data_scheme_init + +!! \section arg_table_blocked_data_scheme_timestep_init Argument Table +!! \htmlinclude blocked_data_scheme_timestep_init.html +!! + subroutine blocked_data_scheme_timestep_init(data_array, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: data_array(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Check size of data array + write(0,'(a,i3)') 'In blocked_data_scheme_timestep_init: checking size of data array to be', sum(data_array_sizes) + if (size(data_array)/=sum(data_array_sizes)) then + write(errmsg,'(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), " but got ", size(data_array) + errflg = 1 + return + end if + end subroutine blocked_data_scheme_timestep_init + +!! \section arg_table_blocked_data_scheme_run Argument Table +!! \htmlinclude blocked_data_scheme_run.html +!! + subroutine blocked_data_scheme_run(nb, data_array, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: nb + integer, intent(in) :: data_array(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Check size of data array + write(0,'(2(a,i3))') 'In blocked_data_scheme_run: checking size of data array for block', nb, ' to be', data_array_sizes(nb) + if (size(data_array)/=data_array_sizes(nb)) then + write(errmsg,'(a,i4)') "Error in blocked_data_scheme_run, expected size(data_array)==6, got ", size(data_array) + errflg = 1 + return + end if + end subroutine blocked_data_scheme_run + + !! \section arg_table_blocked_data_scheme_timestep_finalize Argument Table + !! \htmlinclude blocked_data_scheme_timestep_finalize.html + !! + subroutine blocked_data_scheme_timestep_finalize(data_array, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: data_array(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Check size of data array + write(0,'(a,i3)') 'In blocked_data_scheme_timestep_finalize: checking size of data array to be', sum(data_array_sizes) + if (size(data_array)/=sum(data_array_sizes)) then + write(errmsg,'(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(data_array) + errflg = 1 + return + end if + end subroutine blocked_data_scheme_timestep_finalize + +!! \section arg_table_blocked_data_scheme_finalize Argument Table +!! \htmlinclude blocked_data_scheme_finalize.html +!! + subroutine blocked_data_scheme_finalize(data_array, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: data_array(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Check size of data array + write(0,'(a,i3)') 'In blocked_data_scheme_finalize: checking size of data array to be', sum(data_array_sizes) + if (size(data_array)/=sum(data_array_sizes)) then + write(errmsg,'(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(data_array) + errflg = 1 + return + end if + end subroutine blocked_data_scheme_finalize + +end module blocked_data_scheme diff --git a/test_prebuild/test_blocked_data/blocked_data_scheme.meta b/test_prebuild/test_blocked_data/blocked_data_scheme.meta new file mode 100644 index 00000000..d92b0da6 --- /dev/null +++ b/test_prebuild/test_blocked_data/blocked_data_scheme.meta @@ -0,0 +1,147 @@ +[ccpp-table-properties] + name = blocked_data_scheme + type = scheme + dependencies = + +######################################################################## +[ccpp-arg-table] + name = blocked_data_scheme_init + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[data_array] + standard_name = blocked_data_array + long_name = blocked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in + +######################################################################## +[ccpp-arg-table] + name = blocked_data_scheme_timestep_init + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[data_array] + standard_name = blocked_data_array + long_name = blocked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in + +######################################################################## +[ccpp-arg-table] + name = blocked_data_scheme_run + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[nb] + standard_name = ccpp_block_number + long_name = number of block for explicit data blocking in CCPP + units = index + dimensions = () + type = integer + intent = in +[data_array] + standard_name = blocked_data_array + long_name = blocked data array + units = 1 + dimensions = (horizontal_loop_extent) + type = integer + intent = in + +######################################################################## +[ccpp-arg-table] + name = blocked_data_scheme_timestep_finalize + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[data_array] + standard_name = blocked_data_array + long_name = blocked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in + +######################################################################## +[ccpp-arg-table] + name = blocked_data_scheme_finalize + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[data_array] + standard_name = blocked_data_array + long_name = blocked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in + diff --git a/test_prebuild/test_blocked_data/ccpp_prebuild_config.py b/test_prebuild/test_blocked_data/ccpp_prebuild_config.py new file mode 100644 index 00000000..700d9f76 --- /dev/null +++ b/test_prebuild/test_blocked_data/ccpp_prebuild_config.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python + +# CCPP prebuild config for GFDL Finite-Volume Cubed-Sphere Model (FV3) + +############################################################################### +# Definitions # +############################################################################### + +HOST_MODEL_IDENTIFIER = "FV3" + +# Add all files with metadata tables on the host model side and in CCPP, +# relative to basedir = top-level directory of host model. This includes +# kind and type definitions used in CCPP physics. Also add any internal +# dependencies of these files to the list. +VARIABLE_DEFINITION_FILES = [ + # actual variable definition files + '../../src/ccpp_types.F90', + 'data.F90', + ] + +TYPEDEFS_NEW_METADATA = { + 'ccpp_types' : { + 'ccpp_t' : 'cdata', + 'ccpp_types' : '', + }, + 'data' : { + 'blocked_data_type' : 'blocked_data_instance(cdata%blk_no)', + 'data' : '', + }, + } + +# Add all physics scheme files relative to basedir +SCHEME_FILES = [ + 'blocked_data_scheme.F90', + ] + +# Default build dir, relative to current working directory, +# if not specified as command-line argument +DEFAULT_BUILD_DIR = 'build' + +# Auto-generated makefile/cmakefile snippets that contain all type definitions +TYPEDEFS_MAKEFILE = '{build_dir}/CCPP_TYPEDEFS.mk' +TYPEDEFS_CMAKEFILE = '{build_dir}/CCPP_TYPEDEFS.cmake' +TYPEDEFS_SOURCEFILE = '{build_dir}/CCPP_TYPEDEFS.sh' + +# Auto-generated makefile/cmakefile snippets that contain all schemes +SCHEMES_MAKEFILE = '{build_dir}/CCPP_SCHEMES.mk' +SCHEMES_CMAKEFILE = '{build_dir}/CCPP_SCHEMES.cmake' +SCHEMES_SOURCEFILE = '{build_dir}/CCPP_SCHEMES.sh' + +# Auto-generated makefile/cmakefile snippets that contain all caps +CAPS_MAKEFILE = '{build_dir}/CCPP_CAPS.mk' +CAPS_CMAKEFILE = '{build_dir}/CCPP_CAPS.cmake' +CAPS_SOURCEFILE = '{build_dir}/CCPP_CAPS.sh' + +# Directory where to put all auto-generated physics caps +CAPS_DIR = '{build_dir}' + +# Directory where the suite definition files are stored +SUITES_DIR = '.' + +# Optional arguments - only required for schemes that use +# optional arguments. ccpp_prebuild.py will throw an exception +# if it encounters a scheme subroutine with optional arguments +# if no entry is made here. Possible values are: 'all', 'none', +# or a list of standard_names: [ 'var1', 'var3' ]. +OPTIONAL_ARGUMENTS = {} + +# Directory where to write static API to +STATIC_API_DIR = '{build_dir}' +STATIC_API_CMAKEFILE = '{build_dir}/CCPP_API.cmake' +STATIC_API_SOURCEFILE = '{build_dir}/CCPP_API.sh' + +# Directory for writing HTML pages generated from metadata files +METADATA_HTML_OUTPUT_DIR = '{build_dir}' + +# HTML document containing the model-defined CCPP variables +HTML_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_BLOCKED_DATA.html' + +# LaTeX document containing the provided vs requested CCPP variables +LATEX_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_BLOCKED_DATA.tex' diff --git a/test_prebuild/test_blocked_data/data.F90 b/test_prebuild/test_blocked_data/data.F90 new file mode 100644 index 00000000..97ad051e --- /dev/null +++ b/test_prebuild/test_blocked_data/data.F90 @@ -0,0 +1,41 @@ +module data + +!! \section arg_table_data Argument Table +!! \htmlinclude data.html +!! + use ccpp_types, only: ccpp_t + + implicit none + + private + + public nblks, blksz, ncols + public ccpp_data_domain, ccpp_data_blocks, blocked_data_type, blocked_data_instance + + integer, parameter :: nblks = 4 + type(ccpp_t), target :: ccpp_data_domain + type(ccpp_t), dimension(nblks), target :: ccpp_data_blocks + + integer, parameter, dimension(nblks) :: blksz = (/6,6,6,3/) + integer, parameter :: ncols = sum(blksz) + +!! \section arg_table_blocked_data_type +!! \htmlinclude blocked_data_type.html +!! + type blocked_data_type + integer, dimension(:), allocatable :: array_data + contains + procedure :: create => blocked_data_create + end type blocked_data_type + + type(blocked_data_type), dimension(nblks) :: blocked_data_instance + +contains + + subroutine blocked_data_create(blocked_data_instance, ncol) + class(blocked_data_type), intent(inout) :: blocked_data_instance + integer, intent(in) :: ncol + allocate(blocked_data_instance%array_data(ncol)) + end subroutine blocked_data_create + +end module data diff --git a/test_prebuild/test_blocked_data/data.meta b/test_prebuild/test_blocked_data/data.meta new file mode 100644 index 00000000..c5fa2842 --- /dev/null +++ b/test_prebuild/test_blocked_data/data.meta @@ -0,0 +1,69 @@ +[ccpp-table-properties] + name = blocked_data_type + type = ddt + dependencies = +[ccpp-arg-table] + name = blocked_data_type + type = ddt +[array_data] + standard_name = blocked_data_array + long_name = blocked data array + units = 1 + dimensions = (horizontal_loop_extent) + type = integer + +[ccpp-table-properties] + name = data + type = module + dependencies = +[ccpp-arg-table] + name = data + type = module +[cdata] + standard_name = ccpp_t_instance + long_name = instance of derived data type ccpp_t + units = DDT + dimensions = () + type = ccpp_t +[nblks] + standard_name = ccpp_block_count + long_name = for explicit data blocking: number of blocks + units = count + dimensions = () + type = integer +[blksz] + standard_name = ccpp_block_sizes + long_name = for explicit data blocking: block sizes of all blocks + units = count + dimensions = (ccpp_block_count) + type = integer +[blksz(ccpp_block_number)] + standard_name = horizontal_loop_extent + long_name = horizontal loop extent + units = count + dimensions = () + type = integer +[ncols] + standard_name = horizontal_dimension + long_name = horizontal dimension + units = count + dimensions = () + type = integer +[blocked_data_type] + standard_name = blocked_data_type + long_name = definition of type blocked_data_type + units = DDT + dimensions = () + type = blocked_data_type +[blocked_data_instance(ccpp_block_number)] + standard_name = blocked_data_type_instance + long_name = instance of derived data type blocked_data_type + units = DDT + dimensions = () + type = blocked_data_type +[blocked_data_instance] + standard_name = blocked_data_type_instance_all_blocks + long_name = instance of derived data type blocked_data_type + units = DDT + dimensions = (ccpp_block_count) + type = blocked_data_type diff --git a/test_prebuild/test_blocked_data/main.F90 b/test_prebuild/test_blocked_data/main.F90 new file mode 100644 index 00000000..763841aa --- /dev/null +++ b/test_prebuild/test_blocked_data/main.F90 @@ -0,0 +1,105 @@ +program test_blocked_data + + use ccpp_types, only: ccpp_t + use data, only: nblks, blksz, ncols + use data, only: ccpp_data_domain, ccpp_data_blocks, & + blocked_data_type, blocked_data_instance + + use ccpp_static_api, only: ccpp_physics_init, & + ccpp_physics_timestep_init, & + ccpp_physics_run, & + ccpp_physics_timestep_finalize, & + ccpp_physics_finalize + + implicit none + + character(len=*), parameter :: ccpp_suite = 'blocked_data_suite' + integer :: ib, ierr + type(ccpp_t), pointer :: cdata => null() + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + ! For physics running over the entire domain, block and thread + ! number are not used; set to safe values + ccpp_data_domain%blk_no = 1 + ccpp_data_domain%thrd_no = 1 + + ! Loop over all blocks and threads for ccpp_data_blocks + do ib=1,nblks + ! Assign the correct block numbers, only one thread + ccpp_data_blocks(ib)%blk_no = ib + ccpp_data_blocks(ib)%thrd_no = 1 + end do + + do ib=1,size(blocked_data_instance) + allocate(blocked_data_instance(ib)%array_data(blksz(ib))) + write(0,'(2(a,i3))') "Allocated array_data for block", ib, " to size", size(blocked_data_instance(ib)%array_data) + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + cdata => ccpp_data_domain + call ccpp_physics_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) + if (ierr/=0) then + write(0,'(a)') "An error occurred in ccpp_physics_init:" + write(0,'(a)') trim(cdata%errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + cdata => ccpp_data_domain + call ccpp_physics_timestep_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) + if (ierr/=0) then + write(0,'(a)') "An error occurred in ccpp_physics_timestep_init:" + write(0,'(a)') trim(cdata%errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics run step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ib=1,nblks + cdata => ccpp_data_blocks(ib) + call ccpp_physics_run(cdata, suite_name=trim(ccpp_suite), ierr=ierr) + if (ierr/=0) then + write(0,'(a,i3,a)') "An error occurred in ccpp_physics_run for block", ib, ":" + write(0,'(a)') trim(cdata%errmsg) + stop 1 + end if + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep finalize step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + cdata => ccpp_data_domain + call ccpp_physics_timestep_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) + if (ierr/=0) then + write(0,'(a)') "An error occurred in ccpp_physics_timestep_init:" + write(0,'(a)') trim(cdata%errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics finalize step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + cdata => ccpp_data_domain + call ccpp_physics_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) + if (ierr/=0) then + write(0,'(a)') "An error occurred in ccpp_physics_timestep_init:" + write(0,'(a)') trim(cdata%errmsg) + stop 1 + end if + +contains + +end program test_blocked_data \ No newline at end of file diff --git a/test_prebuild/test_blocked_data/suite_blocked_data_suite.xml b/test_prebuild/test_blocked_data/suite_blocked_data_suite.xml new file mode 100644 index 00000000..cf4fe9a4 --- /dev/null +++ b/test_prebuild/test_blocked_data/suite_blocked_data_suite.xml @@ -0,0 +1,9 @@ + + + + + + blocked_data_scheme + + + From a9e194ad2cda8d8a48ababc8ef4d3ac94f6fb4c6 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 1 Dec 2023 11:35:15 -0700 Subject: [PATCH 23/38] Add CI tests for ccpp-prebuild --- .github/workflows/prebuild.yaml | 44 +++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 .github/workflows/prebuild.yaml diff --git a/.github/workflows/prebuild.yaml b/.github/workflows/prebuild.yaml new file mode 100644 index 00000000..74248fcd --- /dev/null +++ b/.github/workflows/prebuild.yaml @@ -0,0 +1,44 @@ +name: ccpp-prebuild + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + unit-tests: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.10] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: ccpp-prebuild unit tests + run: | + export PYTHONPATH=$(pwd)/scripts:$(pwd)/scripts/parse_tools + cd test_prebuild + python3 test_metadata_parser.py + python3 test_mkstatic.py + + - name: ccpp-prebuild blocked data tests + run: | + cd test_prebuild/test_blocked_data + python3 ../../scripts/ccpp_prebuild.py --config=ccpp_prebuild_config.py --builddir=build + cd build + cmake .. + make + ./test_blocked_data.x From 3714ef8d4aaabd5632e5f2a7b5b6d90d738f9d35 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 1 Dec 2023 11:39:25 -0700 Subject: [PATCH 24/38] Debugging CI workflow for prebuild --- .github/workflows/prebuild.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/prebuild.yaml b/.github/workflows/prebuild.yaml index 74248fcd..80e34608 100644 --- a/.github/workflows/prebuild.yaml +++ b/.github/workflows/prebuild.yaml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.10] + python-version: ["3.10"] steps: - uses: actions/checkout@v4 @@ -26,14 +26,17 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: ccpp-prebuild unit tests run: | export PYTHONPATH=$(pwd)/scripts:$(pwd)/scripts/parse_tools cd test_prebuild python3 test_metadata_parser.py python3 test_mkstatic.py - - name: ccpp-prebuild blocked data tests run: | cd test_prebuild/test_blocked_data From d2e446ededa890c5e379733cbf5df1962cd97775 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 1 Dec 2023 11:51:55 -0700 Subject: [PATCH 25/38] Self-review --- test_prebuild/test_blocked_data/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test_prebuild/test_blocked_data/CMakeLists.txt b/test_prebuild/test_blocked_data/CMakeLists.txt index 1dc0f3db..921c87e6 100644 --- a/test_prebuild/test_blocked_data/CMakeLists.txt +++ b/test_prebuild/test_blocked_data/CMakeLists.txt @@ -53,7 +53,7 @@ else(API) message(STATUS "Got CCPP API from cmakefile include file: ${API}") endif(API) -set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check -ggdb -fbacktrace -ffree-line-length-none") #" -cpp -fno-range-check") +set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check -ggdb -fbacktrace -ffree-line-length-none") #------------------------------------------------------------------------------ add_library(ccpp_blocked_data STATIC ${SCHEMES} ${CAPS} ${API}) @@ -88,4 +88,3 @@ install(EXPORT ccpp_blocked_data-targets # Define where to install the C headers and Fortran modules #install(FILES ${HEADERS_C} DESTINATION include) install(FILES ${MODULES_F90} DESTINATION include) - From 9b206f10e3054bbc113e1b73c1910fa2dca50155 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Mon, 4 Dec 2023 08:39:56 -0700 Subject: [PATCH 26/38] Revert "Merge branch 'main' of https://github.com/NCAR/ccpp-framework into feature_capgen_unit_conversions" This reverts commit 998cc7bf97ac39d5682092f6fd14ca94b09adb3e, reversing changes made to 3b045946dcf6bdfe70afa9603233eef2ab3b25cf. --- scripts/mkstatic.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index f86f738f..576abdf8 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -1611,14 +1611,6 @@ def write(self, metadata_request, metadata_define, arguments, debug): if subcycle_body: body += subcycle_body_prefix + subcycle_body + subcycle_body_suffix - #For the init stage, for the case when the suite doesn't have any schemes with init phases, - #we still need to add the host-supplied ccpp_t variable to the init group caps so that it is - #available for setting the initialized flag for the particular instance being called. Otherwise, - #the initialized_set_block for the init phase tries to reference the unavailable ccpp_t variable. - if (ccpp_stage == 'init' and not self.parents[ccpp_stage]): - ccpp_var.intent = 'in' - self.parents[ccpp_stage].update({ccpp_var.local_name:ccpp_var}) - # Get list of arguments, module use statement and variable definitions for this subroutine (=stage for the group) (self.arguments[ccpp_stage], sub_module_use, sub_var_defs) = create_arguments_module_use_var_defs( self.parents[ccpp_stage], metadata_define, tmpvars.values()) From b5cc909b4553fe3a99c7c5fa17be1f8ce8419be9 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Mon, 4 Dec 2023 08:54:12 -0700 Subject: [PATCH 27/38] Activate var_action_test --- test/run_fortran_tests.sh | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/test/run_fortran_tests.sh b/test/run_fortran_tests.sh index 8b0f5bcb..fe34b3c0 100755 --- a/test/run_fortran_tests.sh +++ b/test/run_fortran_tests.sh @@ -38,14 +38,12 @@ if [ $res -ne 0 ]; then fi # Run var_action test -# TODO: Re-enable after feature fully implemented. -# ./var_action_test/run_test -# res=$? -# errcnt=$((errcnt + res)) -# if [ $res -ne 0 ]; then -# echo "Failure running var_action test" -# fi -echo "Skipping var_action_test/run_test until feature is fully implemented" + ./var_action_test/run_test + res=$? + errcnt=$((errcnt + res)) + if [ $res -ne 0 ]; then + echo "Failure running var_action test" + fi if [ $errcnt -eq 0 ]; then echo "All tests PASSed!" From 4ec1cbae82d1c3cf4a3e3bb8faae71ec5ff08831 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 4 Dec 2023 10:16:00 -0700 Subject: [PATCH 28/38] test_prebuild/test_blocked_data: 'use, intrinsic :: iso_fortran_env, only: error_unit' --- .../test_blocked_data/blocked_data_scheme.F90 | 12 ++++++---- test_prebuild/test_blocked_data/main.F90 | 24 ++++++++++--------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/test_prebuild/test_blocked_data/blocked_data_scheme.F90 b/test_prebuild/test_blocked_data/blocked_data_scheme.F90 index 12a610d2..eeda2206 100644 --- a/test_prebuild/test_blocked_data/blocked_data_scheme.F90 +++ b/test_prebuild/test_blocked_data/blocked_data_scheme.F90 @@ -4,7 +4,9 @@ module blocked_data_scheme + use, intrinsic :: iso_fortran_env, only: error_unit implicit none + private public :: blocked_data_scheme_init, & blocked_data_scheme_timestep_init, & @@ -28,7 +30,7 @@ subroutine blocked_data_scheme_init(data_array, errmsg, errflg) errmsg = '' errflg = 0 ! Check size of data array - write(0,'(a,i3)') 'In blocked_data_scheme_init: checking size of data array to be', sum(data_array_sizes) + write(error_unit,'(a,i3)') 'In blocked_data_scheme_init: checking size of data array to be', sum(data_array_sizes) if (size(data_array)/=sum(data_array_sizes)) then write(errmsg,'(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(data_array) errflg = 1 @@ -47,7 +49,7 @@ subroutine blocked_data_scheme_timestep_init(data_array, errmsg, errflg) errmsg = '' errflg = 0 ! Check size of data array - write(0,'(a,i3)') 'In blocked_data_scheme_timestep_init: checking size of data array to be', sum(data_array_sizes) + write(error_unit,'(a,i3)') 'In blocked_data_scheme_timestep_init: checking size of data array to be', sum(data_array_sizes) if (size(data_array)/=sum(data_array_sizes)) then write(errmsg,'(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), " but got ", size(data_array) errflg = 1 @@ -67,7 +69,7 @@ subroutine blocked_data_scheme_run(nb, data_array, errmsg, errflg) errmsg = '' errflg = 0 ! Check size of data array - write(0,'(2(a,i3))') 'In blocked_data_scheme_run: checking size of data array for block', nb, ' to be', data_array_sizes(nb) + write(error_unit,'(2(a,i3))') 'In blocked_data_scheme_run: checking size of data array for block', nb, ' to be', data_array_sizes(nb) if (size(data_array)/=data_array_sizes(nb)) then write(errmsg,'(a,i4)') "Error in blocked_data_scheme_run, expected size(data_array)==6, got ", size(data_array) errflg = 1 @@ -86,7 +88,7 @@ subroutine blocked_data_scheme_timestep_finalize(data_array, errmsg, errflg) errmsg = '' errflg = 0 ! Check size of data array - write(0,'(a,i3)') 'In blocked_data_scheme_timestep_finalize: checking size of data array to be', sum(data_array_sizes) + write(error_unit,'(a,i3)') 'In blocked_data_scheme_timestep_finalize: checking size of data array to be', sum(data_array_sizes) if (size(data_array)/=sum(data_array_sizes)) then write(errmsg,'(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(data_array) errflg = 1 @@ -105,7 +107,7 @@ subroutine blocked_data_scheme_finalize(data_array, errmsg, errflg) errmsg = '' errflg = 0 ! Check size of data array - write(0,'(a,i3)') 'In blocked_data_scheme_finalize: checking size of data array to be', sum(data_array_sizes) + write(error_unit,'(a,i3)') 'In blocked_data_scheme_finalize: checking size of data array to be', sum(data_array_sizes) if (size(data_array)/=sum(data_array_sizes)) then write(errmsg,'(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(data_array) errflg = 1 diff --git a/test_prebuild/test_blocked_data/main.F90 b/test_prebuild/test_blocked_data/main.F90 index 763841aa..cc57f618 100644 --- a/test_prebuild/test_blocked_data/main.F90 +++ b/test_prebuild/test_blocked_data/main.F90 @@ -1,5 +1,7 @@ program test_blocked_data + use, intrinsic :: iso_fortran_env, only: error_unit + use ccpp_types, only: ccpp_t use data, only: nblks, blksz, ncols use data, only: ccpp_data_domain, ccpp_data_blocks, & @@ -35,7 +37,7 @@ program test_blocked_data do ib=1,size(blocked_data_instance) allocate(blocked_data_instance(ib)%array_data(blksz(ib))) - write(0,'(2(a,i3))') "Allocated array_data for block", ib, " to size", size(blocked_data_instance(ib)%array_data) + write(error_unit,'(2(a,i3))') "Allocated array_data for block", ib, " to size", size(blocked_data_instance(ib)%array_data) end do !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -45,8 +47,8 @@ program test_blocked_data cdata => ccpp_data_domain call ccpp_physics_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) if (ierr/=0) then - write(0,'(a)') "An error occurred in ccpp_physics_init:" - write(0,'(a)') trim(cdata%errmsg) + write(error_unit,'(a)') "An error occurred in ccpp_physics_init:" + write(error_unit,'(a)') trim(cdata%errmsg) stop 1 end if @@ -57,8 +59,8 @@ program test_blocked_data cdata => ccpp_data_domain call ccpp_physics_timestep_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) if (ierr/=0) then - write(0,'(a)') "An error occurred in ccpp_physics_timestep_init:" - write(0,'(a)') trim(cdata%errmsg) + write(error_unit,'(a)') "An error occurred in ccpp_physics_timestep_init:" + write(error_unit,'(a)') trim(cdata%errmsg) stop 1 end if @@ -70,8 +72,8 @@ program test_blocked_data cdata => ccpp_data_blocks(ib) call ccpp_physics_run(cdata, suite_name=trim(ccpp_suite), ierr=ierr) if (ierr/=0) then - write(0,'(a,i3,a)') "An error occurred in ccpp_physics_run for block", ib, ":" - write(0,'(a)') trim(cdata%errmsg) + write(error_unit,'(a,i3,a)') "An error occurred in ccpp_physics_run for block", ib, ":" + write(error_unit,'(a)') trim(cdata%errmsg) stop 1 end if end do @@ -83,8 +85,8 @@ program test_blocked_data cdata => ccpp_data_domain call ccpp_physics_timestep_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) if (ierr/=0) then - write(0,'(a)') "An error occurred in ccpp_physics_timestep_init:" - write(0,'(a)') trim(cdata%errmsg) + write(error_unit,'(a)') "An error occurred in ccpp_physics_timestep_init:" + write(error_unit,'(a)') trim(cdata%errmsg) stop 1 end if @@ -95,8 +97,8 @@ program test_blocked_data cdata => ccpp_data_domain call ccpp_physics_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) if (ierr/=0) then - write(0,'(a)') "An error occurred in ccpp_physics_timestep_init:" - write(0,'(a)') trim(cdata%errmsg) + write(error_unit,'(a)') "An error occurred in ccpp_physics_timestep_init:" + write(error_unit,'(a)') trim(cdata%errmsg) stop 1 end if From e6ffbd54467767e647b0d96ea9eae85f2f73d9b5 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Mon, 4 Dec 2023 14:11:41 -0700 Subject: [PATCH 29/38] Fix unit tests for variable transformations. Reorganization a bit to be consistent with how the DEBUG checking is implemented in PR #512 --- scripts/suite_objects.py | 90 +++++++++++++++++--------- scripts/var_props.py | 22 ++++--- test/unit_tests/test_var_transforms.py | 55 +++++----------- 3 files changed, 88 insertions(+), 79 deletions(-) mode change 100644 => 100755 test/unit_tests/test_var_transforms.py diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index 405909c9..b22119d3 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1210,31 +1210,8 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): # end if # Are there any forward/reverse transforms for this variable? if compat_obj is not None and (compat_obj.has_vert_transforms or compat_obj.has_unit_transforms): - # Add local variable (_local) needed for transformation. - tmp_var = var.clone(var.get_prop_value('local_name')+'_local') - self.__group.manage_variable(tmp_var) - - # Create indices, flipping if necessary. - indices = [':']*var.get_rank() - if compat_obj.has_vert_transforms: - dim = find_vertical_dimension(var.get_dimensions()) - vdim_name = vert_dim.split(':')[-1] - group_vvar = self.__group.call_list.find_variable(vdim_name) - vname = group_vvar.get_prop_value('local_name') - indices[dim[1]] = vname+':1:-1' - - # Add any forward transforms. - if (var.get_prop_value('intent') != 'in'): - self.__forward_transforms.append( - compat_obj.forward_transform(lvar_lname=var.get_prop_value('local_name'), - rvar_lname=tmp_var.get_prop_value('local_name'), - indices=indices)) - # Add any reverse transforms. - if (var.get_prop_value('intent') != 'out'): - self.__reverse_transforms.append( - compat_obj.reverse_transform(lvar_lname=tmp_var.get_prop_value('local_name'), - rvar_lname=var.get_prop_value('local_name'), - indices=indices)) + self.add_var_transform(var, compat_obj, vert_dim) + # end for if self.needs_vertical is not None: self.parent.add_part(self, replace=True) # Should add a vloop @@ -1246,6 +1223,58 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): # end if return scheme_mods + def add_var_transform(self, var, compat_obj, vert_dim): + """Add variable transformation before/after call to Scheme in """ + # Add dummy variable (_local) needed for transformation. + dummy = var.clone(var.get_prop_value('local_name')+'_local') + self.__group.manage_variable(dummy) + + # Create indices for transform. + lindices = [':']*var.get_rank() + rindices = [':']*var.get_rank() + + # If needed, modify vertical dimension for vertical orientation flipping + dim = find_vertical_dimension(var.get_dimensions()) + vdim_name = vert_dim.split(':')[-1] + group_vvar = self.__group.call_list.find_variable(vdim_name) + vname = group_vvar.get_prop_value('local_name') + lindices[dim[1]] = '1:'+vname + rindices[dim[1]] = '1:'+vname + if compat_obj.has_vert_transforms: + rindices[dim[1]] = vname+':1:-1' + + # If needed, modify horizontal dimension for loop substitution. + dim = find_horizontal_dimension(var.get_dimensions()) + if compat_obj.has_dim_transforms: + print("SWALES: ",dim) + + # Add any forward transforms. + if (var.get_prop_value('intent') != 'in'): + self.__forward_transforms.append([var.get_prop_value('local_name'), + dummy.get_prop_value('local_name'), + lindices, rindices, compat_obj]) + + # Add any reverse transforms. + if (var.get_prop_value('intent') != 'out'): + self.__reverse_transforms.append([dummy.get_prop_value('local_name'), + var.get_prop_value('local_name'), + rindices, lindices, compat_obj]) + + def write_var_transform(self, var, dummy, rindices, lindices, compat_obj, + outfile, indent, forward): + """Write variable transformation needed to call this Scheme """ + if forward: + stmt = compat_obj.forward_transform(lvar_lname=dummy, + rvar_lname=var, + lvar_indices=lindices, + rvar_indices=rindices) + else: + stmt = compat_obj.reverse_transform(lvar_lname=var, + rvar_lname=dummy, + lvar_indices=rindices, + rvar_indices=lindices) + outfile.write(stmt, indent+1) + def write(self, outfile, errcode, indent): # Unused arguments are for consistent write interface # pylint: disable=unused-argument @@ -1257,14 +1286,17 @@ def write(self, outfile, errcode, indent): my_args = self.call_list.call_string(cldicts=cldicts, is_func_call=True, subname=self.subroutine_name) - stmt = 'call {}({})' - # Write the scheme call. + outfile.write('if ({} == 0) then'.format(errcode), indent) # Write any reverse transforms. - for reverse_transform in self.__reverse_transforms: outfile.write(reverse_transform, indent+1) + for (dummy, var, rindices, lindices, compat_obj) in self.__reverse_transforms: + tstmt = self.write_var_transform(dummy, var, rindices, lindices, compat_obj, outfile, indent, False) + # Write the scheme call. + stmt = 'call {}({})' outfile.write(stmt.format(self.subroutine_name, my_args), indent+1) # Write any forward transforms. - for forward_transform in self.__forward_transforms: outfile.write(forward_transform, indent+1) + for (var, dummy, lindices, rindices, compat_obj) in self.__forward_transforms: + tstmt = self.write_var_transform(dummy, var, rindices, lindices, compat_obj, outfile, indent, True) outfile.write('end if', indent) def schemes(self): diff --git a/scripts/var_props.py b/scripts/var_props.py index 0c4c33e6..70bb89b9 100755 --- a/scripts/var_props.py +++ b/scripts/var_props.py @@ -953,13 +953,15 @@ def __init__(self, var1_stdname, var1_type, var1_kind, var1_units, # end if self.__incompat_reason = " and ".join([x for x in incompat_reason if x]) - def forward_transform(self, lvar_lname, rvar_lname, indices, + def forward_transform(self, lvar_lname, rvar_lname, rvar_indices, lvar_indices, adjust_hdim=None, flip_vdim=None): """Compute and return the the forward transform from "var1" to "var2". is the local name of "var2". is the local name of "var1". - is a tuple of the loop indices for "var1" (i.e., "var1" - will show up in the RHS of the transform as "var1(indices)". + is a tuple of the loop indices for "var1" (i.e., "var1" + will show up in the RHS of the transform as "var1(rvar_indices)". + is a tuple of the loop indices for "var1" (i.e., "var2" + will show up in the LHS of the transform as "var2(lvar_indices)". If is not None, it should be a string containing the local name of the "horizontal_loop_begin" variable. This is used to compute the offset in the horizontal axis index between one and @@ -972,8 +974,8 @@ def forward_transform(self, lvar_lname, rvar_lname, indices, "vertical_interface_dimension"). """ # Dimension transform (Indices handled externally) - rhs_term = f"{rvar_lname}({','.join(indices)})" - lhs_term = f"{lvar_lname}" + rhs_term = f"{rvar_lname}({','.join(rvar_indices)})" + lhs_term = f"{lvar_lname}({','.join(lvar_indices)})" if self.has_kind_transforms: kind = self.__kind_transforms[1] @@ -991,13 +993,13 @@ def forward_transform(self, lvar_lname, rvar_lname, indices, # end if return f"{lhs_term} = {rhs_term}" - def reverse_transform(self, lvar_lname, rvar_lname, indices, + def reverse_transform(self, lvar_lname, rvar_lname, rvar_indices, lvar_indices, adjust_hdim=None, flip_vdim=None): """Compute and return the the reverse transform from "var2" to "var1". is the local name of "var1". is the local name of "var2". - is a tuple of the loop indices for "var2" (i.e., "var2" - will show up in the RHS of the transform as "var2(indices)". + is a tuple of the loop indices for "var1" (i.e., "var1" + will show up in the RHS of the transform as "var1(rvar_indices)". If is not None, it should be a string containing the local name of the "horizontal_loop_begin" variable. This is used to compute the offset in the horizontal axis index between one and @@ -1010,8 +1012,8 @@ def reverse_transform(self, lvar_lname, rvar_lname, indices, "vertical_interface_dimension"). """ # Dimension transforms (Indices handled exrernally) - lhs_term = f"{lvar_lname}({','.join(indices)})" - rhs_term = f"{rvar_lname}" + lhs_term = f"{lvar_lname}({','.join(lvar_indices)})" + rhs_term = f"{rvar_lname}({','.join(rvar_indices)})" if self.has_kind_transforms: kind = self.__kind_transforms[0] diff --git a/test/unit_tests/test_var_transforms.py b/test/unit_tests/test_var_transforms.py old mode 100644 new mode 100755 index ae71008f..94a6c4a0 --- a/test/unit_tests/test_var_transforms.py +++ b/test/unit_tests/test_var_transforms.py @@ -282,13 +282,12 @@ def test_valid_dim_transforms(self): self.assertIsInstance(compat, VarCompatObj, msg=self.__inst_emsg.format(type(compat))) rindices = ("hind", "vind") - fwd_stmt = compat.forward_transform(v2_lname, v1_lname, rindices, - adjust_hdim=None, flip_vdim=None) + lindices = rindices + fwd_stmt = compat.forward_transform(v2_lname, v1_lname, rindices, lindices) ind_str = ','.join(rindices) expected = f"{v2_lname}({ind_str}) = {v1_lname}({ind_str})" self.assertEqual(fwd_stmt, expected) - rev_stmt = compat.reverse_transform(v1_lname, v2_lname, rindices, - adjust_hdim=None, flip_vdim=None) + rev_stmt = compat.reverse_transform(v1_lname, v2_lname, rindices, lindices) expected = f"{v1_lname}({ind_str}) = {v2_lname}({ind_str})" self.assertEqual(rev_stmt, expected) @@ -298,17 +297,13 @@ def test_valid_dim_transforms(self): msg=self.__inst_emsg.format(type(compat))) rindices = ("hind", "vind") lindices = ("hind-col_start+1", "vind") - fwd_stmt = compat.forward_transform(v2_lname, v1_lname, rindices, - adjust_hdim='col_start', - flip_vdim=None) + fwd_stmt = compat.forward_transform(v2_lname, v1_lname, rindices, lindices) lind_str = ','.join(lindices) rind_str = ','.join(rindices) expected = f"{v2_lname}({lind_str}) = {v1_lname}({rind_str})" self.assertEqual(fwd_stmt, expected) - rev_stmt = compat.reverse_transform(v1_lname, v2_lname, rindices, - adjust_hdim='col_start', - flip_vdim=None) lindices = ("hind+col_start-1", "vind") + rev_stmt = compat.reverse_transform(v1_lname, v2_lname, rindices, lindices) lind_str = ','.join(lindices) expected = f"{v1_lname}({lind_str}) = {v2_lname}({rind_str})" self.assertEqual(rev_stmt, expected) @@ -320,17 +315,13 @@ def test_valid_dim_transforms(self): msg=self.__inst_emsg.format(type(compat))) rindices = ("hind", "vind") lindices = ("hind-col_start+1", "pver-vind+1") - fwd_stmt = compat.forward_transform(v2_lname, v1_lname, rindices, - adjust_hdim='col_start', - flip_vdim='pver') + fwd_stmt = compat.forward_transform(v2_lname, v1_lname, rindices, lindices) lind_str = ','.join(lindices) rind_str = ','.join(rindices) expected = f"{v2_lname}({lind_str}) = {v1_lname}({rind_str})" self.assertEqual(fwd_stmt, expected) - rev_stmt = compat.reverse_transform(v1_lname, v2_lname, rindices, - adjust_hdim='col_start', - flip_vdim='pver') lindices = ("hind+col_start-1", "pver-vind+1") + rev_stmt = compat.reverse_transform(v1_lname, v2_lname, rindices, lindices) lind_str = ','.join(lindices) expected = f"{v1_lname}({lind_str}) = {v2_lname}({rind_str})" self.assertEqual(rev_stmt, expected) @@ -342,17 +333,13 @@ def test_valid_dim_transforms(self): rindices = ("hind", "vind") lindices = ("hind-col_start+1", "vind") conv = f"273.15_{real_array1.get_prop_value('kind')}" - fwd_stmt = compat.forward_transform(v3_lname, v1_lname, rindices, - adjust_hdim='col_start', - flip_vdim=None) + fwd_stmt = compat.forward_transform(v3_lname, v1_lname, rindices, lindices) lind_str = ','.join(lindices) rind_str = ','.join(rindices) expected = f"{v3_lname}({lind_str}) = {v1_lname}({rind_str})+{conv}" self.assertEqual(fwd_stmt, expected) - rev_stmt = compat.reverse_transform(v1_lname, v3_lname, rindices, - adjust_hdim='col_start', - flip_vdim=None) lindices = ("hind+col_start-1", "vind") + rev_stmt = compat.reverse_transform(v1_lname, v3_lname, rindices, lindices) lind_str = ','.join(lindices) conv = f"273.15_{real_array2.get_prop_value('kind')}" expected = f"{v1_lname}({lind_str}) = {v3_lname}({rind_str})-{conv}" @@ -364,18 +351,14 @@ def test_valid_dim_transforms(self): msg=self.__inst_emsg.format(type(compat))) rindices = ("hind", "vind") lindices = ("hind", "vind") - fwd_stmt = compat.forward_transform(v4_lname, v3_lname, rindices, - adjust_hdim='col_start', - flip_vdim=None) + fwd_stmt = compat.forward_transform(v4_lname, v3_lname, rindices, lindices) lind_str = ','.join(lindices) rind_str = ','.join(rindices) rkind = real_array3.get_prop_value('kind') expected = f"{v4_lname}({lind_str}) = real({v3_lname}({rind_str}), {rkind})" self.assertEqual(fwd_stmt, expected) - rev_stmt = compat.reverse_transform(v3_lname, v4_lname, rindices, - adjust_hdim='col_start', - flip_vdim=None) lindices = ("hind", "vind") + rev_stmt = compat.reverse_transform(v3_lname, v4_lname, rindices, lindices) lind_str = ','.join(lindices) rkind = real_array4.get_prop_value('kind') expected = f"{v3_lname}({lind_str}) = real({v4_lname}({rind_str}), {rkind})" @@ -389,17 +372,13 @@ def test_valid_dim_transforms(self): lindices = ("hind-col_start+1", "vind") rkind = real_array4.get_prop_value('kind') conv = f"273.15_{rkind}" - fwd_stmt = compat.forward_transform(v2_lname, v1_lname, rindices, - adjust_hdim='col_start', - flip_vdim=None) + fwd_stmt = compat.forward_transform(v2_lname, v1_lname, rindices, lindices) lind_str = ','.join(lindices) rind_str = ','.join(rindices) expected = f"{v2_lname}({lind_str}) = real({v1_lname}({rind_str}), {rkind})+{conv}" self.assertEqual(fwd_stmt, expected) - rev_stmt = compat.reverse_transform(v1_lname, v2_lname, rindices, - adjust_hdim='col_start', - flip_vdim=None) lindices = ("hind+col_start-1", "vind") + rev_stmt = compat.reverse_transform(v1_lname, v2_lname, rindices, lindices) lind_str = ','.join(lindices) rkind = real_array1.get_prop_value('kind') conv = f"273.15_{rkind}" @@ -413,9 +392,7 @@ def test_valid_dim_transforms(self): msg=self.__inst_emsg.format(type(compat))) rindices = ("hind", "vind") lindices = ("pver-vind+1", "hind-col_start+1") - fwd_stmt = compat.forward_transform(v4_lname, v5_lname, rindices, - adjust_hdim='col_start', - flip_vdim='pver') + fwd_stmt = compat.forward_transform(v4_lname, v5_lname, rindices, lindices) lind_str = ','.join(lindices) rind_str = ','.join(rindices) rkind = real_array3.get_prop_value('kind') @@ -423,10 +400,8 @@ def test_valid_dim_transforms(self): self.assertEqual(fwd_stmt, expected) rindices = ("vind", "hind") rind_str = ','.join(rindices) - rev_stmt = compat.reverse_transform(v5_lname, v4_lname, rindices, - adjust_hdim='col_start', - flip_vdim='pver') lindices = ("hind+col_start-1", "pver-vind+1") + rev_stmt = compat.reverse_transform(v5_lname, v4_lname, rindices, lindices) lind_str = ','.join(lindices) rkind = real_array4.get_prop_value('kind') expected = f"{v5_lname}({lind_str}) = {v4_lname}({rind_str})" From e9c79eb06ae76b9cbad6ccb2778696a80c2d13b1 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Mon, 4 Dec 2023 14:18:32 -0700 Subject: [PATCH 30/38] Cleanup forward/reverse transform terminology --- scripts/suite_objects.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index b22119d3..c9b6d042 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1248,28 +1248,28 @@ def add_var_transform(self, var, compat_obj, vert_dim): if compat_obj.has_dim_transforms: print("SWALES: ",dim) - # Add any forward transforms. + # Add any reverse transforms. if (var.get_prop_value('intent') != 'in'): - self.__forward_transforms.append([var.get_prop_value('local_name'), + self.__reverse_transforms.append([var.get_prop_value('local_name'), dummy.get_prop_value('local_name'), lindices, rindices, compat_obj]) - # Add any reverse transforms. + # Add any forward transforms. if (var.get_prop_value('intent') != 'out'): - self.__reverse_transforms.append([dummy.get_prop_value('local_name'), + self.__forward_transforms.append([dummy.get_prop_value('local_name'), var.get_prop_value('local_name'), rindices, lindices, compat_obj]) def write_var_transform(self, var, dummy, rindices, lindices, compat_obj, outfile, indent, forward): """Write variable transformation needed to call this Scheme """ - if forward: - stmt = compat_obj.forward_transform(lvar_lname=dummy, + if not forward: + stmt = compat_obj.reverse_transform(lvar_lname=dummy, rvar_lname=var, lvar_indices=lindices, rvar_indices=rindices) else: - stmt = compat_obj.reverse_transform(lvar_lname=var, + stmt = compat_obj.forward_transform(lvar_lname=var, rvar_lname=dummy, lvar_indices=rindices, rvar_indices=lindices) @@ -1288,15 +1288,15 @@ def write(self, outfile, errcode, indent): subname=self.subroutine_name) outfile.write('if ({} == 0) then'.format(errcode), indent) - # Write any reverse transforms. - for (dummy, var, rindices, lindices, compat_obj) in self.__reverse_transforms: - tstmt = self.write_var_transform(dummy, var, rindices, lindices, compat_obj, outfile, indent, False) + # Write any forward transforms. + for (dummy, var, rindices, lindices, compat_obj) in self.__forward_transforms: + tstmt = self.write_var_transform(var, dummy, rindices, lindices, compat_obj, outfile, indent, False) # Write the scheme call. stmt = 'call {}({})' outfile.write(stmt.format(self.subroutine_name, my_args), indent+1) - # Write any forward transforms. - for (var, dummy, lindices, rindices, compat_obj) in self.__forward_transforms: - tstmt = self.write_var_transform(dummy, var, rindices, lindices, compat_obj, outfile, indent, True) + # Write any reverse transforms. + for (var, dummy, lindices, rindices, compat_obj) in self.__reverse_transforms: + tstmt = self.write_var_transform(var, dummy, rindices, lindices, compat_obj, outfile, indent, True) outfile.write('end if', indent) def schemes(self): From bb4975af98ae5f286f77f1f08c1a85c32d50d733 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Mon, 4 Dec 2023 15:23:58 -0700 Subject: [PATCH 31/38] Doc tests for transformations --- scripts/var_props.py | 60 +++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 34 deletions(-) diff --git a/scripts/var_props.py b/scripts/var_props.py index 70bb89b9..73a9b4b3 100755 --- a/scripts/var_props.py +++ b/scripts/var_props.py @@ -791,49 +791,41 @@ class VarCompatObj: kind_types=["kind_phys=REAL64", \ "kind_dyn=REAL32", \ "kind_host=REAL64"]) - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", [], \ - "var1_lname", "var_stdname", "real", "kind_phys", \ - "m", [], "var2_lname", _DOCTEST_RUNENV) #doctest: +ELLIPSIS + >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", [], "var1_lname", False,\ + "var_stdname", "real", "kind_phys", "m", [], "var2_lname", False,\ + _DOCTEST_RUNENV) #doctest: +ELLIPSIS # Test that a 2-D var with no horizontal transform works - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", \ - ['horizontal_dimension'], "var1_lname", "var_stdname", \ - "real", "kind_phys", "m", ['horizontal_dimension'], \ - "var2_lname", _DOCTEST_RUNENV) #doctest: +ELLIPSIS + >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['horizontal_dimension'], "var1_lname", False, \ + "var_stdname", "real", "kind_phys", "m", ['horizontal_dimension'], "var2_lname", False, \ + _DOCTEST_RUNENV) #doctest: +ELLIPSIS # Test that a 2-D var with a horizontal transform works - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", \ - ['horizontal_dimension'], "var1_lname", "var_stdname", \ - "real", "kind_phys", "m", ['horizontal_loop_extent'], \ - "var2_lname", _DOCTEST_RUNENV) #doctest: +ELLIPSIS + >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['horizontal_dimension'], "var1_lname", False, \ + "var_stdname", "real", "kind_phys", "m", ['horizontal_loop_extent'], "var2_lname", False, \ + _DOCTEST_RUNENV) #doctest: +ELLIPSIS # Test that a 2-D var with unit conversion m->km works - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", \ - ['horizontal_dimension'], "var1_lname", "var_stdname", \ - "real", "kind_phys", "km", ['horizontal_dimension'], \ - "var2_lname", _DOCTEST_RUNENV) #doctest: +ELLIPSIS + >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['horizontal_dimension'], "var1_lname", False, \ + "var_stdname", "real", "kind_phys", "km", ['horizontal_dimension'], "var2_lname", False, \ + _DOCTEST_RUNENV) #doctest: +ELLIPSIS # Test that a 2-D var with unit conversion m->km works and that it # produces the correct forward transformation - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", \ - ['horizontal_dimension'], "var1_lname", "var_stdname", \ - "real", "kind_phys", "km", ['horizontal_dimension'], \ - "var2_lname", _DOCTEST_RUNENV).forward_transform( \ - "var1_lname", "var2_lname", ('i')) + >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['horizontal_dimension'], "var1_lname", False, \ + "var_stdname", "real", "kind_phys", "km", ['horizontal_dimension'], "var2_lname", False, \ + _DOCTEST_RUNENV).forward_transform("var1_lname", "var2_lname", 'i', 'i') 'var1_lname(i) = 1.0E-3_kind_phys*var2_lname(i)' # Test that a 3-D var with unit conversion m->km and vertical flipping # works and that it produces the correct reverse transformation - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", \ - ['horizontal_dimension', 'vertical_layer_dimension'], \ - "var1_lname", "var_stdname", "real", "kind_phys", "km",\ - ['horizontal_dimension', 'vertical_layer_dimension'], \ - "var2_lname", _DOCTEST_RUNENV).reverse_transform( \ - "var1_lname", "var2_lname", ('i','k'), flip_vdim='nk') + >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['horizontal_dimension', 'vertical_layer_dimension'], "var1_lname", False,\ + "var_stdname", "real", "kind_phys", "km",['horizontal_dimension', 'vertical_layer_dimension'], "var2_lname", True, \ + _DOCTEST_RUNENV).reverse_transform("var1_lname", "var2_lname", ('i','k'), ('i','nk-k+1')) 'var1_lname(i,nk-k+1) = 1.0E+3_kind_phys*var2_lname(i,k)' """ @@ -844,9 +836,9 @@ def __init__(self, var1_stdname, var1_type, var1_kind, var1_units, """Initialize this object with information on the equivalence and/or conformability of two variables. variable 1 is described by , , , - , , , and . + , , , , and . variable 2 is described by , , , - , , , and . + , , , , and . is the CCPPFrameworkEnv object used here to verify kind equivalence or to produce kind transformations. """ @@ -1052,9 +1044,9 @@ def _get_kind_convstrs(self, var1_kind, var2_kind, run_env): >>> _DOCTEST_CONTEXT1 = ParseContext(linenum=3, filename='foo.F90') >>> _DOCTEST_CONTEXT2 = ParseContext(linenum=5, filename='bar.F90') >>> _DOCTEST_VCOMPAT = VarCompatObj("var_stdname", "real", "kind_phys", \ - "m", [], "var1_lname", "var_stdname", \ + "m", [], "var1_lname", False, "var_stdname", \ "real", "kind_phys", "m", [], \ - "var2_lname", _DOCTEST_RUNENV, \ + "var2_lname", False, _DOCTEST_RUNENV, \ v1_context=_DOCTEST_CONTEXT1, \ v2_context=_DOCTEST_CONTEXT2) @@ -1106,9 +1098,9 @@ def _get_unit_convstrs(self, var1_units, var2_units): >>> _DOCTEST_CONTEXT1 = ParseContext(linenum=3, filename='foo.F90') >>> _DOCTEST_CONTEXT2 = ParseContext(linenum=5, filename='bar.F90') >>> _DOCTEST_VCOMPAT = VarCompatObj("var_stdname", "real", "kind_phys", \ - "m", [], "var1_lname", "var_stdname", \ + "m", [], "var1_lname", False, "var_stdname", \ "real", "kind_phys", "m", [], \ - "var2_lname", _DOCTEST_RUNENV, \ + "var2_lname", False, _DOCTEST_RUNENV, \ v1_context=_DOCTEST_CONTEXT1, \ v2_context=_DOCTEST_CONTEXT2) @@ -1178,9 +1170,9 @@ def _get_dim_transforms(self, var1_dims, var2_dims): >>> _DOCTEST_CONTEXT1 = ParseContext(linenum=3, filename='foo.F90') >>> _DOCTEST_CONTEXT2 = ParseContext(linenum=5, filename='bar.F90') >>> _DOCTEST_VCOMPAT = VarCompatObj("var_stdname", "real", "kind_phys", \ - "m", [], "var1_lname", "var_stdname", \ + "m", [], "var1_lname", False, "var_stdname", \ "real", "kind_phys", "m", [], \ - "var2_lname", _DOCTEST_RUNENV, \ + "var2_lname", False, _DOCTEST_RUNENV, \ v1_context=_DOCTEST_CONTEXT1, \ v2_context=_DOCTEST_CONTEXT2) From 265a05b99092354c4745d792ccf11e92768962b7 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Dec 2023 09:09:07 -0700 Subject: [PATCH 32/38] Add more inline documentation. Address reviewers comments. --- scripts/suite_objects.py | 78 +++++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py index c9b6d042..b97c9ce6 100644 --- a/scripts/suite_objects.py +++ b/scripts/suite_objects.py @@ -1144,10 +1144,7 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): # end if scheme_mods = set() scheme_mods.add((my_header.module, self.subroutine_name)) - var_local = {"local_name":[], "std_name":[]} for var in my_header.variable_list(): - var_local["local_name"].append(var.get_prop_value('local_name')) - var_local["std_name"].append(var.get_prop_value('standard_name')) vstdname = var.get_prop_value('standard_name') def_val = var.get_prop_value('default_value') vdims = var.get_dimensions() @@ -1209,7 +1206,9 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): # end if # end if # Are there any forward/reverse transforms for this variable? - if compat_obj is not None and (compat_obj.has_vert_transforms or compat_obj.has_unit_transforms): + if compat_obj is not None and (compat_obj.has_vert_transforms or + compat_obj.has_unit_transforms or + compat_obj.has_kind_transforms): self.add_var_transform(var, compat_obj, vert_dim) # end for @@ -1224,51 +1223,74 @@ def analyze(self, phase, group, scheme_library, suite_vars, level): return scheme_mods def add_var_transform(self, var, compat_obj, vert_dim): - """Add variable transformation before/after call to Scheme in """ + """Register any variable transformation needed by for this Scheme. + For any transformation identified in , create dummy variable + from to perform the transformation. Determine the indices needed + for the transform and save for use during write stage""" + # Add dummy variable (_local) needed for transformation. dummy = var.clone(var.get_prop_value('local_name')+'_local') self.__group.manage_variable(dummy) - # Create indices for transform. + # Create indices (default) for transform. lindices = [':']*var.get_rank() rindices = [':']*var.get_rank() # If needed, modify vertical dimension for vertical orientation flipping - dim = find_vertical_dimension(var.get_dimensions()) + _, vdim = find_vertical_dimension(var.get_dimensions()) vdim_name = vert_dim.split(':')[-1] group_vvar = self.__group.call_list.find_variable(vdim_name) vname = group_vvar.get_prop_value('local_name') - lindices[dim[1]] = '1:'+vname - rindices[dim[1]] = '1:'+vname + lindices[vdim] = '1:'+vname + rindices[vdim] = '1:'+vname if compat_obj.has_vert_transforms: - rindices[dim[1]] = vname+':1:-1' + rindices[vdim] = vname+':1:-1' # If needed, modify horizontal dimension for loop substitution. - dim = find_horizontal_dimension(var.get_dimensions()) - if compat_obj.has_dim_transforms: - print("SWALES: ",dim) + # NOT YET IMPLEMENTED + #hdim = find_horizontal_dimension(var.get_dimensions()) + #if compat_obj.has_dim_transforms: - # Add any reverse transforms. - if (var.get_prop_value('intent') != 'in'): - self.__reverse_transforms.append([var.get_prop_value('local_name'), - dummy.get_prop_value('local_name'), - lindices, rindices, compat_obj]) - - # Add any forward transforms. + # + # Register any reverse (pre-Scheme) transforms. + # if (var.get_prop_value('intent') != 'out'): - self.__forward_transforms.append([dummy.get_prop_value('local_name'), + self.__reverse_transforms.append([dummy.get_prop_value('local_name'), var.get_prop_value('local_name'), rindices, lindices, compat_obj]) + # + # Register any forward (post-Scheme) transforms. + # + if (var.get_prop_value('intent') != 'in'): + self.__forward_transforms.append([var.get_prop_value('local_name'), + dummy.get_prop_value('local_name'), + lindices, rindices, compat_obj]) + def write_var_transform(self, var, dummy, rindices, lindices, compat_obj, outfile, indent, forward): - """Write variable transformation needed to call this Scheme """ + """Write variable transformation needed to call this Scheme in . + is the varaible that needs transformation before and after calling Scheme. + is the local variable needed for the transformation.. + are the LHS indices of for reverse transforms (before Scheme). + are the RHS indices of for reverse transforms (before Scheme). + are the LHS indices of for forward transforms (after Scheme). + are the RHS indices of for forward transforms (after Scheme). + """ + # + # Write reverse (pre-Scheme) transform. + # if not forward: + # dummy(lindices) = var(rindices) stmt = compat_obj.reverse_transform(lvar_lname=dummy, rvar_lname=var, lvar_indices=lindices, rvar_indices=rindices) + # + # Write forward (post-Scheme) transform. + # else: + # var(lindices) = dummy(rindices) stmt = compat_obj.forward_transform(lvar_lname=var, rvar_lname=dummy, lvar_indices=rindices, @@ -1288,21 +1310,21 @@ def write(self, outfile, errcode, indent): subname=self.subroutine_name) outfile.write('if ({} == 0) then'.format(errcode), indent) - # Write any forward transforms. - for (dummy, var, rindices, lindices, compat_obj) in self.__forward_transforms: + # Write any reverse (pre-Scheme) transforms. + for (dummy, var, rindices, lindices, compat_obj) in self.__reverse_transforms: tstmt = self.write_var_transform(var, dummy, rindices, lindices, compat_obj, outfile, indent, False) # Write the scheme call. stmt = 'call {}({})' outfile.write(stmt.format(self.subroutine_name, my_args), indent+1) - # Write any reverse transforms. - for (var, dummy, lindices, rindices, compat_obj) in self.__reverse_transforms: + # Write any forward (post-Scheme) transforms. + for (var, dummy, lindices, rindices, compat_obj) in self.__forward_transforms: tstmt = self.write_var_transform(var, dummy, rindices, lindices, compat_obj, outfile, indent, True) + # outfile.write('end if', indent) def schemes(self): """Return self as a list for consistency with subcycle""" return [self] - # end if def variable_list(self, recursive=False, std_vars=True, loop_vars=True, consts=True): @@ -1671,7 +1693,7 @@ def register_action(self, vaction): # Add the missing dim vaction.add_local(self, _API_LOCAL, self.run_env) return True - + # end if return False def manage_variable(self, newvar): From 0d0cfcd2ce13230de8675e5576542d7a240a875b Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Dec 2023 09:50:31 -0700 Subject: [PATCH 33/38] Address reviewer comment --- scripts/var_props.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/var_props.py b/scripts/var_props.py index 73a9b4b3..e38118c2 100755 --- a/scripts/var_props.py +++ b/scripts/var_props.py @@ -921,7 +921,7 @@ def __init__(self, var1_stdname, var1_type, var1_kind, var1_units, # end if if self.__compat: # Check for vertical array flipping (do later) - if var1_top or var2_top: + if (var1_top or var2_top) and (var1_top != var2_top): self.__compat = True self.has_vert_transforms = True # end if From f280ef57cdfd02054079fed2ba7026155442930b Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Dec 2023 10:01:37 -0700 Subject: [PATCH 34/38] Update authors in CMakeList. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4daf855b..4e0d6b24 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(ccpp_framework #------------------------------------------------------------------------------ # Set package definitions set(PACKAGE "ccpp-framework") -set(AUTHORS "Dom Heinzeller" "Grant Firl" "Mike Kavulich" "Steve Goldhaber") +set(AUTHORS "Dom Heinzeller" "Grant Firl" "Mike Kavulich" "Steve Goldhaber" "Dustin Swales") string(TIMESTAMP YEAR "%Y") #------------------------------------------------------------------------------ From 54e54624e9300817f10d13f8da3a33b312ce7aee Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Dec 2023 10:02:44 -0700 Subject: [PATCH 35/38] Update authors in CMakeList. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e0d6b24..41e163ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(ccpp_framework #------------------------------------------------------------------------------ # Set package definitions set(PACKAGE "ccpp-framework") -set(AUTHORS "Dom Heinzeller" "Grant Firl" "Mike Kavulich" "Steve Goldhaber" "Dustin Swales") +set(AUTHORS "Dom Heinzeller" "Grant Firl" "Mike Kavulich" "Dustin Swales" "Courtney Peverley") string(TIMESTAMP YEAR "%Y") #------------------------------------------------------------------------------ From 1cc6da49fd1de0faeeb857719d85b8c1782b9691 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 5 Dec 2023 10:11:13 -0700 Subject: [PATCH 36/38] Address reviewer comment --- scripts/var_props.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/var_props.py b/scripts/var_props.py index e38118c2..c8d10d2f 100755 --- a/scripts/var_props.py +++ b/scripts/var_props.py @@ -921,7 +921,7 @@ def __init__(self, var1_stdname, var1_type, var1_kind, var1_units, # end if if self.__compat: # Check for vertical array flipping (do later) - if (var1_top or var2_top) and (var1_top != var2_top): + if var1_top != var2_top: self.__compat = True self.has_vert_transforms = True # end if From 44f005fcae836bf6801427a95b052a39e6448cf7 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 12 Dec 2023 08:51:35 -0700 Subject: [PATCH 37/38] Address comments --- scripts/var_props.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/var_props.py b/scripts/var_props.py index c8d10d2f..dc16fd0f 100755 --- a/scripts/var_props.py +++ b/scripts/var_props.py @@ -952,7 +952,7 @@ def forward_transform(self, lvar_lname, rvar_lname, rvar_indices, lvar_indices, is the local name of "var1". is a tuple of the loop indices for "var1" (i.e., "var1" will show up in the RHS of the transform as "var1(rvar_indices)". - is a tuple of the loop indices for "var1" (i.e., "var2" + is a tuple of the loop indices for "var2" (i.e., "var2" will show up in the LHS of the transform as "var2(lvar_indices)". If is not None, it should be a string containing the local name of the "horizontal_loop_begin" variable. This is used to @@ -992,6 +992,8 @@ def reverse_transform(self, lvar_lname, rvar_lname, rvar_indices, lvar_indices, is the local name of "var2". is a tuple of the loop indices for "var1" (i.e., "var1" will show up in the RHS of the transform as "var1(rvar_indices)". + is a tuple of the loop indices for "var2" (i.e., "var2" + will show up in the LHS of the transform as "var2(lvar_indices)". If is not None, it should be a string containing the local name of the "horizontal_loop_begin" variable. This is used to compute the offset in the horizontal axis index between one and @@ -1003,7 +1005,7 @@ def reverse_transform(self, lvar_lname, rvar_lname, rvar_indices, lvar_indices, "var2" (i.e., "vertical_layer_dimension" or "vertical_interface_dimension"). """ - # Dimension transforms (Indices handled exrernally) + # Dimension transforms (Indices handled externally) lhs_term = f"{lvar_lname}({','.join(lvar_indices)})" rhs_term = f"{rvar_lname}({','.join(rvar_indices)})" From 4aae9ad5c833078fef630ea6c57f3b04db21b8c4 Mon Sep 17 00:00:00 2001 From: dustinswales Date: Tue, 12 Dec 2023 09:30:05 -0700 Subject: [PATCH 38/38] Rename var_action test var_compatability --- test/run_fortran_tests.sh | 6 +-- .../.gitignore | 0 .../CMakeLists.txt | 4 +- .../README.md | 4 +- .../effr_calc.F90 | 0 .../effr_calc.meta | 0 .../run_test | 50 +++++++++---------- .../test_host.F90 | 4 +- .../test_host.meta | 0 .../test_host_data.F90 | 0 .../test_host_data.meta | 0 .../test_host_mod.F90 | 0 .../test_host_mod.meta | 0 .../test_reports.py | 14 +++--- .../var_compatability_files.txt} | 0 .../var_compatability_suite.xml} | 2 +- 16 files changed, 42 insertions(+), 42 deletions(-) rename test/{var_action_test => var_compatability_test}/.gitignore (100%) rename test/{var_action_test => var_compatability_test}/CMakeLists.txt (98%) rename test/{var_action_test => var_compatability_test}/README.md (58%) rename test/{var_action_test => var_compatability_test}/effr_calc.F90 (100%) rename test/{var_action_test => var_compatability_test}/effr_calc.meta (100%) rename test/{var_action_test => var_compatability_test}/run_test (72%) rename test/{var_action_test => var_compatability_test}/test_host.F90 (99%) rename test/{var_action_test => var_compatability_test}/test_host.meta (100%) rename test/{var_action_test => var_compatability_test}/test_host_data.F90 (100%) rename test/{var_action_test => var_compatability_test}/test_host_data.meta (100%) rename test/{var_action_test => var_compatability_test}/test_host_mod.F90 (100%) rename test/{var_action_test => var_compatability_test}/test_host_mod.meta (100%) rename test/{var_action_test => var_compatability_test}/test_reports.py (94%) rename test/{var_action_test/var_action_files.txt => var_compatability_test/var_compatability_files.txt} (100%) rename test/{var_action_test/var_action_suite.xml => var_compatability_test/var_compatability_suite.xml} (69%) diff --git a/test/run_fortran_tests.sh b/test/run_fortran_tests.sh index fe34b3c0..cd6436fb 100755 --- a/test/run_fortran_tests.sh +++ b/test/run_fortran_tests.sh @@ -37,12 +37,12 @@ if [ $res -ne 0 ]; then echo "Failure running advection test" fi -# Run var_action test - ./var_action_test/run_test +# Run var_compatability test + ./var_compatability_test/run_test res=$? errcnt=$((errcnt + res)) if [ $res -ne 0 ]; then - echo "Failure running var_action test" + echo "Failure running var_compatability test" fi if [ $errcnt -eq 0 ]; then diff --git a/test/var_action_test/.gitignore b/test/var_compatability_test/.gitignore similarity index 100% rename from test/var_action_test/.gitignore rename to test/var_compatability_test/.gitignore diff --git a/test/var_action_test/CMakeLists.txt b/test/var_compatability_test/CMakeLists.txt similarity index 98% rename from test/var_action_test/CMakeLists.txt rename to test/var_compatability_test/CMakeLists.txt index 4b9daab8..7e3590a6 100644 --- a/test/var_action_test/CMakeLists.txt +++ b/test/var_compatability_test/CMakeLists.txt @@ -19,9 +19,9 @@ get_filename_component(CCPP_ROOT "${TEST_ROOT}" DIRECTORY) # Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) # #------------------------------------------------------------------------------ -LIST(APPEND SCHEME_FILES "var_action_files.txt") +LIST(APPEND SCHEME_FILES "var_compatability_files.txt") LIST(APPEND HOST_FILES "test_host_data" "test_host_mod") -LIST(APPEND SUITE_FILES "var_action_suite.xml") +LIST(APPEND SUITE_FILES "var_compatability_suite.xml") # HOST is the name of the executable we will build. # We assume there are files ${HOST}.meta and ${HOST}.F90 in CMAKE_SOURCE_DIR SET(HOST "${CMAKE_PROJECT_NAME}") diff --git a/test/var_action_test/README.md b/test/var_compatability_test/README.md similarity index 58% rename from test/var_action_test/README.md rename to test/var_compatability_test/README.md index b593a1f9..066cf771 100644 --- a/test/var_action_test/README.md +++ b/test/var_compatability_test/README.md @@ -1,6 +1,6 @@ -var_action test +var_compatability test ================ -To build and run the var_action test, run ./run_test +To build and run the var_compatability test, run ./run_test This script will build and run the test. The exit code is zero (0) on PASS and non-zero on FAIL. diff --git a/test/var_action_test/effr_calc.F90 b/test/var_compatability_test/effr_calc.F90 similarity index 100% rename from test/var_action_test/effr_calc.F90 rename to test/var_compatability_test/effr_calc.F90 diff --git a/test/var_action_test/effr_calc.meta b/test/var_compatability_test/effr_calc.meta similarity index 100% rename from test/var_action_test/effr_calc.meta rename to test/var_compatability_test/effr_calc.meta diff --git a/test/var_action_test/run_test b/test/var_compatability_test/run_test similarity index 72% rename from test/var_action_test/run_test rename to test/var_compatability_test/run_test index 2b4db0ac..ca185966 100755 --- a/test/var_action_test/run_test +++ b/test/var_compatability_test/run_test @@ -118,36 +118,36 @@ frame_src="${framework}/src" ## NB: This has to be after build_dir is finalized ## host_files="${build_dir}/ccpp/test_host_ccpp_cap.F90" -suite_files="${build_dir}/ccpp/ccpp_var_action_suite_cap.F90" +suite_files="${build_dir}/ccpp/ccpp_var_compatability_suite_cap.F90" utility_files="${build_dir}/ccpp/ccpp_kinds.F90" utility_files="${utility_files},${frame_src}/ccpp_constituent_prop_mod.F90" utility_files="${utility_files},${frame_src}/ccpp_hashable.F90" utility_files="${utility_files},${frame_src}/ccpp_hash_table.F90" ccpp_files="${utility_files}" ccpp_files="${ccpp_files},${build_dir}/ccpp/test_host_ccpp_cap.F90" -ccpp_files="${ccpp_files},${build_dir}/ccpp/ccpp_var_action_suite_cap.F90" +ccpp_files="${ccpp_files},${build_dir}/ccpp/ccpp_var_compatability_suite_cap.F90" #process_list="" module_list="effr_calc" #dependencies="" -suite_list="var_action_suite" -required_vars_var_action="ccpp_error_code,ccpp_error_message" -required_vars_var_action="${required_vars_var_action},effective_radius_of_stratiform_cloud_ice_particle" -required_vars_var_action="${required_vars_var_action},effective_radius_of_stratiform_cloud_liquid_water_particle" -required_vars_var_action="${required_vars_var_action},effective_radius_of_stratiform_cloud_rain_particle" -required_vars_var_action="${required_vars_var_action},effective_radius_of_stratiform_cloud_snow_particle" -required_vars_var_action="${required_vars_var_action},horizontal_loop_begin" -required_vars_var_action="${required_vars_var_action},horizontal_loop_end" -required_vars_var_action="${required_vars_var_action},vertical_layer_dimension" -input_vars_var_action="effective_radius_of_stratiform_cloud_liquid_water_particle" -input_vars_var_action="${input_vars_var_action},effective_radius_of_stratiform_cloud_rain_particle" -input_vars_var_action="${input_vars_var_action},effective_radius_of_stratiform_cloud_snow_particle" -input_vars_var_action="${input_vars_var_action},horizontal_loop_begin" -input_vars_var_action="${input_vars_var_action},horizontal_loop_end" -input_vars_var_action="${input_vars_var_action},vertical_layer_dimension" -output_vars_var_action="ccpp_error_code,ccpp_error_message" -output_vars_var_action="${output_vars_var_action},effective_radius_of_stratiform_cloud_ice_particle" -output_vars_var_action="${output_vars_var_action},effective_radius_of_stratiform_cloud_liquid_water_particle" -output_vars_var_action="${output_vars_var_action},effective_radius_of_stratiform_cloud_snow_particle" +suite_list="var_compatability_suite" +required_vars_var_compatability="ccpp_error_code,ccpp_error_message" +required_vars_var_compatability="${required_vars_var_compatability},effective_radius_of_stratiform_cloud_ice_particle" +required_vars_var_compatability="${required_vars_var_compatability},effective_radius_of_stratiform_cloud_liquid_water_particle" +required_vars_var_compatability="${required_vars_var_compatability},effective_radius_of_stratiform_cloud_rain_particle" +required_vars_var_compatability="${required_vars_var_compatability},effective_radius_of_stratiform_cloud_snow_particle" +required_vars_var_compatability="${required_vars_var_compatability},horizontal_loop_begin" +required_vars_var_compatability="${required_vars_var_compatability},horizontal_loop_end" +required_vars_var_compatability="${required_vars_var_compatability},vertical_layer_dimension" +input_vars_var_compatability="effective_radius_of_stratiform_cloud_liquid_water_particle" +input_vars_var_compatability="${input_vars_var_compatability},effective_radius_of_stratiform_cloud_rain_particle" +input_vars_var_compatability="${input_vars_var_compatability},effective_radius_of_stratiform_cloud_snow_particle" +input_vars_var_compatability="${input_vars_var_compatability},horizontal_loop_begin" +input_vars_var_compatability="${input_vars_var_compatability},horizontal_loop_end" +input_vars_var_compatability="${input_vars_var_compatability},vertical_layer_dimension" +output_vars_var_compatability="ccpp_error_code,ccpp_error_message" +output_vars_var_compatability="${output_vars_var_compatability},effective_radius_of_stratiform_cloud_ice_particle" +output_vars_var_compatability="${output_vars_var_compatability},effective_radius_of_stratiform_cloud_liquid_water_particle" +output_vars_var_compatability="${output_vars_var_compatability},effective_radius_of_stratiform_cloud_snow_particle" ## ## Run a database report and check the return string @@ -215,13 +215,13 @@ check_datatable ${report_prog} ${datafile} "--module-list" ${module_list} #check_datatable ${report_prog} ${datafile} "--dependencies" ${dependencies} check_datatable ${report_prog} ${datafile} "--suite-list" ${suite_list} \ --sep ";" -echo -e "\nChecking variables for var_action suite from command line" +echo -e "\nChecking variables for var_compatability suite from command line" check_datatable ${report_prog} ${datafile} "--required-variables" \ - ${required_vars_var_action} "var_action_suite" + ${required_vars_var_compatability} "var_compatability_suite" check_datatable ${report_prog} ${datafile} "--input-variables" \ - ${input_vars_var_action} "var_action_suite" + ${input_vars_var_compatability} "var_compatability_suite" check_datatable ${report_prog} ${datafile} "--output-variables" \ - ${output_vars_var_action} "var_action_suite" + ${output_vars_var_compatability} "var_compatability_suite" # Run make make res=$? diff --git a/test/var_action_test/test_host.F90 b/test/var_compatability_test/test_host.F90 similarity index 99% rename from test/var_action_test/test_host.F90 rename to test/var_compatability_test/test_host.F90 index 885c2e62..6de9597c 100644 --- a/test/var_action_test/test_host.F90 +++ b/test/var_compatability_test/test_host.F90 @@ -8,7 +8,7 @@ module test_prog public test_host ! Public data and interfaces - integer, public, parameter :: cs = 16 + integer, public, parameter :: cs = 32 integer, public, parameter :: cm = 60 !> \section arg_table_suite_info Argument Table @@ -375,7 +375,7 @@ program test logical :: run_okay ! Setup expected test suite info - test_suites(1)%suite_name = 'var_action_suite' + test_suites(1)%suite_name = 'var_compatability_suite' test_suites(1)%suite_parts => test_parts1 test_suites(1)%suite_input_vars => test_invars1 test_suites(1)%suite_output_vars => test_outvars1 diff --git a/test/var_action_test/test_host.meta b/test/var_compatability_test/test_host.meta similarity index 100% rename from test/var_action_test/test_host.meta rename to test/var_compatability_test/test_host.meta diff --git a/test/var_action_test/test_host_data.F90 b/test/var_compatability_test/test_host_data.F90 similarity index 100% rename from test/var_action_test/test_host_data.F90 rename to test/var_compatability_test/test_host_data.F90 diff --git a/test/var_action_test/test_host_data.meta b/test/var_compatability_test/test_host_data.meta similarity index 100% rename from test/var_action_test/test_host_data.meta rename to test/var_compatability_test/test_host_data.meta diff --git a/test/var_action_test/test_host_mod.F90 b/test/var_compatability_test/test_host_mod.F90 similarity index 100% rename from test/var_action_test/test_host_mod.F90 rename to test/var_compatability_test/test_host_mod.F90 diff --git a/test/var_action_test/test_host_mod.meta b/test/var_compatability_test/test_host_mod.meta similarity index 100% rename from test/var_action_test/test_host_mod.meta rename to test/var_compatability_test/test_host_mod.meta diff --git a/test/var_action_test/test_reports.py b/test/var_compatability_test/test_reports.py similarity index 94% rename from test/var_action_test/test_reports.py rename to test/var_compatability_test/test_reports.py index fff8603d..a8dc6eea 100755 --- a/test/var_action_test/test_reports.py +++ b/test/var_compatability_test/test_reports.py @@ -57,16 +57,16 @@ def usage(errmsg=None): # Check data _HOST_FILES = [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90")] -_SUITE_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_var_action_suite_cap.F90")] +_SUITE_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_var_compatability_suite_cap.F90")] _UTILITY_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_kinds.F90"), os.path.join(_SRC_DIR, "ccpp_constituent_prop_mod.F90"), os.path.join(_SRC_DIR, "ccpp_hashable.F90"), os.path.join(_SRC_DIR, "ccpp_hash_table.F90")] _CCPP_FILES = _UTILITY_FILES + \ [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90"), - os.path.join(_BUILD_DIR, "ccpp", "ccpp_var_action_suite_cap.F90")] + os.path.join(_BUILD_DIR, "ccpp", "ccpp_var_compatability_suite_cap.F90")] _MODULE_LIST = ["effr_calc"] -_SUITE_LIST = ["var_action_suite"] +_SUITE_LIST = ["var_compatability_suite"] _INPUT_VARS_VAR_ACTION = ["horizontal_loop_begin", "horizontal_loop_end", "vertical_layer_dimension", "effective_radius_of_stratiform_cloud_liquid_water_particle", "effective_radius_of_stratiform_cloud_rain_particle", @@ -140,15 +140,15 @@ def check_datatable(database, report_type, check_list, _MODULE_LIST) NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("suite_list"), _SUITE_LIST) -print("\nChecking variables for var_action suite from python") +print("\nChecking variables for var_compatability suite from python") NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("required_variables", - value="var_action_suite"), + value="var_compatability_suite"), _REQUIRED_VARS_VAR_ACTION) NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("input_variables", - value="var_action_suite"), + value="var_compatability_suite"), _INPUT_VARS_VAR_ACTION) NUM_ERRORS += check_datatable(_DATABASE, DatatableReport("output_variables", - value="var_action_suite"), + value="var_compatability_suite"), _OUTPUT_VARS_VAR_ACTION) sys.exit(NUM_ERRORS) diff --git a/test/var_action_test/var_action_files.txt b/test/var_compatability_test/var_compatability_files.txt similarity index 100% rename from test/var_action_test/var_action_files.txt rename to test/var_compatability_test/var_compatability_files.txt diff --git a/test/var_action_test/var_action_suite.xml b/test/var_compatability_test/var_compatability_suite.xml similarity index 69% rename from test/var_action_test/var_action_suite.xml rename to test/var_compatability_test/var_compatability_suite.xml index 72a7b80f..ae75d9f7 100644 --- a/test/var_action_test/var_action_suite.xml +++ b/test/var_compatability_test/var_compatability_suite.xml @@ -1,6 +1,6 @@ - + effr_calc