diff --git a/scripts/ccpp_prebuild.py b/scripts/ccpp_prebuild.py index a5b8e5a8..daa870c9 100755 --- a/scripts/ccpp_prebuild.py +++ b/scripts/ccpp_prebuild.py @@ -13,7 +13,7 @@ # CCPP framework imports from common import encode_container, decode_container, decode_container_as_dict, execute -from common import CCPP_INTERNAL_VARIABLES, CCPP_STATIC_API_MODULE, CCPP_INTERNAL_VARIABLE_DEFINITON_FILE +from common import CCPP_STAGES, CCPP_INTERNAL_VARIABLES, CCPP_STATIC_API_MODULE, CCPP_INTERNAL_VARIABLE_DEFINITON_FILE from common import STANDARD_VARIABLE_TYPES, STANDARD_INTEGER_TYPE, CCPP_TYPE from common import SUITE_DEFINITION_FILENAME_PATTERN from common import split_var_name_and_array_reference @@ -341,6 +341,9 @@ def filter_metadata(metadata, arguments, dependencies, schemes_in_files, suites) for var in metadata[var_name][:]: container_string = decode_container(var.container) subroutine = container_string[container_string.find('SUBROUTINE')+len('SUBROUTINE')+1:] + # Replace the full CCPP stage name with the abbreviated version + for ccpp_stage in CCPP_STAGES.keys(): + subroutine = subroutine.replace(ccpp_stage, CCPP_STAGES[ccpp_stage]) for suite in suites: if subroutine in suite.all_subroutines_called: keep = True @@ -402,7 +405,7 @@ def check_optional_arguments(metadata, arguments, optional_arguments): for var_name in optional_arguments[scheme_name][subroutine_name]: if not var_name in arguments[scheme_name][subroutine_name]: raise Exception("Explicitly requested optional argument '{}' not known to {}/{}".format( - var_name, module_name, subroutine_name)) + var_name, scheme_name, subroutine_name)) else: for var in metadata[var_name][:]: for item in var.container.split(' '): diff --git a/scripts/common.py b/scripts/common.py index 9166c900..88d3ac43 100755 --- a/scripts/common.py +++ b/scripts/common.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +from collections import OrderedDict import keyword import logging import os @@ -7,7 +8,16 @@ import subprocess import sys -CCPP_STAGES = [ 'init', 'run', 'finalize' ] +# This dictionary contains short names for the different CCPP stages, +# because Fortran does not allow subroutine names with more than 63 characters +# Important: 'timestep_init' and 'timestep_finalize' need to come first so that +# a pattern match won't pick "init" for a CCPP subroutine name "xyz_timestep_init" +CCPP_STAGES = OrderedDict() +CCPP_STAGES['timestep_init'] = 'tsinit' +CCPP_STAGES['timestep_finalize'] = 'tsfinal' +CCPP_STAGES['init'] = 'init' +CCPP_STAGES['run'] = 'run' +CCPP_STAGES['finalize'] = 'final' CCPP_ERROR_FLAG_VARIABLE = 'ccpp_error_flag' CCPP_ERROR_MSG_VARIABLE = 'ccpp_error_message' diff --git a/scripts/metadata_parser.py b/scripts/metadata_parser.py index 53707e8b..225b2cf5 100755 --- a/scripts/metadata_parser.py +++ b/scripts/metadata_parser.py @@ -2,13 +2,15 @@ import collections import logging +import os +import re import subprocess +import sys from xml.etree import ElementTree as ET -from common import encode_container +from common import encode_container, CCPP_STAGES from mkcap import Var -import sys, os sys.path.append(os.path.join(os.path.split(__file__)[0], 'fortran_tools')) from parse_fortran import Ftype_type_decl from metadata_table import MetadataHeader @@ -182,7 +184,7 @@ def read_new_metadata(filename, module_name, table_name, scheme_name = None, sub legacy_note = ' replaced by horizontal loop extent (legacy extension)' # Adjust dimensions dimensions = new_var.get_prop_value('dimensions') - if scheme_name and (table_name.endswith("_init") or table_name.endswith("finalize")) \ + if scheme_name and (table_name.endswith("_init") or table_name.endswith("_finalize")) \ and 'horizontal_loop_extent' in dimensions: logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \ "for variable {} in table {}".format(standard_name,table_name)) @@ -508,9 +510,6 @@ def parse_scheme_tables(filepath, filename): # Set debug to true if logging level is debug debug = logging.getLogger().getEffectiveLevel() == logging.DEBUG - # Valid suffices for physics scheme routines - subroutine_suffices = [ 'init', 'run', 'finalize'] - # Final metadata container for all variables in file metadata = collections.OrderedDict() @@ -596,9 +595,16 @@ def parse_scheme_tables(filepath, filename): subroutine_name = words[j+1].split('(')[0].strip() # Consider the last substring separated by a '_' of the subroutine name as a 'postfix' if subroutine_name.find('_') >= 0: - subroutine_suffix = subroutine_name.split('_')[-1] - if subroutine_suffix in subroutine_suffices: - scheme_name = subroutine_name[0:subroutine_name.rfind('_')] + scheme_name = None + subroutine_suffix = None + for ccpp_stage in CCPP_STAGES: + pattern = '^(.*)_{}$'.format(ccpp_stage) + match = re.match(pattern, subroutine_name) + if match: + scheme_name = match.group(1) + subroutine_suffix = ccpp_stage + break + if match: if not scheme_name == module_name: raise Exception('Scheme name differs from module name: module_name="{0}" vs. scheme_name="{1}"'.format( module_name, scheme_name)) @@ -720,30 +726,6 @@ def parse_scheme_tables(filepath, filename): raise Exception('Mandatory CCPP variable {0} not declared in metadata table of subroutine {1}'.format( var_name, subroutine_name)) - # For CCPP-compliant files (i.e. files with metadata tables, perform additional checks) - if len(metadata.keys()) > 0: - # Check that all subroutine "root" names in the current module are equal to scheme_name - # and that there are exactly three subroutines for scheme X: X_init, X_run, X_finalize - message = '' - abort = False - for scheme_name in registry[module_name].keys(): - # Pre-generate error message - message += 'Check that all subroutines in module {0} have the same root name:\n'.format(module_name) - message += ' i.e. scheme_A_init, scheme_A_run, scheme_A_finalize\n' - message += 'Here is a list of the subroutine names for scheme {0}:\n'.format(scheme_name) - message += '{0}\n\n'.format(', '.join(sorted(registry[module_name][scheme_name].keys()))) - if (not len(registry[module_name][scheme_name].keys()) == 3): - logging.exception(message) - abort = True - else: - for suffix in subroutine_suffices: - subroutine_name = '{0}_{1}'.format(scheme_name, suffix) - if not subroutine_name in registry[module_name][scheme_name].keys(): - logging.exception(message) - abort = True - if abort: - raise Exception(message) - # Debugging output to screen and to XML if debug and len(metadata.keys()) > 0: # To screen diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index 80a142e7..7cc9285a 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -106,6 +106,7 @@ def create_arguments_module_use_var_defs(variable_dictionary, metadata_define, t module_use = [] var_defs = [] local_kind_and_type_vars = [] + for standard_name in variable_dictionary.keys(): # Add variable local name and variable definitions arguments.append(variable_dictionary[standard_name].local_name) @@ -128,11 +129,22 @@ def create_arguments_module_use_var_defs(variable_dictionary, metadata_define, t type_var = metadata_define[type_var_standard_name][0] module_use.append(type_var.print_module_use()) local_kind_and_type_vars.append(type_var_standard_name) + # Add any local variables (required for unit conversions, array transformations, ...) if tmpvars: var_defs.append('! Local variables for unit conversions, array transformations, ...') for tmpvar in tmpvars: var_defs.append(tmpvar.print_def_local()) + # Add special kind variables + if tmpvar.type in STANDARD_VARIABLE_TYPES and tmpvar.kind and not tmpvar.type == STANDARD_CHARACTER_TYPE: + kind_var_standard_name = tmpvar.kind + if not kind_var_standard_name in local_kind_and_type_vars: + if not kind_var_standard_name in metadata_define.keys(): + raise Exception("Kind {kind} not defined by host model".format(kind=kind_var_standard_name)) + kind_var = metadata_define[kind_var_standard_name][0] + module_use.append(kind_var.print_module_use()) + local_kind_and_type_vars.append(kind_var_standard_name) + return (arguments, module_use, var_defs) class API(object): @@ -265,7 +277,7 @@ def write(self): # which comes in as a scalar for any potential block/thread via the argument list. ccpp_var = None parent_standard_names = [] - for ccpp_stage in CCPP_STAGES: + for ccpp_stage in CCPP_STAGES.keys(): for suite in suites: for parent_standard_name in suite.parents[ccpp_stage].keys(): if not parent_standard_name in parent_standard_names: @@ -288,7 +300,7 @@ def write(self): # Create a subroutine for each stage self._subroutines=[] subs = '' - for ccpp_stage in CCPP_STAGES: + for ccpp_stage in CCPP_STAGES.keys(): suite_switch = '' for suite in suites: # Calls to groups of schemes for this stage @@ -309,7 +321,7 @@ def write(self): ierr = {suite_name}_{group_name}_{stage}_cap({arguments})'''.format(clause=clause, suite_name=group.suite, group_name=group.name, - stage=ccpp_stage, + stage=CCPP_STAGES[ccpp_stage], arguments=argument_list_group) group_calls += ''' else @@ -324,7 +336,7 @@ def write(self): argument_list_suite = create_argument_list_wrapped_explicit(suite.arguments[ccpp_stage]) suite_call = ''' ierr = {suite_name}_{stage}_cap({arguments}) -'''.format(suite_name=suite.name, stage=ccpp_stage, arguments=argument_list_suite) +'''.format(suite_name=suite.name, stage=CCPP_STAGES[ccpp_stage], arguments=argument_list_suite) # Add call to all groups of this suite and to the entire suite if not suite_switch: @@ -494,8 +506,8 @@ def __init__(self, **kwargs): self._caps = None self._module = None self._subroutines = None - self._parents = { ccpp_stage : {} for ccpp_stage in CCPP_STAGES } - self._arguments = { ccpp_stage : [] for ccpp_stage in CCPP_STAGES } + self._parents = { ccpp_stage : {} for ccpp_stage in CCPP_STAGES.keys() } + self._arguments = { ccpp_stage : [] for ccpp_stage in CCPP_STAGES.keys() } self._update_cap = True for key, value in kwargs.items(): setattr(self, "_"+key, value) @@ -580,7 +592,7 @@ def parse(self): schemes.append(scheme_xml.text) loop=int(subcycle_xml.get('loop')) for ccpp_stage in CCPP_STAGES: - self._all_subroutines_called.append(scheme_xml.text + '_' + ccpp_stage) + self._all_subroutines_called.append(scheme_xml.text + '_' + CCPP_STAGES[ccpp_stage]) subcycles.append(Subcycle(loop=loop, schemes=schemes)) self._groups.append(Group(name=group_xml.get('name'), subcycles=subcycles, suite=self._name)) @@ -664,7 +676,7 @@ def write(self, metadata_request, metadata_define, arguments): group.write(metadata_request, metadata_define, arguments) for subroutine in group.subroutines: module_use += ' use {m}, only: {s}\n'.format(m=group.module, s=subroutine) - for ccpp_stage in CCPP_STAGES: + for ccpp_stage in CCPP_STAGES.keys(): for parent_standard_name in group.parents[ccpp_stage].keys(): if parent_standard_name in self.parents[ccpp_stage]: if self.parents[ccpp_stage][parent_standard_name].intent == 'in' and \ @@ -676,7 +688,7 @@ def write(self, metadata_request, metadata_define, arguments): else: self.parents[ccpp_stage][parent_standard_name] = copy.deepcopy(group.parents[ccpp_stage][parent_standard_name]) subs = '' - for ccpp_stage in CCPP_STAGES: + for ccpp_stage in CCPP_STAGES.keys(): # Create a wrapped argument list for calling the suite, # get module use statements and variable definitions (self.arguments[ccpp_stage], sub_module_use, sub_var_defs) = \ @@ -696,9 +708,9 @@ def write(self, metadata_request, metadata_define, arguments): body += ''' ierr = {suite_name}_{group_name}_{stage}_cap({arguments}) if (ierr/=0) return -'''.format(suite_name=self._name, group_name=group.name, stage=ccpp_stage, arguments=argument_list_group) +'''.format(suite_name=self._name, group_name=group.name, stage=CCPP_STAGES[ccpp_stage], arguments=argument_list_group) # Add name of subroutine in the suite cap to list of subroutine names - subroutine = '{name}_{stage}_cap'.format(name=self._name, stage=ccpp_stage) + subroutine = '{name}_{stage}_cap'.format(name=self._name, stage=CCPP_STAGES[ccpp_stage]) self._subroutines.append(subroutine) # Add subroutine to output subs += Suite.sub.format(subroutine=subroutine, @@ -825,6 +837,13 @@ class Group(object): initialized_test_blocks = { 'init' : ''' if (initialized) return +''', + 'timestep_init' : ''' + if (.not.initialized) then + write({target_name_msg},'(*(a))') '{name}_timestep_init called before {name}_init' + {target_name_flag} = 1 + return + end if ''', 'run' : ''' if (.not.initialized) then @@ -832,6 +851,13 @@ class Group(object): {target_name_flag} = 1 return end if +''', + 'timestep_finalize' : ''' + if (.not.initialized) then + write({target_name_msg},'(*(a))') '{name}_timestep_finalize called before {name}_init' + {target_name_flag} = 1 + return + end if ''', 'finalize' : ''' if (.not.initialized) return @@ -842,7 +868,9 @@ class Group(object): 'init' : ''' initialized = .true. ''', + 'timestep_init' : '', 'run' : '', + 'timestep_finalize' : '', 'finalize' : ''' initialized = .false. ''', @@ -880,7 +908,7 @@ def write(self, metadata_request, metadata_define, arguments): self._subroutines = [] local_subs = '' # - for ccpp_stage in CCPP_STAGES: + for ccpp_stage in CCPP_STAGES.keys(): # The special init and finalize routines are only run in that stage if self._init and not ccpp_stage == 'init': continue @@ -915,8 +943,8 @@ def write(self, metadata_request, metadata_define, arguments): module_name = scheme_name subroutine_name = scheme_name + '_' + ccpp_stage container = encode_container(module_name, scheme_name, subroutine_name) - # Skip entirely empty routines - if not arguments[scheme_name][subroutine_name]: + # Skip entirely empty routines or non-existent routines + if not subroutine_name in arguments[scheme_name].keys() or not arguments[scheme_name][subroutine_name]: continue error_check = '' args = '' @@ -925,6 +953,7 @@ def write(self, metadata_request, metadata_define, arguments): # First identify all dimensions needed to handle the arguments # and add them to the list of required variables for the cap additional_variables_required = [] + # for var_standard_name in arguments[scheme_name][subroutine_name]: if not var_standard_name in metadata_define.keys(): raise Exception('Variable {standard_name} not defined in host model metadata'.format( @@ -943,7 +972,7 @@ def write(self, metadata_request, metadata_define, arguments): additional_variables_required.append(dim) # If blocked data structures need to be converted, add necessary variables - if ccpp_stage in ['init', 'finalize'] and CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in var.local_name: + if ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in var.local_name: if not CCPP_BLOCK_COUNT in local_vars.keys() \ and not CCPP_BLOCK_COUNT in additional_variables_required + arguments[scheme_name][subroutine_name]: logging.debug("Adding variable {} for handling blocked data structures".format(CCPP_BLOCK_COUNT)) @@ -1107,22 +1136,20 @@ def write(self, metadata_request, metadata_define, arguments): kind_string = '_' + local_vars[var_standard_name]['kind'] if local_vars[var_standard_name]['kind'] else '' # Convert blocked data in init and finalize steps - only required for variables with block number and horizontal_dimension - if ccpp_stage in ['init', 'finalize'] and CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in local_vars[var_standard_name]['name'] \ - and CCPP_HORIZONTAL_DIMENSION in var.dimensions: + if ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and \ + CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in local_vars[var_standard_name]['name'] and \ + CCPP_HORIZONTAL_DIMENSION in var.dimensions: # Reuse existing temporary variable, if possible if local_vars[var_standard_name]['name'] in tmpvars.keys(): # If the variable already has a local variable (tmpvar), reuse it tmpvar = tmpvars[local_vars[var_standard_name]['name']] actions_in = tmpvar.actions['in'] actions_out = tmpvar.actions['out'] - # DH* 2020-05-26 - raise Exception("Reusing temporary variables used for blocking has not been tested yet") - # *DH 2020-05-26 else: # Add a local variable (tmpvar) for this variable tmpvar_cnt += 1 tmpvar = copy.deepcopy(var) - tmpvar.local_name = 'tmpvar{0}'.format(tmpvar_cnt) + tmpvar.local_name = '{0}_local'.format(var.local_name) # # Create string for allocating the temporary array by converting the dimensions # (in standard_name format) to local names as known to the host model @@ -1166,8 +1193,7 @@ def write(self, metadata_request, metadata_define, arguments): # Define actions before, depending on intent if var.intent in [ 'in', 'inout' ]: - actions_in = ''' - allocate({tmpvar}({dims})) + actions_in = ''' allocate({tmpvar}({dims})) ib = 1 do nb=1,{block_count} {tmpvar}({dimpad_before}ib:ib+{block_size}-1{dimpad_after}) = {var} @@ -1182,16 +1208,13 @@ def write(self, metadata_request, metadata_define, arguments): dimpad_after=dimpad_after, ) else: - actions_in = ''' - allocate({tmpvar}({dims})) + actions_in = ''' allocate({tmpvar}({dims})) '''.format(tmpvar=tmpvar.local_name, dims=','.join(alloc_dimensions), ) - # Define actions after, depending on intent if var.intent in [ 'inout', 'out' ]: - actions_out = ''' - ib = 1 + actions_out = ''' ib = 1 do nb=1,{block_count} {var} = {tmpvar}({dimpad_before}ib:ib+{block_size}-1{dimpad_after}) ib = ib+{block_size} @@ -1205,8 +1228,7 @@ def write(self, metadata_request, metadata_define, arguments): dimpad_after=dimpad_after, ) else: - actions_out = ''' - deallocate({tmpvar}) + actions_out = ''' deallocate({tmpvar}) '''.format(tmpvar=tmpvar.local_name) # Set/update actions for this temporary variable @@ -1250,7 +1272,8 @@ def write(self, metadata_request, metadata_define, arguments): arg = '{local_name}={var_name},'.format(local_name=var.local_name, var_name=tmpvar.local_name) # Variables stored in blocked data structures but without horizontal dimension not supported at this time (doesn't make sense anyway) - elif ccpp_stage in ['init', 'finalize'] and CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in local_vars[var_standard_name]['name']: + elif ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and \ + CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in local_vars[var_standard_name]['name']: raise Exception("Variables stored in blocked data structures but without horizontal dimension not supported at this time: {}".format(var_standard_name)) # Unit conversions without converting blocked data structures @@ -1351,14 +1374,19 @@ def write(self, metadata_request, metadata_define, arguments): # Remove duplicates from additional manual variable definitions var_defs_manual = list(set(var_defs_manual)) - # Write cap - subroutine = self._suite + '_' + self._name + '_' + ccpp_stage + '_cap' + # Write cap - shorten certain ccpp_stages to stay under the 63 character limit for Fortran function names + subroutine = self._suite + '_' + self._name + '_' + CCPP_STAGES[ccpp_stage] + '_cap' self._subroutines.append(subroutine) - # Test and set blocks for initialization status - initialized_test_block = Group.initialized_test_blocks[ccpp_stage].format( - target_name_flag=ccpp_error_flag_target_name, - target_name_msg=ccpp_error_msg_target_name, - name=self._name) + # Test and set blocks for initialization status - check that at least + # the mandatory CCPP error handling arguments are present (i.e. there is + # at least one subroutine that gets called from this group), or skip. + if self.arguments[ccpp_stage]: + initialized_test_block = Group.initialized_test_blocks[ccpp_stage].format( + target_name_flag=ccpp_error_flag_target_name, + target_name_msg=ccpp_error_msg_target_name, + name=self._name) + else: + initialized_test_block = '' initialized_set_block = Group.initialized_set_blocks[ccpp_stage].format( target_name_flag=ccpp_error_flag_target_name, target_name_msg=ccpp_error_msg_target_name,