From 57d268c0558487a7df95d876239688227ce6655c Mon Sep 17 00:00:00 2001 From: John Michalakes Date: Tue, 20 Jun 2023 09:59:18 -0600 Subject: [PATCH] Support multiple concurrent instances per MPI task (#463) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to have an ensemble of model instances in a JEDI executable, we need for packages the model relies on to be “stateless” – meaning that the data for an instances are not shared between instances. A more comprehensive solution would be to implement physics so that all such data is either passed in via arguments to the package, or allocated as an instance of a class defining the physics. Deferring the larger issue of revisiting CCPP requirements to cover support for concurrent instances, this PR takes a more modest approach. It adds a new "instance" dimension to the types already defined for GFS physics (GFS_control, GFS_data, GFS_interstitial). And it includes some changes that were needed to address first-time initialization latches inside certain packages. One of these, in Thompson MP, declears the heap allocated fields by ccpp instance. The issue here was that only the first instance of Thompson MP was initializing itself. The other guards against trying to allocate data that has already been allocated (e.g. h2ointerp and ozinterp). The extent of packages touched by the PR is only one NRL-used suite that is a subset of all the packages in CCPP. --- scripts/common.py | 3 +++ scripts/mkstatic.py | 19 ++++++++++--------- src/ccpp_types.F90 | 1 + src/ccpp_types.meta | 6 ++++++ 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/scripts/common.py b/scripts/common.py index 87e057be..34fb131a 100755 --- a/scripts/common.py +++ b/scripts/common.py @@ -68,6 +68,9 @@ # Filename pattern for suite definition files SUITE_DEFINITION_FILENAME_PATTERN = re.compile('^suite_(.*)\.xml$') +# Maximum number of concurrent CCPP instances per MPI task +CCPP_NUM_INSTANCES = 200 + def execute(cmd, abort = True): """Runs a local command in a shell. Waits for completion and returns status, stdout and stderr. If abort = True, abort in diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py index 51202c41..cd389b3b 100755 --- a/scripts/mkstatic.py +++ b/scripts/mkstatic.py @@ -16,7 +16,7 @@ from common import CCPP_STAGES from common import CCPP_ERROR_CODE_VARIABLE, CCPP_ERROR_MSG_VARIABLE, CCPP_LOOP_COUNTER, CCPP_LOOP_EXTENT from common import CCPP_BLOCK_NUMBER, CCPP_BLOCK_COUNT, CCPP_BLOCK_SIZES, CCPP_THREAD_NUMBER, CCPP_INTERNAL_VARIABLES -from common import CCPP_CONSTANT_ONE, CCPP_HORIZONTAL_DIMENSION, CCPP_HORIZONTAL_LOOP_EXTENT +from common import CCPP_CONSTANT_ONE, CCPP_HORIZONTAL_DIMENSION, CCPP_HORIZONTAL_LOOP_EXTENT, CCPP_NUM_INSTANCES from common import FORTRAN_CONDITIONAL_REGEX_WORDS, FORTRAN_CONDITIONAL_REGEX from common import CCPP_TYPE, STANDARD_VARIABLE_TYPES, STANDARD_CHARACTER_TYPE from common import CCPP_STATIC_API_MODULE, CCPP_STATIC_SUBROUTINE_NAME @@ -896,7 +896,7 @@ class Group(object): private public :: {subroutines} - logical, save :: initialized = .false. + logical, dimension({num_instances}), save :: initialized = .false. contains ''' @@ -930,37 +930,37 @@ class Group(object): initialized_test_blocks = { 'init' : ''' - if (initialized) return + if (initialized(cdata%ccpp_instance)) return ''', 'timestep_init' : ''' - if (.not.initialized) then + if (.not.initialized(cdata%ccpp_instance)) then write({target_name_msg},'(*(a))') '{name}_timestep_init called before {name}_init' {target_name_flag} = 1 return end if ''', 'run' : ''' - if (.not.initialized) then + if (.not.initialized(cdata%ccpp_instance)) then write({target_name_msg},'(*(a))') '{name}_run called before {name}_init' {target_name_flag} = 1 return end if ''', 'timestep_finalize' : ''' - if (.not.initialized) then + if (.not.initialized(cdata%ccpp_instance)) then write({target_name_msg},'(*(a))') '{name}_timestep_finalize called before {name}_init' {target_name_flag} = 1 return end if ''', 'finalize' : ''' - if (.not.initialized) return + if (.not.initialized(cdata%ccpp_instance)) return ''', } initialized_set_blocks = { 'init' : ''' - initialized = .true. + initialized(cdata%ccpp_instance) = .true. ''', 'timestep_init' : '', 'run' : '', @@ -1665,7 +1665,8 @@ def write(self, metadata_request, metadata_define, arguments, debug): f.write(Group.header.format(group=self._name, module=self._module, module_use=module_use, - subroutines=', &\n '.join(self._subroutines))) + subroutines=', &\n '.join(self._subroutines), + num_instances=CCPP_NUM_INSTANCES)) f.write(local_subs) f.write(Group.footer.format(module=self._module)) if (f is not sys.stdout): diff --git a/src/ccpp_types.F90 b/src/ccpp_types.F90 index 992646bd..bebb53cb 100644 --- a/src/ccpp_types.F90 +++ b/src/ccpp_types.F90 @@ -58,6 +58,7 @@ module ccpp_types integer :: loop_max = CCPP_DEFAULT_LOOP_MAX integer :: blk_no = CCPP_DEFAULT_BLOCK_AND_THREAD_NUMBER integer :: thrd_no = CCPP_DEFAULT_BLOCK_AND_THREAD_NUMBER + integer :: ccpp_instance = 1 contains diff --git a/src/ccpp_types.meta b/src/ccpp_types.meta index 963762cc..6ad84522 100644 --- a/src/ccpp_types.meta +++ b/src/ccpp_types.meta @@ -44,6 +44,12 @@ units = index dimensions = () type = integer +[ccpp_instance] + standard_name = ccpp_instance + long_name = ccpp_instance + units = index + dimensions = () + type = integer ########################################################################