Support multiple concurrent instances per MPI task (#463)

In order to have an ensemble of model instances in a JEDI executable, we
need for packages the model relies on to be “stateless” – meaning
that the data for an instances are not shared between instances. A more
comprehensive solution would be to implement physics so that all such
data is either passed in via arguments to the package, or allocated as
an instance of a class defining the physics.

Deferring the larger issue of revisiting CCPP requirements to cover
support for concurrent instances, this PR takes a more modest approach. It
adds a new "instance" dimension to the types already defined for GFS
physics (GFS_control, GFS_data, GFS_interstitial). And it includes some
changes that were needed to address first-time initialization latches
inside certain packages.

One of these, in Thompson MP, declears the heap allocated fields by ccpp
instance. The issue here was that only the first instance of Thompson
MP was initializing itself. The other guards against trying to allocate
data that has already been allocated (e.g. h2ointerp and ozinterp).

The extent of packages touched by the PR is only one NRL-used suite that
is a subset of all the packages in CCPP.

scripts/common.py

@@ -68,6 +68,9 @@
 # Filename pattern for suite definition files
 SUITE_DEFINITION_FILENAME_PATTERN = re.compile('^suite_(.*)\.xml$')
 
+# Maximum number of concurrent CCPP instances per MPI task
+CCPP_NUM_INSTANCES = 200
+
 def execute(cmd, abort = True):
     """Runs a local command in a shell. Waits for completion and
     returns status, stdout and stderr. If abort = True, abort in

scripts/mkstatic.py

@@ -16,7 +16,7 @@
 from common import CCPP_STAGES
 from common import CCPP_ERROR_CODE_VARIABLE, CCPP_ERROR_MSG_VARIABLE, CCPP_LOOP_COUNTER, CCPP_LOOP_EXTENT
 from common import CCPP_BLOCK_NUMBER, CCPP_BLOCK_COUNT, CCPP_BLOCK_SIZES, CCPP_THREAD_NUMBER, CCPP_INTERNAL_VARIABLES
-from common import CCPP_CONSTANT_ONE, CCPP_HORIZONTAL_DIMENSION, CCPP_HORIZONTAL_LOOP_EXTENT
+from common import CCPP_CONSTANT_ONE, CCPP_HORIZONTAL_DIMENSION, CCPP_HORIZONTAL_LOOP_EXTENT, CCPP_NUM_INSTANCES
 from common import FORTRAN_CONDITIONAL_REGEX_WORDS, FORTRAN_CONDITIONAL_REGEX
 from common import CCPP_TYPE, STANDARD_VARIABLE_TYPES, STANDARD_CHARACTER_TYPE
 from common import CCPP_STATIC_API_MODULE, CCPP_STATIC_SUBROUTINE_NAME
@@ -896,7 +896,7 @@ class Group(object):
    private
    public :: {subroutines}
 
-   logical, save :: initialized = .false.
+   logical, dimension({num_instances}), save :: initialized = .false.
 
    contains
 '''
@@ -930,37 +930,37 @@ class Group(object):
 
     initialized_test_blocks = {
         'init' : '''
-      if (initialized) return
+      if (initialized(cdata%ccpp_instance)) return
 ''',
         'timestep_init' : '''
-      if (.not.initialized) then
+      if (.not.initialized(cdata%ccpp_instance)) then
         write({target_name_msg},'(*(a))') '{name}_timestep_init called before {name}_init'
         {target_name_flag} = 1
         return
       end if
 ''',
         'run' : '''
-      if (.not.initialized) then
+      if (.not.initialized(cdata%ccpp_instance)) then
         write({target_name_msg},'(*(a))') '{name}_run called before {name}_init'
         {target_name_flag} = 1
         return
       end if
 ''',
         'timestep_finalize' : '''
-      if (.not.initialized) then
+      if (.not.initialized(cdata%ccpp_instance)) then
         write({target_name_msg},'(*(a))') '{name}_timestep_finalize called before {name}_init'
         {target_name_flag} = 1
         return
       end if
 ''',
         'finalize' : '''
-      if (.not.initialized) return
+      if (.not.initialized(cdata%ccpp_instance)) return
 ''',
     }
 
     initialized_set_blocks = {
         'init' : '''
-      initialized = .true.
+      initialized(cdata%ccpp_instance) = .true.
 ''',
         'timestep_init' : '',
         'run' : '',
@@ -1665,7 +1665,8 @@ def write(self, metadata_request, metadata_define, arguments, debug):
         f.write(Group.header.format(group=self._name,
                                     module=self._module,
                                     module_use=module_use,
-                                    subroutines=', &\n             '.join(self._subroutines)))
+                                    subroutines=', &\n             '.join(self._subroutines),
+                                    num_instances=CCPP_NUM_INSTANCES))
         f.write(local_subs)
         f.write(Group.footer.format(module=self._module))
         if (f is not sys.stdout):

src/ccpp_types.F90

@@ -58,6 +58,7 @@ module ccpp_types
        integer                                             :: loop_max = CCPP_DEFAULT_LOOP_MAX
        integer                                             :: blk_no = CCPP_DEFAULT_BLOCK_AND_THREAD_NUMBER
        integer                                             :: thrd_no = CCPP_DEFAULT_BLOCK_AND_THREAD_NUMBER
+       integer                                             :: ccpp_instance = 1
 
     contains
 

src/ccpp_types.meta

@@ -44,6 +44,12 @@
   units = index
   dimensions = ()
   type = integer
+[ccpp_instance]
+  standard_name = ccpp_instance
+  long_name = ccpp_instance
+  units = index
+  dimensions = ()
+  type = integer
 
 ########################################################################