updates associated with CA restart and PE decomposition bugs

.gitmodules

@@ -8,5 +8,5 @@
   branch = main
 [submodule "ccpp/physics"]
   path = ccpp/physics
-  url = https://github.com/NCAR/ccpp-physics
+  url = https://github.com/pjpegion/ccpp-physics
   branch = main

atmos_model.F90

@@ -685,8 +685,8 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step)
                               GFS_data%Coupling, GFS_data%Grid, GFS_data%Tbd, GFS_data%Cldprop,  GFS_data%Radtend, &
                               GFS_data%IntDiag, Init_parm, GFS_Diag)
    call FV3GFS_restart_read (GFS_data, GFS_restart_var, Atm_block, GFS_control, Atmos%domain, Atm(mygrid)%flagstruct%warm_start)
-   if(GFS_control%ca_sgs)then
-      call read_ca_restart (Atmos%domain,GFS_control%scells)
+   if(GFS_control%do_ca .and. Atm(mygrid)%flagstruct%warm_start)then
+      call read_ca_restart (Atmos%domain,GFS_control%scells,GFS_control%nca,GFS_control%ncells_g,GFS_control%nca_g)
    endif
    ! Populate the GFS_data%Statein container with the prognostic state
    ! in Atm_block, which contains the initial conditions/restart data.
@@ -979,8 +979,8 @@ subroutine atmos_model_end (Atmos)
         GFS_Control%lndp_type > 0  .or. GFS_Control%do_ca ) then
       if(restart_endfcst) then
         call write_stoch_restart_atm('RESTART/atm_stoch.res.nc')
-        if (GFS_control%ca_sgs)then
-          call write_ca_restart(Atmos%domain,GFS_control%scells)
+        if (GFS_control%do_ca)then
+          call write_ca_restart()
         endif
       endif
       call stochastic_physics_wrapper_end(GFS_control)
@@ -1008,8 +1008,8 @@ subroutine atmos_model_restart(Atmos, timestamp)
     call atmosphere_restart(timestamp)
     call FV3GFS_restart_write (GFS_data, GFS_restart_var, Atm_block, &
                                GFS_control, Atmos%domain, timestamp)
-    if(GFS_control%ca_sgs)then
-       call write_ca_restart(Atmos%domain,GFS_control%scells,timestamp)
+    if(GFS_control%do_ca)then
+       call write_ca_restart(timestamp)
     endif
 end subroutine atmos_model_restart
 ! </SUBROUTINE>

ccpp/data/GFS_typedefs.F90

@@ -526,7 +526,6 @@ module GFS_typedefs
     real (kind=kind_phys), pointer :: ca_rad   (:)   => null() !
     real (kind=kind_phys), pointer :: ca_micro (:)   => null() !
     real (kind=kind_phys), pointer :: condition(:)   => null() !
-    real (kind=kind_phys), pointer :: vfact_ca(:)    => null() !
     !--- stochastic physics
     real (kind=kind_phys), pointer :: shum_wts  (:,:) => null()  !
     real (kind=kind_phys), pointer :: sppt_wts  (:,:) => null()  !
@@ -1122,7 +1121,7 @@ module GFS_typedefs
     logical              :: ca_sgs          !< switch for sgs ca
     logical              :: ca_global       !< switch for global ca
     logical              :: ca_smooth       !< switch for gaussian spatial filter
-    integer              :: iseed_ca        !< seed for random number generation in ca scheme
+    integer*8            :: iseed_ca        !< seed for random number generation in ca scheme
     integer              :: nspinup         !< number of iterations to spin up the ca
     real(kind=kind_phys) :: rcell           !< threshold used for CA scheme
     real(kind=kind_phys) :: nthresh         !< threshold used for convection coupling
@@ -1131,6 +1130,7 @@ module GFS_typedefs
     logical              :: ca_closure      !< logical switch for ca on closure
     logical              :: ca_entr         !< logical switch for ca on entrainment
     logical              :: ca_trigger      !< logical switch for ca on trigger
+    real (kind=kind_phys), allocatable :: vfact_ca(:) !< vertical tapering for ca_global
 
 !--- stochastic physics control parameters
     logical              :: do_sppt
@@ -2963,7 +2963,6 @@ subroutine coupling_create (Coupling, IM, Model)
 
    !-- cellular automata
     allocate (Coupling%condition(IM))
-    allocate (Coupling%vfact_ca(Model%levs))
     if (Model%do_ca) then
       allocate (Coupling%ca1      (IM))
       allocate (Coupling%ca2      (IM))
@@ -2973,7 +2972,6 @@ subroutine coupling_create (Coupling, IM, Model)
       allocate (Coupling%ca_shal  (IM))
       allocate (Coupling%ca_rad   (IM))
       allocate (Coupling%ca_micro (IM))
-      Coupling%vfact_ca = clear_val
       Coupling%ca1       = clear_val
       Coupling%ca2       = clear_val
       Coupling%ca3       = clear_val
@@ -3053,7 +3051,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, &
                                  nwat, tracer_names, tracer_types,  &
                                  input_nml_file, tile_num, blksz,   &
                                  ak, bk, restart, hydrostatic,      &
-                                 communicator, ntasks, nthreads)
+                                 communicator, ntasks, nthreads )
 
 !--- modules
     use physcons,         only: con_rerth, con_pi
@@ -4423,6 +4421,11 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, &
     Model%lndp_each_step   = lndp_each_step
 
     !--- cellular automata options
+    ! force namelist constsitency
+    allocate(Model%vfact_ca(levs))
+    if ( .not. ca_global ) nca_g=0
+    if ( .not. ca_sgs ) nca=0
+
     Model%nca              = nca
     Model%scells           = scells
     Model%tlives           = tlives

ccpp/data/GFS_typedefs.meta

@@ -2193,13 +2193,6 @@
   type = real
   kind = kind_phys
   active = (flag_for_cellular_automata)
-[vfact_ca]
-  standard_name = cellular_automata_vertical_weight
-  long_name = vertical weight for ca
-  units = frac
-  dimensions = (vertical_layer_dimension)
-  type = real
-  kind = kind_phys
 [ca1]
   standard_name = cellular_automata_global_pattern_from_coupled_process
   long_name = cellular automata global pattern
@@ -4408,6 +4401,13 @@
   units = flag
   dimensions = ()
   type = logical
+[vfact_ca]
+  standard_name = cellular_automata_vertical_weight
+  long_name = vertical weight for ca
+  units = frac
+  dimensions = (vertical_layer_dimension)
+  type = real
+  kind = kind_phys
 [ca_closure]
   standard_name = flag_for_global_cellular_automata_closure
   long_name = switch for ca on closure

stochastic_physics/stochastic_physics_wrapper.F90

@@ -41,9 +41,7 @@ module stochastic_physics_wrapper_mod
   real(kind=kind_phys), dimension(:,:),   allocatable, save :: lake
   real(kind=kind_phys), dimension(:,:),   allocatable, save :: condition
   real(kind=kind_phys), dimension(:,:),   allocatable, save :: ca_deep_cpl, ca_turb_cpl, ca_shal_cpl
-  real(kind=kind_phys), dimension(:,:),   allocatable, save :: ca_deep_diag,ca_turb_diag,ca_shal_diag
   real(kind=kind_phys), dimension(:,:),   allocatable, save :: ca1_cpl, ca2_cpl, ca3_cpl
-  real(kind=kind_phys), dimension(:,:),   allocatable, save :: ca1_diag,ca2_diag,ca3_diag
 
 
 !----------------
@@ -170,21 +168,21 @@ subroutine stochastic_physics_wrapper (GFS_Control, GFS_Data, Atm_block, ierr)
         if (.not. minval(Atm_block%blksz) == maxblk) then
            call mpp_error(FATAL, 'Logic errror: cellular_automata not compatible with non-uniform blocksizes')
         end if
-        ! *DH
-        do nb=1,nblks
-          GFS_Data(nb)%Intdiag%ca_deep(:)  = 0.
-          GFS_Data(nb)%Intdiag%ca_turb(:)  = 0.
-          GFS_Data(nb)%Intdiag%ca_shal(:)  = 0.
-          GFS_Data(nb)%Coupling%ca_deep(:) = 0.
-          GFS_Data(nb)%Coupling%ca_turb(:) = 0.
-          GFS_Data(nb)%Coupling%ca_shal(:) = 0.
-          GFS_Data(nb)%Coupling%ca1(:) = 0.
-          GFS_Data(nb)%Coupling%ca2(:) = 0.
-          GFS_Data(nb)%Coupling%ca3(:) = 0.
-          GFS_Data(nb)%Intdiag%ca1(:)  = 0.
-          GFS_Data(nb)%Intdiag%ca2(:)  = 0.
-          GFS_Data(nb)%Intdiag%ca3(:)  = 0.
-        enddo
+        if(GFS_Control%ca_sgs)then
+           allocate(sst         (1:nblks, maxblk))
+           allocate(lmsk        (1:nblks, maxblk))
+           allocate(lake        (1:nblks, maxblk))
+           allocate(condition   (1:nblks, maxblk))
+           allocate(ca_deep_cpl (1:nblks, maxblk))
+           allocate(ca_turb_cpl (1:nblks, maxblk))
+           allocate(ca_shal_cpl (1:nblks, maxblk))
+        endif
+        if(GFS_Control%ca_global)then
+          ! Allocate contiguous arrays; no need to copy in (intent out)
+          allocate(ca1_cpl (1:nblks, maxblk))
+          allocate(ca2_cpl (1:nblks, maxblk))
+          allocate(ca3_cpl (1:nblks, maxblk))
+        endif
       endif
 
       is_initialized = .true.
@@ -254,7 +252,7 @@ subroutine stochastic_physics_wrapper (GFS_Control, GFS_Data, Atm_block, ierr)
              endif
 
              call lndp_apply_perts(GFS_Control%blksz, GFS_Control%lsm, GFS_Control%lsm_noah, GFS_Control%lsm_ruc, lsoil,      &
-                               GFS_Control%dtf, GFS_Control%kdt, GFS_Control%lndp_each_step,                                  &
+                               GFS_Control%dtp, GFS_Control%kdt, GFS_Control%lndp_each_step,                                  &
                                GFS_Control%n_var_lndp, GFS_Control%lndp_var_list, GFS_Control%lndp_prt_list,                  &
                                sfc_wts, xlon, xlat, stype, GFS_Control%pores, GFS_Control%resid,param_update_flag,            &
                                smc, slc, stc, vfrac, alvsf, alnsf, alvwf, alnwf, facsf, facwf, snoalb, semis, zorll, ierr)
@@ -297,17 +295,6 @@ subroutine stochastic_physics_wrapper (GFS_Control, GFS_Data, Atm_block, ierr)
       if (GFS_Control%do_ca) then
 
        if(GFS_Control%ca_sgs)then
-         ! Allocate contiguous arrays; copy in as needed
-         allocate(sst         (1:nblks, maxblk))
-         allocate(lmsk        (1:nblks, maxblk))
-         allocate(lake        (1:nblks, maxblk))
-         allocate(ca_deep_diag(1:nblks, maxblk))
-         allocate(ca_turb_diag(1:nblks, maxblk))
-         allocate(ca_shal_diag(1:nblks, maxblk))
-         allocate(condition   (1:nblks, maxblk))
-         allocate(ca_deep_cpl (1:nblks, maxblk))
-         allocate(ca_turb_cpl (1:nblks, maxblk))
-         allocate(ca_shal_cpl (1:nblks, maxblk))
          do nb=1,nblks
              sst        (nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Sfcprop%tsfco(:)
              lmsk       (nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Sfcprop%slmsk(:)
@@ -317,61 +304,37 @@ subroutine stochastic_physics_wrapper (GFS_Control, GFS_Data, Atm_block, ierr)
              ca_turb_cpl(nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Coupling%ca_turb(:)
              ca_shal_cpl(nb,1:GFS_Control%blksz(nb)) = GFS_Data(nb)%Coupling%ca_shal(:)
          enddo
-         call cellular_automata_sgs(GFS_Control%kdt,GFS_control%dtf,GFS_control%restart,GFS_Control%first_time_step,              &
-            sst,lmsk,lake,condition,ca_deep_cpl,ca_turb_cpl,ca_shal_cpl,ca_deep_diag,ca_turb_diag,                                &
-            ca_shal_diag,Atm(mygrid)%domain_for_coupler,nblks,                                                                    &
+         call cellular_automata_sgs(GFS_Control%kdt,GFS_control%dtp,GFS_control%restart,GFS_Control%first_time_step,              &
+            sst,lmsk,lake,condition,ca_deep_cpl,ca_turb_cpl,ca_shal_cpl, Atm(mygrid)%domain_for_coupler,nblks,                    &
             Atm_block%isc,Atm_block%iec,Atm_block%jsc,Atm_block%jec,Atm(mygrid)%npx,Atm(mygrid)%npy, levs,                        &
-            GFS_Control%nthresh,GFS_Control%rcell,GFS_Control%nca,GFS_Control%scells,GFS_Control%tlives,GFS_Control%nfracseed,    &
-            GFS_Control%nseed,GFS_Control%ca_global,GFS_Control%ca_sgs,GFS_Control%iseed_ca,                                      &
-            GFS_Control%ca_smooth,GFS_Control%nspinup,GFS_Control%ca_trigger,Atm_block%blksz(1),GFS_Control%master,GFS_Control%communicator)
+            GFS_Control%nthresh,GFS_Control%rcell,GFS_Control%tile_num,GFS_Control%nca,GFS_Control%scells,GFS_Control%tlives,     &
+            GFS_Control%nfracseed, GFS_Control%nseed,GFS_Control%iseed_ca,                                                        &
+            GFS_Control%nspinup,GFS_Control%ca_trigger,Atm_block%blksz(1),GFS_Control%master,GFS_Control%communicator)
          ! Copy contiguous data back as needed
          do nb=1,nblks
-             GFS_Data(nb)%Intdiag%ca_deep(:)  = ca_deep_diag(nb,1:GFS_Control%blksz(nb))
-             GFS_Data(nb)%Intdiag%ca_turb(:)  = ca_turb_diag(nb,1:GFS_Control%blksz(nb))
-             GFS_Data(nb)%Intdiag%ca_shal(:)  = ca_shal_diag(nb,1:GFS_Control%blksz(nb))
+             GFS_Data(nb)%Intdiag%ca_deep(:)  = ca_deep_cpl(nb,1:GFS_Control%blksz(nb))
+             GFS_Data(nb)%Intdiag%ca_turb(:)  = ca_turb_cpl(nb,1:GFS_Control%blksz(nb))
+             GFS_Data(nb)%Intdiag%ca_shal(:)  = ca_shal_cpl(nb,1:GFS_Control%blksz(nb))
              GFS_Data(nb)%Coupling%ca_deep(:) = ca_deep_cpl (nb,1:GFS_Control%blksz(nb))
              GFS_Data(nb)%Coupling%ca_turb(:) = ca_turb_cpl (nb,1:GFS_Control%blksz(nb))
              GFS_Data(nb)%Coupling%ca_shal(:) = ca_shal_cpl (nb,1:GFS_Control%blksz(nb))
          enddo
-         deallocate(sst         )
-         deallocate(lmsk        )
-         deallocate(lake        )
-         deallocate(condition   )
-         deallocate(ca_deep_cpl )
-         deallocate(ca_turb_cpl )
-         deallocate(ca_shal_cpl )
-         deallocate(ca_deep_diag)
-         deallocate(ca_turb_diag)
-         deallocate(ca_shal_diag)
        endif
        if(GFS_Control%ca_global)then
-          ! Allocate contiguous arrays; no need to copy in (intent out)
-          allocate(ca1_cpl (1:nblks, maxblk))
-          allocate(ca2_cpl (1:nblks, maxblk))
-          allocate(ca3_cpl (1:nblks, maxblk))
-          allocate(ca1_diag(1:nblks, maxblk))
-          allocate(ca2_diag(1:nblks, maxblk))
-          allocate(ca3_diag(1:nblks, maxblk))
-          call cellular_automata_global(GFS_Control%kdt,GFS_Control%first_time_step,ca1_cpl,ca2_cpl,ca3_cpl,ca1_diag,ca2_diag,ca3_diag,         &
+          call cellular_automata_global(GFS_Control%kdt,GFS_control%restart,GFS_Control%first_time_step,ca1_cpl,ca2_cpl,ca3_cpl,         &
             Atm(mygrid)%domain_for_coupler, nblks,Atm_block%isc,Atm_block%iec,Atm_block%jsc,Atm_block%jec,Atm(mygrid)%npx,Atm(mygrid)%npy,levs, &
             GFS_Control%nca_g,GFS_Control%ncells_g,GFS_Control%nlives_g,GFS_Control%nfracseed,GFS_Control%nseed_g,                              &
-            GFS_Control%ca_global,GFS_Control%ca_sgs,GFS_Control%iseed_ca,GFS_Control%ca_smooth,GFS_Control%nspinup,Atm_block%blksz(1),         &
+            GFS_Control%iseed_ca,GFS_control%tile_num,GFS_Control%ca_smooth,GFS_Control%nspinup,Atm_block%blksz(1),         &
             GFS_Control%nsmooth,GFS_Control%ca_amplitude,GFS_Control%master,GFS_Control%communicator)
           ! Copy contiguous data back
           do nb=1,nblks
-             GFS_Data(nb)%Coupling%ca1(:) = ca1_cpl (nb,1:GFS_Control%blksz(nb))
-             GFS_Data(nb)%Coupling%ca2(:) = ca2_cpl (nb,1:GFS_Control%blksz(nb))
-             GFS_Data(nb)%Coupling%ca3(:) = ca3_cpl (nb,1:GFS_Control%blksz(nb))
-             GFS_Data(nb)%Intdiag%ca1(:)  = ca1_diag(nb,1:GFS_Control%blksz(nb))
-             GFS_Data(nb)%Intdiag%ca2(:)  = ca2_diag(nb,1:GFS_Control%blksz(nb))
-             GFS_Data(nb)%Intdiag%ca3(:)  = ca3_diag(nb,1:GFS_Control%blksz(nb))
+             GFS_Data(nb)%Coupling%ca1(:) = ca1_cpl(nb,1:GFS_Control%blksz(nb))
+             GFS_Data(nb)%Coupling%ca2(:) = ca2_cpl(nb,1:GFS_Control%blksz(nb))
+             GFS_Data(nb)%Coupling%ca3(:) = ca3_cpl(nb,1:GFS_Control%blksz(nb))
+             GFS_Data(nb)%Intdiag%ca1(:)  = ca1_cpl(nb,1:GFS_Control%blksz(nb))
+             GFS_Data(nb)%Intdiag%ca2(:)  = ca2_cpl(nb,1:GFS_Control%blksz(nb))
+             GFS_Data(nb)%Intdiag%ca3(:)  = ca3_cpl(nb,1:GFS_Control%blksz(nb))
           enddo
-          deallocate(ca1_cpl )
-          deallocate(ca2_cpl )
-          deallocate(ca3_cpl )
-          deallocate(ca1_diag)
-          deallocate(ca2_diag)
-          deallocate(ca3_diag)
        endif
 
       endif !do_ca
@@ -425,6 +388,20 @@ subroutine stochastic_physics_wrapper_end (GFS_Control)
       endif
       call finalize_stochastic_physics()
    endif
+   if(GFS_Control%ca_sgs)then
+       deallocate(sst         )
+       deallocate(lmsk        )
+       deallocate(lake        )
+       deallocate(condition   )
+       deallocate(ca_deep_cpl )
+       deallocate(ca_turb_cpl )
+       deallocate(ca_shal_cpl )
+    endif
+    if(GFS_Control%ca_global)then
+        deallocate(ca1_cpl )
+        deallocate(ca2_cpl )
+        deallocate(ca3_cpl )
+    endif
   end subroutine stochastic_physics_wrapper_end
 
 end module stochastic_physics_wrapper_mod