feature/transition-to-capgen-1: update from master 2020/10/02, change horizontal_dimension to horizontal_loop_extent

.gitmodules

@@ -4,9 +4,13 @@
 	branch = dev/emc
 [submodule "ccpp/framework"]
 	path = ccpp/framework
-	url = https://github.com/NCAR/ccpp-framework
-	branch = feature/transition-to-capgen-1
+	#url = https://github.com/NCAR/ccpp-framework
+	#branch = feature/transition-to-capgen-1
+	url = https://github.com/climbfuji/ccpp-framework
+	branch = update_from_master_20201002_and_more
 [submodule "ccpp/physics"]
 	path = ccpp/physics
-	url = https://github.com/NCAR/ccpp-physics
-	branch = feature/transition-to-capgen-1
+	#url = https://github.com/NCAR/ccpp-physics
+	#branch = feature/transition-to-capgen-1
+	url = https://github.com/climbfuji/ccpp-physics
+	branch = update_from_master_20201002_and_more

atmos_model.F90

@@ -99,13 +99,13 @@ module atmos_model_mod
                               IPD_interstitial => GFS_interstitial
 use IPD_driver,         only: IPD_initialize, IPD_initialize_rst
 use CCPP_driver,        only: CCPP_step, non_uniform_blocks
+
+use stochastic_physics_wrapper_mod, only: stochastic_physics_wrapper
 #else
 use IPD_driver,         only: IPD_initialize, IPD_initialize_rst, IPD_step
 use physics_abstraction_layer, only: time_vary_step, radiation_step1, physics_step1, physics_step2
 #endif
 
-use stochastic_physics_wrapper_mod, only: stochastic_physics_wrapper
-
 use FV3GFS_io_mod,      only: FV3GFS_restart_read, FV3GFS_restart_write, &
                               FV3GFS_IPD_checksum,                       &
                               FV3GFS_diag_register, FV3GFS_diag_output,  &
@@ -291,16 +291,16 @@ subroutine update_atmos_radiation_physics (Atmos)
 #ifdef CCPP
       call CCPP_step (step="time_vary", nblks=Atm_block%nblks, ierr=ierr)
       if (ierr/=0)  call mpp_error(FATAL, 'Call to CCPP time_vary step failed')
+
+!--- call stochastic physics pattern generation / cellular automata
+      call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block, ierr)
+      if (ierr/=0)  call mpp_error(FATAL, 'Call to stochastic_physics_wrapper failed')
+
 #else
       Func1d => time_vary_step
       call IPD_step (IPD_Control, IPD_Data(:), IPD_Diag, IPD_Restart, IPD_func1d=Func1d)
 #endif
 
-!--- call stochastic physics pattern generation / cellular automata
-    call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block, ierr) 
-    if (ierr/=0)  call mpp_error(FATAL, 'Call to stochastic_physics_wrapper failed')
-
-
 !--- if coupled, assign coupled fields
 
       if( IPD_Control%cplflx .or. IPD_Control%cplwav ) then
@@ -625,14 +625,15 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step)
 #ifdef CCPP
    call IPD_initialize (IPD_Control, IPD_Data, IPD_Diag, IPD_Restart, &
                         IPD_Interstitial, commglobal, mpp_npes(), Init_parm)
-#else
-   call IPD_initialize (IPD_Control, IPD_Data, IPD_Diag, IPD_Restart, Init_parm)
-#endif
 
 !--- Initialize stochastic physics pattern generation / cellular automata for first time step
    call stochastic_physics_wrapper(IPD_Control, IPD_Data, Atm_block, ierr)
    if (ierr/=0)  call mpp_error(FATAL, 'Call to stochastic_physics_wrapper failed')
 
+#else
+   call IPD_initialize (IPD_Control, IPD_Data, IPD_Diag, IPD_Restart, Init_parm)
+#endif
+
    Atmos%Diag => IPD_Diag
 
    Atm(mygrid)%flagstruct%do_skeb = IPD_Control%do_skeb

ccpp/CMakeLists.txt

@@ -120,11 +120,13 @@ elseif (${CMAKE_Fortran_COMPILER_ID} MATCHES "Intel")
     endif (LEGACY_INTEL)
   elseif (${CMAKE_BUILD_TYPE} MATCHES "Release")
     # Specify aggressive optimization flags (to be overwritten for individual files in ccpp-physics' CMakeLists.txt)
-    if (SIMDMULTIARCH)
-      set (CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512")
-    else (SIMDMULTIARCH)
-      set (CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -xCORE-AVX2")
-    endif (SIMDMULTIARCH)
+    if (AVX2)
+      if (SIMDMULTIARCH)
+        set (CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -axSSE4.2,AVX,CORE-AVX2,CORE-AVX512")
+      else (SIMDMULTIARCH)
+        set (CMAKE_Fortran_FLAGS_OPT "-no-prec-div -no-prec-sqrt -xCORE-AVX2")
+      endif (SIMDMULTIARCH)
+    endif (AVX2)
     set (CMAKE_Fortran_FLAGS  "${CMAKE_Fortran_FLAGS} -debug minimal -fp-model source -qoverride-limits -qopt-prefetch=3")
   endif (${CMAKE_BUILD_TYPE} MATCHES "Debug")
   set (CMAKE_Fortran_FLAGS_DEFAULT_PREC "-i4 -real-size 64")

ccpp/driver/CCPP_driver.F90

@@ -93,7 +93,7 @@ subroutine CCPP_step (step, nblks, ierr)
         end do
       end do
 
-   else if (trim(step)=="physics_init") then
+    else if (trim(step)=="physics_init") then
 
       ! Since the physics init steps are independent of the blocking structure,
       ! we can use cdata_domain here. Since we don't use threading on the outside,
@@ -107,7 +107,7 @@ subroutine CCPP_step (step, nblks, ierr)
         return
       end if
 
-   else if (trim(step)=="time_vary") then
+    else if (trim(step)=="time_vary") then
 
       ! Since the time_vary steps only use data structures for all blocks (except the
       ! CCPP-internal variables ccpp_error_flag and ccpp_error_message, which are defined
@@ -123,8 +123,8 @@ subroutine CCPP_step (step, nblks, ierr)
         return
       end if
 
-   ! Radiation and stochastic physics
-   else if (trim(step)=="radiation" .or. trim(step)=="physics" .or. trim(step)=="stochastics") then
+    ! Radiation and stochastic physics
+    else if (trim(step)=="radiation" .or. trim(step)=="physics" .or. trim(step)=="stochastics") then
 
       ! Set number of threads available to physics schemes to one,
       ! because threads are used on the outside for blocking
@@ -162,8 +162,8 @@ subroutine CCPP_step (step, nblks, ierr)
 !$OMP end parallel
       if (ierr/=0) return
 
-   ! Finalize
-   else if (trim(step)=="finalize") then
+    ! Finalize
+    else if (trim(step)=="finalize") then
 
       ! Loop over blocks, don't use threading on the outside but allowing threading
       ! inside the finalization, similar to what is done for the initialization

ccpp/suites/suite_FV3_GFS_2017_couplednsst.xml

@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<suite name="FV3_GFS_2017_couplednsst" lib="ccppphys" ver="4">
+  <!-- <init></init> -->
+  <group name="time_vary">
+    <subcycle loop="1">
+      <scheme>GFS_time_vary_pre</scheme>
+      <scheme>GFS_rrtmg_setup</scheme>
+      <scheme>GFS_rad_time_vary</scheme>
+      <scheme>GFS_phys_time_vary</scheme>
+    </subcycle>
+  </group>
+  <group name="radiation">
+    <subcycle loop="1">
+      <scheme>GFS_suite_interstitial_rad_reset</scheme>
+      <scheme>GFS_rrtmg_pre</scheme>
+      <scheme>rrtmg_sw_pre</scheme>
+      <scheme>rrtmg_sw</scheme>
+      <scheme>rrtmg_sw_post</scheme>
+      <scheme>rrtmg_lw_pre</scheme>
+      <scheme>rrtmg_lw</scheme>
+      <scheme>rrtmg_lw_post</scheme>
+      <scheme>GFS_rrtmg_post</scheme>
+    </subcycle>
+  </group>
+  <group name="physics">
+    <subcycle loop="1">
+      <scheme>GFS_suite_interstitial_phys_reset</scheme>
+      <scheme>GFS_suite_stateout_reset</scheme>
+      <scheme>get_prs_fv3</scheme>
+      <scheme>GFS_suite_interstitial_1</scheme>
+      <scheme>GFS_surface_generic_pre</scheme>
+      <scheme>GFS_surface_composites_pre</scheme>
+      <scheme>dcyc2t3</scheme>
+      <scheme>GFS_surface_composites_inter</scheme>
+      <scheme>GFS_suite_interstitial_2</scheme>
+    </subcycle>
+    <!-- Surface iteration loop -->
+    <subcycle loop="2">
+      <scheme>sfc_diff</scheme>
+      <scheme>GFS_surface_loop_control_part1</scheme>
+      <scheme>lsm_noah</scheme>
+      <scheme>sfc_nst_pre</scheme>
+      <scheme>sfc_nst</scheme>
+      <scheme>sfc_nst_post</scheme>
+      <scheme>sfc_cice</scheme>
+      <scheme>sfc_sice</scheme>
+      <scheme>GFS_surface_loop_control_part2</scheme>
+    </subcycle>
+    <!-- End of surface iteration loop -->
+    <subcycle loop="1">
+      <scheme>GFS_surface_composites_post</scheme>
+      <scheme>sfc_diag</scheme>
+      <scheme>sfc_diag_post</scheme>
+      <scheme>GFS_surface_generic_post</scheme>
+      <scheme>GFS_PBL_generic_pre</scheme>
+      <scheme>hedmf</scheme>
+      <scheme>GFS_PBL_generic_post</scheme>
+      <scheme>GFS_GWD_generic_pre</scheme>
+      <scheme>cires_ugwp</scheme>
+      <scheme>cires_ugwp_post</scheme>
+      <scheme>GFS_GWD_generic_post</scheme>
+      <scheme>rayleigh_damp</scheme>
+      <scheme>GFS_suite_stateout_update</scheme>
+      <scheme>ozphys</scheme>
+      <scheme>get_phi_fv3</scheme>
+      <scheme>GFS_suite_interstitial_3</scheme>
+      <scheme>GFS_DCNV_generic_pre</scheme>
+      <scheme>samfdeepcnv</scheme>
+      <scheme>GFS_DCNV_generic_post</scheme>
+      <scheme>GFS_SCNV_generic_pre</scheme>
+      <scheme>samfshalcnv</scheme>
+      <scheme>GFS_SCNV_generic_post</scheme>
+      <scheme>GFS_suite_interstitial_4</scheme>
+      <scheme>cnvc90</scheme>
+      <scheme>GFS_MP_generic_pre</scheme>
+      <scheme>zhaocarr_gscond</scheme>
+      <scheme>zhaocarr_precpd</scheme>
+      <scheme>GFS_MP_generic_post</scheme>
+      <scheme>maximum_hourly_diagnostics</scheme>
+    </subcycle>
+  </group>
+  <group name="stochastics">
+    <subcycle loop="1">
+      <scheme>GFS_stochastics</scheme>
+    </subcycle>
+  </group>
+  <!-- <finalize></finalize> -->
+</suite>

ccpp/suites/suite_FV3_GFS_cpld_rasmgshocnsst.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
-<suite name="FV3_GFS_cpld_rasmgshocnsst" lib="ccppphys" ver="3">
+<suite name="FV3_GFS_cpld_rasmgshocnsst" lib="ccppphys" ver="4">
   <!-- <init></init> -->
   <group name="time_vary">
     <subcycle loop="1">

ccpp/suites/suite_FV3_RRFS_v1beta.xml

@@ -56,7 +56,8 @@
       <scheme>GFS_surface_generic_post</scheme>
       <scheme>mynnedmf_wrapper</scheme>
       <scheme>GFS_GWD_generic_pre</scheme>
-      <scheme>drag_suite</scheme>
+      <scheme>cires_ugwp</scheme>
+      <scheme>cires_ugwp_post</scheme>
       <scheme>GFS_GWD_generic_post</scheme>
       <scheme>rayleigh_damp</scheme>
       <scheme>GFS_suite_stateout_update</scheme>

gfsphysics/GFS_layer/GFS_diagnostics.F90

@@ -2445,7 +2445,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop
       ExtDiag(idx)%name = 'dt3dt_nophys'
       ExtDiag(idx)%desc = 'temperature tendency due to non-physics processes'
       ExtDiag(idx)%unit = 'K s-1'
-      ExtDiag(idx)%mod_name = 'gfs_phys'
+      ExtDiag(idx)%mod_name = 'gfs_dyn'
       ExtDiag(idx)%time_avg = .TRUE.
       allocate (ExtDiag(idx)%data(nblks))
       do nb = 1,nblks
@@ -2626,7 +2626,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop
       ExtDiag(idx)%name = 'du3dt_nophys'
       ExtDiag(idx)%desc = 'u momentum tendency due to non-physics processes'
       ExtDiag(idx)%unit = 'm s-2'
-      ExtDiag(idx)%mod_name = 'gfs_phys'
+      ExtDiag(idx)%mod_name = 'gfs_dyn'
       ExtDiag(idx)%time_avg = .TRUE.
       allocate (ExtDiag(idx)%data(nblks))
       do nb = 1,nblks
@@ -2638,7 +2638,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop
       ExtDiag(idx)%name = 'dv3dt_nophys'
       ExtDiag(idx)%desc = 'v momentum tendency due to non-physics processes'
       ExtDiag(idx)%unit = 'm s-2'
-      ExtDiag(idx)%mod_name = 'gfs_phys'
+      ExtDiag(idx)%mod_name = 'gfs_dyn'
       ExtDiag(idx)%time_avg = .TRUE.
       allocate (ExtDiag(idx)%data(nblks))
       do nb = 1,nblks
@@ -2785,7 +2785,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop
       ExtDiag(idx)%name = 'dq3dt_nophys'
       ExtDiag(idx)%desc = 'water vapor specific humidity tendency due to non-physics processes'
       ExtDiag(idx)%unit = 'kg kg-1 s-1'
-      ExtDiag(idx)%mod_name = 'gfs_phys'
+      ExtDiag(idx)%mod_name = 'gfs_dyn'
       ExtDiag(idx)%time_avg = .TRUE.
       allocate (ExtDiag(idx)%data(nblks))
       do nb = 1,nblks
@@ -2797,7 +2797,7 @@ subroutine GFS_externaldiag_populate (ExtDiag, Model, Statein, Stateout, Sfcprop
       ExtDiag(idx)%name = 'dq3dt_o3nophys'
       ExtDiag(idx)%desc = 'ozone concentration tendency due to non-physics processes'
       ExtDiag(idx)%unit = 'kg kg-1 s-1'
-      ExtDiag(idx)%mod_name = 'gfs_phys'
+      ExtDiag(idx)%mod_name = 'gfs_dyn'
       ExtDiag(idx)%time_avg = .TRUE.
       allocate (ExtDiag(idx)%data(nblks))
       do nb = 1,nblks

gfsphysics/GFS_layer/GFS_typedefs.F90

@@ -657,6 +657,7 @@ module GFS_typedefs
     integer              :: icliq_sw        !< sw optical property for liquid clouds
     integer              :: iovr_sw         !< sw: max-random overlap clouds
     integer              :: iovr_lw         !< lw: max-random overlap clouds
+    integer              :: iovr            !< max-random overlap clouds for sw & lw (maximum of both)
     integer              :: ictm            !< ictm=0 => use data at initial cond time, if not
                                             !<           available; use latest; no extrapolation.
                                             !< ictm=1 => use data at the forecast time, if not
@@ -1550,9 +1551,9 @@ module GFS_typedefs
 #ifdef CCPP
     real (kind=kind_phys), pointer :: TRAIN  (:,:)   => null()  !< accumulated stratiform T tendency (K s-1)
 #endif
-#ifdef CCPP
+!#ifdef CCPP
     real (kind=kind_phys), pointer :: cldfra  (:,:)   => null()  !< instantaneous 3D cloud fraction
-#endif
+!#endif
     !--- MP quantities for 3D diagnositics 
     real (kind=kind_phys), pointer :: refl_10cm(:,:) => null()  !< instantaneous refl_10cm 
 !
@@ -1681,6 +1682,7 @@ module GFS_typedefs
     real (kind=kind_phys), pointer      :: adjvisdfd(:)       => null()  !<
     real (kind=kind_phys), pointer      :: aerodp(:,:)        => null()  !<
     real (kind=kind_phys), pointer      :: alb1d(:)           => null()  !<
+    real (kind=kind_phys), pointer      :: alpha(:,:)         => null()  !<
     real (kind=kind_phys), pointer      :: bexp1d(:)          => null()  !<
     real (kind=kind_phys), pointer      :: cd(:)              => null()  !<
     real (kind=kind_phys), pointer      :: cd_ice(:)          => null()  !<
@@ -3683,6 +3685,7 @@ subroutine control_initialize (Model, nlunit, fn_nml, me, master, &
     Model%icliq_sw         = icliq_sw
     Model%iovr_sw          = iovr_sw
     Model%iovr_lw          = iovr_lw
+    Model%iovr             = max(Model%iovr_sw,Model%iovr_lw)
     Model%ictm             = ictm
     Model%isubc_sw         = isubc_sw
     Model%isubc_lw         = isubc_lw
@@ -4838,6 +4841,7 @@ subroutine control_print(Model)
       print *, ' icliq_sw          : ', Model%icliq_sw
       print *, ' iovr_sw           : ', Model%iovr_sw
       print *, ' iovr_lw           : ', Model%iovr_lw
+      print *, ' iovr              : ', Model%iovr
       print *, ' ictm              : ', Model%ictm
       print *, ' isubc_sw          : ', Model%isubc_sw
       print *, ' isubc_lw          : ', Model%isubc_lw
@@ -6179,6 +6183,10 @@ subroutine interstitial_create (Interstitial, IM, Model)
     allocate (Interstitial%adjvisdfd       (IM))
     allocate (Interstitial%aerodp          (IM,NSPC1))
     allocate (Interstitial%alb1d           (IM))
+    if (.not. Model%do_RRTMGP) then
+      ! RRTMGP uses its own cloud_overlap_param
+      allocate (Interstitial%alpha         (IM,Model%levr+LTP))
+    end if
     allocate (Interstitial%bexp1d          (IM))
     allocate (Interstitial%cd              (IM))
     allocate (Interstitial%cd_ice          (IM))
@@ -6718,6 +6726,9 @@ subroutine interstitial_rad_reset (Interstitial, Model)
     !
     Interstitial%aerodp       = clear_val
     Interstitial%alb1d        = clear_val
+    if (.not. Model%do_RRTMGP) then
+      Interstitial%alpha      = clear_val
+    end if
     Interstitial%cldsa        = clear_val
     Interstitial%cldtaulw     = clear_val
     Interstitial%cldtausw     = clear_val
@@ -7154,6 +7165,9 @@ subroutine interstitial_print(Interstitial, Model, mpirank, omprank, blkno)
     write (0,*) 'sum(Interstitial%adjvisdfd       ) = ', sum(Interstitial%adjvisdfd       )
     write (0,*) 'sum(Interstitial%aerodp          ) = ', sum(Interstitial%aerodp          )
     write (0,*) 'sum(Interstitial%alb1d           ) = ', sum(Interstitial%alb1d           )
+    if (.not. Model%do_RRTMGP) then
+      write (0,*) 'sum(Interstitial%alpha           ) = ', sum(Interstitial%alpha         )
+    end if
     write (0,*) 'sum(Interstitial%bexp1d          ) = ', sum(Interstitial%bexp1d          )
     write (0,*) 'sum(Interstitial%cd              ) = ', sum(Interstitial%cd              )
     write (0,*) 'sum(Interstitial%cd_ice          ) = ', sum(Interstitial%cd_ice          )