Merge pull request #3626 from ESMCI/jgfouca/scream_cime

CIME changes to support SCREAM

config/e3sm/config_files.xml

@@ -108,6 +108,7 @@
       <value component="allactive">$SRCROOT/cime_config/allactive/config_compsets.xml</value>
       <value component="drv"      >$CIMEROOT/src/drivers/$COMP_INTERFACE/cime_config/config_compsets.xml</value>
       <value component="cam"      >$SRCROOT/components/cam/cime_config/config_compsets.xml</value>
+      <value component="scream"   >$SRCROOT/components/scream/cime_config/config_compsets.xml</value>
       <value component="clm"      >$SRCROOT/components/clm/cime_config/config_compsets.xml</value>
       <value component="cice"     >$SRCROOT/components/cice/cime_config/config_compsets.xml</value>
       <value component="mpaso"    >$SRCROOT/components/mpas-ocean/cime_config/config_compsets.xml</value>
@@ -273,10 +274,11 @@
     <type>char</type>
     <default_value>unset</default_value>
     <values>
-      <value component="cam" >$SRCROOT/components/cam/cime_config/config_component.xml</value>
-      <value component="datm">$CIMEROOT/src/components/data_comps_$COMP_INTERFACE/datm/cime_config/config_component.xml</value>
-      <value component="satm">$CIMEROOT/src/components/stub_comps_$COMP_INTERFACE/satm/cime_config/config_component.xml</value>
-      <value component="xatm">$CIMEROOT/src/components/xcpl_comps_$COMP_INTERFACE/xatm/cime_config/config_component.xml</value>
+      <value component="cam"   >$SRCROOT/components/cam/cime_config/config_component.xml</value>
+      <value component="scream">$SRCROOT/components/scream/cime_config/config_component.xml</value>
+      <value component="datm"  >$CIMEROOT/src/components/data_comps_$COMP_INTERFACE/datm/cime_config/config_component.xml</value>
+      <value component="satm"  >$CIMEROOT/src/components/stub_comps_$COMP_INTERFACE/satm/cime_config/config_component.xml</value>
+      <value component="xatm"  >$CIMEROOT/src/components/xcpl_comps_$COMP_INTERFACE/xatm/cime_config/config_component.xml</value>
     </values>
     <group>case_last</group>
     <file>env_case.xml</file>

scripts/Tools/Makefile

@@ -69,9 +69,6 @@ endif
 ifeq ($(strip $(USE_ALBANY)), TRUE)
    USE_CXX = TRUE
 endif
-ifeq ($(strip $(USE_KOKKOS)), TRUE)
-  USE_CXX = TRUE
-endif
 
 SLIBS ?= $(USER_SLIBS)
 
@@ -270,13 +267,6 @@ ifeq ($(strip $(USE_ALBANY)), TRUE)
   include $(ALBANY_PATH)/export_albany.in
 endif
 
-ifeq ($(strip $(USE_KOKKOS)), TRUE)
-  include $(INSTALL_SHAREDPATH)/Makefile.kokkos
-  SLIBS += $(KOKKOS_LIBS)
-  CXXFLAGS += $(KOKKOS_CXXFLAGS)
-  CXX_LDFLAGS += $(KOKKOS_LDFLAGS)
-endif
-
 # Set MOAB info if it is being used
 ifeq ($(strip $(USE_MOAB)), TRUE)
   ifdef MOAB_PATH

scripts/lib/CIME/build.py

@@ -17,11 +17,10 @@
      "CAM_CONFIG_OPTS", "COMP_LND", "COMPARE_TO_NUOPC", "HOMME_TARGET",
      "OCN_SUBMODEL", "CISM_USE_TRILINOS", "USE_TRILINOS", "USE_ALBANY", "USE_PETSC")
 
-def get_standard_makefile_args(case, shared_lib=False):
+def get_standard_makefile_args(case, shared_lib=False): # pylint: disable=unused-argument
     make_args = "CIME_MODEL={} ".format(case.get_value("MODEL"))
     make_args += " SMP={} ".format(stringify_bool(case.get_build_threaded()))
-    if not shared_lib:
-        make_args += " USE_KOKKOS={} ".format(stringify_bool(uses_kokkos(case)))
+    expect(not uses_kokkos(case), "Kokkos is not supported for classic Makefile build system")
     for var in _CMD_ARGS_FOR_BUILD:
         make_args += xml_to_make_variable(case, var)
 
@@ -46,6 +45,9 @@ def get_standard_cmake_args(case, sharedpath, shared_lib=False):
     for var in _CMD_ARGS_FOR_BUILD:
         cmake_args += xml_to_make_variable(case, var, cmake=True)
 
+    if atm_model == "scream":
+        cmake_args += xml_to_make_variable(case, "HOMME_TARGET", cmake=True)
+
     # Disable compiler checks
     cmake_args += " -DCMAKE_C_COMPILER_WORKS=1 -DCMAKE_CXX_COMPILER_WORKS=1 -DCMAKE_Fortran_COMPILER_WORKS=1"
 
@@ -63,6 +65,8 @@ def xml_to_make_variable(case, varname, cmake=False):
 def uses_kokkos(case):
 ###############################################################################
     cam_target = case.get_value("CAM_TARGET")
+    # atm_comp   = case.get_value("COMP_ATM") # scream does not use the shared kokkoslib for now
+
     return get_model() == "e3sm" and cam_target in ("preqx_kokkos", "theta-l")
 
 ###############################################################################

src/components/data_comps_mct/docn/src/docn_comp_mod.F90

@@ -95,7 +95,7 @@ subroutine docn_comp_init(Eclock, x2o, o2x, &
        seq_flds_x2o_fields, seq_flds_o2x_fields, &
        SDOCN, gsmap, ggrid, mpicom, compid, my_task, master_task, &
        inst_suffix, inst_name, logunit, read_restart, &
-       scmMode, scmlat, scmlon)
+       scmMode, iop_mode, scmlat, scmlon)
 
     ! !DESCRIPTION: initialize docn model
     use pio        , only : iosystem_desc_t
@@ -119,6 +119,9 @@ subroutine docn_comp_init(Eclock, x2o, o2x, &
     integer(IN)            , intent(in)    :: logunit             ! logging unit number
     logical                , intent(in)    :: read_restart        ! start from restart
     logical                , intent(in)    :: scmMode             ! single column mode
+    logical                , intent(in)    :: iop_mode            ! IOP mode
+                                                                  ! cover planet with
+                                                                  ! identical surface
     real(R8)               , intent(in)    :: scmLat              ! single column lat
     real(R8)               , intent(in)    :: scmLon              ! single column lon
 
@@ -172,7 +175,8 @@ subroutine docn_comp_init(Eclock, x2o, o2x, &
        if (my_task == master_task) &
             write(logunit,F05) ' scm lon lat = ',scmlon,scmlat
        call shr_strdata_init(SDOCN,mpicom,compid,name='ocn', &
-            scmmode=scmmode,scmlon=scmlon,scmlat=scmlat, calendar=calendar, reset_domain_mask=.true.)
+            scmmode=scmmode,iop_mode=iop_mode,scmlon=scmlon,scmlat=scmlat, &
+            calendar=calendar, reset_domain_mask=.true.)
     else
        if (datamode == 'SST_AQUAPANAL' .or. datamode == 'SST_AQUAPFILE' .or. &
            datamode == 'SOM_AQUAP' .or. datamode == 'SST_AQUAP_CONSTANT' ) then

src/components/data_comps_mct/docn/src/ocn_comp_mct.F90

@@ -74,6 +74,7 @@ subroutine ocn_init_mct( EClock, cdata, x2o, o2x, NLFilename )
     integer(IN)       :: shrloglev                 ! original log level
     integer(IN)       :: ierr                      ! error code
     logical           :: scmMode = .false.         ! single column mode
+    logical           :: iop_mode = .false.        ! IOP mode
     real(R8)          :: scmLat  = shr_const_SPVAL ! single column lat
     real(R8)          :: scmLon  = shr_const_SPVAL ! single column lon
     character(*), parameter :: F00   = "('(docn_comp_init) ',8a)"
@@ -91,6 +92,7 @@ subroutine ocn_init_mct( EClock, cdata, x2o, o2x, NLFilename )
     ! Obtain infodata variables
     call seq_infodata_getData(infodata, &
          single_column=scmMode, &
+         iop_mode=iop_mode, &
          scmlat=scmlat, scmlon=scmLon, &
          read_restart=read_restart)
 
@@ -154,7 +156,7 @@ subroutine ocn_init_mct( EClock, cdata, x2o, o2x, NLFilename )
          seq_flds_x2o_fields, seq_flds_o2x_fields, &
          SDOCN, gsmap, ggrid, mpicom, compid, my_task, master_task, &
          inst_suffix, inst_name, logunit, read_restart, &
-         scmMode, scmlat, scmlon)
+         scmMode, iop_mode, scmlat, scmlon)
 
     !----------------------------------------------------------------------------
     ! Fill infodata that needs to be returned from docn

src/drivers/mct/cime_config/config_component.xml

@@ -1142,6 +1142,15 @@
     <desc>grid mask - DO NOT EDIT (for experts only)</desc>
   </entry>
 
+  <entry id="IOP_MODE">
+    <type>logical</type>
+    <valid_values>TRUE,FALSE</valid_values>
+    <default_value>FALSE</default_value>
+    <group>run_domain</group>
+    <file>env_run.xml</file>
+    <desc>Propogate a single point to the global grid  - DO NOT EDIT (for experts only)</desc>
+  </entry>
+
   <entry id="PTS_MODE">
     <type>logical</type>
     <valid_values>TRUE,FALSE</valid_values>

src/drivers/mct/cime_config/config_component_e3sm.xml

@@ -327,6 +327,9 @@
       <value compset=".+" grid="a%ne60np4">96</value>
       <value compset=".+" grid="a%ne120np4">96</value>
       <value compset=".+" grid="a%ne240np4">144</value>
+      <value compset=".+" grid="a%ne256np4">288</value>
+      <value compset=".+" grid="a%ne512np4">576</value>
+      <value compset=".+" grid="a%ne1024np4">1152</value>
       <value compset=".+" grid="a%ne0np4_arm_x8v3" >144</value>
       <value compset=".+" grid="a%ne0np4_conus_x4v1" >96</value>
       <value compset=".+" grid="a%ne0np4_northamericax4v1" >96</value>

src/drivers/mct/cime_config/namelist_definition_drv.xml

@@ -1484,6 +1484,20 @@
     </values>
   </entry>
 
+  <entry id="iop_mode" modify_via_xml="IOP_MODE">
+    <type>logical</type>
+    <category>seq_infodata_inparm</category>
+    <group>seq_infodata_inparm</group>
+    <desc>
+      turns on Intensive Observation Period (IOP) mode, a planet covered
+      with homogenous surface and forced identically with a forcing file,
+      set by IOP_MODE in env_case.xml, default: false
+    </desc>
+    <values>
+      <value>$IOP_MODE</value>
+    </values>
+  </entry>
+
   <entry id="single_column" modify_via_xml="PTS_MODE">
     <type>logical</type>
     <category>seq_infodata_inparm</category>

src/drivers/mct/main/cime_comp_mod.F90

@@ -444,6 +444,7 @@ module cime_comp_mod
 
   logical  :: areafact_samegrid      ! areafact samegrid flag
   logical  :: single_column          ! scm mode logical
+  logical  :: iop_mode               ! iop mode logical
   real(r8) :: scmlon                 ! single column lon
   real(r8) :: scmlat                 ! single column lat
   logical  :: aqua_planet            ! aqua planet mode
@@ -1059,6 +1060,7 @@ subroutine cime_pre_init2()
          esp_present=esp_present                   , &
          iac_present=iac_present                   , &
          single_column=single_column               , &
+         iop_mode=iop_mode                         , &
          aqua_planet=aqua_planet                   , &
          cpl_seq_option=cpl_seq_option             , &
          drv_threading=drv_threading               , &
@@ -1274,6 +1276,7 @@ subroutine cime_pre_init2()
        call seq_comm_getinfo(OCNID(ens1), mpicom=mpicom_OCNID)
 
        call shr_scam_checkSurface(scmlon, scmlat, &
+            iop_mode,                             &
             OCNID(ens1), mpicom_OCNID,            &
             lnd_present=lnd_present,              &
             ocn_present=ocn_present,              &

src/drivers/mct/shr/seq_infodata_mod.F90

@@ -83,6 +83,7 @@ MODULE seq_infodata_mod
      character(SHR_KIND_CL)  :: restart_pfile   ! Restart pointer file
      character(SHR_KIND_CL)  :: restart_file    ! Full archive path to restart file
      logical                 :: single_column   ! single column mode
+     logical                 :: iop_mode        ! IOP mode
      real (SHR_KIND_R8)      :: scmlat          ! single column lat
      real (SHR_KIND_R8)      :: scmlon          ! single column lon
      character(SHR_KIND_CS)  :: logFilePostFix  ! postfix for output log files
@@ -331,6 +332,7 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag)
     character(SHR_KIND_CL) :: restart_file       ! Restart filename
 
     logical                :: single_column      ! single column mode
+    logical                :: iop_mode           ! IOP mode
     real (SHR_KIND_R8)     :: scmlat             ! single column mode latitude
     real (SHR_KIND_R8)     :: scmlon             ! single column mode longitude
     character(SHR_KIND_CS) :: logFilePostFix     ! postfix for output log files
@@ -428,7 +430,7 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag)
          brnch_retain_casename, info_debug, bfbflag,       &
          restart_pfile, restart_file, run_barriers,        &
          single_column, scmlat, force_stop_at,             &
-         scmlon, logFilePostFix, outPathRoot, flux_diurnal,&
+         scmlon, iop_mode, logFilePostFix, outPathRoot, flux_diurnal,&
          ocn_surface_flux_scheme, &
          coldair_outbreak_mod, &
          flux_convergence, flux_max_iteration,             &
@@ -491,6 +493,7 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag)
        restart_pfile         = 'rpointer.drv'
        restart_file          = trim(sp_str)
        single_column         = .false.
+       iop_mode              = .false.
        scmlat                = -999.
        scmlon                = -999.
        logFilePostFix        = '.log'
@@ -626,6 +629,7 @@ SUBROUTINE seq_infodata_Init( infodata, nmlfile, ID, pioid, cpl_tag)
           infodata%restart_file       = restart_file
        end if
        infodata%single_column         = single_column
+       infodata%iop_mode              = iop_mode
        infodata%scmlat                = scmlat
        infodata%scmlon                = scmlon
        infodata%logFilePostFix        = logFilePostFix
@@ -964,7 +968,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_
        model_version, username, hostname, rest_case_name, tchkpt_dir,     &
        start_type, restart_pfile, restart_file, perpetual, perpetual_ymd, &
        aqua_planet,aqua_planet_sst, brnch_retain_casename, &
-       single_column, scmlat,scmlon,logFilePostFix, outPathRoot,          &
+       single_column, scmlat,scmlon,iop_mode,logFilePostFix, outPathRoot,&
        atm_present, atm_prognostic,                                       &
        lnd_present, lnd_prognostic,                                       &
        rof_present, rof_prognostic,                                       &
@@ -1033,6 +1037,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_
     logical,                optional, intent(OUT) :: single_column
     real (SHR_KIND_R8),     optional, intent(OUT) :: scmlat
     real (SHR_KIND_R8),     optional, intent(OUT) :: scmlon
+    logical,                optional, intent(OUT) :: iop_mode
     character(len=*),       optional, intent(OUT) :: logFilePostFix          ! output log file postfix
     character(len=*),       optional, intent(OUT) :: outPathRoot             ! output file root
     logical,                optional, intent(OUT) :: perpetual               ! If this is perpetual
@@ -1208,6 +1213,7 @@ SUBROUTINE seq_infodata_GetData_explicit( infodata, cime_model, case_name, case_
     if ( present(restart_pfile)  ) restart_pfile  = infodata%restart_pfile
     if ( present(restart_file)   ) restart_file   = infodata%restart_file
     if ( present(single_column)  ) single_column  = infodata%single_column
+    if ( present(iop_mode  )     ) iop_mode       = infodata%iop_mode
     if ( present(scmlat)         ) scmlat         = infodata%scmlat
     if ( present(scmlon)         ) scmlon         = infodata%scmlon
     if ( present(logFilePostFix) ) logFilePostFix = infodata%logFilePostFix
@@ -1499,7 +1505,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_
        model_version, username, hostname, rest_case_name, tchkpt_dir,     &
        start_type, restart_pfile, restart_file, perpetual, perpetual_ymd, &
        aqua_planet,aqua_planet_sst, brnch_retain_casename, &
-       single_column, scmlat,scmlon,logFilePostFix, outPathRoot,          &
+       single_column, scmlat,scmlon,iop_mode,logFilePostFix, outPathRoot,          &
        atm_present, atm_prognostic,                                       &
        lnd_present, lnd_prognostic,                                       &
        rof_present, rof_prognostic,                                       &
@@ -1567,6 +1573,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_
     logical,                optional, intent(IN)    :: single_column
     real (SHR_KIND_R8),     optional, intent(IN)    :: scmlat
     real (SHR_KIND_R8),     optional, intent(IN)    :: scmlon
+    logical,                optional, intent(IN)    :: iop_mode
     character(len=*),       optional, intent(IN)    :: logFilePostFix          ! output log file postfix
     character(len=*),       optional, intent(IN)    :: outPathRoot             ! output file root
     logical,                optional, intent(IN)    :: perpetual               ! If this is perpetual
@@ -1739,6 +1746,7 @@ SUBROUTINE seq_infodata_PutData_explicit( infodata, cime_model, case_name, case_
     if ( present(restart_pfile)  ) infodata%restart_pfile  = restart_pfile
     if ( present(restart_file)   ) infodata%restart_file   = restart_file
     if ( present(single_column)  ) infodata%single_column  = single_column
+    if ( present(iop_mode)       ) infodata%iop_mode       = iop_mode
     if ( present(scmlat)         ) infodata%scmlat         = scmlat
     if ( present(scmlon)         ) infodata%scmlon         = scmlon
     if ( present(logFilePostFix) ) infodata%logFilePostFix = logFilePostFix
@@ -2041,6 +2049,7 @@ subroutine seq_infodata_bcast(infodata,mpicom)
     call shr_mpi_bcast(infodata%restart_pfile,           mpicom)
     call shr_mpi_bcast(infodata%restart_file,            mpicom)
     call shr_mpi_bcast(infodata%single_column,           mpicom)
+    call shr_mpi_bcast(infodata%iop_mode,                mpicom)
     call shr_mpi_bcast(infodata%scmlat,                  mpicom)
     call shr_mpi_bcast(infodata%scmlon,                  mpicom)
     call shr_mpi_bcast(infodata%logFilePostFix,          mpicom)
@@ -2723,6 +2732,7 @@ SUBROUTINE seq_infodata_print( infodata )
     write(logunit,F0A) subname,'Restart file (full path) = ', trim(infodata%restart_file)
 
     write(logunit,F0L) subname,'single_column            = ', infodata%single_column
+    write(logunit,F0L) subname,'iop_mode                 = ', infodata%iop_mode
     write(logunit,F0R) subname,'scmlat                   = ', infodata%scmlat
     write(logunit,F0R) subname,'scmlon                   = ', infodata%scmlon