Merge pull request #45 from binli2337/feature/update_cdeps

Update CDEPS to include the latest changes from ESCOMP/CDEPS

.github/PULL_REQUEST_TEMPLATE.md

@@ -6,37 +6,13 @@ Contributors other than yourself, if any:
 
 CDEPS Issues Fixed (include github issue #):
 
-Are there dependencies on other component PRs
- - [ ] CIME (list)
- - [ ] CMEPS (list)
-
-Are changes expected to change answers?
- - [ ] bit for bit
- - [ ] different at roundoff level
- - [ ] more substantial
-
-Any User Interface Changes (namelist or namelist defaults changes)?
- - [ ] Yes
- - [ ] No
-
-Testing performed:
-- [ ] (required) aux_cdeps
-   - machines and compilers:
-   - details (e.g. failed tests):
-- [ ] (optional) CESM prealpha test
-   - machines and compilers
-   - details (e.g. failed tests):
+Are there dependencies on other component PRs (if so list):
+
+Are changes expected to change answers (bfb, different to roundoff, more substantial):
+
+Any User Interface Changes (namelist or namelist defaults changes):
+
+Testing performed (e.g. aux_cdeps, CESM prealpha, etc):
 
 Hashes used for testing:
-- [ ] CIME
-  - repository to check out: https://github.com/ESCOMP/CESM.git
-  - branch:
-  - hash:
-- [ ] CMEPS
-  - repository to check out: https://github.com/ESCOMP/CESM.git
-  - branch:
-  - hash:
-- [ ] CESM
-  - repository to check out: https://github.com/ESCOMP/CESM.git
-  - branch:
-  - hash:
+

cime_config/buildlib

@@ -81,6 +81,7 @@ def buildlib(bldroot, libroot, case):
     cmake_flags += " -DCMAKE_INSTALL_PREFIX:PATH=/"
     cmake_flags += " -DLIBROOT={} ".format(libroot)
     cmake_flags += " -DMPILIB={} ".format(mpilib)
+    cmake_flags += " -DSRCROOT={} ".format(case.get_value("SRCROOT"))
     cmake_flags += " -DPIO_C_LIBRARY={path}/lib -DPIO_C_INCLUDE_DIR={path}/include ".format(path=os.path.join(case.get_value("EXEROOT"),sharedpath))
     cmake_flags += " -DPIO_Fortran_LIBRARY={path}/lib -DPIO_Fortran_INCLUDE_DIR={path}/include ".format(path=os.path.join(case.get_value("EXEROOT"),sharedpath))
     cmake_flags += srcpath

datm/atm_comp_nuopc.F90

@@ -111,7 +111,7 @@ module cdeps_datm_comp
   integer                      :: flds_scalar_index_nextsw_cday = 0
   integer                      :: mpicom                    ! mpi communicator
   integer                      :: my_task                   ! my task in mpi communicator mpicom
-  logical                      :: masterproc                ! true of my_task == master_task
+  logical                      :: mainproc                ! true of my_task == main_task
   integer                      :: inst_index                ! number of current instance (ie. 1)
   character(len=16)            :: inst_suffix = ""          ! char string associated with instance (ie. "_0001" or "")
   integer                      :: logunit                   ! logging unit number
@@ -150,7 +150,7 @@ module cdeps_datm_comp
   ! constants
   integer                      :: idt                                 ! integer model timestep
   logical                      :: diagnose_data = .true.
-  integer          , parameter :: master_task   = 0                   ! task number of master task
+  integer          , parameter :: main_task   = 0                   ! task number of main task
   character(len=*) , parameter :: rpfile        = 'rpointer.atm'
 #ifdef CESMCOUPLED
   character(*)     , parameter :: modName       = "(atm_comp_nuopc)"
@@ -247,11 +247,11 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
          logunit, rc=rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
-    ! Determine logical masterproc
-    masterproc = (my_task == master_task)
+    ! Determine logical mainproc
+    mainproc = (my_task == main_task)
 
     ! Read atm_nml from nlfilename
-    if (my_task == master_task) then
+    if (my_task == main_task) then
        nlfilename = "datm_in"//trim(inst_suffix)
        open (newunit=nu,file=trim(nlfilename),status="old",action="read")
        read (nu,nml=datm_nml,iostat=ierr)
@@ -276,7 +276,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
     call shr_mpi_bcast(flds_wiso                 , mpicom, 'flds_wiso')
 
     ! write namelist input to standard out
-    if (my_task == master_task) then
+    if (my_task == main_task) then
        write(logunit,F00)' case_name      = ',trim(case_name)
        write(logunit,F00)' datamode       = ',trim(datamode)
        write(logunit,F00)' model_meshfile = ',trim(model_meshfile)
@@ -293,7 +293,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
     end if
 
     ! Validate sdat datamode
-    if (masterproc) write(logunit,*) ' datm datamode = ',trim(datamode)
+    if (mainproc) write(logunit,*) ' datm datamode = ',trim(datamode)
     if ( trim(datamode) == 'CORE2_NYF'    .or. &
          trim(datamode) == 'CORE2_IAF'    .or. &
          trim(datamode) == 'CORE_IAF_JRA' .or. &
@@ -411,9 +411,9 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
     ! Initialize and update orbital values
-    call dshr_orbital_init(gcomp, logunit, my_task == master_task, rc)
+    call dshr_orbital_init(gcomp, logunit, my_task == main_task, rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
-    call dshr_orbital_update(clock, logunit, my_task == master_task, &
+    call dshr_orbital_update(clock, logunit, my_task == main_task, &
          orbEccen, orbObliqr, orbLambm0, orbMvelpp, rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
@@ -474,7 +474,7 @@ subroutine ModelAdvance(gcomp, rc)
     rc = ESMF_SUCCESS
 
     call ESMF_TraceRegionEnter(subname)
-    call memcheck(subname, 5, my_task==master_task)
+    call memcheck(subname, 5, my_task==main_task)
 
     ! Query the Component for its clock, importState and exportState
     call NUOPC_ModelGet(gcomp, modelClock=clock, importState=importState, exportState=exportState, rc=rc)
@@ -491,7 +491,7 @@ subroutine ModelAdvance(gcomp, rc)
     call shr_cal_ymd2date(yr, mon, day, next_ymd)
 
     ! Update the orbital values
-    call dshr_orbital_update(clock, logunit, my_task == master_task, &
+    call dshr_orbital_update(clock, logunit, my_task == main_task, &
          orbEccen, orbObliqr, orbLambm0, orbMvelpp, rc)
     if (ChkErr(rc,__LINE__,u_FILE_u)) return
 
@@ -642,24 +642,24 @@ subroutine datm_comp_run(importState, exportState, target_ymd, target_tod, targe
        call datm_datamode_jra_advance(exportstate, target_ymd, target_tod, sdat%model_calendar, rc)
        if (ChkErr(rc,__LINE__,u_FILE_u)) return
     case('CLMNCEP')
-       call datm_datamode_clmncep_advance(masterproc, logunit, mpicom,  rc)
+       call datm_datamode_clmncep_advance(mainproc, logunit, mpicom,  rc)
        if (ChkErr(rc,__LINE__,u_FILE_u)) return
     case('CPLHIST')
-       call datm_datamode_cplhist_advance(masterproc, logunit, mpicom,  rc)
+       call datm_datamode_cplhist_advance(mainproc, logunit, mpicom,  rc)
        if (ChkErr(rc,__LINE__,u_FILE_u)) return
     case('ERA5')
-       call datm_datamode_era5_advance(exportstate, masterproc, logunit, mpicom, target_ymd, &
+       call datm_datamode_era5_advance(exportstate, mainproc, logunit, mpicom, target_ymd, &
             target_tod, sdat%model_calendar, rc)
        if (ChkErr(rc,__LINE__,u_FILE_u)) return
     case('GEFS')
-       call datm_datamode_gefs_advance(exportstate, masterproc, logunit, mpicom, target_ymd, &
+       call datm_datamode_gefs_advance(exportstate, mainproc, logunit, mpicom, target_ymd, &
             target_tod, sdat%model_calendar, rc)
     case('CFSR')
-       call datm_datamode_cfsr_advance(exportstate, masterproc, logunit, mpicom, target_ymd, &
+       call datm_datamode_cfsr_advance(exportstate, mainproc, logunit, mpicom, target_ymd, &
             target_tod, sdat%model_calendar, rc)
        if (ChkErr(rc,__LINE__,u_FILE_u)) return
     case('GFS')
-       call datm_datamode_gfs_advance(exportstate, masterproc, logunit, mpicom, target_ymd, &
+       call datm_datamode_gfs_advance(exportstate, mainproc, logunit, mpicom, target_ymd, &
             target_tod, sdat%model_calendar, rc)
        if (ChkErr(rc,__LINE__,u_FILE_u)) return
     end select
@@ -698,7 +698,7 @@ subroutine datm_comp_run(importState, exportState, target_ymd, target_tod, targe
     end if
 
     ! Log output for model date
-    if (masterproc) write(logunit,*) 'atm : model date ', target_ymd, target_tod
+    if (mainproc) write(logunit,*) 'atm : model date ', target_ymd, target_tod
 
     ! Diagnostics
     if (diagnose_data) then
@@ -747,41 +747,41 @@ subroutine datm_init_dfields(rc)
          if (chkerr(rc,__LINE__,u_FILE_u)) return
          if (rank == 1) then
             call dshr_dfield_add( dfields, sdat, trim(lfieldnames(n)), trim(lfieldnames(n)), &
-                 exportState, logunit, masterproc, rc=rc)
+                 exportState, logunit, mainproc, rc=rc)
             if (ChkErr(rc,__LINE__,u_FILE_u)) return
          else if (rank == 2) then
             select case (trim(lfieldnames(n)))
             case('Faxa_bcph')
                strm_flds3 = (/'Faxa_bcphidry', 'Faxa_bcphodry', 'Faxa_bcphiwet'/)
-               call dshr_dfield_add(dfields, sdat, trim(lfieldnames(n)), strm_flds3, exportState, logunit, masterproc, rc)
+               call dshr_dfield_add(dfields, sdat, trim(lfieldnames(n)), strm_flds3, exportState, logunit, mainproc, rc)
                if (ChkErr(rc,__LINE__,u_FILE_u)) return
             case('Faxa_ocph')
                strm_flds3 = (/'Faxa_ocphidry', 'Faxa_ocphodry', 'Faxa_ocphiwet'/)
-               call dshr_dfield_add(dfields, sdat, trim(lfieldnames(n)), strm_flds3, exportState, logunit, masterproc, rc)
+               call dshr_dfield_add(dfields, sdat, trim(lfieldnames(n)), strm_flds3, exportState, logunit, mainproc, rc)
                if (ChkErr(rc,__LINE__,u_FILE_u)) return
             case('Faxa_dstwet')
                strm_flds4 = (/'Faxa_dstwet1', 'Faxa_dstwet2', 'Faxa_dstwet3', 'Faxa_dstwet4'/)
-               call dshr_dfield_add(dfields, sdat, trim(lfieldnames(n)), strm_flds4, exportState, logunit, masterproc, rc)
+               call dshr_dfield_add(dfields, sdat, trim(lfieldnames(n)), strm_flds4, exportState, logunit, mainproc, rc)
                if (ChkErr(rc,__LINE__,u_FILE_u)) return
             case('Faxa_dstdry')
                strm_flds4 = (/'Faxa_dstdry1', 'Faxa_dstdry2', 'Faxa_dstdry3', 'Faxa_dstdry4'/)
-               call dshr_dfield_add(dfields, sdat, trim(lfieldnames(n)), strm_flds4, exportState, logunit, masterproc, rc)
+               call dshr_dfield_add(dfields, sdat, trim(lfieldnames(n)), strm_flds4, exportState, logunit, mainproc, rc)
                if (ChkErr(rc,__LINE__,u_FILE_u)) return
             case('Faxa_rainc_wiso')
                strm_flds3 = (/'Faxa_rainc_16O', 'Faxa_rainc_18O', 'Faxa_rainc_HDO'/)
-               call dshr_dfield_add(dfields, sdat, trim(lfieldnames(n)), strm_flds3, exportState, logunit, masterproc, rc)
+               call dshr_dfield_add(dfields, sdat, trim(lfieldnames(n)), strm_flds3, exportState, logunit, mainproc, rc)
                if (ChkErr(rc,__LINE__,u_FILE_u)) return
             case('Faxa_rainl_wiso')
                strm_flds3 = (/'Faxa_rainl_16O', 'Faxa_rainl_18O', 'Faxa_rainl_HDO'/)
-               call dshr_dfield_add(dfields, sdat, trim(lfieldnames(n)), strm_flds3, exportState, logunit, masterproc, rc)
+               call dshr_dfield_add(dfields, sdat, trim(lfieldnames(n)), strm_flds3, exportState, logunit, mainproc, rc)
                if (ChkErr(rc,__LINE__,u_FILE_u)) return
             case('Faxa_snowc_wiso')
                strm_flds3 = (/'Faxa_snowc_16O', 'Faxa_snowc_18O', 'Faxa_snowc_HDO'/)
-               call dshr_dfield_add(dfields, sdat, trim(lfieldnames(n)), strm_flds3, exportState, logunit, masterproc, rc)
+               call dshr_dfield_add(dfields, sdat, trim(lfieldnames(n)), strm_flds3, exportState, logunit, mainproc, rc)
                if (ChkErr(rc,__LINE__,u_FILE_u)) return
             case('Faxa_snowl_wiso')
                strm_flds3 = (/'Faxa_snowl_16O', 'Faxa_snowl_18O', 'Faxa_snowl_HDO'/)
-               call dshr_dfield_add(dfields, sdat, trim(lfieldnames(n)), strm_flds3, exportState, logunit, masterproc, rc)
+               call dshr_dfield_add(dfields, sdat, trim(lfieldnames(n)), strm_flds3, exportState, logunit, mainproc, rc)
                if (ChkErr(rc,__LINE__,u_FILE_u)) return
             end select
          end if
@@ -850,7 +850,7 @@ subroutine ModelFinalize(gcomp, rc)
     !-------------------------------------------------------------------------------
 
     rc = ESMF_SUCCESS
-    if (my_task == master_task) then
+    if (my_task == main_task) then
        write(logunit,*)
        write(logunit,*) 'datm : end of main integration loop'
        write(logunit,*)

datm/cime_config/buildnml

@@ -56,25 +56,33 @@ def _get_neon_data_availability(case, neonsite):
     oldestdate = None
     newestdate = None
     neonatm = None
-    with open(fullpath, "r") as fd:
-        for line in fd.readlines():
-            fpath, fname = os.path.split(line)
-            if fpath.endswith('cdeps/v1/'+neonsite):
-                neonatm, date, _ = line.strip().split(',')
-                if neonatm.endswith('.nc'):
-                    neonatm = neonatm.split('/')[-1]
-                    newestdate = datetime.strptime(neonatm[-10:],"%Y-%m.nc")
-                    if not oldestdate:
-                        oldestdate = datetime.strptime(neonatm[-10:],"%Y-%m.nc")
-                    neonatm = 'cdeps/v1/'+neonsite+'/'+neonatm
-                    datavaliddate.append(neonatm)
-    expect(newestdate, "No tower data found on server for NEON site {}".format(neonsite))
-    logger.info("Found tower data for {} through {}".format(oldestdate, newestdate))
-    datavaliddate.sort()
-
-    return datavaliddate
-
+    dataversion = case.get_value("NEONVERSION")
+    if dataversion == "latest":
+        dataversions = ["v2", "v1"]
+    else:
+        dataversions = [dataversion]
+
+    for version in dataversions:
+        if not newestdate:
+            with open(fullpath, "r") as fd:
+                for line in fd.readlines():
+                    fpath, fname = os.path.split(line)
+                    if fpath.endswith(f'cdeps/{version}/{neonsite}'):
+                        neonatm, date, _ = line.strip().split(',')
+                        if neonatm.endswith('.nc'):
+                            neonatm = neonatm.split('/')[-1]
+                            newestdate = datetime.strptime(neonatm[-10:],"%Y-%m.nc")
+                            if not oldestdate:
+                                oldestdate = datetime.strptime(neonatm[-10:],"%Y-%m.nc")
+                            neonatm = f'cdeps/{version}/{neonsite}/'+neonatm
+                            datavaliddate.append(neonatm)
+            if newestdate:                                        
+                logger.info("Found tower data version {} for {} through {}".format(version, oldestdate, newestdate))
+                datavaliddate.sort()
+                return datavaliddate
 
+    expect(newestdate, "No tower data found on server for NEON site {}".format(neonsite))
+    return None
 
 # pylint: disable=too-many-arguments,too-many-locals,too-many-branches,too-many-statements
 ####################################################################################

datm/cime_config/user_nl_datm_streams

@@ -20,4 +20,14 @@
 ! As an example:
 !   foo:year_first = 1950
 ! would change the stream year_first stream_entry to 1950 for the foo stream block
+! NOTE: multi-line inputs are enabled by adding a \ at the end of the line
+! As an emaple:
+! foo:datafiles=foo1,foo2, \
+!     foo3
+! Will yield the following new entry for datafiles in stream foo
+!   <datafiles>
+!      <file>foo1</file>
+!      <file>foo2</file>
+!      <file>foo3</file>
+!   </datafiles>
 !------------------------------------------------------------------------

datm/datm_datamode_cfsr_mod.F90

@@ -165,11 +165,11 @@ subroutine datm_datamode_cfsr_init_pointers(exportState, sdat, rc)
   end subroutine datm_datamode_cfsr_init_pointers
 
   !===============================================================================
-  subroutine datm_datamode_cfsr_advance(exportstate, masterproc, logunit, mpicom, target_ymd, target_tod, model_calendar, rc)
+  subroutine datm_datamode_cfsr_advance(exportstate, mainproc, logunit, mpicom, target_ymd, target_tod, model_calendar, rc)
 
     ! input/output variables
     type(ESMF_State)       , intent(inout) :: exportState
-    logical                , intent(in)    :: masterproc
+    logical                , intent(in)    :: mainproc
     integer                , intent(in)    :: logunit
     integer                , intent(in)    :: mpicom
     integer                , intent(in)    :: target_ymd
@@ -194,12 +194,12 @@ subroutine datm_datamode_cfsr_advance(exportstate, masterproc, logunit, mpicom,
        ! determine tbotmax (see below for use)
        rtmp = maxval(Sa_tbot(:))
        call shr_mpi_max(rtmp, tbotmax, mpicom, 'datm_tbot', all=.true.)
-       if (masterproc) write(logunit,*) trim(subname),' tbotmax = ',tbotmax
+       if (mainproc) write(logunit,*) trim(subname),' tbotmax = ',tbotmax
 
        ! determine maskmax (see below for use)
        rtmp = maxval(strm_mask(:))
        call shr_mpi_max(rtmp, maskmax, mpicom, 'datm_mask', all=.true.)
-       if (masterproc) write(logunit,*) trim(subname),' maskmax = ',maskmax
+       if (mainproc) write(logunit,*) trim(subname),' maskmax = ',maskmax
 
        ! reset first_time
        first_time = .false.

datm/datm_datamode_clmncep_mod.F90

@@ -316,10 +316,10 @@ subroutine datm_datamode_clmncep_init_pointers(importState, exportState, sdat, r
   end subroutine datm_datamode_clmncep_init_pointers
 
   !===============================================================================
-  subroutine datm_datamode_clmncep_advance(masterproc, logunit, mpicom, rc)
+  subroutine datm_datamode_clmncep_advance(mainproc, logunit, mpicom, rc)
 
     ! input/output variables
-    logical                , intent(in)    :: masterproc
+    logical                , intent(in)    :: mainproc
     integer                , intent(in)    :: logunit
     integer                , intent(in)    :: mpicom
     integer                , intent(out)   :: rc
@@ -348,7 +348,7 @@ subroutine datm_datamode_clmncep_advance(masterproc, logunit, mpicom, rc)
        ! determine tbotmax (see below for use)
        rtmp = maxval(Sa_tbot(:))
        call shr_mpi_max(rtmp, tbotmax, mpicom, 'datm_tbot', all=.true.)
-       if (masterproc) write(logunit,*) trim(subname),' tbotmax = ',tbotmax
+       if (mainproc) write(logunit,*) trim(subname),' tbotmax = ',tbotmax
        if(tbotmax <= 0) then
           call shr_sys_abort(subname//'ERROR: bad value in tbotmax')
        endif
@@ -360,13 +360,13 @@ subroutine datm_datamode_clmncep_advance(masterproc, logunit, mpicom, rc)
        else
           anidrmax = SHR_CONST_SPVAL
        end if
-       if (masterproc) write(logunit,*) trim(subname),' anidrmax = ',anidrmax
+       if (mainproc) write(logunit,*) trim(subname),' anidrmax = ',anidrmax
 
        ! determine tdewmax (see below for use)
        if (associated(strm_tdew)) then
           rtmp = maxval(strm_tdew(:))
           call shr_mpi_max(rtmp, tdewmax, mpicom, 'datm_tdew', all=.true.)
-          if (masterproc) write(logunit,*) trim(subname),' tdewmax = ',tdewmax
+          if (mainproc) write(logunit,*) trim(subname),' tdewmax = ',tdewmax
        endif
 
        ! reset first_time

datm/datm_datamode_cplhist_mod.F90

@@ -178,10 +178,10 @@ subroutine datm_datamode_cplhist_init_pointers(importState, exportState, sdat, r
   end subroutine datm_datamode_cplhist_init_pointers
 
   !===============================================================================
-  subroutine datm_datamode_cplhist_advance(masterproc, logunit, mpicom, rc)
+  subroutine datm_datamode_cplhist_advance(mainproc, logunit, mpicom, rc)
 
     ! input/output variables
-    logical                , intent(in)    :: masterproc
+    logical                , intent(in)    :: mainproc
     integer                , intent(in)    :: logunit
     integer                , intent(in)    :: mpicom
     integer                , intent(out)   :: rc