Merge https://github.com/neerajabhamidipati/MOM6 into MOM6_neeraja

.gitlab-ci.yml

@@ -60,7 +60,7 @@ gnu:ocean-only-nolibs:
     - time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
     - make -f MRS/Makefile.build build/gnu/env && cd build/gnu
     # mkdir -p build/gnu/repro/symmetric_dynamic/ocean_only && cd build/gnu/repro/symmetric_dynamic/ocean_only
-    - ../../MOM6-examples/src/mkmf/bin/list_paths -l ../../../config_src/{solo_driver,dynamic_symmetric} ../../../src ../../MOM6-examples/src/FMS
+    - ../../MOM6-examples/src/mkmf/bin/list_paths -l ../../../config_src/{solo_driver,dynamic_symmetric,ext*} ../../../src ../../MOM6-examples/src/FMS
     - sed -i '/FMS\/.*\/test_/d' path_names
     - ../../MOM6-examples/src/mkmf/bin/mkmf -t ../../MOM6-examples/src/mkmf/templates/ncrc-gnu.mk -p MOM6 -c"-Duse_libMPI -Duse_netCDF" path_names
     - time (source ./env ; make NETCDF=3 REPRO=1 MOM6 -s -j)
@@ -73,7 +73,7 @@ gnu:ice-ocean-nolibs:
     - time tar zxf $CACHE_DIR/tests_$CI_PIPELINE_ID.tgz && cd tests
     - make -f MRS/Makefile.build build/gnu/env && cd build/gnu
     # mkdir -p build/gnu/repro/symmetric_dynamic/ocean_only && cd build/gnu/repro/symmetric_dynamic/ocean_only
-    - ../../MOM6-examples/src/mkmf/bin/list_paths -l ../../../config_src/{coupled_driver,dynamic} ../../../src ../../MOM6-examples/src/{FMS,coupler,SIS2,icebergs,ice_param,land_null,atmos_null}
+    - ../../MOM6-examples/src/mkmf/bin/list_paths -l ../../../config_src/{coupled_driver,dynamic,ext*} ../../../src ../../MOM6-examples/src/{FMS,coupler,SIS2,icebergs,ice_param,land_null,atmos_null}
     - sed -i '/FMS\/.*\/test_/d' path_names
     - ../../MOM6-examples/src/mkmf/bin/mkmf -t ../../MOM6-examples/src/mkmf/templates/ncrc-gnu.mk -p MOM6 -c"-Duse_libMPI -Duse_netCDF -D_USE_LEGACY_LAND_ -Duse_AM3_physics" path_names
     - time (source ./env ; make NETCDF=3 REPRO=1 MOM6 -s -j)
@@ -116,7 +116,7 @@ run:
     - time tar zxf $CACHE_DIR/build-intel-repro-$CI_PIPELINE_ID.tgz
     - time tar zxf $CACHE_DIR/build-pgi-repro-$CI_PIPELINE_ID.tgz
     # time tar zxf $CACHE_DIR/build-gnu-debug-$CI_PIPELINE_ID.tgz
-    - (echo '#!/bin/tcsh';echo 'make -f MRS/Makefile.tests all -B') > job.sh
+    - (echo '#!/bin/tcsh';echo 'make -f MRS/Makefile.tests all') > job.sh
     - sbatch --clusters=c3,c4 --nodes=29 --time=0:34:00 --account=gfdl_o --qos=debug --job-name=mom6_regressions --output=log.$CI_PIPELINE_ID --wait job.sh
     - cat log.$CI_PIPELINE_ID
     - test -f restart_results_gnu.tar.gz

.gitmodules

@@ -4,9 +4,3 @@
 [submodule "pkg/GSW-Fortran"]
 	path = pkg/GSW-Fortran
 	url = https://github.com/TEOS-10/GSW-Fortran.git
-[submodule "pkg/MOM6_DA_hooks"]
-	path = pkg/MOM6_DA_hooks
-	url = https://github.com/MJHarrison-GFDL/MOM6_DA_hooks.git
-[submodule "pkg/geoKdTree"]
-	path = pkg/geoKdTree
-	url = https://github.com/travissluka/geoKdTree.git

.testing/Makefile

@@ -20,7 +20,7 @@ MKMF := $(abspath $(DEPS)/mkmf/bin/mkmf)
 
 # FMS framework
 FMS_URL ?= https://github.com/NOAA-GFDL/FMS.git
-FMS_COMMIT ?= 2019.01.01
+FMS_COMMIT ?= 2019.01.03
 FMS := $(DEPS)/fms
 
 #---
@@ -85,16 +85,41 @@ endif
 SOURCE = \
   $(foreach ext,F90 inc c h,$(wildcard $(1)/*/*.$(ext) $(1)/*/*/*.$(ext)))
 
-MOM_SOURCE = $(call SOURCE,../src) $(wildcard ../config_src/solo_driver/*.F90)
+MOM_SOURCE = $(call SOURCE,../src) $(wildcard ../config_src/solo_driver/*.F90) \
+             $(wildcard ../config_src/ext*/*/*.F90)
 TARGET_SOURCE = $(call SOURCE,build/target_codebase/src) \
-  $(wildcard build/target_codebase/config_src/solo_driver/*.F90)
+  $(wildcard build/target_codebase/config_src/solo_driver/*.F90) \
+  $(wildcard build/target_codebase/config_src/ext*/*.F90)
 FMS_SOURCE = $(call SOURCE,$(DEPS)/fms/src)
 
+#---
+# Python preprocessing environment configuration
+
+HAS_NUMPY = $(shell python -c "import numpy" 2> /dev/null && echo "yes")
+HAS_NETCDF4 = $(shell python -c "import netCDF4" 2> /dev/null && echo "yes")
+
+USE_VENV =
+ifneq ($(HAS_NUMPY), yes)
+  USE_VENV = yes
+endif
+ifneq ($(HAS_NETCDF4), yes)
+  USE_VENV = yes
+endif
+
+# When disabled, activation is a null operation (`true`)
+VENV_PATH =
+VENV_ACTIVATE = true
+ifeq ($(USE_VENV), yes)
+  VENV_PATH = work/local-env
+  VENV_ACTIVATE = . $(VENV_PATH)/bin/activate
+endif
+
+
 #---
 # Rules
 
 .PHONY: all build.regressions
-all: $(foreach b,$(BUILDS),build/$(b)/MOM6)
+all: $(foreach b,$(BUILDS),build/$(b)/MOM6) $(VENV_PATH)
 build.regressions: $(foreach b,symmetric target,build/$(b)/MOM6)
 
 # Executable
@@ -114,7 +139,7 @@ build/asymmetric/path_names: GRID_SRC=config_src/dynamic
 build/%/path_names: GRID_SRC=config_src/dynamic_symmetric
 
 build/%/MOM6: build/%/Makefile $(FMS)/lib/libfms.a
-	make -C $(@D) $(MOMFLAGS) $(@F)
+	$(MAKE) -C $(@D) $(MOMFLAGS) $(@F)
 
 build/%/Makefile: build/%/path_names
 	cp $(MKMF_TEMPLATE) $(@D)
@@ -133,13 +158,15 @@ build/target/path_names: $(LIST_PATHS) $(TARGET_CODEBASE) $(TARGET_SOURCE)
 	cd $(@D) && $(LIST_PATHS) -l \
 	  ../../$(TARGET_CODEBASE)/src \
 	  ../../$(TARGET_CODEBASE)/config_src/solo_driver \
+	  ../../$(TARGET_CODEBASE)/config_src/ext* \
 	  ../../$(TARGET_CODEBASE)/$(GRID_SRC)
 
 build/%/path_names: $(LIST_PATHS) $(MOM_SOURCE)
 	mkdir -p $(@D)
 	cd $(@D) && $(LIST_PATHS) -l \
 	  ../../../src \
 	  ../../../config_src/solo_driver \
+	  ../../../config_src/ext* \
 	  ../../../$(GRID_SRC)
 
 # Target repository for regression tests
@@ -153,7 +180,7 @@ $(TARGET_CODEBASE):
 
 $(FMS)/lib/libfms.a: $(FMS)/build/Makefile
 	mkdir -p $(FMS)/lib
-	cd $(FMS)/build && make NETCDF=3 DEBUG=1 ../lib/libfms.a
+	cd $(FMS)/build && $(MAKE) NETCDF=3 DEBUG=1 ../lib/libfms.a
 
 $(FMS)/build/Makefile: $(FMS)/build/path_names
 	cp $(MKMF_TEMPLATE) $(@D)
@@ -180,6 +207,18 @@ $(LIST_PATHS) $(MKMF):
 	cd $(DEPS)/mkmf; git checkout $(MKMF_COMMIT)
 
 
+#---
+# Python preprocessing
+# NOTE: Some less mature environments (e.g. Arm64 Ubuntu) require explicit
+#   installation of numpy before netCDF4, as well as wheel and cython support.
+work/local-env:
+	python3 -m venv $@
+	. $@/bin/activate \
+	  && pip3 install wheel \
+	  && pip3 install cython \
+	  && pip3 install numpy \
+	  && pip3 install netCDF4
+
 #----
 # Testing
 
@@ -250,8 +289,6 @@ $(eval $(call CMP_RULE,regression,symmetric target))
 # Restart tests only compare the final stat record
 .PRECIOUS: $(foreach b,symmetric restart target,work/%/$(b)/ocean.stats)
 %.restart: $(foreach b,symmetric restart,work/%/$(b)/ocean.stats)
-	#cmp $(foreach f,$^,<(tr -s ' ' < $(f) | cut -d ' ' -f3- | tail -n 1)) \
-	#  || diff $^
 	@cmp $(foreach f,$^,<(tr -s ' ' < $(f) | cut -d ' ' -f3- | tail -n 1)) \
 	  || !( \
 	    mkdir -p results/$*; \
@@ -262,22 +299,9 @@ $(eval $(call CMP_RULE,regression,symmetric target))
 
 # TODO: chksum_diag parsing of restart files
 
-
 #---
 # Test run output files
 
-# Generalized MPI environment variable support
-# XXX: Using `-env` in the MPICH test can erroneously producing an `nv` file.
-# $(1): Environment variables
-ifeq ($(shell $(MPIRUN) -x tmp=1 true 2> /dev/null ; echo $$?), 0)
-  MPIRUN_CMD=$(MPIRUN) $(if $(1),-x $(1),)
-else ifeq ($(shell $(MPIRUN) -env tmp=1 true 2> /dev/null ; echo $$?), 0)
-  MPIRUN_CMD=$(MPIRUN) $(if $(1),-env $(1),)
-else
-  MPIRUN_CMD=$(1) $(MPIRUN)
-endif
-
-
 # Rule to build work/<tc>/{ocean.stats,chksum_diag}.<tag>
 # $(1): Test configuration name <tag>
 # $(2): Executable type
@@ -291,24 +315,30 @@ work/%/$(1)/ocean.stats work/%/$(1)/chksum_diag: build/$(2)/MOM6
 	if [ $(3) ]; then find build/$(2) -name *.gcda -exec rm -f '{}' \; ; fi
 	mkdir -p $$(@D)
 	cp -rL $$*/* $$(@D)
-	cd $$(@D) && if [ -f Makefile ]; then make; fi
+	if [ -f $$(@D)/Makefile ]; then \
+	  $$(VENV_ACTIVATE) \
+	    && cd $$(@D) \
+	    && $(MAKE); \
+	else \
+	  cd $$(@D); \
+	fi
 	mkdir -p $$(@D)/RESTART
 	echo -e "$(4)" > $$(@D)/MOM_override
 	cd $$(@D) \
-	  && $$(call MPIRUN_CMD,$(5)) -n $(6) ../../../$$< 2> std.err > std.out \
+	  && $(5) $(MPIRUN) -n $(6) ../../../$$< 2> std.err > std.out \
 	  || !( \
 	    mkdir -p ../../../results/$$*/ ; \
-	    cat std.out | tee ../../../results/$$*/std.$(1).out | tail ; \
-		cat std.err | tee ../../../results/$$*/std.$(1).err | tail ; \
-		rm ocean.stats chksum_diag ; \
-		echo -e "${FAIL}: $$*.$(1) failed at runtime." \
+	    cat std.out | tee ../../../results/$$*/std.$(1).out | tail -20 ; \
+	    cat std.err | tee ../../../results/$$*/std.$(1).err | tail -20 ; \
+	    rm ocean.stats chksum_diag ; \
+	    echo -e "${FAIL}: $$*.$(1) failed at runtime." \
 	  )
 	@echo -e "${DONE}: $$*.$(1); no runtime errors."
 	if [ $(3) ]; then \
 	  mkdir -p results/$$* ; \
 	  bash <(curl -s https://codecov.io/bash) -n $$@ \
 	    > work/$$*/codecov.$(1).out \
-		2> work/$$*/codecov.$(1).err ; \
+	    2> work/$$*/codecov.$(1).err ; \
 	fi
 endef
 
@@ -320,7 +350,7 @@ $(eval $(call STAT_RULE,symmetric,symmetric,$(REPORT_COVERAGE),,,1))
 $(eval $(call STAT_RULE,asymmetric,asymmetric,,,,1))
 $(eval $(call STAT_RULE,target,target,,,,1))
 $(eval $(call STAT_RULE,repro,repro,,,,1))
-$(eval $(call STAT_RULE,openmp,openmp,,,,1))
+$(eval $(call STAT_RULE,openmp,openmp,,,GOMP_CPU_AFFINITY=0,1))
 $(eval $(call STAT_RULE,layout,symmetric,,LAYOUT=2$(,)1,,2))
 $(eval $(call STAT_RULE,rotate,symmetric,,ROTATE_INDEX=True\nINDEX_TURNS=1,,1))
 $(eval $(call STAT_RULE,nan,symmetric,,,MALLOC_PERTURB_=256,1))
@@ -337,7 +367,13 @@ work/%/restart/ocean.stats: build/symmetric/MOM6
 	rm -rf $(@D)
 	mkdir -p $(@D)
 	cp -rL $*/* $(@D)
-	cd work/$*/restart && if [ -f Makefile ]; then make; fi
+	if [ -f $(@D)/Makefile ]; then \
+	  $(VENV_ACTIVATE) \
+	    && cd work/$*/restart \
+	    && $(MAKE); \
+	else \
+	  cd work/$*/restart; \
+	fi
 	mkdir -p $(@D)/RESTART
 	# Generate the half-period input namelist
 	# TODO: Assumes that runtime set by DAYMAX, will fail if set by input.nml
@@ -352,8 +388,8 @@ work/%/restart/ocean.stats: build/symmetric/MOM6
 	cd $(@D) && $(MPIRUN) -n 1 ../../../$< 2> std1.err > std1.out \
 	  || !( \
 	    cat std1.out | tee ../../../results/$*/std.restart1.out | tail ; \
-		cat std1.err | tee ../../../results/$*/std.restart1.err | tail ; \
-		echo -e "${FAIL}: $*.restart failed at runtime." \
+	    cat std1.err | tee ../../../results/$*/std.restart1.err | tail ; \
+	    echo -e "${FAIL}: $*.restart failed at runtime." \
 	  )
 	# Setup the next inputs
 	cd $(@D) && rm -rf INPUT && mv RESTART INPUT
@@ -363,8 +399,8 @@ work/%/restart/ocean.stats: build/symmetric/MOM6
 	cd $(@D) && $(MPIRUN) -n 1 ../../../$< 2> std2.err > std2.out \
 	  || !( \
 	    cat std2.out | tee ../../../results/$*/std.restart2.out | tail ; \
-		cat std2.err | tee ../../../results/$*/std.restart2.err | tail ; \
-		echo -e "${FAIL}: $*.restart failed at runtime." \
+	    cat std2.err | tee ../../../results/$*/std.restart2.err | tail ; \
+	    echo -e "${FAIL}: $*.restart failed at runtime." \
 	  )
 
 # TODO: Restart checksum diagnostics

.testing/README.md

@@ -217,3 +217,10 @@ use `srun` (such as on GFDL's gaea HPC):
 ```
 make MPIRUN=srun test
 ```
+
+For convenience you can provide some macro in the file `config.mk`. For example, on
+gaea, to be able to run `make test -s -j` you will find putting the line
+```
+MPIRUN = srun -mblock --exclusive
+```
+in `config.mk` very useful.

.testing/tc0/MOM_input

@@ -227,3 +227,10 @@ ENERGYSAVEDAYS = 1.0
 
 DIAG_AS_CHKSUM = True
 DEBUG = True
+DEFAULT_2018_ANSWERS = True     !   [Boolean] default = True
+GRID_ROTATION_ANGLE_BUGS = True !   [Boolean] default = True
+USE_GM_WORK_BUG = True          !   [Boolean] default = True
+FIX_UNSPLIT_DT_VISC_BUG = False !   [Boolean] default = False
+USE_LAND_MASK_FOR_HVISC = False !   [Boolean] default = False
+GUST_CONST = 0.02               !   [Pa] default = 0.02
+FIX_USTAR_GUSTLESS_BUG = False  !   [Boolean] default = False

.testing/tc1.a/MOM_tc_variant

@@ -1 +1,2 @@
 #override SPLIT=False
+#override FIX_UNSPLIT_DT_VISC_BUG = False !   [Boolean] default = False

.testing/tc1.b/MOM_tc_variant

@@ -1,2 +1,3 @@
 #override SPLIT=False
 #override USE_RK2=True
+#override FIX_UNSPLIT_DT_VISC_BUG = False !   [Boolean] default = False

.testing/tc1/MOM_input

@@ -574,3 +574,16 @@ ENERGYSAVEDAYS = 0.125          !   [days] default = 3600.0
                                 ! energies of the run and other globally summed diagnostics.
 DIAG_AS_CHKSUM = True
 DEBUG = True
+USE_PSURF_IN_EOS = False        !   [Boolean] default = False
+DEFAULT_2018_ANSWERS = True     !   [Boolean] default = True
+GRID_ROTATION_ANGLE_BUGS = True !   [Boolean] default = True
+INTERPOLATE_RES_FN = True       !   [Boolean] default = True
+GILL_EQUATORIAL_LD = False      !   [Boolean] default = False
+USE_GM_WORK_BUG = True          !   [Boolean] default = True
+USE_LAND_MASK_FOR_HVISC = False !   [Boolean] default = False
+KAPPA_SHEAR_ITER_BUG = True     !   [Boolean] default = True
+KAPPA_SHEAR_ALL_LAYER_TKE_BUG = True !   [Boolean] default = True
+BULKML_CONV_MOMENTUM_BUG = True !   [Boolean] default = True
+PEN_SW_ABSORB_MINTHICK = 0.001  !   [m] default = 0.001
+GUST_CONST = 0.02               !   [Pa] default = 0.02
+FIX_USTAR_GUSTLESS_BUG = False  !   [Boolean] default = False

.testing/tc2/MOM_input

@@ -297,6 +297,10 @@ BOUND_CORIOLIS = True           !   [Boolean] default = False
                                 ! v-points, and similarly at v-points.  This option would
                                 ! have no effect on the SADOURNY Coriolis scheme if it
                                 ! were possible to use centered difference thickness fluxes.
+PGF_STANLEY_T2_DET_COEFF = 0.5  !   [nondim] default = -1.0
+                                ! The coefficient correlating SGS temperature variance with the mean temperature
+                                ! gradient in the deterministic part of the Stanley form of the Brankart
+                                ! correction. Negative values disable the scheme.
 
 ! === module MOM_hor_visc ===
 LAPLACIAN = True
@@ -426,6 +430,10 @@ KHTH = 1.0                      !   [m2 s-1] default = 0.0
                                 ! The background horizontal thickness diffusivity.
 KHTH_MAX = 900.0                !   [m2 s-1] default = 0.0
                                 ! The maximum horizontal thickness diffusivity.
+STANLEY_PRM_DET_COEFF = 0.5     !   [nondim] default = -1.0
+                                ! The coefficient correlating SGS temperature variance with the mean temperature
+                                ! gradient in the deterministic part of the Stanley parameterization. Negative
+                                ! values disable the scheme.
 
 ! === module MOM_mixed_layer_restrat ===
 FOX_KEMPER_ML_RESTRAT_COEF = 5.0 !   [nondim] default = 0.0
@@ -601,3 +609,14 @@ ENERGYSAVEDAYS = 0.5            !   [days] default = 3600.0
 DIAG_AS_CHKSUM = True
 DEBUG = True
 USE_GM_WORK_BUG = False
+USE_PSURF_IN_EOS = False        !   [Boolean] default = False
+DEFAULT_2018_ANSWERS = True     !   [Boolean] default = True
+GRID_ROTATION_ANGLE_BUGS = True !   [Boolean] default = True
+REMAP_UV_USING_OLD_ALG = True   !   [Boolean] default = True
+USE_LAND_MASK_FOR_HVISC = False !   [Boolean] default = False
+KAPPA_SHEAR_ITER_BUG = True     !   [Boolean] default = True
+KAPPA_SHEAR_ALL_LAYER_TKE_BUG = True !   [Boolean] default = True
+USE_MLD_ITERATION = False       !   [Boolean] default = False
+PEN_SW_ABSORB_MINTHICK = 0.001  !   [m] default = 0.001
+GUST_CONST = 0.02               !   [Pa] default = 0.02
+FIX_USTAR_GUSTLESS_BUG = False  !   [Boolean] default = False

.testing/tc3/MOM_input

@@ -469,3 +469,12 @@ ENERGYSAVEDAYS = 3.0            !   [hours] default = 1.44E+04
                                 ! energies of the run and other globally summed diagnostics.
 DIAG_AS_CHKSUM = True
 DEBUG = True
+DEFAULT_2018_ANSWERS = True     !   [Boolean] default = True
+OBC_RADIATION_MAX = 10.0        !   [nondim] default = 10.0
+GRID_ROTATION_ANGLE_BUGS = True !   [Boolean] default = True
+USE_GM_WORK_BUG = True          !   [Boolean] default = True
+USE_LAND_MASK_FOR_HVISC = False !   [Boolean] default = False
+KAPPA_SHEAR_ITER_BUG = True     !   [Boolean] default = True
+KAPPA_SHEAR_ALL_LAYER_TKE_BUG = True !   [Boolean] default = True
+GUST_CONST = 0.02               !   [Pa] default = 0.02
+FIX_USTAR_GUSTLESS_BUG = False  !   [Boolean] default = False

.testing/tc4/.gitignore

@@ -0,0 +1,4 @@
+ocean_hgrid.nc
+sponge.nc
+temp_salt_ic.nc
+topog.nc

.testing/tc4/MOM_input

@@ -320,9 +320,6 @@ DTBT = 10.0                     !   [s or nondim] default = -0.98
 ! Parameterization of enhanced mixing due to convection via CVMix
 
 ! === module MOM_entrain_diffusive ===
-CORRECT_DENSITY = False         !   [Boolean] default = True
-                                ! If true, and USE_EOS is true, the layer densities are restored toward their
-                                ! target values by the diapycnal mixing, as described in Hallberg (MWR, 2000).
 
 ! === module MOM_set_diffusivity ===
 BBL_EFFIC = 0.0                 !   [nondim] default = 0.2
@@ -410,3 +407,20 @@ MAXCPU = 2.88E+04               !   [wall-clock seconds] default = -1.0
 
 DIAG_AS_CHKSUM = True
 DEBUG = True
+
+USE_PSURF_IN_EOS = False        !   [Boolean] default = False
+DEFAULT_2018_ANSWERS = True     !   [Boolean] default = True
+GRID_ROTATION_ANGLE_BUGS = True !   [Boolean] default = True
+INTERPOLATE_RES_FN = True       !   [Boolean] default = True
+GILL_EQUATORIAL_LD = False      !   [Boolean] default = False
+USE_GM_WORK_BUG = True          !   [Boolean] default = True
+FIX_UNSPLIT_DT_VISC_BUG = False !   [Boolean] default = False
+REMAP_UV_USING_OLD_ALG = True   !   [Boolean] default = True
+USE_LAND_MASK_FOR_HVISC = False !   [Boolean] default = False
+KAPPA_SHEAR_ITER_BUG = True     !   [Boolean] default = True
+KAPPA_SHEAR_ALL_LAYER_TKE_BUG = True !   [Boolean] default = True
+USE_MLD_ITERATION = False       !   [Boolean] default = False
+PEN_SW_ABSORB_MINTHICK = 0.001  !   [m] default = 0.001
+GUST_CONST = 0.02               !   [Pa] default = 0.02
+FIX_USTAR_GUSTLESS_BUG = False  !   [Boolean] default = False
+