Merge pull request #6 from NOAA-EMC/dev/emc

.codecov.yml

@@ -0,0 +1,8 @@
+coverage:
+    status:
+        project:
+            default:
+                threshold: 100%
+        patch:
+            default:
+                threshold: 100%

.gitignore

@@ -6,16 +6,3 @@ html
 MOM6
 build/
 deps/
-#.testing/*/available_diags.*
-#.testing/*/CPU_stats
-#.testing/*/chksum_diag
-#.testing/*/exitcode
-#.testing/*/logfile.*.out
-#.testing/*/MOM_parameter_doc.*
-#.testing/*/ocean_geometry.nc
-#.testing/*/ocean.stats
-#.testing/*/ocean.stats.nc
-#.testing/*/RESTART/
-#.testing/*/time_stamp.out
-#.testing/*/Vertical_coordinate.nc
-#.testing/*/GOLD_IC.nc

.testing/.gitignore

@@ -1,13 +1,3 @@
-available_diags.*
-CPU_stats
-chksum_diag
-exitcode
-logfile.*.out
-MOM_parameter_doc.*
-ocean_geometry.nc
-ocean.stats
-ocean.stats.nc
-RESTART/
-time_stamp.out
-Vertical_coordinate.nc
-GOLD_IC.nc
+config.mk
+work/
+results/

.testing/Makefile

@@ -1,8 +1,12 @@
 SHELL = bash
-MPIRUN ?= mpirun
 
+# User-defined configuration
 -include config.mk
 
+# Default configurations
+MPIRUN ?= mpirun
+DO_REPRO_TESTS ?= true
+
 #---
 # Dependencies
 BASE = $(dir $(abspath $(lastword $(MAKEFILE_LIST))))/..
@@ -36,42 +40,53 @@ MKMF_TEMPLATE ?= $(DEPS)/mkmf/templates/ncrc-gnu.mk
 # Test configuration
 
 # Executables
-BUILDS = symmetric asymmetric repro
-CONFIGS := $(foreach n,$(shell seq 0 3),tc$(n))
-TESTS = grids layouts restarts repros nans dims
+BUILDS = symmetric asymmetric repro openmp
+CONFIGS := $(wildcard tc*)
+TESTS = grids layouts restarts nans dims openmps
+
+# REPRO tests enable reproducibility with optimization, and often do not match
+# the DEBUG results in older GCCs and vendor compilers, so we can optionally
+# disable them.
+ifeq ($(DO_REPRO_TESTS), true)
+  BUILDS += repro
+  TESTS += repros
+endif
 
 # The following variables are configured by Travis:
 #   DO_REGRESSION_TESTS: true if $(TRAVIS_PULL_REQUEST) is a PR number
 #   MOM_TARGET_SLUG: TRAVIS_REPO_SLUG
 #   MOM_TARGET_LOCAL_BRANCH: TRAVIS_BRANCH
-#
-# These are set to true by Travis if testing a pull request
+
+# These are set to true by our Travis configuration if testing a pull request
 DO_REGRESSION_TESTS ?=
 REPORT_COVERAGE ?=
 
 ifeq ($(DO_REGRESSION_TESTS), true)
-	BUILDS += target
-	TEST += regressions
+  BUILDS += target
+  TESTS += regressions
 
-	MOM_TARGET_SLUG ?= NOAA-GFDL/MOM6
-	MOM_TARGET_URL ?= https://github.com/$(MOM_TARGET_SLUG)
+  MOM_TARGET_SLUG ?= NOAA-GFDL/MOM6
+  MOM_TARGET_URL ?= https://github.com/$(MOM_TARGET_SLUG)
 
-	MOM_TARGET_LOCAL_BRANCH ?= dev/gfdl
-	MOM_TARGET_BRANCH := origin/$(MOM_TARGET_LOCAL_BRANCH)
+  MOM_TARGET_LOCAL_BRANCH ?= dev/gfdl
+  MOM_TARGET_BRANCH := origin/$(MOM_TARGET_LOCAL_BRANCH)
 
-	TARGET_CODEBASE = $(BUILD)/target_codebase
+  TARGET_CODEBASE = $(BUILD)/target_codebase
 else
-	MOM_TARGET_URL =
-	MOM_TARGET_BRANCH =
-	TARGET_CODEBASE =
+  MOM_TARGET_URL =
+  MOM_TARGET_BRANCH =
+  TARGET_CODEBASE =
 endif
 
+SOURCE = $(wildcard $(BASE)/src/*/*.F90 $(BASE)/src/*/*/*.F90 $(BASE)/config_src/solo_driver/*.F90)
+
 
 #---
 # Rules
 
-.PHONY: all
+.PHONY: all build.regressions
 all: $(foreach b,$(BUILDS),$(BUILD)/$(b)/MOM6)
+build.regressions: $(foreach b,symmetric target,$(BUILD)/$(b)/MOM6)
 
 # Executable
 BUILD_TARGETS = MOM6 Makefile path_names
@@ -84,6 +99,7 @@ $(BUILD)/target/MOM6: MOMFLAGS=NETCDF=3 DEBUG=1
 $(BUILD)/symmetric/MOM6: MOMFLAGS=NETCDF=3 DEBUG=1 $(COVFLAG)
 $(BUILD)/asymmetric/MOM6: MOMFLAGS=NETCDF=3 DEBUG=1
 $(BUILD)/repro/MOM6: MOMFLAGS=NETCDF=3 REPRO=1
+$(BUILD)/openmp/MOM6: MOMFLAGS=NETCDF=3 DEBUG=1 OPENMP=1
 
 $(BUILD)/asymmetric/path_names: GRID_SRC=config_src/dynamic
 $(BUILD)/%/path_names: GRID_SRC=config_src/dynamic_symmetric
@@ -110,7 +126,7 @@ $(BUILD)/target/path_names: $(LIST_PATHS) $(TARGET_CODEBASE)
 		$(TARGET_CODEBASE)/config_src/solo_driver \
 		$(TARGET_CODEBASE)/$(GRID_SRC)
 
-$(BUILD)/%/path_names: $(LIST_PATHS)
+$(BUILD)/%/path_names: $(LIST_PATHS) $(SOURCE)
 	mkdir -p $(@D)
 	cd $(@D) && $(LIST_PATHS) -l \
 		$(BASE)/src \
@@ -164,71 +180,90 @@ test: $(foreach t,$(TESTS),test.$(t))
 # NOTE: We remove tc3 (OBC) from grid test since it cannot run asymmetric grids
 
 .PHONY: $(foreach t,$(TESTS),test.$(t))
-test.regressions: $(foreach c,$(CONFIGS),$(c).regression $(c).regression.diag)
 test.grids: $(foreach c,$(filter-out tc3,$(CONFIGS)),$(c).grid $(c).grid.diag)
 test.layouts: $(foreach c,$(CONFIGS),$(c).layout $(c).layout.diag)
 test.restarts: $(foreach c,$(CONFIGS),$(c).restart)
 test.repros: $(foreach c,$(CONFIGS),$(c).repro $(c).repro.diag)
+test.openmps: $(foreach c,$(CONFIGS),$(c).openmp $(c).openmp.diag)
 test.nans: $(foreach c,$(CONFIGS),$(c).nan $(c).nan.diag)
 test.dims: $(foreach c,$(CONFIGS),$(foreach d,t l h z,$(c).dim.$(d) $(c).dim.$(d).diag))
 
-# NOTE: chksum_diag return code of cmp is currently ignored since many fail!
+test.regressions: $(foreach c,$(CONFIGS),$(c).regression $(c).regression.diag)
+	! ls -1 results/*/*.reg
+
 define CMP_RULE
-.PRECIOUS: $(foreach b,$(2),$(BASE)/.testing/%/ocean.stats.$(b))
-%.$(1): $(foreach b,$(2),$(BASE)/.testing/%/ocean.stats.$(b))
-	cmp $$^
+.PRECIOUS: $(foreach b,$(2),results/%/ocean.stats.$(b))
+%.$(1): $(foreach b,$(2),results/%/ocean.stats.$(b))
+	cmp $$^ || diff $$^
 
-.PRECIOUS: $(foreach b,$(2),$(BASE)/.testing/%/chksum_diag.$(b))
-%.$(1).diag: $(foreach b,$(2),$(BASE)/.testing/%/chksum_diag.$(b))
-	cmp $$^ || true
+.PRECIOUS: $(foreach b,$(2),results/%/chksum_diag.$(b))
+%.$(1).diag: $(foreach b,$(2),results/%/chksum_diag.$(b))
+	cmp $$^ || diff $$^
 endef
 
-$(eval $(call CMP_RULE,regression,symmetric target))
 $(eval $(call CMP_RULE,grid,symmetric asymmetric))
 $(eval $(call CMP_RULE,layout,symmetric layout))
 $(eval $(call CMP_RULE,repro,symmetric repro))
+$(eval $(call CMP_RULE,openmp,symmetric openmp))
 $(eval $(call CMP_RULE,nan,symmetric nan))
 $(foreach d,t l h z,$(eval $(call CMP_RULE,dim.$(d),symmetric dim.$(d))))
 
+# Custom comparison rules
+
+.PRECIOUS: $(foreach b,symmetric restart target,results/%/ocean.stats.$(b))
+
 # Restart tests only compare the final stat record
-.PRECIOUS: $(foreach b,symmetric restart,$(BASE)/.testing/%/ocean.stats.$(b))
-%.restart: $(foreach b,symmetric restart,$(BASE)/.testing/%/ocean.stats.$(b))
-	cmp $(foreach f,$^,<(tr -s ' ' < $(f) | cut -d ' ' -f3- | tail -n 1))
+%.restart: $(foreach b,symmetric restart,results/%/ocean.stats.$(b))
+	cmp $(foreach f,$^,<(tr -s ' ' < $(f) | cut -d ' ' -f3- | tail -n 1)) \
+		|| diff $^
 
 # TODO: chksum_diag parsing of restart files
 
+# All regression tests must be completed when considering answer changes
+%.regression: $(foreach b,symmetric target,results/%/ocean.stats.$(b))
+	cmp $^ || (diff $^ > $<.reg || true)
+
+%.regression.diag: $(foreach b,symmetric target,results/%/chksum_diag.$(b))
+	cmp $^ || (diff $^ > $<.reg || true)
 
 #---
 # Test run output files
 
-#(1): Configuration name
-#(2): Executable type
-#(3): Enable coverage flag
-#(4): MOM_override configuration
-#(5): Environment variables
-#(6): Number of MPI ranks
-
-# Simple function for generalised Slurm (srun) and OpenMPI (mpirun) support
-# (1): Environment variables
-
-ifeq ($(MPIRUN), srun)
-MPIRUN_CMD=$(1) $(MPIRUN)
+# Generalized MPI environment variable support
+# $(1): Environment variables
+ifeq ($(shell $(MPIRUN) -x tmp=1 true 2> /dev/null ; echo $$?), 0)
+  MPIRUN_CMD=$(MPIRUN) $(if $(1),-x $(1),)
+else ifeq ($(shell $(MPIRUN) -env tmp=1 true 2> /dev/null ; echo $$?), 0)
+  MPIRUN_CMD=$(MPIRUN) $(if $(1),-env $(1),)
 else
-MPIRUN_CMD=$(MPIRUN) $(if $(1),-x $(1),)
+  MPIRUN_CMD=$(1) $(MPIRUN)
 endif
 
+# Rule to build results/<tc>/{ocean.stats,chksum_diag}.<tag>
+# $(1): Test configuration name <tag>
+# $(2): Executable type
+# $(3): Enable coverage flag
+# $(4): MOM_override configuration
+# $(5): Environment variables
+# $(6): Number of MPI ranks
 define STAT_RULE
-$$(BASE)/.testing/%/ocean.stats.$(1): $$(BUILD)/$(2)/MOM6
-	if [ $(3) ]; then find $$(BUILD) -name *.gcda -exec rm -f '{}' \; ; fi
-	mkdir -p $$(@D)/RESTART
-	echo $(4) > $$(@D)/MOM_override
-	cd $$(@D) && $$(call MPIRUN_CMD,$(5)) -n $(6) $$< 2> debug.out
-	cp $$(@D)/ocean.stats $$@
-	> $$(@D)/MOM_override
-	if [ $(3) ]; then cd $$(BASE) && bash <(curl -s https://codecov.io/bash) -n $$@; fi
-
-$$(BASE)/.testing/%/chksum_diag.$(1): $$(BASE)/.testing/%/ocean.stats.$(1)
-	cp $$(@D)/chksum_diag $$@
+results/%/ocean.stats.$(1): ../build/$(2)/MOM6
+	if [ $(3) ]; then find ../build/$(2) -name *.gcda -exec rm -f '{}' \; ; fi
+	mkdir -p work/$$*/$(1)
+	cp -rL $$*/* work/$$*/$(1)
+	cd work/$$*/$(1) && if [ -f Makefile ]; then make; fi
+	mkdir -p work/$$*/$(1)/RESTART
+	echo $(4) > work/$$*/$(1)/MOM_override
+	cd work/$$*/$(1) && $$(call MPIRUN_CMD,$(5)) -n $(6) ../../../$$< 2> debug.out > std.out \
+		|| ! sed 's/^/$$*.$(1): /' std.out debug.out \
+		&& sed 's/^/$$*.$(1): /' std.out
+	mkdir -p $$(@D)
+	cp work/$$*/$(1)/ocean.stats $$@
+	if [ $(3) ]; then cd .. && bash <(curl -s https://codecov.io/bash) -n $$@; fi
+
+results/%/chksum_diag.$(1): results/%/ocean.stats.$(1)
+	mkdir -p $$(@D)
+	cp work/$$*/$(1)/chksum_diag $$@
 endef
 
 # Define $(,) as comma escape character
@@ -238,51 +273,58 @@ $(eval $(call STAT_RULE,symmetric,symmetric,$(REPORT_COVERAGE),,,1))
 $(eval $(call STAT_RULE,asymmetric,asymmetric,,,,1))
 $(eval $(call STAT_RULE,target,target,,,,1))
 $(eval $(call STAT_RULE,repro,repro,,,,1))
+$(eval $(call STAT_RULE,openmp,openmp,,,,1))
 $(eval $(call STAT_RULE,layout,symmetric,,LAYOUT=2$(,)1,,2))
 $(eval $(call STAT_RULE,nan,symmetric,,,MALLOC_PERTURB_=256,1))
 $(eval $(call STAT_RULE,dim.t,symmetric,,T_RESCALE_POWER=11,,1))
 $(eval $(call STAT_RULE,dim.l,symmetric,,L_RESCALE_POWER=11,,1))
 $(eval $(call STAT_RULE,dim.h,symmetric,,H_RESCALE_POWER=11,,1))
 $(eval $(call STAT_RULE,dim.z,symmetric,,Z_RESCALE_POWER=11,,1))
 
-# Restart tests require signicant preprocessing, and are handled separately.
-$(BASE)/.testing/%/ocean.stats.restart: $(BUILD)/symmetric/MOM6
-	# Cleanup
-	mkdir -p $(@D)/RESTART
-	git checkout $(@D)/input.nml
-	> $(@D)/MOM_override
+# Restart tests require significant preprocessing, and are handled separately.
+results/%/ocean.stats.restart: ../build/symmetric/MOM6
+	rm -rf work/$*/restart
+	mkdir -p work/$*/restart
+	cp -rL $*/* work/$*/restart
+	cd work/$*/restart && if [ -f Makefile ]; then make; fi
+	mkdir -p work/$*/restart/RESTART
 	# Generate the half-period input namelist
 	# TODO: Assumes runtime set by DAYMAX, will fail if set by input.nml
-	cd $(@D) \
+	cd work/$*/restart \
 		&& daymax=$$(grep DAYMAX MOM_input | cut -d '!' -f 1 | cut -d '=' -f 2 | xargs) \
 		&& timeunit=$$(grep TIMEUNIT MOM_input | cut -d '!' -f 1 | cut -d '=' -f 2 | xargs) \
 		&& if [ -z "$${timeunit}" ]; then timeunit="8.64e4"; fi \
 		&& printf -v timeunit_int "%.f" "$${timeunit}" \
 		&& halfperiod=$$(printf "%.f" $$(bc <<< "scale=10; 0.5 * $${daymax} * $${timeunit_int}")) \
-		&& printf "\n&ocean_solo_nml\n    seconds = $${halfperiod}\n/\n" >> input.nml \
-		&& echo $${daymax} $${timeunit}
+		&& printf "\n&ocean_solo_nml\n    seconds = $${halfperiod}\n/\n" >> input.nml
 	# Run the first half-period
-	cd $(@D) && $(MPIRUN) -n 1 $< 2> debug.out
+	cd work/$*/restart && $(MPIRUN) -n 1 ../../../$< 2> debug1.out > std1.out \
+		|| ! sed 's/^/$*.restart1: /' std1.out debug1.out \
+		&& sed 's/^/$*.restart1: /' std1.out
 	# Setup the next inputs
-	rm -rf $(@D)/INPUT && mv $(@D)/RESTART $(@D)/INPUT
-	mkdir $(@D)/RESTART
-	cd $(@D) && sed -i -e "s/input_filename *= *'n'/input_filename = 'r'/g" input.nml
+	cd work/$*/restart && rm -rf INPUT && mv RESTART INPUT
+	mkdir work/$*/restart/RESTART
+	cd work/$*/restart && sed -i -e "s/input_filename *= *'n'/input_filename = 'r'/g" input.nml
 	# Run the second half-period
-	cd $(@D) && $(MPIRUN) -n 1 $< 2> debug.out
+	cd work/$*/restart && $(MPIRUN) -n 1 ../../../$< 2> debug2.out > std2.out \
+		|| ! sed 's/^/$*.restart2: /' std2.out debug2.out \
+		&& sed 's/^/$*.restart2: /' std2.out
 	# Archive the results and cleanup
-	cp $(@D)/ocean.stats $@
-	rm -rf $(@D)/INPUT
-	git checkout $(@D)/input.nml
+	mkdir -p $(@D)
+	cp work/$*/restart/ocean.stats $@
 
 # TODO: Restart checksum diagnostics
 
 
 #----
 .PHONY: clean
 clean: clean.stats
-	rm -rf $(BUILD)
+	@# Assert that we are in .testing for recursive delete
+	@[ $$(basename $$(pwd)) = .testing ]
+	rm -rf ../build
 
 .PHONY: clean.stats
 clean.stats:
-	find $(BASE)/.testing -name ocean.stats* -exec rm {} \;
-	find $(BASE)/.testing -name chksum_diag* -exec rm {} \;
+	@# Assert that we are in .testing for recursive delete
+	@[ $$(basename $$(pwd)) = .testing ]
+	rm -rf work results

.testing/README.md

@@ -56,7 +56,7 @@ Model state is currently defined by the `ocean.stats` output file, which
 reports the total energy (per unit mass) at machine precision alongside similar
 global metrics, such as mass or mean sea level, at lower precision.
 
-Clhecksums for every available diagnostic are also compared and the Makefile
+Checksums for every available diagnostic are also compared and the Makefile
 will report any differences, but such differences are not yet considered a fail
 condition.
 
@@ -138,7 +138,7 @@ This will run through the following tests:
 - `test.restarts`: Resubmission by restarts
 - `test.repros`: Optimized (REPRO) and unoptimized (DEBUG) compilation
 - `test.nans`: NaN initialization of allocated arrays
-- `test.dims`: Dimensional scaling (length, time, thichkness, depth)
+- `test.dims`: Dimensional scaling (length, time, thickness, depth)
 
 To enable the regression tests, use `DO_REGRESSION_TEST=true`.
 ```
@@ -159,18 +159,21 @@ fail if the answers differ from this build.
 
 The following test configurations (TCs) are supported:
 
-- TC0: Unit testing of various model components, based on `unit_tests`
-- TC1: A low-resolution version of the `benchmark` configuration
-- TC2: An ALE configuration based on TC1
-- TC3: An open-boundary condition (OBC) test based on `circle_obcs`
+- tc0: Unit testing of various model components, based on `unit_tests`
+- tc1: A low-resolution version of the `benchmark` configuration
+  - tc1.a: Use the un-split mode with Runge-Kutta 3 time integration
+  - tc1.b: Use the un-split mode with Runge-Kutta 2 time integration
+- tc2: An ALE configuration based on tc1 with tides
+  - tc2.a: Use sigma, PPM_H4 and no tides
+- tc3: An open-boundary condition (OBC) test based on `circle_obcs`
 
 
 ## Code coverage
 
 Code coverage reports the lines of code which have been tested, and can
 explicitly demonstrate when a particular operation is untested.
 
-Coverage is measued using `gcov` and is reported for TCs using the `symmetric`
+Coverage is measured using `gcov` and is reported for TCs using the `symmetric`
 executable.
 
 Coverage reporting is optionally sent to the `codecov.io` site.

.testing/tc0/diag_table

@@ -1,2 +1,2 @@
-"Unit tests"
+"MOM test configuration 0"
 1 1 1 0 0 0