converted ftst_program_setup to MPI test

cmake/LibMPI.cmake

@@ -0,0 +1,217 @@
+# This file contains CMake code related to MPI. (Taken from the
+# ParallelIO project.)
+
+# Jim Edwards
+include (CMakeParseArguments)
+
+# Find Valgrind to perform memory leak check
+if (PIO_VALGRIND_CHECK)
+  find_program (VALGRIND_COMMAND NAMES valgrind)
+  if (VALGRIND_COMMAND)
+    set (VALGRIND_COMMAND_OPTIONS --leak-check=full --show-reachable=yes)
+  else ()
+    message (WARNING "Valgrind not found: memory leak check could not be performed")
+    set (VALGRIND_COMMAND "")
+  endif ()
+endif ()
+
+#
+# - Functions for parallel testing with CTest
+#
+
+#==============================================================================
+# - Get the machine platform-specific
+#
+# Syntax:  platform_name (RETURN_VARIABLE)
+#
+function (platform_name RETURN_VARIABLE)
+
+  # Determine platform name from site name...
+  site_name (SITENAME)
+
+  # hera
+  if (SITENAME MATCHES "^hfe01" OR
+      SITENAME MATCHES "^hfe02" OR
+      SITENAME MATCHES "^hfe03" OR
+      SITENAME MATCHES "^hfe04" OR
+      SITENAME MATCHES "^hfe05" OR
+      SITENAME MATCHES "^hfe06" OR
+      SITENAME MATCHES "^hfe07" OR
+      SITENAME MATCHES "^hfe08" OR
+      SITENAME MATCHES "^hfe09" OR
+      SITENAME MATCHES "^hfe10" OR
+      SITENAME MATCHES "^hfe11" OR
+      SITENAME MATCHES "^hfe12")
+
+    set (${RETURN_VARIABLE} "hera" PARENT_SCOPE)
+
+  # wcoss_cray (Luna)
+  elseif (SITENAME MATCHES "^llogin1" OR
+      SITENAME MATCHES "^llogin2" OR
+      SITENAME MATCHES "^llogin3")
+
+    set (${RETURN_VARIABLE} "wcoss_cray" PARENT_SCOPE)
+
+  # wcoss_cray (Surge)
+  elseif (SITENAME MATCHES "^slogin1" OR
+      SITENAME MATCHES "^slogin2" OR
+      SITENAME MATCHES "^slogin3")
+
+    set (${RETURN_VARIABLE} "wcoss_cray" PARENT_SCOPE)
+
+  # wcoss_dell_p3 (Venus)
+  elseif (SITENAME MATCHES "^v71a1.ncep.noaa.gov" OR
+      SITENAME MATCHES "^v71a2.ncep.noaa.gov" OR
+      SITENAME MATCHES "^v71a3.ncep.noaa.gov" OR
+      SITENAME MATCHES "^v72a1.ncep.noaa.gov" OR
+      SITENAME MATCHES "^v72a2.ncep.noaa.gov" OR
+      SITENAME MATCHES "^v72a3.ncep.noaa.gov")
+
+    set (${RETURN_VARIABLE} "wcoss_dell_p3" PARENT_SCOPE)
+
+  # wcoss_dell_p3 (Mars)
+  elseif (SITENAME MATCHES "^m71a1.ncep.noaa.gov" OR
+      SITENAME MATCHES "^m71a2.ncep.noaa.gov" OR
+      SITENAME MATCHES "^m71a3.ncep.noaa.gov" OR
+      SITENAME MATCHES "^m72a1.ncep.noaa.gov" OR
+      SITENAME MATCHES "^m72a2.ncep.noaa.gov" OR
+      SITENAME MATCHES "^m72a3.ncep.noaa.gov")
+
+    set (${RETURN_VARIABLE} "wcoss_dell_p3" PARENT_SCOPE)
+
+  # wcoss2
+  elseif (SITENAME MATCHES "^along01" OR
+      SITENAME MATCHES "^alogin02")
+
+    set (${RETURN_VARIABLE} "wcoss2" PARENT_SCOPE)
+
+  # gaea
+  elseif (SITENAME MATCHES "^gaea9" OR
+      SITENAME MATCHES "^gaea10" OR
+      SITENAME MATCHES "^gaea11" OR
+      SITENAME MATCHES "^gaea12" OR
+      SITENAME MATCHES "^gaea13" OR
+      SITENAME MATCHES "^gaea14" OR
+      SITENAME MATCHES "^gaea15" OR
+      SITENAME MATCHES "^gaea16" OR
+      SITENAME MATCHES "^gaea9.ncrc.gov" OR
+      SITENAME MATCHES "^gaea10.ncrc.gov" OR
+      SITENAME MATCHES "^gaea11.ncrc.gov" OR
+      SITENAME MATCHES "^gaea12.ncrc.gov" OR
+      SITENAME MATCHES "^gaea13.ncrc.gov" OR
+      SITENAME MATCHES "^gaea14.ncrc.gov" OR
+      SITENAME MATCHES "^gaea15.ncrc.gov" OR
+      SITENAME MATCHES "^gaea16.ncrc.gov")
+
+    set (${RETURN_VARIABLE} "gaea" PARENT_SCOPE)
+
+  # jet 
+  elseif (SITENAME MATCHES "^fe1" OR
+      SITENAME MATCHES "^fe2" OR
+      SITENAME MATCHES "^fe3" OR
+      SITENAME MATCHES "^fe4" OR
+      SITENAME MATCHES "^fe5" OR
+      SITENAME MATCHES "^fe6" OR
+      SITENAME MATCHES "^fe7" OR
+      SITENAME MATCHES "^fe8" OR
+      SITENAME MATCHES "^tfe1" OR
+      SITENAME MATCHES "^tfe2")
+
+    set (${RETURN_VARIABLE} "jet" PARENT_SCOPE)
+
+  elseif (SITENAME MATCHES "^Orion-login-1.HPC.MsState.Edu" OR
+      SITENAME MATCHES "^Orion-login-2.HPC.MsState.Edu" OR
+      SITENAME MATCHES "^Orion-login-3.HPC.MsState.Edu" OR
+      SITENAME MATCHES "^Orion-login-4.HPC.MsState.Edu")
+
+    set (${RETURN_VARIABLE} "orion" PARENT_SCOPE)
+
+  elseif (SITENAME MATCHES "^cheyenne1.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne1.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne2.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne3.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne4.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne5.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne6.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne1.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne2.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne3.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne4.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne5.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^cheyenne6.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^chadmin1.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^chadmin2.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^chadmin3.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^chadmin4.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^chadmin5.ib0.cheyenne.ucar.edu" OR
+      SITENAME MATCHES "^chadmin6.ib0.cheyenne.ucar.edu")
+
+    set (${RETURN_VARIABLE} "cheyenne" PARENT_SCOPE)
+  elseif (SITENAME MATCHES "^login1.stampede2.tacc.utexas.edu" OR
+      SITENAME MATCHES "^login2.stampede2.tacc.utexas.edu" OR
+      SITENAME MATCHES "^login3.stampede2.tacc.utexas.edu" OR
+      SITENAME MATCHES "^login4.stampede2.tacc.utexas.edu")
+
+
+    set (${RETURN_VARIABLE} "stampede" PARENT_SCOPE)
+
+  else ()
+
+    set (${RETURN_VARIABLE} "unknown" PARENT_SCOPE)
+
+  endif ()
+endfunction ()
+
+#==============================================================================
+# - Add a new parallel test
+#
+# Syntax:  add_mpi_test (<TESTNAME>
+#                        EXECUTABLE <command>
+#                        ARGUMENTS <arg1> <arg2> ...
+#                        NUMPROCS <num_procs>
+#                        TIMEOUT <timeout>)
+function (add_mpi_test TESTNAME)
+
+  # Parse the input arguments
+  set (options)
+  set (oneValueArgs NUMPROCS TIMEOUT EXECUTABLE)
+  set (multiValueArgs ARGUMENTS)
+  cmake_parse_arguments (${TESTNAME} "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+
+  # Store parsed arguments for convenience
+  set (exec_file ${${TESTNAME}_EXECUTABLE})
+  set (exec_args ${${TESTNAME}_ARGUMENTS})
+  set (num_procs ${${TESTNAME}_NUMPROCS})
+  set (timeout ${${TESTNAME}_TIMEOUT})
+
+  # Get the platform name
+  platform_name (PLATFORM)
+
+  get_property(WITH_MPIEXEC GLOBAL PROPERTY WITH_MPIEXEC)
+  if (WITH_MPIEXEC)
+    set(MPIEXEC "${WITH_MPIEXEC}")
+  endif ()
+
+  # Default ("unknown" platform) execution
+  if (PLATFORM STREQUAL "unknown")
+
+    # Run tests directly from the command line
+    set(EXE_CMD ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${num_procs}
+      ${MPIEXEC_PREFLAGS} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file}
+      ${MPIEXEC_POSTFLAGS} ${exec_args})
+
+  else ()
+
+    # Run tests from the platform-specific executable
+    set (EXE_CMD ${CMAKE_SOURCE_DIR}/cmake/mpiexec.${PLATFORM}
+      ${num_procs} ${VALGRIND_COMMAND} ${VALGRIND_COMMAND_OPTIONS} ${exec_file} ${exec_args})
+
+  endif ()
+
+  # Add the test to CTest
+  add_test(NAME ${TESTNAME} COMMAND ${EXE_CMD})
+
+  # Adjust the test timeout
+  set_tests_properties(${TESTNAME} PROPERTIES TIMEOUT ${timeout})
+
+endfunction()

cmake/mpiexec.hera

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# Arguments:
+#
+#   $1  - Number of MPI Tasks
+#   $2+ - Executable and its arguments
+#
+
+ACCOUNT=
+QOS=debug
+
+NP=$1
+shift
+
+srun -A $ACCOUNT -q $QOS -n $NP $@

cmake/mpiexec.jet

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# Arguments:
+#
+#   $1  - Number of MPI Tasks
+#   $2+ - Executable and its arguments
+#
+
+ACCOUNT=
+QOS=debug
+
+NP=$1
+shift
+
+srun -A $ACCOUNT -q $QOS -n $NP $@

cmake/mpiexec.orion

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# Arguments:
+#
+#   $1  - Number of MPI Tasks
+#   $2+ - Executable and its arguments
+#
+
+ACCOUNT=
+QOS=debug
+
+NP=$1
+shift
+
+srun -A $ACCOUNT -q $QOS -n $NP $@

tests/chres_cube/CMakeLists.txt

@@ -3,6 +3,9 @@
 #
 # George Gayno, Lin Gan, Ed Hartnett
 
+# Include cmake to allow parallel I/O tests.
+include (LibMPI)
+
 if(CMAKE_Fortran_COMPILER_ID MATCHES "^(Intel)$")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -r8 -convert big_endian -assume byterecl")
 elseif(CMAKE_Fortran_COMPILER_ID MATCHES "^(GNU)$")
@@ -51,7 +54,6 @@ add_executable(ftst_program_setup "${CMAKE_SOURCE_DIR}/sorc/chgres_cube.fd/input
   "${CMAKE_SOURCE_DIR}/sorc/chgres_cube.fd/search_util.F90"
   "${CMAKE_SOURCE_DIR}/sorc/chgres_cube.fd/utils.F90"
   ftst_program_setup.F90)
-add_test(NAME ftst_program_setup COMMAND ftst_program_setup)
 target_link_libraries(
   ftst_program_setup
   nemsio::nemsio
@@ -69,5 +71,10 @@ if(OpenMP_Fortran_FOUND)
   target_link_libraries(ftst_program_setup OpenMP::OpenMP_Fortran)
 endif()
 
+# Cause test to be run with MPI.
+add_mpi_test(ftst_program_setup
+  EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/ftst_program_setup
+  NUMPROCS 4
+  TIMEOUT 60)
 
 

tests/chres_cube/ftst_program_setup.F90

@@ -3,15 +3,21 @@
 ! Ed Hartnett 2/16/21
 
 program ftst_program_setup
+  use mpi
   use esmf
   use netcdf
   use program_setup
   implicit none
   integer :: is
+  integer :: my_rank, nprocs
+  integer :: ierr
 
-  print*, "Starting test of program_setup."
+  call mpi_init(ierr)
+  call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr)
+  call MPI_Comm_size(MPI_COMM_WORLD, nprocs, ierr)
 
-  print*, "testing read_setup_namelist with file fort.41..."
+  if (my_rank .eq. 0) print*, "Starting test of program_setup."
+  if (my_rank .eq. 0) print*, "testing read_setup_namelist with file fort.41..."
   call read_setup_namelist()
   if (cycle_mon .ne. 7 .or. cycle_day .ne. 4 .or. cycle_hour .ne. 12) stop 4
   if (.not. convert_atm .or. .not. convert_sfc .or. .not. convert_nst) stop 5
@@ -43,9 +49,9 @@ program ftst_program_setup
      tracers(is) = "NULL"
      tracers_input(is) = "NULL"
   enddo
-  print*, "OK"
+  if (my_rank .eq. 0) print*, "OK"
 
-  print*, "testing read_setup_namelist with config_fv3_tiled..."
+  if (my_rank .eq. 0) print*, "testing read_setup_namelist with config_fv3_tiled..."
   call read_setup_namelist("config_fv3_tiled.nml")
   if (cycle_mon .ne. 10 .or. cycle_day .ne. 3 .or. cycle_hour .ne. 0) stop 34
   if (.not. convert_atm .or. .not. convert_sfc .or. .not. convert_nst) stop 35
@@ -73,14 +79,14 @@ program ftst_program_setup
      tracers(is) = "NULL"
      tracers_input(is) = "NULL"
   enddo
-  print*, "OK"
+  if (my_rank .eq. 0) print*, "OK"
 
   ! Reading this namelist fails for some reason.
   ! print*, "testing read_setup_namelist with config_fv3_tiled_warm_restart..."
   ! call read_setup_namelist("config_fv3_tiled_warm_restart.nml")
   ! print*, "OK"
 
-  print*, "testing read_setup_namelist with config_gaussian_nemsio..."
+  if (my_rank .eq. 0) print*, "testing read_setup_namelist with config_gaussian_nemsio..."
   call read_setup_namelist("config_gaussian_nemsio.nml")
   if (cycle_mon .ne. 7 .or. cycle_day .ne. 4 .or. cycle_hour .ne. 12) stop 74
   if (.not. convert_atm .or. .not. convert_sfc .or. .not. convert_nst) stop 75
@@ -98,7 +104,6 @@ program ftst_program_setup
   if (atm_weight_file .ne. "NULL") stop 88
   if (trim(input_type) .ne. "gaussian_nemsio") stop 89
   if (trim(external_model) .ne. "GFS") stop 90
-  print *,num_tracers
   if (num_tracers .ne. 7) stop 21
   if (tracers(1) .ne. "sphum" .or. tracers(2) .ne. "liq_wat" .or. tracers(3) .ne. "o3mr" .or. &
        tracers(4) .ne. "ice_wat" .or. tracers(5) .ne. "rainwat" .or. tracers(6) .ne. "snowwat" .or. &
@@ -113,9 +118,9 @@ program ftst_program_setup
      tracers(is) = "NULL"
      tracers_input(is) = "NULL"
   enddo
-  print*, "OK"
+  if (my_rank .eq. 0) print*, "OK"
 
-  print*, "testing read_setup_namelist with config_spectral_sigio..."
+  if (my_rank .eq. 0) print*, "testing read_setup_namelist with config_spectral_sigio..."
   call read_setup_namelist("config_spectral_sigio.nml")
   if (cycle_mon .ne. 7 .or. cycle_day .ne. 17 .or. cycle_hour .ne. 0) stop 114
   if (.not. convert_atm .or. .not. convert_sfc .or. convert_nst) stop 115
@@ -143,9 +148,9 @@ program ftst_program_setup
      tracers(is) = "NULL"
      tracers_input(is) = "NULL"
   enddo
-  print*, "OK"
+  if (my_rank .eq. 0) print*, "OK"
 
-  print*, "testing read_setup_namelist with config_gfs_grib2..."
+  if (my_rank .eq. 0) print*, "testing read_setup_namelist with config_gfs_grib2..."
   call read_setup_namelist("config_gfs_grib2.nml")
   if (cycle_mon .ne. 11 .or. cycle_day .ne. 4 .or. cycle_hour .ne. 0) stop 94
   if (.not. convert_atm .or. .not. convert_sfc .or. convert_nst) stop 95
@@ -177,7 +182,9 @@ program ftst_program_setup
      tracers(is) = "NULL"
      tracers_input(is) = "NULL"
   enddo
-  print*, "OK"
+  if (my_rank .eq. 0) print*, "OK"
 
-  print*, "SUCCESS!"
+  if (my_rank .eq. 0) print*, "SUCCESS!"
+
+ call mpi_finalize(ierr)  
 end program ftst_program_setup