ckp

CMakeLists.txt

@@ -210,9 +210,6 @@ SET(EXTRA_DEPS "")
 ENABLE_TESTING()
 INCLUDE(CTest)
 
-# Copy the CTest customization file into binary directory, as required.
-FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/CTestCustom.cmake DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
-
 # Set Memory test program for non-MSVC based builds.
 # Assume valgrind for now.
 IF((NOT MSVC) AND (NOT MINGW) AND (NOT ISCYGWIN))
@@ -1296,7 +1293,7 @@ IF(NOT ENABLE_REMOTE_FUNCTIONALITY)
 ENDIF()
 
 # Control S3 Testing: Multi-valued option
-SET(WITH_S3_TESTING OFF CACHE STRING "Control S3 Testing: ON (i.e. all) OFF (i.e. none)  PUBLIC")
+SET(WITH_ING OFF CACHE STRING "Control S3 Testing: ON (i.e. all) OFF (i.e. none)  PUBLIC")
     SET_PROPERTY(CACHE WITH_S3_TESTING PROPERTY STRINGS ON OFF PUBLIC)  #
 IF(WITH_S3_TESTING STREQUAL "")
   SET(WITH_S3_TESTING OFF CACHE STRING "") # Default
@@ -1340,11 +1337,6 @@ ELSE()
   SET(ENABLE_S3_AWS OFF CACHE BOOL "S3 AWS" FORCE)
 ENDIF()
 
-# Unless/until we get aws-sdk-cpp working for Windows, force use of internal
-IF(ENABLE_S3 AND MSVC)
-  SET(ENABLE_S3_INTERNAL ON CACHE BOOL "S3 Intern" FORCE)
-ENDIF()
-
 IF(ENABLE_S3)
   IF(NOT ENABLE_S3_AWS AND NOT ENABLE_S3_INTERNAL)
     message(FATAL_ERROR "S3 support library not found; please specify option -DENABLE_S3=NO")
@@ -1363,6 +1355,10 @@ IF(WITH_S3_TESTING STREQUAL "PUBLIC" OR WITH_S3_TESTING)
 ENDIF()
 ENDIF()
 
+# The working S3 path tree within the Unidata bucket.
+# WARNING: this must match the value in configure.ac
+SET(S3TESTSUBTREE netcdf-x CACHE STRING "Working S3 path.")
+
 OPTION(ENABLE_LIBXML2 "Link against libxml2 if it is available, use the packaged tinyxml2 parser otherwise." ON)
 SET(XMLPARSER "tinyxml2 (bundled)")
 
@@ -2600,15 +2596,19 @@ SET(NC_WHICH_S3_SDK "none")
 endif()
 
 if(WITH_S3_TESTING STREQUAL PUBLIC)
-SET(DO_S3_TESTING "public")
+SET(ENABLE_S3_TESTING "public")
 elseif(WITH_S3_TESTING)
-SET(DO_S3_TESTING "yes")
+SET(ENABLE_S3_TESTING "yes")
+SET(ENABLE_S3_TESTALL "yes")
 elseif(NOT WITH_S3_TESTING)
-SET(DO_S3_TESTING "no")
+SET(ENABLE_S3_TESTING "no")
 else()
-SET(DO_S3_TESTING "no")
+SET(ENABLE_S3_TESTING "no")
 endif()
 
+# Copy the CTest customization file into binary directory, as required.
+CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/CTestCustom.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/CTestCustom.cmake")
+
 # Generate file from template.
 CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in"
   "${CMAKE_CURRENT_BINARY_DIR}/libnetcdf.settings"
@@ -2679,6 +2679,13 @@ IF(ENABLE_TESTS)
     # Build dap4_test/pingurl4.c
     #####
     configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ncdap_test/pingurl.c ${CMAKE_CURRENT_BINARY_DIR}/dap4_test/pingurl4.c @ONLY NEWLINE_STYLE LF)
+
+    #####
+    # Build CTestCustom.cmake to cleanup S3 after tests are done.
+    #####
+    configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CTestCustom.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/CTestCustom.cmake NEWLINE_STYLE LF)
+
+
 ENDIF()
 
 if(DEFINED ENV{LIB_FUZZING_ENGINE})

Makefile.am

@@ -31,7 +31,7 @@ AM_DISTCHECK_CONFIGURE_FLAGS = --without-plugin-dir
 EXTRA_DIST = README.md COPYRIGHT INSTALL.md test_prog.c lib_flags.am	\
 cmake CMakeLists.txt COMPILE.cmake.txt config.h.cmake.in		\
 cmake_uninstall.cmake.in FixBundle.cmake.in nc-config.cmake.in		\
-RELEASE_NOTES.md CTestCustom.cmake CTestConfig.cmake.in			\
+RELEASE_NOTES.md CTestConfig.cmake.in CTestCustom.cmake.in		\
 libnetcdf.settings.in netCDFConfig.cmake.in CMakeInstallation.cmake	\
 test-driver-verbose test_common.in fuzz
 
@@ -213,3 +213,13 @@ all-local: liblib/libnetcdf.la
 	echo ${PACKAGE_VERSION} > VERSION
 # Remove the VERSION file
 CLEANFILES = VERSION
+
+if ENABLE_S3_TESTALL
+# Cleanup S3 resources
+distcleancheck: s3clean
+
+.PHONY: s3clean
+# Provide a specific s3 cleanup action
+s3clean:
+	echo ">>>> ${abs_srcdir}/run_s3_cleanup.sh ${abs_builddir} ${S3TESTSUBTREE}"
+endif

config.h.cmake.in

@@ -163,6 +163,9 @@ are set when opening a binary file on Windows. */
 /* if true, enable S3 testing*/
 #cmakedefine WITH_S3_TESTING "PUBLIC"
 
+/* S3 Working subtree */
+#define S3TESTSUBTREE "${S3TESTSUBTREE}"
+
 /* if true, run extra tests which may not work yet */
 #cmakedefine EXTRA_TESTS 1
 

configure.ac

@@ -974,6 +974,7 @@ AC_DEFINE_UNQUOTED([WITH_S3_TESTING], [$with_s3_testing], [control S3 testing.])
 
 if test "x$with_s3_testing" = xyes ; then
   AC_MSG_WARN([*** DO NOT SPECIFY WITH_S3_TESTING=YES UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***])
+  AC_DEFINE([ENABLE_S3_TESTALL], [yes], [control S3 testing.])
 fi
 
 # Check whether we want to enable strict null byte header padding.
@@ -2033,7 +2034,7 @@ AC_SUBST(HAS_S3_AWS,[$enable_s3_aws])
 AC_SUBST(HAS_S3_INTERNAL,[$enable_s3_internal])
 AC_SUBST(HAS_HDF5_ROS3,[$has_hdf5_ros3])
 AC_SUBST(HAS_NCZARR,[$enable_nczarr])
-AC_SUBST(DO_S3_TESTING,[$with_s3_testing])
+AC_SUBST(ENABLE_S3_TESTING,[$with_s3_testing])
 AC_SUBST(HAS_NCZARR_ZIP,[$enable_nczarr_zip])
 AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip])
 AC_SUBST(HAS_QUANTIZE,[$enable_quantize])
@@ -2056,6 +2057,11 @@ if test "x$enable_s3_aws" = xno && test "x$enable_s3_internal" = xno; then
 AC_SUBST(WHICH_S3_SDK,[none])
 fi
 
+# The working S3 path tree within the Unidata bucket.
+# WARNING: this must match the value in CMakeLists.txt
+AC_DEFINE([S3TESTSUBTREE], ["netcdf-x"], [S3 working path])
+AC_SUBST([S3TESTSUBTREE],[netcdf-x])
+
 # Always available
 std_filters="bz2"
 if test "x$have_deflate" = xyes ; then

docs/cloud.md

@@ -85,12 +85,12 @@ Currently the following build cases are known to work.
 <tr><td>OSX		<td>Automake	<td>nch5s3comms	<td>unknown
 <tr><td>OSX		<td>CMake	<td>aws-s3-sdk	<td>unknown
 <tr><td>OSX		<td>CMake	<td>nch5s3comms	<td>unknown
-<tr><td>Visual Studio	<td>CMake	<td>aws-s3-sdk	<td>no (tests-fail)
+<tr><td>Visual Studio	<td>CMake	<td>aws-s3-sdk	<td>no (tests fail)
 <tr><td>Visual Studio	<td>CMake	<td>nch5s3comms	<td>yes
-<tr><td>Cygwin		<td>Automake	<td>aws-s3-sdk	<td>unknown
+<tr><td>Cygwin		<td>Automake	<td>aws-s3-sdk	<td>no (tests fail)
 <tr><td>Cygwin		<td>Automake	<td>nch5s3comms	<td>yes
-<tr><td>Cygwin		<td>CMake	<td>aws-s3-sdk	<td>unknown
-<tr><td>Cygwin		<td>CMake	<td>nch5s3comms	<td>unknown
+<tr><td>Cygwin		<td>CMake	<td>aws-s3-sdk	<td>no
+<tr><td>Cygwin		<td>CMake	<td>nch5s3comms	<td>yes
 <tr><td>Mingw		<td>Automake	<td>aws-s3-sdk	<td>unknown
 <tr><td>Mingw		<td>Automake	<td>nch5s3comms	<td>unknown
 <tr><td>Mingw		<td>CMake	<td>aws-s3-sdk	<td>unknown
@@ -163,7 +163,7 @@ This library, [aws-sdk-cpp library](https://github.com/aws/aws-sdk-cpp.git),
 has a number of properties of interest:
 * It is written in C++
 * It is available on [GitHub](https://github.com/aws/aws-sdk-cpp.git),
-* It uses CMake + ninja as its primary build system.
+* It uses CMake as its primary build system.
 
 ### *\*nix\** Build
 
@@ -179,18 +179,17 @@ pushd aws-sdk-cpp
 mkdir build
 cd build
 PREFIX=/usr/local
-FLAGS="-DCMAKE_INSTALL_PREFIX=${PREFIX} \
-       -DCMAKE_INSTALL_LIBDIR=lib \
-       -DCMAKE_MODULE_PATH=${PREFIX}/lib/cmake \
+FLAGS="-DCMAKE_INSTALL_PREFIX=/usr/local \
        -DCMAKE_POLICY_DEFAULT_CMP0075=NEW \
        -DBUILD_ONLY=s3 \
        -DENABLE_UNITY_BUILD=ON \
-       -DENABLE_TESTING=OFF \
-       -DCMAKE_BUILD_TYPE=$CFG \
-       -DSIMPLE_INSTALL=ON"
-cmake -GNinja $FLAGS ..
-ninja all
-ninja install
+       -DCMAKE_BUILD_TYPE=Release \
+       -DSIMPLE_INSTALL=ON \
+       -DENABLE_TESTING=OFF
+
+cmake $FLAGS ..
+cmake --build . --config Release
+sudo cmake --install . --config Release
 cd ..
 popd
 ````

docs/internal.md

@@ -647,6 +647,44 @@ done:
 #endif /*HAVE_ZSTD*/
 ````
 
+# 5. Test Interference {#intern_isolation}
+
+At some point, Unidata switched from running tests serially to running tests in parallel.
+It soon became apparent that there were resources shared between tests and that parallel
+execution sometimes caused interference between tests.
+
+In order to fix the inter-test interference, several approaches were used.
+1. Renaming resources (primarily files) so that tests would create difference test files.
+2. Telling the test system that there were explicit dependencies between tests so that they would not be run in parallel.
+3.  Isolating test resources by creating independent directories for each test.
+
+## Test Isolation
+The isolation mechanism is currently used mostly in nczarr_tests.
+It requires that tests are all executed inside a shell script.
+When the script starts, it invokes a shell function called "isolate".
+Suppose, for example, that the shell script is called "run_XXXX.sh".
+The isolate function creates a directory with the general name "testdir_XXXX_<random>".
+The "<random>" part is a randomly generated 32-bit positive integer.
+The range of the random number needs to be big enough to ensure that their is a low probability the same number will occur for some other test.
+So the isolation directory is created and then the test is executed inside that directory.
+The cleanup requires that this test directory be deleted in order to release all created resources.
+
+## Cloud Test Isolation
+When testing against the cloud (currently Amazon S3), the interference problem is intensified.
+This is because the current cloud testing uses a single S3 bucket, which means that not only is there inter-test interference, but there is also potential interference across builds.
+This means, for example, that testing by github actions could interfere with local testing by individual users.
+This problem is not completely solvable because it requires that certain things be done at the beginning of all tests and at the end of all tests.
+This is possible with cmake, but not (as yet) possible with automake.
+
+In any case, there is a shell function called s3isolate in nczarr_test/test_nczarr.sh that operates on cloud resources in a way that is similar to the isolate function.
+The s3isolate does two things:
+1. It invokes isolate to ensure local isolation.
+2. It creates a path prefix relative to the Unidata S3 bucket that has the same name as the test directory created by isolate.
+
+The test then ensures that any cloud resources are created as extensions of the path prefix.
+Cleanup is also somewhat more complex and requires removing all the test directories and cloud paths created by any test.
+
+
 # Point of Contact {#intern_poc}
 
 *Author*: Dennis Heimbigner<br>

docs/nczarr.md

@@ -487,9 +487,9 @@ A separate tabulation of S3 support is in the document cloud.md.
 <tr><td>Linux            <td> Automake     <td> yes
 <tr><td>Linux            <td> CMake        <td> yes
 <tr><td>Cygwin           <td> Automake     <td> yes
-<tr><td>Cygwin           <td> CMake        <td> unknown
-<tr><td>OSX              <td> Automake     <td> unknown
-<tr><td>OSX              <td> CMake        <td> unknown
+<tr><td>Cygwin           <td> CMake        <td> yes
+<tr><td>OSX              <td> Automake     <td> yes
+<tr><td>OSX              <td> CMake        <td> yes
 <tr><td>Visual Studio    <td> CMake        <td> yes
 </table>
 

include/nclog.h

@@ -49,7 +49,8 @@ EXTERNL int ncbreakpoint(int err);
 
 /* Debug support */
 #if defined(NCCATCH)
-#define NCTHROW(e) ((e) == NC_NOERR ? (e) : ncthrow(e,__FILE__,__LINE__))
+/* Warning: do not evaluate e more than once */
+#define NCTHROW(e) ncthrow(e,__FILE__,__LINE__)
 #else
 #define NCTHROW(e) (e)
 #endif

include/netcdf_json.h

@@ -173,8 +173,9 @@ and do the command:
 
 #undef NCJDEBUG
 #ifdef NCJDEBUG
+/* Warning: do not evaluate err more than once */
+#define NCJTHROW(err) ncjbreakpoint(err)
 static int ncjbreakpoint(int err) {return err;}
-#define NCJTHROW(err) ((err)==NCJ_ERR?ncjbreakpoint(err):(err))
 #else
 #define NCJTHROW(err) (err)
 #endif

libdap2/dapdebug.h

@@ -55,6 +55,7 @@ extern int dappanic(const char* fmt, ...);
 
 #ifdef CATCHERROR
 /* Place breakpoint on dapbreakpoint to catch errors close to where they occur*/
+/* Warning: do not evaluate more than once */
 #define THROW(e) dapthrow(e,__LINE__,__FILE__)
 #define THROWCHK(e) (void)dapthrow(e,__LINE__,__FILE__)
 

libdap4/d4debug.h

@@ -42,6 +42,7 @@ extern int d4panic(const char* fmt, ...);
 
 #ifdef D4CATCH
 /* Place breakpoint on dapbreakpoint to catch errors close to where they occur*/
+/* WARNING: do not evaluate e more than once */
 #define THROW(e) d4throw(e)
 #define THROWCHK(e) (void)d4throw(e)
 extern int d4breakpoint(int err);

libdispatch/ncexhash.c

@@ -20,6 +20,7 @@ See LICENSE.txt for license information.
 #define INLINED
 
 #ifdef CATCH
+/* Warning: do not evaluate x more than once */
 #define THROW(x) throw(x)
 static void breakpoint(void) {}
 static int ignore[] = {NC_ENOTFOUND, 0};

libdispatch/ncjson.c

@@ -27,8 +27,9 @@ and do the command:
 
 #undef NCJDEBUG
 #ifdef NCJDEBUG
+/* Warning: do not evaluate err more than once */
+#define NCJTHROW(err) ncjbreakpoint(err)
 static int ncjbreakpoint(int err) {return err;}
-#define NCJTHROW(err) ((err)==NCJ_ERR?ncjbreakpoint(err):(err))
 #else
 #define NCJTHROW(err) (err)
 #endif

libdispatch/ncs3sdk_h5.c

@@ -427,7 +427,7 @@ EXTERNL int
 NC_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp)
 {
     NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0);
-    return getkeys(s3client0, bucket, prefixkey0, "/", nkeysp, keysp, errmsgp);
+    return NCUNTRACE(getkeys(s3client0, bucket, prefixkey0, "/", nkeysp, keysp, errmsgp));
 }
 
 /*
@@ -439,7 +439,7 @@ EXTERNL int
 NC_s3sdksearch(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp)
 {
     NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0);
-    return getkeys(s3client0, bucket, prefixkey0, NULL, nkeysp, keysp, errmsgp);
+    return NCUNTRACE(getkeys(s3client0, bucket, prefixkey0, NULL, nkeysp, keysp, errmsgp));
 }
 
 EXTERNL int
@@ -561,7 +561,8 @@ makes3prefix(const char* prefix, char** prefixdirp)
     return NC_NOERR;
 }
 
-/* Copy keys1 concat keys2 into merge; note that merge list may not be empty. */
+/* Move keys1 concat keys2 into merge; note that merge list may not be empty. */
+/* Will leave keys1 and keys2 empty */
 static int
 mergekeysets(NClist* keys1, NClist* keys2, NClist* merge)
 {

libdispatch/ncxcache.c

@@ -21,6 +21,7 @@
 #define SMALLTABLE
 
 #ifdef CATCH
+/* Warning: do not evalue x more than once */
 #define THROW(x) throw(x)
 static void breakpoint(void) {}
 static int ignore[] = {0};

libhdf5/hdf5debug.h

@@ -12,6 +12,7 @@
 
 #ifdef H5CATCH
 /* Place breakpoint to catch errors close to where they occur*/
+/* Warning: do not evaluate e more than once */
 #define THROW(e) nch5throw(e,__LINE__)
 #define THROWCHK(e) (void)nch5throw(e)
 extern int nch5breakpoint(int err);

libnczarr/zdebug.h

@@ -5,8 +5,8 @@
 #ifndef ZDEBUG_H
 #define ZDEBUG_H
 
-#undef ZCATCH /* Warning: significant performance impact */
-#undef ZTRACING /* Warning: significant performance impact */
+#define ZCATCH /* Warning: significant performance impact */
+#define ZTRACING /* Warning: significant performance impact */
 
 #undef ZDEBUG /* general debug */
 #undef ZDEBUG1 /* detailed debug */
@@ -22,6 +22,7 @@
 
 #ifdef ZCATCH
 /* Place breakpoint on zbreakpoint to catch errors close to where they occur*/
+/* WARNING: Do not evaluate e more than once */
 #define THROW(e) zthrow((e),__FILE__, __func__, __LINE__)
 #define REPORT(e,msg) zreport((e),(msg),__FILE__, __func__, __LINE__)
 #define ZCHECK(e) if((e)) {THROW(stat); goto done;} else {}