From cead13ef8ca881c7942e0b46f187d596af125e39 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Wed, 6 Sep 2023 17:54:08 -0600 Subject: [PATCH] ckp --- CMakeLists.txt | 33 +-- CTestCustom.cmake | 14 -- Makefile.am | 12 +- config.h.cmake.in | 3 + configure.ac | 8 +- docs/cloud.md | 27 ++- docs/internal.md | 38 +++ docs/nczarr.md | 6 +- include/nclog.h | 3 +- include/netcdf_json.h | 3 +- libdap2/dapdebug.h | 1 + libdap4/d4debug.h | 1 + libdispatch/ncexhash.c | 1 + libdispatch/ncjson.c | 3 +- libdispatch/ncs3sdk_h5.c | 7 +- libdispatch/ncxcache.c | 1 + libhdf5/hdf5debug.h | 1 + libnczarr/zdebug.h | 5 +- libnczarr/zfilter.c | 6 +- libnczarr/zmap_s3sdk.c | 12 +- libnczarr/zsync.c | 66 ++---- libnczarr/zutil.c | 3 + libnczarr/zvar.c | 11 +- nc_test4/Makefile.am | 6 +- nc_test4/test_filter_vlen.c | 268 +++++++++++----------- nc_test4/tst_filter_misc.sh | 12 +- nc_test4/tst_h5_endians.c | 4 +- nc_test4/tst_put_vars_two_unlim_dim.c | 1 + ncdap_test/test_manyurls.c | 6 + ncdump/tst_mud.sh | 5 +- ncdump/tst_nccopy5.sh | 8 +- nczarr_test/CMakeLists.txt | 38 ++- nczarr_test/Makefile.am | 45 +++- nczarr_test/ref_zarr_test_data.cdl.gz | Bin 2699 -> 2699 bytes nczarr_test/run_cachetest.sh | 2 - nczarr_test/run_chunkcases.sh | 5 - nczarr_test/run_external.sh | 2 - nczarr_test/run_fillonlyz.sh | 2 - nczarr_test/run_filter.sh | 30 ++- nczarr_test/run_interop.sh | 24 +- nczarr_test/run_jsonconvention.sh | 2 - nczarr_test/run_misc.sh | 2 - nczarr_test/run_nccopyz.sh | 2 - nczarr_test/run_ncgen4.sh | 2 - nczarr_test/run_nczarr_fill.sh | 2 - nczarr_test/run_nczfilter.sh | 2 - nczarr_test/run_newformat.sh | 2 - nczarr_test/run_notzarr.sh | 2 - nczarr_test/run_nulls.sh | 2 - nczarr_test/run_perf_chunks1.sh | 2 - nczarr_test/run_quantize.sh | 1 - nczarr_test/run_scalar.sh | 2 - nczarr_test/run_strings.sh | 3 +- nczarr_test/run_unlim_io.sh | 58 ++--- nczarr_test/test_endians.c | 8 +- nczarr_test/test_fillonlyz.c | 2 +- nczarr_test/test_nczarr.sh | 13 +- nczarr_test/test_put_vars_two_unlim_dim.c | 1 + nczarr_test/test_utils.c | 8 + nczarr_test/test_utils.h | 14 +- nczarr_test/testfilter_misc.c | 149 ++++++------ nczarr_test/zmapio.c | 6 +- oc2/ocdebug.h | 1 + plugins/H5Zmisc.c | 191 ++++++++------- plugins/Makefile.am | 2 +- plugins/h5misc.h | 49 +++- test_common.in | 21 +- unit_test/run_s3sdk.sh | 2 +- unit_test/test_s3sdk.c | 7 +- 69 files changed, 691 insertions(+), 590 deletions(-) delete mode 100644 CTestCustom.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index e8d4b30c1f..ddee397646 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -210,9 +210,6 @@ SET(EXTRA_DEPS "") ENABLE_TESTING() INCLUDE(CTest) -# Copy the CTest customization file into binary directory, as required. -FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/CTestCustom.cmake DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) - # Set Memory test program for non-MSVC based builds. # Assume valgrind for now. IF((NOT MSVC) AND (NOT MINGW) AND (NOT ISCYGWIN)) @@ -1296,7 +1293,7 @@ IF(NOT ENABLE_REMOTE_FUNCTIONALITY) ENDIF() # Control S3 Testing: Multi-valued option -SET(WITH_S3_TESTING OFF CACHE STRING "Control S3 Testing: ON (i.e. all) OFF (i.e. none) PUBLIC") +SET(WITH_ING OFF CACHE STRING "Control S3 Testing: ON (i.e. all) OFF (i.e. none) PUBLIC") SET_PROPERTY(CACHE WITH_S3_TESTING PROPERTY STRINGS ON OFF PUBLIC) # IF(WITH_S3_TESTING STREQUAL "") SET(WITH_S3_TESTING OFF CACHE STRING "") # Default @@ -1340,11 +1337,6 @@ ELSE() SET(ENABLE_S3_AWS OFF CACHE BOOL "S3 AWS" FORCE) ENDIF() -# Unless/until we get aws-sdk-cpp working for Windows, force use of internal -IF(ENABLE_S3 AND MSVC) - SET(ENABLE_S3_INTERNAL ON CACHE BOOL "S3 Intern" FORCE) -ENDIF() - IF(ENABLE_S3) IF(NOT ENABLE_S3_AWS AND NOT ENABLE_S3_INTERNAL) message(FATAL_ERROR "S3 support library not found; please specify option -DENABLE_S3=NO") @@ -1363,6 +1355,10 @@ IF(WITH_S3_TESTING STREQUAL "PUBLIC" OR WITH_S3_TESTING) ENDIF() ENDIF() +# The working S3 path tree within the Unidata bucket. +# WARNING: this must match the value in configure.ac +SET(S3TESTSUBTREE netcdf-x CACHE STRING "Working S3 path.") + OPTION(ENABLE_LIBXML2 "Link against libxml2 if it is available, use the packaged tinyxml2 parser otherwise." ON) SET(XMLPARSER "tinyxml2 (bundled)") @@ -2600,15 +2596,19 @@ SET(NC_WHICH_S3_SDK "none") endif() if(WITH_S3_TESTING STREQUAL PUBLIC) -SET(DO_S3_TESTING "public") +SET(ENABLE_S3_TESTING "public") elseif(WITH_S3_TESTING) -SET(DO_S3_TESTING "yes") +SET(ENABLE_S3_TESTING "yes") +SET(ENABLE_S3_TESTALL "yes") elseif(NOT WITH_S3_TESTING) -SET(DO_S3_TESTING "no") +SET(ENABLE_S3_TESTING "no") else() -SET(DO_S3_TESTING "no") +SET(ENABLE_S3_TESTING "no") endif() +# Copy the CTest customization file into binary directory, as required. +CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/CTestCustom.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/CTestCustom.cmake") + # Generate file from template. CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in" "${CMAKE_CURRENT_BINARY_DIR}/libnetcdf.settings" @@ -2679,6 +2679,13 @@ IF(ENABLE_TESTS) # Build dap4_test/pingurl4.c ##### configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ncdap_test/pingurl.c ${CMAKE_CURRENT_BINARY_DIR}/dap4_test/pingurl4.c @ONLY NEWLINE_STYLE LF) + + ##### + # Build CTestCustom.cmake to cleanup S3 after tests are done. + ##### + configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CTestCustom.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/CTestCustom.cmake NEWLINE_STYLE LF) + + ENDIF() if(DEFINED ENV{LIB_FUZZING_ENGINE}) diff --git a/CTestCustom.cmake b/CTestCustom.cmake deleted file mode 100644 index 7483a775f0..0000000000 --- a/CTestCustom.cmake +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, -# 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, -# 2015, 2016, 2017, 2018 -# University Corporation for Atmospheric Research/Unidata. - -# See netcdf-c/COPYRIGHT file for more info. -## This file is used to customize CTest. -## See: -## -## http://public.kitware.com/Wiki/CTest:Using_CTEST_and_CDASH_without_CMAKE -## -## for more information. - -set(CTEST_CUSTOM_MAXIMUM_NUMBER_OF_WARNINGS "500") diff --git a/Makefile.am b/Makefile.am index b72b9eee1b..e65a863747 100644 --- a/Makefile.am +++ b/Makefile.am @@ -31,7 +31,7 @@ AM_DISTCHECK_CONFIGURE_FLAGS = --without-plugin-dir EXTRA_DIST = README.md COPYRIGHT INSTALL.md test_prog.c lib_flags.am \ cmake CMakeLists.txt COMPILE.cmake.txt config.h.cmake.in \ cmake_uninstall.cmake.in FixBundle.cmake.in nc-config.cmake.in \ -RELEASE_NOTES.md CTestCustom.cmake CTestConfig.cmake.in \ +RELEASE_NOTES.md CTestConfig.cmake.in CTestCustom.cmake.in \ libnetcdf.settings.in netCDFConfig.cmake.in CMakeInstallation.cmake \ test-driver-verbose test_common.in fuzz @@ -213,3 +213,13 @@ all-local: liblib/libnetcdf.la echo ${PACKAGE_VERSION} > VERSION # Remove the VERSION file CLEANFILES = VERSION + +if ENABLE_S3_TESTALL +# Cleanup S3 resources +distcleancheck: s3clean + +.PHONY: s3clean +# Provide a specific s3 cleanup action +s3clean: + echo ">>>> ${abs_srcdir}/run_s3_cleanup.sh ${abs_builddir} ${S3TESTSUBTREE}" +endif diff --git a/config.h.cmake.in b/config.h.cmake.in index cfdadfbe71..7160293a6b 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -163,6 +163,9 @@ are set when opening a binary file on Windows. */ /* if true, enable S3 testing*/ #cmakedefine WITH_S3_TESTING "PUBLIC" +/* S3 Working subtree */ +#define S3TESTSUBTREE "${S3TESTSUBTREE}" + /* if true, run extra tests which may not work yet */ #cmakedefine EXTRA_TESTS 1 diff --git a/configure.ac b/configure.ac index 7a67b53d6c..ebffac8843 100644 --- a/configure.ac +++ b/configure.ac @@ -974,6 +974,7 @@ AC_DEFINE_UNQUOTED([WITH_S3_TESTING], [$with_s3_testing], [control S3 testing.]) if test "x$with_s3_testing" = xyes ; then AC_MSG_WARN([*** DO NOT SPECIFY WITH_S3_TESTING=YES UNLESS YOU HAVE ACCESS TO THE UNIDATA S3 BUCKET! ***]) + AC_DEFINE([ENABLE_S3_TESTALL], [yes], [control S3 testing.]) fi # Check whether we want to enable strict null byte header padding. @@ -2033,7 +2034,7 @@ AC_SUBST(HAS_S3_AWS,[$enable_s3_aws]) AC_SUBST(HAS_S3_INTERNAL,[$enable_s3_internal]) AC_SUBST(HAS_HDF5_ROS3,[$has_hdf5_ros3]) AC_SUBST(HAS_NCZARR,[$enable_nczarr]) -AC_SUBST(DO_S3_TESTING,[$with_s3_testing]) +AC_SUBST(ENABLE_S3_TESTING,[$with_s3_testing]) AC_SUBST(HAS_NCZARR_ZIP,[$enable_nczarr_zip]) AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip]) AC_SUBST(HAS_QUANTIZE,[$enable_quantize]) @@ -2056,6 +2057,11 @@ if test "x$enable_s3_aws" = xno && test "x$enable_s3_internal" = xno; then AC_SUBST(WHICH_S3_SDK,[none]) fi +# The working S3 path tree within the Unidata bucket. +# WARNING: this must match the value in CMakeLists.txt +AC_DEFINE([S3TESTSUBTREE], ["netcdf-x"], [S3 working path]) +AC_SUBST([S3TESTSUBTREE],[netcdf-x]) + # Always available std_filters="bz2" if test "x$have_deflate" = xyes ; then diff --git a/docs/cloud.md b/docs/cloud.md index 69e8f50a29..cd4e23f159 100644 --- a/docs/cloud.md +++ b/docs/cloud.md @@ -85,12 +85,12 @@ Currently the following build cases are known to work. OSX Automake nch5s3comms unknown OSX CMake aws-s3-sdk unknown OSX CMake nch5s3comms unknown -Visual Studio CMake aws-s3-sdk no (tests-fail) +Visual Studio CMake aws-s3-sdk no (tests fail) Visual Studio CMake nch5s3comms yes -Cygwin Automake aws-s3-sdk unknown +Cygwin Automake aws-s3-sdk no (tests fail) Cygwin Automake nch5s3comms yes -Cygwin CMake aws-s3-sdk unknown -Cygwin CMake nch5s3comms unknown +Cygwin CMake aws-s3-sdk no +Cygwin CMake nch5s3comms yes Mingw Automake aws-s3-sdk unknown Mingw Automake nch5s3comms unknown Mingw CMake aws-s3-sdk unknown @@ -163,7 +163,7 @@ This library, [aws-sdk-cpp library](https://github.com/aws/aws-sdk-cpp.git), has a number of properties of interest: * It is written in C++ * It is available on [GitHub](https://github.com/aws/aws-sdk-cpp.git), -* It uses CMake + ninja as its primary build system. +* It uses CMake as its primary build system. ### *\*nix\** Build @@ -179,18 +179,17 @@ pushd aws-sdk-cpp mkdir build cd build PREFIX=/usr/local -FLAGS="-DCMAKE_INSTALL_PREFIX=${PREFIX} \ - -DCMAKE_INSTALL_LIBDIR=lib \ - -DCMAKE_MODULE_PATH=${PREFIX}/lib/cmake \ +FLAGS="-DCMAKE_INSTALL_PREFIX=/usr/local \ -DCMAKE_POLICY_DEFAULT_CMP0075=NEW \ -DBUILD_ONLY=s3 \ -DENABLE_UNITY_BUILD=ON \ - -DENABLE_TESTING=OFF \ - -DCMAKE_BUILD_TYPE=$CFG \ - -DSIMPLE_INSTALL=ON" -cmake -GNinja $FLAGS .. -ninja all -ninja install + -DCMAKE_BUILD_TYPE=Release \ + -DSIMPLE_INSTALL=ON \ + -DENABLE_TESTING=OFF + +cmake $FLAGS .. +cmake --build . --config Release +sudo cmake --install . --config Release cd .. popd ```` diff --git a/docs/internal.md b/docs/internal.md index fc6f06618e..55a0a8ccf0 100644 --- a/docs/internal.md +++ b/docs/internal.md @@ -647,6 +647,44 @@ done: #endif /*HAVE_ZSTD*/ ```` +# 5. Test Interference {#intern_isolation} + +At some point, Unidata switched from running tests serially to running tests in parallel. +It soon became apparent that there were resources shared between tests and that parallel +execution sometimes caused interference between tests. + +In order to fix the inter-test interference, several approaches were used. +1. Renaming resources (primarily files) so that tests would create difference test files. +2. Telling the test system that there were explicit dependencies between tests so that they would not be run in parallel. +3. Isolating test resources by creating independent directories for each test. + +## Test Isolation +The isolation mechanism is currently used mostly in nczarr_tests. +It requires that tests are all executed inside a shell script. +When the script starts, it invokes a shell function called "isolate". +Suppose, for example, that the shell script is called "run_XXXX.sh". +The isolate function creates a directory with the general name "testdir_XXXX_". +The "" part is a randomly generated 32-bit positive integer. +The range of the random number needs to be big enough to ensure that their is a low probability the same number will occur for some other test. +So the isolation directory is created and then the test is executed inside that directory. +The cleanup requires that this test directory be deleted in order to release all created resources. + +## Cloud Test Isolation +When testing against the cloud (currently Amazon S3), the interference problem is intensified. +This is because the current cloud testing uses a single S3 bucket, which means that not only is there inter-test interference, but there is also potential interference across builds. +This means, for example, that testing by github actions could interfere with local testing by individual users. +This problem is not completely solvable because it requires that certain things be done at the beginning of all tests and at the end of all tests. +This is possible with cmake, but not (as yet) possible with automake. + +In any case, there is a shell function called s3isolate in nczarr_test/test_nczarr.sh that operates on cloud resources in a way that is similar to the isolate function. +The s3isolate does two things: +1. It invokes isolate to ensure local isolation. +2. It creates a path prefix relative to the Unidata S3 bucket that has the same name as the test directory created by isolate. + +The test then ensures that any cloud resources are created as extensions of the path prefix. +Cleanup is also somewhat more complex and requires removing all the test directories and cloud paths created by any test. + + # Point of Contact {#intern_poc} *Author*: Dennis Heimbigner
diff --git a/docs/nczarr.md b/docs/nczarr.md index 0308be9bd4..4c2a5595f1 100644 --- a/docs/nczarr.md +++ b/docs/nczarr.md @@ -487,9 +487,9 @@ A separate tabulation of S3 support is in the document cloud.md. Linux Automake yes Linux CMake yes Cygwin Automake yes -Cygwin CMake unknown -OSX Automake unknown -OSX CMake unknown +Cygwin CMake yes +OSX Automake yes +OSX CMake yes Visual Studio CMake yes diff --git a/include/nclog.h b/include/nclog.h index e7146c162a..20267c3b92 100644 --- a/include/nclog.h +++ b/include/nclog.h @@ -49,7 +49,8 @@ EXTERNL int ncbreakpoint(int err); /* Debug support */ #if defined(NCCATCH) -#define NCTHROW(e) ((e) == NC_NOERR ? (e) : ncthrow(e,__FILE__,__LINE__)) +/* Warning: do not evaluate e more than once */ +#define NCTHROW(e) ncthrow(e,__FILE__,__LINE__) #else #define NCTHROW(e) (e) #endif diff --git a/include/netcdf_json.h b/include/netcdf_json.h index e063319d5a..46657548a6 100644 --- a/include/netcdf_json.h +++ b/include/netcdf_json.h @@ -173,8 +173,9 @@ and do the command: #undef NCJDEBUG #ifdef NCJDEBUG +/* Warning: do not evaluate err more than once */ +#define NCJTHROW(err) ncjbreakpoint(err) static int ncjbreakpoint(int err) {return err;} -#define NCJTHROW(err) ((err)==NCJ_ERR?ncjbreakpoint(err):(err)) #else #define NCJTHROW(err) (err) #endif diff --git a/libdap2/dapdebug.h b/libdap2/dapdebug.h index ac8a3ea805..830b9f05d3 100644 --- a/libdap2/dapdebug.h +++ b/libdap2/dapdebug.h @@ -55,6 +55,7 @@ extern int dappanic(const char* fmt, ...); #ifdef CATCHERROR /* Place breakpoint on dapbreakpoint to catch errors close to where they occur*/ +/* Warning: do not evaluate more than once */ #define THROW(e) dapthrow(e,__LINE__,__FILE__) #define THROWCHK(e) (void)dapthrow(e,__LINE__,__FILE__) diff --git a/libdap4/d4debug.h b/libdap4/d4debug.h index 5ff9db4f0c..adf655a2c2 100644 --- a/libdap4/d4debug.h +++ b/libdap4/d4debug.h @@ -42,6 +42,7 @@ extern int d4panic(const char* fmt, ...); #ifdef D4CATCH /* Place breakpoint on dapbreakpoint to catch errors close to where they occur*/ +/* WARNING: do not evaluate e more than once */ #define THROW(e) d4throw(e) #define THROWCHK(e) (void)d4throw(e) extern int d4breakpoint(int err); diff --git a/libdispatch/ncexhash.c b/libdispatch/ncexhash.c index d1677805e7..ea3957a054 100644 --- a/libdispatch/ncexhash.c +++ b/libdispatch/ncexhash.c @@ -20,6 +20,7 @@ See LICENSE.txt for license information. #define INLINED #ifdef CATCH +/* Warning: do not evaluate x more than once */ #define THROW(x) throw(x) static void breakpoint(void) {} static int ignore[] = {NC_ENOTFOUND, 0}; diff --git a/libdispatch/ncjson.c b/libdispatch/ncjson.c index 7730f42f72..6b43714f47 100644 --- a/libdispatch/ncjson.c +++ b/libdispatch/ncjson.c @@ -27,8 +27,9 @@ and do the command: #undef NCJDEBUG #ifdef NCJDEBUG +/* Warning: do not evaluate err more than once */ +#define NCJTHROW(err) ncjbreakpoint(err) static int ncjbreakpoint(int err) {return err;} -#define NCJTHROW(err) ((err)==NCJ_ERR?ncjbreakpoint(err):(err)) #else #define NCJTHROW(err) (err) #endif diff --git a/libdispatch/ncs3sdk_h5.c b/libdispatch/ncs3sdk_h5.c index 15e96cefdc..e9da587f11 100644 --- a/libdispatch/ncs3sdk_h5.c +++ b/libdispatch/ncs3sdk_h5.c @@ -427,7 +427,7 @@ EXTERNL int NC_s3sdkgetkeys(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp) { NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0); - return getkeys(s3client0, bucket, prefixkey0, "/", nkeysp, keysp, errmsgp); + return NCUNTRACE(getkeys(s3client0, bucket, prefixkey0, "/", nkeysp, keysp, errmsgp)); } /* @@ -439,7 +439,7 @@ EXTERNL int NC_s3sdksearch(void* s3client0, const char* bucket, const char* prefixkey0, size_t* nkeysp, char*** keysp, char** errmsgp) { NCTRACE(11,"bucket=%s prefixkey0=%s",bucket,prefixkey0); - return getkeys(s3client0, bucket, prefixkey0, NULL, nkeysp, keysp, errmsgp); + return NCUNTRACE(getkeys(s3client0, bucket, prefixkey0, NULL, nkeysp, keysp, errmsgp)); } EXTERNL int @@ -561,7 +561,8 @@ makes3prefix(const char* prefix, char** prefixdirp) return NC_NOERR; } -/* Copy keys1 concat keys2 into merge; note that merge list may not be empty. */ +/* Move keys1 concat keys2 into merge; note that merge list may not be empty. */ +/* Will leave keys1 and keys2 empty */ static int mergekeysets(NClist* keys1, NClist* keys2, NClist* merge) { diff --git a/libdispatch/ncxcache.c b/libdispatch/ncxcache.c index ee1602d6f6..40b82f81c4 100644 --- a/libdispatch/ncxcache.c +++ b/libdispatch/ncxcache.c @@ -21,6 +21,7 @@ #define SMALLTABLE #ifdef CATCH +/* Warning: do not evalue x more than once */ #define THROW(x) throw(x) static void breakpoint(void) {} static int ignore[] = {0}; diff --git a/libhdf5/hdf5debug.h b/libhdf5/hdf5debug.h index 996e787107..34f6c218cd 100644 --- a/libhdf5/hdf5debug.h +++ b/libhdf5/hdf5debug.h @@ -12,6 +12,7 @@ #ifdef H5CATCH /* Place breakpoint to catch errors close to where they occur*/ +/* Warning: do not evaluate e more than once */ #define THROW(e) nch5throw(e,__LINE__) #define THROWCHK(e) (void)nch5throw(e) extern int nch5breakpoint(int err); diff --git a/libnczarr/zdebug.h b/libnczarr/zdebug.h index 71a9e43537..e182cec870 100644 --- a/libnczarr/zdebug.h +++ b/libnczarr/zdebug.h @@ -5,8 +5,8 @@ #ifndef ZDEBUG_H #define ZDEBUG_H -#undef ZCATCH /* Warning: significant performance impact */ -#undef ZTRACING /* Warning: significant performance impact */ +#define ZCATCH /* Warning: significant performance impact */ +#define ZTRACING /* Warning: significant performance impact */ #undef ZDEBUG /* general debug */ #undef ZDEBUG1 /* detailed debug */ @@ -22,6 +22,7 @@ #ifdef ZCATCH /* Place breakpoint on zbreakpoint to catch errors close to where they occur*/ +/* WARNING: Do not evaluate e more than once */ #define THROW(e) zthrow((e),__FILE__, __func__, __LINE__) #define REPORT(e,msg) zreport((e),(msg),__FILE__, __func__, __LINE__) #define ZCHECK(e) if((e)) {THROW(stat); goto done;} else {} diff --git a/libnczarr/zfilter.c b/libnczarr/zfilter.c index 9d2bc68f23..6f6a013a58 100644 --- a/libnczarr/zfilter.c +++ b/libnczarr/zfilter.c @@ -846,7 +846,7 @@ NCZ_applyfilterchain(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NClist* cha { int i, stat = NC_NOERR; void* lastbuffer = NULL; /* if not null, then last allocated buffer */ - + ZTRACE(6,"|chain|=%u inlen=%u indata=%p encode=%d", (unsigned)nclistlength(chain), (unsigned)inlen, indata, encode); /* Make sure all the filters are loaded && setup */ @@ -1155,7 +1155,7 @@ static int NCZ_load_all_plugins(void) { int i,j,ret = NC_NOERR; - const char* pluginroots = NULL; + char* pluginroots = NULL; struct stat buf; NClist* dirs = nclistnew(); char* defaultpluginpath = NULL; @@ -1185,7 +1185,7 @@ NCZ_load_all_plugins(void) pluginroots = getenv(PLUGIN_ENV); /* Usually HDF5_PLUGIN_PATH */ if(pluginroots != NULL && strlen(pluginroots) == 0) pluginroots = NULL; if(pluginroots == NULL) { - pluginroots = strdup(defaultpluginpath); + pluginroots = defaultpluginpath; } assert(pluginroots != NULL); ZTRACEMORE(6,"pluginroots=%s",(pluginroots?pluginroots:"null")); diff --git a/libnczarr/zmap_s3sdk.c b/libnczarr/zmap_s3sdk.c index 76faf56d51..0f456b6f85 100644 --- a/libnczarr/zmap_s3sdk.c +++ b/libnczarr/zmap_s3sdk.c @@ -152,18 +152,19 @@ zs3create(const char *path, int mode, size64_t flags, void* parameters, NCZMAP** if((stat = NC_s3sdkbucketcreate(z3map->s3client,z3map->s3.region,z3map->s3.bucket,&z3map->errmsg))) goto done; } - /* The root object should not exist */ + /* The root object may or may not already exist */ switch (stat = NC_s3sdkinfo(z3map->s3client,z3map->s3.bucket,z3map->s3.rootkey,NULL,&z3map->errmsg)) { case NC_EEMPTY: /* no such object */ stat = NC_NOERR; /* which is what we want */ errclear(z3map); break; - case NC_NOERR: stat = NC_EOBJECT; goto done; /* already exists */ - default: reporterr(z3map); goto done; - } - if(!stat) { + case NC_NOERR: + stat = NC_EOBJECT; + errclear(z3map); /* Delete objects inside root object tree */ s3clear(z3map,z3map->s3.rootkey); + goto done; /* already exists */ + default: reporterr(z3map); goto done; } } @@ -352,6 +353,7 @@ zs3write(NCZMAP* map, const char* key, size64_t start, size64_t count, const voi case NC_EEMPTY: memsize = endwrite; isempty = 1; + stat = NC_NOERR; /* reset */ break; default: reporterr(z3map); goto done; } diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c index d388b5fcb5..f973492e21 100644 --- a/libnczarr/zsync.c +++ b/libnczarr/zsync.c @@ -356,10 +356,8 @@ ncz_sync_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose) } /* chunks key */ - /* It is not clear if the zarr format supports the concept - of contiguous, so we will simulate by: - 1. setting a flag in _nczvar (below) - 2. making the chunk sizes all be same as the max dim size (here) + /* The zarr format does not support the concept + of contiguous (or compact), so it will never appear in the read case. */ /* list of chunk sizes */ if((stat = NCJaddstring(jvar,NCJ_STRING,"chunks"))) goto done; @@ -368,7 +366,7 @@ ncz_sync_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose) if(zvar->scalar) { NCJaddstring(jtmp,NCJ_INT,"1"); /* one chunk of size 1 */ } else for(i=0;indims;i++) { - size64_t len = (var->storage == NC_CONTIGUOUS ? shape[i] : var->chunksizes[i]); + size64_t len = var->chunksizes[i]; snprintf(number,sizeof(number),"%lld",len); NCJaddstring(jtmp,NCJ_INT,number); } @@ -483,16 +481,14 @@ ncz_sync_var_meta(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int isclose) jdimrefs = NULL; /* Avoid memory problems */ /* Add the _Storage flag */ - /* Record if this is a scalar; use the storage field */ + /* Record if this is a scalar */ if(var->ndims == 0) { - if((stat = NCJnewstring(NCJ_STRING,"scalar",&jtmp)))goto done; - } else if(var->storage == NC_CONTIGUOUS) { - if((stat = NCJnewstring(NCJ_STRING,"contiguous",&jtmp)))goto done; - } else if(var->storage == NC_COMPACT) { - if((stat = NCJnewstring(NCJ_STRING,"compact",&jtmp)))goto done; - } else {/* chunked */ - if((stat = NCJnewstring(NCJ_STRING,"chunked",&jtmp)))goto done; + if((stat = NCJnewstring(NCJ_INT,"1",&jtmp)))goto done; + if((stat = NCJinsert(jncvar,"scalar",jtmp))) goto done; + jtmp = NULL; } + /* everything looks like it is chunked */ + if((stat = NCJnewstring(NCJ_STRING,"chunked",&jtmp)))goto done; if((stat = NCJinsert(jncvar,"storage",jtmp))) goto done; jtmp = NULL; @@ -1464,8 +1460,6 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) NCjson* jfilter = NULL; int chainindex; #endif - int varsized; - int suppressvar = 0; /* 1 => make this variable invisible */ ZTRACE(3,"file=%s grp=%s |varnames|=%u",file->controller->path,grp->hdr.name,nclistlength(varnames)); @@ -1548,9 +1542,6 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) } } - /* See if this variable is variable sized */ - varsized = NC4_var_varsized(var); - if(!purezarr) { /* Extract the _NCZARR_ARRAY values */ /* Do this first so we know about storage esp. scalar */ @@ -1570,20 +1561,18 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) } if(jncvar == NULL) {stat = NC_ENCZARR; goto done;} assert((NCJsort(jncvar) == NCJ_DICT)); + /* Extract scalar flag */ + if((stat = NCJdictget(jncvar,"scalar",&jvalue))) + goto done; + if(jvalue != NULL) { + var->storage = NC_CHUNKED; + zvar->scalar = 1; + } /* Extract storage flag */ if((stat = NCJdictget(jncvar,"storage",&jvalue))) goto done; if(jvalue != NULL) { - if(strcmp(NCJstring(jvalue),"chunked") == 0) { - var->storage = NC_CHUNKED; - } else if(strcmp(NCJstring(jvalue),"compact") == 0) { - var->storage = NC_COMPACT; - } else if(strcmp(NCJstring(jvalue),"scalar") == 0) { - var->storage = NC_CONTIGUOUS; - zvar->scalar = 1; - } else { /*storage = NC_CONTIGUOUS;*/ - var->storage = NC_CONTIGUOUS; - } + var->storage = NC_CHUNKED; } /* Extract dimrefs list */ switch ((stat = NCJdictget(jncvar,"dimrefs",&jdimrefs))) { @@ -1716,11 +1705,11 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) #ifdef ENABLE_NCZARR_FILTERS if(var->filters == NULL) var->filters = (void*)nclistnew(); if(zvar->incompletefilters == NULL) zvar->incompletefilters = (void*)nclistnew(); - { int k; chainindex = 0; /* track location of filter in the chain */ if((stat = NCZ_filter_initialize())) goto done; if((stat = NCJdictget(jvar,"filters",&jvalue))) goto done; if(jvalue != NULL && NCJsort(jvalue) != NCJ_NULL) { + int k; if(NCJsort(jvalue) != NCJ_ARRAY) {stat = NC_EFILTER; goto done;} for(k=0;;k++) { jfilter = NULL; @@ -1730,15 +1719,14 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) if((stat = NCZ_filter_build(file,var,jfilter,chainindex++))) goto done; } } - } #endif } /* compressor key */ /* From V2 Spec: A JSON object identifying the primary compression codec and providing configuration parameters, or ``null`` if no compressor is to be used. */ - if(!varsized) { /* Only process if variable is fixed-size */ #ifdef ENABLE_NCZARR_FILTERS + { if(var->filters == NULL) var->filters = (void*)nclistnew(); if((stat = NCZ_filter_initialize())) goto done; if((stat = NCJdictget(jvar,"compressor",&jfilter))) goto done; @@ -1746,12 +1734,8 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;} if((stat = NCZ_filter_build(file,var,jfilter,chainindex++))) goto done; } -#endif } - /* Suppress variable if there are filters and var is not fixed-size */ - if(varsized && nclistlength((NClist*)var->filters) > 0) - suppressvar = 1; - +#endif if((stat = computedimrefs(file, var, purezarr, xarray, rank, dimnames, shapes, var->dim))) goto done; @@ -1762,16 +1746,10 @@ define_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) } #ifdef ENABLE_NCZARR_FILTERS - if(!suppressvar) { - /* At this point, we can finalize the filters */ - if((stat = NCZ_filter_setup(var))) goto done; - } + /* At this point, we can finalize the filters */ + if((stat = NCZ_filter_setup(var))) goto done; #endif - if(suppressvar) { - if((stat = NCZ_zclose_var1(var))) goto done; - } - /* Clean up from last cycle */ nclistfreeall(dimnames); dimnames = nclistnew(); nullfree(varpath); varpath = NULL; diff --git a/libnczarr/zutil.c b/libnczarr/zutil.c index ab5a0fa092..b8b7b6697f 100644 --- a/libnczarr/zutil.c +++ b/libnczarr/zutil.c @@ -280,6 +280,9 @@ fprintf(stderr,"uploadjson: %s\n",key); fflush(stderr); goto done; ZTRACEMORE(4,"\tjson=%s",content); +if(getenv("NCS3JSON") != NULL) +fprintf(stderr,">>>> uploadjson: %s: %s\n",key,content); + /* Write the metadata */ if((stat = nczmap_write(zmap, key, 0, strlen(content), content))) goto done; diff --git a/libnczarr/zvar.c b/libnczarr/zvar.c index 23703f406e..cb8bc762ae 100644 --- a/libnczarr/zvar.c +++ b/libnczarr/zvar.c @@ -522,6 +522,7 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, int d; int retval = NC_NOERR; int storage = NC_CHUNKED; + size_t contigchunksizes[NC_MAX_VAR_DIMS]; /* Fake chunksizes if storage is contiguous or compact */ LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid)); @@ -632,13 +633,15 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, if (nclistlength(((NClist*)var->filters)) > 0) {retval = NC_EINVAL; goto done;} #endif - for (d = 0; d < var->ndims; d++) + for (d = 0; d < var->ndims; d++) { if (var->dim[d]->unlimited) {retval = NC_EINVAL; goto done;} + contigchunksizes[d] = var->dim[d]->len; /* Fake a single big chunk */ + } + chunksizes = (const size_t*)contigchunksizes; storage = NC_CHUNKED; /*only chunked supported */ } - /* Handle chunked storage settings. */ if (storage == NC_CHUNKED && var->ndims == 0) { {retval = NC_EINVAL; goto done;} } else if (storage == NC_CHUNKED && var->ndims > 0) { @@ -659,10 +662,6 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, {retval = NC_EBADCHUNK; goto done;} } } - else if (storage == NC_CONTIGUOUS || storage == NC_COMPACT) - { - var->storage = NC_CHUNKED; - } /* Is this a variable with a chunksize greater than the current * cache size? */ diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index d3f41bf0a0..5fcb69cf12 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -86,11 +86,13 @@ check_PROGRAMS += tst_multifilter tst_filter_vlen TESTS += tst_filter.sh TESTS += tst_specific_filters.sh TESTS += tst_bloscfail.sh -TESTS += tst_filter_vlen.sh if ISMINGW XFAIL_TESTS += tst_filter.sh endif # ISMINGW +TESTS += tst_filter_vlen.sh +TESTS += tst_filter_misc.sh + if AX_MANUAL # This test is too dangerous to run in a parallel make environment. # It causes race conditions. So suppress and only test by hand. @@ -134,7 +136,7 @@ ref_any.cdl tst_specific_filters.sh tst_unknown.sh \ tst_virtual_datasets.c noop1.cdl unknown.cdl \ tst_broken_files.c ref_bloscx.cdl tst_bloscfail.sh \ tst_fixedstring.sh ref_fixedstring.h5 ref_fixedstring.cdl \ -tst_filterinstall.sh tst_filter_vlen.sh +tst_filterinstall.sh tst_filter_vlen.sh tst_filter_misc.sh CLEANFILES = tst_mpi_parallel.bin cdm_sea_soundings.nc bm_chunking.nc \ tst_floats_1D.cdl floats_1D_3.nc floats_1D.cdl tst_*.nc tmp_*.txt \ diff --git a/nc_test4/test_filter_vlen.c b/nc_test4/test_filter_vlen.c index e7775370a2..827087bb51 100644 --- a/nc_test4/test_filter_vlen.c +++ b/nc_test4/test_filter_vlen.c @@ -8,25 +8,16 @@ #include #include +#ifdef HAVE_HDF5_H #include +#endif #include "netcdf.h" #include "netcdf_aux.h" #include "netcdf_filter.h" -#undef TESTODDSIZE - #undef DEBUG -#ifndef H5Z_FILTER_FLETCHER32 -#define H5Z_FILTER_FLETCHER32 3 -#endif - -/* The C standard apparently defines all floating point constants as double; - we rely on that in this code. -*/ -#define DBLVAL 12345678.12345678 - -#define TEST_ID 32768 +#define FILTER_ID 1 /*deflate*/ #define MAXERRS 8 @@ -34,40 +25,29 @@ #define NPARAMS 14 -static unsigned int baseline[NPARAMS]; - static const char* testfile = NULL; #define MAXDIMS 8 -#define DFALT_TESTFILE "tmp_misc.nc" - -#define spec "32768, -17b, 23ub, -25S, 27US, 77, 93U, 789f, 12345678.12345678d, -9223372036854775807L, 18446744073709551615UL" - -#ifdef TESTODDSIZE -#define NDIMS 1 -static size_t dimsize[NDIMS] = {4}; -static size_t chunksize[NDIMS] = {3}; +#ifdef TESTNCZARR +#define DFALT_TESTFILE "file://tmp_filter_vlen.nc#mode=nczarr,file" #else +#define DFALT_TESTFILE "tmp_filter_vlen.nc" +#endif + #define NDIMS 4 static size_t dimsize[NDIMS] = {4,4,4,4}; -static size_t chunksize[NDIMS] = {4,4,4,4}; -#endif static size_t ndims = NDIMS; static size_t totalproduct = 1; /* x-product over max dims */ static size_t actualproduct = 1; /* x-product over actualdims */ -static size_t chunkproduct = 1; /* x-product over actual chunks */ - -static size_t pattern[MAXDIMS]; static int nerrs = 0; static int ncid, varid; static int dimids[MAXDIMS]; -static float* array = NULL; -static float* expected = NULL; +static char** array = NULL; /* Forward */ static int test_test1(void); @@ -91,31 +71,10 @@ check(int err,int line) } #define CHECK(x) check(x,__LINE__) -static int -verifychunks(void) -{ - int i; - int store = -1; - size_t localchunks[MAXDIMS]; - memset(localchunks,0,sizeof(localchunks)); - CHECK(nc_inq_var_chunking(ncid, varid, &store, localchunks)); - if(store != NC_CHUNKED) { - fprintf(stderr,"bad chunk store\n"); - return 0; - } - for(i=0;i 1) @@ -247,19 +261,14 @@ init(int argc, char** argv) /* Setup various variables */ totalproduct = 1; actualproduct = 1; - chunkproduct = 1; for(i=0;i 0 ? "FAILED" : "PASS")); exit(nerrs > 0?1:0); } diff --git a/nc_test4/tst_filter_misc.sh b/nc_test4/tst_filter_misc.sh index 4efe553613..e2ade3d8f4 100755 --- a/nc_test4/tst_filter_misc.sh +++ b/nc_test4/tst_filter_misc.sh @@ -1,9 +1,5 @@ #!/bin/bash -# Test the filter install -# This cannot be run as a regular test -# because installation will not have occurred - if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh @@ -22,11 +18,11 @@ s3isolate fi if test "x$TESTNCZARR" = x1; then - TFAVAIL =${execdir}/test_filter_avail - TFVLEN =${execdir}/test_filter_vlen + TFAVAIL=${execdir}/test_filter_avail + TFVLEN=${execdir}/test_filter_vlen else - TFAVAIL =${execdir}/tst_filter_avail - TFVLEN =${execdir}/tst_filter_vlen + TFAVAIL=${execdir}/tst_filter_avail + TFVLEN=${execdir}/tst_filter_vlen fi # Load the findplugins function diff --git a/nc_test4/tst_h5_endians.c b/nc_test4/tst_h5_endians.c index 841578758a..c648515391 100644 --- a/nc_test4/tst_h5_endians.c +++ b/nc_test4/tst_h5_endians.c @@ -20,10 +20,12 @@ #undef BE_DEBUG #ifdef TESTNCZARR +#include "test_utils.h" #define FILE_NAME_NC "file://tmp_h5_endians.file#mode=nczarr,file" #else #define FILE_NAME_NC "tst_h5_endians.nc" #endif + #define NDIM 10 #define NLON 20 #define DIM_NAME "x" @@ -36,7 +38,7 @@ #define LE_DBL_VARNAME "dbl_le" #define BE_DBL_VARNAME "dbl_be" -#ifdef BE_DEBUG +#if defined BE_DEBUG || defined TESTNCZARR static float f32swap(float x) { diff --git a/nc_test4/tst_put_vars_two_unlim_dim.c b/nc_test4/tst_put_vars_two_unlim_dim.c index eaf138b17e..ef0d6e89b5 100644 --- a/nc_test4/tst_put_vars_two_unlim_dim.c +++ b/nc_test4/tst_put_vars_two_unlim_dim.c @@ -20,6 +20,7 @@ See \ref copyright file for more info. #include #ifdef TESTNCZARR +#include "test_utils.h" #define FILE_NAME "file://tmp_put_vars_two_unlim_dim.file#mode=nczarr,file" #else #define FILE_NAME "tst_put_vars_two_unlim_dim.nc" diff --git a/ncdap_test/test_manyurls.c b/ncdap_test/test_manyurls.c index 00343a0fb8..6f2ec01e38 100644 --- a/ncdap_test/test_manyurls.c +++ b/ncdap_test/test_manyurls.c @@ -5,6 +5,8 @@ #include "manyurls.h" +#undef VERBOSE + int main() { int i,ncid; @@ -14,13 +16,17 @@ int main() char* tp = *p; int mode = 0; int status = -1; +#ifdef VERBOSE printf("Opening: %s\n",tp); +#endif status = nc_open(tp, mode, &ncid); switch(status) { case NC_NOERR: break; case NC_ENOTFOUND: +#ifdef VERBOSE printf("{%d} %s\n",i,tp); +#endif status = NC_NOERR; break; default: diff --git a/ncdump/tst_mud.sh b/ncdump/tst_mud.sh index 0eb5a7a58e..a538fda0f2 100755 --- a/ncdump/tst_mud.sh +++ b/ncdump/tst_mud.sh @@ -11,9 +11,11 @@ set -e if test "x$TESTNCZARR" = x1 ; then . "$srcdir/test_nczarr.sh" s3isolate "testdir_mud4" +else +ISOPATH=`pwd` +fi THISDIR=`pwd` cd $ISOPATH -fi echo "" echo "*** Testing ncdump output for multiple unlimited dimensions" @@ -71,7 +73,6 @@ if test "x$TESTNCZARR" = x1 ; then testcase file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip ; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3 ; fi - if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup else testcase nc fi diff --git a/ncdump/tst_nccopy5.sh b/ncdump/tst_nccopy5.sh index be29e4d9c4..2c2399b18a 100755 --- a/ncdump/tst_nccopy5.sh +++ b/ncdump/tst_nccopy5.sh @@ -4,6 +4,7 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh set -e + if test "x$TESTNCZARR" = x1 ; then . "$srcdir/test_nczarr.sh" s3isolate "testdir_nccopy5" @@ -25,12 +26,12 @@ fi echo "" # Trim off leading and trailing whitespace -# Also remove any +# Also remove any and de-tabify # usage: trim # Leaves result in variable TRIMMED trim() { # trim leading whitespace and remove - TMP=`echo "$1" |tr -d '\r' | sed -e 's/^[ ]*//'` + TMP=`echo "$1" |tr -d '\r' | tr '\t' ' ' |sed -e 's/^[ ]*//'` # trim trailing whitespace TRIMMED=`echo "$TMP" | sed -e 's/[ ]*$//'` } @@ -201,7 +202,8 @@ ${CHUNKTEST} ${file} unlimited ${NCDUMP} -n tmp_nc5_base ${file} > tmp_nc5.cdl echo "*** Test nccopy -c with unlimited dimension; classic ->enhanced" -${NCCOPY} -M500 -c ivar:5,3 $file tmp_nc34.nc +# Warning: make sure that nccopy does not convert small chunking to contiguous => -M +${NCCOPY} -M50 -c ivar:5,3 $file tmp_nc34.nc ${NCDUMP} -n tmp_nc5_base tmp_nc34.nc > tmp_nc34.cdl diff tmp_nc5.cdl tmp_nc34.cdl diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index e79982ba18..bd09fd594a 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -12,26 +12,27 @@ remove_definitions(-DDLL_EXPORT) # Create nczarr tests from nc_test4 tests macro(NCZARR_SH_TEST basename src) FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../${src}/tst_${basename}.sh SHSOURCE) -STRING(PREPEND SHSOURCE "#!/bin/bash\n") +# Make sure the order of prepended lines is correct STRING(PREPEND SHSOURCE "TESTNCZARR=1\n") +STRING(PREPEND SHSOURCE "#!/bin/bash\n") # Replace with FILE(CONFIGURE) when cmake 3.18 is in common use FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/run_${basename}.1 "${SHSOURCE}") CONFIGURE_FILE(${CMAKE_CURRENT_BINARY_DIR}/run_${basename}.1 ${CMAKE_CURRENT_BINARY_DIR}/run_${basename}.sh FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE @ONLY NEWLINE_STYLE LF) FILE(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/run_${basename}.1) endmacro(NCZARR_SH_TEST) -macro(NCZARR_C_TEST basename newname) -FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../nc_test4/${basename}.c CSOURCE) +macro(NCZARR_C_TEST basename newname src) +FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../${src}/${basename}.c CSOURCE) STRING(PREPEND CSOURCE "#define TESTNCZARR\n") FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${newname}.c "${CSOURCE}") endmacro(NCZARR_C_TEST) -NCZARR_C_TEST(tst_quantize test_quantize) -NCZARR_C_TEST(tst_filter_vlen test_filter_vlen) -NCZARR_C_TEST(tst_unlim_vars test_unlim_vars) -NCZARR_C_TEST(tst_h5_endians test_endians) -NCZARR_C_TEST(tst_put_vars_two_unlim_dim test_put_vars_two_unlim_dim) -NCZARR_C_TEST(tst_cnunking test_chunking) +NCZARR_C_TEST(tst_quantize test_quantize nc_test4) +NCZARR_C_TEST(tst_filter_vlen test_filter_vlen nc_test4) +NCZARR_C_TEST(tst_unlim_vars test_unlim_vars nc_test4) +NCZARR_C_TEST(tst_h5_endians test_endians nc_test4) +NCZARR_C_TEST(tst_put_vars_two_unlim_dim test_put_vars_two_unlim_dim nc_test4) +NCZARR_C_TEST(tst_chunking test_chunking ncdump) NCZARR_SH_TEST(specific_filters nc_test4) NCZARR_SH_TEST(unknown nc_test4) @@ -58,7 +59,7 @@ IF(ENABLE_TESTS) SET(TSTCOMMONSRC test_utils.c ${XGETOPTSRC}) - INCLUDE_DIRECTORIES(../libnczarr) + INCLUDE_DIRECTORIES(../libnczarr ../plugins) # Base tests # The tests are set up as a combination of shell scripts and executables that @@ -74,24 +75,21 @@ IF(ENABLE_TESTS) BUILD_BIN_TEST(test_notzarr ${TSTCOMMONSRC}) BUILD_BIN_TEST(test_chunkcases ${TSTCOMMONSRC}) - ADD_BIN_TEST(nczarr_test test_endians ${TSTCOMMONSRC}) +# ADD_BIN_TEST(nczarr_test test_endians ${TSTCOMMONSRC}) # Unlimited Tests BUILD_BIN_TEST(test_zchunks ${COMMONSRC}) BUILD_BIN_TEST(test_zchunks2 ${COMMONSRC}) BUILD_BIN_TEST(test_zchunks3 ${COMMONSRC}) - BUILD_BIN_TEST(test_unlim_io ${TSTCOMMONSRC}) ADD_BIN_TEST(nczarr_test test_unlim_vars ${TSTCOMMONSRC}) ADD_BIN_TEST(nczarr_test test_put_vars_two_unlim_dim ${TSTCOMMONSRC}) - ADD_SH_TEST(nczarr_test run_mud) ADD_SH_TEST(nczarr_test run_nccopyz) - ADD_SH_TEST(nczarr_test run_unlim_io) - ADD_SH_TEST(nczarr_test run_mud) IF(USE_HDF5) BUILD_BIN_TEST(test_unlim_io ${TSTCOMMONSRC}) BUILD_BIN_TEST(test_chunking ${TSTCOMMONSRC}) ADD_SH_TEST(nczarr_test run_unlim_io) ADD_SH_TEST(nczarr_test run_nccopy5) + ADD_SH_TEST(nczarr_test run_mud) ENDIF() IF(FALSE) # Obsolete tests @@ -131,7 +129,6 @@ IF(ENABLE_TESTS) ENDIF() add_sh_test(nczarr_test run_purezarr) - add_sh_test(nczarr_test run_interop) add_sh_test(nczarr_test run_misc) add_sh_test(nczarr_test run_nczarr_fill) add_sh_test(nczarr_test run_jsonconvention) @@ -142,10 +139,11 @@ IF(ENABLE_TESTS) add_sh_test(nczarr_test run_quantize) add_sh_test(nczarr_test run_notzarr) - if(ENABLE_S3) - add_sh_test(nczarr_test run_s3_cleanup) - ENDIF() - + # This has timeout under CMake +# if(NOT ISCMAKE) + add_sh_test(nczarr_test run_interop) +# ENDIF() + IF(USE_HDF5) add_sh_test(nczarr_test run_fillonlyz) ENDIF() diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index ea7bb48273..9a79951c8e 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -8,7 +8,7 @@ include $(top_srcdir)/lib_flags.am AM_CFLAGS += -I.. AM_CPPFLAGS += -I${top_srcdir}/libnczarr -AM_LDFLAGS += ${top_builddir}/liblib/libnetcdf.la +AM_CPPFLAGS += -I${top_srcdir}/plugins LDADD = ${top_builddir}/liblib/libnetcdf.la TESTS_ENVIRONMENT = @@ -43,7 +43,7 @@ test_fillonlyz_SOURCES = test_fillonlyz.c ${testcommonsrc} check_PROGRAMS += test_fillonlyz test_quantize test_notzarr # Unlimited Dimension tests -check_PROGRAMS += test_zchunks test_zchunks2 test_zchunks3 test_chunking test_unlim_vars test_put_vars_two_unlim_dim +check_PROGRAMS += test_zchunks test_zchunks2 test_zchunks3 test_unlim_vars test_put_vars_two_unlim_dim if USE_HDF5 check_PROGRAMS += test_unlim_io test_unlim_io_SOURCES = test_unlim_io.c ${testcommonsrc} @@ -74,7 +74,15 @@ TESTS += run_notzarr.sh TESTS += run_external.sh # Unlimited Dimension tests (at least in part) -TESTS += run_nccopy5.sh run_mud.sh run_nccopy5.sh test_unlim_vars test_put_vars_two_unlim_dim +TESTS += run_mud.sh test_unlim_vars test_put_vars_two_unlim_dim + +# Unlimited Dimension tests +check_PROGRAMS += test_chunking +TESTS += run_nccopy5.sh + +check_PROGRAMS += test_filter_avail test_filter_vlen +TESTS += run_filter_vlen.sh run_filter_misc.sh + if USE_HDF5 TESTS += run_unlim_io.sh run_nccopyz.sh endif @@ -195,12 +203,14 @@ ref_nulls_nczarr.baseline ref_nulls_zarr.baseline ref_nulls.cdl ref_notzarr.tar. # Interoperability files EXTRA_DIST += ref_power_901_constants_orig.zip ref_power_901_constants.cdl ref_quotes_orig.zip ref_quotes.cdl \ -ref_zarr_test_data.cdl.gz +ref_zarr_test_data.cdl.gz ref_zarr_test_data_2d.cdl.gz CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp*.zmap tmp_ngc.c ref_zarr_test_data.cdl tst_*.nc.zip ref_quotes.zip ref_power_901_constants.zip -BUILT_SOURCES = test_quantize.c test_filter_vlen.c test_unlim_vars.c test_endians.c test_put_vars_two_unlim_dim.c \ - run_unknown.sh run_specific_filters.sh run_filter_vlen.sh run_filterinstall.sh run_mud.sh run_nccopy5.sh +BUILT_SOURCES = test_quantize.c test_filter_vlen.c test_unlim_vars.c test_endians.c \ + test_put_vars_two_unlim_dim.c test_chunking.c \ + run_unknown.sh run_specific_filters.sh run_filter_vlen.sh run_filterinstall.sh \ + run_mud.sh run_nccopy5.sh run_filter_misc.sh test_quantize.c: $(top_srcdir)/nc_test4/tst_quantize.c rm -f $@ @@ -259,6 +269,15 @@ run_filter_vlen.sh: $(top_srcdir)/nc_test4/tst_filter_vlen.sh chmod a+x $@ rm -f run_filter_vlen.tmp +run_filter_misc.sh: $(top_srcdir)/nc_test4/tst_filter_misc.sh + rm -f $@ run_filter_misc.tmp + echo "#!/bin/bash" > run_filter_misc.tmp + echo "TESTNCZARR=1" >> run_filter_misc.tmp + cat $(top_srcdir)/nc_test4/tst_filter_misc.sh >> run_filter_misc.tmp + tr -d '\r' < run_filter_misc.tmp > $@ + chmod a+x $@ + rm -f run_filter_misc.tmp + run_filterinstall.sh: $(top_srcdir)/nc_test4/tst_filterinstall.sh rm -f $@ run_filterinstall.tmp echo "#!/bin/bash" > run_filterinstall.tmp @@ -292,12 +311,16 @@ clean-local: rm -fr tmp_*.nc tmp_*.zarr tst_quantize*.zarr tmp*.file results.file results.s3 results.zip rm -fr rcmiscdir ref_power_901_constants.file - -DISTCLEANFILES = findplugin.sh test_quantize.c run_specific_filters.sh run_filterinstall.sh run_unknown.sh test_filter_vlen.c run_filter_vlen.sh test_unlim_vars.c run_mud.sh run_nccopy5.sh - +if ENABLE_S3_TESTALL # Provide a specific s3 cleanup action -s3cleanup:: - ${srcdir}/run_s3_cleanup.sh +.PHONY: s3clean +s3clean: + ${abs_srcdir}/run_s3_cleanup.sh ${abs_builddir} ${S3TESTSUBTREE} +else +s3clean: +endif + +DISTCLEANFILES = findplugin.sh ${BUILT_SOURCES} # If valgrind is present, add valgrind targets. @VALGRIND_CHECK_RULES@ diff --git a/nczarr_test/ref_zarr_test_data.cdl.gz b/nczarr_test/ref_zarr_test_data.cdl.gz index 85ab24c24e9706f8d6acb74e0d1b4c5630354d16..5f1f2dee21f5812d4907f7fa798bea58fbf6e9e3 100644 GIT binary patch delta 17 YcmeAc?H1*b@8;lWTzoENBS$kA0563Fp8x;= delta 17 YcmeAc?H1*b@8;lm<-?G?k)xRl04tLO;s5{u diff --git a/nczarr_test/run_cachetest.sh b/nczarr_test/run_cachetest.sh index fbd446dae5..80562bbd71 100755 --- a/nczarr_test/run_cachetest.sh +++ b/nczarr_test/run_cachetest.sh @@ -36,5 +36,3 @@ ${execdir}/test_readcaching testcase file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi - -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_chunkcases.sh b/nczarr_test/run_chunkcases.sh index 77dd77f971..492c8f8b23 100755 --- a/nczarr_test/run_chunkcases.sh +++ b/nczarr_test/run_chunkcases.sh @@ -112,9 +112,6 @@ echo "Test miscellaneous 1" makefile tmp_misc1 rm -f tmp_misc1_${zext}.txt tmp_misc1_${zext}.cdl $TC -d 6,12,4 -c 2,3,1 -s 0,0,0 -e 6,1,4 -Ocw $F -if test "x$FEATURE_S3TESTS" = xyes ; then -${S3UTIL} -u 'https://s3.us-east-1.amazonaws.com/unidata-zarr-test-data' -k '/netcdf-c' list -fi ${NCDUMP} $F > tmp_misc1_${zext}.cdl diff -b ${srcdir}/ref_misc1.cdl tmp_misc1_${zext}.cdl ${execdir}/ncdumpchunks -v v $F > tmp_misc1_${zext}.dmp @@ -129,5 +126,3 @@ testcases() { testcases file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcases zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcases s3; fi - -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_external.sh b/nczarr_test/run_external.sh index c243bbbde2..bf84d460f2 100755 --- a/nczarr_test/run_external.sh +++ b/nczarr_test/run_external.sh @@ -32,5 +32,3 @@ for t in $TESTCASES ; do testcase "$t" done fi - -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_fillonlyz.sh b/nczarr_test/run_fillonlyz.sh index 3251628739..ef324c5f49 100755 --- a/nczarr_test/run_fillonlyz.sh +++ b/nczarr_test/run_fillonlyz.sh @@ -28,5 +28,3 @@ ${execdir}/test_fillonlyz${ext} "$fileurl" testcase file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi - -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_filter.sh b/nczarr_test/run_filter.sh index 9ed36c1f20..a62f9970ca 100755 --- a/nczarr_test/run_filter.sh +++ b/nczarr_test/run_filter.sh @@ -72,7 +72,7 @@ if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi testapi() { zext=$1 -echo "*** Testing dynamic filters using API for map=$zext" +echo "*** Testing dynamic filters using API for storage format $zext" fileargs tmp_api deletemap $zext $file ${execdir}/testfilter $fileurl @@ -85,7 +85,7 @@ echo "*** Pass: API dynamic filter for map=$zext" testmisc() { zext=$1 -echo "*** Testing dynamic filters parameter passing for map $zext" +echo "*** Testing dynamic filters parameter passing for storage format $zext" fileargs tmp_misc deletemap $zext $file ${execdir}/testfilter_misc $fileurl @@ -100,12 +100,12 @@ cat >./tmp_misc2_$zext.txt < ./tmp_ng_$zext.txt # Remove irrelevant -s output sclean ./tmp_ng_$zext.txt ./tmp_ng2_$zext.txt diff -b -w ${srcdir}/ref_bzip2.cdl ./tmp_ng2_$zext.txt -echo "*** Pass: ncgen dynamic filter for map $zext" +echo "*** Pass: ncgen dynamic filter for storage format $zext" } testncp() { zext=$1 -echo "*** Testing dynamic filters using nccopy for map $zext" +echo "*** Testing dynamic filters using nccopy for storage format $zext" fileargs tmp_unfiltered deletemap $zext $file # Create our input test files @@ -130,22 +130,22 @@ ${NCDUMP} -s -n filtered $fileurl > ./tmp_ncp_$zext.txt # Remove irrelevant -s output sclean ./tmp_ncp_$zext.txt ./tmp_ncp_$zext.dump diff -b -w ${srcdir}/ref_filtered.cdl ./tmp_ncp_$zext.dump -echo " *** Pass: nccopy simple filter for map $zext" +echo " *** Pass: nccopy simple filter for storage format $zext" } testngc() { zext=$1 -echo "*** Testing dynamic filters using ncgen with -lc for map $zext" +echo "*** Testing dynamic filters using ncgen with -lc for storage format $zext" fileargs tmp_ngc deletemap $zext $file ${NCGEN} -lc -4 ${srcdir}/../nc_test4/bzip2.cdl > tmp_ngc.c diff -b -w ${srcdir}/../nc_test4/../nc_test4/ref_bzip2.c ./tmp_ngc.c -echo "*** Pass: ncgen dynamic filter for map $zext" +echo "*** Pass: ncgen dynamic filter for storage format $zext" } testmulti() { zext=$1 -echo "*** Testing multiple filters for map $zext" +echo "*** Testing multiple filters for storage format $zext" fileargs tmp_multi deletemap $zext $file ${execdir}/testfilter_multi $fileurl @@ -153,12 +153,12 @@ ${NCDUMP} -hsF -n multifilter $fileurl >./tmp_multi_$zext.cdl # Remove irrelevant -s output sclean ./tmp_multi_$zext.cdl ./tmp_smulti_$zext.cdl diff -b -w ${srcdir}/ref_multi.cdl ./tmp_smulti_$zext.cdl -echo "*** Pass: multiple filters for map $zext" +echo "*** Pass: multiple filters for storage format $zext" } testrep() { zext=$1 -echo "*** Testing filter re-definition invocation for map $zext" +echo "*** Testing filter re-definition invocation for storage format $zext" fileargs tmp_rep deletemap $zext $file ${execdir}/testfilter_repeat $fileurl >tmp_rep_$zext.txt @@ -167,12 +167,12 @@ diff -b -w ${srcdir}/../nc_test4/ref_filter_repeat.txt tmp_rep_$zext.txt testorder() { zext=$1 -echo "*** Testing multiple filter order of invocation on create for map $zext" +echo "*** Testing multiple filter order of invocation on create for storage format $zext" fileargs tmp_order deletemap $zext $file ${execdir}/testfilter_order create $fileurl >tmp_order_$zext.txt diff -b -w ${srcdir}/../nc_test4/ref_filter_order_create.txt tmp_order_$zext.txt -echo "*** Testing multiple filter order of invocation on read for map $zext" +echo "*** Testing multiple filter order of invocation on read for storage format $zext" ${execdir}/testfilter_order read $fileurl >tmp_order_rd_$zext.txt diff -b -w ${srcdir}/../nc_test4/ref_filter_order_read.txt tmp_order_rd_$zext.txt } @@ -180,5 +180,3 @@ diff -b -w ${srcdir}/../nc_test4/ref_filter_order_read.txt tmp_order_rd_$zext.tx testset file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi - -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_interop.sh b/nczarr_test/run_interop.sh index 9b4f4bdbac..e9cf8e59a3 100755 --- a/nczarr_test/run_interop.sh +++ b/nczarr_test/run_interop.sh @@ -5,9 +5,10 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . "$srcdir/test_nczarr.sh" -set -x set -e +metaonly="-h" + s3isolate "testdir_interop" THISDIR=`pwd` cd $ISOPATH @@ -23,11 +24,9 @@ testcasefile() { zext=file base=$1 mode=$2 - metaonly=$3 - if test "x$metaonly" = xmetaonly ; then flags="-h"; fi fileargs ${ISOPATH}/ref_$base "mode=$mode,$zext" rm -f tmp_${base}_${zext}.cdl - ${NCDUMP} $flags $fileurl > tmp_${base}_${zext}.cdl + ${NCDUMP} $metaonly $fileurl > tmp_${base}_${zext}.cdl diff -b ${srcdir}/ref_${base}.cdl tmp_${base}_${zext}.cdl } @@ -37,7 +36,7 @@ testcasezip() { mode=$2 fileargs ${ISOPATH}/ref_$base "mode=$mode,$zext" rm -f tmp_${base}_${zext}.cdl - ${NCDUMP} -h $flags $fileurl > tmp_${base}_${zext}.cdl + ${NCDUMP} $metaonly $flags $fileurl > tmp_${base}_${zext}.cdl diff -b ${srcdir}/ref_${base}.cdl tmp_${base}_${zext}.cdl } @@ -47,9 +46,10 @@ testcases3() { mode=$2 rm -f tmp_${base}_${zext}.cdl url="https://${UH}/${UB}/${base}.zarr#mode=${mode},s3" - ${NCDUMP} $url > tmp_${base}_${zext}.cdl + # Dumping everything causes timeout so dump a single var + ${NCDUMP} -v "/group_with_dims/var2D" $flags $url > tmp_${base}_${zext}.cdl # Find the proper ref file - diff -b ${ISOPATH}/ref_${base}.cdl tmp_${base}_${zext}.cdl + diff -b ${ISOPATH}/ref_${base}_2d.cdl tmp_${base}_${zext}.cdl } testallcases() { @@ -59,19 +59,20 @@ case "$zext" in # need to unpack unzip ref_power_901_constants.zip >> tmp_ignore.txt mv ${ISOPATH}/ref_power_901_constants ${ISOPATH}/ref_power_901_constants.file - testcasefile power_901_constants zarr metaonly; # test xarray as default + testcasefile power_901_constants zarr; # test xarray as default ;; zip) # Move into position - testcasezip power_901_constants xarray metaonly + testcasezip power_901_constants xarray # Test large constant interoperability - testcasezip quotes zarr metaonly + testcasezip quotes zarr ;; s3) # Read a test case created by netcdf-java zarr. # unpack # Use gunzip because it always appears to be available gunzip -c ${srcdir}/ref_zarr_test_data.cdl.gz > ${ISOPATH}/ref_zarr_test_data.cdl + gunzip -c ${srcdir}/ref_zarr_test_data_2d.cdl.gz > ${ISOPATH}/ref_zarr_test_data_2d.cdl testcases3 zarr_test_data xarray ;; *) echo "unimplemented kind: $1" ; exit 1;; @@ -79,6 +80,7 @@ esac } # common setup + if ! test -f ${ISOPATH}/ref_power_901_constants.zip ; then cp -f ${srcdir}/ref_power_901_constants_orig.zip ${ISOPATH}/ref_power_901_constants.zip fi @@ -89,5 +91,3 @@ fi testallcases file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testallcases zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testallcases s3; fi - -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_jsonconvention.sh b/nczarr_test/run_jsonconvention.sh index ad35ea8910..64b629d858 100755 --- a/nczarr_test/run_jsonconvention.sh +++ b/nczarr_test/run_jsonconvention.sh @@ -38,5 +38,3 @@ diff -b ${srcdir}/ref_jsonconvention.zmap tmp_jsonconvention_clean_${zext}.txt testcase file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi - -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_misc.sh b/nczarr_test/run_misc.sh index 2d58cf72b9..b9741c99e1 100755 --- a/nczarr_test/run_misc.sh +++ b/nczarr_test/run_misc.sh @@ -60,5 +60,3 @@ if test "x$FEATURE_S3TESTS" = xyes ; then testcase1 s3 testcase2 s3 fi - -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_nccopyz.sh b/nczarr_test/run_nccopyz.sh index cce940e77f..66b4286cd7 100755 --- a/nczarr_test/run_nccopyz.sh +++ b/nczarr_test/run_nccopyz.sh @@ -82,5 +82,3 @@ if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi echo "*** All nccopy nczarr tests passed!" - -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_ncgen4.sh b/nczarr_test/run_ncgen4.sh index 67cd5c27be..b2f6b0010d 100755 --- a/nczarr_test/run_ncgen4.sh +++ b/nczarr_test/run_ncgen4.sh @@ -95,5 +95,3 @@ if test "x$FEATURE_NCZARR_ZIP" = xyes ; then runtestset zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then runtestset s3; fi echo "*** PASSED ***" - -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_nczarr_fill.sh b/nczarr_test/run_nczarr_fill.sh index c095fc0804..688fb1c24e 100755 --- a/nczarr_test/run_nczarr_fill.sh +++ b/nczarr_test/run_nczarr_fill.sh @@ -68,5 +68,3 @@ if test "x$FEATURE_HDF5" = xyes ; then testcase2059 s3 fi fi - -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_nczfilter.sh b/nczarr_test/run_nczfilter.sh index 22d198b5cd..8ec523dc58 100755 --- a/nczarr_test/run_nczfilter.sh +++ b/nczarr_test/run_nczfilter.sh @@ -14,5 +14,3 @@ THISDIR=`pwd` cd $ISOPATH ${execdir}/test_nczfilter - -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_newformat.sh b/nczarr_test/run_newformat.sh index 40eb0d8ef6..fde66c9314 100755 --- a/nczarr_test/run_newformat.sh +++ b/nczarr_test/run_newformat.sh @@ -46,5 +46,3 @@ if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcasecvt zip testcasepure zip fi - -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_notzarr.sh b/nczarr_test/run_notzarr.sh index 06a596b746..7eb8a01ac1 100755 --- a/nczarr_test/run_notzarr.sh +++ b/nczarr_test/run_notzarr.sh @@ -60,5 +60,3 @@ echo "Test non-zarr S3 file" RET=`${execdir}/test_notzarr "https://$URL/${S3ISOPATH}/notzarr.s3#mode=zarr,s3"` testfailed "$RET" fi - -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_nulls.sh b/nczarr_test/run_nulls.sh index 796bf967f3..85d81b50b4 100755 --- a/nczarr_test/run_nulls.sh +++ b/nczarr_test/run_nulls.sh @@ -53,5 +53,3 @@ diff -bw ${srcdir}/ref_nulls_nczarr.baseline tmp_nulls_nczarr_${zext}.cdl testcase file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi - -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_perf_chunks1.sh b/nczarr_test/run_perf_chunks1.sh index 5367e78292..e03dc373a8 100755 --- a/nczarr_test/run_perf_chunks1.sh +++ b/nczarr_test/run_perf_chunks1.sh @@ -38,5 +38,3 @@ echo '*** SUCCESS!!!' testcases file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcases zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcases s3; fi - -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_quantize.sh b/nczarr_test/run_quantize.sh index 32af46df7f..902d56434d 100755 --- a/nczarr_test/run_quantize.sh +++ b/nczarr_test/run_quantize.sh @@ -31,4 +31,3 @@ testcase file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi # There is a (currently) untraceable bug when using S3 #if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi -# if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_scalar.sh b/nczarr_test/run_scalar.sh index b1aff8e768..c6de0ebc81 100755 --- a/nczarr_test/run_scalar.sh +++ b/nczarr_test/run_scalar.sh @@ -60,5 +60,3 @@ diff -bw $top_srcdir/nczarr_test/ref_scalar.cdl tmp_scalar_zarr_${zext}.cdl testcase file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi - -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/run_strings.sh b/nczarr_test/run_strings.sh index 6d572bb17d..3651e5db7f 100755 --- a/nczarr_test/run_strings.sh +++ b/nczarr_test/run_strings.sh @@ -54,4 +54,5 @@ testcase file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup + + diff --git a/nczarr_test/run_unlim_io.sh b/nczarr_test/run_unlim_io.sh index a19d49d76e..de5fb6ff81 100755 --- a/nczarr_test/run_unlim_io.sh +++ b/nczarr_test/run_unlim_io.sh @@ -5,7 +5,6 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ${srcdir}/test_nczarr.sh -set -x set -e s3isolate "testdir_unlim_io" @@ -35,68 +34,75 @@ remfile() { esac } -makefile() { - fileargs $1 - remfile $file - case "$zext" in - nc4) F=$file;; - file) F=$fileurl;; - zip) F=$fileurl;; - s3) F=$fileurl;; - *) echo "no such extension: $zext" ; exit 1;; - esac +buildfile() { +zext=$1 +base=$2 +if test "x$TESTNCZARR" = x1 ; then +fileargs "${base}_${zext}" +deletemap $zext $file +file="$fileurl" +else +file="${base}_${zext}.nc" +rm -f $file +fi } testcase1() { zext=$1 +buildfile $zext tmp_unlim_io1 echo ""; echo "*** Test simple use of unlimited" rm -fr tmp_unlim_io1.nc tmp_unlim_io1.$zext tmp_unlim_io1.nc.txt tmp_unlim_io1.${zext}.txt $TC -d 0 -c 2 -s 0 -e 1 -Ocw tmp_unlim_io1.nc -$TC -d 0 -c 2 -s 0 -e 1 -Ocw "file://tmp_unlim_io1.$zext#mode=nczarr,$zext" -${NCDUMP} tmp_unlim_io1.nc >tmp_unlim_io1.nc.txt -${NCDUMP} "file://tmp_unlim_io1.$zext#mode=nczarr,$zext" >tmp_unlim_io1.${zext}.txt +$TC -d 0 -c 2 -s 0 -e 1 -Ocw "$file" +${NCDUMP} -n tmp_unlim_io1 tmp_unlim_io1.nc >tmp_unlim_io1.nc.txt +${NCDUMP} -n tmp_unlim_io1 "$file" >tmp_unlim_io1.${zext}.txt diff -b tmp_unlim_io1.nc.txt tmp_unlim_io1.${zext}.txt } testcase2() { zext=$1 +buildfile $zext tmp_unlim_io2 echo ""; echo "*** Test 2-d chunking" rm -fr tmp_unlim_io2.nc tmp_unlim_io2.$zext tmp_unlim_io2.nc.txt tmp_unlim_io2.${zext}.txt $TC -d 0,0 -c 2,2 -s 0,0 -e 2,2 -v 17 -Ocw tmp_unlim_io2.nc -$TC -d 0,0 -c 2,2 -s 0,0 -e 2,2 -v 17 -Ocw "file://tmp_unlim_io2.$zext#mode=nczarr,$zext" -${NCDUMP} tmp_unlim_io2.nc >tmp_unlim_io2.nc.txt -${NCDUMP} "file://tmp_unlim_io2.$zext#mode=nczarr,$zext" >tmp_unlim_io2.${zext}.txt +$TC -d 0,0 -c 2,2 -s 0,0 -e 2,2 -v 17 -Ocw "$file" +${NCDUMP} -n tmp_unlim_io2 tmp_unlim_io2.nc >tmp_unlim_io2.nc.txt +${NCDUMP} -n tmp_unlim_io2 "$file" >tmp_unlim_io2.${zext}.txt diff -b tmp_unlim_io2.nc.txt tmp_unlim_io2.${zext}.txt -${execdir}/ncdumpchunks -v v "file://tmp_unlim_io2.$zext#mode=nczarr,$zext" +${execdir}/ncdumpchunks -v v "$file" } testcase3() { zext=$1 +buildfile $zext tmp_unlim_io3 echo ""; echo "*** Test multi-chunk extension" rm -fr tmp_unlim_io3.nc tmp_unlim_io3.$zext tmp_unlim_io3.nc.txt tmp_unlim_io3.${zext}.txt $TC -d 0,0 -c 2,2 -s 6,6 -e 2,2 -v 17 -Ocw tmp_unlim_io3.nc -$TC -d 0,0 -c 2,2 -s 6,6 -e 2,2 -v 17 -Ocw "file://tmp_unlim_io3.$zext#mode=nczarr,$zext" -${NCDUMP} tmp_unlim_io3.nc >tmp_unlim_io3.nc.txt -${NCDUMP} "file://tmp_unlim_io3.$zext#mode=nczarr,$zext" >tmp_unlim_io3.${zext}.txt +$TC -d 0,0 -c 2,2 -s 6,6 -e 2,2 -v 17 -Ocw "$file" +${NCDUMP} -n tmp_unlim_io3 tmp_unlim_io3.nc >tmp_unlim_io3.nc.txt +${NCDUMP} -n tmp_unlim_io3 "$file" >tmp_unlim_io3.${zext}.txt diff -b tmp_unlim_io3.nc.txt tmp_unlim_io3.${zext}.txt } testcase4() { zext=$1 +buildfile $zext tmp_unlim_io4 echo ""; echo "*** Test unlimited as second dimension" rm -fr tmp_unlim_io4.nc tmp_unlim_io4.$zext tmp_unlim_io4.nc.txt tmp_unlim_io4.${zext}.txt $TC -d 8,0 -c 2,2 -s 6,6 -e 2,2 -v 17 -Ocw tmp_unlim_io4.nc -$TC -d 8,0 -c 2,2 -s 6,6 -e 2,2 -v 17 -Ocw "file://tmp_unlim_io4.$zext#mode=nczarr,$zext" -${NCDUMP} tmp_unlim_io4.nc >tmp_unlim_io4.nc.txt -${NCDUMP} "file://tmp_unlim_io4.$zext#mode=nczarr,$zext" >tmp_unlim_io4.${zext}.txt +$TC -d 8,0 -c 2,2 -s 6,6 -e 2,2 -v 17 -Ocw "$file" +${NCDUMP} -n tmp_unlim_io3 tmp_unlim_io4.nc >tmp_unlim_io4.nc.txt +${NCDUMP} -n tmp_unlim_io3 "$file" >tmp_unlim_io4.${zext}.txt diff -b tmp_unlim_io4.nc.txt tmp_unlim_io4.${zext}.txt } testcases() { - testcase4 $1 + testcase1 $1 + testcase2 $1 + testcase3 $1 + testcase4 $1 } testcases file if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcases zip; fi if test "x$FEATURE_S3TESTS" = xyes ; then testcases s3; fi -if test "x$FEATURE_S3TESTS" = xyes ; then s3sdkdelete "/${S3ISOPATH}" ; fi # Cleanup diff --git a/nczarr_test/test_endians.c b/nczarr_test/test_endians.c index a65765170a..161321534a 100644 --- a/nczarr_test/test_endians.c +++ b/nczarr_test/test_endians.c @@ -21,10 +21,12 @@ #undef BE_DEBUG #ifdef TESTNCZARR +#include "test_utils.h" #define FILE_NAME_NC "file://tmp_h5_endians.file#mode=nczarr,file" #else #define FILE_NAME_NC "tst_h5_endians.nc" #endif + #define NDIM 10 #define NLON 20 #define DIM_NAME "x" @@ -37,7 +39,7 @@ #define LE_DBL_VARNAME "dbl_le" #define BE_DBL_VARNAME "dbl_be" -#ifdef BE_DEBUG +#if defined BE_DEBUG || defined TESTNCZARR static float f32swap(float x) { @@ -276,12 +278,10 @@ int main() { return retval; if ((retval = nc_get_var(ncid,be_float_varid,fdata_be_out))) return retval; -#ifdef BE_DEBUG for(i=0;i #ifdef TESTNCZARR +#include "test_utils.h" #define FILE_NAME "file://tmp_put_vars_two_unlim_dim.file#mode=nczarr,file" #else #define FILE_NAME "tst_put_vars_two_unlim_dim.nc" diff --git a/nczarr_test/test_utils.c b/nczarr_test/test_utils.c index 9fce3ba583..3afc074b0b 100644 --- a/nczarr_test/test_utils.c +++ b/nczarr_test/test_utils.c @@ -642,3 +642,11 @@ ncz_gets3testurl(void) } return s3testurlp; } + +void +ncz_report(int err, int lineno) +{ + fprintf(stderr,"Error: %d: %s\n", lineno, nc_strerror(err)); + exit(1); +} + diff --git a/nczarr_test/test_utils.h b/nczarr_test/test_utils.h index a23753e03a..1140cde22e 100644 --- a/nczarr_test/test_utils.h +++ b/nczarr_test/test_utils.h @@ -3,8 +3,6 @@ #include "netcdf.h" -#define ERR(e) report(e,__LINE__) - typedef enum Op {None=0, Create=1, Read=2, Write=3, Wholechunk=4, Odom=5, Extend=6} Op; /* Bit mask of defined options; powers of 2*/ @@ -91,15 +89,11 @@ EXTERNL int nc__testurl(const char*,char**); EXTERNL const char* ncz_gets3testurl(void); -static void -report(int err, int lineno) -{ - fprintf(stderr,"Error: %d: %s\n", lineno, nc_strerror(err)); - exit(1); -} - EXTERNL Options* options; EXTERNL Metadata* meta; -EXTERNL NClist* capture; + +EXTERNL void ncz_report(int err, int line); + +#define ERR(e) ncz_report(e,__LINE__) #endif /*TST_UTILS_H*/ diff --git a/nczarr_test/testfilter_misc.c b/nczarr_test/testfilter_misc.c index 933a85586f..1530b22d59 100644 --- a/nczarr_test/testfilter_misc.c +++ b/nczarr_test/testfilter_misc.c @@ -16,6 +16,8 @@ #include "netcdf_aux.h" #include "netcdf_filter.h" +#include "h5misc.h" /* from plugins dir */ + #undef TESTODDSIZE #undef DEBUG @@ -31,8 +33,6 @@ #define MAXPARAMS 32 -#define NPARAMS 14 - static unsigned int baseline[NPARAMS]; static const char* testfile = NULL; @@ -41,8 +41,6 @@ static const char* testfile = NULL; #define DFALT_TESTFILE "tmp_misc.nc" -#define spec "32768, -17b, 23ub, -25S, 27US, 77, 93U, 789f, 12345678.12345678d, -9223372036854775807L, 18446744073709551615UL" - #ifdef TESTODDSIZE #define NDIMS 1 static size_t dimsize[NDIMS] = {4}; @@ -136,7 +134,7 @@ create(void) { int i; - /* Create a file with one big variable, but whose dimensions arte not a multiple of chunksize (to see what happens) */ + /* Create a file with one big variable whose dimensions may or may not be a multiple of chunksize (to see what happens) */ CHECK(nc_create(testfile, NC_NETCDF4|NC_CLOBBER, &ncid)); CHECK(nc_set_fill(ncid, NC_NOFILL, NULL)); for(i=0;i 0?1:0); } diff --git a/nczarr_test/zmapio.c b/nczarr_test/zmapio.c index 0007b7f3d1..d261f8fcc9 100644 --- a/nczarr_test/zmapio.c +++ b/nczarr_test/zmapio.c @@ -332,13 +332,12 @@ objdump(void) for(i=0;i 0;depth++) { + for(depth=0;depth < nclistlength(stack);depth++) { size64_t len = 0; OBJKIND kind = 0; int hascontent = 0; nullfree(content); content = NULL; - nullfree(obj); obj = NULL; - obj = nclistremove(stack,0); /* zero pos is always top of stack */ + obj = nclistget(stack,depth); kind = keykind(obj); /* Now print info for this obj key */ switch (stat=nczmap_len(map,obj,&len)) { @@ -386,7 +385,6 @@ objdump(void) } } done: - nullfree(obj); nullfree(content); nczmap_close(map,0); nclistfreeall(stack); diff --git a/oc2/ocdebug.h b/oc2/ocdebug.h index cb9890e085..148ae2fbff 100644 --- a/oc2/ocdebug.h +++ b/oc2/ocdebug.h @@ -84,6 +84,7 @@ extern OCerror ocbreakpoint(OCerror err); extern OCerror occatch(OCerror err); extern CURLcode ocreportcurlerror(struct OCstate* state, CURLcode cstat); /* Place breakpoint on ocbreakpoint to catch errors close to where they occur*/ +/* Warning: do not evaluate e more than once */ #define OCCATCH(e) occatch(e) #define OCCATCHCHK(e) (void)occatch(e) #define OCGOTO(label) {ocbreakpoint(-1); goto label;} diff --git a/plugins/H5Zmisc.c b/plugins/H5Zmisc.c index f652992cee..5ff5f14b40 100644 --- a/plugins/H5Zmisc.c +++ b/plugins/H5Zmisc.c @@ -5,6 +5,9 @@ #include #include #include "netcdf_filter_build.h" +#include "netcdf_aux.h" + + #include "h5misc.h" /* WARNING: @@ -35,8 +38,8 @@ static size_t H5Z_filter_test(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], size_t nbytes, size_t *buf_size, void **buf); -static int paramcheck(size_t nparams, const unsigned int* params); -static void mismatch(size_t i, const char* which); +static int paramcheck(size_t nparams, const unsigned int* params, struct All* extracted); +static void mismatch(const char* which); const H5Z_class2_t H5Z_TEST[1] = {{ H5Z_CLASS_T_VERS, /* H5Z_class_t version */ @@ -97,6 +100,7 @@ H5Z_filter_test(unsigned int flags, size_t cd_nelmts, void* newbuf; unsigned int testcase = 0; size_t size = 1024 * sizeof(float) * 2; + struct All values; if(cd_nelmts == 0) goto fail; @@ -104,8 +108,8 @@ H5Z_filter_test(unsigned int flags, size_t cd_nelmts, testcase = cd_values[0]; switch (testcase) { - case TC_ENDIAN: - if(!paramcheck(cd_nelmts,cd_values)) + case TC_PARAMS: + if(!paramcheck(cd_nelmts,cd_values,&values)) goto fail; break; case TC_ODDSIZE: @@ -142,7 +146,7 @@ fprintf(stderr,"TC_EXPANDED: decompress: nbytes=%u buf_size=%u xdata[0..8]=|",(u H5free_memory(*buf); *buf = newbuf; - } else { /* Compress */ + } else { /* (flags & H5Z_FLAG_REVERSE) Compress */ if(testcase == TC_EXPANDED) { int i; float* b; @@ -176,103 +180,93 @@ fprintf(stderr,"TC_EXPANDED: compress: nbytes=%u buf_size=%u size=%u\n",(unsigne return 0; } +static void +extract1(void* field, size_t size, const unsigned int* params) +{ + union { + unsigned long long ll; + unsigned char char8[8]; + unsigned param[2]; + } u; + unsigned char b = 0; + unsigned short s = 0; + unsigned int i = 0; + unsigned char* bp = 0; + unsigned short* sp = NULL; + unsigned int* ip = NULL; + unsigned long long* llp = NULL; + memset(&u,0,sizeof(u)); + switch (size) { + case 1: + b = (unsigned char)(params[0]); + bp = (unsigned char*)field; + *bp = b; + break; + case 2: + s = (unsigned short)(params[0]); + sp = (unsigned short*)field; + *sp = s; + break; + case 4: + i = (unsigned)(params[0]); + ip = (unsigned*)field; + *ip = i; + break; + case 8: + u.param[0] = params[0]; + u.param[1] = params[1]; + ncaux_h5filterspec_fix8(u.char8,0); + llp = (unsigned long long*)field; + *llp = u.ll; + break; + default: fprintf(stderr,"insert: unexpected size: %u\n",(unsigned)size); abort(); + } +} + +static void +extractparams(size_t nparams, const unsigned int* params, struct All* all) +{ + size_t offset = 0; + extract1(&all->tbyte,sizeof(all->tbyte),¶ms[offset]); offset += 1; + extract1(&all->tubyte,sizeof(all->tubyte),¶ms[offset]); offset += 1; + extract1(&all->tshort,sizeof(all->tshort),¶ms[offset]); offset += 1; + extract1(&all->tushort,sizeof(all->tushort),¶ms[offset]); offset += 1; + extract1(&all->tint,sizeof(all->tint),¶ms[offset]); offset += 1; + extract1(&all->tuint,sizeof(all->tuint),¶ms[offset]); offset += 1; + extract1(&all->tfloat32,sizeof(all->tfloat32),¶ms[offset]); offset += 1; + extract1(&all->tint64,sizeof(all->tint64),¶ms[offset]); offset += 2*1; + extract1(&all->tuint64,sizeof(all->tuint64),¶ms[offset]); offset += 2*1; + extract1(&all->tfloat64,sizeof(all->tfloat64),¶ms[offset]); offset += 2*1; +} + +/* Verify values of the parameters */ static int -paramcheck(size_t nparams, const unsigned int* params) +paramcheck(size_t nparams, const unsigned int* params, struct All* extracted) { - size_t i; - unsigned char mem[8]; + struct All all; - if(nparams != 14) { - fprintf(stderr,"Too few parameters: need=14 sent=%ld\n",(unsigned long)nparams); + memset(&all,0,sizeof(all)); + + if(nparams != NPARAMS) { + fprintf(stderr,"Incorrect number of parameters: expected=%ld sent=%ld\n",(unsigned long)NPARAMS,(unsigned long)nparams); goto fail; } - for(i=0;i", +# where XXX is some string uniquely associated with the test name. +# The isolation dir is created within the ${builddir} directory. +# The is a generated 32 bit unsigned random integer to make the chance +# of collision very small. +# Process ID was not used because of the small but real chance of collision. isolate() { local rnd diff --git a/unit_test/run_s3sdk.sh b/unit_test/run_s3sdk.sh index a6f37ec85c..adc34708db 100755 --- a/unit_test/run_s3sdk.sh +++ b/unit_test/run_s3sdk.sh @@ -13,7 +13,7 @@ isolate "testdir_uts3sdk" # Create an isolation path for S3; build on the isolation directory S3ISODIR="$ISODIR" -S3ISOPATH="/netcdf-c" +S3ISOPATH="/${S3TESTSUBTREE}" S3ISOPATH="${S3ISOPATH}/$S3ISODIR" test_cleanup() { diff --git a/unit_test/test_s3sdk.c b/unit_test/test_s3sdk.c index 4237396187..4278d2c228 100644 --- a/unit_test/test_s3sdk.c +++ b/unit_test/test_s3sdk.c @@ -66,10 +66,7 @@ check(int code, const char* fcn, int line) { if(code == NC_NOERR) return code; fprintf(stderr,"***FAIL: (%d) %s @ %s:%d\n",code,nc_strerror(code),fcn,line); -#ifdef DEBUG abort(); -#endif - exit(1); } static enum Actions @@ -261,9 +258,9 @@ testgetkeys(void) printf("\n"); done: - cleanup(); for(i=0;i