From 3ffe7be4464db9984c2cdd380f6c0e063c10805d Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 14 Mar 2022 12:39:37 -0600 Subject: [PATCH] Enhance/Fix filter support re: Discussion https://github.com/Unidata/netcdf-c/discussions/2214 The primary change is to support so-called "standard filters". A standard filter is one that is defined by the following netcdf-c API: ```` int nc_def_var_XXX(int ncid, int varid, size_t nparams, unsigned* params); int nc_inq_var_XXXX(int ncid, int varid, int* usefilterp, unsigned* params); ```` So for example, zstandard would be a standard filter by defining the functions *nc_def_var_zstandard* and *nc_inq_var_zstandard*. In order to define these functions, we need a new dispatch function: ```` int nc_inq_filter_avail(int ncid, unsigned filterid); ```` This function, combined with the existing filter API can be used to implement arbitrary standard filters using a simple code pattern. Note that I would have preferred that this function return a list of all available filters, but HDF5 does not support that functionality. So this PR implements the dispatch function and implements the following standard functions: + bzip2 + zstandard + blosc Specific test cases are also provided for HDF5 and NCZarr. Over time, other specific standard filters will be defined. ## Primary Changes * Add nc_inq_filter_avail() to netcdf-c API. * Add standard filter implementations to test use of *nc_inq_filter_avail*. * Bump the dispatch table version number and add to all the relevant dispatch tables (libsrc, libsrcp, etc). * Create a program to invoke nc_inq_filter_avail so that it is accessible to shell scripts. * Cleanup szip support to properly support szip when HDF5 is disabled. This involves detecting libsz separately from testing if HDF5 supports szip. * Integrate shuffle and fletcher32 into the existing filter API. This means that, for example, nc_def_var_fletcher32 is now a wrapper around nc_def_var_filter. * Extend the Codec defaulting to allow multiple default shared libraries. ## Misc. Changes * Modify configure.ac/CMakeLists.txt to look for the relevant libraries implementing standard filters. * Modify libnetcdf.settings to list available standard filters (including deflate and szip). * Add CMake test modules to locate libbz2 and libzstd. * Cleanup the HDF5 memory manager function use in the plugins. * remove unused file include//ncfilter.h * remove tests for the HDF5 memory operations e.g. H5allocate_memory. * Add flag to ncdump to force use of _Filter instead of _Deflate or _Shuffle or _Fletcher32. Used for testing. --- .github/workflows/mingw.yml | 29 --- .github/workflows/run_tests_win_mingw.yml | 2 +- CMakeLists.txt | 138 ++++++----- RELEASE_NOTES.md | 1 + cmake/modules/FindBz2.cmake | 64 +++++ cmake/modules/FindZstd.cmake | 64 +++++ config.h.cmake.in | 25 +- configure.ac | 124 +++++++--- h5_test/tst_h_enums.c | 8 - h5_test/tst_h_par_compress.c | 4 +- h5_test/tst_h_vars.c | 4 +- include/Makefile.am | 2 +- include/hdf5internal.h | 1 + include/nc4internal.h | 2 - include/ncconfigure.h | 9 - include/ncfilter.h | 20 -- include/nchttp.h | 2 +- include/netcdf_dispatch.h.in | 3 + include/netcdf_filter.h | 49 ++++ include/netcdf_filter_hdf5_build.h | 13 + lib_flags.am | 4 + libdap2/ncd2dispatch.c | 2 + libdap4/ncd4dispatch.c | 14 ++ libdispatch/CMakeLists.txt | 6 +- libdispatch/derror.c | 2 +- libdispatch/dfilter.c | 180 ++++++++++++++ libdispatch/dhttp.c | 2 +- libdispatch/dinfermodel.c | 2 +- libdispatch/dnotnc4.c | 7 + libhdf4/hdf4dispatch.c | 6 + libhdf5/H5FDhttp.c | 1 + libhdf5/hdf5dispatch.c | 1 + libhdf5/hdf5filter.c | 57 +++-- libhdf5/hdf5open.c | 15 -- libhdf5/hdf5var.c | 29 ++- libhdf5/nc4hdf.c | 72 +++--- liblib/CMakeLists.txt | 19 +- libncpoco/cp_unix.c | 35 ++- libnczarr/Makefile.am | 1 + libnczarr/zcache.h | 2 +- libnczarr/zdebug.h | 2 +- libnczarr/zdispatch.c | 13 +- libnczarr/zdispatch.h | 1 + libnczarr/zfilter.c | 277 ++++++++++++++++------ libnczarr/zfilter.h | 2 +- libnczarr/zvar.c | 28 +-- libnetcdf.settings.in | 4 +- libsrc/nc3dispatch.c | 2 + libsrc4/nc4var.c | 17 +- libsrcp/ncpdispatch.c | 2 + nc_test4/CMakeLists.txt | 9 +- nc_test4/Makefile.am | 12 +- nc_test4/noop.cdl | 83 +++++++ nc_test4/test_filter_vlen.c | 218 ----------------- nc_test4/tst_filter.sh | 52 ++-- nc_test4/tst_filter_avail.c | 106 +++++++++ nc_test4/tst_interops5.c | 4 +- nc_test4/tst_parallel5.c | 8 +- nc_test4/tst_quantize.c | 5 +- nc_test4/tst_specific_filters.sh | 119 +++++++--- nc_test4/tst_udf.c | 16 +- nc_test4/tst_vars3.c | 13 +- nc_test4/tst_vars4.c | 2 +- ncdump/CMakeLists.txt | 65 ++--- ncdump/Makefile.am | 4 + ncdump/nccopy.c | 1 - ncdump/ncdump.1 | 4 +- ncdump/ncdump.c | 93 +++++--- ncdump/ncdump.h | 3 + ncdump/ncfilteravail.c | 178 ++++++++++++++ ncdump/ref_tst_ncf213.cdl | 10 +- ncdump/ref_tst_special_atts.cdl | 6 +- ncdump/tst_nccopy3.sh | 3 - ncdump/tst_nccopy4.sh | 1 - ncdump/tst_netcdf4_4.sh | 2 +- ncdump/utils.h | 6 +- ncgen/Makefile.am | 2 +- ncgen/genbin.c | 10 + ncgen/ncgenl.c | 69 ------ nczarr_test/CMakeLists.txt | 19 +- nczarr_test/Makefile.am | 28 ++- nczarr_test/ncdumpchunks.c | 25 +- nczarr_test/run_filter.sh | 30 ++- nczarr_test/run_nczfilter.sh | 14 ++ nczarr_test/run_quantize.sh | 4 +- nczarr_test/run_specific_filters.sh | 161 ------------- nczarr_test/tst_nczfilter.c | 7 +- plugins/BZIP2_LICENSE | 42 ++++ plugins/CMakeLists.txt | 30 ++- plugins/H5Zszip.c | 72 +++--- plugins/H5Zzstd.c | 257 ++++++++++++++++++++ plugins/H5Zzstd.h | 22 ++ plugins/Makefile.am | 72 ++++-- plugins/NCZdefaults.c | 227 +----------------- plugins/NCZszip.c | 255 ++++++++++++++++++++ test_common.in | 5 + 96 files changed, 2419 insertions(+), 1324 deletions(-) delete mode 100644 .github/workflows/mingw.yml create mode 100644 cmake/modules/FindBz2.cmake create mode 100644 cmake/modules/FindZstd.cmake delete mode 100644 include/ncfilter.h create mode 100644 nc_test4/noop.cdl create mode 100644 nc_test4/tst_filter_avail.c create mode 100644 ncdump/ncfilteravail.c create mode 100755 nczarr_test/run_nczfilter.sh delete mode 100755 nczarr_test/run_specific_filters.sh create mode 100644 plugins/BZIP2_LICENSE create mode 100644 plugins/H5Zzstd.c create mode 100644 plugins/H5Zzstd.h create mode 100644 plugins/NCZszip.c diff --git a/.github/workflows/mingw.yml b/.github/workflows/mingw.yml deleted file mode 100644 index 0f9c9a2c5d..0000000000 --- a/.github/workflows/mingw.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: NetCDF-Build MinGW -on: [workflow_dispatch,push] - -jobs: - build: - runs-on: windows-latest - defaults: - run: - shell: msys2 {0} - steps: - - uses: actions/checkout@v2 - - uses: msys2/setup-msys2@v2 - with: - msystem: MINGW64 - update: true - install: git mingw-w64-x86_64-toolchain automake libtool autoconf make mingw-w64-x86_64-hdf5 unzip - - name: CI-Build - shell: msys2 {0} - run: | - echo 'Running in MSYS2!' - set -e - pwd - git clone --single-branch --branch moreosfixes.tmp https://github.com/DennisHeimbigner/netcdf-c.git - cd netcdf-c - autoreconf -i --force - ./configure --prefix=/builddir -enable-shared --disable-static --disable-plugins --disable-logging --disable-dap-remote-tests --disable-byterange - make -j LDFLAGS="-no-undefined -Wl,--export-all-symbols" - make check - diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml index 919c50f235..913920a3aa 100644 --- a/.github/workflows/run_tests_win_mingw.yml +++ b/.github/workflows/run_tests_win_mingw.yml @@ -7,7 +7,7 @@ name: Run MSYS2, MinGW64-based Tests -on: [ pull_request ] +on: [pull_request] jobs: diff --git a/CMakeLists.txt b/CMakeLists.txt index a986144370..af19fc29d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,7 +42,7 @@ SET(PACKAGE_VERSION ${VERSION}) # Version of the dispatch table. This must match the value in # configure.ac. -SET(NC_DISPATCH_VERSION 4) +SET(NC_DISPATCH_VERSION 5) # Get system configuration, Use it to determine osname, os release, cpu. These # will be used when committing to CDash. @@ -588,6 +588,33 @@ ENDIF(ENABLE_STRICT_NULL_BYTE_HEADER_PADDING) # SET(BUILD_RPC ON CACHE BOOL "") #ENDIF() +# Note that szip management is tricky. +# This is because we have three things to consider: +# 1. is libsz available? +# 2. is szip enabled in HDF5? +# 3. is nczarr enabled? +# We need separate flags for cases 1 and 2 + +# We need to determine if libsz is available both for HDF5 and NCZarr +# If user has specified the `SZIP_LIBRARY`, use it; otherwise try to find... +IF(NOT SZIP_LIBRARY) + FIND_LIBRARY(SZIP PATH NAMES szip sz sz2) + IF(SZIP) + SET(SZIP_LIBRARY ${SZIP}) + ELSE() + UNSET(SZIP_LIBRARY) + UNSET(SZIP) + ENDIF() +ENDIF() + +IF(SZIP_LIBRARY) + SET(SZIP_FOUND yes) + SET(HAVE_SZ yes) +ELSE() + SET(SZIP_FOUND no) + SET(HAVE_SZ no) +ENDIF() + ## # Option to Enable HDF5 # @@ -847,19 +874,18 @@ IF(USE_HDF5) #error #endif int main() { - int x = 1;}" USE_SZIP) - IF(USE_SZIP) + int x = 1;}" USE_HDF5_SZIP) + IF(USE_HDF5_SZIP) + SET(HAVE_H5Z_SZIP yes) # If user has specified the `SZIP_LIBRARY`, use it; otherwise try to find... - IF(NOT SZIP_LIBRARY) - FIND_LIBRARY(SZIP PATH NAMES szip sz) - SET(SZIP_LIBRARY ${SZIP}) - IF(NOT SZIP) + IF(SZIP_FOUND) + SET(CMAKE_REQUIRED_LIBRARIES ${SZIP_LIBRARY} ${CMAKE_REQUIRED_LIBRARIES}) + MESSAGE(STATUS "HDF5 has szip.") + ELSE() MESSAGE(FATAL_ERROR "HDF5 Requires SZIP, but cannot find libszip or libsz.") - ENDIF() ENDIF() - SET(HAVE_H5Z_SZIP 1) - SET(CMAKE_REQUIRED_LIBRARIES ${SZIP_LIBRARY} ${CMAKE_REQUIRED_LIBRARIES}) - MESSAGE(STATUS "HDF5 has szip.") + ELSE() + SET(HAVE_H5Z_SZIP no) ENDIF() #### @@ -891,9 +917,6 @@ IF(USE_HDF5) #Check to see if HDF5 library has collective metadata APIs, (HDF5 >= 1.10.0) CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5Pset_all_coll_metadata_ops "" HDF5_HAS_COLL_METADATA_OPS) - CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5free_memory "" HAVE_H5FREE_MEMORY) - CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5allocate_memory "" HAVE_H5ALLOCATE_MEMORY) - CHECK_LIBRARY_EXISTS(${HDF5_C_LIBRARY_hdf5} H5resize_memory "" HAVE_H5RESIZE_MEMORY) IF(HDF5_PARALLEL) SET(HDF5_CC h5pcc) @@ -1055,30 +1078,33 @@ ELSE() SET(ENABLE_ZLIB FALSE) ENDIF() -# See if we have libblosc -IF(!MSVC) -FIND_PACKAGE(Blosc) -ENDIF() -# Define a test flag for have blosc library -IF(Blosc_FOUND) - INCLUDE_DIRECTORIES(${Blosc_INCLUDE_DIRS}) - SET(ENABLE_BLOSC TRUE) +macro(set_std_filter filter) +# Upper case the filter name +string(TOUPPER "${filter}" upfilter) +string(TOLOWER "${filter}" downfilter) +# Define a test flag for filter +IF(${filter}_FOUND) + INCLUDE_DIRECTORIES(${filter}_INCLUDE_DIRS}) + SET(ENABLE_${upfilter} TRUE) + SET(STD_FILTERS "${STD_FILTERS},${downfilter}") ELSE() - SET(ENABLE_BLOSC FALSE) + SET(ENABLE_${upfilter} FALSE) ENDIF() +endmacro(set_std_filter) -# See if we have libszip -IF(!MSVC) -#FIND_PACKAGE(SZIP) -#FIND_LIBRARY(SZIP PATH NAMES szip sz) -SET(SZIP_LIBRARY ${SZIP}) -ENDIF() -# Define a test flag for have szip library -IF(SZIP_FOUND) - INCLUDE_DIRECTORIES(${SZIP_INCLUDE_DIRS}) - SET(ENABLE_SZIP TRUE) -ELSE() - SET(ENABLE_SZIP FALSE) +# Locate some compressors +FIND_PACKAGE(Bz2) +FIND_PACKAGE(Blosc) +FIND_PACKAGE(Zstd) + +# Accumulate standard filters +set(STD_FILTERS "deflate") # Always have deflate */ +set_std_filter(SZIP) +set_std_filter(Blosc) +set_std_filter(Zstd) +set_std_filter(Bz2) +IF(NOT Bz2_FOUND) +set(STD_FILTERS "${STD_FILTERS},bzip2") # Always have bzip2 */ ENDIF() # See if we have libzip @@ -1160,21 +1186,22 @@ IF(ENABLE_NCZARR_S3_TESTS AND NOT ENABLE_NCZARR_S3) SET(ENABLE_NCZARR_S3_TESTS OFF CACHE BOOL "NCARR S3 TESTS" FORCE) ENDIF() -# See if aws-s3-sdk is available -# But only if enabled +# Note we check for the library after checking for enable_nczarr_s3 +# because for some reason this screws up if we unconditionally test for sdk +# and it is not available. Fix someday IF(ENABLE_NCZARR_S3) -find_package(AWSSDK REQUIRED COMPONENTS s3;core) -IF(AWSSDK_FOUND) - SET(service s3;core) - AWSSDK_DETERMINE_LIBS_TO_LINK(service AWS_LINK_LIBRARIES) - SET(ENABLE_S3_SDK ON CACHE BOOL "S3 SDK" FORCE) + # See if aws-s3-sdk is available + find_package(AWSSDK REQUIRED COMPONENTS s3;core) + IF(AWSSDK_FOUND) + SET(service s3;core) + AWSSDK_DETERMINE_LIBS_TO_LINK(service AWS_LINK_LIBRARIES) + SET(ENABLE_S3_SDK ON CACHE BOOL "S3 SDK" FORCE) + ELSE() + SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE) + ENDIF() ELSE() SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE) ENDIF() -ELSE(ENABLE_NCZARR_S3) -# Unconditionally disable -SET(ENABLE_S3_SDK OFF CACHE BOOL "S3 SDK" FORCE) -ENDIF(ENABLE_NCZARR_S3) IF(NOT ENABLE_S3_SDK) IF(ENABLE_NCZARR_S3 OR ENABLE_NCZARR_S3_TESTS) @@ -1493,11 +1520,19 @@ IF(NOT BUILD_SHARED_LIBS) ENDIF() OPTION(ENABLE_NCZARR_FILTERS "Enable NCZarr filters" yes) +OPTION(ENABLE_NCZARR_FILTERS_TESTING "Enable NCZarr filter testing." yes) + +# Constraints IF (NOT ENABLE_PLUGINS) - SET(ENABLE_NCZARR_FILTERS OFF CACHE BOOL "Enable NCZarr Filters." FORCE) + MESSAGE(WARNING "ENABLE_FILTER_TESTING requires shared libraries. Disabling.") + SET(ENABLE_NCZARR_FILTERS OFF CACHE BOOL "Enable NCZarr Filters." FORCE) +ENDIF() + +IF (NOT ENABLE_NCZARR) + MESSAGE(WARNING "ENABLE_NCZARR==NO => ENABLE_NCZARR_FILTERS==NO AND ENABLE_NCZARR_FILTER_TESTING==NO") + SET(ENABLE_NCZARR_FILTERS OFF CACHE BOOL "Disable NCZARR_FILTERS" FORCE) ENDIF() -OPTION(ENABLE_NCZARR_FILTERS_TESTING "Enable NCZarr filter testing." yes) IF (NOT ENABLE_NCZARR_FILTERS) SET(ENABLE_NCZARR_FILTER_TESTING OFF CACHE BOOL "Enable NCZarr Filter Testing" FORCE) ENDIF() @@ -2357,9 +2392,6 @@ is_enabled(ENABLE_V2_API HAS_NC2) is_enabled(ENABLE_NETCDF_4 HAS_NC4) is_enabled(ENABLE_HDF4 HAS_HDF4) is_enabled(USE_HDF5 HAS_HDF5) -is_enabled(USE_SZIP HAS_SZIP) -is_enabled(USE_SZIP HAS_SZIP_WRITE) -is_enabled(USE_SZIP HAS_SZLIB_WRITE) is_enabled(STATUS_PNETCDF HAS_PNETCDF) is_enabled(STATUS_PARALLEL HAS_PARALLEL) is_enabled(ENABLE_PARALLEL4 HAS_PARALLEL4) @@ -2373,7 +2405,6 @@ is_enabled(JNA HAS_JNA) is_enabled(ENABLE_ZERO_LENGTH_COORD_BOUND RELAX_COORD_BOUND) is_enabled(USE_CDF5 HAS_CDF5) is_enabled(ENABLE_ERANGE_FILL HAS_ERANGE_FILL) -is_enabled(HAVE_H5Z_SZIP HAS_SZLIB) is_enabled(HDF5_HAS_PAR_FILTERS HAS_PAR_FILTERS) is_enabled(ENABLE_NCZARR HAS_NCZARR) is_enabled(ENABLE_NCZARR_S3_TESTS DO_NCZARR_S3_TESTS) @@ -2382,7 +2413,8 @@ is_enabled(ENABLE_NCZARR_ZIP DO_NCZARR_ZIP_TESTS) is_enabled(ENABLE_QUANTIZE HAS_QUANTIZE) is_enabled(ENABLE_LOGGING HAS_LOGGING) is_enabled(ENABLE_FILTER_TESTING DO_FILTER_TESTS) -is_enabled(ENABLE_BLOSC HAS_BLOSC) +is_enabled(HAVE_SZ HAS_SZIP) +is_enabled(HAVE_SZ HAS_SZLIB_WRITE) # Generate file from template. CONFIGURE_FILE("${CMAKE_CURRENT_SOURCE_DIR}/libnetcdf.settings.in" diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 70bf7c3fef..1ed2b989b0 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.8.2 - TBD +* [Enhancement] Improve filter support. More specifically (1) add nc_inq_filter_avail to check if a filter is available, (2) add the notion of standard filters, (3) cleanup szip support to fix interaction with NCZarr. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????). * [Bug Fix] Require that the type of the variable in nc_def_var_filter is not variable length. See [Github #/2231](https://github.com/Unidata/netcdf-c/pull/2231). * [File Change] Apply HDF5 v1.8 format compatibility when writing to previous files, as well as when creating new files. The superblock version remains at 2 for newly created files. Full backward read/write compatibility for netCDF-4 is maintained in all cases. See [Github #2176](https://github.com/Unidata/netcdf-c/issues/2176). * [Enhancement] Add ability to set dataset alignment for netcdf-4/HDF5 files. See [Github #2206](https://github.com/Unidata/netcdf-c/pull/2206). diff --git a/cmake/modules/FindBz2.cmake b/cmake/modules/FindBz2.cmake new file mode 100644 index 0000000000..89d66e1810 --- /dev/null +++ b/cmake/modules/FindBz2.cmake @@ -0,0 +1,64 @@ +# Searches for an installation of the bz2 library. On success, it sets the following variables: +# +# Bz2_FOUND Set to true to indicate the bz2 library was found +# Bz2_INCLUDE_DIRS The directory containing the header file bz2/bz2.h +# Bz2_LIBRARIES The libraries needed to use the bz2 library +# +# To specify an additional directory to search, set Bz2_ROOT. +# +# Author: Siddhartha Chaudhuri, 2009 +# + +# Look for the header, first in the user-specified location and then in the system locations +SET(Bz2_INCLUDE_DOC "The directory containing the header file bz2.h") +FIND_PATH(Bz2_INCLUDE_DIRS NAMES bz2.h bz2/bz2.h PATHS ${Bz2_ROOT} ${Bz2_ROOT}/include DOC ${Bz2_INCLUDE_DOC} NO_DEFAULT_PATH) +IF(NOT Bz2_INCLUDE_DIRS) # now look in system locations + FIND_PATH(Bz2_INCLUDE_DIRS NAMES bzlib.h DOC ${Bz2_INCLUDE_DOC}) +ENDIF(NOT Bz2_INCLUDE_DIRS) + +SET(Bz2_FOUND FALSE) + +IF(Bz2_INCLUDE_DIRS) + SET(Bz2_LIBRARY_DIRS ${Bz2_INCLUDE_DIRS}) + + IF("${Bz2_LIBRARY_DIRS}" MATCHES "/include$") + # Strip off the trailing "/include" in the path. + GET_FILENAME_COMPONENT(Bz2_LIBRARY_DIRS ${Bz2_LIBRARY_DIRS} PATH) + ENDIF("${Bz2_LIBRARY_DIRS}" MATCHES "/include$") + + IF(EXISTS "${Bz2_LIBRARY_DIRS}/lib") + SET(Bz2_LIBRARY_DIRS ${Bz2_LIBRARY_DIRS}/lib) + ENDIF(EXISTS "${Bz2_LIBRARY_DIRS}/lib") + + # Find Bz2 libraries + FIND_LIBRARY(Bz2_DEBUG_LIBRARY NAMES bz2d bz2_d libbz2d libbz2_d libbz2 + PATH_SUFFIXES Debug ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Debug + PATHS ${Bz2_LIBRARY_DIRS} NO_DEFAULT_PATH) + FIND_LIBRARY(Bz2_RELEASE_LIBRARY NAMES bz2 libbz2 + PATH_SUFFIXES Release ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Release + PATHS ${Bz2_LIBRARY_DIRS} NO_DEFAULT_PATH) + + SET(Bz2_LIBRARIES ) + IF(Bz2_DEBUG_LIBRARY AND Bz2_RELEASE_LIBRARY) + SET(Bz2_LIBRARIES debug ${Bz2_DEBUG_LIBRARY} optimized ${Bz2_RELEASE_LIBRARY}) + ELSEIF(Bz2_DEBUG_LIBRARY) + SET(Bz2_LIBRARIES ${Bz2_DEBUG_LIBRARY}) + ELSEIF(Bz2_RELEASE_LIBRARY) + SET(Bz2_LIBRARIES ${Bz2_RELEASE_LIBRARY}) + ENDIF(Bz2_DEBUG_LIBRARY AND Bz2_RELEASE_LIBRARY) + + IF(Bz2_LIBRARIES) + SET(Bz2_FOUND TRUE) + ENDIF(Bz2_LIBRARIES) +ENDIF(Bz2_INCLUDE_DIRS) + +IF(Bz2_FOUND) +# IF(NOT Bz2_FIND_QUIETLY) + MESSAGE(STATUS "Found Bz2: headers at ${Bz2_INCLUDE_DIRS}, libraries at ${Bz2_LIBRARY_DIRS}") + MESSAGE(STATUS " library is ${Bz2_LIBRARIES}") +# ENDIF(NOT Bz2_FIND_QUIETLY) +ELSE(Bz2_FOUND) + IF(Bz2_FIND_REQUIRED) + MESSAGE(FATAL_ERROR "Bz2 library not found") + ENDIF(Bz2_FIND_REQUIRED) +ENDIF(Bz2_FOUND) diff --git a/cmake/modules/FindZstd.cmake b/cmake/modules/FindZstd.cmake new file mode 100644 index 0000000000..2630343bdd --- /dev/null +++ b/cmake/modules/FindZstd.cmake @@ -0,0 +1,64 @@ +# Searches for an installation of the zstd library. On success, it sets the following variables: +# +# Zstd_FOUND Set to true to indicate the zstd library was found +# Zstd_INCLUDE_DIRS The directory containing the header file zstd/zstd.h +# Zstd_LIBRARIES The libraries needed to use the zstd library +# +# To specify an additional directory to search, set Zstd_ROOT. +# +# Author: Siddhartha Chaudhuri, 2009 +# + +# Look for the header, first in the user-specified location and then in the system locations +SET(Zstd_INCLUDE_DOC "The directory containing the header file zstd.h") +FIND_PATH(Zstd_INCLUDE_DIRS NAMES zstd.h zstd/zstd.h PATHS ${Zstd_ROOT} ${Zstd_ROOT}/include DOC ${Zstd_INCLUDE_DOC} NO_DEFAULT_PATH) +IF(NOT Zstd_INCLUDE_DIRS) # now look in system locations + FIND_PATH(Zstd_INCLUDE_DIRS NAMES zstd.h zstd/zstd.h DOC ${Zstd_INCLUDE_DOC}) +ENDIF(NOT Zstd_INCLUDE_DIRS) + +SET(Zstd_FOUND FALSE) + +IF(Zstd_INCLUDE_DIRS) + SET(Zstd_LIBRARY_DIRS ${Zstd_INCLUDE_DIRS}) + + IF("${Zstd_LIBRARY_DIRS}" MATCHES "/include$") + # Strip off the trailing "/include" in the path. + GET_FILENAME_COMPONENT(Zstd_LIBRARY_DIRS ${Zstd_LIBRARY_DIRS} PATH) + ENDIF("${Zstd_LIBRARY_DIRS}" MATCHES "/include$") + + IF(EXISTS "${Zstd_LIBRARY_DIRS}/lib") + SET(Zstd_LIBRARY_DIRS ${Zstd_LIBRARY_DIRS}/lib) + ENDIF(EXISTS "${Zstd_LIBRARY_DIRS}/lib") + + # Find Zstd libraries + FIND_LIBRARY(Zstd_DEBUG_LIBRARY NAMES zstdd zstd_d libzstdd libzstd_d libzstd + PATH_SUFFIXES Debug ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Debug + PATHS ${Zstd_LIBRARY_DIRS} NO_DEFAULT_PATH) + FIND_LIBRARY(Zstd_RELEASE_LIBRARY NAMES zstd libzstd + PATH_SUFFIXES Release ${CMAKE_LIBRARY_ARCHITECTURE} ${CMAKE_LIBRARY_ARCHITECTURE}/Release + PATHS ${Zstd_LIBRARY_DIRS} NO_DEFAULT_PATH) + + SET(Zstd_LIBRARIES ) + IF(Zstd_DEBUG_LIBRARY AND Zstd_RELEASE_LIBRARY) + SET(Zstd_LIBRARIES debug ${Zstd_DEBUG_LIBRARY} optimized ${Zstd_RELEASE_LIBRARY}) + ELSEIF(Zstd_DEBUG_LIBRARY) + SET(Zstd_LIBRARIES ${Zstd_DEBUG_LIBRARY}) + ELSEIF(Zstd_RELEASE_LIBRARY) + SET(Zstd_LIBRARIES ${Zstd_RELEASE_LIBRARY}) + ENDIF(Zstd_DEBUG_LIBRARY AND Zstd_RELEASE_LIBRARY) + + IF(Zstd_LIBRARIES) + SET(Zstd_FOUND TRUE) + ENDIF(Zstd_LIBRARIES) +ENDIF(Zstd_INCLUDE_DIRS) + +IF(Zstd_FOUND) +# IF(NOT Zstd_FIND_QUIETLY) + MESSAGE(STATUS "Found Zstd: headers at ${Zstd_INCLUDE_DIRS}, libraries at ${Zstd_LIBRARY_DIRS}") + MESSAGE(STATUS " library is ${Zstd_LIBRARIES}") +# ENDIF(NOT Zstd_FIND_QUIETLY) +ELSE(Zstd_FOUND) + IF(Zstd_FIND_REQUIRED) + MESSAGE(FATAL_ERROR "Zstd library not found") + ENDIF(Zstd_FIND_REQUIRED) +ENDIF(Zstd_FOUND) diff --git a/config.h.cmake.in b/config.h.cmake.in index 87f473e1f3..fb2b57f3bf 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -178,6 +178,15 @@ are set when opening a binary file on Windows. */ /* Define to 1 if you have the `atexit function. */ #cmakedefine HAVE_ATEXIT 1 +/* Define to 1 if bzip2 library available. */ +#cmakedefine HAVE_BZIP2 1 + +/* Define to 1 if zstd library available. */ +#cmakedefine HAVE_ZSTD 1 + +/* Define to 1 if blosc library available. */ +#cmakedefine HAVE_BLOSC 1 + /* Define to 1 if you have hdf5_coll_metadata_ops */ #cmakedefine HDF5_HAS_COLL_METADATA_OPS 1 @@ -259,6 +268,9 @@ are set when opening a binary file on Windows. */ /* Define to 1 if you have the `H5Z_SZIP' function. */ #cmakedefine HAVE_H5Z_SZIP 1 +/* Define to 1 if you have libsz */ +#cmakedefine HAVE_SZ 1 + /* Define to 1 if you have the header file. */ #cmakedefine HAVE_HDF5_H 1 @@ -422,16 +434,6 @@ are set when opening a binary file on Windows. */ /* Define to 1 if the system has the type `ushort'. */ #cmakedefine HAVE_USHORT 1 -/* if true, H5free_memory() will be used to free hdf5-allocated memory in - nc4file. */ -#cmakedefine HAVE_H5FREE_MEMORY 1 - -/* if true, H5allocate_memory() will be used. */ -#cmakedefine HAVE_H5ALLOCATE_MEMORY 1 - -/* if true, H5resize_memory() will be used. */ -#cmakedefine HAVE_H5RESIZE_MEMORY 1 - /* if true, hdf5 has parallelism enabled */ #cmakedefine HDF5_PARALLEL 1 @@ -620,9 +622,6 @@ with zip */ /* if true, enable nczarr blosc support */ #cmakedefine ENABLE_BLOSC 1 -/* if true, enable nczarr szip support */ -#cmakedefine USE_SZIP 1 - /* Version number of package */ #cmakedefine VERSION "${netCDF_VERSION}" diff --git a/configure.ac b/configure.ac index 929d12f055..a065175ec5 100644 --- a/configure.ac +++ b/configure.ac @@ -599,16 +599,64 @@ if test "x$enable_nczarr" = xyes; then fi AM_CONDITIONAL(ENABLE_NCZARR, [test x$enable_nczarr = xyes]) +########## +# Look for Standardized libraries +########## + # See if we have libblosc -AC_CHECK_LIB([blosc],[blosc_init],[enable_blosc=yes],[enable_blosc=no]) -if test "x$enable_blosc" = "xyes" ; then +AC_CHECK_LIB([blosc],[blosc_init],[have_blosc=yes],[have_blosc=no]) +if test "x$have_blosc" = "xyes" ; then AC_SEARCH_LIBS([blosc_init],[blosc blosc.dll cygblosc.dll], [], []) - AC_DEFINE([ENABLE_BLOSC], [1], [if true, blosc library is available]) + AC_DEFINE([HAVE_BLOSC], [1], [if true, blosc library is available]) +fi + +# See if we have libzstd +AC_CHECK_LIB([zstd],[ZSTD_compress],[have_zstd=yes],[have_zstd=no]) +if test "x$have_zstd" = "xyes" ; then + AC_SEARCH_LIBS([ZSTD_compress],[zstd zstd.dll cygzstd.dll], [], []) + AC_DEFINE([HAVE_ZSTD], [1], [if true, zstd library is available]) +fi +AC_MSG_CHECKING([whether libzstd library is available]) +AC_MSG_RESULT([${have_zstd}]) + +# See if we have libbzip2 or libbz2 +AC_CHECK_LIB([bz2],[BZ2_bzCompress],[have_bzip2=yes],[have_bzip2=no]) +if test "x$have_bzip2" = "xyes" ; then + AC_SEARCH_LIBS([BZ2_bzCompress],[bz2 bz2.dll cygbz2.dll], [], []) + AC_DEFINE([HAVE_BZIP2], [1], [if true, bz2 library is installed]) +fi +AC_MSG_CHECKING([whether libbz2 library is available]) +AC_MSG_RESULT([${have_bzip2}]) + +if test "x$have_bzip2" = "xno" ; then +AC_CHECK_LIB([bzip2],[BZ2_bzCompress],[have_bzip2=yes],[have_bzip2=no]) +if test "x$have_bzip2" = "xyes" ; then + AC_SEARCH_LIBS([BZ2_bzCompress],[bzip2 bzip2.dll cygbzip2.dll], [], []) + AC_DEFINE([HAVE_BZIP2], [1], [if true, bzip2 library is installed]) +fi +AC_MSG_CHECKING([whether libbzip2 library is available]) +AC_MSG_RESULT([${have_bzip2}]) +fi + +# Note that szip management is tricky. +# This is because we have three things to consider: +# 1. is libsz available? +# 2. is szip enabled in HDF5? +# 3. is nczarr enabled? +# We need separate flags for cases 1 and 2 + +# See if we have libsz (usually via libaec) +AC_CHECK_LIB([sz],[SZ_BufftoBuffCompress],[have_sz=yes],[have_sz=no]) +if test "x$have_sz" = "xyes" ; then + AC_SEARCH_LIBS([SZ_BufftoBuffCompress],[sz sz.dll cygsz.dll], [], []) + AC_DEFINE([HAVE_SZ], [1], [if true, libsz (==szip) is available]) fi -AC_MSG_CHECKING([whether libblosc library is available]) -AC_MSG_RESULT([${enable_blosc}]) +AC_MSG_CHECKING([whether libsz library is available]) +AC_MSG_RESULT([${have_sz}]) + +########## -# See if we have libzip +# See if we have libzip for NCZarr AC_CHECK_LIB([zip],[zip_open],[have_zip=yes],[have_zip=no]) if test "x$have_zip" = "xyes" ; then AC_SEARCH_LIBS([zip_open],[zip zip.dll cygzip.dll], [], []) @@ -1333,7 +1381,7 @@ fi # Set defaults hdf5_parallel=no hdf5_supports_par_filters=no -enable_szlib=no +enable_hdf5_szip=no has_ross3=no if test "x$enable_hdf5" = xyes; then @@ -1360,7 +1408,7 @@ if test "x$enable_hdf5" = xyes; then # H5Pset_fapl_mpiposix and H5Pget_fapl_mpiposix have been removed since HDF5 1.8.12. # Use H5Pset_fapl_mpio and H5Pget_fapl_mpio, instead. - AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5free_memory H5resize_memory H5allocate_memory H5Pset_all_coll_metadata_ops H5Literate]) + AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5Pset_all_coll_metadata_ops H5Literate]) # Check to see if HDF5 library has collective metadata APIs, (HDF5 >= 1.10.0) if test "x$ac_cv_func_H5Pset_all_coll_metadata_ops" = xyes; then @@ -1404,10 +1452,10 @@ if test "x$enable_hdf5" = xyes; then # must be able to link to szip library. AC_MSG_CHECKING([whether szlib was used when building HDF5]) if test "x$ac_cv_func_H5Z_SZIP" = xyes; then - enable_szlib=yes - AC_DEFINE([USE_SZIP], [1], [if true, compile in szip compression in netCDF-4 variables]) + enable_hdf5_szip=yes + AC_DEFINE([HAVE_H5Z_SZIP], [1], [if true, compile in szip compression in netCDF-4 variables]) fi - AC_MSG_RESULT([$enable_szlib]) + AC_MSG_RESULT([$enable_hdf5_szip]) # Check to see if HDF5 library is 1.10.6 or greater. # Used to control path name conversion @@ -1422,15 +1470,6 @@ if test "x$enable_hdf5" = xyes; then AC_DEFINE([HDF5_UTF8_PATHS], [1], [if true, HDF5 paths can be utf-8]) fi -else # !enable_hdf5 - # See if we have libsz - AC_CHECK_LIB([sz],[SZ_BufftoBuffCompress],[enable_szlib=yes],[enable_szlib=no]) - if test "x$enable_szlib" = "xyes" ; then - AC_SEARCH_LIBS([SZ_BufftoBuffCompress],[sz sz.dll cygsz.dll], [], []) - AC_DEFINE([USE_SZIP], [1], [if true, szip library is available]) - fi - AC_MSG_CHECKING([whether szip library is available]) - AC_MSG_RESULT([${enable_szlib}]) fi AM_CONDITIONAL(ENABLE_NCDUMPCHUNKS, [test "x$has_readchunks" = xyes ]) @@ -1635,6 +1674,12 @@ AC_MSG_WARN([--disable-filter-testing => --disable-nczarr-filter-testing]) enable_nczarr_filter_testing=no fi +if test "x$enable_nczarr" = xno; then +AC_MSG_WARN([--disable-nczarr => --disable-nczarr-filters]) +enable_nczarr_filters=no +enable_nczarr_filter_testing=no +fi + if test "x$enable_nczarr_filters" = xyes; then AC_DEFINE([ENABLE_NCZARR_FILTERS], [1], [if true, enable NCZarr filters]) fi @@ -1660,7 +1705,6 @@ AM_CONDITIONAL(ENABLE_CDF5, [test "x$enable_cdf5" = xyes]) AM_CONDITIONAL(ENABLE_DAP_REMOTE_TESTS, [test "x$enable_dap_remote_tests" = xyes]) AM_CONDITIONAL(ENABLE_DAP_AUTH_TESTS, [test "x$enable_dap_auth_tests" = xyes]) AM_CONDITIONAL(ENABLE_DAP_LONG_TESTS, [test "x$enable_dap_long_tests" = xyes]) -AM_CONDITIONAL(USE_SZIP, [test "x$ac_cv_func_H5Z_SZIP" = xyes]) AM_CONDITIONAL(USE_PNETCDF_DIR, [test ! "x$PNETCDFDIR" = x]) AM_CONDITIONAL(USE_LOGGING, [test "x$enable_logging" = xyes]) AM_CONDITIONAL(CROSS_COMPILING, [test "x$cross_compiling" = xyes]) @@ -1682,8 +1726,11 @@ AM_CONDITIONAL(HAS_PAR_FILTERS, [test x$hdf5_supports_par_filters = xyes ]) AM_CONDITIONAL(ENABLE_NCZARR, [test "x$enable_nczarr" = xyes]) AM_CONDITIONAL(ENABLE_S3_SDK, [test "x$enable_s3_sdk" = xyes]) AM_CONDITIONAL(HAS_MULTIFILTERS, [test "x$has_multifilters" = xyes]) -AM_CONDITIONAL(ENABLE_BLOSC, [test "x$enable_blosc" = xyes]) -AM_CONDITIONAL(ENABLE_SZIP, [test "x$enable_szip" = xyes]) +AM_CONDITIONAL(HAVE_SZ, [test "x$have_sz" = xyes]) +AM_CONDITIONAL(HAVE_H5Z_SZIP, [test "x$enable_hdf5_szip" = xyes]) +AM_CONDITIONAL(HAVE_BLOSC, [test "x$have_blosc" = xyes]) +AM_CONDITIONAL(HAVE_BZIP2, [test "x$have_bzip2" = xyes]) +AM_CONDITIONAL(HAVE_ZSTD, [test "x$have_zstd" = xyes]) # If the machine doesn't have a long long, and we want netCDF-4, then # we've got problems! @@ -1771,8 +1818,6 @@ AC_SUBST(HAS_HDF4,[$enable_hdf4]) AC_SUBST(HAS_HDF5,[$enable_hdf5]) AC_SUBST(HAS_PNETCDF,[$enable_pnetcdf]) AC_SUBST(HAS_LOGGING, [$enable_logging]) -AC_SUBST(HAS_SZLIB,[$enable_szlib]) -AC_SUBST(HAS_SZLIB_WRITE, [$enable_szlib]) AC_SUBST(HAS_PARALLEL,[$enable_parallel]) AC_SUBST(HAS_PARALLEL4,[$enable_parallel4]) AC_SUBST(HAS_DISKLESS,[yes]) @@ -1789,8 +1834,29 @@ AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip]) AC_SUBST([HAS_QUANTIZE],[yes]) AC_SUBST(HAS_LOGGING,[$enable_logging]) AC_SUBST(DO_FILTER_TESTS,[$enable_filter_testing]) -AC_SUBST(HAVE_BLOSC,[$enable_blosc]) -AC_SUBST(HAVE_SZIP,[$enable_szip]) +AC_SUBST(HAS_SZLIB,[$have_sz]) +AC_SUBST(HAS_SZLIB_WRITE, [$have_sz]) + +# Always available +std_filters="deflate,bzip2" +if test "x$enable_szlib" = xyes ; then +std_filters="${std_filters},szip" +fi +# We need to include szip iff HDF5 && HDF5_HAS_SZIP || !HDF5 && NCZARR && libsz +if test "x$enable_hdf5" = xyes && test "x$enable_hdf5_szip" = xyes ; then + std_filters="${std_filters},szip" +fi +if test "x$enable_hdf5" = xno && test "x$have_sz" = xyes ; then +std_filters="${std_filters},szip" +fi +if test "x$have_blosc" = xyes ; then +std_filters="${std_filters},blosc" +fi +if test "x$have_zstd" = xyes ; then +std_filters="${std_filters},zst" +fi + +AC_SUBST(STD_FILTERS,[$std_filters]) # Access netcdf specific version of config.h AH_BOTTOM([#include "ncconfigure.h"]) @@ -1838,7 +1904,6 @@ AX_SET_META([NC_HAS_NC2],[$nc_build_v2],[yes]) AX_SET_META([NC_HAS_NC4],[$enable_netcdf_4],[yes]) AX_SET_META([NC_HAS_HDF4],[$enable_hdf4],[yes]) AX_SET_META([NC_HAS_HDF5],[$enable_hdf5],[yes]) -AX_SET_META([NC_HAS_SZIP],[$ac_cv_func_H5Z_SZIP],[yes]) AX_SET_META([NC_HAS_DAP2],[$enable_dap],[yes]) AX_SET_META([NC_HAS_DAP4],[$enable_dap4],[yes]) AX_SET_META([NC_HAS_DISKLESS],[yes],[yes]) @@ -1854,6 +1919,7 @@ AX_SET_META([NC_HAS_BYTERANGE],[$enable_byterange],[yes]) AX_SET_META([NC_HAS_NCZARR],[$enable_nczarr],[yes]) AX_SET_META([NC_HAS_MULTIFILTERS],[$has_multifilters],[yes]) AX_SET_META([NC_HAS_LOGGING],[$enable_logging],[yes]) +AX_SET_META([NC_HAS_SZIP],[$enable_hdf5_szip],[yes]) # This is the version of the dispatch table. If the dispatch table is # changed, this should be incremented, so that user-defined format @@ -1861,7 +1927,7 @@ AX_SET_META([NC_HAS_LOGGING],[$enable_logging],[yes]) # dispatch table to submit. If this is changed, make sure the value in # CMakeLists.txt also changes to match. -AC_SUBST([NC_DISPATCH_VERSION], [4]) +AC_SUBST([NC_DISPATCH_VERSION], [5]) AC_DEFINE_UNQUOTED([NC_DISPATCH_VERSION], [${NC_DISPATCH_VERSION}], [Dispatch table version.]) ##### diff --git a/h5_test/tst_h_enums.c b/h5_test/tst_h_enums.c index cb5326e68d..352b4dce3b 100644 --- a/h5_test/tst_h_enums.c +++ b/h5_test/tst_h_enums.c @@ -121,11 +121,7 @@ main() if (the_value != val[i]) ERR; member_name = H5Tget_member_name(typeid, i); if (strcmp(member_name, love_how[i])) ERR; -#ifdef HAVE_H5FREE_MEMORY H5free_memory(member_name); -#else - free(member_name); -#endif } /* Now read the data in the attribute and make sure it's what we @@ -239,11 +235,7 @@ main() if (the_value != val[i]) ERR; member_name = H5Tget_member_name(typeid, i); if (strcmp(member_name, lang[i])) ERR; -#ifdef HAVE_H5FREE_MEMORY H5free_memory(member_name); -#else - free(member_name); -#endif } /* Now read the data in the dataset and make sure it's what we diff --git a/h5_test/tst_h_par_compress.c b/h5_test/tst_h_par_compress.c index c09ef1cc4c..6a15e6a810 100644 --- a/h5_test/tst_h_par_compress.c +++ b/h5_test/tst_h_par_compress.c @@ -25,11 +25,11 @@ /* The following code, when uncommented, adds szip testing for * parallel I/O. However, this currently fails. I have a support * request in to HDF5 about this. Ed 7/8/20 */ -/* #ifdef USE_SZIP */ +/* #ifdef HAVE_H5Z_SZIP */ /* #define NUM_COMPRESS_FILTERS 2 */ /* #else */ /* #define NUM_COMPRESS_FILTERS 1 */ -/* #endif /\* USE_SZIP *\/ */ +/* #endif /\* HAVE_H5Z_SZIP *\/ */ #define NUM_COMPRESS_FILTERS 1 int diff --git a/h5_test/tst_h_vars.c b/h5_test/tst_h_vars.c index 6465235c94..b6e2294d8f 100644 --- a/h5_test/tst_h_vars.c +++ b/h5_test/tst_h_vars.c @@ -409,7 +409,7 @@ main() } SUMMARIZE_ERR; -#ifdef USE_SZIP +#ifdef HAVE_H5Z_SZIP printf("*** Checking szip functionality..."); #define SZIP_VAR_NAME "szip_var" #define SZIP_DIM1_LEN 32 @@ -596,6 +596,6 @@ main() } /* next file */ } SUMMARIZE_ERR; -#endif /* USE_SZIP */ +#endif /* HAVE_H5Z_SZIP */ FINAL_RESULTS; } diff --git a/include/Makefile.am b/include/Makefile.am index ac4067ca8c..0a4a58e050 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -20,7 +20,7 @@ nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h \ ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h \ ncpathmgr.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h \ hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h \ -ncfilter.h ncjson.h ncxml.h ncs3sdk.h +ncjson.h ncxml.h ncs3sdk.h if USE_DAP noinst_HEADERS += ncdap.h diff --git a/include/hdf5internal.h b/include/hdf5internal.h index 9b8a4b7e17..1a15a172ba 100644 --- a/include/hdf5internal.h +++ b/include/hdf5internal.h @@ -182,6 +182,7 @@ int nc4_hdf5_get_chunk_cache(int ncid, size_t *sizep, size_t *nelemsp, int NC4_hdf5_def_var_filter(int ncid, int varid, unsigned int filterid, size_t nparams, const unsigned int *params); int NC4_hdf5_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int *filterids); int NC4_hdf5_inq_var_filter_info(int ncid, int varid, unsigned int filterid, size_t* nparamsp, unsigned int *params); +int NC4_hdf5_inq_filter_avail(int ncid, unsigned id); /* Filterlist management */ diff --git a/include/nc4internal.h b/include/nc4internal.h index 8d89060329..b78a69c07e 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -204,8 +204,6 @@ typedef struct NC_VAR_INFO int storage; /**< Storage of this var, compact, contiguous, or chunked. */ int endianness; /**< What endianness for the var? */ int parallel_access; /**< Type of parallel access for I/O on variable (collective or independent). */ - nc_bool_t shuffle; /**< True if var has shuffle filter applied. */ - nc_bool_t fletcher32; /**< True if var has fletcher32 filter applied. */ struct ChunkCache { size_t size; /**< Size in bytes of the var chunk cache. */ size_t nelems; /**< Number of slots in var chunk cache. */ diff --git a/include/ncconfigure.h b/include/ncconfigure.h index b58bd379b3..8a8e8595a5 100644 --- a/include/ncconfigure.h +++ b/include/ncconfigure.h @@ -152,13 +152,4 @@ typedef long long fileoffset_t; #define NC_UNUSED(var) (void)var #endif -/* Protect old HDF5 code (pre 1.8.12) */ -#ifndef HAVE_H5ALLOCATE_MEMORY -#ifndef H5allocate_memory -#define H5allocate_memory(size,clear) ((clear)?calloc(1,(size)):malloc(size)) -#define H5free_memory(buf) free(buf) -#define H5resize_memory(buf,size) realloc(buf,size) -#endif -#endif - #endif /* NCCONFIGURE_H */ diff --git a/include/ncfilter.h b/include/ncfilter.h deleted file mode 100644 index 8d9a5fc456..0000000000 --- a/include/ncfilter.h +++ /dev/null @@ -1,20 +0,0 @@ -/* Copyright 2018-2018 University Corporation for Atmospheric - Research/Unidata. */ -/** - * @file - * @internal This header file contains macros, types and prototypes - * used to build and manipulate filters. It contains definitions - * for multiple file formats. - * - * @author Dennis Heimbigner - */ - -#ifndef NCFILTER_H -#define NCFILTER_H - - -/**************************************************/ -/* Internal filter related structures */ -/* Obsolete */ - -#endif /*NCFILTER_H*/ diff --git a/include/nchttp.h b/include/nchttp.h index 0fc01351e2..82f7150654 100644 --- a/include/nchttp.h +++ b/include/nchttp.h @@ -34,7 +34,7 @@ typedef struct NC_HTTP_STATE { extern int nc_http_init(NC_HTTP_STATE** state); extern int nc_http_init_verbose(NC_HTTP_STATE** state, int verbose); -extern int nc_http_size(NC_HTTP_STATE* state, const char* url, long long unsigned* sizep); +extern int nc_http_size(NC_HTTP_STATE* state, const char* url, long long* sizep); extern int nc_http_read(NC_HTTP_STATE* state, const char* url, size64_t start, size64_t count, NCbytes* buf); extern int nc_http_write(NC_HTTP_STATE* state, const char* url, NCbytes* payload); extern int nc_http_close(NC_HTTP_STATE* state); diff --git a/include/netcdf_dispatch.h.in b/include/netcdf_dispatch.h.in index c08dea0b9b..95b1bc6002 100644 --- a/include/netcdf_dispatch.h.in +++ b/include/netcdf_dispatch.h.in @@ -150,6 +150,8 @@ struct NC_Dispatch /* Version 4 Add quantization. */ int (*def_var_quantize)(int ncid, int varid, int quantize_mode, int nsd); int (*inq_var_quantize)(int ncid, int varid, int *quantize_modep, int *nsdp); + /* Version 5 adds filter availability */ + int (*inq_filter_avail)(int ncid, unsigned id); }; #if defined(__cplusplus) @@ -189,6 +191,7 @@ extern "C" { EXTERNL int NC_NOTNC4_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams, unsigned int* params); EXTERNL int NC_NOOP_inq_var_filter_ids(int ncid, int varid, size_t* nfilters, unsigned int* filterids); EXTERNL int NC_NOOP_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams, unsigned int* params); + EXTERNL int NC_NOOP_inq_filter_avail(int ncid, unsigned id); EXTERNL int NC_NOTNC4_def_grp(int, const char *, int *); EXTERNL int NC_NOTNC4_rename_grp(int, const char *); diff --git a/include/netcdf_filter.h b/include/netcdf_filter.h index b68b9c2ac1..8b52b73039 100644 --- a/include/netcdf_filter.h +++ b/include/netcdf_filter.h @@ -17,14 +17,20 @@ /* Must match values in */ #ifndef H5Z_FILTER_DEFLATE #define H5Z_FILTER_DEFLATE 1 +#define H5Z_FILTER_SHUFFLE 2 +#define H5Z_FILTER_FLETCHER32 3 #endif + #ifndef H5Z_FILTER_SZIP #define H5Z_FILTER_SZIP 4 #define H5_SZIP_ALLOW_K13_OPTION_MASK 1 #define H5_SZIP_CHIP_OPTION_MASK 2 #define H5_SZIP_EC_OPTION_MASK 4 +#define H5_SZIP_LSB_OPTION_MASK 8 +#define H5_SZIP_MSB_OPTION_MASK 16 #define H5_SZIP_NN_OPTION_MASK 32 #define H5_SZIP_MAX_PIXELS_PER_BLOCK 32 +#define H5_SZIP_RAW_OPTION_MASK 128 #define NC_SZIP_EC 4 /**< Selects entropy coding method for szip. */ #define NC_SZIP_NN 32 /**< Selects nearest neighbor coding method for szip. */ @@ -35,6 +41,29 @@ /** The maximum allowed setting for pixels_per_block when calling nc_def_var_szip(). */ #define NC_MAX_PIXELS_PER_BLOCK 32 +/* Other Standard Filters */ +#ifndef H5Z_FILTER_ZSTD +#define H5Z_FILTER_ZSTD 32015 +#endif +#ifndef H5Z_FILTER_BZIP2 +#define H5Z_FILTER_BZIP2 307 +#endif +#ifndef H5Z_FILTER_BLOSC +#define H5Z_FILTER_BLOSC 32001 +#endif +#ifndef BLOSC_SHUFFLE +enum BLOSC_SHUFFLE { +BLOSC_NOSHUFFLE=0, /* no shuffle */ +BLOSC_SHUFFLE=1, /* byte-wise shuffle */ +BLOSC_BITSHUFFLE=2 /* bit-wise shuffle */ +}; +#endif + +/*Assumes use of c-blosc library */ +#ifndef BLOSC_BLOSCLZ +enum BLOSC_SUBCOMPRESSORS {BLOSC_LZ=0, BLOSC_LZ4=1, BLOSC_LZ4HC=2, BLOSC_SNAPPY=3, BLOSC_ZLIB=4, BLOSC_ZSTD=5}; +#endif + #if defined(__cplusplus) extern "C" { #endif @@ -60,6 +89,26 @@ EXTERNL int nc_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* /* End HDF5 Format Declarations */ +/**************************************************/ +/* Allow user to interrogate if a filter is available*/ +/* Would prefer to get list of all known filters, but HDF5 does not support that */ + +/* See if filter is available */ +EXTERNL int nc_inq_filter_avail(int ncid, unsigned id); + +/**************************************************/ +/* Functions for accessing standardized filters */ + +EXTERNL int nc_def_var_bzip2(int ncid, int varid, int level); +EXTERNL int nc_inq_var_bzip2(int ncid, int varid, int* hasfilterp, int *levelp); + +/* Note use of 'zstandard' instead of 'zstd' */ +EXTERNL int nc_def_var_zstandard(int ncid, int varid, int level); +EXTERNL int nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *levelp); + +EXTERNL int nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, unsigned blocksize, unsigned addshuffle); +EXTERNL int nc_inq_var_blosc(int ncid, int varid, int* hasfilterp, unsigned* subcompressorp, unsigned* levelp, unsigned* blocksizep, unsigned* addshufflep); + #if defined(__cplusplus) } #endif diff --git a/include/netcdf_filter_hdf5_build.h b/include/netcdf_filter_hdf5_build.h index 8db64f72bf..24000c7b8c 100644 --- a/include/netcdf_filter_hdf5_build.h +++ b/include/netcdf_filter_hdf5_build.h @@ -204,4 +204,17 @@ typedef const void* (*H5PL_get_plugin_info_proto)(void); } #endif +/* Protect old HDF5 code (pre 1.8.12) */ +#ifdef USE_HDF5 +# if H5_VERSION_LE(1,8,11) +# define H5allocate_memory(size,clear) ((clear)?calloc(1,(size)):malloc(size)) +# define H5free_memory(buf) free(buf) +# define H5resize_memory(buf,size) realloc(buf,size) +# endif +#else +# define H5allocate_memory(size,clear) ((clear)?calloc(1,(size)):malloc(size)) +# define H5free_memory(buf) free(buf) +# define H5resize_memory(buf,size) realloc(buf,size) +#endif + #endif /*NETCDF_FILTER_HDF5_BUILD_H*/ diff --git a/lib_flags.am b/lib_flags.am index 050f4b6d9d..3e190d4727 100644 --- a/lib_flags.am +++ b/lib_flags.am @@ -13,6 +13,10 @@ if USE_DAP AM_CPPFLAGS += -I${top_srcdir}/oc2 endif +if ENABLE_NCZARR +AM_CPPFLAGS += -I${top_srcdir}/libnczarr +endif + if ENABLE_S3_SDK AM_LDFLAGS += -lstdc++ endif diff --git a/libdap2/ncd2dispatch.c b/libdap2/ncd2dispatch.c index c2105d9a3c..519193def4 100644 --- a/libdap2/ncd2dispatch.c +++ b/libdap2/ncd2dispatch.c @@ -7,6 +7,7 @@ #include "ncd2dispatch.h" #include "ncrc.h" #include "ncoffsets.h" +#include "netcdf_dispatch.h" #ifdef DEBUG2 #include "dapdump.h" #endif @@ -181,6 +182,7 @@ NC_NOOP_inq_var_filter_info, NC_NOTNC4_def_var_quantize, NC_NOTNC4_inq_var_quantize, +NC_NOOP_inq_filter_avail, }; const NC_Dispatch* NCD2_dispatch_table = NULL; /* moved here from ddispatch.c */ diff --git a/libdap4/ncd4dispatch.c b/libdap4/ncd4dispatch.c index 7f85759c8e..f875c641e0 100644 --- a/libdap4/ncd4dispatch.c +++ b/libdap4/ncd4dispatch.c @@ -799,6 +799,18 @@ NCD4_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams, return (ret); } +static int +NCD4_inq_filter_avail(int ncid, unsigned id) +{ + NC* ncp; + int ret; + int substrateid; + if((ret = NC_check_id(ncid, (NC**)&ncp)) != NC_NOERR) return (ret); + substrateid = makenc4id(ncp,ncid); + ret = nc_inq_filter_avail(substrateid, id); + return (ret); +} + /**************************************************/ /* Following functions are overridden to handle @@ -1011,4 +1023,6 @@ NCD4_inq_var_filter_info, NC_NOTNC4_def_var_quantize, NCD4_inq_var_quantize, + +NCD4_inq_filter_avail, }; diff --git a/libdispatch/CMakeLists.txt b/libdispatch/CMakeLists.txt index dcb69c6484..e89e185524 100644 --- a/libdispatch/CMakeLists.txt +++ b/libdispatch/CMakeLists.txt @@ -21,7 +21,7 @@ ENDIF(ENABLE_BYTERANGE) IF(ENABLE_S3_SDK) SET(libdispatch_SOURCES ${libdispatch_SOURCES} ncs3sdk.cpp awsincludes.h) -ENDIF(ENABLE_S3_SDK) +ENDIF() IF(REGEDIT) SET(libdispatch_SOURCES ${libdispatch_SOURCES} dreg.c) @@ -36,6 +36,10 @@ IF(MPI_C_LIBRARIES) target_link_libraries(dispatch PUBLIC ${MPI_C_LIBRARIES}) ENDIF(MPI_C_LIBRARIES) +IF(ENABLE_NCZARR) + target_include_directories(dispatch PUBLIC ../libnczarr) +ENDIF(ENABLE_NCZARR) + IF(ENABLE_S3_SDK) target_include_directories(dispatch PUBLIC ${AWSSDK_INCLUDE_DIRS}) IF(NOT MSVC) diff --git a/libdispatch/derror.c b/libdispatch/derror.c index 921d15a573..3b4ebd2b6c 100644 --- a/libdispatch/derror.c +++ b/libdispatch/derror.c @@ -261,7 +261,7 @@ const char *nc_strerror(int ncerr1) case NC_EFILTER: return "NetCDF: Filter error: bad id or parameters or duplicate filter"; case NC_ENOFILTER: - return "NetCDF: Filter error: unimplemented filter encountered"; + return "NetCDF: Filter error: undefined filter encountered"; case NC_ECANTEXTEND: return "NetCDF: Attempt to extend dataset during NC_INDEPENDENT I/O operation. Use nc_var_par_access to set mode NC_COLLECTIVE before extending variable."; case NC_EMPI: return "NetCDF: MPI operation failed."; diff --git a/libdispatch/dfilter.c b/libdispatch/dfilter.c index e3cf9270da..f953402bce 100644 --- a/libdispatch/dfilter.c +++ b/libdispatch/dfilter.c @@ -20,6 +20,10 @@ #include "hdf5internal.h" #endif +#ifdef ENABLE_NCZARR +#include "zdispatch.h" +#endif + /* Unified filter related code */ @@ -189,6 +193,30 @@ nc_inq_var_filter(int ncid, int varid, unsigned int* idp, size_t* nparamsp, unsi return stat; } +/**************************************************/ +/* Test if filter is available. Would prefer + returning a list of all available filters, but HDF5 + does not support that capability. + @param file for which filter list is desired + @param id filter id of interest + @return NC_NOERR if the filter is available + @return NC_EBADID if ncid is invalid + @return NC_ENOFILTER if filter is not available. +*/ + +EXTERNL int +nc_inq_filter_avail(int ncid, unsigned id) +{ + int stat = NC_NOERR; + NC* ncp; + + stat = NC_check_id(ncid,&ncp); + if(stat != NC_NOERR) return stat; + if((stat = ncp->dispatch->inq_filter_avail(ncid,id))) goto done; +done: + return stat; +} + /**************************************************/ /* Support direct user defined filters */ @@ -255,3 +283,155 @@ int stat = NC_NOERR; } #endif /*ENABLE_CLIENTSIDE_FILTERS*/ +/**************************************************/ +/* Functions for accessing standardized filters */ + +int +nc_def_var_bzip2(int ncid, int varid, int level) +{ + int stat = NC_NOERR; + unsigned ulevel; + + if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BZIP2))) goto done; + /* Filter is available */ + /* 1 <= Level <= 9 */ + if (level < 1 || level > 9) + return NC_EINVAL; + ulevel = (unsigned) level; /* Keep bit pattern */ + if((stat = nc_def_var_filter(ncid,varid,H5Z_FILTER_BZIP2,1,&ulevel))) goto done; +done: + return stat; +} + +int +nc_inq_var_bzip2(int ncid, int varid, int* hasfilterp, int *levelp) +{ + int stat = NC_NOERR; + size_t nparams; + unsigned params = 0; + int hasfilter = 0; + + if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BZIP2))) goto done; + /* Filter is available */ + /* Get filter info */ + stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_BZIP2,&nparams,NULL); + if(stat == NC_ENOFILTER) {stat = NC_NOERR; hasfilter = 0; goto done;} + if(stat != NC_NOERR) goto done; + hasfilter = 1; + if(nparams != 1) {stat = NC_EFILTER; goto done;} + if((stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_BZIP2,&nparams,¶ms))) goto done; +done: + if(levelp) *levelp = (int)params; + if(hasfilterp) *hasfilterp = hasfilter; + return stat; +} + +int +nc_def_var_zstandard(int ncid, int varid, int level) +{ +#ifdef HAVE_ZSTD + int stat = NC_NOERR; + unsigned ulevel; + + if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_ZSTD))) goto done; + /* Filter is available */ + /* Level must be between -131072 and 22 on Zstandard v. 1.4.5 (~202009) + Earlier versions have fewer levels (especially fewer negative levels) */ + if (level < -131072 || level > 22) + return NC_EINVAL; + ulevel = (unsigned) level; /* Keep bit pattern */ + if((stat = nc_def_var_filter(ncid,varid,H5Z_FILTER_ZSTD,1,&ulevel))) goto done; +done: + return stat; +#else + return NC_NOERR; +#endif /*HAVE_ZSTD*/ +} + +int +nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *levelp) +{ +#ifdef HAVE_ZSTD + int stat = NC_NOERR; + size_t nparams; + unsigned params = 0; + int hasfilter = 0; + + if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_ZSTD))) goto done; + /* Filter is available */ + /* Get filter info */ + stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_ZSTD,&nparams,NULL); + if(stat == NC_ENOFILTER) {stat = NC_NOERR; hasfilter = 0; goto done;} + if(stat != NC_NOERR) goto done; + hasfilter = 1; + if(nparams != 1) {stat = NC_EFILTER; goto done;} + if((stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_ZSTD,&nparams,¶ms))) goto done; +done: + if(levelp) *levelp = (int)params; + if(hasfilterp) *hasfilterp = hasfilter; + return stat; +#else + return NC_NOERR; +#endif /*HAVE_ZSTD*/ +} + +int +nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, unsigned blocksize, unsigned addshuffle) +{ +#ifdef HAVE_BLOSC + int stat = NC_NOERR; + unsigned params[7];; + + if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BLOSC))) goto done; + /* Filter is available */ + + /* Verify parameters */ + if(addshuffle > (unsigned)BLOSC_BITSHUFFLE) {stat = NC_EINVAL; goto done;} + if(subcompressor > (unsigned)BLOSC_ZSTD) {stat = NC_EINVAL; goto done;} + + /* Set the parameters */ + params[0] = 0; + params[1] = 0; + params[2] = 0; + params[3] = blocksize; + params[4] = level; + params[5] = addshuffle; + params[6] = subcompressor; + if((stat = nc_def_var_filter(ncid,varid,H5Z_FILTER_BLOSC,7,params))) goto done; +done: + return stat; +#else + return NC_NOERR; +#endif +} + +int +nc_inq_var_blosc(int ncid, int varid, int* hasfilterp, unsigned* subcompressorp, unsigned* levelp, unsigned* blocksizep, unsigned* addshufflep) +{ +#ifdef HAVE_BLOSC + int stat = NC_NOERR; + size_t nparams; + unsigned params[7]; + int hasfilter = 0; + + if((stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BLOSC))) goto done; + /* Filter is available */ + + /* Get filter info */ + stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_BLOSC,&nparams,NULL); + if(stat == NC_ENOFILTER) {stat = NC_NOERR; hasfilter = 0; goto done;} + if(stat != NC_NOERR) goto done; + hasfilter = 1; + if(nparams != 7) {stat = NC_EFILTER; goto done;} + if((stat = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_BLOSC,&nparams,params))) goto done; + if(blocksizep) *blocksizep = params[3]; + if(levelp) *levelp = params[4]; + if(addshufflep) *addshufflep = params[5]; + if(subcompressorp) *subcompressorp = params[6]; +done: + if(hasfilterp) *hasfilterp = hasfilter; + return stat; +#else + return NC_NOERR; +#endif +} diff --git a/libdispatch/dhttp.c b/libdispatch/dhttp.c index 95fcd059a9..250ddab171 100644 --- a/libdispatch/dhttp.c +++ b/libdispatch/dhttp.c @@ -276,7 +276,7 @@ Assume URL etc has already been set. */ int -nc_http_size(NC_HTTP_STATE* state, const char* objecturl, long long unsigned* sizep) +nc_http_size(NC_HTTP_STATE* state, const char* objecturl, long long* sizep) { int stat = NC_NOERR; const char* hdr = NULL; diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index fb58d18c23..e98ff912f8 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -52,7 +52,7 @@ struct MagicFile { struct NCURI* uri; int omode; NCmodel* model; - long long unsigned filelen; + long long filelen; int use_parallel; int iss3; void* parameters; /* !NULL if inmemory && !diskless */ diff --git a/libdispatch/dnotnc4.c b/libdispatch/dnotnc4.c index 5c66364f63..e91872eca5 100644 --- a/libdispatch/dnotnc4.c +++ b/libdispatch/dnotnc4.c @@ -97,6 +97,13 @@ NC_NOOP_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparam return NC_ENOFILTER; } +int +NC_NOOP_inq_filter_avail(int ncid, unsigned id) +{ + NC_UNUSED(ncid); + return NC_ENOFILTER; +} + /** * @internal Not allowed for classic model. * diff --git a/libhdf4/hdf4dispatch.c b/libhdf4/hdf4dispatch.c index feff5ad724..1a41d123c7 100644 --- a/libhdf4/hdf4dispatch.c +++ b/libhdf4/hdf4dispatch.c @@ -13,6 +13,7 @@ #include "hdf4dispatch.h" #include "nc4dispatch.h" #include "hdf5dispatch.h" +#include "netcdf_filter.h" /* This is the dispatch object that holds pointers to all the * functions that make up the HDF4 dispatch interface. */ @@ -106,6 +107,11 @@ static const NC_Dispatch HDF4_dispatcher = { NC_NOOP_inq_var_filter_ids, NC_NOOP_inq_var_filter_info, + + NC_NOTNC4_def_var_quantize, + NC_NOTNC4_inq_var_quantize, + + NC_NOOP_inq_filter_avail, }; const NC_Dispatch *HDF4_dispatch_table = NULL; diff --git a/libhdf5/H5FDhttp.c b/libhdf5/H5FDhttp.c index 1a18f708a7..81dba1f5bf 100644 --- a/libhdf5/H5FDhttp.c +++ b/libhdf5/H5FDhttp.c @@ -49,6 +49,7 @@ /* Define a simple #ifdef test for the version of H5FD_class_t we are using */ + #if H5_VERS_MAJOR == 1 #if H5_VERS_MINOR < 10 #define H5FDCLASS1 1 diff --git a/libhdf5/hdf5dispatch.c b/libhdf5/hdf5dispatch.c index dc16de6555..130f0f0766 100644 --- a/libhdf5/hdf5dispatch.c +++ b/libhdf5/hdf5dispatch.c @@ -109,6 +109,7 @@ static const NC_Dispatch HDF5_dispatcher = { NC4_def_var_quantize, NC4_inq_var_quantize, + NC4_hdf5_inq_filter_avail, }; const NC_Dispatch* HDF5_dispatch_table = NULL; /* moved here from ddispatch.c */ diff --git a/libhdf5/hdf5filter.c b/libhdf5/hdf5filter.c index dc38fd2322..40884503a1 100644 --- a/libhdf5/hdf5filter.c +++ b/libhdf5/hdf5filter.c @@ -287,7 +287,18 @@ NC4_hdf5_addfilter(NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const un } fi->flags = flags; if(!olddef) { - nclistpush(flist,fi); + size_t pos = nclistlength(flist); + /* Need to be careful about where we insert fletcher32 and shuffle */ + if(nclistlength(flist) > 0) { + if(id == H5Z_FILTER_FLETCHER32) + pos = 0; /* alway first filter */ + else if(id == H5Z_FILTER_SHUFFLE) { + /* See if first filter is fletcher32 */ + struct NC_HDF5_Filter* f0 = (struct NC_HDF5_Filter*)nclistget(flist,0); + if(f0->filterid == H5Z_FILTER_FLETCHER32) pos = 1; else pos = 0; + } + } + nclistinsert(flist,pos,fi); PRINTFILTERLIST(var,"add"); } fi = NULL; /* either way,its in the var->filters list */ @@ -354,10 +365,8 @@ NC4_hdf5_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, struct NC_HDF5_Filter* oldspec = NULL; int flags = 0; htri_t avail = -1; -#ifdef HAVE_H5Z_SZIP int havedeflate = 0; int haveszip = 0; -#endif LOG((2, "%s: ncid 0x%x varid %d", __func__, ncid, varid)); @@ -386,13 +395,20 @@ NC4_hdf5_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, #endif /* HDF5_SUPPORTS_PAR_FILTERS */ #endif /* USE_PARALLEL */ - /* Lookup incoming id to see if already defined */ + /* See if this filter is missing or not */ + if((avail = H5Zfilter_avail(id)) < 0) + {stat = NC_EHDFERR; goto done;} /* Something in HDF5 went wrong */ + if(avail == 0) + {stat = NC_ENOFILTER; goto done;} /* filter not available */ + + /* Lookup incoming id to see if already defined for this variable*/ switch((stat=NC4_hdf5_filter_lookup(var,id,&oldspec))) { case NC_NOERR: break; /* already defined */ case NC_ENOFILTER: break; /*not defined*/ default: goto done; } -#ifdef HAVE_H5Z_SZIP + stat = NC_NOERR; /* reset */ + /* See if deflate &/or szip is defined */ switch ((stat = NC4_hdf5_filter_lookup(var,H5Z_FILTER_DEFLATE,NULL))) { case NC_NOERR: havedeflate = 1; break; @@ -404,11 +420,8 @@ NC4_hdf5_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, case NC_ENOFILTER: haveszip = 0; break; default: goto done; } -#endif /* HAVE_H5Z_SZIP */ + stat = NC_NOERR; /* reset */ - /* See if this filter is missing or not */ - if((avail = H5Zfilter_avail(id)) < 0) - {stat = NC_EHDFERR; goto done;} /* Something in HDF5 went wrong */ if(!avail) { NC_HDF5_VAR_INFO_T* hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info; flags |= NC_HDF5_FILTER_MISSING; @@ -425,25 +438,18 @@ NC4_hdf5_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, level = (int)params[0]; if (level < NC_MIN_DEFLATE_LEVEL || level > NC_MAX_DEFLATE_LEVEL) {stat = THROW(NC_EINVAL); goto done;} -#ifdef HAVE_H5Z_SZIP /* If szip compression is already applied, return error. */ if(haveszip) {stat = THROW(NC_EINVAL); goto done;} -#endif } -#ifdef HAVE_H5Z_SZIP if(id == H5Z_FILTER_SZIP) { /* Do error checking */ if(nparams != 2) {stat = THROW(NC_EFILTER); goto done;}/* incorrect no. of parameters */ - /* Pixels per block must be an even number, < 32. */ + /* Pixels per block must be an even number, <= 32. */ if (params[1] % 2 || params[1] > NC_MAX_PIXELS_PER_BLOCK) {stat = THROW(NC_EINVAL); goto done;} /* If zlib compression is already applied, return error. */ if(havedeflate) {stat = THROW(NC_EINVAL); goto done;} } -#else /*!HAVE_H5Z_SZIP*/ - if(id == H5Z_FILTER_SZIP) - {stat = THROW(NC_EFILTER); goto done;} /* Not allowed */ -#endif /* Filter => chunking */ var->storage = NC_CHUNKED; /* Determine default chunksizes for this variable unless already specified */ @@ -456,7 +462,6 @@ NC4_hdf5_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, goto done; } } -#ifdef HAVE_H5Z_SZIP /* More error checking */ if(id == H5Z_FILTER_SZIP) { /* szip X chunking error checking */ /* For szip, the pixels_per_block parameter must not be greater @@ -470,7 +475,6 @@ NC4_hdf5_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, if (params[1] > num_elem) {stat = THROW(NC_EINVAL); goto done;} } -#endif /* addfilter can handle case where filter is already defined, and will just replace parameters */ if((stat = NC4_hdf5_addfilter(var,id,nparams,params,flags))) goto done; @@ -588,3 +592,18 @@ NC4_hdf5_filter_finalize(void) { return NC_NOERR; } + +/* Test if filter available */ +int +NC4_hdf5_inq_filter_avail(int ncid, unsigned id) +{ + int stat = NC_NOERR; + htri_t avail = -1; + NC_UNUSED(ncid); + /* See if this filter is available or not */ + if((avail = H5Zfilter_avail(id)) < 0) + {stat = NC_EHDFERR; goto done;} /* Something in HDF5 went wrong */ + if(avail == 0) stat = NC_ENOFILTER; +done: + return stat; +} diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 1fe4bed9de..5ca07c642d 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -16,7 +16,6 @@ #include "ncrc.h" #include "ncauth.h" #include "ncmodel.h" -#include "ncfilter.h" #include "ncpathmgr.h" #ifdef ENABLE_BYTERANGE @@ -1087,14 +1086,6 @@ static int get_filter_info(hid_t propid, NC_VAR_INFO_T *var) {stat = NC_EHDFERR; goto done;} /* Something in HDF5 went wrong */ switch (filter) { - case H5Z_FILTER_SHUFFLE: - var->shuffle = NC_TRUE; - break; - - case H5Z_FILTER_FLETCHER32: - var->fletcher32 = NC_TRUE; - break; - case H5Z_FILTER_DEFLATE: if (cd_nelems != CD_NELEMS_ZLIB || cd_values[0] > NC_MAX_DEFLATE_LEVEL) @@ -2015,13 +2006,7 @@ hdf5free(void* memory) #ifndef JNA /* On Windows using the microsoft runtime, it is an error for one library to free memory allocated by a different library.*/ -#ifdef HAVE_H5FREE_MEMORY if(memory != NULL) H5free_memory(memory); -#else -#ifndef _MSC_VER - if(memory != NULL) free(memory); -#endif -#endif #endif } diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index fd3d06deb7..e25c106fcc 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -533,31 +533,30 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, } /* Shuffle filter? */ - if (shuffle) - { - if(*shuffle) var->shuffle = *shuffle; /* Once set, cannot be unset */ - if(var->shuffle) - var->storage = NC_CHUNKED; + if (shuffle && *shuffle) { + retval = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_SHUFFLE,NULL,NULL); + if(!retval || retval == NC_ENOFILTER) { + if((retval = nc_def_var_filter(ncid,varid,H5Z_FILTER_SHUFFLE,0,NULL))) return retval; + var->storage = NC_CHUNKED; + } } /* Fletcher32 checksum error protection? */ - if (fletcher32) - { - if(*fletcher32) var->fletcher32 = *fletcher32; /* cannot be unset */ - if(var->fletcher32) + if (fletcher32 && fletcher32) { + retval = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_FLETCHER32,NULL,NULL); + if(!retval || retval == NC_ENOFILTER) { + if((retval = nc_def_var_filter(ncid,varid,H5Z_FILTER_FLETCHER32,0,NULL))) return retval; var->storage = NC_CHUNKED; + } } #ifdef USE_PARALLEL - /* If deflate, shuffle, or fletcher32 was turned on with + /* If filter is being applied with * parallel I/O writes, then switch to collective access. HDF5 * requires collevtive access for filter use with parallel * I/O. */ - if (shuffle || fletcher32) - { - if (h5->parallel && (nclistlength((NClist*)var->filters) > 0 || var->shuffle || var->fletcher32)) + if (h5->parallel && (nclistlength((NClist*)var->filters) > 0)) var->parallel_access = NC_COLLECTIVE; - } #endif /* USE_PARALLEL */ /* Handle storage settings. */ @@ -568,7 +567,7 @@ nc_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, * no filters in use for this data. */ if (*storage != NC_CHUNKED) { - if (nclistlength(((NClist*)var->filters)) > 0 || var->fletcher32 || var->shuffle) + if (nclistlength(((NClist*)var->filters)) > 0) return NC_EINVAL; for (d = 0; d < var->ndims; d++) if (var->dim[d]->unlimited) diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index 9c93bf861b..54758160bb 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -824,18 +824,6 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid } } - /* If the user wants to fletcher error correction, set that up now. */ - /* Since it is a checksum of sorts, flatcher is always applied first */ - if (var->fletcher32) - if (H5Pset_fletcher32(plistid) < 0) - BAIL(NC_EHDFERR); - - /* If the user wants to shuffle the data, set that up now. */ - if (var->shuffle) { - if (H5Pset_shuffle(plistid) < 0) - BAIL(NC_EHDFERR); - } - /* If the user wants to compress the data, using either zlib * (a.k.a deflate) or szip, or another filter, set that up now. * Szip and zip can be turned on @@ -847,35 +835,39 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid NClist* filters = (NClist*)var->filters; for(j=0;jfilterid == H5Z_FILTER_DEFLATE) {/* Handle zip case here */ - unsigned level; - if(fi->nparams != 1) - BAIL(NC_EFILTER); - level = (int)fi->params[0]; - if(H5Pset_deflate(plistid, level) < 0) - BAIL(NC_EFILTER); - } else if(fi->filterid == H5Z_FILTER_SZIP) {/* Handle szip case here */ - int options_mask; - int bits_per_pixel; - if(fi->nparams != 2) - BAIL(NC_EFILTER); - options_mask = (int)fi->params[0]; - bits_per_pixel = (int)fi->params[1]; - if(H5Pset_szip(plistid, options_mask, bits_per_pixel) < 0) - BAIL(NC_EFILTER); - } else { - herr_t code = H5Pset_filter(plistid, fi->filterid, -#if 0 - H5Z_FLAG_MANDATORY, + if(fi->filterid == H5Z_FILTER_FLETCHER32) { + if(H5Pset_fletcher32(plistid) < 0) + BAIL(NC_EHDFERR); + } else if(fi->filterid == H5Z_FILTER_SHUFFLE) { + if(H5Pset_shuffle(plistid) < 0) + BAIL(NC_EHDFERR); + } else if(fi->filterid == H5Z_FILTER_DEFLATE) {/* Handle zip case here */ + unsigned level; + if(fi->nparams != 1) + BAIL(NC_EFILTER); + level = (int)fi->params[0]; + if(H5Pset_deflate(plistid, level) < 0) + BAIL(NC_EFILTER); + } else if(fi->filterid == H5Z_FILTER_SZIP) {/* Handle szip case here */ + int options_mask; + int bits_per_pixel; + if(fi->nparams != 2) + BAIL(NC_EFILTER); + options_mask = (int)fi->params[0]; + bits_per_pixel = (int)fi->params[1]; + if(H5Pset_szip(plistid, options_mask, bits_per_pixel) < 0) + BAIL(NC_EFILTER); + } else { + herr_t code = H5Pset_filter(plistid, fi->filterid, +#if 1 + H5Z_FLAG_MANDATORY, #else - H5Z_FLAG_OPTIONAL, + H5Z_FLAG_OPTIONAL, #endif - fi->nparams, fi->params); - if(code < 0) - BAIL(NC_EFILTER); - } - } + fi->nparams, fi->params); + if(code < 0) + BAIL(NC_EFILTER); + } } } @@ -898,7 +890,7 @@ var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid /* If there are no unlimited dims, and no filters, and the user * has not specified chunksizes, use contiguous variable for * better performance. */ - if (!var->shuffle && !var->fletcher32 && nclistlength((NClist*)var->filters) == 0 && + if (nclistlength((NClist*)var->filters) == 0 && (var->chunksizes == NULL || !var->chunksizes[0]) && !unlimdim) var->storage = NC_CONTIGUOUS; diff --git a/liblib/CMakeLists.txt b/liblib/CMakeLists.txt index acbc09a25d..7773316422 100644 --- a/liblib/CMakeLists.txt +++ b/liblib/CMakeLists.txt @@ -77,9 +77,22 @@ SET(TLL_LIBS ${TLL_LIBS} ${HAVE_LIBM} ${ZLIB_LIBRARY}) # Add extra dependencies specified via NC_EXTRA_DEPS SET(TLL_LIBS ${TLL_LIBS} ${EXTRA_DEPS}) -IF(ENABLE_BLOSC) +IF(ENABLE_NCZARR_ZIP) + SET(TLL_LIBS ${TLL_LIBS} ${Zip_LIBRARIES}) +ENDIF() + +IF(Blosc_FOUND) SET(TLL_LIBS ${TLL_LIBS} ${Blosc_LIBRARIES}) ENDIF() +IF(Zstd_FOUND) +SET(TLL_LIBS ${TLL_LIBS} ${Zstd_LIBRARIES}) +ENDIF() +IF(Bz2_FOUND) +SET(TLL_LIBS ${TLL_LIBS} ${Bz2_LIBRARIES}) +ENDIF() +IF(SZIP_FOUND) +SET(TLL_LIBS ${TLL_LIBS} ${SZIP_LIBRARIES}) +ENDIF() IF(HAVE_LIBDL) SET(TLL_LIBS ${LIBDL} ${TLL_LIBS}) @@ -117,10 +130,6 @@ IF(ENABLE_PNETCDF AND PNETCDF) SET(TLL_LIBS ${TLL_LIBS} ${PNETCDF}) ENDIF() -IF(ENABLE_NCZARR_ZIP) - SET(TLL_LIBS ${TLL_LIBS} ${Zip_LIBRARIES}) -ENDIF() - IF(ENABLE_S3_SDK) # TARGET_LINK_DIRECTORIES(netcdf PUBLIC ${AWSSDK_LIB_DIR}) TARGET_LINK_LIBRARIES(netcdf ${AWS_LINK_LIBRARIES}) diff --git a/libncpoco/cp_unix.c b/libncpoco/cp_unix.c index f26993de4d..3ccea2ed03 100755 --- a/libncpoco/cp_unix.c +++ b/libncpoco/cp_unix.c @@ -19,6 +19,8 @@ #include "ncpoco.h" #include "ncpathmgr.h" +#undef DEBUG + /* Note: cygwin is missing RTLD_LOCAL, set it to 0 */ #if !defined(RTLD_LOCAL) #define RTLD_LOCAL 0 @@ -29,13 +31,32 @@ #endif #if !defined(RTLD_LAZY) -#define RTLD_LAZY 0 +#define RTLD_LAZY 1 +#endif + +#if !defined(RTLD_NOW) +#define RTLD_NOW 2 #endif #ifdef USE_MUTEX static pthread_mutex_t mutex; #endif +static void +ncperr(const char* fcn, NCPSharedLib* lib) +{ + const char* msg = dlerror(); + lib->err.msg[0] = '\0'; + if(msg != NULL) { + strlcat(lib->err.msg,fcn,sizeof(lib->err.msg)); + strlcat(lib->err.msg,": ",sizeof(lib->err.msg)); + strlcat(lib->err.msg,msg,sizeof(lib->err.msg)); +#ifdef DEBUG + fprintf(stderr,">>> %s\n",lib->err.msg); +#endif + } +} + int ncp_unix_initialize(void) { @@ -96,7 +117,7 @@ static int load(NCPSharedLib* lib , const char* path0, int flags) { int ret = NC_NOERR; - int realflags = RTLD_LAZY; + int realflags = RTLD_LAZY; /* versus RTLD_NOW which does not appear to work */ char* path = NULL; if((path = NCpathcvt(path0))==NULL) {ret = NC_ENOMEM; goto done;} @@ -113,9 +134,7 @@ load(NCPSharedLib* lib , const char* path0, int flags) lib->state.flags = realflags; lib->state.handle = dlopen(lib->path, lib->state.flags); if(lib->state.handle == NULL) { - const char* msg = dlerror(); - if(msg == NULL) msg = ""; - strncpy(lib->err.msg,msg,sizeof(lib->err.msg)); + ncperr("dlopen",lib); ret = NC_ENOTFOUND; goto ldone; } @@ -150,8 +169,12 @@ getsymbol(NCPSharedLib* lib, const char* name) { void* result = NULL; lock(); - if(lib->state.handle != NULL) + if(lib->state.handle != NULL) { result = dlsym(lib->state.handle, name); + if(result == NULL) { + ncperr("dlsym",lib); + } + } unlock(); return result; } diff --git a/libnczarr/Makefile.am b/libnczarr/Makefile.am index f5950fd847..fa0102beba 100644 --- a/libnczarr/Makefile.am +++ b/libnczarr/Makefile.am @@ -7,6 +7,7 @@ # Get AM_CPPFLAGS and AM_LDFLAGS include $(top_srcdir)/lib_flags.am + AM_CPPFLAGS += -D_LARGEFILE_SOURCE AM_CPPFLAGS += -I$(top_srcdir)/libsrc4 -I$(top_srcdir)/libncpoco libnczarr_la_CPPFLAGS = ${AM_CPPFLAGS} diff --git a/libnczarr/zcache.h b/libnczarr/zcache.h index f25f30185d..f5f7362dd2 100644 --- a/libnczarr/zcache.h +++ b/libnczarr/zcache.h @@ -53,7 +53,7 @@ typedef struct NCZChunkCache { /**************************************************/ -#define FILTERED(cache) (nclistlength((NClist*)(cache)->var->filters) || (cache)->var->shuffle || (cache)->var->fletcher32); +#define FILTERED(cache) (nclistlength((NClist*)(cache)->var->filters)) extern int NCZ_set_var_chunk_cache(int ncid, int varid, size_t size, size_t nelems, float preemption); extern int NCZ_adjust_var_cache(NC_VAR_INFO_T *var); diff --git a/libnczarr/zdebug.h b/libnczarr/zdebug.h index 97b255db41..7874524f2d 100644 --- a/libnczarr/zdebug.h +++ b/libnczarr/zdebug.h @@ -9,7 +9,7 @@ #undef ZDEBUG1 /* detailed debug */ #undef ZCATCH /* Warning: significant performance impact */ -#define ZTRACING /* Warning: significant performance impact */ +#undef ZTRACING /* Warning: significant performance impact */ #include "ncexternl.h" #include "nclog.h" diff --git a/libnczarr/zdispatch.c b/libnczarr/zdispatch.c index b2fff3d9ef..8f06e613ca 100644 --- a/libnczarr/zdispatch.c +++ b/libnczarr/zdispatch.c @@ -106,6 +106,7 @@ static const NC_Dispatch NCZ_dispatcher = { NCZ_inq_var_filter_info, NCZ_def_var_quantize, NCZ_inq_var_quantize, + NCZ_inq_filter_avail, }; const NC_Dispatch* NCZ_dispatch_table = NULL; /* moved here from ddispatch.c */ @@ -157,7 +158,7 @@ NCZ_show_metadata(int ncid) } #ifndef ENABLE_NCZARR_FILTERS - int +int NCZ_def_var_filter(int ncid, int varid, unsigned int id , size_t n , const unsigned int *params) { NC_UNUSED(ncid); @@ -188,6 +189,16 @@ NCZ_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparams, u NC_UNUSED(params); return REPORT(NC_ENOFILTER,"inq_var_filter_info"); } + +int +NCZ_inq_filter_avail(int ncid, unsigned id) +{ + NC_UNUSED(ncid); + NC_UNUSED(id); + return REPORT(NC_ENOFILTER,"inq_filter_avail"); +} + + #endif /*ENABLE_NCZARR_FILTERS*/ /**************************************************/ diff --git a/libnczarr/zdispatch.h b/libnczarr/zdispatch.h index 09391d03f8..3933ae96f8 100644 --- a/libnczarr/zdispatch.h +++ b/libnczarr/zdispatch.h @@ -172,6 +172,7 @@ EXTERNL int NCZ_inq_unlimdims(int, int *, int *); EXTERNL int NCZ_def_var_filter(int ncid, int varid, unsigned int filterid, size_t nparams, const unsigned int *params); EXTERNL int NCZ_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int *filterids); EXTERNL int NCZ_inq_var_filter_info(int ncid, int varid, unsigned int filterid, size_t* nparamsp, unsigned int *params); +EXTERNL int NCZ_inq_filter_avail(int ncid, unsigned id); EXTERNL int NCZ_def_var_quantize(int ncid, int varid, int quantize_mode, int nsd); EXTERNL int NCZ_inq_var_quantize(int ncid, int varid, int *quantize_modep, int *nsdp); diff --git a/libnczarr/zfilter.c b/libnczarr/zfilter.c index d2d24b9048..a4ca1d358b 100644 --- a/libnczarr/zfilter.c +++ b/libnczarr/zfilter.c @@ -56,6 +56,9 @@ #undef DEBUGF #undef DEBUGL +/* If set, then triage potential shared libraries based on extension */ +#define NAMEOPT + #define NULLIFY(x) ((x)?(x):"NULL") /* Hold the loaded filter plugin information */ @@ -65,6 +68,7 @@ typedef struct NCZ_Plugin { NCPSharedLib* hdf5lib; /* source of the filter */ } hdf5; struct CodecAPI { + int defaulted; /* codeclib was a defaulting library */ const NCZ_codec_t* codec; NCPSharedLib* codeclib; /* of the source codec; null if same as hdf5 */ } codec; @@ -134,13 +138,15 @@ typedef struct NCZ_Filter { } NCZ_Filter; /* WARNING: GLOBAL DATA */ +/* TODO: move to common global state */ + /* All possible HDF5 filter plugins */ -/* Convert to linked list or hash table or equivalent since very sparse */ +/* Consider onverting to linked list or hash table or equivalent since very sparse */ NCZ_Plugin* loaded_plugins[H5Z_FILTER_MAX]; int loaded_plugins_max = -1; -static NCZ_codec_t** codec_defaults; /* NULL terminated */ -static NCPSharedLib* default_lib; /* source of the defaults */ +static NClist* codec_defaults = NULL; /* NClist */ +static NClist* default_libs = NULL; /* NClist; sources of the defaults */ static int NCZ_filter_initialized = 0; @@ -166,12 +172,12 @@ NCJtrace(const NCjson* j) #endif -#ifdef DEBUGF +#if defined(DEBUGF) || defined(DEBUGL) const char* printplugin(const NCZ_Plugin* plugin) { - static char plbuf[4096]; + char* plbuf = malloc(4096); char plbuf2[2000]; char plbuf1[2000]; @@ -181,23 +187,23 @@ printplugin(const NCZ_Plugin* plugin) snprintf(plbuf1,sizeof(plbuf1),"hdf5={id=%u name=%s}",plugin->hdf5.filter->id,plugin->hdf5.filter->name); if(plugin->codec.codec) snprintf(plbuf2,sizeof(plbuf2),"codec={codecid=%s hdf5id=%u}",plugin->codec.codec->codecid,plugin->codec.codec->hdf5id); - snprintf(plbuf,sizeof(plbuf),"plugin={%s %s}",plbuf1,plbuf2); + snprintf(plbuf,4096,"plugin={%s %s}",plbuf1,plbuf2); return plbuf; } -static const char* +static char* printparams(size_t nparams, const unsigned* params) { - static char ppbuf[4096]; + char* ppbuf = malloc(4096); if(nparams == 0) - snprintf(ppbuf,sizeof(ppbuf),"{0,%p}",params); + snprintf(ppbuf,4096,"{0,%p}",params); else - snprintf(ppbuf,sizeof(ppbuf),"{%u %s}",(unsigned)nparams,nczprint_paramvector(nparams,params)); + snprintf(ppbuf,4096,"{%u %s}",(unsigned)nparams,nczprint_paramvector(nparams,params)); return ppbuf; } -static const char* +static char* printnczparams(const NCZ_Params p) { return printparams(p.nparams,p.params); @@ -255,6 +261,10 @@ static int ensure_working(const NC_VAR_INFO_T* var, NCZ_Filter* filter); static int paramnczclone(NCZ_Params* dst, const NCZ_Params* src); static int paramclone(size_t nparams, unsigned** dstp, const unsigned* src); +#ifdef NAMEOPT +static int pluginnamecheck(const char* name); +#endif + /**************************************************/ /** * @file @@ -660,7 +670,7 @@ NCZ_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp, spec->hdf5.working.nparams,spec->hdf5.working.params, &spec->hdf5.visible.nparams,&spec->hdf5.visible.params); #ifdef DEBUGF - fprintf(stderr,"DEBUGF: NCZ_visible_parameters: ncid=%d varid=%d working=%s visible=%s\n",ncid,varid, + fprintf(stderr,">>> DEBUGF: NCZ_visible_parameters: ncid=%d varid=%d working=%s visible=%s\n",ncid,varid, printnczparams(spec->hdf5.visible),printnczparams(spec->hdf5.working)); #endif if(stat) goto done; @@ -679,6 +689,32 @@ NCZ_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp, done: return ZUNTRACEX(stat,"nparams=%u",(unsigned)(nparamsp?*nparamsp:0)); } + +/* Test if a specific filter is available. + @param file for which use of a filter is desired + @param id the filter id of interest + @return NC_NOERR if the filter is available + @return NC_ENOFILTER if the filter is not available + @return NC_EBADID if ncid is invalid + @return NC_EFILTER if ncid format does not support filters +*/ +int +NCZ_inq_filter_avail(int ncid, unsigned id) +{ + int stat = NC_NOERR; + struct NCZ_Plugin* plug = NULL; + + NC_UNUSED(ncid); + ZTRACE(1,"ncid=%d id=%u",ncid,id); + if((stat = NCZ_filter_initialize())) goto done; + /* Check the available filters list */ + if((stat = NCZ_plugin_loaded((int)id, &plug))) goto done; + if(plug == NULL) + stat = NC_ENOFILTER; +done: + return ZUNTRACE(stat); +} + #endif /*ENABLE_NCZARR_FILTERS*/ /**************************************************/ @@ -689,14 +725,17 @@ NCZ_filter_initialize(void) { int stat = NC_NOERR; ZTRACE(6,""); + if(NCZ_filter_initialized) goto done; - { - NCZ_filter_initialized = 1; - memset(loaded_plugins,0,sizeof(loaded_plugins)); + + default_libs = nclistnew(); + codec_defaults = nclistnew(); + NCZ_filter_initialized = 1; + memset(loaded_plugins,0,sizeof(loaded_plugins)); #ifdef ENABLE_NCZARR_FILTERS - if((stat = NCZ_load_all_plugins())) goto done; + if((stat = NCZ_load_all_plugins())) goto done; #endif - } + done: return ZUNTRACE(stat); } @@ -710,15 +749,34 @@ NCZ_filter_finalize(void) if(!NCZ_filter_initialized) goto done; #ifdef ENABLE_NCZARR_FILTERS /* Reclaim all loaded filters */ +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: finalize reclaim:\n"); +#endif for(i=0;i<=loaded_plugins_max;i++) { - NCZ_unload_plugin(loaded_plugins[i]); - loaded_plugins[i] = NULL; + if(loaded_plugins[i]) { + NCZ_unload_plugin(loaded_plugins[i]); + loaded_plugins[i] = NULL; + } + } + /* Reclaim the codec defaults */ + if(nclistlength(codec_defaults) > 0) { + for(i=0;i 0) { + for(i=0;i>> current: alloc=%u used=%u buf=%p\n",(unsigned)current_alloc,(unsigned)current_used,current_buf); #endif /* Apply in proper order */ if(encode) { @@ -793,7 +851,7 @@ fprintf(stderr,"current: alloc=%u used=%u buf=%p\n",(unsigned)current_alloc,(uns next_used = 0; next_used = ff->filter(0,f->hdf5.working.nparams,f->hdf5.working.params,current_used,&next_alloc,&next_buf); #ifdef DEBUG -fprintf(stderr,"next: alloc=%u used=%u buf=%p\n",(unsigned)next_alloc,(unsigned)next_used,next_buf); +fprintf(stderr,">>> next: alloc=%u used=%u buf=%p\n",(unsigned)next_alloc,(unsigned)next_used,next_buf); #endif if(next_used == 0) {stat = NC_EFILTER; lastbuffer = next_buf; goto done; } /* If the filter did not need to create a new buffer, then next == current else current was reclaimed */ @@ -812,7 +870,7 @@ fprintf(stderr,"next: alloc=%u used=%u buf=%p\n",(unsigned)next_alloc,(unsigned) next_used = 0; next_used = ff->filter(H5Z_FLAG_REVERSE,f->hdf5.working.nparams,f->hdf5.working.params,current_used,&next_alloc,&next_buf); #ifdef DEBUG -fprintf(stderr,"next: alloc=%u used=%u buf=%p\n",(unsigned)next_alloc,(unsigned)next_used,next_buf); +fprintf(stderr,">>> next: alloc=%u used=%u buf=%p\n",(unsigned)next_alloc,(unsigned)next_used,next_buf); #endif if(next_used == 0) {stat = NC_EFILTER; lastbuffer = next_buf; goto done;} /* If the filter did not need to create a new buffer, then next == current else current was reclaimed */ @@ -822,7 +880,7 @@ fprintf(stderr,"next: alloc=%u used=%u buf=%p\n",(unsigned)next_alloc,(unsigned) } } #ifdef DEBUG -fprintf(stderr,"current: alloc=%u used=%u buf=%p\n",(unsigned)current_alloc,(unsigned)current_used,current_buf); +fprintf(stderr,">>> current: alloc=%u used=%u buf=%p\n",(unsigned)current_alloc,(unsigned)current_used,current_buf); #endif /* return results */ if(outlenp) {*outlenp = current_used;} /* or should it be current_alloc? */ @@ -864,7 +922,7 @@ NCZ_filter_jsonize(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Fil if(filter->plugin->codec.codec->NCZ_hdf5_to_codec) { stat = filter->plugin->codec.codec->NCZ_hdf5_to_codec(filter->hdf5.visible.nparams,filter->hdf5.visible.params,&filter->codec.codec); #ifdef DEBUGF - fprintf(stderr,"DEBUGF: NCZ_hdf5_to_codec: visible=%s codec=%s\n",printnczparams(filter->hdf5.visible),filter->codec.codec); + fprintf(stderr,">>> DEBUGF: NCZ_hdf5_to_codec: visible=%s codec=%s\n",printnczparams(filter->hdf5.visible),filter->codec.codec); #endif if(stat) goto done; } else @@ -921,7 +979,7 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j if(plugin->codec.codec->NCZ_codec_to_hdf5) { stat = plugin->codec.codec->NCZ_codec_to_hdf5(codec.codec,&hdf5.visible.nparams,&hdf5.visible.params); #ifdef DEBUGF - fprintf(stderr,"DEBUGF: NCZ_codec_to_hdf5: codec=%s, hdf5=%s\n",printcodec(codec),printhdf5(hdf5)); + fprintf(stderr,">>> DEBUGF: NCZ_codec_to_hdf5: codec=%s, hdf5=%s\n",printcodec(codec),printhdf5(hdf5)); #endif if(stat) goto done; } @@ -1052,7 +1110,7 @@ getentries(const char* path, NClist* contents) static int NCZ_load_all_plugins(void) { - int i,ret = NC_NOERR; + int i,j,ret = NC_NOERR; const char* pluginroot = NULL; struct stat buf; NClist* dirs = nclistnew(); @@ -1063,7 +1121,7 @@ NCZ_load_all_plugins(void) ZTRACE(6,""); #ifdef DEBUGL - fprintf(stderr,"DEBUGL: NCZ_load_all_plugins\n"); + fprintf(stderr,">>> DEBUGL: NCZ_load_all_plugins\n"); #endif /* Find the plugin directory root(s) */ @@ -1107,20 +1165,35 @@ NCZ_load_all_plugins(void) /* Try to load plugins from this directory */ if((ret = NCZ_load_plugin_dir(dir))) goto done; } - - if(codec_defaults != NULL) { /* Try to provide default for any HDF5 filters without matching Codec. */ - NCZ_codec_t** dfalts = NULL; +#ifdef DEBUGL + { int i; + fprintf(stderr,"codec_defaults:"); + for(i=0;icodec->hdf5id); + } + fprintf(stderr,"\n"); + } +#endif + if(nclistlength(codec_defaults)) { /* Try to provide default for any HDF5 filters without matching Codec. */ /* Search the defaults */ - for(dfalts = codec_defaults;*dfalts;dfalts++) { - NCZ_codec_t* codec = *dfalts; - int hdf5id = codec->hdf5id; - NCZ_Plugin* p = loaded_plugins[hdf5id]; /* get candidate */ - if(p != NULL && p->hdf5.filter != NULL && p->codec.codec == NULL) { + for(j=0;jcodec != NULL) { + const NCZ_codec_t* codec = dfalt->codec; + int hdf5id = codec->hdf5id; + NCZ_Plugin* p = NULL; + if(hdf5id < 0 || hdf5id > loaded_plugins_max) {ret = NC_EFILTER; goto done;} + p = loaded_plugins[hdf5id]; /* get candidate */ + if(p != NULL && p->hdf5.filter != NULL + && p->codec.codec == NULL) { #ifdef DEBUGL - fprintf(stderr,"DEBUGL: plugin defaulted: id=%u, codec=%s\n",hdf5id,codec->codecid); + fprintf(stderr,">>> DEBUGL: plugin defaulted: id=%u, codec=%s src=%s\n",hdf5id,codec->codecid,dfalt->codeclib->path); #endif - p->codec.codec = codec; codec = NULL; - p->codec.codeclib = NULL; + p->codec.codec = codec; + p->codec.codeclib = dfalt->codeclib; + p->codec.defaulted = 1; + } } } } @@ -1133,12 +1206,17 @@ NCZ_load_all_plugins(void) if((p = loaded_plugins[i]) != NULL) { if(p->hdf5.filter == NULL || p->codec.codec == NULL) { #ifdef DEBUGL - fprintf(stderr,"DEBUGL: plugin expunged: id=%u\n",p->hdf5.filter->id); + fprintf(stderr,">>> DEBUGL: Incomplete plugin: expunged: id=%u; reasons: %s %s\n",i, + (p->hdf5.filter==NULL?"hdf5":""),(p->codec.codec==NULL?"codec":"")); #endif /* expunge this entry */ (void)NCZ_unload_plugin(p); loaded_plugins[i] = NULL; } +#ifdef DEBUGL + else + fprintf(stderr,">>> DEBUGL: plugin accepted: id=%u\n",i); +#endif } } } @@ -1152,7 +1230,7 @@ NCZ_load_all_plugins(void) if(p->codec.codec && p->codec.codec->NCZ_codec_initialize) p->codec.codec->NCZ_codec_initialize(); #ifdef DEBUGL - fprintf(stderr,"DEBUGL: plugin initialized: id=%u\n",p->hdf5.filter->id); + fprintf(stderr,">>> DEBUGL: plugin initialized: id=%u\n",p->hdf5.filter->id); #endif } } @@ -1213,7 +1291,7 @@ NCZ_load_plugin_dir(const char* path) ZTRACE(7,"path=%s",path); #ifdef DEBUGL - fprintf(stderr,"DEBUGL: NCZ_load_plugin_dir: path=%s\n",path); + fprintf(stderr,">>> DEBUGL: NCZ_load_plugin_dir: path=%s\n",path); #endif if(path == NULL) {stat = NC_EINVAL; goto done;} @@ -1236,9 +1314,12 @@ NCZ_load_plugin_dir(const char* path) strlcat(file,"/",flen); strlcat(file,name,flen); /* See if can load the file */ - switch ((stat = NCZ_load_plugin(file,&plugin))) { + stat = NCZ_load_plugin(file,&plugin); + switch (stat) { case NC_NOERR: break; - case NC_ENOFILTER: case NC_ENOTFOUND: stat = NC_NOERR; break; /* will cause it to be ignored */ + case NC_ENOFILTER: case NC_ENOTFOUND: + stat = NC_NOERR; + break; /* will cause it to be ignored */ default: goto done; } if(plugin != NULL) { @@ -1247,11 +1328,11 @@ NCZ_load_plugin_dir(const char* path) loaded_plugins[id] = plugin; if(id > loaded_plugins_max) loaded_plugins_max = id; #ifdef DEBUGL - fprintf(stderr,"DEBUGL: plugin loaded: %s\n",printplugin(plugin)); + fprintf(stderr,">>> DEBUGL: plugin loaded: %s\n",printplugin(plugin)); #endif } else { #ifdef DEBUGL - fprintf(stderr,"DEBUGL: plugin duplicate: %s\n",printplugin(plugin)); + fprintf(stderr,">>> DEBUGL: plugin duplicate: %s\n",printplugin(plugin)); #endif NCZ_unload_plugin(plugin); /* its a duplicate */ } @@ -1280,23 +1361,21 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp) ZTRACE(8,"path=%s",path); -#ifdef DEBUGL - fprintf(stderr,"DEBUGL: NCZ_load_plugin: path=%s\n",path); -#endif - if(plugp) *plugp = NULL; -#ifdef _WIN32 +#if defined NAMEOPT || defined _WIN32 /*triage because visual studio does a popup if the file will not load*/ - if(memcmp(path+(strlen(path)-4),".dll",4) != 0) { - stat = NC_ENOFILTER; goto done; - } + if(!pluginnamecheck(path)) {stat = NC_ENOFILTER; goto done;} #endif /* load the shared library */ if((stat = ncpsharedlibnew(&lib))) goto done; if((stat = ncpload(lib,path,flags))) goto done; +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: NCZ_load_plugin: path=%s\n",path); +#endif + /* See what we have */ { H5PL_get_plugin_type_proto gpt = (H5PL_get_plugin_type_proto)ncpgetsymbol(lib,"H5PLget_plugin_type"); @@ -1304,14 +1383,25 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp) NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info"); NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults"); - /* Deal with defaults first */ + if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL) + {stat = NC_ENOFILTER; goto done;} + if(cpd != NULL) { - if(codec_defaults == NULL) { - codec_defaults = (NCZ_codec_t**)cpd(); - default_lib = lib; lib = NULL; + /* Deal with defaults first */ + NCZ_codec_t** cp = NULL; + nclistpush(default_libs,lib); + for(cp=cpd();*cp;cp++) { + struct CodecAPI* c0; + c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI)); + if(c0 == NULL) {stat = NC_ENOMEM; goto done1;} + c0->codec = *cp; + c0->codeclib = lib; + nclistpush(codec_defaults,c0); c0 = NULL; } +done1: + lib = NULL; goto done; - } + } if(gpt != NULL && gpi != NULL) { /* get HDF5 info */ @@ -1331,14 +1421,11 @@ NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp) } } - /* Ignore this library if neither h5class nor codec are defined */ - if(h5class == NULL && codec == NULL) {stat = NC_ENOFILTER; goto done;} - #ifdef DEBUGL -fprintf(stderr,"DEBUGL: load: %s:",path); -if(h5class) fprintf(stderr," %u",(unsigned)h5class->id); -if(codec) fprintf(stderr," %u/%s",codec->hdf5id,codec->codecid); -fprintf(stderr,"\n"); +fprintf(stderr,">>> DEBUGL: load: %s:",path); +if(h5class) fprintf(stderr,">>> %u",(unsigned)h5class->id); +if(codec) fprintf(stderr,">>> %u/%s",codec->hdf5id,codec->codecid); +fprintf(stderr,">>> \n"); #endif if(h5class != NULL && codec != NULL) { @@ -1379,7 +1466,7 @@ fprintf(stderr,"\n"); #ifdef DEBUGL if(plugin) - fprintf(stderr,"DEBUGL: load_plugin: %s\n",printplugin(plugin)); + fprintf(stderr,">>> DEBUGL: load_plugin: %s\n",printplugin(plugin)); #endif @@ -1404,19 +1491,48 @@ NCZ_unload_plugin(NCZ_Plugin* plugin) if(plugin) { #ifdef DEBUGL - fprintf(stderr,"DEBUGL: unload: %s\n",printplugin(plugin)); + fprintf(stderr,">>> DEBUGL: unload: %s\n",printplugin(plugin)); #endif if(plugin->codec.codec && plugin->codec.codec->NCZ_codec_finalize) plugin->codec.codec->NCZ_codec_finalize(); if(plugin->hdf5.filter != NULL) loaded_plugins[plugin->hdf5.filter->id] = NULL; if(plugin->hdf5.hdf5lib != NULL) (void)ncpsharedlibfree(plugin->hdf5.hdf5lib); - if(plugin->codec.codeclib != NULL) (void)ncpsharedlibfree(plugin->codec.codeclib); - memset(plugin,0,sizeof(NCZ_Plugin)); + if(!plugin->codec.defaulted && plugin->codec.codeclib != NULL) (void)ncpsharedlibfree(plugin->codec.codeclib); +memset(plugin,0,sizeof(NCZ_Plugin)); free(plugin); } return ZUNTRACE(NC_NOERR); } +#ifdef NAMEOPT +static int +pluginnamecheck(const char* name) +{ + size_t i,count,len; + const char* p; + if(name == NULL) return 0; + /* get basename */ + p = strrchr(name,'/'); + if(p != NULL) name = (p+1); + len = strlen(name); + if(len == 0) return 0; + i = len-1; + count = 1; + p = name+i; + for(;i>=0;i--,count++,p--) { + char c = *p; + if(c == '/') break; + if(c == '.') { + if(count >= 3 && memcmp(p,".so",3)==0) return 1; + if(count >= 4 && memcmp(p,".dll",4)==0) return 1; + if(count >= 6 && memcmp(p,".dylib",6)==0) return 1; + } + } + return 0; +} +#endif + + /**************************************************/ /* _Codecs attribute */ @@ -1458,12 +1574,17 @@ ensure_working(const NC_VAR_INFO_T* var, NCZ_Filter* filter) assert(filter->flags & FLAG_VISIBLE); /* Convert the visible parameters to working parameters; may also modify the visible params */ +#ifdef DEBUGF + fprintf(stderr,">>> DEBUGF: NCZ_modify_parameters: before: visible=%s working=%s\n", + printparams(filter->hdf5.visible.nparams,filter->hdf5.visible.params), + printparams(filter->hdf5.working.nparams,filter->hdf5.working.params)); +#endif if(filter->plugin && filter->plugin->codec.codec->NCZ_modify_parameters) { stat = filter->plugin->codec.codec->NCZ_modify_parameters(ncidfor(var),var->hdr.id, &filter->hdf5.visible.nparams, &filter->hdf5.visible.params, &filter->hdf5.working.nparams, &filter->hdf5.working.params); #ifdef DEBUGF - fprintf(stderr,"DEBUGF: NCZ_modify_parameters: ncid=%d varid=%d filter=%s\n", (int)ncidfor(var),(int)var->hdr.id, + fprintf(stderr,">>> DEBUGF: NCZ_modify_parameters: stat=%d ncid=%d varid=%d filter=%s\n",stat, (int)ncidfor(var),(int)var->hdr.id, printfilter(filter)); #endif if(stat) goto done; @@ -1477,10 +1598,15 @@ ensure_working(const NC_VAR_INFO_T* var, NCZ_Filter* filter) nullfree(filter->hdf5.working.params); if((stat = paramnczclone(&filter->hdf5.working,&filter->hdf5.visible))) goto done; } +#ifdef DEBUGF + fprintf(stderr,">>> DEBUGF: NCZ_modify_parameters: after: visible=%s working=%s\n", + printparams(filter->hdf5.visible.nparams,filter->hdf5.visible.params), + printparams(filter->hdf5.working.nparams,filter->hdf5.working.params)); +#endif filter->flags |= FLAG_WORKING; } #ifdef DEBUGF - fprintf(stderr,"DEBUGF: ensure_working_parameters: ncid=%lu varid=%u filter=%s\n", ncidfor(var), (unsigned)var->hdr.id,printfilter(filter)); + fprintf(stderr,">>> DEBUGF: ensure_working_parameters: ncid=%lu varid=%u filter=%s\n", ncidfor(var), (unsigned)var->hdr.id,printfilter(filter)); #endif done: return THROW(stat); @@ -1528,7 +1654,7 @@ rebuild_visible(const NC_VAR_INFO_T* var, NCZ_Filter* filter) } filter->flags |= FLAG_VISIBLE; #ifdef DEBUGF - fprintf(stderr,"DEBUGF: rebuild_visible_parameters: ncid=%lu varid=%u filter=%s\n", ncidfor(var), (unsigned)var->hdr.id,printfilter(filter)); + fprintf(stderr,">>> DEBUGF: rebuild_visible_parameters: ncid=%lu varid=%u filter=%s\n", ncidfor(var), (unsigned)var->hdr.id,printfilter(filter)); #endif done: nullfree(visible0); @@ -1555,7 +1681,7 @@ NCZ_filter_setup(NC_VAR_INFO_T* var) /* Initialize the working parameters */ if((stat = ensure_working(var,filter))) goto done; #ifdef DEBUGF - fprintf(stderr,"DEBUGF: NCZ_filter_setup: ncid=%d varid=%d filter=%s\n", (int)ncidfor(var),(int)var->hdr.id, + fprintf(stderr,">>> DEBUGF: NCZ_filter_setup: ncid=%d varid=%d filter=%s\n", (int)ncidfor(var),(int)var->hdr.id, printfilter(filter)); #endif } @@ -1564,6 +1690,7 @@ NCZ_filter_setup(NC_VAR_INFO_T* var) return ZUNTRACE(stat); } + /**************************************************/ /* Clone an hdf5 parameter set */ diff --git a/libnczarr/zfilter.h b/libnczarr/zfilter.h index 7a4e2e517a..4aa7de80aa 100644 --- a/libnczarr/zfilter.h +++ b/libnczarr/zfilter.h @@ -48,5 +48,5 @@ int NCZ_applyfilterchain(const NC_FILE_INFO_T*, NC_VAR_INFO_T*, NClist* chain, s int NCZ_filter_jsonize(const NC_FILE_INFO_T*, const NC_VAR_INFO_T*, struct NCZ_Filter* filter, struct NCjson**); int NCZ_filter_build(const NC_FILE_INFO_T*, NC_VAR_INFO_T* var, const NCjson* jfilter); int NCZ_codec_attr(const NC_VAR_INFO_T* var, size_t* lenp, void* data); - + #endif /*ZFILTER_H*/ diff --git a/libnczarr/zvar.c b/libnczarr/zvar.c index 4526840576..c0aa61f3bd 100644 --- a/libnczarr/zvar.c +++ b/libnczarr/zvar.c @@ -600,23 +600,21 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, #endif /* Shuffle filter? */ - if (shuffle) - { - if(*shuffle) var->shuffle = *shuffle; - var->storage = NC_CHUNKED; - if(var->shuffle) { - if((retval = NCZ_def_var_filter(ncid,varid,2,0,NULL))) goto done; - } + if (shuffle && *shuffle) { + retval = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_SHUFFLE,NULL,NULL); + if(!retval || retval == NC_ENOFILTER) { + if((retval = NCZ_def_var_filter(ncid,varid,H5Z_FILTER_SHUFFLE,0,NULL))) return retval; + var->storage = NC_CHUNKED; + } } /* Fletcher32 checksum error protection? */ - if (fletcher32) - { - if(*fletcher32) var->fletcher32 = *fletcher32; - var->storage = NC_CHUNKED; - if(var->fletcher32) { - if((retval = NCZ_def_var_filter(ncid,varid,3,0,NULL))) goto done; - } + if (fletcher32 && fletcher32) { + retval = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_FLETCHER32,NULL,NULL); + if(!retval || retval == NC_ENOFILTER) { + if((retval = NCZ_def_var_filter(ncid,varid,H5Z_FILTER_FLETCHER32,0,NULL))) return retval; + var->storage = NC_CHUNKED; + } } /* Handle storage settings. */ @@ -628,7 +626,7 @@ ncz_def_var_extra(int ncid, int varid, int *shuffle, int *unused1, * no filters in use for this data. */ if (storage != NC_CHUNKED) { - if (nclistlength(((NClist*)var->filters)) > 0 || var->fletcher32 || var->shuffle) + if (nclistlength(((NClist*)var->filters)) > 0) {retval = NC_EINVAL; goto done;} for (d = 0; d < var->ndims; d++) if (var->dim[d]->unlimited) diff --git a/libnetcdf.settings.in b/libnetcdf.settings.in index 6896b869e9..0e07df85e2 100644 --- a/libnetcdf.settings.in +++ b/libnetcdf.settings.in @@ -41,10 +41,10 @@ JNA Support: @HAS_JNA@ CDF5 Support: @HAS_CDF5@ ERANGE Fill Support: @HAS_ERANGE_FILL@ Relaxed Boundary Check: @RELAX_COORD_BOUND@ -SZIP Support: @HAS_SZLIB@ -SZIP Write Support: @HAS_SZLIB_WRITE@ Parallel Filters: @HAS_PAR_FILTERS@ NCZarr Support: @HAS_NCZARR@ Multi-Filter Support: @HAS_MULTIFILTERS@ Quantization: @HAS_QUANTIZE@ Logging: @HAS_LOGGING@ +SZIP Write Support: @HAS_SZLIB_WRITE@ +Standard Filters: @STD_FILTERS@ diff --git a/libsrc/nc3dispatch.c b/libsrc/nc3dispatch.c index cd1141c7de..e7358a0860 100644 --- a/libsrc/nc3dispatch.c +++ b/libsrc/nc3dispatch.c @@ -169,6 +169,8 @@ NC_NOOP_inq_var_filter_info, NC_NOTNC4_def_var_quantize, NC_NOTNC4_inq_var_quantize, + +NC_NOOP_inq_filter_avail, }; const NC_Dispatch* NC3_dispatch_table = NULL; /*!< NC3 Dispatch table, moved here from ddispatch.c */ diff --git a/libsrc4/nc4var.c b/libsrc4/nc4var.c index fac6520bf5..ace45c60ad 100644 --- a/libsrc4/nc4var.c +++ b/libsrc4/nc4var.c @@ -242,11 +242,16 @@ NC4_inq_var_all(int ncid, int varid, char *name, nc_type *xtypep, *storagep = var->storage; /* Filter stuff. */ - if (shufflep) - *shufflep = (int)var->shuffle; - if (fletcher32p) - *fletcher32p = (int)var->fletcher32; - + if (shufflep) { + retval = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_SHUFFLE,0,NULL); + if(retval && retval != NC_ENOFILTER) return retval; + *shufflep = (retval == NC_NOERR?1:0); + } + if (fletcher32p) { + retval = nc_inq_var_filter_info(ncid,varid,H5Z_FILTER_FLETCHER32,0,NULL); + if(retval && retval != NC_ENOFILTER) return retval; + *fletcher32p = (retval == NC_NOERR?1:0); + } if (deflatep) return NC_EFILTER; @@ -476,7 +481,7 @@ NC4_var_par_access(int ncid, int varid, int par_access) /* If zlib, shuffle, or fletcher32 filters are in use, then access * must be collective. Fail an attempt to set such a variable to * independent access. */ - if ((nclistlength((NClist*)var->filters) > 0 || var->shuffle || var->fletcher32) && + if (nclistlength((NClist*)var->filters) > 0) par_access == NC_INDEPENDENT) return NC_EINVAL; diff --git a/libsrcp/ncpdispatch.c b/libsrcp/ncpdispatch.c index 7e4eeaf961..e99cd925bf 100644 --- a/libsrcp/ncpdispatch.c +++ b/libsrcp/ncpdispatch.c @@ -1466,6 +1466,8 @@ NC_NOOP_inq_var_filter_info, NC_NOTNC4_def_var_quantize, NC_NOTNC4_inq_var_quantize, + +NC_NOOP_inq_filter_avail, }; const NC_Dispatch *NCP_dispatch_table = NULL; /* moved here from ddispatch.c */ diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt index 478f380930..e4d78b3587 100644 --- a/nc_test4/CMakeLists.txt +++ b/nc_test4/CMakeLists.txt @@ -39,22 +39,21 @@ IF(BUILD_UTILITIES) ADD_SH_TEST(nc_test4 tst_misc) build_bin_test(tst_fillonly) ADD_SH_TEST(nc_test4 test_fillonly) -IF(ENABLE_FILTER_TESTING) +IF(USE_HDF5 AND ENABLE_FILTER_TESTING) build_bin_test(tst_filterparser) build_bin_test(test_filter) build_bin_test(test_filter_misc) build_bin_test(tst_multifilter) build_bin_test(test_filter_order) build_bin_test(test_filter_repeat) + build_bin_test(tst_filter_avail) build_bin_test(test_filter_vlen) ADD_SH_TEST(nc_test4 tst_filter) -IF(ENABLE_BLOSC) ADD_SH_TEST(nc_test4 tst_specific_filters) -ENDIF() IF(ENABLE_CLIENTSIDE_FILTERS) add_bin_test(nc_test4 test_filter_reg) ENDIF(ENABLE_CLIENTSIDE_FILTERS) -ENDIF(ENABLE_FILTER_TESTING) +ENDIF(USE_HDF5 AND ENABLE_FILTER_TESTING) ENDIF(BUILD_UTILITIES) @@ -83,7 +82,7 @@ IF(LARGE_FILE_TESTS) ENDIF() -IF(USE_SZIP) +IF(HAVE_H5Z_SZIP) BUILD_BIN_TEST(test_szip) BUILD_BIN_TEST(h5testszip) IF(BUILD_UTILITIES) diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index 5305d8c8b3..23328202ec 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -71,21 +71,21 @@ check_PROGRAMS += tst_fillonly TESTS += test_fillonly.sh # Szip Tests (requires ncdump) -if USE_SZIP +if HAVE_H5Z_SZIP check_PROGRAMS += test_szip h5testszip TESTS += tst_szip.sh endif -# Filter Tests (requires ncdump and ncgen) +# Filter Tests (requires ncdump and ncgen and HDF5) +if USE_HDF5 if ENABLE_FILTER_TESTING extradir = check_PROGRAMS += test_filter test_filter_misc test_filter_order test_filter_repeat test_filter_vlen -check_PROGRAMS += tst_multifilter +check_PROGRAMS += tst_multifilter tst_filter_avail TESTS += tst_filter.sh -if ENABLE_BLOSC TESTS += tst_specific_filters.sh endif -endif +endif # USE_HDF5 endif # BUILD_UTILITIES # This are extra tests that will only be run if netcdf-4 is configured @@ -102,7 +102,7 @@ ref_tst_xplatform2_1.nc ref_tst_xplatform2_2.nc ref_tst_dims.nc \ ref_tst_interops4.nc CMakeLists.txt run_grp_rename.sh \ run_empty_vlen_test.sh ref_hdf5_compat1.nc ref_hdf5_compat2.nc \ ref_hdf5_compat3.nc tst_misc.sh tdset.h5 tst_szip.sh ref_szip.h5 \ -ref_szip.cdl tst_filter.sh bzip2.cdl ref_filtered.cdl \ +ref_szip.cdl tst_filter.sh bzip2.cdl noop.cdl ref_filtered.cdl \ ref_unfiltered.cdl ref_bzip2.c findplugin.in ref_unfilteredvv.cdl \ ref_filteredvv.cdl ref_multi.cdl \ ref_ncgenF.cdl ref_nccopyF.cdl \ diff --git a/nc_test4/noop.cdl b/nc_test4/noop.cdl new file mode 100644 index 0000000000..0f10ef5c49 --- /dev/null +++ b/nc_test4/noop.cdl @@ -0,0 +1,83 @@ +netcdf bzip2 { +dimensions: + dim0 = 4 ; + dim1 = 4 ; + dim2 = 4 ; + dim3 = 4 ; +variables: + float var(dim0, dim1, dim2, dim3) ; + var:_Storage = "chunked" ; + var:_ChunkSizes = 4, 4, 4, 4 ; + var:_Filter = "40000,17" ; + var:_NoFill = "true" ; + +// global attributes: + :_Format = "netCDF-4" ; +data: + + var = + 0, 1, 2, 3, + 4, 5, 6, 7, + 8, 9, 10, 11, + 12, 13, 14, 15, + 16, 17, 18, 19, + 20, 21, 22, 23, + 24, 25, 26, 27, + 28, 29, 30, 31, + 32, 33, 34, 35, + 36, 37, 38, 39, + 40, 41, 42, 43, + 44, 45, 46, 47, + 48, 49, 50, 51, + 52, 53, 54, 55, + 56, 57, 58, 59, + 60, 61, 62, 63, + 64, 65, 66, 67, + 68, 69, 70, 71, + 72, 73, 74, 75, + 76, 77, 78, 79, + 80, 81, 82, 83, + 84, 85, 86, 87, + 88, 89, 90, 91, + 92, 93, 94, 95, + 96, 97, 98, 99, + 100, 101, 102, 103, + 104, 105, 106, 107, + 108, 109, 110, 111, + 112, 113, 114, 115, + 116, 117, 118, 119, + 120, 121, 122, 123, + 124, 125, 126, 127, + 128, 129, 130, 131, + 132, 133, 134, 135, + 136, 137, 138, 139, + 140, 141, 142, 143, + 144, 145, 146, 147, + 148, 149, 150, 151, + 152, 153, 154, 155, + 156, 157, 158, 159, + 160, 161, 162, 163, + 164, 165, 166, 167, + 168, 169, 170, 171, + 172, 173, 174, 175, + 176, 177, 178, 179, + 180, 181, 182, 183, + 184, 185, 186, 187, + 188, 189, 190, 191, + 192, 193, 194, 195, + 196, 197, 198, 199, + 200, 201, 202, 203, + 204, 205, 206, 207, + 208, 209, 210, 211, + 212, 213, 214, 215, + 216, 217, 218, 219, + 220, 221, 222, 223, + 224, 225, 226, 227, + 228, 229, 230, 231, + 232, 233, 234, 235, + 236, 237, 238, 239, + 240, 241, 242, 243, + 244, 245, 246, 247, + 248, 249, 250, 251, + 252, 253, 254, 255 ; +} diff --git a/nc_test4/test_filter_vlen.c b/nc_test4/test_filter_vlen.c index c41e962d7f..e7775370a2 100644 --- a/nc_test4/test_filter_vlen.c +++ b/nc_test4/test_filter_vlen.c @@ -66,24 +66,13 @@ static int nerrs = 0; static int ncid, varid; static int dimids[MAXDIMS]; -static size_t odom[MAXDIMS]; static float* array = NULL; static float* expected = NULL; -static unsigned int filterid = 0; -static size_t nparams = 0; -static unsigned int params[MAXPARAMS]; - /* Forward */ static int test_test1(void); static void init(int argc, char** argv); static void reset(void); -static void odom_reset(void); -static int odom_more(void); -static int odom_next(void); -static int odom_offset(void); -static float expectedvalue(void); -static void verifyparams(void); #define ERRR do { \ fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ @@ -100,16 +89,7 @@ check(int err,int line) } return NC_NOERR; } - -static void -report(const char* msg, int lineno) -{ - fprintf(stderr,"fail: line=%d %s\n",lineno,msg); - exit(1); -} - #define CHECK(x) check(x,__LINE__) -#define REPORT(x) report(x,__LINE__) static int verifychunks(void) @@ -156,69 +136,6 @@ defvar(nc_type xtype) return NC_NOERR; } -static void -setvarfilter(void) -{ - CHECK(nc_def_var_filter(ncid,varid,TEST_ID,NPARAMS,baseline)); - verifyparams(); -} - -static void -verifyparams(void) -{ - int i; - CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params)); - if(filterid != TEST_ID) REPORT("id mismatch"); - if(nparams != NPARAMS) REPORT("nparams mismatch"); - for(i=0;i 0) { - params = (unsigned int*)malloc(sizeof(unsigned int)*nparams); - if(params == NULL) - return NC_ENOMEM; - CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params)); - } - if(filterid != TEST_ID) { - fprintf(stderr,"open: test id mismatch: %d\n",filterid); - free(params); - return NC_EFILTER; - } - if(nparams != NPARAMS) { - size_t i; - fprintf(stderr,"nparams mismatch\n"); - for(nerrs=0,i=0;i 0) return NC_EFILTER; - - /* Verify chunking */ - if(!verifychunks()) - return 0; - fflush(stderr); - return 1; -} - static int setchunking(void) { @@ -231,91 +148,6 @@ setchunking(void) return NC_NOERR; } -static void -fill(void) -{ - odom_reset(); - if(1) { - int i; - if(actualproduct <= 1) abort(); - for(i=0;i= MAXERRS) - break; - } - } - } else - { - odom_reset(); - while(odom_more()) { - int offset = odom_offset(); - float expect = expectedvalue(); - if(array[offset] != expect) { - fprintf(stderr,"data mismatch: array[%d]=%f expected=%f\n", - offset,array[offset],expect); - errs++; - if(errs >= MAXERRS) - break; - } - odom_next(); - } - } - - if(errs == 0) - fprintf(stderr,"no data errors\n"); - return (errs == 0); -} - -static void -showparameters(void) -{ - int i; - fprintf(stderr,"test: nparams=%ld: params=",(unsigned long)nparams); - for(i=0;i=0;i--) { - odom[i] += 1; - if(odom[i] < dimsize[i]) break; - if(i == 0) return 0; /* leave the 0th entry if it overflows*/ - odom[i] = 0; /* reset this position*/ - } - return 1; -} - -static int -odom_offset(void) -{ - int i; - int offset = 0; - for(i=0;i$2 } -# Hide/unhide the bzip2 filter -hidebzip2() { +# Hide/unhide the noop filter +hidenoop() { rm -fr ${HDF5_PLUGIN_PATH}/save mkdir ${HDF5_PLUGIN_PATH}/save - mv ${BZIP2PATH} ${HDF5_PLUGIN_PATH}/save + mv ${NOOPPATH} ${HDF5_PLUGIN_PATH}/save } -unhidebzip2() { - mv ${HDF5_PLUGIN_PATH}/save/${BZIP2LIB} ${HDF5_PLUGIN_PATH} +unhidenoop() { + mv ${HDF5_PLUGIN_PATH}/save/${NOOPLIB} ${HDF5_PLUGIN_PATH} rm -fr ${HDF5_PLUGIN_PATH}/save } @@ -61,6 +61,10 @@ BZIP2PATH="${HDF5_PLUGIN_PATH}/${BZIP2LIB}" # Find misc and capture findplugin h5misc MISCPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}" +# Find noop and capture +findplugin h5noop +NOOPLIB="${HDF5_PLUGIN_LIB}" +NOOPPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}" echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" export HDF5_PLUGIN_PATH @@ -68,14 +72,10 @@ export HDF5_PLUGIN_PATH # Verify if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi +if ! test -f ${NOOPPATH} ; then echo "Unable to locate ${NOOPPATH}"; exit 1; fi # See if we have szip -HAVE_SZIP=0 -if test -f ${TOPBUILDDIR}/libnetcdf.settings ; then - if grep "SZIP Support:[ ]*yes" <${TOPBUILDDIR}/libnetcdf.settings ; then - HAVE_SZIP=1 - fi -fi +if avail szip; then HAVE_SZIP=1; else HAVE_SZIP=0; fi # Execute the specified tests @@ -181,31 +181,31 @@ fi if test "x$UNK" = x1 ; then echo "*** Testing access to filter info when filter dll is not available" -rm -f bzip2.nc ./tmp_filter.txt -# xfail build bzip2.nc -hidebzip2 -if ${NCGEN} -lb -4 -o bzip2.nc ${srcdir}/bzip2.cdl ; then +rm -f noop.nc ./tmp_filter.txt +# xfail build noop.nc +hidenoop +if ${NCGEN} -lb -4 -o noop.nc ${srcdir}/noop.cdl ; then echo "*** FAIL: ncgen" else echo "*** XFAIL: ncgen" fi -unhidebzip2 -# build bzip2.nc -${NCGEN} -lb -4 -o bzip2.nc ${srcdir}/bzip2.cdl +unhidenoop +# build noop.nc +${NCGEN} -lb -4 -o noop.nc ${srcdir}/noop.cdl # Now hide the filter code -hidebzip2 +hidenoop rm -f ./tmp_filter.txt # This will xfail -if ${NCDUMP} -s bzip2.nc > ./tmp_filter.txt ; then - echo "*** FAIL: ncdump -hs bzip2.nc" +if ${NCDUMP} -s noop.nc > ./tmp_filter.txt ; then + echo "*** FAIL: ncdump -hs noop.nc" else - echo "*** XFAIL: ncdump -hs bzip2.nc" + echo "*** XFAIL: ncdump -hs noop.nc" fi # Restore the filter code -unhidebzip2 +unhidenoop # Verify we can see filter when using -h rm -f ./tmp_filter.txt -${NCDUMP} -hs bzip2.nc > ./tmp_filter.txt +${NCDUMP} -hs noop.nc > ./tmp_filter.txt echo "*** Pass: unknown filter" fi @@ -222,7 +222,7 @@ echo "*** Testing multiple filters" rm -f ./tmp_multifilter.nc ./tmp_multi.txt ./tmp_smulti.cdl rm -f tmp_nccopyF.cdl tmp_nccopyF.nc tmp_ncgenF.cdl tmp_ncgenF.nc ${execdir}/tst_multifilter -${NCDUMP} -hs -n multifilter tmp_multifilter.nc >./tmp_multi.cdl +${NCDUMP} -hsF -n multifilter tmp_multifilter.nc >./tmp_multi.cdl # Remove irrelevant -s output sclean ./tmp_multi.cdl ./tmp_smulti.cdl diff -b -w ${srcdir}/ref_multi.cdl ./tmp_smulti.cdl @@ -237,7 +237,7 @@ diff -b -w ${srcdir}/ref_nccopyF.cdl ./tmp_nccopyFs.cdl echo "*** ncgen with multiple filters" ${NCGEN} -4 -lb -o tmp_ncgenF.nc ${srcdir}/ref_nccopyF.cdl # Need to fix name using -n -${NCDUMP} -hs -n nccopyF tmp_ncgenF.nc > ./tmp_ncgenF.cdl +${NCDUMP} -hsF -n nccopyF tmp_ncgenF.nc > ./tmp_ncgenF.cdl sclean tmp_ncgenF.cdl tmp_ncgenFs.cdl diff -b -w ${srcdir}/ref_nccopyF.cdl ./tmp_ncgenFs.cdl echo "*** Pass: multiple filters" diff --git a/nc_test4/tst_filter_avail.c b/nc_test4/tst_filter_avail.c new file mode 100644 index 0000000000..39fa323f95 --- /dev/null +++ b/nc_test4/tst_filter_avail.c @@ -0,0 +1,106 @@ +/* + Copyright 2018, UCAR/Unidata + See COPYRIGHT file for copying and redistribution conditions. +*/ + +#include "config.h" +#include +#include +#include + +#ifdef USE_HDF5 +#include +#endif + +#include "netcdf.h" +#include "netcdf_aux.h" +#include "netcdf_filter.h" + +#undef DEBUG + +#define MAXPARAMS 32 + +#ifdef TESTNCZARR +#define DFALT_TESTFILE "file://tmp_filter_avail.file#mode=nczarr,file" +#else +#define DFALT_TESTFILE "tmp_filter_avail.nc" +#endif + +static const char* testfile = NULL; +static int nerrs = 0; + +static int ncid; + +/* Forward */ +static int test_test1(void); +static void init(int argc, char** argv); + +#define ERRR do { \ +fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ +fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ +nerrs++;\ +} while (0) + +static int +check(int err,int line) +{ + if(err != NC_NOERR) { + fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err)); + } + return NC_NOERR; +} + +#define CHECK(x) check(x,__LINE__) + +static int +test_test1(void) +{ + int stat = NC_NOERR; + + printf("test1: bzip2 availability\n"); + CHECK(nc_create(testfile,NC_NETCDF4|NC_CLOBBER,&ncid)); + CHECK(nc_enddef(ncid)); + switch (stat = nc_inq_filter_avail(ncid,H5Z_FILTER_BZIP2)) { + case NC_NOERR: break; + case NC_ENOFILTER: break; + default: CHECK(stat); goto done; + } + if(stat == NC_ENOFILTER) { + printf("*** FAIL: filter %d not available\n",H5Z_FILTER_BZIP2); + } else { + printf("*** PASS: filter %d available\n",H5Z_FILTER_BZIP2); + } + + CHECK(nc_abort(ncid)); +done: + return stat; +} + +/**************************************************/ +/* Utilities */ + +static void +init(int argc, char** argv) +{ + /* get the testfile path */ + if(argc > 1) + testfile = argv[1]; + else + testfile = DFALT_TESTFILE; +} + +/**************************************************/ +int +main(int argc, char **argv) +{ +#ifdef USE_HDF5 +#ifdef DEBUG + H5Eprint1(stderr); + nc_set_log_level(1); +#endif +#endif + init(argc,argv); + if(test_test1() != NC_NOERR) ERRR; + exit(nerrs > 0?1:0); +} diff --git a/nc_test4/tst_interops5.c b/nc_test4/tst_interops5.c index 2a478bce8d..d696eb09a7 100644 --- a/nc_test4/tst_interops5.c +++ b/nc_test4/tst_interops5.c @@ -180,7 +180,7 @@ main(int argc, char **argv) } } SUMMARIZE_ERR; -#ifdef USE_SZIP +#ifdef HAVE_H5Z_SZIP printf("*** testing HDF5 compatibility with szip..."); { @@ -254,7 +254,7 @@ main(int argc, char **argv) } SUMMARIZE_ERR; -#endif /* USE_SZIP */ +#endif /* HAVE_H5Z_SZIP */ /* This test suggested by user brentd42 to find a memory problem in * function rec_read_metadata(). This test demonstrates the bug on * address sanitizer runs. See diff --git a/nc_test4/tst_parallel5.c b/nc_test4/tst_parallel5.c index 47d661f927..2809ee3162 100644 --- a/nc_test4/tst_parallel5.c +++ b/nc_test4/tst_parallel5.c @@ -370,7 +370,7 @@ main(int argc, char **argv) } if (!mpi_rank) SUMMARIZE_ERR; -#ifdef USE_SZIP +#ifdef HAVE_H5Z_SZIP #ifdef HDF5_SUPPORTS_PAR_FILTERS #define SZIP_DIM_LEN 256 #define SZIP_DIM_NAME "Barrels" @@ -419,9 +419,9 @@ main(int argc, char **argv) if (!mpi_rank) SUMMARIZE_ERR; #endif /* HDF5_SUPPORTS_PAR_FILTERS */ -#endif /* USE_SZIP */ +#endif /* HAVE_H5Z_SZIP */ -#ifdef USE_SZIP +#ifdef HAVE_H5Z_SZIP #ifdef HDF5_SUPPORTS_PAR_FILTERS #define SZIP_DIM_LEN 256 #define SZIP_DIM_NAME "Barrels" @@ -469,7 +469,7 @@ main(int argc, char **argv) if (!mpi_rank) SUMMARIZE_ERR; #endif /* HDF5_SUPPORTS_PAR_FILTERS */ -#endif /* USE_SZIP */ +#endif /* HAVE_H5Z_SZIP */ /* Shut down MPI. */ MPI_Finalize(); diff --git a/nc_test4/tst_quantize.c b/nc_test4/tst_quantize.c index 4abea412a6..5b7c1e4c0a 100644 --- a/nc_test4/tst_quantize.c +++ b/nc_test4/tst_quantize.c @@ -210,7 +210,6 @@ main(int argc, char **argv) char url[4096]; snprintf(url,sizeof(url),template,file_name); strcpy(file_name,url); -fprintf(stderr,"\n>>> type url = |%s|\n",file_name); } #endif /* Create file. */ @@ -228,7 +227,8 @@ fprintf(stderr,"\n>>> type url = |%s|\n",file_name); { if (nc_open(file_name, NC_NETCDF4, &ncid)) ERR; if (nc_inq_varid(ncid,VAR_NAME,&varid)) ERR; - if (nc_inq_var_quantize(ncid, varid, &quantize_mode, &nsd_in)) ERR; + if (nc_inq_var_quantize(ncid, varid, &quantize_mode, &nsd_in)) + ERR; if (quantize_mode) ERR; if (nc_close(ncid)) ERR; } @@ -931,7 +931,6 @@ fprintf(stderr,"\n>>> type url = |%s|\n",file_name); * set compression level to 1, which is usually the best * choice. */ if (nc_def_var_deflate(ncid, varid1, 0, 1, 1)) ERR; - if (nc_def_var_deflate(ncid, varid2, 0, 1, 1)) ERR; #endif #endif /* Write the data. */ diff --git a/nc_test4/tst_specific_filters.sh b/nc_test4/tst_specific_filters.sh index 36ad8eaf10..874aabcd0c 100755 --- a/nc_test4/tst_specific_filters.sh +++ b/nc_test4/tst_specific_filters.sh @@ -1,10 +1,24 @@ #!/bin/bash # Test the implementations of specific filters +# Also test nc_inq_filter_avail + +# WARNING: This file is also used to build nczarr_test/run_specific_filters.sh if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh +set -e + +if test "x$TESTNCZARR" = x1 ; then +. "$srcdir/test_nczarr.sh" +BLOSCARGS="32001,0,0,0,256,5,1,1" +BLOSCCODEC='[{\"id\": \"blosc\",\"clevel\": 5,\"blocksize\": 256,\"cname\": \"lz4\",\"shuffle\": 1}]' +else +BLOSCARGS="32001,2,2,4,256,5,1,1" +BLOSCCODEC='[{\"id\": \"blosc\",\"clevel\": 5,\"blocksize\": 256,\"cname\": \"lz4\",\"shuffle\": 1}]' +fi + # Load the findplugins function . ${builddir}/findplugin.sh echo "findplugin.sh loaded" @@ -17,8 +31,6 @@ findplugin h5bzip2 echo "final HDF5_PLUGIN_PATH=${HDF5_PLUGIN_PATH}" export HDF5_PLUGIN_PATH -set -e - # Function to remove selected -s attributes from file; # These attributes might be platform dependent sclean() { @@ -50,6 +62,7 @@ trimleft() { sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 } + setfilter() { FF="$1" FSRC="$2" @@ -76,56 +89,98 @@ zext=$1 zfilt="$2" zparams="$3" zcodec="$4" -echo "*** Testing processing of filter $zfilt" -file="tmp_${zfilt}.nc" -rm -f "tmp_${zfilt},nc" -setfilter $zfilt ref_any.cdl "tmp_${zfilt}.cdl" "$zparams" "$zcodec" -if ${NCGEN} -4 -lb -o $file "tmp_${zfilt}.cdl" ; then - ${NCDUMP} -n $zfilt -s $file > "tmp_${zfilt}.tmp" - sclean "tmp_${zfilt}.tmp" "tmp_${zfilt}.dump" +echo "*** Testing processing of filter $zfilt for map $zext" +if test "x$TESTNCZARR" = x1 ; then +fileargs "tmp_filt_${zfilt}" +deletemap $zext $file +else +file="tmp_filt_${zfilt}.nc" +rm -f $file +fi +setfilter $zfilt ref_any.cdl "tmp_filt_${zfilt}.cdl" "$zparams" "$zcodec" +if test "x$TESTNCZARR" = x1 ; then +${NCGEN} -4 -lb -o $fileurl "tmp_filt_${zfilt}.cdl" +${NCDUMP} -n $zfilt -sF $fileurl > "tmp_filt_${zfilt}.tmp" +else +${NCGEN} -4 -lb -o $file "tmp_filt_${zfilt}.cdl" +${NCDUMP} -n $zfilt -sF $file > "tmp_filt_${zfilt}.tmp" fi -unset NCTRACING +sclean "tmp_filt_${zfilt}.tmp" "tmp_filt_${zfilt}.dump" +} + +testfletcher32() { + zext=$1 + runfilter $zext fletcher32 '3' '[{\"id\": \"fletcher32\"}]' + # need to do fixup + sed -e '/_Fletcher32 = "true"/d' -e '/_Filter = 3'/d -e '/_Codecs = \"[{\"id\": \"fletcher32\"}]\"/d' \ + < tmp_filt_fletcher32.cdl > tmp_filt_fletcher32x.dump + diff -b -w "tmp_filt_fletcher32.cdl" "tmp_filt_fletcher32x.dump" +} + +testshuffle() { + zext=$1 + runfilter $zext shuffle '2' '[{\"id\": \"shuffle\",\"elementsize\": \"0\"}]' + # need to replace _Filter + sed -e '/_Shuffle = "true"/d' -e '/_Filter = 2'/d -e '/_Codecs = \"[{\"id\": \"shuffle\"}]\"/d' \ + < tmp_filt_shuffle.cdl > tmp_filt_shufflex.dump + diff -b -w "tmp_filt_shuffle.cdl" "tmp_filt_shufflex.dump" } testdeflate() { zext=$1 + if ! avail deflate; then return 0; fi runfilter $zext deflate '1,9' '[{\"id\": \"zlib\",\"level\": \"9\"}]' - if test -f "tmp_deflate.dump" ; then - # need to replace _DeflateLevel - sed -e 's/_DeflateLevel = 9/_Filter = "1,9"/' < tmp_deflate.dump > tmp_deflatex.dump - diff -b -w "tmp_deflate.cdl" "tmp_deflatex.dump" - else - echo "XFAIL: filter=deflate" - fi + # need to replace _DeflateLevel +# sed -e 's/_DeflateLevel = 9/_Filter = "1,9"/' < tmp_filt_deflate.dump > tmp_filt_deflatex.dump + diff -b -w "tmp_filt_deflate.cdl" "tmp_filt_deflate.dump" } testbzip2() { zext=$1 + if ! avail bzip2; then return 0; fi runfilter $zext bzip2 '307,9' '[{\"id\": \"bz2\",\"level\": \"9\"}]' - if test -f "tmp_bzip2.dump" ; then - diff -b -w "tmp_bzip2.cdl" "tmp_bzip2.dump" - else - echo "XFAIL: filter=bzip2" - fi + diff -b -w "tmp_filt_bzip2.cdl" "tmp_filt_bzip2.dump" +} + +testszip() { + zext=$1 + if ! avail szip; then return 0; fi +# H5_SZIP_NN_OPTION_MASK=32; H5_SZIP_MAX_PIXELS_PER_BLOCK_IN=32 + runfilter $zext szip '4,32,32' '[{\"id\": \"szip\",\"mask\": 32,\"pixels-per-block\": 32}]' + diff -b -w "tmp_filt_szip.cdl" "tmp_filt_szip.dump" } testblosc() { zext=$1 - runfilter $zext blosc '32001,2,2,4,256,5,1,1' '[{\"id\": \"blosc\",\"clevel\": 5,\"blocksize\": 0,\"cname\": \"lz4\",\"shuffle\": 1}]' - if test -f "tmp_blosc.dump" ; then - diff -b -w "tmp_blosc.cdl" "tmp_blosc.dump" - else - echo "XFAIL: filter=blosc" - fi + if ! avail blosc; then return 0; fi + runfilter $zext blosc $BLOSCARGS "$BLOSCCODEC" + diff -b -w "tmp_filt_blosc.cdl" "tmp_filt_blosc.dump" +} + +testzstd() { + zext=$1 + if ! avail zstd; then return 0; fi + runfilter $zext zstd '32015,1' '[{\"id\": \"zstd\",\"level\": \"1\"}]' + diff -b -w "tmp_filt_zstd.cdl" "tmp_filt_zstd.dump" } testset() { # Which test cases to exercise +if test "x$TESTNCZARR" = x1 ; then +# testfletcher32 $1 + testshuffle $1 +fi # testdeflate $1 - testbzip2 $1 +# testszip $1 +# testbzip2 $1 # testblosc $1 +# testzstd $1 } -testset nc - -exit 0 +if test "x$TESTNCZARR" = x1 ; then + testset file + if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi + if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi +else + testset nc +fi diff --git a/nc_test4/tst_udf.c b/nc_test4/tst_udf.c index 43d44a016c..1c7acc47b9 100644 --- a/nc_test4/tst_udf.c +++ b/nc_test4/tst_udf.c @@ -158,6 +158,13 @@ static NC_Dispatch tst_dispatcher = { NC_NOOP_inq_var_filter_ids, NC_NOOP_inq_var_filter_info, #endif +#if NC_DISPATCH_VERSION >= 4 + NC_NOTNC4_def_var_quantize, + NC_NOTNC4_inq_var_quantize, +#endif +#if NC_DISPATCH_VERSION >= 5 + NC_NOOP_inq_filter_avail, +#endif }; /* This is the dispatch object that holds pointers to all the @@ -250,10 +257,17 @@ static NC_Dispatch tst_dispatcher_bad_version = { NC_NOTNC4_def_var_filter, NC_NOTNC4_set_var_chunk_cache, NC_NOTNC4_get_var_chunk_cache, -#if NC_DISPATCH_VERSION >= 2 +#if NC_DISPATCH_VERSION >= 3 NC_NOOP_inq_var_filter_ids, NC_NOOP_inq_var_filter_info, #endif +#if NC_DISPATCH_VERSION >= 4 + NC_NOTNC4_def_var_quantize, + NC_NOTNC4_inq_var_quantize, +#endif +#if NC_DISPATCH_VERSION >= 5 + NC_NOOP_inq_filter_avail, +#endif }; #define NUM_UDFS 2 diff --git a/nc_test4/tst_vars3.c b/nc_test4/tst_vars3.c index 37eca905aa..f3fc064e9a 100644 --- a/nc_test4/tst_vars3.c +++ b/nc_test4/tst_vars3.c @@ -420,7 +420,7 @@ main(int argc, char **argv) if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; -#ifdef USE_SZIP +#ifdef HAVE_H5Z_SZIP printf("**** testing simple szip filter setup..."); { int ncid; @@ -731,7 +731,7 @@ main(int argc, char **argv) SUMMARIZE_ERR; #else /* This code is run if szip is not present in HDF5. It checks that - * nc_def_var_szip() returns NC_EFILTER in that case. */ + * nc_def_var_szip() returns NC_ENOFILTER in that case. */ printf("**** testing szip handling when szip not built..."); { int ncid; @@ -749,15 +749,16 @@ main(int argc, char **argv) params[0] = NC_SZIP_NN; /* options_mask */ params[1] = NC_SZIP_EC_BPP_IN; /* pixels_per_block */ if (nc_def_var_chunking(ncid, varid, NC_CHUNKED, NULL)) ERR; - { int stat; if ((stat = nc_def_var_filter(ncid, varid, H5_FILTER_SZIP, NUM_PARAMS_IN, - params)) != NC_EFILTER) + { int stat; + if ((stat = nc_def_var_filter(ncid, varid, H5_FILTER_SZIP, NUM_PARAMS_IN, + params)) != NC_ENOFILTER) ERR; } if (nc_def_var_szip(ncid, varid, NC_SZIP_NN, - NC_SZIP_EC_BPP_IN) != NC_EFILTER) ERR; + NC_SZIP_EC_BPP_IN) != NC_ENOFILTER) ERR; if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; -#endif /* USE_SZIP */ +#endif /* HAVE_H5Z_SZIP */ FINAL_RESULTS; } diff --git a/nc_test4/tst_vars4.c b/nc_test4/tst_vars4.c index 28dabde919..755521b201 100644 --- a/nc_test4/tst_vars4.c +++ b/nc_test4/tst_vars4.c @@ -34,7 +34,7 @@ main(int argc, char **argv) nc_type type_in; size_t len_in; - nc_set_log_level(3); + nc_set_log_level(-1); if (nc_create(FILE_NAME, NC_NETCDF4 | NC_CLOBBER, &ncid)) ERR; if (nc_def_dim(ncid, X_NAME, XDIM_LEN, &dims[0])) ERR; if (nc_def_dim(ncid, Y_NAME, YDIM_LEN, &dims[1])) ERR; diff --git a/ncdump/CMakeLists.txt b/ncdump/CMakeLists.txt index c0aee2a7f3..2e9001b4ab 100644 --- a/ncdump/CMakeLists.txt +++ b/ncdump/CMakeLists.txt @@ -17,6 +17,7 @@ SET(ocprint_FILES ocprint.c) SET(ncvalidator_FILES ncvalidator.c) SET(printfqn_FILES printfqn.c) SET(ncpathcvt_FILES ncpathcvt.c) +SET(ncfilteravail_FILES ncfilteravail.c) SET(nchdf5version_FILES nchdf5version.c) IF(USE_X_GETOPT) @@ -26,12 +27,14 @@ IF(USE_X_GETOPT) SET(ncvalidator_FILES ${ncvalidator_FILES} XGetopt.c) SET(printfqn_FILES ${printfqn_FILES} XGetopt.c) SET(ncpathcvt_FILES ${ncpathcvt_FILES} XGetopt.c) + SET(ncfilteravail_FILES ${ncfilteravail_FILES} XGetopt.c) ENDIF(USE_X_GETOPT) ADD_EXECUTABLE(ncdump ${ncdump_FILES}) ADD_EXECUTABLE(nccopy ${nccopy_FILES}) ADD_EXECUTABLE(ncvalidator ${ncvalidator_FILES}) ADD_EXECUTABLE(ncpathcvt ${ncpathcvt_FILES}) +ADD_EXECUTABLE(ncfilteravail ${ncfilteravail_FILES}) IF(USE_HDF5) ADD_EXECUTABLE(nc4print nc4print.c nc4printer.c) @@ -47,6 +50,7 @@ TARGET_LINK_LIBRARIES(ncdump netcdf ${ALL_TLL_LIBS}) TARGET_LINK_LIBRARIES(nccopy netcdf ${ALL_TLL_LIBS}) TARGET_LINK_LIBRARIES(ncvalidator netcdf ${ALL_TLL_LIBS}) TARGET_LINK_LIBRARIES(ncpathcvt netcdf ${ALL_TLL_LIBS}) +TARGET_LINK_LIBRARIES(ncfilteravail netcdf ${ALL_TLL_LIBS}) IF(USE_HDF5) TARGET_LINK_LIBRARIES(nc4print netcdf ${ALL_TLL_LIBS}) @@ -64,57 +68,30 @@ ENDIF(ENABLE_DAP) # in MSVC builds. This is required to get # test scripts to work. #### -IF(MSVC) - SET_TARGET_PROPERTIES(ncdump PROPERTIES RUNTIME_OUTPUT_DIRECTORY - ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(ncdump PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG - ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(ncdump PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE - ${CMAKE_CURRENT_BINARY_DIR}) - - SET_TARGET_PROPERTIES(nccopy PROPERTIES RUNTIME_OUTPUT_DIRECTORY - ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(nccopy PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG - ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(nccopy PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE - ${CMAKE_CURRENT_BINARY_DIR}) - - SET_TARGET_PROPERTIES(ncvalidator PROPERTIES RUNTIME_OUTPUT_DIRECTORY - ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(ncvalidator PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG - ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(ncvalidator PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE - ${CMAKE_CURRENT_BINARY_DIR}) - - SET_TARGET_PROPERTIES(ncpathcvt PROPERTIES RUNTIME_OUTPUT_DIRECTORY - ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(ncpathcvt PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG - ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(ncpathcvt PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE - ${CMAKE_CURRENT_BINARY_DIR}) - IF(USE_HDF5) - SET_TARGET_PROPERTIES(printfqn PROPERTIES RUNTIME_OUTPUT_DIRECTORY - ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(printfqn PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG - ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(printfqn PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE - ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(nchdf5version PROPERTIES RUNTIME_OUTPUT_DIRECTORY +IF(MSVC) + macro(setbinprops name) + SET_TARGET_PROPERTIES(${name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(nchdf5version PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG + SET_TARGET_PROPERTIES(${name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(nchdf5version PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE + SET_TARGET_PROPERTIES(${name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR}) + endmacro() + + setbinprops(ncdump) + setbinprops(nccopy) + setbinprops(ncvalidator) + setbinprops(ncpathcvt) + setbinprops(ncfilteravail) + + IF(USE_HDF5) + setbinprops(printfqn) + setbinprops(nchdf5version) ENDIF(USE_HDF5) IF(ENABLE_DAP) - SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY - ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG - ${CMAKE_CURRENT_BINARY_DIR}) - SET_TARGET_PROPERTIES(ocprint PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE - ${CMAKE_CURRENT_BINARY_DIR}) + setbinprops(ocprint) ENDIF(ENABLE_DAP) ENDIF() diff --git a/ncdump/Makefile.am b/ncdump/Makefile.am index d5db8cf3a8..1efa5f4632 100644 --- a/ncdump/Makefile.am +++ b/ncdump/Makefile.am @@ -41,6 +41,10 @@ ncvalidator_SOURCES = ncvalidator.c noinst_PROGRAMS += ncpathcvt ncpathcvt_SOURCES = ncpathcvt.c +# A non-installed utility program to test if a filter is available +noinst_PROGRAMS += ncfilteravail +ncfilteravail_SOURCES = ncfilteravail.c + # A simple netcdf-4 metadata -> xml printer. Do not install. if USE_HDF5 bin_PROGRAMS += nc4print diff --git a/ncdump/nccopy.c b/ncdump/nccopy.c index 5b0a331124..2ac4a73d9c 100644 --- a/ncdump/nccopy.c +++ b/ncdump/nccopy.c @@ -31,7 +31,6 @@ #include "nccomps.h" #include "list.h" #include "ncpathmgr.h" -#include "ncfilter.h" #undef DEBUGFILTER #undef DEBUGCHUNK diff --git a/ncdump/ncdump.1 b/ncdump/ncdump.1 index bc835e0965..6e904cd9bb 100644 --- a/ncdump/ncdump.1 +++ b/ncdump/ncdump.1 @@ -7,7 +7,7 @@ ncdump \- Convert netCDF file to text form (CDL) .HP ncdump .nh -\%[\-chistxw] +\%[\-chistxwF] \%[\-v \fIvar1,...\fP] \%[\-b \fIlang\fP] \%[\-f \fIlang\fP] @@ -230,6 +230,8 @@ with client-side caching of entire variables. .IP "\fB-x\fP" Output XML (NcML) instead of CDL. The NcML does not include data values. The NcML output option currently only works for netCDF classic model data. +.IP "\fB-F\fP" +Use _Filter and _Codecs attributes in place of _Fletcher32, _Shuffle, and _Deflate. .SH EXAMPLES .LP Look at the structure of the data in the netCDF file `\fBfoo.nc\fP': diff --git a/ncdump/ncdump.c b/ncdump/ncdump.c index 6a1e720c6c..1098d35dd3 100644 --- a/ncdump/ncdump.c +++ b/ncdump/ncdump.c @@ -95,6 +95,7 @@ fspec_t formatting_specs = /* defaults, overridden by command-line options */ false, /* human-readable output for date-time values? */ false, /* use 'T' separator between date and time values as strings? */ false, /* output special attributes, eg chunking? */ + false, /* if -F specified */ LANG_C, /* language conventions for indices */ false, /* for DAP URLs, client-side cache used */ 0, /* if -v specified, number of variables in list */ @@ -124,12 +125,13 @@ usage(void) [-g grp1[,...]] Data and metadata for group(s) ,... only\n\ [-w] With client-side caching of variables for DAP URLs\n\ [-x] Output XML (NcML) instead of CDL\n\ + [-F] Output _Filter and _Codecs instead of _Fletcher32, _Shuffle, and _Deflate\n\ [-Xp] Unconditionally suppress output of the properties attribute\n\ [-Ln] Set log level to n (>= 0); ignore if logging not enabled.\n\ file Name of netCDF file (or URL if DAP access enabled)\n" (void) fprintf(stderr, - "%s [-c|-h] [-v ...] [[-b|-f] [c|f]] [-l len] [-n name] [-p n[,n]] [-k] [-x] [-s] [-t|-i] [-g ...] [-w] [-Ln] file\n%s", + "%s [-c|-h] [-v ...] [[-b|-f] [c|f]] [-l len] [-n name] [-p n[,n]] [-k] [-x] [-s] [-t|-i] [-g ...] [-w] [-F] [-Ln] file\n%s", progname, USAGE); @@ -1005,48 +1007,73 @@ pr_att_specials( printf(" = \"unknown\" ;\n"); } - /* _Filter (including deflate and shuffle) */ + /* _Checksum (fletcher32) */ + if(!formatting_specs.filter_atts) { + int fletcher32 = 0; + NC_CHECK( nc_inq_var_fletcher32(ncid, varid, &fletcher32) ); + if(fletcher32 != 0) { + pr_att_name(ncid, varp->name, NC_ATT_CHECKSUM); + printf(" = \"true\" ;\n"); + } + } + /* _Shuffle */ + if(!formatting_specs.filter_atts) { + int haveshuffle = 0; + NC_CHECK( nc_inq_var_deflate(ncid, varid, &haveshuffle, NULL, NULL)); + if(haveshuffle) { + pr_att_name(ncid, varp->name, NC_ATT_SHUFFLE); + printf(" = \"true\" ;\n"); + } + } + + /* _Deflate*/ + if(!formatting_specs.filter_atts) { + int havedeflate = 0; + int level = -1; + NC_CHECK( nc_inq_var_deflate(ncid, varid, NULL, &havedeflate, &level)); + if(havedeflate) { + pr_att_name(ncid, varp->name, NC_ATT_DEFLATE); + printf(" = %d ;\n",level); + } + } + + /* _Filter */ { size_t nparams, nfilters, nbytes; - int shuffle=NC_NOSHUFFLE; unsigned int* filterids = NULL; unsigned int* params = NULL; - int usedeflateatt = 0; /* Get applicable filter ids */ NC_CHECK(nc_inq_var_filter_ids(ncid, varid, &nfilters, NULL)); /* Get set of filters for this variable */ + filterids = NULL; if(nfilters > 0) { filterids = (unsigned int*)malloc(sizeof(unsigned int)*nfilters); if(filterids == NULL) NC_CHECK(NC_ENOMEM); - } else - filterids = NULL; - NC_CHECK(nc_inq_var_filter_ids(ncid, varid, &nfilters, filterids)); + NC_CHECK(nc_inq_var_filter_ids(ncid, varid, &nfilters, filterids)); + } if(nfilters > 0) { int k; - int pratt = 0; + int first = 1; + int _filter = 0; for(k=0;k 0) { params = (unsigned int*)calloc(1,sizeof(unsigned int)*nparams); NC_CHECK(nc_inq_var_filter_info(ncid, varid, filterids[k], &nbytes, params)); } else params = NULL; - /* Use _Deflate if the first filter is zip */ - if(k == 0 && filterids[k] == H5Z_FILTER_DEFLATE) { - pr_att_name(ncid, varp->name, NC_ATT_DEFLATE); - printf(" = %d", (int)params[0]); - pratt = 1; - usedeflateatt = 1; - nullfree(params); params = NULL; - continue; - } - if(pratt || k == 0) { + if(first) { pr_att_name(ncid,varp->name,NC_ATT_FILTER); printf(" = \""); - pratt = 0; - } - if(k > 0) printf("|"); + } else + printf("|"); printf("%u",filterids[k]); if(nparams > 0) { int i; @@ -1054,17 +1081,11 @@ pr_att_specials( printf(",%u",params[i]); } nullfree(params); params = NULL; + first = 0; } - if(!usedeflateatt) printf("\""); - printf(" ;\n"); + if(_filter) printf("\" ;\n"); } if(filterids) free(filterids); - /* Finally, do Shuffle */ - NC_CHECK( nc_inq_var_deflate(ncid, varid, &shuffle, NULL, NULL)); - if(shuffle != NC_NOSHUFFLE) { - pr_att_name(ncid, varp->name, NC_ATT_SHUFFLE); - printf(" = \"true\" ;\n"); - } } /* _Codecs*/ { @@ -1088,15 +1109,6 @@ pr_att_specials( } } } - /* _Checksum */ - { - int fletcher32 = 0; - NC_CHECK( nc_inq_var_fletcher32(ncid, varid, &fletcher32) ); - if(fletcher32 != 0) { - pr_att_name(ncid, varp->name, NC_ATT_CHECKSUM); - printf(" = \"true\" ;\n"); - } - } /* _Endianness */ if(varp->tinfo->size > 1) /* Endianness is meaningless for 1-byte types */ { @@ -2261,7 +2273,7 @@ main(int argc, char *argv[]) } opterr = 1; - while ((c = getopt(argc, argv, "b:cd:f:g:hikl:n:p:stv:xwKL:X:")) != EOF) + while ((c = getopt(argc, argv, "b:cd:f:g:hikl:n:p:stv:xwFKL:X:")) != EOF) switch(c) { case 'h': /* dump header only, no data */ formatting_specs.header_only = true; @@ -2375,6 +2387,9 @@ main(int argc, char *argv[]) #endif ncsetlogging(1); break; + case 'F': + formatting_specs.filter_atts = true; + break; case '?': usage(); exit(EXIT_FAILURE); diff --git a/ncdump/ncdump.h b/ncdump/ncdump.h index d5a95ef1be..4381c41c99 100644 --- a/ncdump/ncdump.h +++ b/ncdump/ncdump.h @@ -45,6 +45,9 @@ typedef struct { /* specification for how to format dump */ * _Endianness, _Format, _Checksum, * _NoFill, _NetCDF4 */ + bool_t filter_atts; /* if true, output filters using _Filter in place of + * _Fletcher32, _Shuffle, and _Deflate. */ + Nclang data_lang; /* Specifies index conventions used in data * comments, either LANG_C (C, 0-based, * column major) or LANG_F (Fortran, diff --git a/ncdump/ncfilteravail.c b/ncdump/ncfilteravail.c new file mode 100644 index 0000000000..b7e937627f --- /dev/null +++ b/ncdump/ncfilteravail.c @@ -0,0 +1,178 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include + +#ifdef HAVE_UNISTD_H +#include +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) +#include "XGetopt.h" +#else +#include +#endif + +#include "netcdf.h" +#include "ncpathmgr.h" +#include "netcdf_filter.h" + +static const char* USAGE = +"ncfilteravail [-t ] | n" +"Options\n" +" -h help" +" -t the formatx name: nc4|nczarr (optional)\n" +"\n" +" the filter name \n" +" the filter id number \n" +"\n" +"Default format is nc4 if built, then nczarr if built else error.\n" +; + +#undef DEBUG + +struct Options { + char* format; + char* filtername; + unsigned filterid; + char* url; +} faoptions; + +static const struct Filter { + char* name; + unsigned id; +} known_filters[] = { +{"deflate",1}, +{"szip",4}, +{"bzip2",307}, +{"zstd",32015}, +{"zstandard",32015}, +{"blosc",32001}, +{NULL,0} +}; + + +static void usage(const char* msg); + +static void +usage(const char* msg) +{ + if(msg != NULL) fprintf(stderr,"%s\n",msg); + fprintf(stderr,"%s",USAGE); + if(msg == NULL) exit(0); else exit(1); +} + +static int +getformat(const char* sformat) +{ + if(sformat != NULL && strlen(sformat) > 0) { + if(strcmp(sformat,"nc4")==0) return NC_FORMATX_NC4; + if(strcmp(sformat,"nczarr")==0) return NC_FORMATX_NCZARR; + } +#ifdef USE_HDF5 + return NC_FORMATX_NC4; +#else +# ifdef ENABLE_NCZARR + return NC_FORMATX_NCZARR; +# endif +#endif + return NC_FORMATX_UNDEFINED; +} + + +static int +makeurl(struct Options* faoptions) +{ + int stat = NC_NOERR; + int format; + char url[4096]; + int pid; + +#if defined(_WIN32) || defined(__MINGW32__) + pid = (int)_getpid(); +#else + pid = (int)getpid(); +#endif + switch (format = getformat(faoptions->format)) { + case NC_FORMATX_NC4: + snprintf(url,sizeof(url),"tmp_%d.nc",pid); + break; + case NC_FORMATX_NCZARR: + snprintf(url,sizeof(url),"file://tmp_%d.file#mode=zarr,file",pid); + break; + default: stat = NC_EINVAL; break; + } + if(stat == NC_NOERR) faoptions->url = strdup(url); + return stat; +} + +static int +makefilterid(struct Options* faoptions) +{ + int stat = NC_NOERR; + const struct Filter* f = NULL; + for(f=known_filters;f->name;f++) { + if(strcmp(f->name,faoptions->filtername)==0) { + faoptions->filterid = f->id; + goto done; + } + } + /* See if it is a number */ + if(1!=sscanf(faoptions->filtername,"%u",&faoptions->filterid)) + {stat = NC_EINVAL; goto done;} +done: + return stat; +} + +int +main(int argc, char** argv) +{ + int c, ncid, stat = NC_NOERR; + int yesno = 0; + + memset((void*)&faoptions,0,sizeof(faoptions)); + + while ((c = getopt(argc, argv, "t:")) != EOF) { + switch(c) { + case 'h': usage(""); break; + case 'x': faoptions.format = strdup(optarg); break; + case '?': + usage("unknown option"); + break; + } + } + + argc -= optind; + argv += optind; + + /* If no filter arguments left or more than one, print usage message. */ + if (argc == 0) + usage("no filter specified"); + if (argc > 1) + usage("more than one filter specified"); + faoptions.filtername = strdup(argv[0]); + + if(makeurl(&faoptions)) {yesno = 0; goto done;} + if(makefilterid(&faoptions)) {yesno = 0; goto done;} + if((stat = nc_create(faoptions.url,NC_NETCDF4,&ncid))) goto done; + switch(stat = nc_inq_filter_avail(ncid,faoptions.filterid)) { + case NC_NOERR: yesno = 1; break; + case NC_ENOFILTER: yesno = 0; stat = NC_NOERR; break; + default: break; + } + if((stat = nc_abort(ncid))) goto done; + +done: + printf("%s",(yesno?"yes":"no")); + if(stat) fprintf(stderr,"error: (%d) %s\n",stat,nc_strerror(stat)); + return (stat?1:0); +} + diff --git a/ncdump/ref_tst_ncf213.cdl b/ncdump/ref_tst_ncf213.cdl index 291e689015..4b16832a38 100644 --- a/ncdump/ref_tst_ncf213.cdl +++ b/ncdump/ref_tst_ncf213.cdl @@ -22,25 +22,23 @@ variables: int var2(dim1, dim2) ; var2:_Storage = "chunked" ; var2:_ChunkSizes = 6, 7 ; - var2:_Fletcher32 = "true" ; + var2:_Filter = "3" ; var2:_Endianness = "big" ; int var3(dim1, dim2, dim3) ; var3:_Storage = "chunked" ; var3:_ChunkSizes = 6, 7, 8 ; - var3:_DeflateLevel = 2 ; + var3:_Filter = "1,2" ; var3:_Endianness = "little" ; int var4(dim1, dim2, dim3) ; var4:_Storage = "chunked" ; var4:_ChunkSizes = 6, 7, 8 ; - var4:_DeflateLevel = 2 ; + var4:_Filter = "1,2" ; var4:_Endianness = "little" ; var4:_NoFill = "true" ; obs_t var5(dim1) ; var5:_Storage = "chunked" ; var5:_ChunkSizes = 6 ; - var5:_DeflateLevel = 2 ; - var5:_Shuffle = "true" ; - var5:_Fletcher32 = "true" ; + var5:_Filter = "3|2,40|1,2" ; var5:_NoFill = "true" ; // global attributes: diff --git a/ncdump/ref_tst_special_atts.cdl b/ncdump/ref_tst_special_atts.cdl index 5498507965..a5c9e229cd 100644 --- a/ncdump/ref_tst_special_atts.cdl +++ b/ncdump/ref_tst_special_atts.cdl @@ -32,16 +32,16 @@ variables: int var4(dim1, dim2, dim3) ; var4:_Storage = "chunked" ; var4:_ChunkSizes = 6, 7, 8 ; - var4:_DeflateLevel = 2 ; var4:_Shuffle = "true" ; + var4:_DeflateLevel = 2 ; var4:_Endianness = "little" ; var4:_NoFill = "true" ; obs_t var5(dim1) ; var5:_Storage = "chunked" ; var5:_ChunkSizes = 6 ; - var5:_DeflateLevel = 2 ; - var5:_Shuffle = "true" ; var5:_Fletcher32 = "true" ; + var5:_Shuffle = "true" ; + var5:_DeflateLevel = 2 ; int var6(dim1) ; var6:_Storage = "compact" ; var6:_Endianness = "little" ; diff --git a/ncdump/tst_nccopy3.sh b/ncdump/tst_nccopy3.sh index 88bf949284..449e896865 100755 --- a/ncdump/tst_nccopy3.sh +++ b/ncdump/tst_nccopy3.sh @@ -8,9 +8,6 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi . ../test_common.sh -export SETX=1 - - set -e echo "" diff --git a/ncdump/tst_nccopy4.sh b/ncdump/tst_nccopy4.sh index 7b482a78b7..81493b0382 100755 --- a/ncdump/tst_nccopy4.sh +++ b/ncdump/tst_nccopy4.sh @@ -7,7 +7,6 @@ set -e # For a netCDF-4 build, test nccopy on netCDF files in this directory -echo "@@@@@@" if test -f tst_group_data${ext} ; then ${execdir}/tst_group_data ; fi if test -f tst_enum_data${ext} ; then ${execdir}/tst_enum_data ; fi if test -f tst_comp${ext} ; then ${execdir}/tst_comp ; fi diff --git a/ncdump/tst_netcdf4_4.sh b/ncdump/tst_netcdf4_4.sh index c49fe4f976..82f20bfcf9 100755 --- a/ncdump/tst_netcdf4_4.sh +++ b/ncdump/tst_netcdf4_4.sh @@ -68,7 +68,7 @@ diff -b tst_compounds4.cdl ${top_srcdir}/ncdump/ref_tst_compounds4.cdl # rm -f tst_ncf213.cdl tst_ncf213.nc # Remove specific _NCProperties values ${NCGEN} -b -o tst_ncf213.nc $srcdir/ref_tst_ncf213.cdl -${NCDUMP} -s -h tst_ncf213.nc > tst_ncf213.cdl +${NCDUMP} -shF tst_ncf213.nc > tst_ncf213.cdl cleanncprops tst_ncf213.cdl tst_ncf213.tmp cleanncprops ${srcdir}/ref_tst_ncf213.cdl ref_tst_ncf213.tmp # Now compare diff --git a/ncdump/utils.h b/ncdump/utils.h index d27821c65b..e6b746c38a 100644 --- a/ncdump/utils.h +++ b/ncdump/utils.h @@ -18,7 +18,11 @@ #define NC_GRP_DELIM '/' typedef int bool_t; -enum {false=0, true=1}; +#ifndef false +#define false 0 +#define true 1 +//enum {false=0, true=1}; +#endif struct safebuf_t; /* Buffer structure for implementing growable strings, used in diff --git a/ncgen/Makefile.am b/ncgen/Makefile.am index 49a3d64976..5beb613cb9 100644 --- a/ncgen/Makefile.am +++ b/ncgen/Makefile.am @@ -61,7 +61,7 @@ CLEANFILES = c0.nc c0_64.nc c0_4.nc c0_4c.nc ref_camrun.c \ makeparser:: rm -f ncgenl.c lex.ncg.c - flex -Pncg -8 ncgen.l + flex -L -Pncg -8 ncgen.l sed -e s/lex.ncg.c/ncgenl.c/g ncgenl.c bison -pncg -t -d ncgen.y rm -f ncgeny.c ncgeny.h diff --git a/ncgen/genbin.c b/ncgen/genbin.c index a126116294..8a66daa272 100644 --- a/ncgen/genbin.c +++ b/ncgen/genbin.c @@ -7,6 +7,7 @@ #include "includes.h" #include /* for isprint() */ #include "netcdf_aux.h" +#include "netcdf_filter.h" #ifdef ENABLE_BINARY @@ -248,6 +249,15 @@ genbin_definespecialattributes(Symbol* var) int k; for(k=0;knfilters;k++) { NC_H5_Filterspec* nfs = special->_Filters[k]; + /* See if the filter is available */ + stat = nc_inq_filter_avail(var->container->nc_id, nfs->filterid); + switch (stat) { + case NC_NOERR: break; + case NC_ENOFILTER: + derror("Filter id=%u; filter not available",nfs->filterid); + /* fall thru */ + default: CHECK_ERR(stat); break; + } stat = nc_def_var_filter(var->container->nc_id, var->nc_id, nfs->filterid, diff --git a/ncgen/ncgenl.c b/ncgen/ncgenl.c index 5bf7fcce73..976f3bca85 100644 --- a/ncgen/ncgenl.c +++ b/ncgen/ncgenl.c @@ -1,6 +1,4 @@ -#line 3 "ncgenl.c" - #define YY_INT_ALIGNED short int /* A lexical scanner generated by flex */ @@ -1533,8 +1531,6 @@ int yy_flex_debug = 0; #define YY_MORE_ADJ 0 #define YY_RESTORE_YY_MORE_OFFSET char *yytext; -#line 1 "ncgen.l" -#line 2 "ncgen.l" /********************************************************************* * Copyright 1993, UCAR/Unidata * See netcdf/COPYRIGHT file for copying and redistribution conditions. @@ -1679,8 +1675,6 @@ struct Specialtoken specials[] = { {NULL,0} /* null terminate */ }; -#line 1683 "ncgenl.c" - /* The most correct (validating) version of UTF8 character set (Taken from: http://www.w3.org/2005/03/23-lex-U) @@ -1722,7 +1716,6 @@ ID ([A-Za-z_]|{UTF8})([A-Z.@#\[\]a-z_0-9+-]|{UTF8})* /* Note: this definition of string will work for utf8 as well, although it is a very relaxed definition */ -#line 1726 "ncgenl.c" #define INITIAL 0 #define ST_C_COMMENT 1 @@ -1941,9 +1934,6 @@ YY_DECL } { -#line 225 "ncgen.l" - -#line 1947 "ncgenl.c" while ( /*CONSTCOND*/1 ) /* loops until end-of-file is reached */ { @@ -2002,14 +1992,12 @@ YY_DECL case 1: YY_RULE_SETUP -#line 226 "ncgen.l" { /* whitespace */ break; } YY_BREAK case 2: YY_RULE_SETUP -#line 230 "ncgen.l" { /* comment */ break; } @@ -2017,7 +2005,6 @@ YY_RULE_SETUP case 3: /* rule 3 can match eol */ YY_RULE_SETUP -#line 234 "ncgen.l" {int len; char* s = NULL; /* In netcdf4, this will be used in a variety of places, so only remove escapes */ @@ -2041,7 +2028,6 @@ yytext[MAXTRST-1] = '\0'; YY_BREAK case 4: YY_RULE_SETUP -#line 255 "ncgen.l" { /* drop leading 0x; pad to even number of chars */ char* p = yytext+2; int len = yyleng - 2; @@ -2056,143 +2042,115 @@ YY_RULE_SETUP YY_BREAK case 5: YY_RULE_SETUP -#line 267 "ncgen.l" {return lexdebug(COMPOUND);} YY_BREAK case 6: YY_RULE_SETUP -#line 268 "ncgen.l" {return lexdebug(ENUM);} YY_BREAK case 7: YY_RULE_SETUP -#line 269 "ncgen.l" {return lexdebug(OPAQUE_);} YY_BREAK case 8: YY_RULE_SETUP -#line 271 "ncgen.l" {return lexdebug(FLOAT_K);} YY_BREAK case 9: YY_RULE_SETUP -#line 272 "ncgen.l" {return lexdebug(DOUBLE_K);} YY_BREAK case 10: YY_RULE_SETUP -#line 273 "ncgen.l" {return lexdebug(CHAR_K);} YY_BREAK case 11: YY_RULE_SETUP -#line 274 "ncgen.l" {return lexdebug(BYTE_K);} YY_BREAK case 12: YY_RULE_SETUP -#line 275 "ncgen.l" {return lexdebug(SHORT_K);} YY_BREAK case 13: YY_RULE_SETUP -#line 276 "ncgen.l" {return lexdebug(INT_K);} YY_BREAK case 14: YY_RULE_SETUP -#line 277 "ncgen.l" {return lexdebug(identcheck(UBYTE_K));} YY_BREAK case 15: YY_RULE_SETUP -#line 278 "ncgen.l" {return lexdebug(identcheck(USHORT_K));} YY_BREAK case 16: YY_RULE_SETUP -#line 279 "ncgen.l" {return lexdebug(identcheck(UINT_K));} YY_BREAK case 17: YY_RULE_SETUP -#line 280 "ncgen.l" {return lexdebug(identcheck(INT64_K));} YY_BREAK case 18: YY_RULE_SETUP -#line 281 "ncgen.l" {return lexdebug(identcheck(UINT64_K));} YY_BREAK case 19: YY_RULE_SETUP -#line 282 "ncgen.l" {return lexdebug(identcheck(STRING_K));} YY_BREAK case 20: YY_RULE_SETUP -#line 284 "ncgen.l" {return lexdebug(FLOAT_K);} YY_BREAK case 21: YY_RULE_SETUP -#line 285 "ncgen.l" {return lexdebug(INT_K);} YY_BREAK case 22: YY_RULE_SETUP -#line 286 "ncgen.l" {return lexdebug(INT_K);} YY_BREAK case 23: YY_RULE_SETUP -#line 287 "ncgen.l" {return lexdebug(identcheck(UINT_K));} YY_BREAK case 24: YY_RULE_SETUP -#line 288 "ncgen.l" {return lexdebug(identcheck(UINT_K));} YY_BREAK case 25: YY_RULE_SETUP -#line 291 "ncgen.l" {int32_val = -1; return lexdebug(NC_UNLIMITED_K);} YY_BREAK case 26: YY_RULE_SETUP -#line 294 "ncgen.l" {return lexdebug(TYPES);} YY_BREAK case 27: YY_RULE_SETUP -#line 295 "ncgen.l" {return lexdebug(DIMENSIONS);} YY_BREAK case 28: YY_RULE_SETUP -#line 296 "ncgen.l" {return lexdebug(VARIABLES);} YY_BREAK case 29: YY_RULE_SETUP -#line 297 "ncgen.l" {return lexdebug(DATA);} YY_BREAK case 30: YY_RULE_SETUP -#line 298 "ncgen.l" {return lexdebug(GROUP);} YY_BREAK case 31: YY_RULE_SETUP -#line 300 "ncgen.l" {BEGIN(TEXT);return lexdebug(NETCDF);} YY_BREAK case 32: YY_RULE_SETUP -#line 302 "ncgen.l" { /* missing value (pre-2.4 backward compatibility) */ if (yytext[0] == '-') { double_val = -INFINITY; @@ -2205,7 +2163,6 @@ YY_RULE_SETUP YY_BREAK case 33: YY_RULE_SETUP -#line 311 "ncgen.l" { /* missing value (pre-2.4 backward compatibility) */ double_val = NAN; specialconstants = 1; @@ -2214,7 +2171,6 @@ YY_RULE_SETUP YY_BREAK case 34: YY_RULE_SETUP -#line 317 "ncgen.l" {/* missing value (pre-2.4 backward compatibility)*/ if (yytext[0] == '-') { float_val = -INFINITYF; @@ -2227,7 +2183,6 @@ YY_RULE_SETUP YY_BREAK case 35: YY_RULE_SETUP -#line 326 "ncgen.l" { /* missing value (pre-2.4 backward compatibility) */ float_val = NANF; specialconstants = 1; @@ -2236,7 +2191,6 @@ YY_RULE_SETUP YY_BREAK case 36: YY_RULE_SETUP -#line 332 "ncgen.l" { #ifdef USE_NETCDF4 if(l_flag == L_C || l_flag == L_BINARY) @@ -2249,7 +2203,6 @@ YY_RULE_SETUP YY_BREAK case 37: YY_RULE_SETUP -#line 342 "ncgen.l" { bbClear(lextext); bbAppendn(lextext,(char*)yytext,yyleng+1); /* include null */ @@ -2260,7 +2213,6 @@ YY_RULE_SETUP YY_BREAK case 38: YY_RULE_SETUP -#line 351 "ncgen.l" {struct Specialtoken* st; bbClear(lextext); bbAppendn(lextext,(char*)yytext,yyleng+1); /* include null */ @@ -2274,7 +2226,6 @@ YY_RULE_SETUP case 39: /* rule 39 can match eol */ YY_RULE_SETUP -#line 361 "ncgen.l" { int c; char* p; char* q; @@ -2294,7 +2245,6 @@ YY_RULE_SETUP YY_BREAK case 40: YY_RULE_SETUP -#line 378 "ncgen.l" { char* id = NULL; int len; len = strlen(yytext); len = unescape(yytext,len,ISIDENT,&id); @@ -2309,7 +2259,6 @@ YY_RULE_SETUP YY_BREAK case 41: YY_RULE_SETUP -#line 390 "ncgen.l" { /* We need to try to see what size of integer ((u)int). @@ -2390,7 +2339,6 @@ done: return 0; YY_BREAK case 42: YY_RULE_SETUP -#line 468 "ncgen.l" { int c; int token = 0; @@ -2441,7 +2389,6 @@ YY_RULE_SETUP YY_BREAK case 43: YY_RULE_SETUP -#line 515 "ncgen.l" { if (sscanf((char*)yytext, "%le", &double_val) != 1) { sprintf(errstr,"bad long or double constant: %s",(char*)yytext); @@ -2452,7 +2399,6 @@ YY_RULE_SETUP YY_BREAK case 44: YY_RULE_SETUP -#line 522 "ncgen.l" { if (sscanf((char*)yytext, "%e", &float_val) != 1) { sprintf(errstr,"bad float constant: %s",(char*)yytext); @@ -2464,7 +2410,6 @@ YY_RULE_SETUP case 45: /* rule 45 can match eol */ YY_RULE_SETUP -#line 529 "ncgen.l" { (void) sscanf((char*)&yytext[1],"%c",&byte_val); return lexdebug(BYTE_CONST); @@ -2472,7 +2417,6 @@ YY_RULE_SETUP YY_BREAK case 46: YY_RULE_SETUP -#line 533 "ncgen.l" { int oct = unescapeoct(&yytext[2]); if(oct < 0) { @@ -2485,7 +2429,6 @@ YY_RULE_SETUP YY_BREAK case 47: YY_RULE_SETUP -#line 542 "ncgen.l" { int hex = unescapehex(&yytext[3]); if(byte_val < 0) { @@ -2498,7 +2441,6 @@ YY_RULE_SETUP YY_BREAK case 48: YY_RULE_SETUP -#line 551 "ncgen.l" { switch ((char)yytext[2]) { case 'a': byte_val = '\007'; break; /* not everyone under- @@ -2520,7 +2462,6 @@ YY_RULE_SETUP case 49: /* rule 49 can match eol */ YY_RULE_SETUP -#line 569 "ncgen.l" { lineno++ ; break; @@ -2528,7 +2469,6 @@ YY_RULE_SETUP YY_BREAK case 50: YY_RULE_SETUP -#line 574 "ncgen.l" {/*initial*/ BEGIN(ST_C_COMMENT); break; @@ -2537,21 +2477,18 @@ YY_RULE_SETUP case 51: /* rule 51 can match eol */ YY_RULE_SETUP -#line 579 "ncgen.l" {/* continuation */ break; } YY_BREAK case 52: YY_RULE_SETUP -#line 583 "ncgen.l" {/* final */ BEGIN(INITIAL); break; } YY_BREAK case YY_STATE_EOF(ST_C_COMMENT): -#line 588 "ncgen.l" {/* final, error */ fprintf(stderr,"unterminated /**/ comment"); BEGIN(INITIAL); @@ -2560,17 +2497,14 @@ case YY_STATE_EOF(ST_C_COMMENT): YY_BREAK case 53: YY_RULE_SETUP -#line 594 "ncgen.l" {/* Note: this next rule will not work for UTF8 characters */ return lexdebug(yytext[0]) ; } YY_BREAK case 54: YY_RULE_SETUP -#line 597 "ncgen.l" ECHO; YY_BREAK -#line 2574 "ncgenl.c" case YY_STATE_EOF(INITIAL): case YY_STATE_EOF(TEXT): yyterminate(); @@ -3576,8 +3510,6 @@ void yyfree (void * ptr ) #define YYTABLES_NAME "yytables" -#line 597 "ncgen.l" - static int lexdebug(int token) { @@ -3676,7 +3608,6 @@ parseULL(int radix, char* text, int* failp) return uint64; } - /** Given the raw bits, the sign char, the tag, and hasU fill in the appropriate *_val field diff --git a/nczarr_test/CMakeLists.txt b/nczarr_test/CMakeLists.txt index b1f23d5aca..03db4557fb 100644 --- a/nczarr_test/CMakeLists.txt +++ b/nczarr_test/CMakeLists.txt @@ -13,6 +13,19 @@ FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../nc_test4/tst_quantize.c QSOURCE) FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test_quantize.c "#define TESTNCZARR\n") FILE(APPEND ${CMAKE_CURRENT_BINARY_DIR}/test_quantize.c "${QSOURCE}") +FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../nc_test4/tst_filter_avail.c ASOURCE) +STRING(PREPEND ASOURCE "#define TESTNCZARR\n") +FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test_filter_avail.c "${ASOURCE}") + +FILE(READ ${CMAKE_CURRENT_SOURCE_DIR}/../nc_test4/tst_specific_filters.sh SPSOURCE) +STRING(PREPEND SPSOURCE "#!/bin/bash\n") +STRING(PREPEND SPSOURCE "TESTNCZARR=1\n") +FILE(WRITE ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.1 "${SPSOURCE}") +CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.1 ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.sh @ONLY NEWLINE_STYLE LF) +FILE(COPY ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.sh DESTINATION ${CMAKE_CURRENT_BINARY_DIR} FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE) +FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.1) +FILE(REMOVE ${CMAKE_CURRENT_SOURCE_DIR}/run_specific_filters.sh) + FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_SOURCE_DIR}/ref*.cdl ${CMAKE_CURRENT_SOURCE_DIR}/ref*.txt) @@ -104,16 +117,16 @@ IF(ENABLE_TESTS) ENDIF() IF(ENABLE_FILTER_TESTING) - add_bin_test(nczarr_test tst_nczfilter) + build_bin_test(tst_nczfilter) build_bin_test(testfilter) build_bin_test(testfilter_misc) build_bin_test(testfilter_multi) build_bin_test(testfilter_order) build_bin_test(testfilter_repeat) + build_bin_test(test_filter_avail) + ADD_SH_TEST(nczarr_test run_nczfilter) ADD_SH_TEST(nczarr_test run_filter) - IF(ENABLE_BLOSC) ADD_SH_TEST(nczarr_test run_specific_filters) - ENDIF() ENDIF(ENABLE_FILTER_TESTING) if(ENABLE_NCZARR_ZIP) add_sh_test(nczarr_test run_newformat) diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index 1249598a62..e9efa73f7e 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -35,7 +35,7 @@ ut_projections_SOURCES = ut_projections.c ${commonsrc} ut_chunking_SOURCES = ut_chunking.c ${commonsrc} tst_fillonlyz_SOURCES = tst_fillonlyz.c ${tstcommonsrc} -check_PROGRAMS += tst_zchunks tst_zchunks2 tst_zchunks3 tst_fillonlyz test_quantize +check_PROGRAMS += tst_zchunks tst_zchunks2 tst_zchunks3 tst_fillonlyz test_quantize TESTS += run_ut_map.sh TESTS += run_ut_mapapi.sh @@ -86,16 +86,15 @@ endif endif # BUILD_BENCHMARKS if ENABLE_FILTER_TESTING + # Do simple filter functionality tests for nczarr check_PROGRAMS += tst_nczfilter -TESTS += tst_nczfilter +TESTS += run_nczfilter.sh # Echo filter tests from nc_test4 -check_PROGRAMS += testfilter testfilter_misc testfilter_order testfilter_repeat testfilter_multi +check_PROGRAMS += testfilter testfilter_misc testfilter_order testfilter_repeat testfilter_multi test_filter_avail TESTS += run_filter.sh -if ENABLE_BLOSC TESTS += run_specific_filters.sh -endif endif #ENABLE_FILTER_TESTING @@ -155,19 +154,32 @@ EXTRA_DIST += ref_power_901_constants.zip ref_power_901_constants.cdl ref_quotes CLEANFILES = ut_*.txt ut*.cdl tmp*.nc tmp*.cdl tmp*.txt tmp*.dmp tmp*.zip tmp*.nc tmp*.dump tmp*.tmp tmp_ngc.c ref_zarr_test_data.cdl -BUILT_SOURCES = test_quantize.c +BUILT_SOURCES = test_quantize.c test_filter_avail.c run_specific_filters.sh test_quantize.c: ../nc_test4/tst_quantize.c rm -f test_quantize.c echo "#define TESTNCZARR" > test_quantize.c cat ../nc_test4/tst_quantize.c >> test_quantize.c +test_filter_avail.c: ../nc_test4/tst_filter_avail.c + rm -f test_filter_avail.c + echo "#define TESTNCZARR" > test_filter_avail.c + cat ../nc_test4/tst_filter_avail.c >> test_filter_avail.c + +run_specific_filters.sh: ../nc_test4/tst_specific_filters.sh + rm -f run_specific_filters.sh run_specific_filters.tmp + echo "#!/bin/bash" > run_specific_filters.tmp + echo "TESTNCZARR=1" >> run_specific_filters.tmp + cat ../nc_test4/tst_specific_filters.sh >> run_specific_filters.tmp + tr -d '\r' < run_specific_filters.tmp > run_specific_filters.sh + chmod a+x run_specific_filters.sh + rm -f run_specific_filters.tmp + # Remove directories clean-local: rm -fr tmp*.file results.file results.s3 results.zip rm -fr rcmiscdir -DISTCLEANFILES = findplugin.sh test_quantize.c +DISTCLEANFILES = findplugin.sh test_quantize.c run_specific_filters.sh # If valgrind is present, add valgrind targets. @VALGRIND_CHECK_RULES@ - diff --git a/nczarr_test/ncdumpchunks.c b/nczarr_test/ncdumpchunks.c index 1f9e878632..a29046797c 100755 --- a/nczarr_test/ncdumpchunks.c +++ b/nczarr_test/ncdumpchunks.c @@ -229,12 +229,14 @@ static void printchunk(Format* format, int* chunkdata, size_t indent) { int k[3]; + int rank = format->rank; unsigned cols[3], pos; size_t* chl = format->chunklens; memset(cols,0,sizeof(cols)); - switch (format->rank) { + if(format->xtype == NC_UBYTE) rank = 0; + switch (rank) { case 1: cols[0] = 1; cols[1] = 1; @@ -265,7 +267,15 @@ printchunk(Format* format, int* chunkdata, size_t indent) k[2] = 0; if(k[1] > 0) printf(" |"); for(k[2]=0;k[2]xtype == NC_UBYTE) { + int l; + unsigned char* bchunkdata = (unsigned char*)(&chunkdata[pos]); + for(l=0;l$2 } # Locate the plugin path and the library names; argument order is critical + +# Find misc and capture +findplugin h5misc +MISCPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}" + +# Find noop and capture +findplugin h5noop +NOOPLIB="${HDF5_PLUGIN_LIB}" +NOOPPATH="${HDF5_PLUGIN_PATH}/${NOOPLIB}" + # Find bzip2 and capture findplugin h5bzip2 BZIP2LIB="${HDF5_PLUGIN_LIB}" BZIP2PATH="${HDF5_PLUGIN_PATH}/${BZIP2LIB}" -# Find misc and capture -findplugin h5misc -MISCPATH="${HDF5_PLUGIN_PATH}/${HDF5_PLUGIN_LIB}" # Verify if ! test -f ${BZIP2PATH} ; then echo "Unable to locate ${BZIP2PATH}"; exit 1; fi if ! test -f ${MISCPATH} ; then echo "Unable to locate ${MISCPATH}"; exit 1; fi -echo "@@@@@@@@@@@" -find ${HDF5_PLUGIN_PATH} - # Execute the specified tests testapi() { @@ -133,18 +137,18 @@ zext=$1 echo "*** Testing access to filter info when filter implementation is not available for map $zext" fileargs tmp_known deletemap $zext $file -# build bzip2.nc -${NCGEN} -lb -4 -o $fileurl ${srcdir}/../nc_test4/bzip2.cdl -# dump and clean bzip2.nc header when filter is avail +# build noop.nc +${NCGEN} -lb -4 -o $fileurl ${srcdir}/../nc_test4/noop.cdl +# dump and clean noop.nc header when filter is avail ${NCDUMP} -hs $fileurl > ./tmp_known_$zext.txt # Remove irrelevant -s output sclean ./tmp_known_$zext.txt tmp_known_$zext.dump # Now hide the filter code -mv ${BZIP2PATH} ./${BZIP2LIB}.save -# dump and clean bzip2.nc header when filter is not avail +mv ${NOOPPATH} ./${NOOPLIB}.save +# dump and clean noop.nc header when filter is not avail ${NCDUMP} -hs $fileurl > ./tmp_unk_$zext.txt # Restore the filter code -mv ./${BZIP2LIB}.save ${BZIP2PATH} +mv ./${NOOPLIB}.save ${NOOPPATH} # Verify that the filter is no longer defined UNK=`sed -e '/var:_Filter/p' -e d ./tmp_unk_$zext.txt` test "x$UNK" = x @@ -167,7 +171,7 @@ echo "*** Testing multiple filters for map $zext" fileargs tmp_multi deletemap $zext $file ${execdir}/testfilter_multi $fileurl -${NCDUMP} -hs -n multifilter $fileurl >./tmp_multi_$zext.cdl +${NCDUMP} -hsF -n multifilter $fileurl >./tmp_multi_$zext.cdl # Remove irrelevant -s output sclean ./tmp_multi_$zext.cdl ./tmp_smulti_$zext.cdl diff -b -w ${srcdir}/ref_multi.cdl ./tmp_smulti_$zext.cdl diff --git a/nczarr_test/run_nczfilter.sh b/nczarr_test/run_nczfilter.sh new file mode 100755 index 0000000000..f6575fc18f --- /dev/null +++ b/nczarr_test/run_nczfilter.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +. "$srcdir/test_nczarr.sh" + +# This shell script runs tst_nczfilter.c + +set -e + +pwd + +${execdir}/tst_nczfilter diff --git a/nczarr_test/run_quantize.sh b/nczarr_test/run_quantize.sh index 5df10f2358..67ab361713 100755 --- a/nczarr_test/run_quantize.sh +++ b/nczarr_test/run_quantize.sh @@ -22,5 +22,5 @@ testcase() { } testcase file -#if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi -#if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi +if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testcase zip; fi +if test "x$FEATURE_S3TESTS" = xyes ; then testcase s3; fi diff --git a/nczarr_test/run_specific_filters.sh b/nczarr_test/run_specific_filters.sh deleted file mode 100755 index 78bc581f1d..0000000000 --- a/nczarr_test/run_specific_filters.sh +++ /dev/null @@ -1,161 +0,0 @@ -#!/bin/bash - -# Test the implementations of specific filters - -if test "x$srcdir" = x ; then srcdir=`pwd`; fi -. ../test_common.sh - -. "$srcdir/test_nczarr.sh" - -set -e - -# Function to remove selected -s attributes from file; -# These attributes might be platform dependent -sclean() { - cat $1 \ - | sed -e '/:_IsNetcdf4/d' \ - | sed -e '/:_Endianness/d' \ - | sed -e '/_NCProperties/d' \ - | sed -e '/_SuperblockVersion/d' \ - | sed -e '/_Format/d' \ - | sed -e '/global attributes:/d' \ - | cat > $2 -} - -# Function to extract _Filter attribute from a file -# These attributes might be platform dependent -getfilterattr() { -V="$1" -sed -e '/${V}.*:_Filter/p' -ed <$2 >$3 -} - -# Function to extract _Codecs attribute from a file -# These attributes might be platform dependent -getcodecsattr() { -V="$1" -sed -e '/${V}.*:_Codecs/p' -ed <$2 >$3 -} - -trimleft() { -sed -e 's/[ ]*\([^ ].*\)/\1/' <$1 >$2 -} - - -setfilter() { - FF="$1" - FSRC="$2" - FDST="$3" - FIH5="$4" - FICX="$5" - FFH5="$6" - FFCX="$7" - if test "x$FFH5" = x ; then FFH5="$FIH5" ; fi - if test "x$FFCX" = x ; then FFCX="$FICX" ; fi - rm -f $FDST - cat ${srcdir}/$FSRC \ - | sed -e "s/ref_any/${FF}/" \ - | sed -e "s/IH5/${FIH5}/" -e "s/FH5/${FFH5}/" \ - | sed -e "s/ICX/${FICX}/" -e "s/FCX/${FFCX}/" \ - | sed -e 's/"/\\"/g' -e 's/@/"/g' \ - | cat > $FDST -} - -# Execute the specified tests - -runfilter() { -zext="$1" -zfilt="$2" -zparams="$3" -zcodec="$4" -echo "*** Testing processing of filter $zfilt for map $zext" -deletemap $zext "tmp_${zfilt}" -fileargs "tmp_${zfilt}" -setfilter $zfilt ref_any.cdl "tmp_${zfilt}.cdl" "$zparams" "$zcodec" -if ${NCGEN} -4 -lb -o $fileurl "tmp_${zfilt}.cdl" ; then - ${NCDUMP} -n $zfilt -s $fileurl > "tmp_${zfilt}.tmp" - sclean "tmp_${zfilt}.tmp" "tmp_${zfilt}.dump" -fi -} - -testfletcher32() { - zext=$1 - runfilter $zext fletcher32 '3' '[{\"id\": \"fletcher32\"}]' - if test -f "tmp_fletcher32.dump" ; then - # need to remove _Filter - sed -e '/_Fletcher32 = "true"/d' < tmp_fletcher32.dump > tmp_fletcher32x.dump - diff -b -w "tmp_fletcher32.cdl" "tmp_fletcher32x.dump" - else - echo "XFAIL: filter=fletcher32 zext=$zext" - fi -} - -testshuffle() { - zext=$1 - runfilter $zext shuffle '2' '[{\"id\": \"shuffle\",\"elementsize\": \"0\"}]' - if test -f "tmp_shuffle.dump" ; then - # need to replace _Filter - sed -e 's/_Filter = "2,4"/_Filter = "2"/' -e '/_Shuffle = "true"/d' < tmp_shuffle.dump > tmp_shufflex.dump - diff -b -w "tmp_shuffle.cdl" "tmp_shufflex.dump" - else - echo "XFAIL: filter=shuffle zext=$zext" - fi -} - -testdeflate() { - zext=$1 - runfilter $zext deflate '1,9' '[{\"id\": \"zlib\",\"level\": \"9\"}]' - if test -f "tmp_deflate.dump" ; then - # need to replace _DeflateLevel - sed -e 's/_DeflateLevel = 9/_Filter = "1,9"/' < tmp_deflate.dump > tmp_deflatex.dump - diff -b -w "tmp_deflate.cdl" "tmp_deflatex.dump" - else - echo "XFAIL: filter=deflate zext=$zext" - fi -} - -testbzip2() { - zext=$1 - runfilter $zext bzip2 '307,9' '[{\"id\": \"bz2\",\"level\": \"9\"}]' - if test -f "tmp_bzip2.dump" ; then - diff -b -w "tmp_bzip2.cdl" "tmp_bzip2.dump" - else - echo "XFAIL: filter=bzip2 zext=$zext" - fi -} - -testszip() { - zext=$1 -# H5_SZIP_NN_OPTION_MASK=32; H5_SZIP_MAX_PIXELS_PER_BLOCK_IN=32 - runfilter $zext szip '4,32,32' '[{\"id\": \"szip\",\"mask\": 32,\"pixels-per-block\": 32}]' - if test -f "tmp_szip.dump" ; then - diff -b -w "tmp_szip.cdl" "tmp_szip.dump" - else - echo "XFAIL: filter=szip zext=$zext" - fi -} - -testblosc() { - zext=$1 - runfilter $zext blosc '32001,0,0,0,0,5,1,1' '[{\"id\": \"blosc\",\"clevel\": 5,\"blocksize\": 0,\"cname\": \"lz4\",\"shuffle\": 1}]' - if test -f "tmp_blosc.dump" ; then - diff -b -w "tmp_blosc.cdl" "tmp_blosc.dump" - else - echo "XFAIL: filter=blosc zext=$zext" - fi -} - -testset() { -# Which test cases to exercise - testfletcher32 $1 - testshuffle $1 - testdeflate $1 - testszip $1 - testbzip2 $1 - testblosc $1 -} - -testset file -#if test "x$FEATURE_NCZARR_ZIP" = xyes ; then testset zip ; fi -#if test "x$FEATURE_S3TESTS" = xyes ; then testset s3 ; fi - -exit 0 diff --git a/nczarr_test/tst_nczfilter.c b/nczarr_test/tst_nczfilter.c index 84c1ea5f84..8e1e987c1a 100644 --- a/nczarr_test/tst_nczfilter.c +++ b/nczarr_test/tst_nczfilter.c @@ -15,6 +15,8 @@ #define ERR(r) {fprintf(stderr,"fail: line %d: (%d) %s\n",__LINE__,(r),nc_strerror((r)));} +#define FILENAME "file://tmp_nczfilter.nc#mode=nczarr,file" + #define FILTERID 1 int @@ -33,7 +35,10 @@ main(int argc, char **argv) size_t nparams; unsigned inqparams[8]; - furl = argv[1]; + if(argc == 1) + furl = FILENAME; + else + furl = argv[1]; if ((ret=nc_create(furl, NC_NETCDF4, &ncid))) ERR(ret); if ((ret=nc_def_dim(ncid, "d", 4, &dimid))) ERR(ret); diff --git a/plugins/BZIP2_LICENSE b/plugins/BZIP2_LICENSE new file mode 100644 index 0000000000..81a37eab7a --- /dev/null +++ b/plugins/BZIP2_LICENSE @@ -0,0 +1,42 @@ + +-------------------------------------------------------------------------- + +This program, "bzip2", the associated library "libbzip2", and all +documentation, are copyright (C) 1996-2019 Julian R Seward. All +rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + +4. The name of the author may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Julian Seward, jseward@acm.org +bzip2/libbzip2 version 1.0.8 of 13 July 2019 + +-------------------------------------------------------------------------- diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index 7024bf1195..71a6af7c40 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -10,7 +10,11 @@ FILE(READ H5Znoop.c NOOP_SOURCE) FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "#define NOOP_INSTANCE 1\n") FILE(APPEND ${CMAKE_CURRENT_BINARY_DIR}/H5Znoop1.c "${NOOP_SOURCE}") -SET(h5bzip2_SOURCES blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c H5Zbzip2.c ${NCJ}) +SET(h5bzip2_SOURCES H5Zbzip2.c) + +IF(NOT HAVE_BZIP2) +SET(h5bzip2_SOURCES ${h5bzip2_SOURCES} blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c) +ENDIF() SET(h5misc_SOURCES H5Zmisc.c H5Zutil.c h5misc.h) @@ -51,11 +55,6 @@ MACRO(buildplugin TARGET TARGETLIB) ENDIF() ENDMACRO() -# Note we use name h5bzip2 instead of bzip2 to avoid logical -# target name clash with examples/C/hdf5plugins -buildplugin(h5bzip2 "h5bzip2") -SET_TARGET_PROPERTIES(h5bzip2 PROPERTIES OUTPUT_NAME "bzip2") - buildplugin(h5misc "h5misc") buildplugin(h5noop "h5noop") buildplugin(h5noop1 "h5noop1") @@ -72,11 +71,24 @@ SET(h5blosc_SOURCES H5Zblosc.c) buildplugin(h5blosc "h5blosc" netcdf;${Blosc_LIBRARIES}) ENDIF() -IF(ENABLE_SZIP) -SET(h5szip_SOURCES H5Zszip.c H5Zszip.h) -buildplugin(h5szip "h5szip" ${Szip_LIBRARIES}) +IF(ENABLE_ZSTD) +SET(h5zstd_SOURCES H5Zzstd.c H5Zzstd.h) +buildplugin(h5zstd "h5zstd" netcdf;${Zstd_LIBRARIES}) ENDIF() +# Need our version of szip if libsz available and we are not using HDF5 +IF(HAVE_SZ) + SET(h5szip_SOURCES H5Zszip.c H5Zszip.h) + buildplugin(h5szip "h5szip" ${Szip_LIBRARIES}) + SET(nczszip_SOURCES NCZszip.c) + buildplugin(nczszip "nczszip" netcdf) +ENDIF() + +buildplugin(h5bzip2 "h5bzip2" ${Bzip2_LIBRARIES}) +# Note we use name h5bzip2 instead of bzip2 to avoid logical +# target name clash with examples/C/hdf5plugins +SET_TARGET_PROPERTIES(h5bzip2 PROPERTIES OUTPUT_NAME "bzip2") + ENDIF(BUILD_UTILITIES) ENDIF(ENABLE_FILTER_TESTING) diff --git a/plugins/H5Zszip.c b/plugins/H5Zszip.c index 5c7c0071d0..5571f0cbfd 100644 --- a/plugins/H5Zszip.c +++ b/plugins/H5Zszip.c @@ -20,15 +20,15 @@ #include #include "netcdf_filter_build.h" -#ifdef USE_SZIP +#ifdef HAVE_SZ #include #include "H5Zszip.h" /* Local function prototypes */ -static htri_t H5Z__can_apply_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id); -static herr_t H5Z__set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id); -static size_t H5Z__filter_szip(unsigned flags, size_t cd_nelmts, +static htri_t H5Z___can_apply_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static herr_t H5Z___set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static size_t H5Z___filter_szip(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); /* This message derives from H5Z */ @@ -38,15 +38,15 @@ H5Z_class2_t H5Z_SZIP[1] = {{ 1, /* Assume encoder present: check before registering */ 1, /* decoder_present flag (set to true) */ "szip", /* Filter name for debugging */ - H5Z__can_apply_szip, /* The "can apply" callback */ - H5Z__set_local_szip, /* The "set local" callback */ - H5Z__filter_szip, /* The actual filter function */ + H5Z___can_apply_szip, /* The "can apply" callback */ + H5Z___set_local_szip, /* The "set local" callback */ + H5Z___filter_szip, /* The actual filter function */ }}; /*------------------------------------------------------------------------- - * Function: H5Z__can_apply_szip + * Function: H5Z___can_apply_szip * * Purpose: Check the parameters for szip compression for validity and * whether they fit a particular dataset. @@ -67,12 +67,12 @@ H5Z_class2_t H5Z_SZIP[1] = {{ *------------------------------------------------------------------------- */ static htri_t -H5Z__can_apply_szip(hid_t H5_ATTR_UNUSED dcpl_id, hid_t type_id, hid_t H5_ATTR_UNUSED space_id) +H5Z___can_apply_szip(hid_t H5_ATTR_UNUSED dcpl_id, hid_t type_id, hid_t H5_ATTR_UNUSED space_id) { + htri_t ret_value = TRUE; /* Return value */ #ifdef USE_HDF5 unsigned dtype_size; /* Datatype's size (in bits) */ H5T_order_t dtype_order; /* Datatype's endianness order */ - htri_t ret_value = TRUE; /* Return value */ FUNC_ENTER_STATIC @@ -95,14 +95,14 @@ H5Z__can_apply_szip(hid_t H5_ATTR_UNUSED dcpl_id, hid_t type_id, hid_t H5_ATTR_U done: FUNC_LEAVE_NOAPI(ret_value) -#else - return SUCCEED; -#endif -} /* end H5Z__can_apply_szip() */ +#else /*!USE_HDF5*/ + return ret_value; +#endif /*USE_HDF5*/ +} /* end H5Z___can_apply_szip() */ /*------------------------------------------------------------------------- - * Function: H5Z__set_local_szip + * Function: H5Z___set_local_szip * * Purpose: Set the "local" dataset parameters for szip compression. * @@ -115,11 +115,11 @@ H5Z__can_apply_szip(hid_t H5_ATTR_UNUSED dcpl_id, hid_t type_id, hid_t H5_ATTR_U *------------------------------------------------------------------------- */ -#ifdef USE_HDF5 - static herr_t -H5Z__set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id) +H5Z___set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id) { + herr_t ret_value = SUCCEED; /* Return value */ + #ifdef USE_HDF5 unsigned flags; /* Filter flags */ size_t cd_nelmts = H5Z_SZIP_USER_NPARMS; /* Number of filter parameters */ @@ -131,7 +131,6 @@ H5Z__set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id) size_t dtype_precision; /* Datatype's precision (in bits) */ size_t dtype_offset; /* Datatype's offset (in bits) */ hsize_t scanline; /* Size of dataspace's fastest changing dimension */ - herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -203,14 +202,14 @@ H5Z__set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id) /* Set the correct endianness flag for szip */ /* (Note: this may not handle non-atomic datatypes well) */ - cd_values[H5Z_SZIP_PARM_MASK] &= ~(SZ_LSB_OPTION_MASK|SZ_MSB_OPTION_MASK); + cd_values[H5Z_SZIP_PARM_MASK] &= ~(H5_SZIP_LSB_OPTION_MASK|H5_SZIP_MSB_OPTION_MASK); switch(dtype_order) { case H5T_ORDER_LE: /* Little-endian byte order */ - cd_values[H5Z_SZIP_PARM_MASK] |= SZ_LSB_OPTION_MASK; + cd_values[H5Z_SZIP_PARM_MASK] |= H5_SZIP_LSB_OPTION_MASK; break; case H5T_ORDER_BE: /* Big-endian byte order */ - cd_values[H5Z_SZIP_PARM_MASK] |= SZ_MSB_OPTION_MASK; + cd_values[H5Z_SZIP_PARM_MASK] |= H5_SZIP_MSB_OPTION_MASK; break; case H5T_ORDER_ERROR: @@ -227,16 +226,14 @@ H5Z__set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id) done: FUNC_LEAVE_NOAPI(ret_value) -#else - return SUCCEED; -#endif -} /* end H5Z__set_local_szip() */ - +#else /*!USE_HDF5*/ + return ret_value; #endif /*USE_HDF5*/ +} /* end H5Z___set_local_szip() */ /*------------------------------------------------------------------------- - * Function: H5Z__filter_szip + * Function: H5Z___filter_szip * * Purpose: Implement an I/O filter around the 'rice' algorithm in * libsz @@ -250,7 +247,7 @@ H5Z__set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id) *------------------------------------------------------------------------- */ static size_t -H5Z__filter_szip(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], +H5Z___filter_szip(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) { size_t ret_value = 0; /* Return value */ @@ -344,4 +341,19 @@ H5Z__filter_szip(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], FUNC_LEAVE_NOAPI(ret_value) } -#endif /* USE_SZIP */ +/* External Discovery Functions */ +DLLEXPORT +H5PL_type_t +H5PLget_plugin_type(void) +{ + return H5PL_TYPE_FILTER; +} + +DLLEXPORT +const void* +H5PLget_plugin_info(void) +{ + return H5Z_SZIP; +} + +#endif /* HAVE_SZ */ diff --git a/plugins/H5Zzstd.c b/plugins/H5Zzstd.c new file mode 100644 index 0000000000..9d56425c71 --- /dev/null +++ b/plugins/H5Zzstd.c @@ -0,0 +1,257 @@ +#include "config.h" +#include +#include +#include +#include +#include + +#include "netcdf_filter_build.h" + +/* WARNING: +Starting with HDF5 version 1.10.x, the plugin code MUST be +careful when using the standard *malloc()*, *realloc()*, and +*free()* function. + +In the event that the code is allocating, reallocating, or +free'ing memory that either came from or will be exported to the +calling HDF5 library, then one MUST use the corresponding HDF5 +functions *H5allocate_memory()*, *H5resize_memory()*, +*H5free_memory()* [5] to avoid memory failures. + +Additionally, if your filter code leaks memory, then the HDF5 library +will generate an error. + +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include "netcdf_filter_build.h" +#include + +#include "H5Zzstd.h" + +/* Forward */ +static htri_t H5Z_zstd_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static size_t H5Z_filter_zstd(unsigned flags,size_t cd_nelmts,const unsigned cd_values[], + size_t nbytes,size_t *buf_size,void**buf); + +const H5Z_class2_t H5Z_ZSTD[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + (H5Z_filter_t)H5Z_FILTER_ZSTD, /* Filter id number */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + "zstd", /* Filter name for debugging */ + (H5Z_can_apply_func_t)H5Z_zstd_can_apply, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + (H5Z_func_t)H5Z_filter_zstd, /* The actual filter function */ +}}; + +/* External Discovery Functions */ +DLLEXPORT +H5PL_type_t +H5PLget_plugin_type(void) +{ + return H5PL_TYPE_FILTER; +} + +DLLEXPORT +const void* +H5PLget_plugin_info(void) +{ + return H5Z_ZSTD; +} + +/* Make this explicit */ +/* + * The "can_apply" callback returns positive a valid combination, zero for an + * invalid combination and negative for an error. + */ +static htri_t +H5Z_zstd_can_apply(hid_t dcpl_id, hid_t type_id, hid_t space_id) +{ + return 1; /* Assume it can always apply */ +} + +static size_t +H5Z_filter_zstd(unsigned int flags, size_t cd_nelmts, + const unsigned int cd_values[], size_t nbytes, + size_t *buf_size, void **buf) +{ + char *outbuf = NULL; + size_t outbuflen; + char *inbuf = NULL; + size_t inbuflen; + size_t outdatalen; + size_t err; + unsigned long long dsize; + + /* Get level parameter */ + if(cd_nelmts != 1) { + fprintf(stderr, "zstd: no level specified\n"); + goto cleanupAndFail; + } + + /* Prepare the input buffer. */ + inbuflen = *buf_size; + inbuf = (char*)*buf; + + if (flags & H5Z_FLAG_REVERSE) { /** Decompress data. */ + dsize = ZSTD_getFrameContentSize(inbuf, inbuflen); + err = (size_t)dsize; + if(ZSTD_isError(err)) { + fprintf(stderr,"zstd: %s\n", ZSTD_getErrorName(err)); + goto cleanupAndFail; + } + + /* Prepare the output buffer. */ + outbuflen = (size_t)dsize; + outbuf = H5allocate_memory(outbuflen,0); + if (outbuf == NULL) { + fprintf(stderr, "memory allocation failed for zstd compression\n"); + goto cleanupAndFail; + } + dsize = ZSTD_decompress(outbuf, outbuflen, inbuf, inbuflen); + err = (size_t)dsize; + if(ZSTD_isError(err)) { + fprintf(stderr,"zstd: %s\n", ZSTD_getErrorName(err)); + goto cleanupAndFail; + } + + } else { /* Compress */ + int level; + + /* Prepare the output buffer. */ + dsize = ZSTD_compressBound(inbuflen); + err = (size_t)dsize; + if(ZSTD_isError(err)) { + fprintf(stderr,"zstd: %s\n", ZSTD_getErrorName(err)); + goto cleanupAndFail; + } + + /* Prepare the output buffer. */ + outbuflen = (size_t)dsize; + outbuf = H5allocate_memory(outbuflen,0); + if (outbuf == NULL) { + fprintf(stderr, "memory allocation failed for zstd compression\n"); + goto cleanupAndFail; + } + + level = (int)cd_values[0]; + dsize = ZSTD_compress(outbuf, outbuflen, inbuf, inbuflen, level); + err = (size_t)dsize; + if(ZSTD_isError(err)) { + fprintf(stderr,"zstd: %s\n", ZSTD_getErrorName(err)); + goto cleanupAndFail; + } + } + + outdatalen = (size_t)dsize; + + /* Always replace the input buffer with the output buffer. */ + H5free_memory(*buf); + *buf = outbuf; + *buf_size = outbuflen; + return outdatalen; + +cleanupAndFail: + if (outbuf) + H5free_memory(outbuf); + return 0; +} + +/**************************************************/ +/* NCZarr Filter Objects */ + +/* Provide the codec support for the HDF5 zstandard library */ + +static int NCZ_zstd_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); + +static NCZ_codec_t NCZ_zstd_codec = {/* NCZ_codec_t codec fields */ + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "zstd", /* Standard name/id of the codec */ + H5Z_FILTER_ZSTD, /* HDF5 alias for zstd */ + NULL, /*NCZ_zstd_codec_initialize*/ + NULL, /*NCZ_zstd_codec_finalize*/ + NCZ_zstd_codec_to_hdf5, + NCZ_zstd_hdf5_to_codec, + NULL, /*NCZ_zstd_modify_parameters*/ +}; + +/* External Export API */ +DLLEXPORT +const void* +NCZ_get_codec_info(void) +{ + return (void*)&NCZ_zstd_codec; +} + +static int +NCZ_zstd_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + NCjson* jcodec = NULL; + NCjson* jtmp = NULL; + unsigned* params = NULL; + struct NCJconst jc; + + if(nparamsp == NULL || paramsp == NULL) + {stat = NC_EINTERNAL; goto done;} + + if((params = (unsigned*)calloc(1,sizeof(unsigned)))== NULL) + {stat = NC_ENOMEM; goto done;} + + /* parse the JSON */ + if(NCJparse(codec_json,0,&jcodec)) + {stat = NC_EFILTER; goto done;} + if(NCJsort(jcodec) != NCJ_DICT) {stat = NC_EPLUGIN; goto done;} + /* Verify the codec ID */ + if(NCJdictget(jcodec,"id",&jtmp)) + {stat = NC_EFILTER; goto done;} + if(jtmp == NULL || !NCJisatomic(jtmp)) {stat = NC_EFILTER; goto done;} + if(strcmp(NCJstring(jtmp),NCZ_zstd_codec.codecid)!=0) {stat = NC_EINVAL; goto done;} + + /* Get Level */ + if(NCJdictget(jcodec,"level",&jtmp)) + {stat = NC_EFILTER; goto done;} + if(NCJcvt(jtmp,NCJ_INT,&jc)) + {stat = NC_EFILTER; goto done;} + if(jc.ival < 0 || jc.ival > NC_MAX_UINT) {stat = NC_EINVAL; goto done;} + params[0] = (unsigned)jc.ival; + *nparamsp = 1; + *paramsp = params; params = NULL; + +done: + if(params) free(params); + NCJreclaim(jcodec); + return stat; +} + +static int +NCZ_zstd_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int stat = NC_NOERR; + unsigned level = 0; + char json[1024]; + + if(nparams == 0 || params == NULL) + {stat = NC_EFILTER; goto done;} + + level = params[0]; + snprintf(json,sizeof(json),"{\"id\": \"%s\", \"level\": \"%u\"}",NCZ_zstd_codec.codecid,level); + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} + diff --git a/plugins/H5Zzstd.h b/plugins/H5Zzstd.h new file mode 100644 index 0000000000..d72e266bf5 --- /dev/null +++ b/plugins/H5Zzstd.h @@ -0,0 +1,22 @@ +#ifndef H5ZSTD_H +#define H5ZSTD_H + +#include "zstd.h" + +#ifdef _MSC_VER + #ifdef DLL_EXPORT /* define when building the library */ + #define DECLSPEC __declspec(dllexport) + #else + #define DECLSPEC __declspec(dllimport) + #endif +#else + #define DECLSPEC extern +#endif + +/* declare the hdf5 interface */ +DECLSPEC H5PL_type_t H5PLget_plugin_type(void); +DECLSPEC const void* H5PLget_plugin_info(void); +DECLSPEC const H5Z_class2_t H5Z_ZSTD[1]; + +#endif /*H5ZSTD_H*/ + diff --git a/plugins/Makefile.am b/plugins/Makefile.am index fc40872e50..b367c98d04 100644 --- a/plugins/Makefile.am +++ b/plugins/Makefile.am @@ -7,6 +7,9 @@ include $(top_srcdir)/lib_flags.am AM_LDFLAGS += -module -avoid-version -shared -export-dynamic \ -rpath ${abs_builddir} ${NOUNDEFINED} +lib_LTLIBRARIES = +noinst_LTLIBRARIES = + if ISMINGW LDADD = ${top_builddir}/liblib/libnetcdf.la endif @@ -14,46 +17,68 @@ endif # BZIP2 version 1.0.8 (https://sourceware.org/bzip2/) BZIP2SRC = blocksort.c huffman.c crctable.c randtable.c compress.c decompress.c bzlib.c bzlib.h bzlib_private.h -EXTRA_DIST = CMakeLists.txt \ - H5Ztemplate.c H5Zmisc.c H5Zutil.c H5Znoop.c h5noop.h NCZmisc.c \ - NCZdefaults.c \ - H5Zbzip2.c h5bzip2.h H5Zblosc.c H5Zblosc.h \ - H5Zshuffle.c H5Zdeflate.c H5Zszip.c H5Zszip.h \ - ${BZIP2SRC} H5checksum.c +EXTRA_DIST = CMakeLists.txt + +# The HDF5 filter wrappers +EXTRA_DIST += \ + H5Ztemplate.c H5Zmisc.c H5Zutil.c H5Znoop.c h5noop.h NCZmisc.c \ + H5Zshuffle.c H5Zdeflate.c H5Zszip.c H5Zszip.h \ + H5Zbzip2.c h5bzip2.h H5Zblosc.c H5Zblosc.h H5Zzstd.c H5Zzstd.h +# The Codec filter wrappers +EXTRA_DIST += NCZdefaults.c NCZszip.c +# The Filter implementations +EXTRA_DIST += H5checksum.c + + +EXTRA_DIST += ${BZIP2SRC} BZIP2_LICENSE if ENABLE_FILTER_TESTING -lib_LTLIBRARIES = libh5bzip2.la -libh5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h ${BZIP2SRC} - -noinst_LTLIBRARIES = libh5misc.la libh5noop.la libh5noop1.la libnczmisc.la \ - libh5shuffle.la libh5fletcher32.la libh5deflate.la +lib_LTLIBRARIES += libh5bzip2.la +libh5bzip2_la_SOURCES = H5Zbzip2.c h5bzip2.h +if ! HAVE_BZIP2 +libh5bzip2_la_SOURCES += ${BZIP2SRC} +endif + +noinst_LTLIBRARIES += libh5misc.la libh5noop.la libh5noop1.la libnczmisc.la noinst_LTLIBRARIES += libnczdefaults.la +if ENABLE_NCZARR_FILTERS +noinst_LTLIBRARIES += libh5fletcher32.la libh5shuffle.la libh5deflate.la libh5shuffle_la_SOURCES = H5Zshuffle.c libh5fletcher32_la_SOURCES = H5Zfletcher32.c H5checksum.c libh5deflate_la_SOURCES = H5Zdeflate.c -if ENABLE_BLOSC +# Need our version of szip if libsz available and we are not using HDF5 +if HAVE_SZ +noinst_LTLIBRARIES += libh5szip.la libnczszip.la +libh5szip_la_SOURCES = H5Zszip.c H5Zszip.h +libnczszip_la_SOURCES = NCZszip.c +endif + +libnczdefaults_la_SOURCES = NCZdefaults.c + +endif # ENABLE_NCZARR_FILTERS + +if HAVE_BLOSC noinst_LTLIBRARIES += libh5blosc.la libh5blosc_la_SOURCES = H5Zblosc.c H5Zblosc.h endif -if ENABLE_SZIP -noinst_LTLIBRARIES += libh5szip.la -libh5szip_la_SOURCES = H5Zszip.c H5Zszip.h +if HAVE_ZSTD +noinst_LTLIBRARIES += libh5zstd.la +libh5zstd_la_SOURCES = H5Zzstd.c H5Zzstd.h endif libh5misc_la_SOURCES = H5Zmisc.c H5Zutil.c h5misc.h libnczmisc_la_SOURCES = NCZmisc.c -libnczdefaults_la_SOURCES = NCZdefaults.c - # The noop filter is to allow testing of multifilters and filter order # Need two distinct instances libh5noop_la_SOURCES = H5Znoop.c H5Zutil.c h5noop.h libh5noop1_la_SOURCES = H5Znoop1.c H5Zutil.c h5noop.h + endif #ENABLE_FILTER_TESTING BUILT_SOURCES = H5Znoop1.c @@ -62,7 +87,12 @@ H5Znoop1.c: Makefile H5Znoop.c echo '#define NOOP_INSTANCE 1' > $@ cat ${srcdir}/H5Znoop.c >> $@ -# For reference: updating bzip2 -BZIP2DIR=/cygdrive/d/bzip2-1.0.8 -updatebzip2: - for b in ${BZIP2HDRS} ${BZIP2SRC} ; do cp -f ${BZIP2DIR}/$${b} . ; done +BZIP2VER = 1.0.8 +BZIP2DIR = bzip2-${BZIP2VER} +BZIP2URL = https://sourceware.org/pub/bzip2/${BZIP2DIR}.tar.gz +bzip2:: + rm -fr ./${BZIP2DIR} ${BZIP2SRC} BZIP2_LICENSE + wget ${BZIP2URL} + tar -zxf ${BZIP2DIR}.tar.gz + cd ${BZIP2DIR}; cp ${BZIP2SRC} ..; cp LICENSE ../BZIP2_LICENSE ; cd .. + rm -fr ./${BZIP2DIR} diff --git a/plugins/NCZdefaults.c b/plugins/NCZdefaults.c index f0b6f52e7b..48dc3439b4 100644 --- a/plugins/NCZdefaults.c +++ b/plugins/NCZdefaults.c @@ -16,11 +16,6 @@ Author: Dennis Heimbigner #include #include -#ifdef USE_SZIP -#include -#include "H5Zszip.h" -#endif - #include "netcdf.h" #include "netcdf_filter.h" #include "netcdf_filter_build.h" @@ -29,7 +24,6 @@ Author: Dennis Heimbigner #define H5Z_FILTER_DEFLATE 1 /*deflation like gzip */ #define H5Z_FILTER_SHUFFLE 2 /*shuffle the data */ #define H5Z_FILTER_FLETCHER32 3 /*fletcher32 checksum of EDC */ -#define H5Z_FILTER_SZIP 4 /*szip compression */ /**************************************************/ /* NCZarr Filter Objects */ @@ -46,12 +40,6 @@ static int NCZ_fletcher32_modify_parameters(int ncid, int varid, size_t* vnparam static int NCZ_deflate_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); static int NCZ_deflate_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); -#ifdef USE_SZIP -static int NCZ_szip_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); -static int NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); -static int NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); -#endif - /**************************************************/ static NCZ_codec_t NCZ_shuffle_codec = { @@ -117,10 +105,13 @@ NCZ_shuffle_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** if((params=(unsigned*)malloc(sizeof(unsigned)))==NULL) {stat = NC_ENOMEM; goto done;} + params[0] = (unsigned)typesize; - *wnparamsp = 1; - nullfree(*wparamsp); - *wparamsp = params; params = NULL; + if(wnparamsp) *wnparamsp = 1; + if(wparamsp) { + nullfree(*wparamsp); + *wparamsp = params; params = NULL; + } done: nullfree(params); @@ -311,216 +302,10 @@ NCZ_deflate_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) /**************************************************/ -#ifdef USE_SZIP - -static NCZ_codec_t NCZ_szip_codec = { - NCZ_CODEC_CLASS_VER, /* Struct version number */ - NCZ_CODEC_HDF5, /* Struct sort */ - "szip", /* Standard name/id of the codec */ - H5Z_FILTER_SZIP, /* HDF5 alias for szip */ - NULL, /*NCZ_szip_codec_initialize*/ - NULL, /*NCZ_szip_codec_finalize*/ - NCZ_szip_codec_to_hdf5, - NCZ_szip_hdf5_to_codec, - NCZ_szip_modify_parameters, -}; - -static int -NCZ_szip_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp) -{ - int stat = NC_NOERR; - unsigned* params = NULL; - size_t nparams = 2; /* No. of visible parameters */ - NCjson* json = NULL; - NCjson* jtmp = NULL; - struct NCJconst jc = {0,0,0,NULL}; - - if(nparamsp == NULL || paramsp == NULL) - {stat = NC_EINTERNAL; goto done;} - - if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL) - {stat = NC_ENOMEM; goto done;} - - if(NCJparse(codec_json,0,&json)) - {stat = NC_EFILTER; goto done;} - - if(NCJdictget(json,"mask",&jtmp) || jtmp == NULL) - {stat = NC_EFILTER; goto done;} - if(NCJcvt(jtmp,NCJ_INT,&jc)) - {stat = NC_EFILTER; goto done;} - params[H5Z_SZIP_PARM_MASK] = (unsigned)jc.ival; - - jtmp = NULL; - if(NCJdictget(json,"pixels-per-block",&jtmp) || jtmp == NULL) - {stat = NC_EFILTER; goto done;} - if(NCJcvt(jtmp,NCJ_INT,&jc)) - {stat = NC_EFILTER; goto done;} - params[H5Z_SZIP_PARM_PPB] = (unsigned)jc.ival; - - *nparamsp = nparams; - *paramsp = params; params = NULL; - -done: - NCJreclaim(json); - nullfree(params); - return stat; -} - -static int -NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) -{ - int stat = NC_NOERR; - char json[2048]; - - snprintf(json,sizeof(json),"{\"id\": \"%s\", \"mask\": \"%u\", \"pixels-per-block\": \"%u\"}", - NCZ_szip_codec.codecid, - params[H5Z_SZIP_PARM_MASK], - params[H5Z_SZIP_PARM_PPB]); - if(codecp) { - if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} - } - -done: - return stat; -} - -static int -NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) -{ - int i,ret_value = NC_NOERR; - nc_type vtype; - size_t typesize, scanline, dtype_precision, npoints; - int ndims, storage, dtype_order; - int dimids[NC_MAX_VAR_DIMS]; - char vname[NC_MAX_NAME+1]; - size_t chunklens[NC_MAX_VAR_DIMS]; - unsigned* params = NULL; - unsigned* vparams = NULL; - size_t wnparams = 4; - - if(wnparamsp == NULL || wparamsp == NULL) - {ret_value = NC_EFILTER; goto done;} - if(vnparamsp == NULL || vparamsp == NULL) - {ret_value = NC_EFILTER; goto done;} - if(*vnparamsp > 0 && *vparamsp == NULL) - {ret_value = NC_EFILTER; goto done;} - - vparams = *vparamsp; - - /* Get variable info */ - if((ret_value = nc_inq_var(ncid,varid,vname,&vtype,&ndims,dimids,NULL))) goto done; - - /* Get the typesize */ - if((ret_value = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done; - - /* Get datatype's precision, in case is less than full bits */ - dtype_precision = typesize; - - if(dtype_precision > 24) { - if(dtype_precision <= 32) - dtype_precision = 32; - else if(dtype_precision <= 64) - dtype_precision = 64; - } /* end if */ - - if(ndims == 0) {ret_value = NC_EFILTER; goto done;} - /* Set "local" parameter for this dataset's "pixels-per-scanline" */ - if((ret_value = nc_inq_dimlen(ncid,dimids[ndims-1],&scanline))) goto done; - - /* Get number of elements for the dataspace; use - total number of elements in the chunk to define the new 'scanline' size */ - /* Compute chunksize */ - if((ret_value = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done; - if(storage != NC_CHUNKED) {ret_value = NC_EFILTER; goto done;} - npoints = 1; - for(i=0;i +#include +#include +#include + +#include +#include "H5Zszip.h" + +#include "netcdf.h" +#include "netcdf_filter.h" +#include "netcdf_filter_build.h" +#include "netcdf_json.h" + +#define H5Z_FILTER_SZIP 4 /*szip compression */ + +/**************************************************/ +/* NCZarr Filter Objects */ + +static int NCZ_szip_codec_to_hdf5(const char* codec, size_t* nparamsp, unsigned** paramsp); +static int NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp); +static int NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp); + +static NCZ_codec_t NCZ_szip_codec = { + NCZ_CODEC_CLASS_VER, /* Struct version number */ + NCZ_CODEC_HDF5, /* Struct sort */ + "szip", /* Standard name/id of the codec */ + H5Z_FILTER_SZIP, /* HDF5 alias for szip */ + NULL, /*NCZ_szip_codec_initialize*/ + NULL, /*NCZ_szip_codec_finalize*/ + NCZ_szip_codec_to_hdf5, + NCZ_szip_hdf5_to_codec, + NCZ_szip_modify_parameters, +}; + +static int +NCZ_szip_codec_to_hdf5(const char* codec_json, size_t* nparamsp, unsigned** paramsp) +{ + int stat = NC_NOERR; + unsigned* params = NULL; + size_t nparams = 2; /* No. of visible parameters */ + NCjson* json = NULL; + NCjson* jtmp = NULL; + struct NCJconst jc = {0,0,0,NULL}; + + if(nparamsp == NULL || paramsp == NULL) + {stat = NC_EINTERNAL; goto done;} + + if((params = (unsigned*)calloc(nparams,sizeof(unsigned)))== NULL) + {stat = NC_ENOMEM; goto done;} + + if(NCJparse(codec_json,0,&json)) + {stat = NC_EFILTER; goto done;} + + if(NCJdictget(json,"mask",&jtmp) || jtmp == NULL) + {stat = NC_EFILTER; goto done;} + if(NCJcvt(jtmp,NCJ_INT,&jc)) + {stat = NC_EFILTER; goto done;} + params[H5Z_SZIP_PARM_MASK] = (unsigned)jc.ival; + + jtmp = NULL; + if(NCJdictget(json,"pixels-per-block",&jtmp) || jtmp == NULL) + {stat = NC_EFILTER; goto done;} + if(NCJcvt(jtmp,NCJ_INT,&jc)) + {stat = NC_EFILTER; goto done;} + params[H5Z_SZIP_PARM_PPB] = (unsigned)jc.ival; + + *nparamsp = nparams; + *paramsp = params; params = NULL; + +done: + NCJreclaim(json); + nullfree(params); + return stat; +} + +static int +NCZ_szip_hdf5_to_codec(size_t nparams, const unsigned* params, char** codecp) +{ + int stat = NC_NOERR; + char json[2048]; + + snprintf(json,sizeof(json),"{\"id\": \"%s\", \"mask\": %u, \"pixels-per-block\": %u}", + NCZ_szip_codec.codecid, + params[H5Z_SZIP_PARM_MASK], + params[H5Z_SZIP_PARM_PPB]); + if(codecp) { + if((*codecp = strdup(json))==NULL) {stat = NC_ENOMEM; goto done;} + } + +done: + return stat; +} + +static int +NCZ_szip_modify_parameters(int ncid, int varid, size_t* vnparamsp, unsigned** vparamsp, size_t* wnparamsp, unsigned** wparamsp) +{ + int i,ret_value = NC_NOERR; + nc_type vtype; + size_t typesize, scanline, dtype_precision, npoints; + int ndims, storage, dtype_order; + int dimids[NC_MAX_VAR_DIMS]; + char vname[NC_MAX_NAME+1]; + size_t chunklens[NC_MAX_VAR_DIMS]; + unsigned* params = NULL; + unsigned* vparams = NULL; + size_t wnparams = 4; + + if(wnparamsp == NULL || wparamsp == NULL) + {ret_value = NC_EFILTER; goto done;} + if(vnparamsp == NULL || vparamsp == NULL) + {ret_value = NC_EFILTER; goto done;} + if(*vnparamsp > 0 && *vparamsp == NULL) + {ret_value = NC_EFILTER; goto done;} + + vparams = *vparamsp; + + /* Get variable info */ + if((ret_value = nc_inq_var(ncid,varid,vname,&vtype,&ndims,dimids,NULL))) goto done; + + /* Get the typesize */ + if((ret_value = nc_inq_type(ncid,vtype,NULL,&typesize))) goto done; + + /* Get datatype's precision, in case is less than full bits */ + dtype_precision = typesize*8; + + if(dtype_precision > 24) { + if(dtype_precision <= 32) + dtype_precision = 32; + else if(dtype_precision <= 64) + dtype_precision = 64; + } /* end if */ + + if(ndims == 0) {ret_value = NC_EFILTER; goto done;} + + /* Set "local" parameter for this dataset's "pixels-per-scanline" */ + if((ret_value = nc_inq_dimlen(ncid,dimids[ndims-1],&scanline))) goto done; + + /* Get number of elements for the dataspace; use + total number of elements in the chunk to define the new 'scanline' size */ + /* Compute chunksize */ + if((ret_value = nc_inq_var_chunking(ncid,varid,&storage,chunklens))) goto done; + if(storage != NC_CHUNKED) {ret_value = NC_EFILTER; goto done;} + npoints = 1; + for(i=0;i