diff --git a/CMakeLists.txt b/CMakeLists.txt index 680c4e1791..9348558d4b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -139,7 +139,6 @@ endif() include(CTest) -adios_option(Blosc "Enable support for Blosc transforms" AUTO) adios_option(Blosc2 "Enable support for c-blosc-2 transforms" AUTO) adios_option(BZip2 "Enable support for BZip2 transforms" AUTO) adios_option(ZFP "Enable support for ZFP transforms" AUTO) @@ -227,7 +226,7 @@ endif() set(ADIOS2_CONFIG_OPTS - BP5 DataMan DataSpaces HDF5 HDF5_VOL MHS SST CUDA Fortran MPI Python Blosc Blosc2 + BP5 DataMan DataSpaces HDF5 HDF5_VOL MHS SST CUDA Fortran MPI Python Blosc2 BZip2 LIBPRESSIO MGARD PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem UCX ZeroMQ Profiling Endian_Reverse GPU_Support AWSSDK ) diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index ee6637a73a..fed08da4c2 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -67,16 +67,6 @@ function(lists_get_prefix listVars outVar) set(${outVar} "${prefix}" PARENT_SCOPE) endfunction() -# Blosc -if(ADIOS2_USE_Blosc STREQUAL AUTO) - find_package(Blosc 1.7) -elseif(ADIOS2_USE_Blosc) - find_package(Blosc 1.7 REQUIRED) -endif() -if(BLOSC_FOUND) - set(ADIOS2_HAVE_Blosc TRUE) -endif() - # Blosc2 if(ADIOS2_USE_Blosc2 STREQUAL AUTO) find_package(Blosc2 2.4) diff --git a/cmake/FindBlosc.cmake b/cmake/FindBlosc.cmake deleted file mode 100644 index 7a6d3fe9f9..0000000000 --- a/cmake/FindBlosc.cmake +++ /dev/null @@ -1,93 +0,0 @@ -#------------------------------------------------------------------------------# -# Distributed under the OSI-approved Apache License, Version 2.0. See -# accompanying file Copyright.txt for details. -#------------------------------------------------------------------------------# -# -# FindBLOSC -# ----------- -# -# Try to find the BLOSC library -# -# This module defines the following variables: -# -# BLOSC_FOUND - System has BLOSC -# BLOSC_INCLUDE_DIRS - The BLOSC include directory -# BLOSC_LIBRARIES - Link these to use BLOSC -# BLOSC_VERSION - Version of the BLOSC library to support -# -# and the following imported targets: -# Blosc::Blosc - The core BLOSC library -# -# You can also set the following variable to help guide the search: -# BLOSC_ROOT - The install prefix for BLOSC containing the -# include and lib folders -# Note: this can be set as a CMake variable or an -# environment variable. If specified as a CMake -# variable, it will override any setting specified -# as an environment variable. - -if(NOT BLOSC_FOUND) - if((NOT BLOSC_ROOT) AND (DEFINED ENV{BLOSC_ROOT})) - set(BLOSC_ROOT "$ENV{BLOSC_ROOT}") - endif() - if(BLOSC_ROOT) - set(BLOSC_INCLUDE_OPTS HINTS ${BLOSC_ROOT}/include NO_DEFAULT_PATHS) - set(BLOSC_LIBRARY_OPTS - HINTS ${BLOSC_ROOT}/lib ${BLOSC_ROOT}/lib64 - NO_DEFAULT_PATHS - ) - endif() - if(WIN32) # uses a Unix-like library prefix on Windows - set(BLOSC_LIBRARY_OPTS - libblosc ${BLOSC_LIBRARY_OPTS} - ) - endif() - - find_path(BLOSC_INCLUDE_DIR blosc.h ${BLOSC_INCLUDE_OPTS}) - find_library(BLOSC_LIBRARY NAMES blosc ${BLOSC_LIBRARY_OPTS}) - if(BLOSC_INCLUDE_DIR) - file(STRINGS ${BLOSC_INCLUDE_DIR}/blosc.h _ver_string - REGEX [=[BLOSC_VERSION_STRING +"[^"]*"]=] - ) - if(_ver_string MATCHES [=[BLOSC_VERSION_STRING +"([0-9]+.[0-9]+.[0-9]+)]=]) - set(BLOSC_VERSION ${CMAKE_MATCH_1}) - endif() - endif() - - # Blosc depends on pthreads - # https://github.com/Blosc/c-blosc/blob/master/blosc/CMakeLists.txt - # https://github.com/Blosc/c-blosc/pull/318 - set(THREADS_PREFER_PTHREAD_FLAG TRUE) - if(WIN32) - # try to use the system library - find_package(Threads) - if(NOT Threads_FOUND) - message(STATUS "Blosc: used the internal pthread library for win32 systems.") - set(BLOSC_LIBRARIES) - else() - set(BLOSC_LIBRARIES Threads::Threads) - endif() - else() - find_package(Threads REQUIRED) - set(BLOSC_LIBRARIES Threads::Threads) - endif() - - include(FindPackageHandleStandardArgs) - find_package_handle_standard_args(Blosc - FOUND_VAR BLOSC_FOUND - VERSION_VAR BLOSC_VERSION - REQUIRED_VARS BLOSC_LIBRARY BLOSC_INCLUDE_DIR - ) - if(BLOSC_FOUND) - set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIR}) - set(BLOSC_LIBRARIES ${BLOSC_LIBRARY}) - if(BLOSC_FOUND AND NOT TARGET Blosc::Blosc) - add_library(Blosc::Blosc UNKNOWN IMPORTED) - set_target_properties(Blosc::Blosc PROPERTIES - IMPORTED_LOCATION "${BLOSC_LIBRARY}" - INTERFACE_INCLUDE_DIRECTORIES "${BLOSC_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "${BLOSC_LIBRARIES}" - ) - endif() - endif() -endif() diff --git a/cmake/adios2-config-common.cmake.in b/cmake/adios2-config-common.cmake.in index edaf1312f6..97e98e54b4 100644 --- a/cmake/adios2-config-common.cmake.in +++ b/cmake/adios2-config-common.cmake.in @@ -76,11 +76,6 @@ else() endif() if(NOT @BUILD_SHARED_LIBS@) - set(ADIOS2_HAVE_Blosc @ADIOS2_HAVE_Blosc@) - if(ADIOS2_HAVE_Blosc) - find_dependency(Blosc) - endif() - set(ADIOS2_HAVE_Blosc2 @ADIOS2_HAVE_Blosc2@) if(ADIOS2_HAVE_Blosc2) find_dependency(Blosc2) diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index b6e619ec35..3f7562636a 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -244,13 +244,8 @@ if(ADIOS2_HAVE_DataSpaces) target_link_libraries(adios2_core_mpi PRIVATE DataSpaces::DataSpaces) endif() -if(ADIOS2_HAVE_Blosc) - target_sources(adios2_core PRIVATE operator/compress/CompressBlosc.cpp) - target_link_libraries(adios2_core PRIVATE Blosc::Blosc) -endif() - if(ADIOS2_HAVE_Blosc2) - target_sources(adios2_core PRIVATE operator/compress/CompressBlosc2.cpp) + target_sources(adios2_core PRIVATE operator/compress/CompressBlosc.cpp) target_link_libraries(adios2_core PRIVATE Blosc2::Blosc2) endif() diff --git a/source/adios2/common/ADIOSTypes.h b/source/adios2/common/ADIOSTypes.h index 36f5fbeb3f..24eefef010 100644 --- a/source/adios2/common/ADIOSTypes.h +++ b/source/adios2/common/ADIOSTypes.h @@ -480,8 +480,8 @@ constexpr char blockSize100k_9[] = "9"; } // end namespace bzip2 #endif -// BBlosc PARAMETERS -#ifdef ADIOS2_HAVE_BLOSC +// Blosc PARAMETERS +#ifdef ADIOS2_HAVE_BLOSC2 constexpr char LosslessBlosc[] = "blosc"; namespace blosc @@ -528,54 +528,6 @@ constexpr char doshuffle_bitshuffle[] = "BLOSC_BITSHUFFLE"; #endif -// Blosc2 PARAMETERS -#ifdef ADIOS2_HAVE_BLOSC2 - -constexpr char LosslessBlosc2[] = "blosc2"; -namespace blosc2 -{ - -namespace key -{ -constexpr char nthreads[] = "nthreads"; -constexpr char compressor[] = "compressor"; -constexpr char clevel[] = "clevel"; -constexpr char doshuffle[] = "doshuffle"; -constexpr char blocksize[] = "blocksize"; -constexpr char threshold[] = "threshold"; -} - -namespace value -{ - -constexpr char compressor_blosclz[] = "blosclz"; -constexpr char compressor_lz4[] = "lz4"; -constexpr char compressor_lz4hc[] = "lz4hc"; -constexpr char compressor_snappy[] = "snappy"; -constexpr char compressor_zlib[] = "zlib"; -constexpr char compressor_zstd[] = "zstd"; - -constexpr char clevel_0[] = "0"; -constexpr char clevel_1[] = "1"; -constexpr char clevel_2[] = "2"; -constexpr char clevel_3[] = "3"; -constexpr char clevel_4[] = "4"; -constexpr char clevel_5[] = "5"; -constexpr char clevel_6[] = "6"; -constexpr char clevel_7[] = "7"; -constexpr char clevel_8[] = "8"; -constexpr char clevel_9[] = "9"; - -constexpr char doshuffle_shuffle[] = "BLOSC_SHUFFLE"; -constexpr char doshuffle_noshuffle[] = "BLOSC_NOSHUFFLE"; -constexpr char doshuffle_bitshuffle[] = "BLOSC_BITSHUFFLE"; - -} // end namespace value - -} // end namespace blosc2 - -#endif - } // end namespace ops } // end namespace adios2 diff --git a/source/adios2/core/Operator.h b/source/adios2/core/Operator.h index 6a4b7926e6..0be396dc5e 100644 --- a/source/adios2/core/Operator.h +++ b/source/adios2/core/Operator.h @@ -37,7 +37,6 @@ class Operator COMPRESS_SZ = 6, COMPRESS_ZFP = 7, COMPRESS_MGARDPLUS = 8, - COMPRESS_BLOSC2 = 9, CALLBACK_SIGNATURE1 = 51, CALLBACK_SIGNATURE2 = 52, PLUGIN_INTERFACE = 53, diff --git a/source/adios2/operator/OperatorFactory.cpp b/source/adios2/operator/OperatorFactory.cpp index e9d899eeb8..1979aeeb02 100644 --- a/source/adios2/operator/OperatorFactory.cpp +++ b/source/adios2/operator/OperatorFactory.cpp @@ -14,12 +14,8 @@ #include "adios2/operator/plugin/PluginOperator.h" #include -#ifdef ADIOS2_HAVE_BLOSC -#include "adios2/operator/compress/CompressBlosc.h" -#endif - #ifdef ADIOS2_HAVE_BLOSC2 -#include "adios2/operator/compress/CompressBlosc2.h" +#include "adios2/operator/compress/CompressBlosc.h" #endif #ifdef ADIOS2_HAVE_BZIP2 @@ -80,8 +76,6 @@ std::string OperatorTypeToString(const Operator::OperatorType type) return "zfp"; case Operator::PLUGIN_INTERFACE: return "plugin"; - case Operator::COMPRESS_BLOSC2: - return "blosc2"; default: return "null"; } @@ -96,7 +90,7 @@ std::shared_ptr MakeOperator(const std::string &type, if (typeLowerCase == "blosc") { -#ifdef ADIOS2_HAVE_BLOSC +#ifdef ADIOS2_HAVE_BLOSC2 ret = std::make_shared(parameters); #endif } @@ -152,12 +146,6 @@ std::shared_ptr MakeOperator(const std::string &type, { ret = std::make_shared(parameters); } - else if (typeLowerCase == "blosc2") - { -#ifdef ADIOS2_HAVE_BLOSC2 - ret = std::make_shared(parameters); -#endif - } else if (typeLowerCase == "null") { ret = std::make_shared(parameters); diff --git a/source/adios2/operator/compress/CompressBlosc.cpp b/source/adios2/operator/compress/CompressBlosc.cpp index 0dc9b3540f..1e70bf878e 100644 --- a/source/adios2/operator/compress/CompressBlosc.cpp +++ b/source/adios2/operator/compress/CompressBlosc.cpp @@ -12,7 +12,7 @@ #include "CompressBlosc.h" extern "C" { -#include +#include } #include "adios2/helper/adiosFunctions.h" @@ -54,22 +54,22 @@ size_t CompressBlosc::Operate(const char *dataIn, const Dims &blockStart, const size_t sizeIn = helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type)); - // blosc V1 metadata + // blosc2 V1 metadata PutParameter(bufferOut, bufferOutOffset, sizeIn); PutParameter(bufferOut, bufferOutOffset, - static_cast(BLOSC_VERSION_MAJOR)); + static_cast(BLOSC2_VERSION_MAJOR)); PutParameter(bufferOut, bufferOutOffset, - static_cast(BLOSC_VERSION_MINOR)); + static_cast(BLOSC2_VERSION_MINOR)); PutParameter(bufferOut, bufferOutOffset, - static_cast(BLOSC_VERSION_RELEASE)); - // blosc V1 metadata end + static_cast(BLOSC2_VERSION_RELEASE)); + // blosc2 V1 metadata end bool useMemcpy = false; // input size under this bound will not compress size_t thresholdSize = 128; - blosc_init(); + blosc2_init(); size_t threads = 1; // defaults int compressionLevel = 1; @@ -166,7 +166,7 @@ size_t CompressBlosc::Operate(const char *dataIn, const Dims &blockStart, if (!useMemcpy) { - const int result = blosc_set_compressor(compressor.c_str()); + const int result = blosc1_set_compressor(compressor.c_str()); if (result == -1) { helper::Throw( @@ -174,24 +174,23 @@ size_t CompressBlosc::Operate(const char *dataIn, const Dims &blockStart, "blosc library linked does not support compressor " + compressor); } - blosc_set_nthreads(threads); - blosc_set_blocksize(blockSize); + blosc2_set_nthreads(threads); + blosc1_set_blocksize(blockSize); uint32_t chunk = 0; for (; inputOffset < sizeIn; ++chunk) { size_t inputChunkSize = std::min(sizeIn - inputOffset, - static_cast(BLOSC_MAX_BUFFERSIZE)); + static_cast(BLOSC2_MAX_BUFFERSIZE)); bloscSize_t maxIntputSize = static_cast(inputChunkSize); - bloscSize_t maxChunkSize = maxIntputSize + BLOSC_MAX_OVERHEAD; + bloscSize_t maxChunkSize = maxIntputSize + BLOSC2_MAX_OVERHEAD; - bloscSize_t compressedChunkSize = - blosc_compress(compressionLevel, doShuffle, typesize, - maxIntputSize, dataIn + inputOffset, - bufferOut + bufferOutOffset, maxChunkSize); + bloscSize_t compressedChunkSize = blosc2_compress( + compressionLevel, doShuffle, typesize, dataIn + inputOffset, + maxIntputSize, bufferOut + bufferOutOffset, maxChunkSize); if (compressedChunkSize > 0) bufferOutOffset += static_cast(compressedChunkSize); @@ -220,7 +219,7 @@ size_t CompressBlosc::Operate(const char *dataIn, const Dims &blockStart, headerPtr->SetNumChunks(0u); } - blosc_destroy(); + blosc2_destroy(); return bufferOutOffset; } @@ -239,7 +238,7 @@ size_t CompressBlosc::InverseOperate(const char *bufferIn, const size_t sizeIn, } else if (bufferVersion == 2) { - // TODO: if a Version 2 blosc buffer is being implemented, put it here + // TODO: if a Version 2 blosc2 buffer is being implemented, put it here // and keep the DecompressV1 routine for backward compatibility } else @@ -325,7 +324,7 @@ size_t CompressBlosc::DecompressChunkedFormat(const char *bufferIn, if (isCompressed) { - blosc_init(); + blosc2_init(); size_t threads = 1; // defaults for (const auto &itParameter : m_Parameters) @@ -338,17 +337,17 @@ size_t CompressBlosc::DecompressChunkedFormat(const char *bufferIn, value, "when setting Blosc nthreads parameter\n")); } } - blosc_set_nthreads(threads); + blosc2_set_nthreads(threads); while (inputOffset < inputDataSize) { /* move over the size of the compressed data */ const char *in_ptr = inputDataBuff + inputOffset; - /** read the size of the compress block from the blosc meta data + /** read the size of the compress block from the blosc2 meta data * - * blosc meta data format (all little endian): - * - 1 byte blosc format version + * blosc2 meta data format (all little endian): + * - 1 byte blosc2 format version * - 1 byte blosclz format version * - 1 byte flags * - 1 byte typesize @@ -365,12 +364,12 @@ size_t CompressBlosc::DecompressChunkedFormat(const char *bufferIn, size_t outputChunkSize = std::min(uncompressedSize - currentOutputSize, - static_cast(BLOSC_MAX_BUFFERSIZE)); + static_cast(BLOSC2_MAX_BUFFERSIZE)); bloscSize_t max_output_size = static_cast(outputChunkSize); - bloscSize_t decompressdSize = - blosc_decompress(in_ptr, out_ptr, max_output_size); + bloscSize_t decompressdSize = blosc2_decompress( + in_ptr, max_inputDataSize, out_ptr, max_output_size); if (decompressdSize > 0) currentOutputSize += static_cast(decompressdSize); @@ -383,7 +382,7 @@ size_t CompressBlosc::DecompressChunkedFormat(const char *bufferIn, } inputOffset += static_cast(max_inputDataSize); } - blosc_destroy(); + blosc2_destroy(); } else { @@ -402,7 +401,7 @@ size_t CompressBlosc::DecompressOldFormat(const char *bufferIn, const size_t sizeIn, char *dataOut, const size_t sizeOut) const { - blosc_init(); + blosc2_init(); size_t threads = 1; // defaults for (const auto &itParameter : m_Parameters) { @@ -414,9 +413,10 @@ size_t CompressBlosc::DecompressOldFormat(const char *bufferIn, value, "when setting Blosc nthreads parameter\n")); } } - blosc_set_nthreads(threads); - const int decompressedSize = blosc_decompress(bufferIn, dataOut, sizeOut); - blosc_destroy(); + blosc2_set_nthreads(threads); + const int decompressedSize = + blosc2_decompress(bufferIn, sizeIn, dataOut, sizeOut); + blosc2_destroy(); return static_cast(decompressedSize); } diff --git a/source/adios2/operator/compress/CompressBlosc.h b/source/adios2/operator/compress/CompressBlosc.h index 69f225cc5b..6af032a271 100644 --- a/source/adios2/operator/compress/CompressBlosc.h +++ b/source/adios2/operator/compress/CompressBlosc.h @@ -93,19 +93,19 @@ class CompressBlosc : public Operator ADIOS2_CLASS_PACKED(DataHeader) { - /** compatible to the first 4 byte of blosc header + /** compatible to the first 4 byte of blosc2 header * - * blosc meta data format (all little endian): - * - 1 byte blosc format version - * - 1 byte blosclz format version + * blosc2 meta data format (all little endian): + * - 1 byte blosc2 format version + * - 1 byte blosc2lz format version * - 1 byte flags * - 1 byte typesize * - * If zero we writing the new adios blosc format which can handle more + * If zero we writing the new adios blosc2 format which can handle more * than 2GiB data chunks. */ uint32_t format = 0u; - /** number of blosc chunks within the data blob + /** number of blosc2 chunks within the data blob * * If zero the data is not compressed and must be decompressed by using * 'memcpy' diff --git a/source/adios2/operator/compress/CompressBlosc2.cpp b/source/adios2/operator/compress/CompressBlosc2.cpp deleted file mode 100644 index 5f21524c30..0000000000 --- a/source/adios2/operator/compress/CompressBlosc2.cpp +++ /dev/null @@ -1,425 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * CompressBlosc2.cpp - * - * Created on: Jun 18, 2019 - * Author: William F Godoy godoywf@ornl.gov - * Rene Widera r.widera@hzdr.de - */ - -#include "CompressBlosc2.h" - -extern "C" { -#include -} - -#include "adios2/helper/adiosFunctions.h" - -#include -#include -#include -#include - -namespace adios2 -{ -namespace core -{ -namespace compress -{ - -const std::map CompressBlosc2::m_Shuffles = { - {"BLOSC_NOSHUFFLE", BLOSC_NOSHUFFLE}, - {"BLOSC_SHUFFLE", BLOSC_SHUFFLE}, - {"BLOSC_BITSHUFFLE", BLOSC_BITSHUFFLE}}; - -const std::set CompressBlosc2::m_Compressors = { - "blosclz", "lz4", "lz4hc", "snappy", "zlib", "zstd"}; - -CompressBlosc2::CompressBlosc2(const Params ¶meters) -: Operator("blosc2", COMPRESS_BLOSC2, "compress", parameters) -{ -} - -size_t CompressBlosc2::Operate(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, const DataType type, - char *bufferOut) -{ - size_t bufferOutOffset = 0; - const uint8_t bufferVersion = 1; - - MakeCommonHeader(bufferOut, bufferOutOffset, bufferVersion); - - const size_t sizeIn = - helper::GetTotalSize(blockCount, helper::GetDataTypeSize(type)); - - // blosc2 V1 metadata - PutParameter(bufferOut, bufferOutOffset, sizeIn); - PutParameter(bufferOut, bufferOutOffset, - static_cast(BLOSC2_VERSION_MAJOR)); - PutParameter(bufferOut, bufferOutOffset, - static_cast(BLOSC2_VERSION_MINOR)); - PutParameter(bufferOut, bufferOutOffset, - static_cast(BLOSC2_VERSION_RELEASE)); - // blosc2 V1 metadata end - - bool useMemcpy = false; - - // input size under this bound will not compress - size_t thresholdSize = 128; - - blosc2_init(); - - size_t threads = 1; // defaults - int compressionLevel = 1; - int doShuffle = BLOSC_SHUFFLE; - std::string compressor = "blosclz"; - size_t blockSize = 0; - - for (const auto &itParameter : m_Parameters) - { - const std::string key = itParameter.first; - const std::string value = itParameter.second; - - if (key == "compression_level" || key == "clevel") - { - compressionLevel = static_cast(helper::StringTo( - value, "when setting Blosc2 clevel parameter\n")); - if (compressionLevel < 0 || compressionLevel > 9) - { - helper::Throw( - "Operator", "CompressBlosc2", "Operate", - "compression_level must be an integer between 0 (no " - "compression) and 9 (more compression, more memory " - "consumption) inclusive"); - } - } - else if (key == "doshuffle") - { - auto itShuffle = m_Shuffles.find(value); - if (itShuffle == m_Shuffles.end()) - { - helper::Throw( - "Operator", "CompressBlosc2", "Operate", - "Parameter doshuffle must be BLOSC_SHUFFLE, " - "BLOSC_NOSHUFFLE or BLOSC_BITSHUFFLE"); - } - doShuffle = itShuffle->second; - } - else if (key == "nthreads") - { - threads = static_cast(helper::StringTo( - value, "when setting Blosc2 nthreads parameter\n")); - } - else if (key == "compressor") - { - compressor = value; - if (m_Compressors.count(compressor) == 0) - { - helper::Throw( - "Operator", "CompressBlosc2", "Operate", - "Parameter compressor must be blosclz (default), lz4, " - "lz4hc, snappy, zlib, or zstd"); - } - } - else if (key == "blocksize") - { - blockSize = static_cast(helper::StringTo( - value, "when setting Blosc2 blocksize parameter\n")); - } - else if (key == "threshold") - { - thresholdSize = static_cast(helper::StringTo( - value, "when setting Blosc2 threshold parameter\n")); - if (thresholdSize < 128u) - thresholdSize = 128u; - } - else - { - helper::Log("Operator", "CompressBlosc2", "Operate", - "Unknown parameter keyword '" + key + "' with value '" + - value + "' passed to Blosc2 compression operator.", - helper::WARNING); - } - } - - // write header to detect new compression format (set first 8 byte to zero) - DataHeader *headerPtr = - reinterpret_cast(bufferOut + bufferOutOffset); - - // set default header - *headerPtr = DataHeader{}; - bufferOutOffset += sizeof(DataHeader); - - int32_t typesize = helper::GetDataTypeSize(type); - if (typesize > BLOSC_MAX_TYPESIZE) - typesize = 1; - - size_t inputOffset = 0u; - - if (sizeIn < thresholdSize) - { - /* disable compression */ - useMemcpy = true; - } - - if (!useMemcpy) - { - const int result = blosc1_set_compressor(compressor.c_str()); - if (result == -1) - { - helper::Throw( - "Operator", "CompressBlosc2", "Operate", - "blosc2 library linked does not support compressor " + - compressor); - } - blosc2_set_nthreads(threads); - blosc1_set_blocksize(blockSize); - - uint32_t chunk = 0; - for (; inputOffset < sizeIn; ++chunk) - { - size_t inputChunkSize = - std::min(sizeIn - inputOffset, - static_cast(BLOSC2_MAX_BUFFERSIZE)); - blosc2Size_t maxIntputSize = - static_cast(inputChunkSize); - - blosc2Size_t maxChunkSize = maxIntputSize + BLOSC2_MAX_OVERHEAD; - - blosc2Size_t compressedChunkSize = blosc2_compress( - compressionLevel, doShuffle, typesize, dataIn + inputOffset, - maxIntputSize, bufferOut + bufferOutOffset, maxChunkSize); - - if (compressedChunkSize > 0) - bufferOutOffset += static_cast(compressedChunkSize); - else - { - // something went wrong with the compression switch to memcopy - useMemcpy = true; - break; - } - /* add size to written output data */ - inputOffset += static_cast(maxIntputSize); - } - - if (!useMemcpy) - { - // validate that all bytes are compressed - assert(inputOffset == sizeIn); - headerPtr->SetNumChunks(chunk); - } - } - - if (useMemcpy) - { - std::memcpy(bufferOut + bufferOutOffset, dataIn + inputOffset, sizeIn); - bufferOutOffset += sizeIn; - headerPtr->SetNumChunks(0u); - } - - blosc2_destroy(); - return bufferOutOffset; -} - -size_t CompressBlosc2::InverseOperate(const char *bufferIn, const size_t sizeIn, - char *dataOut) -{ - size_t bufferInOffset = 1; // skip operator type - const uint8_t bufferVersion = - GetParameter(bufferIn, bufferInOffset); - bufferInOffset += 2; // skip two reserved bytes - - if (bufferVersion == 1) - { - return DecompressV1(bufferIn + bufferInOffset, sizeIn - bufferInOffset, - dataOut); - } - else if (bufferVersion == 2) - { - // TODO: if a Version 2 blosc2 buffer is being implemented, put it here - // and keep the DecompressV1 routine for backward compatibility - } - else - { - helper::Throw("Operator", "CompressBlosc2", - "InverseOperate", - "invalid blosc2 buffer version"); - } - - return 0; -} - -bool CompressBlosc2::IsDataTypeValid(const DataType type) const { return true; } - -size_t CompressBlosc2::DecompressV1(const char *bufferIn, const size_t sizeIn, - char *dataOut) -{ - // Do NOT remove even if the buffer version is updated. Data might be still - // in lagacy formats. This function must be kept for backward compatibility. - // If a newer buffer format is implemented, create another function, e.g. - // DecompressV2 and keep this function for decompressing lagacy data. - - size_t bufferInOffset = 0; - size_t sizeOut = GetParameter(bufferIn, bufferInOffset); - - m_VersionInfo = - " Data is compressed using BLOSC2 Version " + - std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + - std::to_string(GetParameter(bufferIn, bufferInOffset)) + "." + - std::to_string(GetParameter(bufferIn, bufferInOffset)) + - ". Please make sure a compatible version is used for decompression."; - - if (sizeIn - bufferInOffset < sizeof(DataHeader)) - { - helper::Throw( - "Operator", "CompressBlosc2", "InverseOperate", - "corrupted compressed buffer." + m_VersionInfo); - } - const bool isChunked = - reinterpret_cast(bufferIn + bufferInOffset) - ->IsChunked(); - - size_t decompressedSize = 0; - if (isChunked) - { - decompressedSize = - DecompressChunkedFormat(bufferIn + bufferInOffset, - sizeIn - bufferInOffset, dataOut, sizeOut); - } - else - { - decompressedSize = - DecompressOldFormat(bufferIn + bufferInOffset, - sizeIn - bufferInOffset, dataOut, sizeOut); - } - if (decompressedSize != sizeOut) - { - helper::Throw("Operator", "CompressBlosc2", - "DecompressV1", m_VersionInfo); - } - return sizeOut; -} - -size_t CompressBlosc2::DecompressChunkedFormat(const char *bufferIn, - const size_t sizeIn, - char *dataOut, - const size_t sizeOut) const -{ - const DataHeader *dataPtr = reinterpret_cast(bufferIn); - uint32_t num_chunks = dataPtr->GetNumChunks(); - size_t inputDataSize = sizeIn - sizeof(DataHeader); - - bool isCompressed = true; - if (num_chunks == 0) - isCompressed = false; - - size_t inputOffset = 0u; - size_t currentOutputSize = 0u; - - const char *inputDataBuff = bufferIn + sizeof(DataHeader); - - size_t uncompressedSize = sizeOut; - - if (isCompressed) - { - blosc2_init(); - - size_t threads = 1; // defaults - for (const auto &itParameter : m_Parameters) - { - const std::string key = itParameter.first; - const std::string value = itParameter.second; - if (key == "nthreads") - { - threads = static_cast(helper::StringTo( - value, "when setting Blosc2 nthreads parameter\n")); - } - } - blosc2_set_nthreads(threads); - - while (inputOffset < inputDataSize) - { - /* move over the size of the compressed data */ - const char *in_ptr = inputDataBuff + inputOffset; - - /** read the size of the compress block from the blosc2 meta data - * - * blosc2 meta data format (all little endian): - * - 1 byte blosc2 format version - * - 1 byte blosclz format version - * - 1 byte flags - * - 1 byte typesize - * - 4 byte uncompressed data size - * - 4 byte block size - * - 4 byte compressed data size - * - * we need only the compressed size ( source address + 12 byte) - */ - blosc2Size_t max_inputDataSize; - std::memcpy(&max_inputDataSize, in_ptr + 12, sizeof(blosc2Size_t)); - - char *out_ptr = dataOut + currentOutputSize; - - size_t outputChunkSize = - std::min(uncompressedSize - currentOutputSize, - static_cast(BLOSC2_MAX_BUFFERSIZE)); - blosc2Size_t max_output_size = - static_cast(outputChunkSize); - - blosc2Size_t decompressdSize = blosc2_decompress( - in_ptr, max_inputDataSize, out_ptr, max_output_size); - - if (decompressdSize > 0) - currentOutputSize += static_cast(decompressdSize); - else - { - helper::Throw( - "Operator", "CompressBlosc2", "DecompressChunkedFormat", - "blosc2 decompress failed with zero buffer size. " + - m_VersionInfo); - } - inputOffset += static_cast(max_inputDataSize); - } - blosc2_destroy(); - } - else - { - std::memcpy(dataOut, inputDataBuff, inputDataSize); - currentOutputSize = inputDataSize; - inputOffset += inputDataSize; - } - - assert(currentOutputSize == uncompressedSize); - assert(inputOffset == inputDataSize); - - return currentOutputSize; -} - -size_t CompressBlosc2::DecompressOldFormat(const char *bufferIn, - const size_t sizeIn, char *dataOut, - const size_t sizeOut) const -{ - blosc2_init(); - size_t threads = 1; // defaults - for (const auto &itParameter : m_Parameters) - { - const std::string key = itParameter.first; - const std::string value = itParameter.second; - if (key == "nthreads") - { - threads = static_cast(helper::StringTo( - value, "when setting Blosc2 nthreads parameter\n")); - } - } - blosc2_set_nthreads(threads); - const int decompressedSize = - blosc2_decompress(bufferIn, sizeIn, dataOut, sizeOut); - blosc2_destroy(); - return static_cast(decompressedSize); -} - -} // end namespace compress -} // end namespace core -} // end namespace adios2 diff --git a/source/adios2/operator/compress/CompressBlosc2.h b/source/adios2/operator/compress/CompressBlosc2.h deleted file mode 100644 index a32c27d652..0000000000 --- a/source/adios2/operator/compress/CompressBlosc2.h +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * CompressBlosc2.h - * - * Created on: Jun 18, 2019 - * Author: William F Godoy godoywf@ornl.gov - * Rene Widera r.widera@hzdr.de - */ - -#ifndef ADIOS2_OPERATOR_COMPRESS_COMPRESSBLOSC2_H_ -#define ADIOS2_OPERATOR_COMPRESS_COMPRESSBLOSC2_H_ - -#include -#include - -#include "adios2/core/Operator.h" - -#if defined(_MSC_VER) -#define ADIOS2_CLASS_PACKED(name) __pragma(pack(push, 1)) class name -#define ADIOS2_CLASS_PACKED_SUFFIX __pragma(pack(pop)) -#else -#define ADIOS2_CLASS_PACKED(name) class __attribute__((packed)) name -#define ADIOS2_CLASS_PACKED_SUFFIX -#endif - -namespace adios2 -{ -namespace core -{ -namespace compress -{ - -class CompressBlosc2 : public Operator -{ - -public: - /** - * Unique constructor - */ - CompressBlosc2(const Params ¶meters); - - ~CompressBlosc2() = default; - - /** - * @param dataIn - * @param blockStart - * @param blockCount - * @param type - * @param bufferOut format will be: 'DataHeader ; (Blosc2CompressedChunk | - * UncompressedData), [ Blosc2CompressedChunk, ...]' - * @return size of compressed buffer in bytes - */ - size_t Operate(const char *dataIn, const Dims &blockStart, - const Dims &blockCount, const DataType type, - char *bufferOut) final; - - /** - * @param bufferIn - * @param sizeIn - * @param dataOut - * @return size of decompressed buffer - */ - size_t InverseOperate(const char *bufferIn, const size_t sizeIn, - char *dataOut) final; - - bool IsDataTypeValid(const DataType type) const final; - -private: - using blosc2Size_t = int32_t; - - /** Decompress chunked data */ - size_t DecompressChunkedFormat(const char *bufferIn, const size_t sizeIn, - char *dataOut, const size_t sizeOut) const; - - /** Decompress data written before ADIOS2 supported large variables larger - * 2GiB. */ - size_t DecompressOldFormat(const char *bufferIn, const size_t sizeIn, - char *dataOut, const size_t sizeOut) const; - - /** - * Decompress function for V1 buffer. Do NOT remove even if the buffer - * version is updated. Data might be still in lagacy formats. This function - * must be kept for backward compatibility - * @param bufferIn : compressed data buffer (V1 only) - * @param sizeIn : number of bytes in bufferIn - * @param dataOut : decompressed data buffer - * @return : number of bytes in dataOut - */ - size_t DecompressV1(const char *bufferIn, const size_t sizeIn, - char *dataOut); - - ADIOS2_CLASS_PACKED(DataHeader) - { - /** compatible to the first 4 byte of blosc2 header - * - * blosc2 meta data format (all little endian): - * - 1 byte blosc2 format version - * - 1 byte blosc2lz format version - * - 1 byte flags - * - 1 byte typesize - * - * If zero we writing the new adios blosc2 format which can handle more - * than 2GiB data chunks. - */ - uint32_t format = 0u; - /** number of blosc2 chunks within the data blob - * - * If zero the data is not compressed and must be decompressed by using - * 'memcpy' - */ - uint32_t numberOfChunks = 0u; - - public: - void SetNumChunks(const uint32_t numChunks) - { - numberOfChunks = numChunks; - } - uint32_t GetNumChunks() const { return numberOfChunks; } - - bool IsChunked() const { return format == 0; } - } - ADIOS2_CLASS_PACKED_SUFFIX; - - static const std::map m_Shuffles; - static const std::set m_Compressors; - - std::string m_VersionInfo; -}; - -} // end namespace compress -} // end namespace core -} // end namespace adios2 - -#undef ADIOS2_CLASS_PACKED -#undef ADIOS2_CLASS_PACKED_SUFFIX - -#endif /* ADIOS2_OPERATOR_COMPRESS_COMPRESSBLOSC2_H_ */ diff --git a/source/adios2/toolkit/format/bp/BPBase.cpp b/source/adios2/toolkit/format/bp/BPBase.cpp index fb8879ee56..5a8b0d0efc 100644 --- a/source/adios2/toolkit/format/bp/BPBase.cpp +++ b/source/adios2/toolkit/format/bp/BPBase.cpp @@ -446,15 +446,20 @@ std::string BPBase::ReadBPString(const std::vector &buffer, // static members const std::set BPBase::m_TransformTypes = { {"unknown", "none", "identity", "bzip2", "sz", "zfp", "mgard", "png", - "blosc", "blosc2", "sirius", "mgardplus", "plugin"}}; + "blosc", "sirius", "mgardplus", "plugin"}}; const std::map BPBase::m_TransformTypesToNames = { - {transform_unknown, "unknown"}, {transform_none, "none"}, - {transform_identity, "identity"}, {transform_sz, "sz"}, - {transform_zfp, "zfp"}, {transform_mgard, "mgard"}, - {transform_png, "png"}, {transform_bzip2, "bzip2"}, - {transform_blosc, "blosc"}, {transform_blosc2, "blosc2"}, - {transform_sirius, "sirius"}, {transform_mgardplus, "mgardplus"}, + {transform_unknown, "unknown"}, + {transform_none, "none"}, + {transform_identity, "identity"}, + {transform_sz, "sz"}, + {transform_zfp, "zfp"}, + {transform_mgard, "mgard"}, + {transform_png, "png"}, + {transform_bzip2, "bzip2"}, + {transform_blosc, "blosc"}, + {transform_sirius, "sirius"}, + {transform_mgardplus, "mgardplus"}, {transform_plugin, "plugin"}}; BPBase::TransformTypes diff --git a/source/adios2/toolkit/format/bp/BPBase.h b/source/adios2/toolkit/format/bp/BPBase.h index bdfa25084b..8d35dd6c69 100644 --- a/source/adios2/toolkit/format/bp/BPBase.h +++ b/source/adios2/toolkit/format/bp/BPBase.h @@ -460,7 +460,6 @@ class BPBase transform_sirius = 14, transform_mgardplus = 15, transform_plugin = 16, - transform_blosc2 = 17 }; /** Supported transform types */ diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp index 41d99d46ff..d0603ae2bb 100644 --- a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp +++ b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.cpp @@ -125,8 +125,26 @@ void BP3Deserializer::ParseMinifooter(const BufferSTL &bufferSTL) position = bufferSize - m_MetadataSet.MiniFooterSize; + // Writer's ADIOS version m_Minifooter.VersionTag.assign(&buffer[position], 28); - position += 28; + position += 24; + m_Minifooter.ADIOSVersionMajor = + helper::ReadValue(buffer, position, + m_Minifooter.IsLittleEndian) - + (uint8_t)'0'; + m_Minifooter.ADIOSVersionMinor = + helper::ReadValue(buffer, position, + m_Minifooter.IsLittleEndian) - + (uint8_t)'0'; + m_Minifooter.ADIOSVersionPatch = + helper::ReadValue(buffer, position, + m_Minifooter.IsLittleEndian) - + (uint8_t)'0'; + m_Minifooter.ADIOSVersion = m_Minifooter.ADIOSVersionMajor * 1000000 + + m_Minifooter.ADIOSVersionMinor * 1000 + + m_Minifooter.ADIOSVersionPatch; + ++position; + ; m_Minifooter.PGIndexStart = helper::ReadValue( buffer, position, m_Minifooter.IsLittleEndian); diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.h b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.h index 3a9ce32246..f91f84d40c 100644 --- a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.h +++ b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.h @@ -104,6 +104,9 @@ class BP3Deserializer : virtual public BP3Base const bool isRowMajorDestination, const size_t threadID = 0); + void BackCompatDecompress(const helper::SubStreamBoxInfo &subStreamBoxInfo, + const size_t threadID = 0); + /** * Clips and assigns memory to blockInfo.Data from a contiguous memory * input diff --git a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc index 32b66e3592..56dfdcd80c 100644 --- a/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc +++ b/source/adios2/toolkit/format/bp/bp3/BP3Deserializer.tcc @@ -155,8 +155,30 @@ void BP3Deserializer::SetVariableBlockInfo( blockOperation.PreSizeOf = sizeof(T); // read metadata from supported type and populate Info - std::memcpy(&blockOperation.PayloadSize, bpOpInfo.Metadata.data() + 8, - 8); + if (m_Minifooter.ADIOSVersion >= 2008000) + { + std::memcpy(&blockOperation.PayloadSize, + bpOpInfo.Metadata.data() + 8, 8); + } + else + { + // Files made before 2.8 have incompatible compression operators + // Add backward compatible fixes here to parse compression metadata + // directly from the BP4 metadata format + std::shared_ptr bpOp = + SetBPBackCompatOperation(bpOpInfo.Type); + if (bpOp) + { + bpOp->GetMetadata(bpOpInfo.Metadata, blockOperation.Info); + blockOperation.PayloadSize = static_cast( + std::stoull(blockOperation.Info.at("OutputSize"))); + } + else + { + std::memcpy(&blockOperation.PayloadSize, + bpOpInfo.Metadata.data() + 8, 8); + } + } subStreamInfo.OperationsInfo.push_back(std::move(blockOperation)); }; @@ -516,34 +538,43 @@ void BP3Deserializer::PostDataRead( { if (subStreamBoxInfo.OperationsInfo.size() > 0) { - const helper::BlockOperationInfo &blockOperationInfo = - InitPostOperatorBlockData(subStreamBoxInfo.OperationsInfo); + if (m_Minifooter.ADIOSVersion >= 2008000) + { + const helper::BlockOperationInfo &blockOperationInfo = + InitPostOperatorBlockData(subStreamBoxInfo.OperationsInfo); - const size_t preOpPayloadSize = - helper::GetTotalSize(blockOperationInfo.PreCount) * - blockOperationInfo.PreSizeOf; - m_ThreadBuffers[threadID][0].resize(preOpPayloadSize); + const size_t preOpPayloadSize = + helper::GetTotalSize(blockOperationInfo.PreCount) * + blockOperationInfo.PreSizeOf; + m_ThreadBuffers[threadID][0].resize(preOpPayloadSize); - // get original block back - char *preOpData = m_ThreadBuffers[threadID][0].data(); - const char *postOpData = m_ThreadBuffers[threadID][1].data(); + // get original block back + char *preOpData = m_ThreadBuffers[threadID][0].data(); + const char *postOpData = m_ThreadBuffers[threadID][1].data(); - std::shared_ptr op = nullptr; - for (auto &o : blockInfo.Operations) - { - if (o->m_Category == "compress" || o->m_Category == "plugin") + std::shared_ptr op = nullptr; + for (auto &o : blockInfo.Operations) { - op = o; - break; + if (o->m_Category == "compress" || o->m_Category == "plugin") + { + op = o; + break; + } } - } - core::Decompress(postOpData, blockOperationInfo.PayloadSize, preOpData, - op); + core::Decompress(postOpData, blockOperationInfo.PayloadSize, + preOpData, op); - // clip block to match selection - helper::ClipVector(m_ThreadBuffers[threadID][0], - subStreamBoxInfo.Seeks.first, - subStreamBoxInfo.Seeks.second); + // clip block to match selection + helper::ClipVector(m_ThreadBuffers[threadID][0], + subStreamBoxInfo.Seeks.first, + subStreamBoxInfo.Seeks.second); + } + else + { + // Files made before 2.8 have incompatible compression operators + // Add backward compatible fixes in the function below + BackCompatDecompress(subStreamBoxInfo, threadID); + } } #ifdef ADIOS2_HAVE_ENDIAN_REVERSE @@ -610,6 +641,48 @@ void BP3Deserializer::PostDataRead( } } +void BP3Deserializer::BackCompatDecompress( + const helper::SubStreamBoxInfo &subStreamBoxInfo, const size_t threadID) +{ + // Files made before 2.8 have incompatible compression operators + // Add backward compatible fixes here + const helper::BlockOperationInfo &blockOperationInfo = + InitPostOperatorBlockData(subStreamBoxInfo.OperationsInfo); + + const size_t preOpPayloadSize = + helper::GetTotalSize(blockOperationInfo.PreCount) * + blockOperationInfo.PreSizeOf; + m_ThreadBuffers[threadID][0].resize(preOpPayloadSize); + + std::string opType = blockOperationInfo.Info.at("Type"); + + // get original block back + char *preOpData = m_ThreadBuffers[threadID][0].data(); + const char *postOpData = m_ThreadBuffers[threadID][1].data(); + // get the right bp3Op + std::shared_ptr bp3Op = + SetBPBackCompatOperation(opType); + + if (bp3Op) + { + bp3Op->GetData(postOpData, blockOperationInfo, preOpData); + // clip block to match selection + helper::ClipVector(m_ThreadBuffers[threadID][0], + subStreamBoxInfo.Seeks.first, + subStreamBoxInfo.Seeks.second); + } + else + { + helper::Throw( + "Toolkit", "format::bp::BP3Deserializer", "PostDataRead", + "This file was created by pre-ADIOS 2.8.0 using " + "compression type " + + opType + + ", for which there is no backward compatible reader in this " + "ADIOS version"); + } +} + template std::map::BPInfo>> BP3Deserializer::AllStepsBlocksInfo(const core::Variable &variable) const diff --git a/source/adios2/toolkit/format/bp/bpBackCompatOperation/compress/BPBackCompatBlosc.cpp b/source/adios2/toolkit/format/bp/bpBackCompatOperation/compress/BPBackCompatBlosc.cpp index 3a64ec2100..ccdb0c4515 100644 --- a/source/adios2/toolkit/format/bp/bpBackCompatOperation/compress/BPBackCompatBlosc.cpp +++ b/source/adios2/toolkit/format/bp/bpBackCompatOperation/compress/BPBackCompatBlosc.cpp @@ -12,10 +12,10 @@ #include "adios2/helper/adiosFunctions.h" -#ifdef ADIOS2_HAVE_BLOSC +#ifdef ADIOS2_HAVE_BLOSC2 #include "adios2/operator/compress/CompressBlosc.h" extern "C" { -#include +#include } #endif @@ -43,7 +43,7 @@ void BPBackCompatBlosc::GetData( const char *input, const helper::BlockOperationInfo &blockOperationInfo, char *dataOutput) const { -#ifdef ADIOS2_HAVE_BLOSC +#ifdef ADIOS2_HAVE_BLOSC2 core::compress::CompressBlosc op((Params())); const size_t sizeOut = (sizeof(size_t) == 8) ? static_cast(helper::StringTo( @@ -65,7 +65,7 @@ void BPBackCompatBlosc::GetData( #endif } -#ifdef ADIOS2_HAVE_BLOSC +#ifdef ADIOS2_HAVE_BLOSC2 size_t BPBackCompatBlosc::Decompress(const void *bufferIn, const size_t sizeIn, void *dataOut, const size_t sizeOut, @@ -115,7 +115,7 @@ size_t BPBackCompatBlosc::DecompressChunkedFormat(const void *bufferIn, if (isCompressed) { - blosc_init(); + blosc2_init(); uint8_t *outputBuff = reinterpret_cast(dataOut); while (inputOffset < inputDataSize) @@ -143,12 +143,12 @@ size_t BPBackCompatBlosc::DecompressChunkedFormat(const void *bufferIn, size_t outputChunkSize = std::min(uncompressedSize - currentOutputSize, - static_cast(BLOSC_MAX_BUFFERSIZE)); + static_cast(BLOSC2_MAX_BUFFERSIZE)); bloscSize_t max_output_size = static_cast(outputChunkSize); bloscSize_t decompressdSize = - blosc_decompress(in_ptr, out_ptr, max_output_size); + blosc1_decompress(in_ptr, out_ptr, max_output_size); if (decompressdSize > 0) currentOutputSize += static_cast(decompressdSize); @@ -160,7 +160,7 @@ size_t BPBackCompatBlosc::DecompressChunkedFormat(const void *bufferIn, } inputOffset += static_cast(max_inputDataSize); } - blosc_destroy(); + blosc2_destroy(); } else { @@ -181,13 +181,13 @@ size_t BPBackCompatBlosc::DecompressOldFormat(const void *bufferIn, const size_t sizeOut, Params &info) const { - blosc_init(); - const int decompressedSize = blosc_decompress(bufferIn, dataOut, sizeOut); - blosc_destroy(); + blosc2_init(); + const int decompressedSize = blosc1_decompress(bufferIn, dataOut, sizeOut); + blosc2_destroy(); return static_cast(decompressedSize); } -#endif // ADIOS2_HAVE_BLOSC +#endif // ADIOS2_HAVE_BLOSC2 } // end namespace format } // end namespace adios2 diff --git a/testing/adios2/engine/bp/operations/CMakeLists.txt b/testing/adios2/engine/bp/operations/CMakeLists.txt index b0efbaa0b1..5fac82a205 100644 --- a/testing/adios2/engine/bp/operations/CMakeLists.txt +++ b/testing/adios2/engine/bp/operations/CMakeLists.txt @@ -60,10 +60,6 @@ if(ADIOS2_HAVE_PNG) bp_gtest_add_tests_helper(WriteReadPNG MPI_ALLOW) endif() -if(ADIOS2_HAVE_Blosc) - bp_gtest_add_tests_helper(WriteReadBlosc MPI_ALLOW) -endif() - if(ADIOS2_HAVE_Blosc2) bp_gtest_add_tests_helper(WriteReadBlosc2 MPI_ALLOW) endif() diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc2.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc2.cpp index 83871d7f58..5648156c62 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc2.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc2.cpp @@ -72,16 +72,16 @@ void Blosc2Accuracy1D(const std::string accuracy, const std::string threshold, // add operations adios2::Operator Blosc2Op = adios.DefineOperator( - "Blosc2Compressor", adios2::ops::LosslessBlosc2); + "Blosc2Compressor", adios2::ops::LosslessBlosc); - var_r32.AddOperation( - Blosc2Op, {{adios2::ops::blosc2::key::clevel, accuracy}, - {adios2::ops::blosc2::key::threshold, threshold}, - {adios2::ops::blosc2::key::doshuffle, doshuffle}}); - var_r64.AddOperation( - Blosc2Op, {{adios2::ops::blosc2::key::clevel, accuracy}, - {adios2::ops::blosc2::key::threshold, threshold}, - {adios2::ops::blosc2::key::doshuffle, doshuffle}}); + var_r32.AddOperation(Blosc2Op, + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); + var_r64.AddOperation(Blosc2Op, + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); @@ -214,16 +214,16 @@ void Blosc2Accuracy2D(const std::string accuracy, const std::string threshold, // add operations adios2::Operator Blosc2Op = adios.DefineOperator( - "Blosc2Compressor", adios2::ops::LosslessBlosc2); + "Blosc2Compressor", adios2::ops::LosslessBlosc); - var_r32.AddOperation( - Blosc2Op, {{adios2::ops::blosc2::key::clevel, accuracy}, - {adios2::ops::blosc2::key::threshold, threshold}, - {adios2::ops::blosc2::key::doshuffle, doshuffle}}); - var_r64.AddOperation( - Blosc2Op, {{adios2::ops::blosc2::key::clevel, accuracy}, - {adios2::ops::blosc2::key::threshold, threshold}, - {adios2::ops::blosc2::key::doshuffle, doshuffle}}); + var_r32.AddOperation(Blosc2Op, + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); + var_r64.AddOperation(Blosc2Op, + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); @@ -359,16 +359,16 @@ void Blosc2Accuracy3D(const std::string accuracy, const std::string threshold, // add operations adios2::Operator Blosc2Op = adios.DefineOperator( - "Blosc2Compressor", adios2::ops::LosslessBlosc2); + "Blosc2Compressor", adios2::ops::LosslessBlosc); - var_r32.AddOperation( - Blosc2Op, {{adios2::ops::blosc2::key::clevel, accuracy}, - {adios2::ops::blosc2::key::threshold, threshold}, - {adios2::ops::blosc2::key::doshuffle, doshuffle}}); - var_r64.AddOperation( - Blosc2Op, {{adios2::ops::blosc2::key::clevel, accuracy}, - {adios2::ops::blosc2::key::threshold, threshold}, - {adios2::ops::blosc2::key::doshuffle, doshuffle}}); + var_r32.AddOperation(Blosc2Op, + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); + var_r64.AddOperation(Blosc2Op, + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); @@ -505,16 +505,16 @@ void Blosc2Accuracy1DSel(const std::string accuracy, // add operations adios2::Operator Blosc2Op = adios.DefineOperator( - "Blosc2Compressor", adios2::ops::LosslessBlosc2); + "Blosc2Compressor", adios2::ops::LosslessBlosc); - var_r32.AddOperation( - Blosc2Op, {{adios2::ops::blosc2::key::clevel, accuracy}, - {adios2::ops::blosc2::key::threshold, threshold}, - {adios2::ops::blosc2::key::doshuffle, doshuffle}}); - var_r64.AddOperation( - Blosc2Op, {{adios2::ops::blosc2::key::clevel, accuracy}, - {adios2::ops::blosc2::key::threshold, threshold}, - {adios2::ops::blosc2::key::doshuffle, doshuffle}}); + var_r32.AddOperation(Blosc2Op, + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); + var_r64.AddOperation(Blosc2Op, + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); @@ -648,16 +648,16 @@ void Blosc2Accuracy2DSel(const std::string accuracy, // add operations adios2::Operator Blosc2Op = adios.DefineOperator( - "Blosc2Compressor", adios2::ops::LosslessBlosc2); + "Blosc2Compressor", adios2::ops::LosslessBlosc); - var_r32.AddOperation( - Blosc2Op, {{adios2::ops::blosc2::key::clevel, accuracy}, - {adios2::ops::blosc2::key::threshold, threshold}, - {adios2::ops::blosc2::key::doshuffle, doshuffle}}); - var_r64.AddOperation( - Blosc2Op, {{adios2::ops::blosc2::key::clevel, accuracy}, - {adios2::ops::blosc2::key::threshold, threshold}, - {adios2::ops::blosc2::key::doshuffle, doshuffle}}); + var_r32.AddOperation(Blosc2Op, + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); + var_r64.AddOperation(Blosc2Op, + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); @@ -794,16 +794,16 @@ void Blosc2Accuracy3DSel(const std::string accuracy, // add operations adios2::Operator Blosc2Op = adios.DefineOperator( - "Blosc2Compressor", adios2::ops::LosslessBlosc2); + "Blosc2Compressor", adios2::ops::LosslessBlosc); - var_r32.AddOperation( - Blosc2Op, {{adios2::ops::blosc2::key::clevel, accuracy}, - {adios2::ops::blosc2::key::doshuffle, doshuffle}, - {adios2::ops::blosc2::key::threshold, threshold}}); - var_r64.AddOperation( - Blosc2Op, {{adios2::ops::blosc2::key::clevel, accuracy}, - {adios2::ops::blosc2::key::threshold, threshold}, - {adios2::ops::blosc2::key::doshuffle, doshuffle}}); + var_r32.AddOperation(Blosc2Op, + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::doshuffle, doshuffle}, + {adios2::ops::blosc::key::threshold, threshold}}); + var_r64.AddOperation(Blosc2Op, + {{adios2::ops::blosc::key::clevel, accuracy}, + {adios2::ops::blosc::key::threshold, threshold}, + {adios2::ops::blosc::key::doshuffle, doshuffle}}); adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); @@ -929,22 +929,22 @@ INSTANTIATE_TEST_SUITE_P( Blosc2Accuracy, BPWriteReadBlosc2, ::testing::Combine( // test compression level - ::testing::Values(adios2::ops::blosc2::value::clevel_1, - adios2::ops::blosc2::value::clevel_2, - adios2::ops::blosc2::value::clevel_3, - adios2::ops::blosc2::value::clevel_4, - adios2::ops::blosc2::value::clevel_5, - adios2::ops::blosc2::value::clevel_6, - adios2::ops::blosc2::value::clevel_7, - adios2::ops::blosc2::value::clevel_8, - adios2::ops::blosc2::value::clevel_9), + ::testing::Values(adios2::ops::blosc::value::clevel_1, + adios2::ops::blosc::value::clevel_2, + adios2::ops::blosc::value::clevel_3, + adios2::ops::blosc::value::clevel_4, + adios2::ops::blosc::value::clevel_5, + adios2::ops::blosc::value::clevel_6, + adios2::ops::blosc::value::clevel_7, + adios2::ops::blosc::value::clevel_8, + adios2::ops::blosc::value::clevel_9), // test threshold: 128 is equal to the default, 1 GiB is used to disable // compression ::testing::Values("128", "1073741824"), // test shuffling - ::testing::Values(adios2::ops::blosc2::value::doshuffle_shuffle, - adios2::ops::blosc2::value::doshuffle_noshuffle, - adios2::ops::blosc2::value::doshuffle_bitshuffle))); + ::testing::Values(adios2::ops::blosc::value::doshuffle_shuffle, + adios2::ops::blosc::value::doshuffle_noshuffle, + adios2::ops::blosc::value::doshuffle_bitshuffle))); int main(int argc, char **argv) {