diff --git a/CMakeLists.txt b/CMakeLists.txt index f33cb0f5..a99e3551 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.15) cmake_policy(SET CMP0091 NEW) # enable new "MSVC runtime library selection" (https://cmake.org/cmake/help/latest/variable/CMAKE_MSVC_RUNTIME_LIBRARY.html) project(libCZI - VERSION 0.56.0 + VERSION 0.57.0 HOMEPAGE_URL "https://github.com/ZEISS/libczi" DESCRIPTION "libCZI is an Open Source Cross-Platform C++ library to read and write CZI") diff --git a/Src/CZICmd/CMakeLists.txt b/Src/CZICmd/CMakeLists.txt index 5dcefdda..3d6f062d 100644 --- a/Src/CZICmd/CMakeLists.txt +++ b/Src/CZICmd/CMakeLists.txt @@ -58,7 +58,7 @@ endif() FetchContent_Declare( cli11 GIT_REPOSITORY https://github.com/CLIUtils/CLI11 - GIT_TAG v2.2.0 + GIT_TAG v2.3.2 ) if (NOT cli11_POPULATED) @@ -94,7 +94,11 @@ set (CZICMDSRCFILES inc_rapidjson.h BitmapGenNull.h CZIcmd.cpp - platform_defines.h) + platform_defines.h + executePlaneScan.h + executePlaneScan.cpp + executeBase.h + executeBase.cpp) add_executable(CZIcmd ${CZICMDSRCFILES}) diff --git a/Src/CZICmd/cmdlineoptions.cpp b/Src/CZICmd/cmdlineoptions.cpp index e0a350ff..7e0b33c6 100644 --- a/Src/CZICmd/cmdlineoptions.cpp +++ b/Src/CZICmd/cmdlineoptions.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #if defined(LINUXENV) #include #endif @@ -428,6 +429,47 @@ struct GeneratorPixelTypeValidator : public CLI::Validator } }; +/// CLI11-validator for the option "--cachesize". +struct CachesizeValidator : public CLI::Validator +{ + CachesizeValidator() + { + this->name_ = "CachesizeValidator"; + this->func_ = [](const std::string& str) -> string + { + const bool parsed_ok = CCmdLineOptions::TryParseSubBlockCacheSize(str, nullptr); + if (!parsed_ok) + { + ostringstream string_stream; + string_stream << "Invalid subblock-cache-size given \"" << str << "\""; + throw CLI::ValidationError(string_stream.str()); + } + + return {}; + }; + } +}; + +struct TileSizeForPlaneScanValidator : public CLI::Validator +{ + TileSizeForPlaneScanValidator() + { + this->name_ = "TileSizeForPlaneScanValidator"; + this->func_ = [](const std::string& str) -> string + { + const bool parsed_ok = CCmdLineOptions::TryParseCreateSize(str, nullptr); + if (!parsed_ok) + { + ostringstream string_stream; + string_stream << "Invalid tile-size-plane-scan given \"" << str << "\""; + throw CLI::ValidationError(string_stream.str()); + } + + return {}; + }; + } +}; + /// A custom formatter for CLI11 - used to have nicely formatted descriptions. class CustomFormatter : public CLI::Formatter { @@ -497,6 +539,7 @@ CCmdLineOptions::ParseResult CCmdLineOptions::Parse(int argc, char** argv) { "ScalingChannelComposite", Command::ScalingChannelComposite }, { "ExtractAttachment", Command::ExtractAttachment}, { "CreateCZI", Command::CreateCZI }, + { "PlaneScan", Command::PlaneScan }, }; const static PlaneCoordinateValidator plane_coordinate_validator; @@ -518,6 +561,8 @@ CCmdLineOptions::ParseResult CCmdLineOptions::Parse(int argc, char** argv) const static CreateSubblockMetadataValidator createsubblockmetadata_validator; const static CompressionOptionsValidator compressionoptions_validator; const static GeneratorPixelTypeValidator generatorpixeltype_validator; + const static CachesizeValidator cachesize_validator; + const static TileSizeForPlaneScanValidator tile_size_for_plane_scan_validator; Command argument_command; string argument_source_filename; @@ -546,9 +591,12 @@ CCmdLineOptions::ParseResult CCmdLineOptions::Parse(int argc, char** argv) string argument_createczisubblockmetadata; string argument_compressionoptions; string argument_generatorpixeltype; + string argument_subblock_cachesize; + string argument_tilesize_for_scan; bool argument_versionflag = false; string argument_source_stream_class; string argument_source_stream_creation_propbag; + bool argument_use_visibility_check_optimization = false; // editorconfig-checker-disable cli_app.add_option("-c,--command", argument_command, @@ -569,7 +617,12 @@ CCmdLineOptions::ParseResult CCmdLineOptions::Parse(int argc, char** argv) \N'ScalingChannelComposite' operates like the previous command, but in addition gets all channels and creates a multi-channel-composite from them using display-settings. \N'ExtractAttachment' allows to extract (and save to a file) the contents of attachments.) - \N'CreateCZI' is used to demonstrate the CZI-creation capabilities of libCZI.)") + \N'CreateCZI' is used to demonstrate the CZI-creation capabilities of libCZI.) + \N'PlaneScan' does the following: over a ROI given with the --rect option a rectangle of size given with + the --tilesize-for-plane-scan option is moved, and the image content of this rectangle is written out to + files. The operation takes place on a plane which is given with the --plane-coordinate option. The filenames of the + tile-bitmaps are generated from the filename given with the --output option, where a string _X[x-position]_Y[y-position]_W[width]_H[height] + is added.)") ->default_val(Command::Invalid) ->option_text("COMMAND") ->transform(CLI::CheckedTransformer(map_string_to_command, CLI::ignore_case)); @@ -708,6 +761,18 @@ CCmdLineOptions::ParseResult CCmdLineOptions::Parse(int argc, char** argv) "'Bgr24' or 'Bgr48'. Default is 'Bgr24'.") ->option_text("PIXELTYPE") ->check(generatorpixeltype_validator); + cli_app.add_option("--cachesize", argument_subblock_cachesize, + "Only used for 'PlaneScan' - specify the size of the subblock-cache in bytes. The argument is to " + "be given with a suffix k, M, G, ...") + ->option_text("CACHESIZE") + ->check(cachesize_validator); + cli_app.add_option("--tilesize-for-plane-scan", argument_tilesize_for_scan, + "Only used for 'PlaneScan' - specify the size of ROI which is used for scanning the plane in " + "units of pixels. Format is e.g. '1600x1200' and default is 512x512.") + ->option_text("TILESIZE") + ->check(tile_size_for_plane_scan_validator); + cli_app.add_flag("--use-visibility-check-optimization", argument_use_visibility_check_optimization, + "Whether to enable the experimental \"visibility check optimization\" for the accessors."); cli_app.add_flag("--version", argument_versionflag, "Print extended version-info and supported operations, then exit."); @@ -742,6 +807,7 @@ CCmdLineOptions::ParseResult CCmdLineOptions::Parse(int argc, char** argv) this->calcHashOfResult = argument_calc_hash; this->drawTileBoundaries = argument_drawtileboundaries; this->command = argument_command; + this->useVisibilityCheckOptimization = argument_use_visibility_check_optimization; try { @@ -901,6 +967,18 @@ CCmdLineOptions::ParseResult CCmdLineOptions::Parse(int argc, char** argv) const bool b = TryParseGeneratorPixeltype(argument_generatorpixeltype, &this->pixelTypeForBitmapGenerator); ThrowIfFalse(b, "--generatorpixeltype", argument_generatorpixeltype); } + + if (!argument_subblock_cachesize.empty()) + { + const bool b = TryParseSubBlockCacheSize(argument_subblock_cachesize, &this->subBlockCacheSize); + ThrowIfFalse(b, "--cachesize", argument_subblock_cachesize); + } + + if (!argument_tilesize_for_scan.empty()) + { + const bool b = TryParseCreateSize(argument_tilesize_for_scan, &this->tilesSizeForPlaneScan); + ThrowIfFalse(b, "--tilesize-for-plane-scan", argument_tilesize_for_scan); + } } catch (runtime_error& exception) { @@ -1027,7 +1105,7 @@ void CCmdLineOptions::Clear() this->sbBlkMetadataKeyValue.clear(); this->rectX = this->rectY = 0; this->rectW = this->rectH = -1;; - this->zoom = -1; + this->zoom = 1; this->pyramidLayerNo = -1; this->pyramidMinificationFactor = -1; this->createTileInfo.rows = this->createTileInfo.columns = 1; @@ -1035,6 +1113,9 @@ void CCmdLineOptions::Clear() this->compressionMode = libCZI::CompressionMode::Invalid; this->compressionParameters = nullptr; this->pixelTypeForBitmapGenerator = libCZI::PixelType::Bgr24; + this->subBlockCacheSize = 0; + this->tilesSizeForPlaneScan = make_tuple(512, 512); + this->useVisibilityCheckOptimization = false; } bool CCmdLineOptions::IsLogLevelEnabled(int level) const @@ -1458,13 +1539,15 @@ bool CCmdLineOptions::TryParseDisplaySettings(const std::string& s, std::map from it. - static constexpr struct + static constexpr struct { const char* name; int stream_property_id; @@ -2278,3 +2361,88 @@ void CCmdLineOptions::PrintHelpStreamsObjects() return true; } + +/*static*/bool CCmdLineOptions::TryParseSubBlockCacheSize(const std::string& text, std::uint64_t* size) +{ + // This regular expression is used to match strings that represent sizes in bytes, kilobytes, megabytes, gigabytes, terabytes, kibibytes, mebibytes, gibibytes, and tebibytes. + // + // Here is a breakdown of the regular expression: + // + // - ^\s*: Matches the start of the string, followed by any amount of whitespace. + // - ([+]?(?:[0-9]+(?:[.][0-9]*)?|[.][0-9]+)): Matches a positive number, which can be an integer or a decimal. The number may optionally be preceded by a plus sign. + // - \s*: Matches any amount of whitespace following the number. + // - (k|m|g|t|ki|mi|gi|ti): Matches one of the following units of size: k (kilobytes), m (megabytes), g (gigabytes), t (terabytes), ki (kibibytes), mi (mebibytes), gi (gibibytes), ti (tebibytes). + // - (?:b?): Optionally matches a 'b', which can be used to explicitly specify that the size is in bytes. + // - \s*$: Matches any amount of whitespace at the end of the string, followed by the end of the string. + // + // This regular expression is case-insensitive. + regex regex(R"(^\s*([+]?(?:[0-9]+(?:[.][0-9]*)?|[.][0-9]+))\s*(k|m|g|t|ki|mi|gi|ti)(?:b?)\s*$)", regex_constants::icase); + smatch match; + regex_search(text, match, regex); + if (match.size() != 3) + { + return false; + } + + double number; + + try + { + number = stod(match[1].str()); + } + catch (invalid_argument&) + { + return false; + } + catch (out_of_range&) + { + return false; + } + + uint64_t factor; + string suffix_string = match[2].str(); + if (icasecmp(suffix_string, "k")) + { + factor = 1000; + } + else if (icasecmp(suffix_string, "ki")) + { + factor = 1024; + } + else if (icasecmp(suffix_string, "m")) + { + factor = 1000 * 1000; + } + else if (icasecmp(suffix_string, "mi")) + { + factor = 1024 * 1024; + } + else if (icasecmp(suffix_string, "g")) + { + factor = 1000 * 1000 * 1000; + } + else if (icasecmp(suffix_string, "gi")) + { + factor = 1024 * 1024 * 1024; + } + else if (icasecmp(suffix_string, "t")) + { + factor = 1000ULL * 1000 * 1000 * 1000; + } + else if (icasecmp(suffix_string, "ti")) + { + factor = 1024ULL * 1024 * 1024 * 1024; + } + else + { + return false; + } + + const uint64_t memory_size = llround(number * static_cast(factor)); + if (size != nullptr) + { + *size = memory_size; + } + + return true; +} diff --git a/Src/CZICmd/cmdlineoptions.h b/Src/CZICmd/cmdlineoptions.h index 099b0e4d..d302827e 100644 --- a/Src/CZICmd/cmdlineoptions.h +++ b/Src/CZICmd/cmdlineoptions.h @@ -32,7 +32,9 @@ enum class Command CreateCZI, - ReadWriteCZI + ReadWriteCZI, + + PlaneScan, }; enum class InfoLevel : std::uint32_t @@ -190,6 +192,11 @@ class CCmdLineOptions libCZI::CompressionMode compressionMode; std::shared_ptr compressionParameters; libCZI::PixelType pixelTypeForBitmapGenerator; + + std::uint64_t subBlockCacheSize; ///< The size of the sub-block cache in bytes. + std::tuple tilesSizeForPlaneScan; ///< The size of the tiles in pixels for the plane scan operation. + + bool useVisibilityCheckOptimization; public: /// Values that represent the result of the "Parse"-operation. enum class ParseResult @@ -257,6 +264,9 @@ class CCmdLineOptions libCZI::CompressionMode GetCompressionMode() const { return this->compressionMode; } std::shared_ptr GetCompressionParameters() const { return this->compressionParameters; } libCZI::PixelType GetPixelGeneratorPixeltype() const { return this->pixelTypeForBitmapGenerator; } + std::uint64_t GetSubBlockCacheSize() const { return this->subBlockCacheSize; } + const std::tuple& GetTileSizeForPlaneScan() const { return this->tilesSizeForPlaneScan; } + bool GetUseVisibilityCheckOptimization() const { return this->useVisibilityCheckOptimization; } private: friend struct RegionOfInterestValidator; friend struct DisplaySettingsValidator; @@ -276,6 +286,8 @@ class CCmdLineOptions friend struct CreateSubblockMetadataValidator; friend struct CompressionOptionsValidator; friend struct GeneratorPixelTypeValidator; + friend struct CachesizeValidator; + friend struct TileSizeForPlaneScanValidator; bool CheckArgumentConsistency() const; void SetOutputFilename(const std::wstring& s); @@ -307,6 +319,7 @@ class CCmdLineOptions static bool TryParseCompressionOptions(const std::string& s, libCZI::Utils::CompressionOption* compression_option); static bool TryParseGeneratorPixeltype(const std::string& s, libCZI::PixelType* pixel_type); static bool TryParseInputStreamCreationPropertyBag(const std::string& s, std::map* property_bag); + static bool TryParseSubBlockCacheSize(const std::string& text, std::uint64_t* size); static void ThrowIfFalse(bool b, const std::string& argument_switch, const std::string& argument); }; diff --git a/Src/CZICmd/execute.cpp b/Src/CZICmd/execute.cpp index 7c45d27d..4aadd4c7 100644 --- a/Src/CZICmd/execute.cpp +++ b/Src/CZICmd/execute.cpp @@ -3,14 +3,16 @@ // SPDX-License-Identifier: LGPL-3.0-or-later #include "stdafx.h" -#include #include "execute.h" +#include "executeBase.h" #include "executeCreateCzi.h" +#include "executePlaneScan.h" #include "inc_libCZI.h" #include "SaveBitmap.h" #include "utils.h" #include "DisplaySettingsHelper.h" #include "inc_rapidjson.h" +#include #include #include #include @@ -19,110 +21,6 @@ using namespace libCZI; using namespace std; using namespace rapidjson; -class CExecuteBase -{ -protected: - static std::shared_ptr CreateAndOpenCziReader(const CCmdLineOptions& options) - { - shared_ptr stream; - if (options.GetInputStreamClassName().empty()) - { - stream = CExecuteBase::CreateStandardFileBasedStreamObject(options.GetCZIFilename().c_str()); - } - else - { - stream = CExecuteBase::CreateInputStreamObject( - options.GetCZIFilename().c_str(), - options.GetInputStreamClassName(), - &options.GetInputStreamPropertyBag()); - } - - auto spReader = libCZI::CreateCZIReader(); - spReader->Open(stream); - return spReader; - } - - static std::shared_ptr CreateStandardFileBasedStreamObject(const wchar_t* fileName) - { - auto stream = libCZI::CreateStreamFromFile(fileName); - return stream; - } - - static std::shared_ptr CreateInputStreamObject(const wchar_t* uri, const string& class_name, const std::map* property_bag) - { - libCZI::StreamsFactory::Initialize(); - libCZI::StreamsFactory::CreateStreamInfo stream_info; - stream_info.class_name = class_name; - if (property_bag != nullptr) - { - stream_info.property_bag = *property_bag; - } - - auto stream = libCZI::StreamsFactory::CreateStream(stream_info, uri); - if (!stream) - { - stringstream string_stream; - string_stream << "Failed to create stream object of the class \"" << class_name << "\"."; - throw std::runtime_error(string_stream.str()); - } - - return stream; - } - - static IntRect GetRoiFromOptions(const CCmdLineOptions& options, const SubBlockStatistics& subBlockStatistics) - { - IntRect roi{ options.GetRectX(), options.GetRectY(), options.GetRectW(), options.GetRectH() }; - if (options.GetIsRelativeRectCoordinate()) - { - roi.x += subBlockStatistics.boundingBox.x; - roi.y += subBlockStatistics.boundingBox.y; - } - - return roi; - } - - static libCZI::RgbFloatColor GetBackgroundColorFromOptions(const CCmdLineOptions& options) - { - return options.GetBackGroundColor(); - } - - static void DoCalcHashOfResult(shared_ptr bm, const CCmdLineOptions& options) - { - DoCalcHashOfResult(bm.get(), options); - } - - static void HandleHashOfResult(const std::function& f, const CCmdLineOptions& options) - { - if (!options.GetCalcHashOfResult()) - { - return; - } - - uint8_t md5sumHash[16]; - if (!f(md5sumHash, sizeof(md5sumHash))) - { - return; - } - - string hashHex = BytesToHexString(md5sumHash, sizeof(md5sumHash)); - std::stringstream ss; - ss << "hash of result: " << hashHex; - auto log = options.GetLog(); - log->WriteLineStdOut(ss.str().c_str()); - } - - static void DoCalcHashOfResult(libCZI::IBitmapData* bm, const CCmdLineOptions& options) - { - HandleHashOfResult( - [&](uint8_t* ptrHash, size_t size)->bool - { - Utils::CalcMd5SumHash(bm, ptrHash, (int)size); - return true; - }, - options); - } -}; - class CExecutePrintInformation : CExecuteBase { public: @@ -655,6 +553,9 @@ class CExecuteSingleChannelTileAccessor : CExecuteBase libCZI::ISingleChannelTileAccessor::Options sctaOptions; sctaOptions.Clear(); sctaOptions.sortByM = true; sctaOptions.drawTileBorder = options.GetDrawTileBoundaries(); + sctaOptions.backGroundColor = GetBackgroundColorFromOptions(options); + sctaOptions.sceneFilter = options.GetSceneIndexSet(); + sctaOptions.useVisibilityCheckOptimization = options.GetUseVisibilityCheckOptimization(); IntRect roi{ options.GetRectX() ,options.GetRectY(),options.GetRectW(),options.GetRectH() }; if (options.GetIsRelativeRectCoordinate()) @@ -850,6 +751,7 @@ class CExecuteSingleChannelScalingTileAccessor : public CExecuteBase libCZI::ISingleChannelScalingTileAccessor::Options scstaOptions; scstaOptions.Clear(); scstaOptions.backGroundColor = GetBackgroundColorFromOptions(options); scstaOptions.sceneFilter = options.GetSceneIndexSet(); + scstaOptions.useVisibilityCheckOptimization = options.GetUseVisibilityCheckOptimization(); auto re = accessor->Get(roi, &coordinate, options.GetZoom(), &scstaOptions); @@ -956,6 +858,7 @@ class CExecuteScalingChannelComposite : CExecuteBase sctaOptions.backGroundColor = GetBackgroundColorFromOptions(options); sctaOptions.drawTileBorder = options.GetDrawTileBoundaries(); sctaOptions.sceneFilter = options.GetSceneIndexSet(); + sctaOptions.useVisibilityCheckOptimization = options.GetUseVisibilityCheckOptimization(); IntRect roi{ options.GetRectX() ,options.GetRectY() ,options.GetRectW(),options.GetRectH() }; if (options.GetIsRelativeRectCoordinate()) { @@ -1229,6 +1132,9 @@ bool execute(const CCmdLineOptions& options) case Command::CreateCZI: success = executeCreateCzi(options); break; + case Command::PlaneScan: + success = executePlaneScan(options); + break; default: break; } diff --git a/Src/CZICmd/executeBase.cpp b/Src/CZICmd/executeBase.cpp new file mode 100644 index 00000000..216ce2c7 --- /dev/null +++ b/Src/CZICmd/executeBase.cpp @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: 2023 Carl Zeiss Microscopy GmbH +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "executeBase.h" + +#include "cmdlineoptions.h" + +using namespace std; +using namespace libCZI; + +std::shared_ptr CExecuteBase::CreateAndOpenCziReader(const CCmdLineOptions& options) +{ + shared_ptr stream; + if (options.GetInputStreamClassName().empty()) + { + stream = CExecuteBase::CreateStandardFileBasedStreamObject(options.GetCZIFilename().c_str()); + } + else + { + stream = CExecuteBase::CreateInputStreamObject( + options.GetCZIFilename().c_str(), + options.GetInputStreamClassName(), + &options.GetInputStreamPropertyBag()); + } + + auto spReader = libCZI::CreateCZIReader(); + spReader->Open(stream); + return spReader; +} + +std::shared_ptr CExecuteBase::CreateStandardFileBasedStreamObject(const wchar_t* fileName) +{ + auto stream = libCZI::CreateStreamFromFile(fileName); + return stream; +} + +std::shared_ptr CExecuteBase::CreateInputStreamObject(const wchar_t* uri, const string& class_name, const std::map* property_bag) +{ + libCZI::StreamsFactory::Initialize(); + libCZI::StreamsFactory::CreateStreamInfo stream_info; + stream_info.class_name = class_name; + if (property_bag != nullptr) + { + stream_info.property_bag = *property_bag; + } + + auto stream = libCZI::StreamsFactory::CreateStream(stream_info, uri); + if (!stream) + { + stringstream string_stream; + string_stream << "Failed to create stream object of the class \"" << class_name << "\"."; + throw std::runtime_error(string_stream.str()); + } + + return stream; +} + +IntRect CExecuteBase::GetRoiFromOptions(const CCmdLineOptions& options, const SubBlockStatistics& subBlockStatistics) +{ + IntRect roi{ options.GetRectX(), options.GetRectY(), options.GetRectW(), options.GetRectH() }; + if (options.GetIsRelativeRectCoordinate()) + { + roi.x += subBlockStatistics.boundingBox.x; + roi.y += subBlockStatistics.boundingBox.y; + } + + return roi; +} + +libCZI::RgbFloatColor CExecuteBase::GetBackgroundColorFromOptions(const CCmdLineOptions& options) +{ + return options.GetBackGroundColor(); +} + +void CExecuteBase::DoCalcHashOfResult(shared_ptr bm, const CCmdLineOptions& options) +{ + DoCalcHashOfResult(bm.get(), options); +} + +void CExecuteBase::HandleHashOfResult(const std::function& f, const CCmdLineOptions& options) +{ + if (!options.GetCalcHashOfResult()) + { + return; + } + + uint8_t md5sumHash[16]; + if (!f(md5sumHash, sizeof(md5sumHash))) + { + return; + } + + string hashHex = BytesToHexString(md5sumHash, sizeof(md5sumHash)); + std::stringstream ss; + ss << "hash of result: " << hashHex; + auto log = options.GetLog(); + log->WriteLineStdOut(ss.str().c_str()); +} + +void CExecuteBase::DoCalcHashOfResult(libCZI::IBitmapData* bm, const CCmdLineOptions& options) +{ + HandleHashOfResult( + [&](uint8_t* ptrHash, size_t size)->bool + { + Utils::CalcMd5SumHash(bm, ptrHash, (int)size); + return true; + }, + options); +} diff --git a/Src/CZICmd/executeBase.h b/Src/CZICmd/executeBase.h new file mode 100644 index 00000000..f97ad1b6 --- /dev/null +++ b/Src/CZICmd/executeBase.h @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2023 Carl Zeiss Microscopy GmbH +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include "executeBase.h" +#include "inc_libCZI.h" + +class CCmdLineOptions; + +class CExecuteBase +{ +protected: + static std::shared_ptr CreateAndOpenCziReader(const CCmdLineOptions& options); + static std::shared_ptr CreateStandardFileBasedStreamObject(const wchar_t* fileName); + static std::shared_ptr CreateInputStreamObject(const wchar_t* uri, const std::string& class_name, const std::map* property_bag); + static libCZI::IntRect GetRoiFromOptions(const CCmdLineOptions& options, const libCZI::SubBlockStatistics& subBlockStatistics); + static libCZI::RgbFloatColor GetBackgroundColorFromOptions(const CCmdLineOptions& options); + static void DoCalcHashOfResult(std::shared_ptr bm, const CCmdLineOptions& options); + static void HandleHashOfResult(const std::function& f, const CCmdLineOptions& options); + static void DoCalcHashOfResult(libCZI::IBitmapData* bm, const CCmdLineOptions& options); +}; diff --git a/Src/CZICmd/executePlaneScan.cpp b/Src/CZICmd/executePlaneScan.cpp new file mode 100644 index 00000000..aa227b12 --- /dev/null +++ b/Src/CZICmd/executePlaneScan.cpp @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: 2023 Carl Zeiss Microscopy GmbH +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "stdafx.h" +#include "executePlaneScan.h" +#include "executeBase.h" +#include "SaveBitmap.h" + +using namespace std; +using namespace libCZI; + +class CExecutePlaneScan : public CExecuteBase +{ +protected: + struct CacheContext + { + shared_ptr cache; + ISubBlockCache::PruneOptions prune_options; + }; +public: + static bool execute(const CCmdLineOptions& options) + { + const auto reader = CExecuteBase::CreateAndOpenCziReader(options); + const auto accessor = reader->CreateSingleChannelScalingTileAccessor(); + + const auto roi = CExecuteBase::GetRoiFromOptions(options, reader->GetStatistics()); + const auto& coordinate = options.GetPlaneCoordinate(); + + CacheContext cache_context; + const uint64_t max_cache_size = options.GetSubBlockCacheSize(); + if (max_cache_size > 0) + { + cache_context.cache = CreateSubBlockCache(); + cache_context.prune_options.maxMemoryUsage = max_cache_size; + } + const auto tile_size_for_plane_scan = options.GetTileSizeForPlaneScan(); + const IntSize tileSize = { get<0>(tile_size_for_plane_scan), get<1>(tile_size_for_plane_scan) }; + const auto saver = CSaveBitmapFactory::CreateSaveBitmapObj(nullptr); + + for (int y = 0; y < (roi.h + static_cast(tileSize.h) - 1) / static_cast(tileSize.h); ++y) + { + for (int x = 0; x < (roi.w + static_cast(tileSize.w) - 1) / static_cast(tileSize.w); ++x) + { + IntRect tileRect = + { + roi.x + x * static_cast(tileSize.w), + roi.y + y * static_cast(tileSize.h), + min(static_cast(tileSize.w), roi.w - x * static_cast(tileSize.w)), + min(static_cast(tileSize.h), roi.h - y * static_cast(tileSize.h)) + }; + + CExecutePlaneScan::WriteRoi(accessor, coordinate, tileRect, cache_context, saver, options); + } + } + + return true; + } +protected: + static void WriteRoi( + const shared_ptr& accessor, + const CDimCoordinate& plane_coordinate, + const IntRect& roi, + const CacheContext& cache_context, + const shared_ptr& saver, + const CCmdLineOptions& options) + { + libCZI::ISingleChannelScalingTileAccessor::Options scstaOptions; + scstaOptions.Clear(); + scstaOptions.backGroundColor = GetBackgroundColorFromOptions(options); + scstaOptions.sceneFilter = options.GetSceneIndexSet(); + scstaOptions.subBlockCache = cache_context.cache; + scstaOptions.useVisibilityCheckOptimization = options.GetUseVisibilityCheckOptimization(); + + const auto bitmap = accessor->Get(roi, &plane_coordinate, options.GetZoom(), &scstaOptions); + + if (cache_context.cache) + { + cache_context.cache->Prune(cache_context.prune_options); + } + + const auto filename = GetFileName(options, roi); + saver->Save(filename.c_str(), SaveDataFormat::PNG, bitmap.get()); + } + + static wstring GetFileName(const CCmdLineOptions& options, const IntRect& roi) + { + wstringstream string_stream; + string_stream << "_X" << roi.x << "_Y" << roi.y << "_W" << roi.w << "_H" << roi.h; + wstring output_filename = options.MakeOutputFilename(string_stream.str().c_str(), L"PNG"); + return output_filename; + } +}; + +bool executePlaneScan(const CCmdLineOptions& options) +{ + return CExecutePlaneScan::execute(options); +} diff --git a/Src/CZICmd/executePlaneScan.h b/Src/CZICmd/executePlaneScan.h new file mode 100644 index 00000000..765cd98d --- /dev/null +++ b/Src/CZICmd/executePlaneScan.h @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: 2023 Carl Zeiss Microscopy GmbH +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#pragma once +#include "cmdlineoptions.h" + +bool executePlaneScan(const CCmdLineOptions& options); diff --git a/Src/libCZI/CMakeLists.txt b/Src/libCZI/CMakeLists.txt index a4ac3181..629e02fb 100644 --- a/Src/libCZI/CMakeLists.txt +++ b/Src/libCZI/CMakeLists.txt @@ -114,6 +114,8 @@ set(LIBCZISRCFILES StreamsLib/simplefileinputstream.h StreamsLib/preadfileinputstream.cpp StreamsLib/preadfileinputstream.h + subblock_cache.h + subblock_cache.cpp ) # prepare the configuration-file "libCZI_Config.h" diff --git a/Src/libCZI/Doc/CZICmd_usage.markdown b/Src/libCZI/Doc/CZICmd_usage.markdown index 22ad1413..a9770660 100644 --- a/Src/libCZI/Doc/CZICmd_usage.markdown +++ b/Src/libCZI/Doc/CZICmd_usage.markdown @@ -7,222 +7,248 @@ The console application "CZIcmd" is provided for two purposes: The synopsis of the program is: - Usage: CZIcmd.exe [OPTIONS] - - using libCZI version 0.54.0 - - Options: - -h,--help Print this help message and exit - - -c,--command COMMAND - COMMAND can be one of 'PrintInformation', 'ExtractSubBlock', - 'SingleChannelTileAccessor', 'ChannelComposite', - 'SingleChannelPyramidTileAccessor', - 'SingleChannelScalingTileAccessor', - 'ScalingChannelComposite', 'ExtractAttachment' and - 'CreateCZI'. - - 'PrintInformation' will print information about the CZI-file - to the console. The argument 'info-level' can be used to - specify which information is to be printed. - - 'ExtractSubBlock' will write the bitmap contained in the - specified sub-block to the OUTPUTFILE. - - 'ChannelComposite' will create a channel-composite of the - specified region and plane and apply display-settings to it. - The resulting bitmap will be written to the specified - OUTPUTFILE. - - 'SingleChannelTileAccessor' will create a tile-composite - (only from sub-blocks on pyramid-layer 0) of the specified - region and plane. The resulting bitmap will be written to - the specified OUTPUTFILE. - - 'SingleChannelPyramidTileAccessor' adds to the previous - command the ability to explictely address a specific - pyramid-layer (which must exist in the CZI-document). - - 'SingleChannelScalingTileAccessor' gets the specified region - with an arbitrary zoom factor. It uses the pyramid-layers in - the CZI-document and scales the bitmap if necessary. The - resulting bitmap will be written to the specified - OUTPUTFILE. - - 'ScalingChannelComposite' operates like the previous - command, but in addition gets all channels and creates a - multi-channel-composite from them using display-settings. - - 'ExtractAttachment' allows to extract (and save to a file) - the contents of attachments.) - - 'CreateCZI' is used to demonstrate the CZI-creation - capabilities of libCZI. - - -s,--source SOURCEFILE - Specifies the source CZI-file. - - --source-stream-class STREAMCLASS - Specifies the stream-class used for reading the source - CZI-file. If not specified, the default file-reader - stream-class is used. Run with argument '--version' to get a - list of available stream-classes. - - --propbag-source-stream-creation PROPBAG - Specifies the property-bag used for creating the stream used - for reading the source CZI-file. The data is given in - JSON-notation. - - -o,--output OUTPUTFILE - specifies the output-filename. A suffix will be appended to - the name given here depending on the type of the file. - - -p,--plane-coordinate PLANE-COORDINATE - Uniquely select a 2D-plane from the document. It is given in - the form [DimChar][number], where 'DimChar' specifies a - dimension and can be any of 'Z', 'C', 'T', 'R', 'I', 'H', - 'V' or 'B'. 'number' is an integer. - Examples: C1T3, C0T-2, C1T44Z15H1. - - -r,--rect ROI Select a paraxial rectangular region as the - region-of-interest. The coordinates may be given either - absolute or relative. If using relative coordinates, they - are relative to what is determined as the upper-left point - in the document.\nRelative coordinates are specified with - the syntax 'rel([x],[y],[width],[height])', absolute - coordinates are specified 'abs([x],[y],[width],[height])'. - Examples: rel(0, 0, 1024, 1024), rel(-100, -100, 500, 500), - abs(-230, 100, 800, 800). - - -d,--display-settings DISPLAYSETTINGS - Specifies the display-settings used for creating a - channel-composite. The data is given in JSON-notation. - - --calc-hash Calculate a hash of the output-picture. The MD5Sum-algorithm - is used for this. - - -t,--drawtileboundaries - Draw a one-pixel black line around each tile. - - -j,--jpgxrcodec DECODERNAME - Choose which decoder implementation is used. Specifying - "WIC" will request the Windows-provided decoder - which is - only available on Windows.By default the internal - JPG-XR-decoder is used. - - -v,--verbosity VERBOSITYLEVEL - Set the verbosity of this program. The argument is a comma- - or semicolon-separated list of the following strings : - 'All', 'Errors', 'Warnings', 'Infos', 'Errors1', - 'Warnings1', 'Infos1', 'Errors2', 'Warnings2', 'Infos2'. - - -b,--background BACKGROUND - Specify the background color. BACKGROUND is either a single - float or three floats, separated by a comma or semicolon. In - case of a single float, it gives a grayscale value, in case - of three floats it gives a RGB - value.The floats are given - normalized to a range from 0 to 1. - - -y,--pyramidinfo PYRAMIDINFO - For the command 'SingleChannelPyramidTileAccessor' the - argument PYRAMIDINFO specifies the pyramid layer. It - consists of two integers(separated by a comma, semicolon or - pipe-symbol), where the first specifies the - minification-factor (between pyramid-layers) and the second - the pyramid-layer (starting with 0 for the layer with the - highest resolution). - - -z,--zoom ZOOM The zoom-factor (which is used for the commands - 'SingleChannelScalingTileAccessor' and - 'ScalingChannelComposite'). It is a float between 0 and 1. - - -i,--info-level INFO-LEVEL - When using the command 'PrintInformation' the INFO-LEVEL can - be used to specify which information is printed. Possible - values are "Statistics", "RawXML", "DisplaySettings", - "DisplaySettingsJson", "AllSubBlocks", "Attachments", - "AllAttachments", "PyramidStatistics", "GeneralInfo", - "ScalingInfo" and "All". The values are given as a list - separated by comma or semicolon. - - -e,--selection SELECTION - For the command 'ExtractAttachment' this allows to specify a - subset which is to be extracted (and saved to a file). It is - possible to specify the name and the index - only - attachments for which the name/index is equal to those - values specified are processed. The arguments are given in - JSON-notation, e.g. {"name":"Thumbnail"} or {"index":3.0}. - - -f,--tile-filter FILTER - Specify to filter subblocks according to the scene-index. A - comma separated list of either an interval or a single - integer may be given here, e.g. "2,3" or "2-4,6" or - "0-3,5-8". - - -m,--channelcompositionformat CHANNELCOMPOSITIONFORMAT - In case of a channel-composition, specifies the pixeltype of - the output. Possible values are "bgr24" (the default) and - "bgra32". If specifying "bgra32" it is possible to give the - value of the alpha-pixels in the form "bgra32(128)" - for an - alpha-value of 128. - - --createbounds BOUNDS - Only used for 'CreateCZI': specify the range of coordinates - used to create a CZI. Format is e.g. 'T0:3Z0:3C0:2'. - - --createsubblocksize SIZE - Only used for 'CreateCZI': specify the size of the subblocks - created in pixels. Format is e.g. '1600x1200'. - - --createtileinfo TILEINFO - Only used for 'CreateCZI': specify the number of tiles on - each plane. Format is e.g. '3x3;10%' for a 3 by 3 tiles - arrangement with 10% overlap. - - --font NAME/FILENAME - Only used for 'CreateCZI': (on Linux) specify the filename - of a TrueType-font (.ttf) to be used for generating text in - the subblocks; (on Windows) name of the font. - - --fontheight HEIGHT - Only used for 'CreateCZI': specifies the height of the font - in pixels (default: 36). - - -g,--guidofczi CZI-File-GUID - Only used for 'CreateCZI': specify the GUID of the file - (which is useful for bit-exact reproducible results); the - GUID must be given in the form - "cfc4a2fe-f968-4ef8-b685-e73d1b77271a" or - "{cfc4a2fe-f968-4ef8-b685-e73d1b77271a}" - - --bitmapgenerator BITMAPGENERATORCLASSNAME - Only used for 'CreateCZI': specifies the bitmap-generator to - use. Possibly values are "gdi", "freetype", "null" or - "default". Run with argument '--version' to get a list of - available bitmap-generators. - - --createczisbblkmetadata KEY_VALUE_SUBBLOCKMETADATA - Only used for 'CreateCZI': a key-value list in JSON-notation - which will be written as subblock-metadata. For example: - {"StageXPosition":-8906.346,"StageYPosition":-648.51} - - --compressionopts COMPRESSIONDESCRIPTION - Only used for 'CreateCZI': a string in a defined format - which states the compression-method and (compression-method - specific) parameters.The format is "compression_method: - key=value; ...". It starts with the name of the - compression-method, followed by a colon, then followed by a - list of key-value pairs which are separated by a semicolon. - Examples: "zstd0:ExplicitLevel=3", - "zstd1:ExplicitLevel=2;PreProcess=HiLoByteUnpack". - - --generatorpixeltype PIXELTYPE - Only used for 'CreateCZI': a string defining the pixeltype - used by the bitmap - generator. Possible values are 'Gray8', - 'Gray16', 'Bgr24' or 'Bgr48'. Default is 'Bgr24'. - - --version Print extended version-info and supported operations, then - exit. +``` +Usage: CZIcmd.exe [OPTIONS] + + using libCZI version 0.57.0 + +Options: + -h,--help Print this help message and exit + + -c,--command COMMAND + COMMAND can be one of 'PrintInformation', 'ExtractSubBlock', + 'SingleChannelTileAccessor', 'ChannelComposite', + 'SingleChannelPyramidTileAccessor', + 'SingleChannelScalingTileAccessor', + 'ScalingChannelComposite', 'ExtractAttachment' and + 'CreateCZI'. + + 'PrintInformation' will print information about the CZI-file + to the console. The argument 'info-level' can be used to + specify which information is to be printed. + + 'ExtractSubBlock' will write the bitmap contained in the + specified sub-block to the OUTPUTFILE. + + 'ChannelComposite' will create a channel-composite of the + specified region and plane and apply display-settings to it. + The resulting bitmap will be written to the specified + OUTPUTFILE. + + 'SingleChannelTileAccessor' will create a tile-composite + (only from sub-blocks on pyramid-layer 0) of the specified + region and plane. The resulting bitmap will be written to + the specified OUTPUTFILE. + + 'SingleChannelPyramidTileAccessor' adds to the previous + command the ability to explicitly address a specific + pyramid-layer (which must exist in the CZI-document). + + 'SingleChannelScalingTileAccessor' gets the specified region + with an arbitrary zoom factor. It uses the pyramid-layers in + the CZI-document and scales the bitmap if necessary. The + resulting bitmap will be written to the specified + OUTPUTFILE. + + 'ScalingChannelComposite' operates like the previous + command, but in addition gets all channels and creates a + multi-channel-composite from them using display-settings. + + 'ExtractAttachment' allows to extract (and save to a file) + the contents of attachments.) + + 'CreateCZI' is used to demonstrate the CZI-creation + capabilities of libCZI.) + + 'PlaneScan' does the following: over a ROI given with the + --rect option a rectangle of size given with the + --tilesize-for-plane-scan option is moved, and the image + content of this rectangle is written out to files. The + operation takes place on a plane which is given with the + --plane-coordinate option. The filenames of the tile-bitmaps + are generated from the filename given with the --output + option, where a string + _X[x-position]_Y[y-position]_W[width]_H[height] is added. + + -s,--source SOURCEFILE + Specifies the source CZI-file. + + --source-stream-class STREAMCLASS + Specifies the stream-class used for reading the source + CZI-file. If not specified, the default file-reader + stream-class is used. Run with argument '--version' to get a + list of available stream-classes. + + --propbag-source-stream-creation PROPBAG + Specifies the property-bag used for creating the stream used + for reading the source CZI-file. The data is given in + JSON-notation. + + -o,--output OUTPUTFILE + specifies the output-filename. A suffix will be appended to + the name given here depending on the type of the file. + + -p,--plane-coordinate PLANE-COORDINATE + Uniquely select a 2D-plane from the document. It is given in + the form [DimChar][number], where 'DimChar' specifies a + dimension and can be any of 'Z', 'C', 'T', 'R', 'I', 'H', + 'V' or 'B'. 'number' is an integer. + Examples: C1T3, C0T-2, C1T44Z15H1. + + -r,--rect ROI Select a paraxial rectangular region as the + region-of-interest. The coordinates may be given either + absolute or relative. If using relative coordinates, they + are relative to what is determined as the upper-left point + in the document.\nRelative coordinates are specified with + the syntax 'rel([x],[y],[width],[height])', absolute + coordinates are specified 'abs([x],[y],[width],[height])'. + Examples: rel(0, 0, 1024, 1024), rel(-100, -100, 500, 500), + abs(-230, 100, 800, 800). + + -d,--display-settings DISPLAYSETTINGS + Specifies the display-settings used for creating a + channel-composite. The data is given in JSON-notation. + + --calc-hash Calculate a hash of the output-picture. The MD5Sum-algorithm + is used for this. + + -t,--drawtileboundaries + Draw a one-pixel black line around each tile. + + -j,--jpgxrcodec DECODERNAME + Choose which decoder implementation is used. Specifying + "WIC" will request the Windows-provided decoder - which is + only available on Windows.By default the internal + JPG-XR-decoder is used. + + -v,--verbosity VERBOSITYLEVEL + Set the verbosity of this program. The argument is a comma- + or semicolon-separated list of the following strings : + 'All', 'Errors', 'Warnings', 'Infos', 'Errors1', + 'Warnings1', 'Infos1', 'Errors2', 'Warnings2', 'Infos2'. + + -b,--background BACKGROUND + Specify the background color. BACKGROUND is either a single + float or three floats, separated by a comma or semicolon. In + case of a single float, it gives a grayscale value, in case + of three floats it gives a RGB - value.The floats are given + normalized to a range from 0 to 1. + + -y,--pyramidinfo PYRAMIDINFO + For the command 'SingleChannelPyramidTileAccessor' the + argument PYRAMIDINFO specifies the pyramid layer. It + consists of two integers(separated by a comma, semicolon or + pipe-symbol), where the first specifies the + minification-factor (between pyramid-layers) and the second + the pyramid-layer (starting with 0 for the layer with the + highest resolution). + + -z,--zoom ZOOM The zoom-factor (which is used for the commands + 'SingleChannelScalingTileAccessor' and + 'ScalingChannelComposite'). It is a float between 0 and 1. + + -i,--info-level INFO-LEVEL + When using the command 'PrintInformation' the INFO-LEVEL can + be used to specify which information is printed. Possible + values are "Statistics", "RawXML", "DisplaySettings", + "DisplaySettingsJson", "AllSubBlocks", "Attachments", + "AllAttachments", "PyramidStatistics", "GeneralInfo", + "ScalingInfo" and "All". The values are given as a list + separated by comma or semicolon. + + -e,--selection SELECTION + For the command 'ExtractAttachment' this allows to specify a + subset which is to be extracted (and saved to a file). It is + possible to specify the name and the index - only + attachments for which the name/index is equal to those + values specified are processed. The arguments are given in + JSON-notation, e.g. {"name":"Thumbnail"} or {"index":3.0}. + + -f,--tile-filter FILTER + Specify to filter subblocks according to the scene-index. A + comma separated list of either an interval or a single + integer may be given here, e.g. "2,3" or "2-4,6" or + "0-3,5-8". + + -m,--channelcompositionformat CHANNELCOMPOSITIONFORMAT + In case of a channel-composition, specifies the pixeltype of + the output. Possible values are "bgr24" (the default) and + "bgra32". If specifying "bgra32" it is possible to give the + value of the alpha-pixels in the form "bgra32(128)" - for an + alpha-value of 128. + + --createbounds BOUNDS + Only used for 'CreateCZI': specify the range of coordinates + used to create a CZI. Format is e.g. 'T0:3Z0:3C0:2'. + + --createsubblocksize SIZE + Only used for 'CreateCZI': specify the size of the subblocks + created in pixels. Format is e.g. '1600x1200'. + + --createtileinfo TILEINFO + Only used for 'CreateCZI': specify the number of tiles on + each plane. Format is e.g. '3x3;10%' for a 3 by 3 tiles + arrangement with 10% overlap. + + --font NAME/FILENAME + Only used for 'CreateCZI': (on Linux) specify the filename + of a TrueType-font (.ttf) to be used for generating text in + the subblocks; (on Windows) name of the font. + + --fontheight HEIGHT + Only used for 'CreateCZI': specifies the height of the font + in pixels (default: 36). + + -g,--guidofczi CZI-File-GUID + Only used for 'CreateCZI': specify the GUID of the file + (which is useful for bit-exact reproducible results); the + GUID must be given in the form + "cfc4a2fe-f968-4ef8-b685-e73d1b77271a" or + "{cfc4a2fe-f968-4ef8-b685-e73d1b77271a}" + + --bitmapgenerator BITMAPGENERATORCLASSNAME + Only used for 'CreateCZI': specifies the bitmap-generator to + use. Possibly values are "gdi", "freetype", "null" or + "default". Run with argument '--version' to get a list of + available bitmap-generators. + + --createczisbblkmetadata KEY_VALUE_SUBBLOCKMETADATA + Only used for 'CreateCZI': a key-value list in JSON-notation + which will be written as subblock-metadata. For example: + {"StageXPosition":-8906.346,"StageYPosition":-648.51} + + --compressionopts COMPRESSIONDESCRIPTION + Only used for 'CreateCZI': a string in a defined format + which states the compression-method and (compression-method + specific) parameters.The format is "compression_method: + key=value; ...". It starts with the name of the + compression-method, followed by a colon, then followed by a + list of key-value pairs which are separated by a semicolon. + Examples: "zstd0:ExplicitLevel=3", + "zstd1:ExplicitLevel=2;PreProcess=HiLoByteUnpack". + + --generatorpixeltype PIXELTYPE + Only used for 'CreateCZI': a string defining the pixeltype + used by the bitmap - generator. Possible values are 'Gray8', + 'Gray16', 'Bgr24' or 'Bgr48'. Default is 'Bgr24'. + + --cachesize CACHESIZE + Only used for 'PlaneScan' - specify the size of the + subblock-cache in bytes. The argument is to be given with a + suffix k, M, G, ... + + --tilesize-for-plane-scan TILESIZE + Only used for 'PlaneScan' - specify the size of ROI which is + used for scanning the plane in units of pixels. Format is + e.g. '1600x1200' and default is 512x512. + + --use-visibility-check-optimization + Whether to enable the experimental "visibility check + optimization" for the accessors. + + --version Print extended version-info and supported operations, then + exit. +``` The above text is printed if the program is executed with the argument '-?' or '\--help': diff --git a/Src/libCZI/Doc/version-history.markdown b/Src/libCZI/Doc/version-history.markdown index e2994c5e..b741992d 100644 --- a/Src/libCZI/Doc/version-history.markdown +++ b/Src/libCZI/Doc/version-history.markdown @@ -11,4 +11,5 @@ version history {#version_history} 0.54.3 | [79](https://github.com/ZEISS/libczi/pull/79) | add option _kCurlHttp_FollowLocation_ to follow HTTP redirects tp curl_http_inputstream 0.55.0 | [78](https://github.com/ZEISS/libczi/pull/78) | optimization: for multi-tile-composition, check relevant tiles for visibility before loading them (and do not load/decode non-visible tiles) 0.55.1 | [80](https://github.com/ZEISS/libczi/pull/80) | bugfix for above optimization - 0.56.0 | [82](https://github.com/ZEISS/libczi/pull/82) | add option "kCurlHttp_CaInfo" & "kCurlHttp_CaInfoBlob", allow to retrieve properties from a stream-class \ No newline at end of file + 0.56.0 | [82](https://github.com/ZEISS/libczi/pull/82) | add option "kCurlHttp_CaInfo" & "kCurlHttp_CaInfoBlob", allow to retrieve properties from a stream-class + 0.57.0 | [84](https://github.com/ZEISS/libczi/pull/84) | add caching for accessors, update CLI11 to version 2.3.2 \ No newline at end of file diff --git a/Src/libCZI/SingleChannelAccessorBase.cpp b/Src/libCZI/SingleChannelAccessorBase.cpp index 8a4e5a9f..67e64885 100644 --- a/Src/libCZI/SingleChannelAccessorBase.cpp +++ b/Src/libCZI/SingleChannelAccessorBase.cpp @@ -105,12 +105,12 @@ std::vector CSingleChannelAccessorBase::CheckForVisibility(const libCZI::In { return result; } - + const int64_t total_pixel_count = static_cast(roi.w) * roi.h; result.reserve(count); RectangleCoverageCalculator coverage_calculator; int64_t covered_pixel_count = 0; - for (int i = count -1; i >= 0; --i) // we start at the end, because that is the subblock which is rendered last (and thus is on top) + for (int i = count - 1; i >= 0; --i) // we start at the end, because that is the subblock which is rendered last (and thus is on top) { const int subblock_index = get_subblock_index(i); coverage_calculator.AddRectangle(get_rect_of_subblock(subblock_index)); @@ -134,3 +134,48 @@ std::vector CSingleChannelAccessorBase::CheckForVisibility(const libCZI::In std::reverse(result.begin(), result.end()); return result; } + +/*static*/CSingleChannelAccessorBase::SubBlockData CSingleChannelAccessorBase::GetSubBlockDataForSubBlockIndex( + const std::shared_ptr& sbBlkRepository, + const std::shared_ptr& cache, + int subBlockIndex, + bool onlyAddCompressedSubBlockToCache) +{ + SubBlockData result; + + // if no cache-object is given, then we simply read the subblock and create a bitmap from it + if (!cache) + { + const auto subblock = sbBlkRepository->ReadSubBlock(subBlockIndex); + result.bitmap = subblock->CreateBitmap(); + result.subBlockInfo = subblock->GetSubBlockInfo(); + } + else + { + const auto bitmap_from_cache = cache->Get(subBlockIndex); + if (bitmap_from_cache) + { + const bool b = sbBlkRepository->TryGetSubBlockInfo(subBlockIndex, &result.subBlockInfo); + if (!b) + { + stringstream ss; + ss << "SubBlockInfo not found in repository for subblock index " << subBlockIndex << "."; + throw logic_error(ss.str()); + } + + result.bitmap = bitmap_from_cache; + } + else + { + const auto subblock = sbBlkRepository->ReadSubBlock(subBlockIndex); + result.bitmap = subblock->CreateBitmap(); + result.subBlockInfo = subblock->GetSubBlockInfo(); + if (!onlyAddCompressedSubBlockToCache || result.subBlockInfo.GetCompressionMode() != CompressionMode::UnCompressed) + { + cache->Add(subBlockIndex, result.bitmap); + } + } + } + + return result; +} diff --git a/Src/libCZI/SingleChannelAccessorBase.h b/Src/libCZI/SingleChannelAccessorBase.h index 7a18d993..90fb8765 100644 --- a/Src/libCZI/SingleChannelAccessorBase.h +++ b/Src/libCZI/SingleChannelAccessorBase.h @@ -66,4 +66,16 @@ class CSingleChannelAccessorBase /// given here, then the result is guaranteed to be the same as if all subblocks were rendered. Non-visible subblocks are not /// part of this list. static std::vector CheckForVisibilityCore(const libCZI::IntRect& roi, int count, const std::function& get_subblock_index, const std::function& get_rect_of_subblock); + + struct SubBlockData + { + std::shared_ptr bitmap; + libCZI::SubBlockInfo subBlockInfo; + }; + + static SubBlockData GetSubBlockDataForSubBlockIndex( + const std::shared_ptr& sbBlkRepository, + const std::shared_ptr& cache, + int subBlockIndex, + bool onlyAddCompressedSubBlockToCache); }; diff --git a/Src/libCZI/SingleChannelPyramidLevelTileAccessor.cpp b/Src/libCZI/SingleChannelPyramidLevelTileAccessor.cpp index cf83d8fc..50a0b50f 100644 --- a/Src/libCZI/SingleChannelPyramidLevelTileAccessor.cpp +++ b/Src/libCZI/SingleChannelPyramidLevelTileAccessor.cpp @@ -103,11 +103,15 @@ void CSingleChannelPyramidLevelTileAccessor::ComposeTiles(libCZI::IBitmapData* b { if (index < bitmapCnt) { - SbInfo sbinfo = getSbInfo(index); - auto sb = this->sbBlkRepository->ReadSubBlock(sbinfo.index); - spBm = sb->CreateBitmap(); - xPosTile = (sb->GetSubBlockInfo().logicalRect.x - xPos) / sizeOfPixel; - yPosTile = (sb->GetSubBlockInfo().logicalRect.y - yPos) / sizeOfPixel; + const SbInfo sbinfo = getSbInfo(index); + const auto subblock_bitmap_data = CSingleChannelAccessorBase::GetSubBlockDataForSubBlockIndex( + this->sbBlkRepository, + options.subBlockCache, + sbinfo.index, + options.onlyUseSubBlockCacheForCompressedData); + spBm = subblock_bitmap_data.bitmap; + xPosTile = (subblock_bitmap_data.subBlockInfo.logicalRect.x - xPos) / sizeOfPixel; + yPosTile = (subblock_bitmap_data.subBlockInfo.logicalRect.y - yPos) / sizeOfPixel; return true; } @@ -129,7 +133,7 @@ libCZI::IntRect CSingleChannelPyramidLevelTileAccessor::CalcDestinationRectFromS libCZI::IntRect CSingleChannelPyramidLevelTileAccessor::NormalizePyramidRect(int x, int y, int w, int h, const PyramidLayerInfo& pyramidInfo) { - const int p = this->CalcSizeOfPixelOnLayer0(pyramidInfo); + const int p = CSingleChannelPyramidLevelTileAccessor::CalcSizeOfPixelOnLayer0(pyramidInfo); return IntRect{ x,y,w * p,h * p }; } diff --git a/Src/libCZI/SingleChannelScalingTileAccessor.cpp b/Src/libCZI/SingleChannelScalingTileAccessor.cpp index d99d988f..95df5de3 100644 --- a/Src/libCZI/SingleChannelScalingTileAccessor.cpp +++ b/Src/libCZI/SingleChannelScalingTileAccessor.cpp @@ -78,17 +78,21 @@ CSingleChannelScalingTileAccessor::CSingleChannelScalingTileAccessor(const std:: return IntSize{ static_cast(roi.w * zoom),static_cast(roi.h * zoom) }; } -void CSingleChannelScalingTileAccessor::ScaleBlt(libCZI::IBitmapData* bmDest, float zoom, const libCZI::IntRect& roi, const SbInfo& sbInfo) +void CSingleChannelScalingTileAccessor::ScaleBlt(libCZI::IBitmapData* bmDest, float zoom, const libCZI::IntRect& roi, const SbInfo& sbInfo, const libCZI::ISingleChannelScalingTileAccessor::Options& options) { - const auto sb = this->sbBlkRepository->ReadSubBlock(sbInfo.index); + auto subblock_bitmap_data = CSingleChannelAccessorBase::GetSubBlockDataForSubBlockIndex( + this->sbBlkRepository, + options.subBlockCache, + sbInfo.index, + options.onlyUseSubBlockCacheForCompressedData); if (GetSite()->IsEnabled(LOGLEVEL_CHATTYINFORMATION)) { stringstream ss; - ss << " bounds: " << Utils::DimCoordinateToString(&sb->GetSubBlockInfo().coordinate) << " M=" << (Utils::IsValidMindex(sbInfo.mIndex) ? to_string(sbInfo.mIndex) : "invalid"); + ss << " bounds: " << Utils::DimCoordinateToString(&subblock_bitmap_data.subBlockInfo.coordinate) << " M=" << (Utils::IsValidMindex(subblock_bitmap_data.subBlockInfo.mIndex) ? to_string(subblock_bitmap_data.subBlockInfo.mIndex) : "invalid"); GetSite()->Log(LOGLEVEL_CHATTYINFORMATION, ss); } - const auto source = sb->CreateBitmap(); + const auto& source = subblock_bitmap_data.bitmap; // In order not to run into trouble with floating point precision, if the scale is exactly 1, we refrain from using the scaling operation // and do instead a simple copy operation. This should ensure a pixel-accurate result if zoom is exactly 1. @@ -118,15 +122,15 @@ void CSingleChannelScalingTileAccessor::ScaleBlt(libCZI::IBitmapData* bmDest, fl // calculate the intersection of the subblock (logical rect) and the destination const auto intersect = Utilities::Intersect(sbInfo.logicalRect, roi); - const double roiSrcTopLeftX = double(intersect.x - sbInfo.logicalRect.x) / sbInfo.logicalRect.w; - const double roiSrcTopLeftY = double(intersect.y - sbInfo.logicalRect.y) / sbInfo.logicalRect.h; - const double roiSrcBttmRightX = double(intersect.x + intersect.w - sbInfo.logicalRect.x) / sbInfo.logicalRect.w; - const double roiSrcBttmRightY = double(intersect.y + intersect.h - sbInfo.logicalRect.y) / sbInfo.logicalRect.h; + const double roiSrcTopLeftX = static_cast(intersect.x - sbInfo.logicalRect.x) / sbInfo.logicalRect.w; + const double roiSrcTopLeftY = static_cast(intersect.y - sbInfo.logicalRect.y) / sbInfo.logicalRect.h; + const double roiSrcBttmRightX = static_cast(intersect.x + intersect.w - sbInfo.logicalRect.x) / sbInfo.logicalRect.w; + const double roiSrcBttmRightY = static_cast(intersect.y + intersect.h - sbInfo.logicalRect.y) / sbInfo.logicalRect.h; - const double destTopLeftX = double(intersect.x - roi.x) / roi.w; - const double destTopLeftY = double(intersect.y - roi.y) / roi.h; - const double destBttmRightX = double(intersect.x + intersect.w - roi.x) / roi.w; - const double destBttmRightY = double(intersect.y + intersect.h - roi.y) / roi.h; + const double destTopLeftX = static_cast(intersect.x - roi.x) / roi.w; + const double destTopLeftY = static_cast(intersect.y - roi.y) / roi.h; + const double destBttmRightX = static_cast(intersect.x + intersect.w - roi.x) / roi.w; + const double destBttmRightY = static_cast(intersect.y + intersect.h - roi.y) / roi.h; DblRect srcRoi{ roiSrcTopLeftX ,roiSrcTopLeftY,roiSrcBttmRightX - roiSrcTopLeftX ,roiSrcBttmRightY - roiSrcTopLeftY }; DblRect dstRoi{ destTopLeftX ,destTopLeftY,destBttmRightX - destTopLeftX ,destBttmRightY - destTopLeftY }; @@ -292,19 +296,19 @@ void CSingleChannelScalingTileAccessor::InternalGet(libCZI::IBitmapData* bmDest, // we only have to deal with a single scene (or: the document does not include a scene-dimension at all), in this // case we do not have group by scene and save some cycles auto sbSetsortedByZoom = this->GetSubSetFilteredBySceneSortedByZoom(roi, planeCoordinate, scenesInvolved, options.sortByM); - this->Paint(bmDest, roi, sbSetsortedByZoom, zoom, options.useVisibilityCheckOptimization); + this->Paint(bmDest, roi, sbSetsortedByZoom, zoom, options/*.useVisibilityCheckOptimization*/); } else { const auto sbSetSortedByZoomPerScene = this->GetSubSetSortedByZoomPerScene(scenesInvolved, roi, planeCoordinate, options.sortByM); for (const auto& it : sbSetSortedByZoomPerScene) { - this->Paint(bmDest, roi, get<1>(it), zoom, options.useVisibilityCheckOptimization); + this->Paint(bmDest, roi, get<1>(it), zoom, options/*.useVisibilityCheckOptimization*/); } } } -void CSingleChannelScalingTileAccessor::Paint(libCZI::IBitmapData* bmDest, const libCZI::IntRect& roi, const SubSetSortedByZoom& sbSetSortedByZoom, float zoom, bool useCoverageOptimization) +void CSingleChannelScalingTileAccessor::Paint(libCZI::IBitmapData* bmDest, const libCZI::IntRect& roi, const SubSetSortedByZoom& sbSetSortedByZoom, float zoom, const libCZI::ISingleChannelScalingTileAccessor::Options& options) { const int idxOf1stSubBlockOfZoomGreater = this->GetIdxOf1stSubBlockWithZoomGreater(sbSetSortedByZoom.subBlocks, sbSetSortedByZoom.sortedByZoom, zoom); if (idxOf1stSubBlockOfZoomGreater < 0) @@ -334,7 +338,7 @@ void CSingleChannelScalingTileAccessor::Paint(libCZI::IBitmapData* bmDest, const } } - if (!useCoverageOptimization) + if (!options.useVisibilityCheckOptimization) { for (auto it = start_iterator; it != end_iterator; ++it) { @@ -347,7 +351,7 @@ void CSingleChannelScalingTileAccessor::Paint(libCZI::IBitmapData* bmDest, const GetSite()->Log(LOGLEVEL_CHATTYINFORMATION, ss); } - this->ScaleBlt(bmDest, zoom, roi, sbInfo); + this->ScaleBlt(bmDest, zoom, roi, sbInfo, options); } } else @@ -375,7 +379,7 @@ void CSingleChannelScalingTileAccessor::Paint(libCZI::IBitmapData* bmDest, const GetSite()->Log(LOGLEVEL_CHATTYINFORMATION, ss); } - this->ScaleBlt(bmDest, zoom, roi, sbInfo); + this->ScaleBlt(bmDest, zoom, roi, sbInfo, options); } } } diff --git a/Src/libCZI/SingleChannelScalingTileAccessor.h b/Src/libCZI/SingleChannelScalingTileAccessor.h index 476ddec4..5c852e88 100644 --- a/Src/libCZI/SingleChannelScalingTileAccessor.h +++ b/Src/libCZI/SingleChannelScalingTileAccessor.h @@ -38,7 +38,7 @@ class CSingleChannelScalingTileAccessor : public CSingleChannelAccessorBase, pub std::vector CreateSortByZoom(const std::vector& sbBlks, bool sortByM); std::vector GetSubSet(const libCZI::IntRect& roi, const libCZI::IDimCoordinate* planeCoordinate, const std::vector* allowedScenes); int GetIdxOf1stSubBlockWithZoomGreater(const std::vector& sbBlks, const std::vector& byZoom, float zoom); - void ScaleBlt(libCZI::IBitmapData* bmDest, float zoom, const libCZI::IntRect& roi, const SbInfo& sbInfo); + void ScaleBlt(libCZI::IBitmapData* bmDest, float zoom, const libCZI::IntRect& roi, const SbInfo& sbInfo, const libCZI::ISingleChannelScalingTileAccessor::Options& options); void InternalGet(libCZI::IBitmapData* bmDest, const libCZI::IntRect& roi, const libCZI::IDimCoordinate* planeCoordinate, float zoom, const libCZI::ISingleChannelScalingTileAccessor::Options& options); @@ -55,5 +55,5 @@ class CSingleChannelScalingTileAccessor : public CSingleChannelAccessorBase, pub SubSetSortedByZoom GetSubSetFilteredBySceneSortedByZoom(const libCZI::IntRect& roi, const libCZI::IDimCoordinate* planeCoordinate, const std::vector& allowedScenes, bool sortByM); std::vector> GetSubSetSortedByZoomPerScene(const std::vector& scenes, const libCZI::IntRect& roi, const libCZI::IDimCoordinate* planeCoordinate, bool sortByM); - void Paint(libCZI::IBitmapData* bmDest, const libCZI::IntRect& roi, const SubSetSortedByZoom& sbSetSortedByZoom, float zoom, bool useCoverageOptimization); + void Paint(libCZI::IBitmapData* bmDest, const libCZI::IntRect& roi, const SubSetSortedByZoom& sbSetSortedByZoom, float zoom, const libCZI::ISingleChannelScalingTileAccessor::Options& options); }; diff --git a/Src/libCZI/SingleChannelTileAccessor.cpp b/Src/libCZI/SingleChannelTileAccessor.cpp index fcfc6c0b..02a44a57 100644 --- a/Src/libCZI/SingleChannelTileAccessor.cpp +++ b/Src/libCZI/SingleChannelTileAccessor.cpp @@ -69,10 +69,14 @@ void CSingleChannelTileAccessor::ComposeTiles(libCZI::IBitmapData* pBm, int xPos { if (index < static_cast(indices_of_visible_tiles.size())) { - const auto sb = this->sbBlkRepository->ReadSubBlock(subBlocksSet[indices_of_visible_tiles[index]].index); - spBm = sb->CreateBitmap(); - xPosTile = sb->GetSubBlockInfo().logicalRect.x; - yPosTile = sb->GetSubBlockInfo().logicalRect.y; + const auto subblock_data = CSingleChannelAccessorBase::GetSubBlockDataForSubBlockIndex( + this->sbBlkRepository, + options.subBlockCache, + subBlocksSet[indices_of_visible_tiles[index]].index, + options.onlyUseSubBlockCacheForCompressedData); + spBm = subblock_data.bitmap; + xPosTile = subblock_data.subBlockInfo.logicalRect.x; + yPosTile = subblock_data.subBlockInfo.logicalRect.y; return true; } @@ -90,10 +94,14 @@ void CSingleChannelTileAccessor::ComposeTiles(libCZI::IBitmapData* pBm, int xPos { if (index < static_cast(subBlocksSet.size())) { - const auto sb = this->sbBlkRepository->ReadSubBlock(subBlocksSet[index].index); - spBm = sb->CreateBitmap(); - xPosTile = sb->GetSubBlockInfo().logicalRect.x; - yPosTile = sb->GetSubBlockInfo().logicalRect.y; + const auto subblock_data = CSingleChannelAccessorBase::GetSubBlockDataForSubBlockIndex( + this->sbBlkRepository, + options.subBlockCache, + subBlocksSet[index].index, + options.onlyUseSubBlockCacheForCompressedData); + spBm = subblock_data.bitmap; + xPosTile = subblock_data.subBlockInfo.logicalRect.x; + yPosTile = subblock_data.subBlockInfo.logicalRect.y; return true; } diff --git a/Src/libCZI/libCZI.h b/Src/libCZI/libCZI.h index 04e45edf..2e94cda7 100644 --- a/Src/libCZI/libCZI.h +++ b/Src/libCZI/libCZI.h @@ -66,6 +66,7 @@ namespace libCZI class IMetadataSegment; class ISubBlockRepository; class IAttachment; + class ISubBlockCache; /// This structure contains information about the compiler settings and the version of the source /// which was used to create the library. @@ -173,6 +174,10 @@ namespace libCZI /// \return The newly created metadata-builder-object. LIBCZI_API std::shared_ptr CreateMetadataBuilder(); + /// Creates a sub block cache object. + /// \returns The newly created sub block cache. + LIBCZI_API std::shared_ptr CreateSubBlockCache(); + /// Creates metadata builder object from the specified UTF8-encoded XML-string. If the XML is /// invalid or if the root-node "ImageDocument" is not present, then an exception is thrown. /// \param xml The UTF8-encoded XML string. diff --git a/Src/libCZI/libCZI_Compositor.h b/Src/libCZI/libCZI_Compositor.h index c5e0d2e1..2fa38565 100644 --- a/Src/libCZI/libCZI_Compositor.h +++ b/Src/libCZI/libCZI_Compositor.h @@ -25,6 +25,127 @@ namespace libCZI SingleChannelScalingTileAccessor ///< The scaling-single-channel-tile accessor (associated interface: ISingleChannelScalingTileAccessor). }; + /// This interface defines how status information about the cache-state can be queried. + class ISubBlockCacheStatistics + { + public: + static constexpr std::uint8_t kMemoryUsage = 1; ///< Bit-mask identifying the memory-usage field in the statistics struct. + static constexpr std::uint8_t kElementsCount = 2; ///< Bit-mask identifying the elements-count field in the statistics struct. + + /// This struct defines the statistics which can be queried from the cache. There is a bitfield which + /// defines which elements are valid. If the bit is set, then the corresponding member is valid. + struct Statistics + { + /// A bit mask which indicates which members are valid. C.f. the constants kMemoryUsage and kElementsCount. + std::uint8_t validityMask; + + /// The memory usage of all elements in the cache. This field is only valid if the bit kMemoryUsage is set in the validityMask. + std::uint64_t memoryUsage; + + /// The number of elements in the cache. This field is only valid if the bit kElementsCount is set in the validityMask. + std::uint32_t elementsCount; + }; + + /// Gets momentarily valid statistics about the cache. The mask defines which statistic/s is/are to be retrieved. + /// In case of multiple fields being requested, it is guaranteed that all requested fields are a transactional + /// snapshot of the state. + /// + /// \param mask A bitmask specifying which fields are requested. Only the fields requested are guaranteed to be valid + /// in the returned struct. + /// + /// \returns A consistent snapshot of the statistics. + virtual Statistics GetStatistics(std::uint8_t mask) const = 0; + + virtual ~ISubBlockCacheStatistics() = default; + + ISubBlockCacheStatistics() = default; + ISubBlockCacheStatistics(const ISubBlockCacheStatistics&) = delete; + ISubBlockCacheStatistics& operator=(const ISubBlockCacheStatistics&) = delete; + ISubBlockCacheStatistics(ISubBlockCacheStatistics&&) noexcept = delete; + ISubBlockCacheStatistics& operator=(ISubBlockCacheStatistics&&) noexcept = delete; + }; + + /// This interface defines the global operations on the cache. It is used to control the memory usage of the cache. + class ISubBlockCacheControl + { + public: + /// Options for controlling the prune operation. There are two metrics which can be used to control what + /// remains in the cache and what is discarded: the maximum memory usage (for all elements in the cache) and + /// the maximum number of sub-blocks. If the cache exceeds one of those limits, then elements are evicted from the cache + /// until both conditions are met. Eviction is done in the order starting with elements which have been least recently accessed. + /// As "access" we define either the Add-operation or the Get-operation - so, when an element is retrieved from the + /// cache, it is considered as "accessed". + /// If only one condition is desired, then the other condition can be set to the maximum value of the respective type (which is the + /// default value). + struct PruneOptions + { + /// The maximum memory usage (in bytes) for the cache. If the cache exceeds this limit, + /// then the least recently used sub-blocks are removed from the cache. + std::uint64_t maxMemoryUsage{ (std::numeric_limits::max)() }; + + /// The maximum number of sub-blocks in the cache. If the cache exceeds this limit, + /// then the least recently used sub-blocks are removed from the cache. + std::uint32_t maxSubBlockCount{ (std::numeric_limits::max)() }; + }; + + /// Prunes the cache. This means that sub-blocks are removed from the cache until the cache satisfies the conditions given in the options. + /// Note that the prune operation is not done automatically - it must be called manually. I.e. when adding an element to the cache, the cache + /// is **not** pruned automatically. + /// \param options Options for controlling the operation. + virtual void Prune(const PruneOptions& options) = 0; + + virtual ~ISubBlockCacheControl() = default; + + ISubBlockCacheControl() = default; + ISubBlockCacheControl(const ISubBlockCacheControl&) = delete; + ISubBlockCacheControl& operator=(const ISubBlockCacheControl&) = delete; + ISubBlockCacheControl(ISubBlockCacheControl&&) noexcept = delete; + ISubBlockCacheControl& operator=(ISubBlockCacheControl&&) noexcept = delete; + }; + + /// This interface defines the operations of adding and querying an element to/from the cache. + class ISubBlockCacheOperation + { + public: + /// Gets the bitmap for the specified subblock-index. If the subblock is not in the cache, then a nullptr is returned. + /// \param subblock_index The subblock index to get. + /// \returns If the subblock is in the cache, then a std::shared_ptr<libCZI::IBitmapData> is returned. Otherwise a nullptr is returned. + virtual std::shared_ptr Get(int subblock_index) = 0; + + /// Adds the specified bitmap for the specified subblock_index to the cache. If the subblock is already in the cache, then it is overwritten. + /// \param subblock_index The subblock index to add. + /// \param pBitmap The bitmap. + virtual void Add(int subblock_index, std::shared_ptr pBitmap) = 0; + + virtual ~ISubBlockCacheOperation() = default; + + ISubBlockCacheOperation() = default; + ISubBlockCacheOperation(const ISubBlockCacheOperation&) = delete; + ISubBlockCacheOperation& operator=(const ISubBlockCacheOperation&) = delete; + ISubBlockCacheOperation(ISubBlockCacheOperation&&) noexcept = delete; + ISubBlockCacheOperation& operator=(ISubBlockCacheOperation&&) noexcept = delete; + }; + + /// Interface for a caching component (which can be used with the compositors). The intended use is as follows: + /// * Whenever the bitmap corresponding to a subblock (c.f. ISubBlock::CreateBitmap) is accessed, the bitmap may be added + /// to a cache object, where the subblock-index is the key. + /// * Whenever a bitmap is needed (for a given subblock-index), the cache object is first queried whether it contains the bitmap. If yes, then the bitmap + /// returned may be used instead of executing the subblock-read-and-decode operation. + /// In order to control the memory usage of the cache, the cache object must be pruned (i.e. subblocks are removed from the cache). Currently this means, + /// that the Prune-method must be called manually. The cache object does not do any pruning automatically. + /// The operations of Adding, Querying and Pruning the cache object are thread-safe. + class ISubBlockCache : public ISubBlockCacheStatistics, public ISubBlockCacheControl, public ISubBlockCacheOperation + { + public: + ~ISubBlockCache() override = default; + + ISubBlockCache() = default; + ISubBlockCache(const ISubBlockCache&) = delete; + ISubBlockCache& operator=(const ISubBlockCache&) = delete; + ISubBlockCache(ISubBlockCache&&) noexcept = delete; + ISubBlockCache& operator=(ISubBlockCache&&) noexcept = delete; + }; + /// The base interface (all accessor-interface must derive from this). class IAccessor { @@ -46,7 +167,7 @@ namespace libCZI /// The pixel type of the output bitmap is either specified as an argument or it is automatically /// determined. In the latter case the first sub-block found on the specified plane is examined for its /// pixeltype, and this pixeltype is used.\n - /// The pixels in the output bitmap get converted from the source pixels (if their pixeltypes differs). + /// The pixels in the output bitmap get converted from the source pixels (if their pixeltypes differ). class ISingleChannelTileAccessor : public IAccessor { public: @@ -67,7 +188,7 @@ namespace libCZI /// all relevant tiles are checked whether they are visible in the destination bitmap. If a tile is not visible, then /// the corresponding sub-block is not read. This can speed up the operation considerably. The result is the same as /// without this optimization - i.e. there should be no reason to turn it off besides potential bugs. - bool useVisibilityCheckOptimization; + bool useVisibilityCheckOptimization; /// If true, then a one-pixel wide boundary will be drawn around /// each tile (in black color). @@ -76,6 +197,16 @@ namespace libCZI /// If specified, only subblocks with a scene-index contained in the set will be considered. std::shared_ptr sceneFilter; + /// If specified, then the sub-block cache is used. This can speed up the operation considerably. + /// Bitmaps that are read by the accessor are added to the cache. If a bitmap is needed which is already + /// in the cache, then the bitmap from the cache is used instead of reading the sub-block from the file. + std::shared_ptr subBlockCache; + + /// If true, then only bitmaps from sub-blocks with compressed data are added to the cache. For uncompressed + /// data, the time for reading the bitmap can be negligible, so the benefit of caching might not outweigh the + /// increased memory usage. + bool onlyUseSubBlockCacheForCompressedData; + /// Clears this object to its blank state. void Clear() { @@ -84,6 +215,8 @@ namespace libCZI this->useVisibilityCheckOptimization = false; this->drawTileBorder = false; this->sceneFilter.reset(); + this->subBlockCache.reset(); + this->onlyUseSubBlockCacheForCompressedData = true; } }; @@ -147,7 +280,7 @@ namespace libCZI /// \return A std::shared_ptr<libCZI::IBitmapData> inline std::shared_ptr Get(libCZI::PixelType pixeltype, int xPos, int yPos, int width, int height, const IDimCoordinate* planeCoordinate, const Options* pOptions) { return this->Get(pixeltype, libCZI::IntRect{ xPos,yPos,width,height }, planeCoordinate, pOptions); } protected: - virtual ~ISingleChannelTileAccessor() {} + virtual ~ISingleChannelTileAccessor() = default; }; /// Interface for single-channel-pyramidlayer tile accessors. @@ -176,6 +309,14 @@ namespace libCZI /// If specified, only subblocks with a scene-index contained in the set will be considered. std::shared_ptr sceneFilter; + /// If specified, then the sub-block cache is used. This can speed up the operation considerably. + /// Bitmaps that are read by the accessor are added to the cache. If a bitmap is needed which is already + /// in the cache, then the bitmap from the cache is used instead of reading the sub-block from the file. + std::shared_ptr subBlockCache; + + /// If true, then only bitmaps from sub-blocks with compressed data are added to the cache. + bool onlyUseSubBlockCacheForCompressedData; + /// Clears this object to its blank state. void Clear() { @@ -183,6 +324,8 @@ namespace libCZI this->sortByM = true; this->backGroundColor.r = this->backGroundColor.g = this->backGroundColor.b = std::numeric_limits::quiet_NaN(); this->sceneFilter.reset(); + this->subBlockCache.reset(); + this->onlyUseSubBlockCacheForCompressedData = true; } }; @@ -242,7 +385,7 @@ namespace libCZI /// with the maximum pixel value (of the specific pixeltype). If it is a RGB-color type, then R, G and B are separately multiplied with /// the maximum pixel value. /// If any of R, G or B is NaN, then the background is not cleared. - RgbFloatColor backGroundColor; + RgbFloatColor backGroundColor; /// If true, then the tiles are sorted by their M-index (tile with highest M-index will be 'on top'). /// Otherwise the Z-order is arbitrary. @@ -260,7 +403,15 @@ namespace libCZI /// all relevant tiles are checked whether they are visible in the destination bitmap. If a tile is not visible, then /// the corresponding sub-block is not read. This can speed up the operation considerably. The result is the same as /// without this optimization - i.e. there should be no reason to turn it off besides potential bugs. - bool useVisibilityCheckOptimization; + bool useVisibilityCheckOptimization; + + /// If specified, then the sub-block cache is used. This can speed up the operation considerably. + /// Bitmaps that are read by the accessor are added to the cache. If a bitmap is needed which is already + /// in the cache, then the bitmap from the cache is used instead of reading the sub-block from the file. + std::shared_ptr subBlockCache; + + /// If true, then only bitmaps from sub-blocks with compressed data are added to the cache. + bool onlyUseSubBlockCacheForCompressedData; /// Clears this object to its blank state. void Clear() @@ -270,6 +421,8 @@ namespace libCZI this->backGroundColor.r = this->backGroundColor.g = this->backGroundColor.b = std::numeric_limits::quiet_NaN(); this->sceneFilter.reset(); this->useVisibilityCheckOptimization = false; + this->subBlockCache.reset(); + this->onlyUseSubBlockCacheForCompressedData = true; } }; diff --git a/Src/libCZI/subblock_cache.cpp b/Src/libCZI/subblock_cache.cpp new file mode 100644 index 00000000..fe7b4c24 --- /dev/null +++ b/Src/libCZI/subblock_cache.cpp @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: 2023 Carl Zeiss Microscopy GmbH +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "subblock_cache.h" + +using namespace libCZI; +using namespace std; + +std::shared_ptr libCZI::CreateSubBlockCache() +{ + return make_shared(); +} + +ISubBlockCacheStatistics::Statistics SubBlockCache::GetStatistics(std::uint8_t mask) const +{ + Statistics result{ 0 }; + if (mask == ISubBlockCacheStatistics::kMemoryUsage) + { + result.validityMask = ISubBlockCacheStatistics::kMemoryUsage; + result.memoryUsage = this->cache_size_in_bytes_.load(); + } + else if (mask == ISubBlockCacheStatistics::kElementsCount) + { + result.validityMask = ISubBlockCacheStatistics::kElementsCount; + result.elementsCount = this->cache_subblock_count_.load(); + } + else if (mask == (ISubBlockCacheStatistics::kMemoryUsage | ISubBlockCacheStatistics::kElementsCount)) + { + result.validityMask = ISubBlockCacheStatistics::kMemoryUsage | ISubBlockCacheStatistics::kElementsCount; + + // We want to ensure that the memory usage and the element count are consistent, therefore we need to lock reading both values. + lock_guard lck(this->mutex_); + result.memoryUsage = this->cache_size_in_bytes_.load(); + result.elementsCount = this->cache_subblock_count_.load(); + } + + return result; +} + +std::shared_ptr SubBlockCache::Get(int subblock_index) +{ + lock_guard lck(this->mutex_); + const auto element = this->cache_.find(subblock_index); + if (element != this->cache_.end()) + { + element->second.lru_value = this->lru_counter_.fetch_add(1); + return element->second.bitmap; + } + + return {}; +} + +void SubBlockCache::Add(int subblock_index, std::shared_ptr bitmap) +{ + const auto size_in_bytes_of_added_bitmap = SubBlockCache::CalculateSizeInBytes(bitmap.get()); + const auto entry_to_be_added = CacheEntry{ bitmap, this->lru_counter_.fetch_add(1) }; + + lock_guard lck(this->mutex_); + const auto result = this->cache_.insert({ subblock_index, entry_to_be_added }); + if (result.second) + { + // New element inserted + this->cache_size_in_bytes_ += size_in_bytes_of_added_bitmap; + ++this->cache_subblock_count_; + } + else + { + // Element with the same key already existed + this->cache_size_in_bytes_ -= SubBlockCache::CalculateSizeInBytes(result.first->second.bitmap.get()); + result.first->second = entry_to_be_added; + this->cache_size_in_bytes_ += size_in_bytes_of_added_bitmap; + } +} + +void SubBlockCache::Prune(const PruneOptions& options) +{ + if (options.maxMemoryUsage != numeric_limits::max() || + options.maxSubBlockCount != numeric_limits::max()) + { + lock_guard lck(this->mutex_); + this->PruneByMemoryUsageAndElementCount(options.maxMemoryUsage, options.maxSubBlockCount); + } +} + +void SubBlockCache::PruneByMemoryUsageAndElementCount(std::uint64_t max_memory_usage, std::uint32_t max_element_count) +{ + // TODO(JBL): This is a very simple implementation of the prune operation. We determine the oldest element and remove it. + // This is repeated until the cache size is below the maximum memory usage and the element count is below the maximum element count. + // Detrimental is the fact that we have to iterate over all elements in the cache to determine the oldest element, and we might have + // to do this multiple times. If the number of elements in the cache is large, this might be a performance bottleneck. + while (this->cache_size_in_bytes_.load() > max_memory_usage || this->cache_subblock_count_.load() > max_element_count) + { + auto oldest_element = std::min_element(this->cache_.begin(), this->cache_.end(), SubBlockCache::CompareForLruValue); + if (oldest_element == this->cache_.end()) + { + break; + } + + this->cache_size_in_bytes_ -= SubBlockCache::CalculateSizeInBytes(oldest_element->second.bitmap.get()); + --this->cache_subblock_count_; + this->cache_.erase(oldest_element); + } +} + +/*static*/std::uint64_t SubBlockCache::CalculateSizeInBytes(const libCZI::IBitmapData* bitmap) +{ + const IntSize size = bitmap->GetSize(); + return static_cast(size.w) * size.h * Utils::GetBytesPerPixel(bitmap->GetPixelType()); +} + +/*static*/bool SubBlockCache::CompareForLruValue(const std::pair& a, const std::pair& b) +{ + return a.second.lru_value < b.second.lru_value; +} diff --git a/Src/libCZI/subblock_cache.h b/Src/libCZI/subblock_cache.h new file mode 100644 index 00000000..28c80e40 --- /dev/null +++ b/Src/libCZI/subblock_cache.h @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: 2023 Carl Zeiss Microscopy GmbH +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#pragma once + +#include "libCZI.h" +#include +#include +#include +#include + +/// A simplistic sub-block cache implementation. It is thread-safe and uses a LRU eviction strategy. +class SubBlockCache : public libCZI::ISubBlockCache +{ +private: + struct CacheEntry + { + std::shared_ptr bitmap; ////< The cached bitmap. + std::uint64_t lru_value; ////< The "LRU value" - when marking a cache entry as "used", this value is set to the current value of the "LRU counter". + }; + + std::map cache_; + mutable std::mutex mutex_; + std::atomic_uint64_t lru_counter_{ 0 }; ///< The "LRU counter" - when marking a cache entry as "used", this counter is incremented and the new value is stored in the cache entry. + std::atomic_uint64_t cache_size_in_bytes_{ 0 }; ////< The current size of the cache in bytes. + std::atomic_uint32_t cache_subblock_count_{ 0 }; ///< The current number of sub-blocks in the cache. +public: + SubBlockCache() = default; + ~SubBlockCache() override = default; + + std::shared_ptr Get(int subblock_index) override; + void Add(int subblock_index, std::shared_ptr bitmap) override; + void Prune(const PruneOptions& options) override; + Statistics GetStatistics(std::uint8_t mask) const override; +private: + void PruneByMemoryUsageAndElementCount(std::uint64_t max_memory_usage, std::uint32_t max_element_count); + static std::uint64_t CalculateSizeInBytes(const libCZI::IBitmapData* bitmap); + static bool CompareForLruValue(const std::pair& a, const std::pair& b); +}; diff --git a/Src/libCZI_UnitTests/CMakeLists.txt b/Src/libCZI_UnitTests/CMakeLists.txt index 8d30312e..0b09a94f 100644 --- a/Src/libCZI_UnitTests/CMakeLists.txt +++ b/Src/libCZI_UnitTests/CMakeLists.txt @@ -63,7 +63,8 @@ ADD_EXECUTABLE(libCZI_UnitTests test_streamslib.cpp test_curlhttpstream.cpp test_rectanglecoverage.cpp - test_TileAccessorCoverageOptimization.cpp) + test_TileAccessorCoverageOptimization.cpp + test_SubBlockCache.cpp) TARGET_LINK_LIBRARIES(libCZI_UnitTests PRIVATE libCZIStatic GTest::gtest GTest::gmock) diff --git a/Src/libCZI_UnitTests/test_Accessors.cpp b/Src/libCZI_UnitTests/test_Accessors.cpp index 49f325a7..7c51e4e2 100644 --- a/Src/libCZI_UnitTests/test_Accessors.cpp +++ b/Src/libCZI_UnitTests/test_Accessors.cpp @@ -670,3 +670,76 @@ TEST(Accessor, CreateDocumentAndCheckSingleChannelScalingAccessor1) EXPECT_EQ(pixel_x0_y1, 3); EXPECT_EQ(pixel_x1_y1, 4); } + +TEST(Accessor, CreateDocumentAndCheckSingleChannelScalingAccessorWithSubBlockCache) +{ + // we use the same CZI-document as before, but we use subblock-cache + auto czi_document_as_blob = CreateCziWithFourSubblockInMosaicArragengement(); + + const auto memory_stream = make_shared(get<0>(czi_document_as_blob).get(), get<1>(czi_document_as_blob)); + const auto reader = CreateCZIReader(); + reader->Open(memory_stream); + + const auto accessor = reader->CreateSingleChannelScalingTileAccessor(); + const auto subblock_cache = CreateSubBlockCache(); + const CDimCoordinate plane_coordinate{ {DimensionIndex::C, 0} }; + ISingleChannelScalingTileAccessor::Options options; + options.Clear(); + options.backGroundColor = RgbFloatColor{ 0,0,0 }; // request to have background cleared with black + options.subBlockCache = subblock_cache; + options.onlyUseSubBlockCacheForCompressedData = false; + + // act + auto composite_bitmap = accessor->Get( + PixelType::Gray8, + IntRect{ 1,1,2,2 }, + &plane_coordinate, + 1, + &options); + + // assert + + // first, check that the result is correct + ASSERT_EQ(composite_bitmap->GetWidth(), 2); + ASSERT_EQ(composite_bitmap->GetHeight(), 2); + { + const ScopedBitmapLockerSP lock_info_bitmap{ composite_bitmap }; + const uint8_t pixel_x0_y0 = *(static_cast(lock_info_bitmap.ptrDataRoi) + 0); + const uint8_t pixel_x1_y0 = *(static_cast(lock_info_bitmap.ptrDataRoi) + 1); + const uint8_t pixel_x0_y1 = *(static_cast(lock_info_bitmap.ptrDataRoi) + static_cast(1) * lock_info_bitmap.stride + 0); + const uint8_t pixel_x1_y1 = *(static_cast(lock_info_bitmap.ptrDataRoi) + static_cast(1) * lock_info_bitmap.stride + 1); + + EXPECT_EQ(pixel_x0_y0, 1); + EXPECT_EQ(pixel_x1_y0, 2); + EXPECT_EQ(pixel_x0_y1, 3); + EXPECT_EQ(pixel_x1_y1, 4); + } + + const auto cache_statistics = subblock_cache->GetStatistics(ISubBlockCacheStatistics::kMemoryUsage | ISubBlockCacheStatistics::kElementsCount); + EXPECT_GE(cache_statistics.memoryUsage, 16); + EXPECT_EQ(cache_statistics.elementsCount, 4); + + // now, we do the same request again, and this time we expect that the subblock-cache is used + composite_bitmap = accessor->Get( + PixelType::Gray8, + IntRect{ 1,1,2,2 }, + &plane_coordinate, + 1, + &options); + + // we check that the result is the same as before + ASSERT_EQ(composite_bitmap->GetWidth(), 2); + ASSERT_EQ(composite_bitmap->GetHeight(), 2); + { + const ScopedBitmapLockerSP lock_info_bitmap{ composite_bitmap }; + const uint8_t pixel_x0_y0 = *(static_cast(lock_info_bitmap.ptrDataRoi) + 0); + const uint8_t pixel_x1_y0 = *(static_cast(lock_info_bitmap.ptrDataRoi) + 1); + const uint8_t pixel_x0_y1 = *(static_cast(lock_info_bitmap.ptrDataRoi) + static_cast(1) * lock_info_bitmap.stride + 0); + const uint8_t pixel_x1_y1 = *(static_cast(lock_info_bitmap.ptrDataRoi) + static_cast(1) * lock_info_bitmap.stride + 1); + + EXPECT_EQ(pixel_x0_y0, 1); + EXPECT_EQ(pixel_x1_y0, 2); + EXPECT_EQ(pixel_x0_y1, 3); + EXPECT_EQ(pixel_x1_y1, 4); + } +} diff --git a/Src/libCZI_UnitTests/test_SubBlockCache.cpp b/Src/libCZI_UnitTests/test_SubBlockCache.cpp new file mode 100644 index 00000000..1cfae4cc --- /dev/null +++ b/Src/libCZI_UnitTests/test_SubBlockCache.cpp @@ -0,0 +1,158 @@ +// SPDX-FileCopyrightText: 2017-2022 Carl Zeiss Microscopy GmbH +// +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "include_gtest.h" +#include "inc_libCZI.h" +#include "utils.h" + +using namespace libCZI; +using namespace std; + +TEST(SubBlockCache, SimpleUseCase1) +{ + const auto cache = CreateSubBlockCache(); + const auto bm1 = CreateTestBitmap(PixelType::Bgr24, 163, 128); + cache->Add(0, bm1); + const auto bm2 = CreateTestBitmap(PixelType::Bgr24, 161, 114); + cache->Add(1, bm2); + + const auto bitmap_from_cache_1 = cache->Get(0); + EXPECT_TRUE(AreBitmapDataEqual(bm1, bitmap_from_cache_1)); + + const auto bitmap_from_cache_2 = cache->Get(1); + EXPECT_TRUE(AreBitmapDataEqual(bm2, bitmap_from_cache_2)); +} + +TEST(SubBlockCache, SimpleUseCase2) +{ + const auto cache = CreateSubBlockCache(); + const auto bm1 = CreateTestBitmap(PixelType::Bgr24, 163, 128); + cache->Add(0, bm1); + const auto bm2 = CreateTestBitmap(PixelType::Bgr24, 161, 114); + cache->Add(1, bm2); + + const auto bitmap_from_cache_3 = cache->Get(3); + EXPECT_TRUE(!bitmap_from_cache_3); +} + +TEST(SubBlockCache, OverwriteExisting) +{ + const auto cache = CreateSubBlockCache(); + const auto bm1 = CreateTestBitmap(PixelType::Bgr24, 163, 128); + cache->Add(0, bm1); + const auto bm2 = CreateTestBitmap(PixelType::Bgr24, 161, 114); + cache->Add(1, bm2); + const auto bm3 = CreateTestBitmap(PixelType::Gray8, 11, 14); + cache->Add(1, bm3); + + const auto bitmap_from_cache_1 = cache->Get(0); + EXPECT_TRUE(AreBitmapDataEqual(bm1, bitmap_from_cache_1)); + + const auto bitmap_from_cache_2 = cache->Get(1); + EXPECT_TRUE(AreBitmapDataEqual(bm3, bitmap_from_cache_2)); + + const auto statistics_memory_usage = cache->GetStatistics(ISubBlockCacheStatistics::kMemoryUsage); + EXPECT_EQ(statistics_memory_usage.validityMask, ISubBlockCacheStatistics::kMemoryUsage); + EXPECT_EQ(statistics_memory_usage.memoryUsage, 163 * 128 * 3 + 11 * 14); + + const auto statistics_elements_count = cache->GetStatistics(ISubBlockCacheStatistics::kElementsCount); + EXPECT_EQ(statistics_elements_count.validityMask, ISubBlockCacheStatistics::kElementsCount); + EXPECT_EQ(statistics_elements_count.elementsCount, 2); +} + +TEST(SubBlockCache, GetStatistics) +{ + const auto cache = CreateSubBlockCache(); + const auto bm1 = CreateTestBitmap(PixelType::Gray8, 4, 2); + cache->Add(0, bm1); + const auto bm2 = CreateTestBitmap(PixelType::Gray16, 2, 2); + cache->Add(1, bm2); + + const auto statistics_memory_usage = cache->GetStatistics(ISubBlockCacheStatistics::kMemoryUsage); + EXPECT_EQ(statistics_memory_usage.validityMask, ISubBlockCacheStatistics::kMemoryUsage); + EXPECT_EQ(statistics_memory_usage.memoryUsage, 4 * 2 + 2 * 2 * 2); + + const auto statistics_elements_count = cache->GetStatistics(ISubBlockCacheStatistics::kElementsCount); + EXPECT_EQ(statistics_elements_count.validityMask, ISubBlockCacheStatistics::kElementsCount); + EXPECT_EQ(statistics_elements_count.elementsCount, 2); + + const auto statistics_both = cache->GetStatistics(ISubBlockCacheStatistics::kMemoryUsage | ISubBlockCacheStatistics::kElementsCount); + EXPECT_EQ(statistics_both.validityMask, ISubBlockCacheStatistics::kMemoryUsage | ISubBlockCacheStatistics::kElementsCount); + EXPECT_EQ(statistics_both.memoryUsage, 4 * 2 + 2 * 2 * 2); + EXPECT_EQ(statistics_both.elementsCount, 2); +} + +TEST(SubBlockCache, PruneCacheCase1) +{ + // We add two elements to the cache, making the last-added element the least recently used one. When then pruning the cache to 1 element, + // the first added element (with key=0) should be removed. + const auto cache = CreateSubBlockCache(); + const auto bm1 = CreateTestBitmap(PixelType::Bgr24, 163, 128); + cache->Add(0, bm1); + const auto bm2 = CreateTestBitmap(PixelType::Bgr24, 161, 114); + cache->Add(1, bm2); + + cache->Prune({ numeric_limits::max(), 1 }); + const auto statistics_elements_count = cache->GetStatistics(ISubBlockCacheStatistics::kElementsCount); + EXPECT_EQ(statistics_elements_count.validityMask, ISubBlockCacheStatistics::kElementsCount); + EXPECT_EQ(statistics_elements_count.elementsCount, 1); + + auto bitmap_from_cache = cache->Get(1); + EXPECT_TRUE(bitmap_from_cache != nullptr); + bitmap_from_cache = cache->Get(0); + EXPECT_TRUE(bitmap_from_cache == nullptr); +} + +TEST(SubBlockCache, PruneCacheCase2) +{ + // We add two items to the cache (with key 0, 1). Then, we retrieve element 0 from the cache, which makes it the least recently used one. + // When then pruning the cache to 1 element, element 1 should be removed. + const auto cache = CreateSubBlockCache(); + const auto bm1 = CreateTestBitmap(PixelType::Bgr24, 163, 128); + cache->Add(0, bm1); + const auto bm2 = CreateTestBitmap(PixelType::Bgr24, 161, 114); + cache->Add(1, bm2); + + // Now, element 0 is the oldest one, and 1 is the least recently used one. + // We now retrieve element 0 from the cache, which makes it the least recently used one. + auto bitmap_from_cache = cache->Get(0); + + cache->Prune({ numeric_limits::max(), 1 }); + const auto statistics_elements_count = cache->GetStatistics(ISubBlockCacheStatistics::kElementsCount); + EXPECT_EQ(statistics_elements_count.validityMask, ISubBlockCacheStatistics::kElementsCount); + EXPECT_EQ(statistics_elements_count.elementsCount, 1); + + bitmap_from_cache = cache->Get(0); + EXPECT_TRUE(bitmap_from_cache != nullptr); + bitmap_from_cache = cache->Get(1); + EXPECT_TRUE(bitmap_from_cache == nullptr); +} + +TEST(SubBlockCache, PruneCacheCase3) +{ + // We add three items to the cache (with key 0, 1, 2), each one byte in size. Then, we request to prune the cache + // to 1 byte max memory usage. This should remove the first two items from the cache (since they are the oldest entries), + // and keep the last one. + const auto cache = CreateSubBlockCache(); + const auto bm1 = CreateTestBitmap(PixelType::Gray8, 1, 1); + cache->Add(0, bm1); + const auto bm2 = CreateTestBitmap(PixelType::Gray8, 1, 1); + cache->Add(1, bm2); + const auto bm3 = CreateTestBitmap(PixelType::Gray8, 1, 1); + cache->Add(2, bm3); + + ISubBlockCache::PruneOptions prune_options; + prune_options.maxMemoryUsage = 1; + cache->Prune(prune_options); + const auto statistics_elements_count = cache->GetStatistics(ISubBlockCacheStatistics::kElementsCount); + EXPECT_EQ(statistics_elements_count.validityMask, ISubBlockCacheStatistics::kElementsCount); + EXPECT_EQ(statistics_elements_count.elementsCount, 1); + + auto bitmap_from_cache = cache->Get(0); + EXPECT_TRUE(bitmap_from_cache == nullptr); + bitmap_from_cache = cache->Get(1); + EXPECT_TRUE(bitmap_from_cache == nullptr); + bitmap_from_cache = cache->Get(2); + EXPECT_TRUE(bitmap_from_cache != nullptr); +}