diff --git a/libs/Common/List.h b/libs/Common/List.h index fbe108dfb..8725a2f5a 100644 --- a/libs/Common/List.h +++ b/libs/Common/List.h @@ -1257,10 +1257,15 @@ class cList { ASSERT(newVectorSize > _vectorSize); // grow by 50% or at least to minNewVectorSize - const IDX expoVectorSize(_vectorSize + (_vectorSize>>1)); + IDX expoVectorSize(_vectorSize + (_vectorSize>>1)); + // cap growth for very large vectors + const IDX maxGrowCapacity(3*1024*1024*1024ull/*3GB*/); + const IDX growCapacity((expoVectorSize - _vectorSize) * sizeof(TYPE)); + if (growCapacity > maxGrowCapacity) + expoVectorSize = _vectorSize + maxGrowCapacity / sizeof(TYPE); + // allocate a larger chunk of memory, copy the data and delete the old chunk if (newVectorSize < expoVectorSize) newVectorSize = expoVectorSize; - // allocate a larger chunk of memory, copy the data and delete the old chunk TYPE* const tmp(_vector); _vector = (TYPE*) operator new[] (newVectorSize * sizeof(TYPE)); _ArrayMoveConstruct(_vector, tmp, _size); diff --git a/libs/Common/ListFIFO.h b/libs/Common/ListFIFO.h new file mode 100644 index 000000000..94a79dd58 --- /dev/null +++ b/libs/Common/ListFIFO.h @@ -0,0 +1,115 @@ +/* +* ListFIFO.h +* +* Copyright (c) 2014-2024 SEACAVE +* +* Author(s): +* +* cDc +* +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* +* Additional Terms: +* +* You are required to preserve legal notices and author attributions in +* that material or in the Appropriate Legal Notices displayed by works +* containing it. +*/ + +#pragma once +#ifndef _MVS_LISTFIFO_H_ +#define _MVS_LISTFIFO_H_ + + +// I N C L U D E S ///////////////////////////////////////////////// + +#include +#include + + +// S T R U C T S /////////////////////////////////////////////////// + +namespace SEACAVE { + +// Tracks accesses of some hash-able type T and records the least recently accessed. +template +class ListFIFO { +public: + // add or move an key to the front + void Put(const T& key) { + const auto it = map.find(key); + if (it != map.end()) { + // if key exists, remove it from its current position + order.erase(it->second); + } + // add the key to the front + order.push_front(key); + map[key] = order.begin(); + } + + // remove and return the least used key (from the back) + T Pop() { + ASSERT(!IsEmpty()); + const T leastUsed = order.back(); + order.pop_back(); + map.erase(leastUsed); + return leastUsed; + } + + // return the least used key (from the back) + const T& Back() { + ASSERT(!IsEmpty()); + return order.back(); + } + + // check if the list is empty + bool IsEmpty() const { + return order.empty(); + } + + // get the size of the list + size_t Size() const { + return order.size(); + } + + // return true if the key is in the list + bool Contains(const T& key) const { + return map.find(key) != map.end(); + } + + // return the keys currently in cache + const std::list& GetCachedValues() const { + return order; + } + + // print the current order of elements + void PrintOrder() const { + std::cout << "Current order: "; + for (const auto& element : order) { + std::cout << element << " "; + } + std::cout << std::endl; + } + +private: + std::list order; + std::unordered_map::iterator> map; +}; +/*----------------------------------------------------------------*/ + +} // namespace SEACAVE + +#endif diff --git a/libs/Common/Types.h b/libs/Common/Types.h index 0aa955aa4..a0923b2b7 100644 --- a/libs/Common/Types.h +++ b/libs/Common/Types.h @@ -59,14 +59,17 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include #include +#include #ifdef _USE_OPENMP #include #endif @@ -1604,6 +1607,8 @@ class TDMatrix : public cv::Mat_ inline size_t elem_stride() const { ASSERT(dims == 2 && step[1] == sizeof(TYPE)); return step[1]; } /// Compute the area of the 2D matrix inline int area() const { ASSERT(dims == 2); return cols*rows; } + /// Compute the memory size of this matrix (in bytes) + inline size_t memory_size() const { return cv::Mat::total() * cv::Mat::elemSize(); } /// Is this coordinate inside the 2D matrix? template diff --git a/libs/Common/Types.inl b/libs/Common/Types.inl index b2d6ed292..360a7c0f1 100644 --- a/libs/Common/Types.inl +++ b/libs/Common/Types.inl @@ -13,7 +13,6 @@ namespace std { -//namespace tr1 { // Specializations for unordered containers template <> struct hash { @@ -23,7 +22,20 @@ template <> struct hash return std::hash()((const uint64_t&)v); } }; -//} // namespace tr1 + +// Adds the given key-value pair in the map, overwriting the current value if the key exists +template +void MapPut(std::map* map, const Key& key, const T& value) { + auto result = map->emplace(key, value); + if (!result.second) + result.first->second = value; +} +template +void MapPut(std::unordered_map* map, const Key& key, const T& value) { + auto result = map->emplace(key, value); + if (!result.second) + result.first->second = value; +} } // namespace std diff --git a/libs/Common/Util.cpp b/libs/Common/Util.cpp index 1ae22cb66..176d43508 100644 --- a/libs/Common/Util.cpp +++ b/libs/Common/Util.cpp @@ -264,48 +264,12 @@ String Util::GetCPUInfo() #endif return cpu; } -/*----------------------------------------------------------------*/ String Util::GetRAMInfo() { - #if defined(_MSC_VER) - - #ifdef _WIN64 - MEMORYSTATUSEX memoryStatus; - memset(&memoryStatus, sizeof(MEMORYSTATUSEX), 0); - memoryStatus.dwLength = sizeof(memoryStatus); - ::GlobalMemoryStatusEx(&memoryStatus); - const size_t nTotalPhys((size_t)memoryStatus.ullTotalPhys); - const size_t nTotalVirtual((size_t)memoryStatus.ullTotalVirtual); - #else - MEMORYSTATUS memoryStatus; - memset(&memoryStatus, sizeof(MEMORYSTATUS), 0); - memoryStatus.dwLength = sizeof(MEMORYSTATUS); - ::GlobalMemoryStatus(&memoryStatus); - const size_t nTotalPhys((size_t)memoryStatus.dwTotalPhys); - const size_t nTotalVirtual((size_t)memoryStatus.dwTotalVirtual); - #endif - - #elif defined(__APPLE__) - - int mib[2] = {CTL_HW, HW_MEMSIZE}; - const unsigned namelen = sizeof(mib) / sizeof(mib[0]); - size_t len = sizeof(size_t); - size_t nTotalPhys; - sysctl(mib, namelen, &nTotalPhys, &len, NULL, 0); - const size_t nTotalVirtual(nTotalPhys); - - #else // __GNUC__ - - struct sysinfo info; - sysinfo(&info); - const size_t nTotalPhys((size_t)info.totalram); - const size_t nTotalVirtual((size_t)info.totalswap); - - #endif // _MSC_VER - return formatBytes(nTotalPhys) + _T(" Physical Memory ") + formatBytes(nTotalVirtual) + _T(" Virtual Memory"); + const MemoryInfo memInfo(GetMemoryInfo()); + return formatBytes(memInfo.totalPhysical) + _T(" Physical Memory ") + formatBytes(memInfo.totalVirtual) + _T(" Virtual Memory"); } -/*----------------------------------------------------------------*/ String Util::GetOSInfo() { @@ -430,7 +394,6 @@ String Util::GetOSInfo() #endif // _MSC_VER } -/*----------------------------------------------------------------*/ String Util::GetDiskInfo(const String& path) { @@ -734,6 +697,62 @@ void Util::LogMemoryInfo() #endif // _PLATFORM_X86 + +// get the total & free physical & virtual memory (in bytes) +Util::MemoryInfo Util::GetMemoryInfo() +{ + #if defined(_MSC_VER) // windows + + #ifdef _WIN64 + MEMORYSTATUSEX status; + status.dwLength = sizeof(MEMORYSTATUSEX); + if (::GlobalMemoryStatusEx(&status) == FALSE) { + ASSERT(false); + return MemoryInfo(); + } + return MemoryInfo(status.ullTotalPhys, status.ullAvailPhys, status.ullTotalVirtual, status.ullAvailVirtual); + #else + MEMORYSTATUS status; + status.dwLength = sizeof(MEMORYSTATUS); + if (::GlobalMemoryStatus(&status) == FALSE) { + ASSERT(false); + return MemoryInfo(); + } + return MemoryInfo(status.dwTotalPhys, status.dwAvailPhys, status.dwTotalVirtual, status.dwAvailVirtual); + #endif + + + #elif defined(__APPLE__) // mac + + int mib[2] ={CTL_HW, HW_MEMSIZE}; + u_int namelen = sizeof(mib) / sizeof(mib[0]); + size_t len = sizeof(size_t); + size_t total_mem; + if (sysctl(mib, namelen, &total_mem, &len, NULL, 0) < 0) { + ASSERT(false); + return MemoryInfo(); + } + return MemoryInfo(total_mem); + + #else // __GNUC__ // linux + + struct sysinfo info; + if (sysinfo(&info) != 0) { + ASSERT(false); + return MemoryInfo(); + } + return MemoryInfo( + (size_t)info.totalram*(size_t)info.mem_unit, + (size_t)info.freeram*(size_t)info.mem_unit, + (size_t)info.totalswap*(size_t)info.mem_unit, + (size_t)info.freeswap*(size_t)info.mem_unit + ); + + #endif +} +/*----------------------------------------------------------------*/ + + // Parses a ASCII command line string and returns an array of pointers to the command line arguments, // along with a count of such arguments, in a way that is similar to the standard C run-time // argv and argc values. diff --git a/libs/Common/Util.h b/libs/Common/Util.h index fd313f903..0f6af868b 100644 --- a/libs/Common/Util.h +++ b/libs/Common/Util.h @@ -759,6 +759,16 @@ class GENERAL_API Util static void LogBuild(); static void LogMemoryInfo(); + struct MemoryInfo { + size_t totalPhysical; + size_t freePhysical; + size_t totalVirtual; + size_t freeVirtual; + MemoryInfo(size_t tP = 0, size_t fP = 0, size_t tV = 0, size_t fV = 0) + : totalPhysical(tP), freePhysical(fP), totalVirtual(tV), freeVirtual(fV) {} + }; + static MemoryInfo GetMemoryInfo(); + static LPSTR* CommandLineToArgvA(LPCSTR CmdLine, size_t& _argc); static String CommandLineToString(size_t argc, LPCTSTR* argv) { String strCmdLine; diff --git a/libs/MVS/Camera.h b/libs/MVS/Camera.h index edb9b00d4..8e4c186fe 100644 --- a/libs/MVS/Camera.h +++ b/libs/MVS/Camera.h @@ -437,6 +437,10 @@ class MVS_API Camera : public CameraIntern // compute the projection scale in this camera of the given world point template + inline TYPE GetFootprintImage(TYPE depth) const { + return static_cast(GetFocalLength() / depth); + } + template inline TYPE GetFootprintImage(const TPoint3& X) const { #if 0 const TYPE fSphereRadius(1); diff --git a/libs/MVS/DMapCache.cpp b/libs/MVS/DMapCache.cpp new file mode 100644 index 000000000..4c2f56ed1 --- /dev/null +++ b/libs/MVS/DMapCache.cpp @@ -0,0 +1,128 @@ +/* +* DMapCache.cpp +* +* Copyright (c) 2014-2024 SEACAVE +* +* Author(s): +* +* cDc +* +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* +* Additional Terms: +* +* You are required to preserve legal notices and author attributions in +* that material or in the Appropriate Legal Notices displayed by works +* containing it. +*/ + +#include "Common.h" +#include "DMapCache.h" + +using namespace MVS; + + +// S T R U C T S /////////////////////////////////////////////////// + +DMapCache::DMapCache(DepthDataArr& _arrDepthData, unsigned _loadFlags, size_t _max_memory_bytes) + : + loadFlags(_loadFlags), arrDepthData(_arrDepthData), + maxMemory(_max_memory_bytes), disabledMaxMemory(0), usedMemory(0), + skipMemoryCheckIdxImage(NO_ID), numImageRead(0) +{ +} + +void DMapCache::SetMaxMemory(size_t max_memory_bytes) { + maxMemory = max_memory_bytes; + ASSERT(skipMemoryCheckIdxImage == NO_ID); + Eject(); +} + +bool DMapCache::UseImage(IIndex idxImage) const { + ASSERT(idxImage < arrDepthData.size()); + std::lock_guard guard(mutex); + ASSERT(arrDepthData[idxImage].IsValid()); + if (!arrDepthData[idxImage].IsEmpty()) { + fifo.Put(idxImage); + return false; + } + mutex.unlock(); + const String fileName(ComposeDepthFilePath(arrDepthData[idxImage].GetView().GetID(), "dmap")); + while (!std::filesystem::is_regular_file(static_cast(fileName))) + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + arrDepthData[idxImage].Load(fileName, loadFlags); + ASSERT(!arrDepthData[idxImage].IsEmpty()); + mutex.lock(); + ++numImageRead; + usedMemory += arrDepthData[idxImage].GetMemorySize(); + fifo.Put(idxImage); + Eject(); + return true; +} + +IIndexArr DMapCache::GetCachedImageIndices(bool ordered) const { + std::lock_guard guard(mutex); + IIndexArr cachedImageIndices; + FOREACH(idxImage, arrDepthData) + if (!arrDepthData[idxImage].IsEmpty()) + cachedImageIndices.push_back(idxImage); + if (ordered) + cachedImageIndices.Sort(); + return cachedImageIndices; +} + +bool DMapCache::IsImageCached(IIndex idxImage) const { + return fifo.Contains(idxImage); +} + +void DMapCache::ClearCache() { + std::lock_guard guard(mutex); + skipMemoryCheckIdxImage = NO_ID; + while (!IsEmpty()) + EjectOldest(); +} + +size_t DMapCache::ComputeUsedMemory() const { + std::lock_guard guard(mutex); + size_t computedUsedMemory = 0; + for (const auto& depthData : arrDepthData) + if (!depthData.IsEmpty()) + computedUsedMemory += depthData.GetMemorySize(); + ASSERT(computedUsedMemory == usedMemory); + return computedUsedMemory; +} + +bool DMapCache::Eject() const { + if (maxMemory == 0) + return true; + while (usedMemory > maxMemory) { + if (!EjectOldest()) + return false; + } + return true; +} + +bool DMapCache::EjectOldest() const { + ASSERT(!fifo.IsEmpty()); + if (fifo.Back() == skipMemoryCheckIdxImage) + return false; + const IIndex idxImage = fifo.Pop(); + usedMemory -= arrDepthData[idxImage].GetMemorySize(); + // release the depth-data; no need to save the depth-data to disk as it is already saved + arrDepthData[idxImage].Release(); + return true; +} +/*----------------------------------------------------------------*/ diff --git a/libs/MVS/DMapCache.h b/libs/MVS/DMapCache.h new file mode 100644 index 000000000..04d5d52e4 --- /dev/null +++ b/libs/MVS/DMapCache.h @@ -0,0 +1,115 @@ +/* +* DMapCache.h +* +* Copyright (c) 2014-2024 SEACAVE +* +* Author(s): +* +* cDc +* +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU Affero General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU Affero General Public License for more details. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +* +* +* Additional Terms: +* +* You are required to preserve legal notices and author attributions in +* that material or in the Appropriate Legal Notices displayed by works +* containing it. +*/ + +#pragma once +#ifndef _MVS_DMAPCACHE_H_ +#define _MVS_DMAPCACHE_H_ + + +// I N C L U D E S ///////////////////////////////////////////////// + +#include "DepthMap.h" +#include "../Common/ListFIFO.h" + + +// S T R U C T S /////////////////////////////////////////////////// + +namespace MVS { + +// Caches depth-maps to disk. +class DMapCache { +public: + explicit DMapCache(DepthDataArr& arrDepthData, unsigned loadFlags, size_t max_memory_bytes); + + // check if the list is empty + bool IsEmpty() const { ASSERT((usedMemory == 0) == fifo.IsEmpty()) return fifo.IsEmpty(); } + + // set the maximum memory usage (in bytes) + void SetMaxMemory(size_t max_memory_bytes = 0/*unlimited*/); + + // enable/disable memory usage + void DisableMemoryCheck() { disabledMaxMemory = maxMemory; maxMemory = 0; } + void EnableMemoryCheck() { if (disabledMaxMemory) { maxMemory = disabledMaxMemory; disabledMaxMemory = 0; Eject(); } } + + // skip memory check if this image index is to be ejected + void SkipMemoryCheckIdxImage(IIndex idxImage = NO_ID) { skipMemoryCheckIdxImage = idxImage; } + + // ensure the depth-data is loaded and mark it as recently used: + // return true if the image was loaded from disk + bool UseImage(IIndex idxImage) const; + + // get the image indices loaded in cache. + IIndexArr GetCachedImageIndices(bool ordered = false) const; + + // return true if the key is in the cache + bool IsImageCached(IIndex idxImage) const; + + // get the number of times images were read from disk + uint32_t GetNumImageReads() const { return numImageRead; } + + // eject all images from the cache + void ClearCache(); + + // get the current memory usage (in bytes) + size_t GetUsedMemory() const { return usedMemory; } + size_t ComputeUsedMemory() const; + +private: + // eject the least recently used images if the cache size is above max-limit + bool Eject() const; + // eject the least recently used image + bool EjectOldest() const; + +private: + unsigned loadFlags; + DepthDataArr& arrDepthData; + + // maximum and used memory (in bytes) + size_t maxMemory, disabledMaxMemory; + mutable size_t usedMemory; + + // index of the image to skip memory check + IIndex skipMemoryCheckIdxImage; + + // guard access to variables that are dynamically loaded from disk + mutable std::mutex mutex; + + // track which images are last accessed + mutable ListFIFO fifo; + + // number of times images were read from disk (debug only) + mutable uint32_t numImageRead; +}; +/*----------------------------------------------------------------*/ + +} // namespace MVS + +#endif diff --git a/libs/MVS/DepthMap.cpp b/libs/MVS/DepthMap.cpp index 0a99d48b2..db39d19f5 100644 --- a/libs/MVS/DepthMap.cpp +++ b/libs/MVS/DepthMap.cpp @@ -129,6 +129,7 @@ DepthData::DepthData(const DepthData& srcDepthData) : confMap(srcDepthData.confMap), dMin(srcDepthData.dMin), dMax(srcDepthData.dMax), + size(srcDepthData.size), references(srcDepthData.references) {} @@ -261,6 +262,7 @@ bool DepthData::Load(const String& fileName, unsigned flags) return false; ASSERT(!IsValid() || (IDs.size() == images.size() && IDs.front() == GetView().GetID())); ASSERT(depthMap.size() == imageSize); + ASSERT(depthMap.size() == size); return true; } /*----------------------------------------------------------------*/ @@ -290,6 +292,23 @@ unsigned DepthData::DecRef() /*----------------------------------------------------------------*/ +// Compute the memory size occupied by the depth-data images (in bytes) +size_t MVS::DepthData::GetMemorySize() const +{ + if (IsEmpty()) + return 0; + size_t nBytes = depthMap.memory_size(); + if (!normalMap.empty()) + nBytes += normalMap.memory_size(); + if (!confMap.empty()) + nBytes += confMap.memory_size(); + if (!viewsMap.empty()) + nBytes += viewsMap.memory_size(); + return nBytes; +} +/*----------------------------------------------------------------*/ + + // S T R U C T S /////////////////////////////////////////////////// diff --git a/libs/MVS/DepthMap.h b/libs/MVS/DepthMap.h index 3bbd77c44..a5fabf2c8 100644 --- a/libs/MVS/DepthMap.h +++ b/libs/MVS/DepthMap.h @@ -214,6 +214,7 @@ struct MVS_API DepthData { ConfidenceMap confMap; // confidence-map ViewsMap viewsMap; // view-IDs map (indexing images vector starting after first view) float dMin, dMax; // global depth range for this image + cv::Size size; // image size used to estimate this depth-map unsigned references; // how many times this depth-map is referenced (on 0 can be safely unloaded) CriticalSection cs; // used to count references @@ -255,6 +256,8 @@ struct MVS_API DepthData { unsigned IncRef(const String& fileName); unsigned DecRef(); + size_t GetMemorySize() const; + #ifdef _USE_BOOST // implement BOOST serialization template diff --git a/libs/MVS/Scene.cpp b/libs/MVS/Scene.cpp index 0434395e0..5df98349f 100644 --- a/libs/MVS/Scene.cpp +++ b/libs/MVS/Scene.cpp @@ -845,15 +845,19 @@ bool Scene::SelectNeighborViews(uint32_t ID, IndexArr& points, unsigned nMinView ++nPoints; // score shared views const Point3f V1(imageData.camera.C - Cast(point)); - const float footprint1(imageData.camera.GetFootprintImage(point)); + const float footprint1(imageData.camera.GetFootprintImage(depth)); for (const PointCloud::View& view: views) { if (view == ID) continue; const Image& imageData2 = images[view]; + const Depth depth2((float)imageData2.camera.PointDepth(point)); + ASSERT(depth2 > 0); + if (depth2 <= 0) + continue; const Point3f V2(imageData2.camera.C - Cast(point)); const float fAngle(ACOS(ComputeAngle(V1.ptr(), V2.ptr()))); const float wAngle(EXP(SQUARE(fAngle-fOptimAngle)*(fAngle 1.6f) diff --git a/libs/MVS/SceneDensify.cpp b/libs/MVS/SceneDensify.cpp index 372e26057..f2950531b 100644 --- a/libs/MVS/SceneDensify.cpp +++ b/libs/MVS/SceneDensify.cpp @@ -33,6 +33,7 @@ #include "Scene.h" #include "SceneDensify.h" #include "PatchMatchCUDA.h" +#include "DMapCache.h" using namespace MVS; @@ -242,6 +243,7 @@ bool DepthMapsData::InitViews(DepthData& depthData, IIndex idxNeighbor, IIndex n viewRef.camera = viewRef.pImageData->camera; if (loadImages) viewRef.pImageData->image.toGray(viewRef.image, cv::COLOR_BGR2GRAY, true); + depthData.size = viewRef.pImageData->image.size(); // initialize views for (IIndex i=1; i& fusedDMaps, IIndex numDMapsReserveFusion, size_t currentCacheMemory = 0) +{ + size_t resolution(0); + IIndex numDMaps(0); + FOREACH(idxImage, arrDepthData) { + const DepthData& depthData = arrDepthData[idxImage]; + if (!depthData.IsValid()) + continue; + if (fusedDMaps.count(idxImage)) + continue; + resolution += depthData.size.area(); + if (++numDMaps >= numDMapsReserveFusion) + break; + } + if (numDMaps == 0) + return 0; + const Util::MemoryInfo memInfo(Util::GetMemoryInfo()); + const size_t neededPointCloudMemory(ROUND2INT(resolution * (1/*depth*/+1/*color*/+3/*normal*/+1/*confidence*/) * 4/*bytes*/ * 0.35/*unique pixels per depth-map*/)); + const size_t freeMemory(currentCacheMemory + memInfo.freePhysical); + const size_t safetyMemory(MAXF(ROUND2INT(memInfo.totalPhysical * 0.08), 1*1024*1024*1024ul/*1GB*/)); + const size_t neededMemory(neededPointCloudMemory + safetyMemory); + const size_t minDMapsMemory(resolution / numDMaps * 8/*min dmaps in memory*/ * (1/*depth*/ + 3/*normal*/ + 1/*confidence*/) * 4/*bytes*/); + if (freeMemory < neededMemory) { + DEBUG("warning: not enough memory to cache depth-maps (%luMB needed, %luMB available)", neededMemory/1024/1024, freeMemory/1024/1024); + return MINF(currentCacheMemory, minDMapsMemory); + } + return freeMemory - neededMemory; +} // GetAvailableMemory + +// finds the best depth-map to fuse next that maximizes the number of neighbors already in cache +std::tuple FetchBestNextDMapIndex(const DepthDataArr& arrDepthData, DMapCache& cacheDMaps, std::unordered_set& fusedDMaps) { + const IIndexArr cachedImages = cacheDMaps.GetCachedImageIndices(true); + IIndex bestImageIdx = NO_ID; + unsigned bestImageScore = 0, bestImageSize = std::numeric_limits::max(); + FOREACH(idxImage, arrDepthData) { + const DepthData& depthData = arrDepthData[idxImage]; + if (!depthData.IsValid()) + continue; + if (fusedDMaps.count(idxImage)) + continue; + ASSERT(!depthData.neighbors.empty()); + IIndexArr cachedNeighbors; + if (!cachedImages.empty()) { + IIndexArr neighbors(0, depthData.neighbors.size()); + for (ViewScore& neighbor: depthData.neighbors) + neighbors.push_back(neighbor.ID); + neighbors.Sort(); + std::set_intersection(neighbors.begin(), neighbors.end(), + cachedImages.begin(), cachedImages.end(), + std::back_inserter(cachedNeighbors)); + } + if (bestImageScore < cachedNeighbors.size() || + (bestImageScore == cachedNeighbors.size() && bestImageSize > depthData.neighbors.size())) { + bestImageScore = cachedNeighbors.size(); + bestImageSize = depthData.neighbors.size(); + bestImageIdx = idxImage; + } + } + return std::make_tuple(bestImageIdx, bestImageScore, static_cast(cachedImages.size())); +} // FetchBestNextDMapIndex + // fuse all valid depth-maps in the same 3D point cloud; // join points very likely to represent the same 3D point and // filter out points blocking the view @@ -1257,121 +1378,81 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b typedef SEACAVE::cList ProjArr; typedef SEACAVE::cList ProjsArr; - // find best connected images - IndexScoreArr connections(scene.images.size()); - size_t nPointsEstimate(0); - bool bNormalMap(true); - #ifdef DENSE_USE_OPENMP - bool bAbort(false); - #pragma omp parallel for shared(connections, nPointsEstimate, bNormalMap, bAbort) - for (int64_t i=0; i<(int64_t)scene.images.size(); ++i) { - #pragma omp flush (bAbort) - if (bAbort) - continue; - const IIndex idxImage((IIndex)i); - #else - FOREACH(idxImage, scene.images) { - #endif - IndexScore& connection = connections[idxImage]; - DepthData& depthData = arrDepthData[idxImage]; - if (!depthData.IsValid()) { - connection.idx = NO_ID; - connection.score = 0; - continue; - } - const String fileName(ComposeDepthFilePath(depthData.GetView().GetID(), "dmap")); - if (depthData.IncRef(fileName) == 0) { - #ifdef DENSE_USE_OPENMP - bAbort = true; - #pragma omp flush (bAbort) - continue; - #else - return; - #endif - } - ASSERT(!depthData.IsEmpty()); - connection.idx = idxImage; - connection.score = (float)scene.images[idxImage].neighbors.size(); - if (bEstimateNormal && depthData.normalMap.empty()) { - EstimateNormalMap(depthData.images.front().camera.K, depthData.depthMap, depthData.normalMap); - if (!depthData.Save(fileName)) { - #ifdef DENSE_USE_OPENMP - bAbort = true; - #pragma omp flush (bAbort) - continue; - #else - return; - #endif - } - } - #ifdef DENSE_USE_OPENMP - #pragma omp critical - #endif - { - nPointsEstimate += ROUND2INT(depthData.depthMap.area()*(0.5f/*valid*/*0.3f/*new*/)); - if (depthData.normalMap.empty()) - bNormalMap = false; - } - } - #ifdef DENSE_USE_OPENMP - if (bAbort) - return; - #endif - connections.Sort(); - while (!connections.empty() && connections.back().score <= 0) - connections.pop_back(); - if (connections.empty()) { - DEBUG("error: no valid depth-maps found"); - return; - } - // fuse all depth-maps, processing the best connected images first const unsigned nMinViewsFuse(MINF(OPTDENSE::nMinViewsFuse, scene.images.size())); const float normalError(COS(FD2R(OPTDENSE::fNormalDiffThreshold))); + const IIndex numDMapsReserveFusion(10); CLISTDEF0(Depth*) invalidDepths(0, 32); size_t nDepths(0); typedef TImage DepthIndex; typedef cList DepthIndexArr; DepthIndexArr arrDepthIdx(scene.images.size()); + const size_t nPointsEstimate(scene.images.size() * 9000); //TODO: better estimate number of points ProjsArr projs(0, nPointsEstimate); - if (bEstimateNormal && !bNormalMap) - bEstimateNormal = false; pointcloud.points.reserve(nPointsEstimate); pointcloud.pointViews.reserve(nPointsEstimate); pointcloud.pointWeights.reserve(nPointsEstimate); + unsigned depthDataLoadFlags(HeaderDepthDataRaw::HAS_DEPTH | HeaderDepthDataRaw::HAS_CONF); if (bEstimateColor) pointcloud.colors.reserve(nPointsEstimate); - if (bEstimateNormal) + if (bEstimateNormal) { pointcloud.normals.reserve(nPointsEstimate); - Util::Progress progress(_T("Fused depth-maps"), connections.size()); + depthDataLoadFlags |= HeaderDepthDataRaw::HAS_NORMAL; + } + Util::Progress progress(_T("Fused depth-maps"), scene.images.size()); GET_LOGCONSOLE().Pause(); - for (const IndexScore& connection: connections) { + std::unordered_set fusedDMaps; + DMapCache cacheDMaps(arrDepthData, depthDataLoadFlags, GetAvailableMemory(arrDepthData, fusedDMaps, numDMapsReserveFusion)); + unsigned totalNumImageNeighborsInCache = 0, totalNumImagesInCache = 0; + while (fusedDMaps.size() < arrDepthData.size()) { TD_TIMER_STARTD(); - const uint32_t idxImage(connection.idx); + // find the best depth-map to fuse next as the one with the most neighbors already in cache + const auto [idxImage, numImageNeighborsInCache, numImagesInCache] = FetchBestNextDMapIndex(arrDepthData, cacheDMaps, fusedDMaps); + if (idxImage == NO_ID) + break; // no more depth-maps to fuse (only invalid depth-maps left) + totalNumImageNeighborsInCache += numImageNeighborsInCache; + totalNumImagesInCache += numImagesInCache; + fusedDMaps.emplace(idxImage); + // fuse depth-map + cacheDMaps.UseImage(idxImage); + cacheDMaps.SkipMemoryCheckIdxImage(idxImage); const DepthData& depthData(arrDepthData[idxImage]); + ASSERT(depthData.GetView().GetLocalID(scene.images) == idxImage); + ASSERT(!depthData.IsEmpty()); + if (bEstimateNormal && depthData.normalMap.empty()) + EstimateNormalMaps(); ASSERT(!depthData.images.empty() && !depthData.neighbors.empty()); + #ifdef DENSE_USE_OPENMP + #pragma omp parallel for + for (int64_t i=0; i<(int64_t)depthData.neighbors.size(); ++i) { + const ViewScore& neighbor = depthData.neighbors[(IIndex)i]; + #else for (const ViewScore& neighbor: depthData.neighbors) { - DepthIndex& depthIdxs = arrDepthIdx[neighbor.ID]; - if (!depthIdxs.empty()) - continue; + #endif const DepthData& depthDataB(arrDepthData[neighbor.ID]); + if (!depthDataB.IsValid()) + continue; + cacheDMaps.UseImage(neighbor.ID); if (depthDataB.IsEmpty()) continue; + DepthIndex& depthIdxs = arrDepthIdx[neighbor.ID]; + if (!depthIdxs.empty()) + continue; depthIdxs.create(depthDataB.depthMap.size()); depthIdxs.memset((uint8_t)NO_ID); } ASSERT(!depthData.IsEmpty()); - const Image8U::Size sizeMap(depthData.depthMap.size()); + ASSERT(depthData.depthMap.size() == depthData.size); const Image& imageData = *depthData.images.front().pImageData; ASSERT(&imageData-scene.images.data() == idxImage); DepthIndex& depthIdxs = arrDepthIdx[idxImage]; if (depthIdxs.empty()) { - depthIdxs.create(Image8U::Size(imageData.width, imageData.height)); + depthIdxs.create(imageData.GetSize()); depthIdxs.memset((uint8_t)NO_ID); } const size_t nNumPointsPrev(pointcloud.points.size()); - for (int i=0; i(imageData.camera.R.t()*Cast(depthData.normalMap(x))) : Normal(0,0,-1)); + const PointCloud::Normal normal(!depthData.normalMap.empty() ? Cast(imageData.camera.R.t() * Cast(depthData.normalMap(x))) : Normal(0, 0, -1)); ASSERT(ISEQUAL(norm(normal), 1.f)); // check the projection in the neighbor depth-maps Point3 X(point*confidence); @@ -1419,7 +1500,7 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b continue; if (IsDepthSimilar(pt.z, depthB, OPTDENSE::fDepthDiffThreshold)) { // check if normals agree - const PointCloud::Normal normalB(bNormalMap ? Cast(imageDataB.camera.R.t()*Cast(depthDataB.normalMap(xB))) : Normal(0,0,-1)); + const PointCloud::Normal normalB(!depthData.normalMap.empty() ? Cast(imageDataB.camera.R.t() * Cast(depthDataB.normalMap(xB))) : Normal(0, 0, -1)); ASSERT(ISEQUAL(norm(normalB), 1.f)); if (normal.dot(normalB) > normalError) { // add view to the 3D point @@ -1471,15 +1552,23 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b } } ASSERT(pointcloud.points.size() == pointcloud.pointViews.size() && pointcloud.points.size() == pointcloud.pointWeights.size() && pointcloud.points.size() == projs.size()); - DEBUG_ULTIMATE("Depths map for reference image %3u fused using %u depths maps: %u new points (%s)", idxImage, depthData.images.size()-1, pointcloud.points.size()-nNumPointsPrev, TD_TIMER_GET_FMT().c_str()); - progress.display(&connection-connections.data()); + DEBUG_ULTIMATE("Depths map for reference image %3u fused using %u depths maps: %u new points, %u/%u cached images (%s)", + idxImage, depthData.images.size()-1, pointcloud.points.size()-nNumPointsPrev, numImageNeighborsInCache, numImagesInCache, TD_TIMER_GET_FMT().c_str()); + progress.display(fusedDMaps.size()); + // ensure enough memory is available for the next depth-maps chunk + cacheDMaps.SkipMemoryCheckIdxImage(); + if (fusedDMaps.size() % numDMapsReserveFusion == 0) + cacheDMaps.SetMaxMemory(GetAvailableMemory(arrDepthData, fusedDMaps, numDMapsReserveFusion, cacheDMaps.GetUsedMemory())); } GET_LOGCONSOLE().Play(); progress.close(); arrDepthIdx.Release(); + cacheDMaps.ClearCache(); - DEBUG_EXTRA("Depth-maps fused and filtered: %u depth-maps, %u depths, %u points (%d%%%%) (%s)", - connections.size(), nDepths, pointcloud.points.size(), ROUND2INT((100.f*pointcloud.points.size())/nDepths), TD_TIMER_GET_FMT().c_str()); + DEBUG_EXTRA("Depth-maps fused and filtered: %u depth-maps, %u depths, %u points (%d%%%%), %.2f hits in %.2f cached (%s)", + fusedDMaps.size(), nDepths, pointcloud.points.size(), ROUND2INT((100.f*pointcloud.points.size())/nDepths), + static_cast(totalNumImageNeighborsInCache) / fusedDMaps.size(), + static_cast(totalNumImagesInCache) / fusedDMaps.size(), TD_TIMER_GET_FMT().c_str()); if (bEstimateNormal && !pointcloud.points.empty() && pointcloud.normals.empty()) { // estimate normal also if requested (quite expensive if normal-maps not available) @@ -1507,11 +1596,6 @@ void DepthMapsData::FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, b } DEBUG_EXTRA("Normals estimated for the dense point-cloud: %u normals (%s)", pointcloud.GetSize(), TD_TIMER_GET_FMT().c_str()); } - - // release all depth-maps - for (DepthData& depthData: arrDepthData) - if (depthData.IsValid()) - depthData.DecRef(); } // FuseDepthMaps /*----------------------------------------------------------------*/ diff --git a/libs/MVS/SceneDensify.h b/libs/MVS/SceneDensify.h index 024d0f1eb..74f29764a 100644 --- a/libs/MVS/SceneDensify.h +++ b/libs/MVS/SceneDensify.h @@ -63,6 +63,8 @@ class MVS_API DepthMapsData bool RemoveSmallSegments(DepthData& depthData); bool GapInterpolation(DepthData& depthData); + void EstimateNormalMaps(); + bool FilterDepthMap(DepthData& depthData, const IIndexArr& idxNeighbors, bool bAdjust=true); void MergeDepthMaps(PointCloud& pointcloud, bool bEstimateColor, bool bEstimateNormal); void FuseDepthMaps(PointCloud& pointcloud, bool bEstimateColor, bool bEstimateNormal);