diff --git a/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp index ead72e2a5e..a967433cf4 100644 --- a/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS1IOHandler.hpp @@ -50,7 +50,7 @@ class OPENPMDAPI_EXPORT ADIOS1IOHandler : public AbstractIOHandler return "ADIOS1"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; void enqueue(IOTask const &) override; @@ -72,7 +72,7 @@ class OPENPMDAPI_EXPORT ADIOS1IOHandler : public AbstractIOHandler return "DUMMY_ADIOS1"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; private: std::unique_ptr m_impl; diff --git a/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp b/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp index 27bbdbcf59..42305bc78c 100644 --- a/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS1IOHandlerImpl.hpp @@ -51,7 +51,7 @@ class OPENPMDAPI_EXPORT ADIOS1IOHandlerImpl virtual void init(); - std::future flush() override; + std::future flush(); virtual int64_t open_write(Writable *); virtual ADIOS_FILE *open_read(std::string const &name); diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index c3a91070e2..8b94d3b51b 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -142,7 +142,7 @@ class ADIOS2IOHandlerImpl ~ADIOS2IOHandlerImpl() override; - std::future flush() override; + std::future flush(internal::FlushParams const &); void createFile(Writable *, Parameter const &) override; @@ -1262,7 +1262,7 @@ class ADIOS2IOHandler : public AbstractIOHandler // we must not throw in a destructor try { - this->flush(); + this->flush(internal::defaultFlushParams); } catch (std::exception const &ex) { @@ -1301,6 +1301,6 @@ class ADIOS2IOHandler : public AbstractIOHandler return "ADIOS2"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; }; // ADIOS2IOHandler } // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp index c56440c4fe..eefac95854 100644 --- a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandler.hpp @@ -54,7 +54,7 @@ class OPENPMDAPI_EXPORT ParallelADIOS1IOHandler : public AbstractIOHandler return "MPI_ADIOS1"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; #if openPMD_HAVE_ADIOS1 void enqueue(IOTask const &) override; #endif diff --git a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp index e0c7504c90..0b1bb2ca34 100644 --- a/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp +++ b/include/openPMD/IO/ADIOS/ParallelADIOS1IOHandlerImpl.hpp @@ -53,7 +53,7 @@ class OPENPMDAPI_EXPORT ParallelADIOS1IOHandlerImpl virtual void init(); - std::future flush() override; + std::future flush(); virtual int64_t open_write(Writable *); virtual ADIOS_FILE *open_read(std::string const &name); diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index dafa896a76..ff8b2fcccb 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -87,6 +87,24 @@ enum class FlushLevel : unsigned char SkeletonOnly }; +namespace internal +{ + /** + * Parameters recursively passed through the openPMD hierarchy when + * flushing. + * + */ + struct FlushParams + { + FlushLevel flushLevel = FlushLevel::InternalFlush; + }; + + /* + * To be used for reading + */ + constexpr FlushParams defaultFlushParams{}; +} // namespace internal + /** Interface for communicating between logical and physically persistent data. * * Input and output operations are channeled through a task queue that is @@ -123,7 +141,7 @@ class AbstractIOHandler * @return Future indicating the completion state of the operation for * backends that decide to implement this operation asynchronously. */ - virtual std::future flush() = 0; + virtual std::future flush(internal::FlushParams const &) = 0; /** The currently used backend */ virtual std::string backendName() const = 0; @@ -132,7 +150,6 @@ class AbstractIOHandler Access const m_backendAccess; Access const m_frontendAccess; std::queue m_work; - FlushLevel m_flushLevel = FlushLevel::InternalFlush; }; // AbstractIOHandler } // namespace openPMD diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index 01aeb4e9fc..170cf4b81a 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -40,7 +40,7 @@ class AbstractIOHandlerImpl virtual ~AbstractIOHandlerImpl() = default; - virtual std::future flush() + std::future flush() { using namespace auxiliary; diff --git a/include/openPMD/IO/DummyIOHandler.hpp b/include/openPMD/IO/DummyIOHandler.hpp index 8a84bb0919..9a4f3c3852 100644 --- a/include/openPMD/IO/DummyIOHandler.hpp +++ b/include/openPMD/IO/DummyIOHandler.hpp @@ -44,6 +44,6 @@ class DummyIOHandler : public AbstractIOHandler /** No-op consistent with the IOHandler interface to enable library use * without IO. */ - std::future flush() override; + std::future flush(internal::FlushParams const &) override; }; // DummyIOHandler } // namespace openPMD diff --git a/include/openPMD/IO/HDF5/HDF5IOHandler.hpp b/include/openPMD/IO/HDF5/HDF5IOHandler.hpp index 9c2433a7da..85d6ab9d40 100644 --- a/include/openPMD/IO/HDF5/HDF5IOHandler.hpp +++ b/include/openPMD/IO/HDF5/HDF5IOHandler.hpp @@ -42,7 +42,7 @@ class HDF5IOHandler : public AbstractIOHandler return "HDF5"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; private: std::unique_ptr m_impl; diff --git a/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp b/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp index 512e3edbb2..e1c8d52257 100644 --- a/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp +++ b/include/openPMD/IO/HDF5/ParallelHDF5IOHandler.hpp @@ -48,7 +48,7 @@ class ParallelHDF5IOHandler : public AbstractIOHandler return "MPI_HDF5"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; private: std::unique_ptr m_impl; diff --git a/include/openPMD/IO/JSON/JSONIOHandler.hpp b/include/openPMD/IO/JSON/JSONIOHandler.hpp index 1c1302bb55..37b00fa165 100644 --- a/include/openPMD/IO/JSON/JSONIOHandler.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandler.hpp @@ -38,7 +38,7 @@ class JSONIOHandler : public AbstractIOHandler return "JSON"; } - std::future flush() override; + std::future flush(internal::FlushParams const &) override; private: JSONIOHandlerImpl m_impl; diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 20dfd441da..d090d0b687 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -210,7 +210,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl void listAttributes(Writable *, Parameter &) override; - std::future flush() override; + std::future flush(); private: using FILEHANDLE = std::fstream; diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index 246ae195e5..3d8ccb4567 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -242,10 +242,11 @@ class Iteration : public Attributable return *m_iterationData; } - void flushFileBased(std::string const &, uint64_t); - void flushGroupBased(uint64_t); - void flushVariableBased(uint64_t); - void flush(); + void flushFileBased( + std::string const &, uint64_t, internal::FlushParams const &); + void flushGroupBased(uint64_t, internal::FlushParams const &); + void flushVariableBased(uint64_t, internal::FlushParams const &); + void flush(internal::FlushParams const &); void deferParseAccess(internal::DeferredParseAccess); /* * Control flow for read(), readFileBased(), readGroupBased() and diff --git a/include/openPMD/Mesh.hpp b/include/openPMD/Mesh.hpp index 1dac4760ee..17ce9373de 100644 --- a/include/openPMD/Mesh.hpp +++ b/include/openPMD/Mesh.hpp @@ -228,7 +228,8 @@ class Mesh : public BaseRecord private: Mesh(); - void flush_impl(std::string const &) override; + void + flush_impl(std::string const &, internal::FlushParams const &) override; void read() override; }; // Mesh diff --git a/include/openPMD/ParticleSpecies.hpp b/include/openPMD/ParticleSpecies.hpp index fc80960ca5..0257cd474f 100644 --- a/include/openPMD/ParticleSpecies.hpp +++ b/include/openPMD/ParticleSpecies.hpp @@ -43,7 +43,7 @@ class ParticleSpecies : public Container ParticleSpecies(); void read(); - void flush(std::string const &) override; + void flush(std::string const &, internal::FlushParams const &) override; /** * @brief Check recursively whether this ParticleSpecies is dirty. diff --git a/include/openPMD/Record.hpp b/include/openPMD/Record.hpp index 10bf0a5666..4f7ee51c28 100644 --- a/include/openPMD/Record.hpp +++ b/include/openPMD/Record.hpp @@ -50,7 +50,8 @@ class Record : public BaseRecord private: Record(); - void flush_impl(std::string const &) override; + void + flush_impl(std::string const &, internal::FlushParams const &) override; void read() override; }; // Record diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index 70ea8281f3..d30f7684f2 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -289,7 +289,7 @@ class RecordComponent : public BaseRecordComponent static constexpr char const *const SCALAR = "\vScalar"; private: - void flush(std::string const &); + void flush(std::string const &, internal::FlushParams const &); virtual void read(); /** diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index e73b59a5ae..7e757fd7fd 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -280,7 +280,7 @@ RecordComponent::storeChunk( Offset o, Extent e, F && createBuffer ) * Flush the openPMD hierarchy to the backend without flushing any actual * data yet. */ - seriesFlush( FlushLevel::SkeletonOnly ); + seriesFlush({FlushLevel::SkeletonOnly}); size_t size = 1; for( auto ext : e ) @@ -305,16 +305,16 @@ RecordComponent::storeChunk( Offset o, Extent e, F && createBuffer ) getBufferView.offset = o; getBufferView.extent = e; getBufferView.dtype = getDatatype(); - IOHandler()->enqueue( IOTask( this, getBufferView ) ); - IOHandler()->flush(); + IOHandler()->enqueue(IOTask(this, getBufferView)); + IOHandler()->flush(internal::defaultFlushParams); auto &out = *getBufferView.out; - if( !out.backendManagedBuffer ) + if (!out.backendManagedBuffer) { - auto data = std::forward< F >( createBuffer )( size ); - out.ptr = static_cast< void * >( data.get() ); - storeChunk( std::move( data ), std::move( o ), std::move( e ) ); + auto data = std::forward(createBuffer)(size); + out.ptr = static_cast(data.get()); + storeChunk(std::move(data), std::move(o), std::move(e)); } - return DynamicMemoryView< T >{ std::move( getBufferView ), size, *this }; + return DynamicMemoryView{std::move(getBufferView), size, *this}; } template< typename T > diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 527ef4538f..0649a9919b 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -532,16 +532,19 @@ OPENPMD_private * * @param begin Start of the range of iterations to flush. * @param end End of the range of iterations to flush. - * @param level Flush level, as documented in AbstractIOHandler.hpp. + * @param flushParams Flush params, as documented in AbstractIOHandler.hpp. * @param flushIOHandler Tasks will always be enqueued to the backend. * If this flag is true, tasks will be flushed to the backend. */ std::future flush_impl( iterations_iterator begin, iterations_iterator end, - FlushLevel level, + internal::FlushParams flushParams, bool flushIOHandler = true); - void flushFileBased(iterations_iterator begin, iterations_iterator end); + void flushFileBased( + iterations_iterator begin, + iterations_iterator end, + internal::FlushParams flushParams); /* * Group-based and variable-based iteration layouts share a lot of logic * (realistically, the variable-based iteration layout only throws out @@ -549,7 +552,10 @@ OPENPMD_private * As a convention, methods that deal with both layouts are called * .*GorVBased, short for .*GroupOrVariableBased */ - void flushGorVBased(iterations_iterator begin, iterations_iterator end); + void flushGorVBased( + iterations_iterator begin, + iterations_iterator end, + internal::FlushParams flushParams); void flushMeshesPath(); void flushParticlesPath(); void readFileBased(); diff --git a/include/openPMD/Span.hpp b/include/openPMD/Span.hpp index 48b24b02f9..93bc0d24f7 100644 --- a/include/openPMD/Span.hpp +++ b/include/openPMD/Span.hpp @@ -125,7 +125,7 @@ class DynamicMemoryView // might need to update m_recordComponent.IOHandler()->enqueue( IOTask(&m_recordComponent, m_param)); - m_recordComponent.IOHandler()->flush(); + m_recordComponent.IOHandler()->flush(internal::defaultFlushParams); } return Span{static_cast(m_param.out->ptr), m_size}; } diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index 8b71e41a6e..58648c8b2a 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -273,9 +273,9 @@ OPENPMD_protected Iteration &containingIteration(); /** @} */ - void seriesFlush(FlushLevel); + void seriesFlush(internal::FlushParams); - void flushAttributes(); + void flushAttributes(internal::FlushParams const &); enum ReadMode { /** diff --git a/include/openPMD/backend/BaseRecord.hpp b/include/openPMD/backend/BaseRecord.hpp index 80946374f8..65ae298da9 100644 --- a/include/openPMD/backend/BaseRecord.hpp +++ b/include/openPMD/backend/BaseRecord.hpp @@ -141,8 +141,9 @@ class BaseRecord : public Container void readBase(); private: - void flush(std::string const &) final; - virtual void flush_impl(std::string const &) = 0; + void flush(std::string const &, internal::FlushParams const &) final; + virtual void + flush_impl(std::string const &, internal::FlushParams const &) = 0; virtual void read() = 0; /** @@ -250,7 +251,7 @@ BaseRecord::erase(key_type const &key) Parameter dDelete; dDelete.name = "."; this->IOHandler()->enqueue(IOTask(&rc, dDelete)); - this->IOHandler()->flush(); + this->IOHandler()->flush(internal::defaultFlushParams); } res = Container::erase(key); } @@ -280,7 +281,7 @@ BaseRecord::erase(iterator res) Parameter dDelete; dDelete.name = "."; this->IOHandler()->enqueue(IOTask(&rc, dDelete)); - this->IOHandler()->flush(); + this->IOHandler()->flush(internal::defaultFlushParams); } ret = Container::erase(res); } @@ -315,7 +316,7 @@ inline void BaseRecord::readBase() aRead.name = "unitDimension"; this->IOHandler()->enqueue(IOTask(this, aRead)); - this->IOHandler()->flush(); + this->IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::ARR_DBL_7) this->setAttribute( "unitDimension", @@ -340,7 +341,7 @@ inline void BaseRecord::readBase() aRead.name = "timeOffset"; this->IOHandler()->enqueue(IOTask(this, aRead)); - this->IOHandler()->flush(); + this->IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::FLOAT) this->setAttribute( "timeOffset", Attribute(*aRead.resource).template get()); @@ -353,7 +354,8 @@ inline void BaseRecord::readBase() } template -inline void BaseRecord::flush(std::string const &name) +inline void BaseRecord::flush( + std::string const &name, internal::FlushParams const &flushParams) { if (!this->written() && this->empty()) throw std::runtime_error( @@ -361,7 +363,7 @@ inline void BaseRecord::flush(std::string const &name) "RecordComponents: " + name); - this->flush_impl(name); + this->flush_impl(name, flushParams); // flush_impl must take care to correctly set the dirty() flag so this // method doesn't do it } diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 4973c05e15..9dd163ecae 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -388,7 +388,7 @@ class Container : public Attributable Parameter pDelete; pDelete.path = "."; IOHandler()->enqueue(IOTask(&res->second, pDelete)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); } return container().erase(key); } @@ -405,7 +405,7 @@ class Container : public Attributable Parameter pDelete; pDelete.path = "."; IOHandler()->enqueue(IOTask(&res->second, pDelete)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); } return container().erase(res); } @@ -436,7 +436,8 @@ OPENPMD_protected container().clear(); } - virtual void flush(std::string const &path) + virtual void + flush(std::string const &path, internal::FlushParams const &flushParams) { if (!written()) { @@ -445,7 +446,7 @@ OPENPMD_protected IOHandler()->enqueue(IOTask(this, pCreate)); } - flushAttributes(); + flushAttributes(flushParams); } // clang-format off diff --git a/include/openPMD/backend/PatchRecord.hpp b/include/openPMD/backend/PatchRecord.hpp index 27ddb0ba96..84d180bac5 100644 --- a/include/openPMD/backend/PatchRecord.hpp +++ b/include/openPMD/backend/PatchRecord.hpp @@ -41,7 +41,8 @@ class PatchRecord : public BaseRecord private: PatchRecord() = default; - void flush_impl(std::string const &) override; + void + flush_impl(std::string const &, internal::FlushParams const &) override; void read() override; }; // PatchRecord } // namespace openPMD diff --git a/include/openPMD/backend/PatchRecordComponent.hpp b/include/openPMD/backend/PatchRecordComponent.hpp index 5f5ed91952..280b674ceb 100644 --- a/include/openPMD/backend/PatchRecordComponent.hpp +++ b/include/openPMD/backend/PatchRecordComponent.hpp @@ -91,7 +91,7 @@ class PatchRecordComponent : public BaseRecordComponent OPENPMD_private // clang-format on - void flush(std::string const &); + void flush(std::string const &, internal::FlushParams const &); void read(); /** diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp index 7006f3648d..8944d92dea 100644 --- a/include/openPMD/backend/Writable.hpp +++ b/include/openPMD/backend/Writable.hpp @@ -114,7 +114,7 @@ class Writable final OPENPMD_private // clang-format on - void seriesFlush(FlushLevel); + void seriesFlush(internal::FlushParams); /* * These members need to be shared pointers since distinct instances of * Writable may share them. diff --git a/src/IO/ADIOS/ADIOS1IOHandler.cpp b/src/IO/ADIOS/ADIOS1IOHandler.cpp index 3a6dc803d2..cde3b75156 100644 --- a/src/IO/ADIOS/ADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS1IOHandler.cpp @@ -330,7 +330,7 @@ ADIOS1IOHandler::ADIOS1IOHandler( ADIOS1IOHandler::~ADIOS1IOHandler() = default; -std::future ADIOS1IOHandler::flush() +std::future ADIOS1IOHandler::flush(internal::FlushParams const &) { return m_impl->flush(); } @@ -431,7 +431,7 @@ ADIOS1IOHandler::ADIOS1IOHandler(std::string path, Access at, json::TracingJSON) ADIOS1IOHandler::~ADIOS1IOHandler() = default; -std::future ADIOS1IOHandler::flush() +std::future ADIOS1IOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index b0e10f7a8e..5afa60628a 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -253,7 +253,8 @@ std::string ADIOS2IOHandlerImpl::fileSuffix() const } } -std::future ADIOS2IOHandlerImpl::flush() +std::future +ADIOS2IOHandlerImpl::flush(internal::FlushParams const &flushParams) { auto res = AbstractIOHandlerImpl::flush(); for (auto &p : m_fileData) @@ -261,7 +262,7 @@ std::future ADIOS2IOHandlerImpl::flush() if (m_dirty.find(p.first) != m_dirty.end()) { p.second->flush( - m_handler->m_flushLevel, /* writeAttributes = */ false); + flushParams.flushLevel, /* writeAttributes = */ false); } else { @@ -2869,9 +2870,10 @@ ADIOS2IOHandler::ADIOS2IOHandler( , m_impl{this, std::move(options), std::move(engineType)} {} -std::future ADIOS2IOHandler::flush() +std::future +ADIOS2IOHandler::flush(internal::FlushParams const &flushParams) { - return m_impl.flush(); + return m_impl.flush(flushParams); } #else // openPMD_HAVE_ADIOS2 @@ -2889,7 +2891,7 @@ ADIOS2IOHandler::ADIOS2IOHandler( : AbstractIOHandler(std::move(path), at) {} -std::future ADIOS2IOHandler::flush() +std::future ADIOS2IOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp index df6f817098..78ac9f4c59 100644 --- a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp @@ -349,7 +349,7 @@ ParallelADIOS1IOHandler::ParallelADIOS1IOHandler( ParallelADIOS1IOHandler::~ParallelADIOS1IOHandler() = default; -std::future ParallelADIOS1IOHandler::flush() +std::future ParallelADIOS1IOHandler::flush(internal::FlushParams const &) { return m_impl->flush(); } @@ -471,7 +471,7 @@ ParallelADIOS1IOHandler::ParallelADIOS1IOHandler( ParallelADIOS1IOHandler::~ParallelADIOS1IOHandler() = default; -std::future ParallelADIOS1IOHandler::flush() +std::future ParallelADIOS1IOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/DummyIOHandler.cpp b/src/IO/DummyIOHandler.cpp index f10cd50ac9..308f584ce4 100644 --- a/src/IO/DummyIOHandler.cpp +++ b/src/IO/DummyIOHandler.cpp @@ -32,7 +32,7 @@ DummyIOHandler::DummyIOHandler(std::string path, Access at) void DummyIOHandler::enqueue(IOTask const &) {} -std::future DummyIOHandler::flush() +std::future DummyIOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 0d855aa004..de541ab0f0 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -2292,7 +2292,7 @@ HDF5IOHandler::HDF5IOHandler( HDF5IOHandler::~HDF5IOHandler() = default; -std::future HDF5IOHandler::flush() +std::future HDF5IOHandler::flush(internal::FlushParams const &) { return m_impl->flush(); } @@ -2306,7 +2306,7 @@ HDF5IOHandler::HDF5IOHandler( HDF5IOHandler::~HDF5IOHandler() = default; -std::future HDF5IOHandler::flush() +std::future HDF5IOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/HDF5/ParallelHDF5IOHandler.cpp b/src/IO/HDF5/ParallelHDF5IOHandler.cpp index c0ce9e49e6..e0a17ed980 100644 --- a/src/IO/HDF5/ParallelHDF5IOHandler.cpp +++ b/src/IO/HDF5/ParallelHDF5IOHandler.cpp @@ -54,7 +54,7 @@ ParallelHDF5IOHandler::ParallelHDF5IOHandler( ParallelHDF5IOHandler::~ParallelHDF5IOHandler() = default; -std::future ParallelHDF5IOHandler::flush() +std::future ParallelHDF5IOHandler::flush(internal::FlushParams const &) { return m_impl->flush(); } @@ -196,7 +196,7 @@ ParallelHDF5IOHandler::ParallelHDF5IOHandler( ParallelHDF5IOHandler::~ParallelHDF5IOHandler() = default; -std::future ParallelHDF5IOHandler::flush() +std::future ParallelHDF5IOHandler::flush(internal::FlushParams const &) { return std::future(); } diff --git a/src/IO/JSON/JSONIOHandler.cpp b/src/IO/JSON/JSONIOHandler.cpp index 158c5454ed..15d18194c7 100644 --- a/src/IO/JSON/JSONIOHandler.cpp +++ b/src/IO/JSON/JSONIOHandler.cpp @@ -29,7 +29,7 @@ JSONIOHandler::JSONIOHandler(std::string path, Access at) : AbstractIOHandler{path, at}, m_impl{JSONIOHandlerImpl{this}} {} -std::future JSONIOHandler::flush() +std::future JSONIOHandler::flush(internal::FlushParams const &) { return m_impl.flush(); } diff --git a/src/Iteration.cpp b/src/Iteration.cpp index eb5e43d189..73c8a46847 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -127,7 +127,7 @@ Iteration &Iteration::close(bool _flush) auto end = begin; ++end; - s.flush_impl(begin, end, FlushLevel::UserFlush); + s.flush_impl(begin, end, {FlushLevel::UserFlush}); } } else @@ -154,7 +154,8 @@ Iteration &Iteration::open() // figure out my iteration number auto begin = s.indexOf(*this); s.openIteration(begin->first, *this); - IOHandler()->flush(); + // @todo, maybe collective here + IOHandler()->flush(internal::defaultFlushParams); return *this; } @@ -191,7 +192,10 @@ bool Iteration::closedByWriter() const } } -void Iteration::flushFileBased(std::string const &filename, uint64_t i) +void Iteration::flushFileBased( + std::string const &filename, + uint64_t i, + internal::FlushParams const &flushParams) { /* Find the root point [Series] of this file, * meshesPath and particlesPath are stored there */ @@ -224,7 +228,7 @@ void Iteration::flushFileBased(std::string const &filename, uint64_t i) fOpen.name = filename; fOpen.encoding = IterationEncoding::fileBased; IOHandler()->enqueue(IOTask(&s.writable(), fOpen)); - flush(); + flush(flushParams); return; } @@ -234,10 +238,11 @@ void Iteration::flushFileBased(std::string const &filename, uint64_t i) s.openIteration(i, *this); } - flush(); + flush(flushParams); } -void Iteration::flushGroupBased(uint64_t i) +void Iteration::flushGroupBased( + uint64_t i, internal::FlushParams const &flushParams) { if (!written()) { @@ -247,10 +252,11 @@ void Iteration::flushGroupBased(uint64_t i) IOHandler()->enqueue(IOTask(this, pCreate)); } - flush(); + flush(flushParams); } -void Iteration::flushVariableBased(uint64_t i) +void Iteration::flushVariableBased( + uint64_t i, internal::FlushParams const &flushParams) { if (!written()) { @@ -261,17 +267,17 @@ void Iteration::flushVariableBased(uint64_t i) this->setAttribute("snapshot", i); } - flush(); + flush(flushParams); } -void Iteration::flush() +void Iteration::flush(internal::FlushParams const &flushParams) { if (IOHandler()->m_frontendAccess == Access::READ_ONLY) { for (auto &m : meshes) - m.second.flush(m.first); + m.second.flush(m.first, flushParams); for (auto &species : particles) - species.second.flush(species.first); + species.second.flush(species.first, flushParams); } else { @@ -286,9 +292,9 @@ void Iteration::flush() s.setMeshesPath("meshes/"); s.flushMeshesPath(); } - meshes.flush(s.meshesPath()); + meshes.flush(s.meshesPath(), flushParams); for (auto &m : meshes) - m.second.flush(m.first); + m.second.flush(m.first, flushParams); } else { @@ -302,16 +308,16 @@ void Iteration::flush() s.setParticlesPath("particles/"); s.flushParticlesPath(); } - particles.flush(s.particlesPath()); + particles.flush(s.particlesPath(), flushParams); for (auto &species : particles) - species.second.flush(species.first); + species.second.flush(species.first, flushParams); } else { particles.dirty() = false; } - flushAttributes(); + flushAttributes(flushParams); } } @@ -379,7 +385,7 @@ void Iteration::read_impl(std::string const &groupPath) aRead.name = "dt"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::FLOAT) setDt(Attribute(*aRead.resource).get()); else if (*aRead.dtype == DT::DOUBLE) @@ -391,7 +397,7 @@ void Iteration::read_impl(std::string const &groupPath) aRead.name = "time"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::FLOAT) setTime(Attribute(*aRead.resource).get()); else if (*aRead.dtype == DT::DOUBLE) @@ -403,7 +409,7 @@ void Iteration::read_impl(std::string const &groupPath) aRead.name = "timeUnitSI"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::DOUBLE) setTimeUnitSI(Attribute(*aRead.resource).get()); else @@ -421,7 +427,7 @@ void Iteration::read_impl(std::string const &groupPath) if (version == "1.0.0" || version == "1.0.1") { IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); hasMeshes = std::count( pList.paths->begin(), pList.paths->end(), @@ -450,7 +456,7 @@ void Iteration::read_impl(std::string const &groupPath) /* obtain all non-scalar meshes */ IOHandler()->enqueue(IOTask(&meshes, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter aList; for (auto const &mesh_name : *pList.paths) @@ -460,7 +466,7 @@ void Iteration::read_impl(std::string const &groupPath) aList.attributes->clear(); IOHandler()->enqueue(IOTask(&m, pOpen)); IOHandler()->enqueue(IOTask(&m, aList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); auto att_begin = aList.attributes->begin(); auto att_end = aList.attributes->end(); @@ -471,7 +477,7 @@ void Iteration::read_impl(std::string const &groupPath) MeshRecordComponent &mrc = m[MeshRecordComponent::SCALAR]; mrc.parent() = m.parent(); IOHandler()->enqueue(IOTask(&mrc, pOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); mrc.get().m_isConstant = true; } m.read(); @@ -480,7 +486,7 @@ void Iteration::read_impl(std::string const &groupPath) /* obtain all scalar meshes */ Parameter dList; IOHandler()->enqueue(IOTask(&meshes, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &mesh_name : *dList.datasets) @@ -488,11 +494,11 @@ void Iteration::read_impl(std::string const &groupPath) Mesh &m = map[mesh_name]; dOpen.name = mesh_name; IOHandler()->enqueue(IOTask(&m, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); MeshRecordComponent &mrc = m[MeshRecordComponent::SCALAR]; mrc.parent() = m.parent(); IOHandler()->enqueue(IOTask(&mrc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); mrc.written() = false; mrc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); mrc.written() = true; @@ -514,7 +520,7 @@ void Iteration::read_impl(std::string const &groupPath) /* obtain all particle species */ pList.paths->clear(); IOHandler()->enqueue(IOTask(&particles, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); internal::EraseStaleEntries map{particles}; for (auto const &species_name : *pList.paths) @@ -522,7 +528,7 @@ void Iteration::read_impl(std::string const &groupPath) ParticleSpecies &p = map[species_name]; pOpen.path = species_name; IOHandler()->enqueue(IOTask(&p, pOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); p.read(); } } diff --git a/src/Mesh.cpp b/src/Mesh.cpp index bdb6081148..b30c7e0f5d 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -213,12 +213,13 @@ template Mesh &Mesh::setTimeOffset(double); template Mesh &Mesh::setTimeOffset(float); -void Mesh::flush_impl(std::string const &name) +void Mesh::flush_impl( + std::string const &name, internal::FlushParams const &flushParams) { if (IOHandler()->m_frontendAccess == Access::READ_ONLY) { for (auto &comp : *this) - comp.second.flush(comp.first); + comp.second.flush(comp.first, flushParams); } else { @@ -228,8 +229,8 @@ void Mesh::flush_impl(std::string const &name) { MeshRecordComponent &mrc = at(RecordComponent::SCALAR); mrc.parent() = parent(); - mrc.flush(name); - IOHandler()->flush(); + mrc.flush(name, flushParams); + IOHandler()->flush(flushParams); writable().abstractFilePosition = mrc.writable().abstractFilePosition; written() = true; @@ -248,7 +249,7 @@ void Mesh::flush_impl(std::string const &name) { for (auto &comp : *this) { - comp.second.flush(name); + comp.second.flush(name, flushParams); writable().abstractFilePosition = comp.second.writable().abstractFilePosition; } @@ -256,10 +257,10 @@ void Mesh::flush_impl(std::string const &name) else { for (auto &comp : *this) - comp.second.flush(comp.first); + comp.second.flush(comp.first, flushParams); } - flushAttributes(); + flushAttributes(flushParams); } } @@ -272,7 +273,7 @@ void Mesh::read() aRead.name = "geometry"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { std::string tmpGeometry = Attribute(*aRead.resource).get(); @@ -293,7 +294,7 @@ void Mesh::read() aRead.name = "dataOrder"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::CHAR) setDataOrder( static_cast(Attribute(*aRead.resource).get())); @@ -313,7 +314,7 @@ void Mesh::read() aRead.name = "axisLabels"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::VEC_STRING || *aRead.dtype == DT::STRING) setAxisLabels( Attribute(*aRead.resource).get >()); @@ -323,7 +324,7 @@ void Mesh::read() aRead.name = "gridSpacing"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Attribute a = Attribute(*aRead.resource); if (*aRead.dtype == DT::VEC_FLOAT || *aRead.dtype == DT::FLOAT) setGridSpacing(a.get >()); @@ -338,7 +339,7 @@ void Mesh::read() aRead.name = "gridGlobalOffset"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::VEC_DOUBLE || *aRead.dtype == DT::DOUBLE) setGridGlobalOffset( Attribute(*aRead.resource).get >()); @@ -348,7 +349,7 @@ void Mesh::read() aRead.name = "gridUnitSI"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::DOUBLE) setGridUnitSI(Attribute(*aRead.resource).get()); else @@ -364,7 +365,7 @@ void Mesh::read() { Parameter pList; IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter pOpen; for (auto const &component : *pList.paths) @@ -378,7 +379,7 @@ void Mesh::read() Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &component : *dList.datasets) @@ -386,7 +387,7 @@ void Mesh::read() MeshRecordComponent &rc = map[component]; dOpen.name = component; IOHandler()->enqueue(IOTask(&rc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); rc.written() = false; rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); rc.written() = true; diff --git a/src/ParticlePatches.cpp b/src/ParticlePatches.cpp index 952aff37ac..76017bbf94 100644 --- a/src/ParticlePatches.cpp +++ b/src/ParticlePatches.cpp @@ -35,7 +35,7 @@ void ParticlePatches::read() { Parameter pList; IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter pOpen; for (auto const &record_name : *pList.paths) @@ -48,7 +48,7 @@ void ParticlePatches::read() Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &component_name : *dList.datasets) @@ -65,7 +65,7 @@ void ParticlePatches::read() dOpen.name = component_name; IOHandler()->enqueue(IOTask(&pr, dOpen)); IOHandler()->enqueue(IOTask(&prc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (determineDatatype() != *dOpen.dtype) throw std::runtime_error( diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 8b00413e80..fdc4fa6f88 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -38,7 +38,7 @@ void ParticleSpecies::read() /* obtain all non-scalar records */ Parameter pList; IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); internal::EraseStaleEntries map{*this}; @@ -61,7 +61,7 @@ void ParticleSpecies::read() aList.attributes->clear(); IOHandler()->enqueue(IOTask(&r, pOpen)); IOHandler()->enqueue(IOTask(&r, aList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); auto att_begin = aList.attributes->begin(); auto att_end = aList.attributes->end(); @@ -73,7 +73,7 @@ void ParticleSpecies::read() RecordComponent &rc = scalarMap[RecordComponent::SCALAR]; rc.parent() = r.parent(); IOHandler()->enqueue(IOTask(&rc, pOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); rc.get().m_isConstant = true; } r.read(); @@ -90,7 +90,7 @@ void ParticleSpecies::read() /* obtain all scalar records */ Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &record_name : *dList.datasets) @@ -100,12 +100,12 @@ void ParticleSpecies::read() Record &r = map[record_name]; dOpen.name = record_name; IOHandler()->enqueue(IOTask(&r, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); internal::EraseStaleEntries scalarMap(r); RecordComponent &rc = scalarMap[RecordComponent::SCALAR]; rc.parent() = r.parent(); IOHandler()->enqueue(IOTask(&rc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); rc.written() = false; rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); rc.written() = true; @@ -138,14 +138,15 @@ namespace } } // namespace -void ParticleSpecies::flush(std::string const &path) +void ParticleSpecies::flush( + std::string const &path, internal::FlushParams const &flushParams) { if (IOHandler()->m_frontendAccess == Access::READ_ONLY) { for (auto &record : *this) - record.second.flush(record.first); + record.second.flush(record.first, flushParams); for (auto &patch : particlePatches) - patch.second.flush(patch.first); + patch.second.flush(patch.first, flushParams); } else { @@ -156,16 +157,16 @@ void ParticleSpecies::flush(std::string const &path) if (it != end()) it->second.setUnitDimension({{UnitDimension::L, 1}}); - Container::flush(path); + Container::flush(path, flushParams); for (auto &record : *this) - record.second.flush(record.first); + record.second.flush(record.first, flushParams); if (flushParticlePatches(particlePatches)) { - particlePatches.flush("particlePatches"); + particlePatches.flush("particlePatches", flushParams); for (auto &patch : particlePatches) - patch.second.flush(patch.first); + patch.second.flush(patch.first, flushParams); } } } diff --git a/src/Record.cpp b/src/Record.cpp index ce18bdccc5..da57ebeef1 100644 --- a/src/Record.cpp +++ b/src/Record.cpp @@ -43,12 +43,13 @@ Record &Record::setUnitDimension(std::map const &udim) return *this; } -void Record::flush_impl(std::string const &name) +void Record::flush_impl( + std::string const &name, internal::FlushParams const &flushParams) { if (IOHandler()->m_frontendAccess == Access::READ_ONLY) { for (auto &comp : *this) - comp.second.flush(comp.first); + comp.second.flush(comp.first, flushParams); } else { @@ -58,8 +59,8 @@ void Record::flush_impl(std::string const &name) { RecordComponent &rc = at(RecordComponent::SCALAR); rc.parent() = parent(); - rc.flush(name); - IOHandler()->flush(); + rc.flush(name, flushParams); + IOHandler()->flush(flushParams); writable().abstractFilePosition = rc.writable().abstractFilePosition; written() = true; @@ -78,7 +79,7 @@ void Record::flush_impl(std::string const &name) { for (auto &comp : *this) { - comp.second.flush(name); + comp.second.flush(name, flushParams); writable().abstractFilePosition = comp.second.writable().abstractFilePosition; } @@ -86,10 +87,10 @@ void Record::flush_impl(std::string const &name) else { for (auto &comp : *this) - comp.second.flush(comp.first); + comp.second.flush(comp.first, flushParams); } - flushAttributes(); + flushAttributes(flushParams); } } @@ -104,7 +105,7 @@ void Record::read() { Parameter pList; IOHandler()->enqueue(IOTask(this, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter pOpen; for (auto const &component : *pList.paths) @@ -118,7 +119,7 @@ void Record::read() Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &component : *dList.datasets) @@ -126,7 +127,7 @@ void Record::read() RecordComponent &rc = (*this)[component]; dOpen.name = component; IOHandler()->enqueue(IOTask(&rc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); rc.written() = false; rc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); rc.written() = true; diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index ad4ca07a83..c69c09e997 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -191,10 +191,11 @@ bool RecordComponent::empty() const return get().m_isEmpty; } -void RecordComponent::flush(std::string const &name) +void RecordComponent::flush( + std::string const &name, internal::FlushParams const &flushParams) { auto &rc = get(); - if (IOHandler()->m_flushLevel == FlushLevel::SkeletonOnly) + if (flushParams.flushLevel == FlushLevel::SkeletonOnly) { rc.m_name = name; return; @@ -273,7 +274,7 @@ void RecordComponent::flush(std::string const &name) rc.m_chunks.pop(); } - flushAttributes(); + flushAttributes(flushParams); } } @@ -292,7 +293,7 @@ void RecordComponent::readBase() { aRead.name = "value"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Attribute a(*aRead.resource); DT dtype = *aRead.dtype; @@ -357,7 +358,7 @@ void RecordComponent::readBase() aRead.name = "shape"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); a = Attribute(*aRead.resource); Extent e; @@ -383,7 +384,7 @@ void RecordComponent::readBase() aRead.name = "unitSI"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::DOUBLE) setUnitSI(Attribute(*aRead.resource).get()); else diff --git a/src/Series.cpp b/src/Series.cpp index 2913eeb9b2..68059e2af9 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -377,7 +377,7 @@ void Series::flush() flush_impl( series.iterations.begin(), series.iterations.end(), - FlushLevel::UserFlush); + {FlushLevel::UserFlush}); } std::unique_ptr Series::parseInput(std::string filepath) @@ -564,10 +564,9 @@ void Series::initDefaults(IterationEncoding ie) std::future Series::flush_impl( iterations_iterator begin, iterations_iterator end, - FlushLevel level, + internal::FlushParams flushParams, bool flushIOHandler) { - IOHandler()->m_flushLevel = level; auto &series = get(); series.m_lastFlushSuccessful = true; try @@ -576,34 +575,33 @@ std::future Series::flush_impl( { using IE = IterationEncoding; case IE::fileBased: - flushFileBased(begin, end); + flushFileBased(begin, end, flushParams); break; case IE::groupBased: case IE::variableBased: - flushGorVBased(begin, end); + flushGorVBased(begin, end, flushParams); break; } if (flushIOHandler) { - auto res = IOHandler()->flush(); - IOHandler()->m_flushLevel = FlushLevel::InternalFlush; - return res; + return IOHandler()->flush(flushParams); } else { - IOHandler()->m_flushLevel = FlushLevel::InternalFlush; return {}; } } catch (...) { - IOHandler()->m_flushLevel = FlushLevel::InternalFlush; series.m_lastFlushSuccessful = false; throw; } } -void Series::flushFileBased(iterations_iterator begin, iterations_iterator end) +void Series::flushFileBased( + iterations_iterator begin, + iterations_iterator end, + internal::FlushParams flushParams) { auto &series = get(); if (end == begin) @@ -618,7 +616,7 @@ void Series::flushFileBased(iterations_iterator begin, iterations_iterator end) { using IO = IterationOpened; case IO::HasBeenOpened: - it->second.flush(); + it->second.flush(flushParams); break; case IO::RemainsClosed: break; @@ -635,7 +633,7 @@ void Series::flushFileBased(iterations_iterator begin, iterations_iterator end) } // Phase 3 - IOHandler()->flush(); + IOHandler()->flush(flushParams); } else { @@ -656,12 +654,13 @@ void Series::flushFileBased(iterations_iterator begin, iterations_iterator end) dirty() |= it->second.dirty(); std::string filename = iterationFilename(it->first); - it->second.flushFileBased(filename, it->first); + it->second.flushFileBased(filename, it->first, flushParams); series.iterations.flush( - auxiliary::replace_first(basePath(), "%T/", "")); + auxiliary::replace_first(basePath(), "%T/", ""), + flushParams); - flushAttributes(); + flushAttributes(flushParams); break; } case IO::RemainsClosed: @@ -679,7 +678,7 @@ void Series::flushFileBased(iterations_iterator begin, iterations_iterator end) } // Phase 3 - IOHandler()->flush(); + IOHandler()->flush(flushParams); /* reset the dirty bit for every iteration (i.e. file) * otherwise only the first iteration will have updates attributes @@ -690,7 +689,10 @@ void Series::flushFileBased(iterations_iterator begin, iterations_iterator end) } } -void Series::flushGorVBased(iterations_iterator begin, iterations_iterator end) +void Series::flushGorVBased( + iterations_iterator begin, + iterations_iterator end, + internal::FlushParams flushParams) { auto &series = get(); if (IOHandler()->m_frontendAccess == Access::READ_ONLY) @@ -701,7 +703,7 @@ void Series::flushGorVBased(iterations_iterator begin, iterations_iterator end) { using IO = IterationOpened; case IO::HasBeenOpened: - it->second.flush(); + it->second.flush(flushParams); break; case IO::RemainsClosed: break; @@ -717,7 +719,7 @@ void Series::flushGorVBased(iterations_iterator begin, iterations_iterator end) } // Phase 3 - IOHandler()->flush(); + IOHandler()->flush(flushParams); } else { @@ -730,7 +732,7 @@ void Series::flushGorVBased(iterations_iterator begin, iterations_iterator end) } series.iterations.flush( - auxiliary::replace_first(basePath(), "%T/", "")); + auxiliary::replace_first(basePath(), "%T/", ""), flushParams); for (auto it = begin; it != end; ++it) { @@ -747,10 +749,10 @@ void Series::flushGorVBased(iterations_iterator begin, iterations_iterator end) { using IE = IterationEncoding; case IE::groupBased: - it->second.flushGroupBased(it->first); + it->second.flushGroupBased(it->first, flushParams); break; case IE::variableBased: - it->second.flushVariableBased(it->first); + it->second.flushVariableBased(it->first, flushParams); break; default: throw std::runtime_error( @@ -771,8 +773,8 @@ void Series::flushGorVBased(iterations_iterator begin, iterations_iterator end) } } - flushAttributes(); - IOHandler()->flush(); + flushAttributes(flushParams); + IOHandler()->flush(flushParams); } } @@ -846,7 +848,7 @@ void Series::readFileBased() iteration.runDeferredParseAccess(); Parameter fClose; iteration.IOHandler()->enqueue(IOTask(&iteration, fClose)); - iteration.IOHandler()->flush(); + iteration.IOHandler()->flush(internal::defaultFlushParams); iteration.get().m_closed = internal::CloseStatus::ClosedTemporarily; }; if (series.m_parseLazily) @@ -892,7 +894,7 @@ void Series::readOneIterationFileBased(std::string const &filePath) fOpen.name = filePath; IOHandler()->enqueue(IOTask(this, fOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); series.iterations.parent() = getWritable(this); readBase(); @@ -900,7 +902,7 @@ void Series::readOneIterationFileBased(std::string const &filePath) using DT = Datatype; aRead.name = "iterationEncoding"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { std::string encoding = Attribute(*aRead.resource).get(); @@ -937,7 +939,7 @@ void Series::readOneIterationFileBased(std::string const &filePath) aRead.name = "iterationFormat"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { written() = false; @@ -967,7 +969,7 @@ void Series::readGorVBased(bool do_init) fOpen.name = series.m_name; fOpen.encoding = iterationEncoding(); IOHandler()->enqueue(IOTask(this, fOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (do_init) { @@ -977,7 +979,7 @@ void Series::readGorVBased(bool do_init) Parameter aRead; aRead.name = "iterationEncoding"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { std::string encoding = @@ -1011,7 +1013,7 @@ void Series::readGorVBased(bool do_init) aRead.name = "iterationFormat"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { written() = false; @@ -1039,7 +1041,7 @@ void Series::readGorVBased(bool do_init) /* obtain all paths inside the basepath (i.e. all iterations) */ Parameter pList; IOHandler()->enqueue(IOTask(&series.iterations, pList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); auto readSingleIteration = [&series, &pOpen, this]( uint64_t index, @@ -1112,7 +1114,7 @@ void Series::readBase() aRead.name = "openPMD"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) setOpenPMD(Attribute(*aRead.resource).get()); else @@ -1120,7 +1122,7 @@ void Series::readBase() aRead.name = "openPMDextension"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == determineDatatype()) setOpenPMDextension(Attribute(*aRead.resource).get()); else @@ -1129,7 +1131,7 @@ void Series::readBase() aRead.name = "basePath"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) setAttribute("basePath", Attribute(*aRead.resource).get()); else @@ -1138,14 +1140,14 @@ void Series::readBase() Parameter aList; IOHandler()->enqueue(IOTask(this, aList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (std::count( aList.attributes->begin(), aList.attributes->end(), "meshesPath") == 1) { aRead.name = "meshesPath"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { /* allow setting the meshes path after completed IO */ @@ -1169,7 +1171,7 @@ void Series::readBase() { aRead.name = "particlesPath"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { /* allow setting the meshes path after completed IO */ @@ -1220,6 +1222,7 @@ AdvanceStatus Series::advance( iterations_iterator begin, Iteration &iteration) { + constexpr internal::FlushParams flushParams = {FlushLevel::UserFlush}; auto &series = get(); auto end = begin; ++end; @@ -1238,7 +1241,8 @@ AdvanceStatus Series::advance( itData.m_closed = internal::CloseStatus::Open; } - flush_impl(begin, end, FlushLevel::UserFlush, /* flushIOHandler = */ false); + // @todo really collective? + flush_impl(begin, end, flushParams, /* flushIOHandler = */ false); if (oldCloseStatus == internal::CloseStatus::ClosedInFrontend) { @@ -1309,17 +1313,7 @@ AdvanceStatus Series::advance( // We cannot call Series::flush now, since the IO handler is still filled // from calling flush(Group|File)based, but has not been emptied yet // Do that manually - IOHandler()->m_flushLevel = FlushLevel::UserFlush; - try - { - IOHandler()->flush(); - } - catch (...) - { - IOHandler()->m_flushLevel = FlushLevel::InternalFlush; - throw; - } - IOHandler()->m_flushLevel = FlushLevel::InternalFlush; + IOHandler()->flush(flushParams); return *param.status; } diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 52748e9806..d037ad68bc 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -66,7 +66,7 @@ bool Attributable::deleteAttribute(std::string const &key) Parameter aDelete; aDelete.name = key; IOHandler()->enqueue(IOTask(this, aDelete)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); attri.m_attributes.erase(it); return true; } @@ -206,14 +206,14 @@ auto Attributable::myPath() const -> MyPath return res; } -void Attributable::seriesFlush(FlushLevel level) +void Attributable::seriesFlush(internal::FlushParams flushParams) { - writable().seriesFlush(level); + writable().seriesFlush(flushParams); } -void Attributable::flushAttributes() +void Attributable::flushAttributes(internal::FlushParams const &flushParams) { - if (IOHandler()->m_flushLevel == FlushLevel::SkeletonOnly) + if (flushParams.flushLevel == FlushLevel::SkeletonOnly) { return; } @@ -237,7 +237,7 @@ void Attributable::readAttributes(ReadMode mode) auto &attri = get(); Parameter aList; IOHandler()->enqueue(IOTask(this, aList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); std::vector written_attributes = attributes(); /* std::set_difference requires sorted ranges */ @@ -277,7 +277,7 @@ void Attributable::readAttributes(ReadMode mode) IOHandler()->enqueue(IOTask(this, aRead)); try { - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); } catch (unsupported_data_error const &e) { diff --git a/src/backend/BaseRecordComponent.cpp b/src/backend/BaseRecordComponent.cpp index 1c86431e02..4460b7ede0 100644 --- a/src/backend/BaseRecordComponent.cpp +++ b/src/backend/BaseRecordComponent.cpp @@ -61,7 +61,7 @@ ChunkTable BaseRecordComponent::availableChunks() Parameter param; IOTask task(this, param); IOHandler()->enqueue(task); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); return std::move(*param.chunks); } diff --git a/src/backend/MeshRecordComponent.cpp b/src/backend/MeshRecordComponent.cpp index aa5afcce34..9602868a07 100644 --- a/src/backend/MeshRecordComponent.cpp +++ b/src/backend/MeshRecordComponent.cpp @@ -34,7 +34,7 @@ void MeshRecordComponent::read() aRead.name = "position"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Attribute a = Attribute(*aRead.resource); if (*aRead.dtype == DT::VEC_FLOAT || *aRead.dtype == DT::FLOAT) setPosition(a.get >()); diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 9c94241338..f31b135b62 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -36,19 +36,21 @@ PatchRecord::setUnitDimension(std::map const &udim) return *this; } -void PatchRecord::flush_impl(std::string const &path) +void PatchRecord::flush_impl( + std::string const &path, internal::FlushParams const &flushParams) { if (this->find(RecordComponent::SCALAR) == this->end()) { if (IOHandler()->m_frontendAccess != Access::READ_ONLY) Container::flush( - path); // warning (clang-tidy-10): bugprone-parent-virtual-call + path, flushParams); // warning (clang-tidy-10): + // bugprone-parent-virtual-call for (auto &comp : *this) - comp.second.flush(comp.first); + comp.second.flush(comp.first, flushParams); } else - this->operator[](RecordComponent::SCALAR).flush(path); - if (IOHandler()->m_flushLevel == FlushLevel::UserFlush) + this->operator[](RecordComponent::SCALAR).flush(path, flushParams); + if (flushParams.flushLevel == FlushLevel::UserFlush) { this->dirty() = false; } @@ -59,7 +61,7 @@ void PatchRecord::read() Parameter aRead; aRead.name = "unitDimension"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == Datatype::ARR_DBL_7 || *aRead.dtype == Datatype::VEC_DOUBLE) @@ -72,7 +74,7 @@ void PatchRecord::read() Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); Parameter dOpen; for (auto const &component_name : *dList.datasets) @@ -80,7 +82,7 @@ void PatchRecord::read() PatchRecordComponent &prc = (*this)[component_name]; dOpen.name = component_name; IOHandler()->enqueue(IOTask(&prc, dOpen)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); /* allow all attributes to be set */ prc.written() = false; prc.resetDataset(Dataset(*dOpen.dtype, *dOpen.extent)); diff --git a/src/backend/PatchRecordComponent.cpp b/src/backend/PatchRecordComponent.cpp index 6eb696f174..ecc44625a2 100644 --- a/src/backend/PatchRecordComponent.cpp +++ b/src/backend/PatchRecordComponent.cpp @@ -80,7 +80,8 @@ PatchRecordComponent::PatchRecordComponent( : BaseRecordComponent{data}, m_patchRecordComponentData{std::move(data)} {} -void PatchRecordComponent::flush(std::string const &name) +void PatchRecordComponent::flush( + std::string const &name, internal::FlushParams const &flushParams) { auto &rc = get(); if (IOHandler()->m_frontendAccess == Access::READ_ONLY) @@ -109,7 +110,7 @@ void PatchRecordComponent::flush(std::string const &name) rc.m_chunks.pop(); } - flushAttributes(); + flushAttributes(flushParams); } } @@ -119,7 +120,7 @@ void PatchRecordComponent::read() aRead.name = "unitSI"; IOHandler()->enqueue(IOTask(this, aRead)); - IOHandler()->flush(); + IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == Datatype::DOUBLE) setUnitSI(Attribute(*aRead.resource).get()); else diff --git a/src/backend/Writable.cpp b/src/backend/Writable.cpp index 7f3733904b..58ce2e0ad4 100644 --- a/src/backend/Writable.cpp +++ b/src/backend/Writable.cpp @@ -35,15 +35,15 @@ Writable::Writable(internal::AttributableData *a) void Writable::seriesFlush() { - seriesFlush(FlushLevel::UserFlush); + seriesFlush({FlushLevel::UserFlush}); } -void Writable::seriesFlush(FlushLevel level) +void Writable::seriesFlush(internal::FlushParams flushParams) { auto series = Attributable({attributable, [](auto const *) {}}).retrieveSeries(); series.flush_impl( - series.iterations.begin(), series.iterations.end(), level); + series.iterations.begin(), series.iterations.end(), flushParams); } } // namespace openPMD