diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index d4a199be53..d3ccc93258 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -150,6 +150,11 @@ class ADIOS2IOHandlerImpl void createFile(Writable *, Parameter const &) override; + void checkFile(Writable *, Parameter &) override; + + // MPI Collective + bool checkFile(std::string fullFilePath) const; + void createPath(Writable *, Parameter const &) override; @@ -226,6 +231,9 @@ class ADIOS2IOHandlerImpl private: adios2::ADIOS m_ADIOS; +#if openPMD_HAVE_MPI + std::optional m_communicator; +#endif /* * If the iteration encoding is variableBased, we default to using the * 2021_02_09 schema since it allows mutable attributes. @@ -329,7 +337,7 @@ class ADIOS2IOHandlerImpl // use m_config std::optional > getOperators(); - std::string fileSuffix() const; + std::string fileSuffix(bool verbose = true) const; /* * We need to give names to IO objects. These names are irrelevant diff --git a/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp b/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp index ea89f033d6..9f43b9ae18 100644 --- a/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp @@ -52,6 +52,7 @@ class CommonADIOS1IOHandlerImpl : public AbstractIOHandlerImpl public: void createFile(Writable *, Parameter const &) override; + void checkFile(Writable *, Parameter &) override; void createPath(Writable *, Parameter const &) override; void createDataset( diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index 518b9dac0a..55ff021d06 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -58,6 +58,12 @@ class AbstractIOHandlerImpl deref_dynamic_cast >( i.parameter.get())); break; + case O::CHECK_FILE: + checkFile( + i.writable, + deref_dynamic_cast >( + i.parameter.get())); + break; case O::CREATE_PATH: createPath( i.writable, @@ -220,6 +226,17 @@ class AbstractIOHandlerImpl virtual void closeFile(Writable *, Parameter const &) = 0; + /** + * Check if the file specified by the parameter is already present on disk. + * The Writable is irrelevant for this method. + * A backend can choose to ignore this task and specify FileExists::DontKnow + * in the out parameter. + * The consequence will be that some top-level attributes might be defined + * a second time when appending to an existing file, because the frontend + * cannot be sure that the file already has these attributes. + */ + virtual void checkFile(Writable *, Parameter &) = 0; + /** Advance the file/stream that this writable belongs to. * * If the backend is based around usage of IO steps (especially streaming diff --git a/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp b/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp index adee717ff0..7502e36e3f 100644 --- a/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp +++ b/include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp @@ -43,6 +43,7 @@ class HDF5IOHandlerImpl : public AbstractIOHandlerImpl void createFile(Writable *, Parameter const &) override; + void checkFile(Writable *, Parameter &) override; void createPath(Writable *, Parameter const &) override; void createDataset( @@ -92,6 +93,15 @@ class HDF5IOHandlerImpl : public AbstractIOHandlerImpl hid_t m_H5T_CDOUBLE; hid_t m_H5T_CLONG_DOUBLE; +protected: +#if openPMD_HAVE_MPI + /* + * Not defined in ParallelHDF5IOHandlerImpl, so we don't have to write + * some methods twice. + */ + std::optional m_communicator; +#endif + private: json::TracingJSON m_config; std::string m_chunks = "auto"; diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index d4db58483b..3ba7d09e24 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -44,7 +44,8 @@ Writable *getWritable(Attributable *); /** Type of IO operation between logical and persistent data. */ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){ - CREATE_FILE, OPEN_FILE, CLOSE_FILE, DELETE_FILE, + CREATE_FILE, CHECK_FILE, OPEN_FILE, CLOSE_FILE, + DELETE_FILE, CREATE_PATH, CLOSE_PATH, OPEN_PATH, DELETE_PATH, LIST_PATHS, @@ -118,6 +119,32 @@ struct OPENPMDAPI_EXPORT Parameter IterationEncoding encoding = IterationEncoding::groupBased; }; +template <> +struct OPENPMDAPI_EXPORT Parameter + : public AbstractParameter +{ + Parameter() = default; + Parameter(Parameter const &p) + : AbstractParameter(), name(p.name), fileExists(p.fileExists) + {} + + std::unique_ptr clone() const override + { + return std::unique_ptr( + new Parameter(*this)); + } + + std::string name = ""; + enum class FileExists + { + DontKnow, + Yes, + No + }; + std::shared_ptr fileExists = + std::make_shared(FileExists::DontKnow); +}; + template <> struct OPENPMDAPI_EXPORT Parameter : public AbstractParameter diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index d090d0b687..738891f33e 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -160,6 +160,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl void createFile(Writable *, Parameter const &) override; + void checkFile(Writable *, Parameter &) override; + void createPath(Writable *, Parameter const &) override; diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index e828bcd7f7..cda0956b52 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -534,7 +534,7 @@ OPENPMD_private bool hasExpansionPattern(std::string filenameWithExtension); bool reparseExpansionPattern(std::string filenameWithExtension); void init(std::shared_ptr, std::unique_ptr); - void initDefaults(IterationEncoding); + void initDefaults(IterationEncoding, bool initAll = false); /** * @brief Internal call for flushing a Series. * diff --git a/include/openPMD/auxiliary/Mpi.hpp b/include/openPMD/auxiliary/Mpi.hpp new file mode 100644 index 0000000000..5201f68bf2 --- /dev/null +++ b/include/openPMD/auxiliary/Mpi.hpp @@ -0,0 +1,77 @@ +/* Copyright 2022 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#pragma once + +#include "openPMD/config.hpp" + +#if openPMD_HAVE_MPI +#include +#endif + +#include + +namespace openPMD::auxiliary +{ +#if openPMD_HAVE_MPI + +namespace detail +{ + namespace + { + // see https://en.cppreference.com/w/cpp/language/if + template + inline constexpr bool dependent_false_v = false; + } // namespace +} // namespace detail + +namespace +{ + template + constexpr MPI_Datatype openPMD_MPI_type() + { + using T_decay = std::decay_t; + if constexpr (std::is_same_v) + { + return MPI_CHAR; + } + else if constexpr (std::is_same_v) + { + return MPI_UNSIGNED; + } + else if constexpr (std::is_same_v) + { + return MPI_UNSIGNED_LONG; + } + else if constexpr (std::is_same_v) + { + return MPI_UNSIGNED_LONG_LONG; + } + else + { + static_assert( + detail::dependent_false_v, + "openPMD_MPI_type: Unsupported type."); + } + } +} // namespace + +#endif +} // namespace openPMD::auxiliary diff --git a/src/IO/ADIOS/ADIOS1IOHandler.cpp b/src/IO/ADIOS/ADIOS1IOHandler.cpp index 92166ca259..019e5a8078 100644 --- a/src/IO/ADIOS/ADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS1IOHandler.cpp @@ -104,6 +104,12 @@ std::future ADIOS1IOHandlerImpl::flush() deref_dynamic_cast >( i.parameter.get())); break; + case O::CHECK_FILE: + checkFile( + i.writable, + deref_dynamic_cast >( + i.parameter.get())); + break; case O::CREATE_PATH: createPath( i.writable, @@ -346,6 +352,7 @@ void ADIOS1IOHandler::enqueue(IOTask const &i) switch (i.operation) { case Operation::CREATE_FILE: + case Operation::CHECK_FILE: case Operation::CREATE_PATH: case Operation::OPEN_PATH: case Operation::CREATE_DATASET: diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 047373cdff..fcdfffc1f8 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -27,6 +27,7 @@ #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp" #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/auxiliary/Mpi.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" @@ -73,6 +74,7 @@ ADIOS2IOHandlerImpl::ADIOS2IOHandlerImpl( std::string specifiedExtension) : AbstractIOHandlerImplCommon(handler) , m_ADIOS{communicator} + , m_communicator{communicator} , m_engineType(std::move(engineType)) , m_userSpecifiedExtension{std::move(specifiedExtension)} { @@ -245,7 +247,7 @@ ADIOS2IOHandlerImpl::getOperators() using AcceptedEndingsForEngine = std::map; -std::string ADIOS2IOHandlerImpl::fileSuffix() const +std::string ADIOS2IOHandlerImpl::fileSuffix(bool verbose) const { // SST engine adds its suffix unconditionally // so we don't add it @@ -267,7 +269,8 @@ std::string ADIOS2IOHandlerImpl::fileSuffix() const if (auto ending = acceptedEndings.find(m_userSpecifiedExtension); ending != acceptedEndings.end()) { - if ((m_engineType == "file" || m_engineType == "filestream") && + if (verbose && + (m_engineType == "file" || m_engineType == "filestream") && (m_userSpecifiedExtension == ".bp3" || m_userSpecifiedExtension == ".bp4" || m_userSpecifiedExtension == ".bp5")) @@ -288,7 +291,7 @@ std::string ADIOS2IOHandlerImpl::fileSuffix() const { std::cerr << "[ADIOS2] No file ending specified. Will not add one." << std::endl; - if (m_engineType == "bp3") + if (verbose && m_engineType == "bp3") { std::cerr << "Note that the ADIOS2 BP3 engine will add its " @@ -300,19 +303,22 @@ std::string ADIOS2IOHandlerImpl::fileSuffix() const } else { - std::cerr << "[ADIOS2] Specified ending '" - << m_userSpecifiedExtension - << "' does not match the selected engine '" - << m_engineType - << "'. Will use the specified ending anyway." - << std::endl; - if (m_engineType == "bp3") + if (verbose) { - std::cerr - << "Note that the ADIOS2 BP3 engine will add its " - "ending '.bp' if not specified (e.g. 'simData.bp3' " - "will appear on disk as 'simData.bp3.bp')." - << std::endl; + std::cerr << "[ADIOS2] Specified ending '" + << m_userSpecifiedExtension + << "' does not match the selected engine '" + << m_engineType + << "'. Will use the specified ending anyway." + << std::endl; + if (m_engineType == "bp3") + { + std::cerr + << "Note that the ADIOS2 BP3 engine will add its " + "ending '.bp' if not specified (e.g. 'simData.bp3' " + "will appear on disk as 'simData.bp3.bp')." + << std::endl; + } } return m_userSpecifiedExtension; } @@ -496,6 +502,60 @@ void ADIOS2IOHandlerImpl::createFile( } } +void ADIOS2IOHandlerImpl::checkFile( + Writable *, Parameter ¶meters) +{ + std::string name = + fullPath(parameters.name + fileSuffix(/* verbose = */ false)); + + using FileExists = Parameter::FileExists; + *parameters.fileExists = checkFile(name) ? FileExists::Yes : FileExists::No; +} + +bool ADIOS2IOHandlerImpl::checkFile(std::string fullFilePath) const +{ + if (m_engineType == "bp3") + { + if (!auxiliary::ends_with(fullFilePath, ".bp")) + { + /* + * BP3 will add this ending if not specified + */ + fullFilePath += ".bp"; + } + } + else if (m_engineType == "sst") + { + /* + * SST will add this ending indiscriminately + */ + fullFilePath += ".sst"; + } + bool fileExists = auxiliary::directory_exists(fullFilePath) || + auxiliary::file_exists(fullFilePath); + +#if openPMD_HAVE_MPI + if (m_communicator.has_value()) + { + bool fileExistsRes = false; + int status = MPI_Allreduce( + &fileExists, + &fileExistsRes, + 1, + MPI_C_BOOL, + MPI_LOR, // logical or + m_communicator.value()); + if (status != 0) + { + throw std::runtime_error("MPI Reduction failed!"); + } + fileExists = fileExistsRes; + } +#endif + + return fileExists; +} + void ADIOS2IOHandlerImpl::createPath( Writable *writable, const Parameter ¶meters) { @@ -2271,11 +2331,14 @@ namespace detail : m_file(impl.fullPath(std::move(file))) , m_IOName(std::to_string(impl.nameCounter++)) , m_ADIOS(impl.m_ADIOS) - , m_IO(impl.m_ADIOS.DeclareIO(m_IOName)) - , m_mode(impl.adios2AccessMode(m_file)) , m_impl(&impl) , m_engineType(impl.m_engineType) { + // Declaring these members in the constructor body to avoid + // initialization order hazards. Need the IO_ prefix since in some + // situation there seems to be trouble with number-only IO names + m_IO = impl.m_ADIOS.DeclareIO("IO_" + m_IOName); + m_mode = impl.adios2AccessMode(m_file); if (!m_IO) { throw std::runtime_error( @@ -2616,9 +2679,22 @@ namespace detail { if (!m_engine) { + auto tempMode = m_mode; switch (m_mode) { case adios2::Mode::Append: +#ifdef _WIN32 + /* + * On Windows, ADIOS2 v2.8. Append mode only works with existing + * files. So, we first check for file existence and switch to + * create mode if it does not exist. + * + * See issue: https://github.com/ornladios/ADIOS2/issues/3358 + */ + tempMode = m_impl->checkFile(m_file) ? adios2::Mode::Append + : adios2::Mode::Write; + [[fallthrough]]; +#endif case adios2::Mode::Write: { // usesSteps attribute only written upon ::advance() // this makes sure that the attribute is only put in case @@ -2626,7 +2702,7 @@ namespace detail m_IO.DefineAttribute( ADIOS2Defaults::str_adios2Schema, m_impl->m_schema); m_engine = std::make_optional( - adios2::Engine(m_IO.Open(m_file, m_mode))); + adios2::Engine(m_IO.Open(m_file, tempMode))); break; } case adios2::Mode::Read: { diff --git a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp index 7bed2f15c7..3dbee3e2db 100644 --- a/src/IO/ADIOS/CommonADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/CommonADIOS1IOHandler.cpp @@ -449,6 +449,14 @@ void CommonADIOS1IOHandlerImpl::createFile( } } +template +void CommonADIOS1IOHandlerImpl::checkFile( + Writable *, Parameter ¶meter) +{ + *parameter.fileExists = + Parameter::FileExists::DontKnow; +} + template void CommonADIOS1IOHandlerImpl::createPath( Writable *writable, Parameter const ¶meters) diff --git a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp index a451dd2c15..20f571e980 100644 --- a/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp +++ b/src/IO/ADIOS/ParallelADIOS1IOHandler.cpp @@ -126,6 +126,12 @@ std::future ParallelADIOS1IOHandlerImpl::flush() deref_dynamic_cast >( i.parameter.get())); break; + case O::CHECK_FILE: + checkFile( + i.writable, + deref_dynamic_cast >( + i.parameter.get())); + break; case O::CREATE_PATH: createPath( i.writable, @@ -365,6 +371,7 @@ void ParallelADIOS1IOHandler::enqueue(IOTask const &i) switch (i.operation) { case Operation::CREATE_FILE: + case Operation::CHECK_FILE: case Operation::CREATE_PATH: case Operation::OPEN_PATH: case Operation::CREATE_DATASET: diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index b663740077..94dce55c1c 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -29,6 +29,7 @@ #include "openPMD/IO/HDF5/HDF5FilePosition.hpp" #include "openPMD/IO/IOTask.hpp" #include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/auxiliary/Mpi.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/backend/Attribute.hpp" @@ -284,6 +285,40 @@ void HDF5IOHandlerImpl::createFile( } } +void HDF5IOHandlerImpl::checkFile( + Writable *, Parameter ¶meters) +{ + std::string name = m_handler->directory + parameters.name; + if (!auxiliary::ends_with(name, ".h5")) + { + name += ".h5"; + } + bool fileExists = + auxiliary::file_exists(name) || auxiliary::directory_exists(name); + +#if openPMD_HAVE_MPI + if (m_communicator.has_value()) + { + bool fileExistsRes = false; + int status = MPI_Allreduce( + &fileExists, + &fileExistsRes, + 1, + MPI_C_BOOL, + MPI_LOR, // logical or + m_communicator.value()); + if (status != 0) + { + throw std::runtime_error("MPI Reduction failed!"); + } + fileExists = fileExistsRes; + } +#endif + + using FileExists = Parameter::FileExists; + *parameters.fileExists = fileExists ? FileExists::Yes : FileExists::No; +} + void HDF5IOHandlerImpl::createPath( Writable *writable, Parameter const ¶meters) { diff --git a/src/IO/HDF5/ParallelHDF5IOHandler.cpp b/src/IO/HDF5/ParallelHDF5IOHandler.cpp index 369ee2e317..f7a6dc1a1c 100644 --- a/src/IO/HDF5/ParallelHDF5IOHandler.cpp +++ b/src/IO/HDF5/ParallelHDF5IOHandler.cpp @@ -65,6 +65,9 @@ ParallelHDF5IOHandlerImpl::ParallelHDF5IOHandlerImpl( , m_mpiComm{comm} , m_mpiInfo{MPI_INFO_NULL} /* MPI 3.0+: MPI_INFO_ENV */ { + // Set this so the parent class can use the MPI communicator in functions + // that are written with special implemenations for MPI-enabled HDF5. + m_communicator = m_mpiComm; m_datasetTransferProperty = H5Pcreate(H5P_DATASET_XFER); m_fileAccessProperty = H5Pcreate(H5P_FILE_ACCESS); m_fileCreateProperty = H5Pcreate(H5P_FILE_CREATE); diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 6f8666fae5..062c736433 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -102,11 +102,12 @@ void JSONIOHandlerImpl::createFile( } auto res_pair = getPossiblyExisting(name); + auto fullPathToFile = fullPath(std::get<0>(res_pair)); File shared_name = File(name); VERIFY_ALWAYS( !(m_handler->m_backendAccess == Access::READ_WRITE && (!std::get<2>(res_pair) || - auxiliary::file_exists(fullPath(std::get<0>(res_pair))))), + auxiliary::file_exists(fullPathToFile))), "[JSON] Can only overwrite existing file in CREATE mode."); if (!std::get<2>(res_pair)) @@ -127,9 +128,12 @@ void JSONIOHandlerImpl::createFile( associateWithFile(writable, shared_name); this->m_dirty.emplace(shared_name); - if (m_handler->m_backendAccess != Access::APPEND) + if (m_handler->m_backendAccess != Access::APPEND || + !auxiliary::file_exists(fullPathToFile)) { - // make sure to overwrite! + // if in create mode: make sure to overwrite + // if in append mode and the file does not exist: create an empty + // dataset this->m_jsonVals[shared_name] = std::make_shared(); } // else: the JSON value is not available in m_jsonVals and will be @@ -140,6 +144,22 @@ void JSONIOHandlerImpl::createFile( } } +void JSONIOHandlerImpl::checkFile( + Writable *, Parameter ¶meters) +{ + std::string name = parameters.name; + if (!auxiliary::ends_with(name, ".json")) + { + name += ".json"; + } + name = fullPath(name); + using FileExists = Parameter::FileExists; + *parameters.fileExists = + (auxiliary::file_exists(name) || auxiliary::directory_exists(name)) + ? FileExists::Yes + : FileExists::No; +} + void JSONIOHandlerImpl::createPath( Writable *writable, Parameter const ¶meter) { diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 1cf77ec180..575610ea16 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -28,6 +28,7 @@ #include "openPMD/backend/Writable.hpp" #include +#include #include namespace openPMD diff --git a/src/Series.cpp b/src/Series.cpp index 814b0e2489..6387258cf6 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -502,15 +502,18 @@ namespace int padding; uint64_t iterationIndex; std::set paddings; - for (auto const &entry : auxiliary::list_directory(directory)) + if (auxiliary::directory_exists(directory)) { - std::tie(isContained, padding, iterationIndex) = - isPartOfSeries(entry); - if (isContained) + for (auto const &entry : auxiliary::list_directory(directory)) { - paddings.insert(padding); - // no std::forward as this is called repeatedly - mappingFunction(iterationIndex, entry); + std::tie(isContained, padding, iterationIndex) = + isPartOfSeries(entry); + if (isContained) + { + paddings.insert(padding); + // no std::forward as this is called repeatedly + mappingFunction(iterationIndex, entry); + } } } if (paddings.size() == 1u) @@ -637,12 +640,8 @@ Given file pattern: ')END" series.m_lastFlushSuccessful = true; } -void Series::initDefaults(IterationEncoding ie) +void Series::initDefaults(IterationEncoding ie, bool initAll) { - if (!containsAttribute("openPMD")) - setOpenPMD(getStandard()); - if (!containsAttribute("openPMDextension")) - setOpenPMDextension(0); if (!containsAttribute("basePath")) { if (ie == IterationEncoding::variableBased) @@ -655,6 +654,21 @@ void Series::initDefaults(IterationEncoding ie) setAttribute("basePath", std::string(BASEPATH)); } } + if (!containsAttribute("openPMD")) + setOpenPMD(getStandard()); + /* + * In Append mode, only init the rest of the defaults after checking that + * the file does not yet exist to avoid overriding more than needed. + * In file-based iteration encoding, files are always truncated in Append + * mode (Append mode works on a per-iteration basis). + */ + if (!initAll && IOHandler()->m_frontendAccess == Access::APPEND && + ie != IterationEncoding::fileBased) + { + return; + } + if (!containsAttribute("openPMDextension")) + setOpenPMDextension(0); if (!containsAttribute("date")) setDate(auxiliary::getDateString()); if (!containsAttribute("software")) @@ -845,6 +859,23 @@ void Series::flushGorVBased( case Access::APPEND: { if (!written()) { + if (IOHandler()->m_frontendAccess == Access::APPEND) + { + Parameter param; + param.name = series.m_name; + IOHandler()->enqueue(IOTask(this, param)); + IOHandler()->flush(internal::defaultFlushParams); + switch (*param.fileExists) + { + using FE = Parameter::FileExists; + case FE::DontKnow: + case FE::No: + initDefaults(iterationEncoding(), /* initAll = */ true); + break; + case FE::Yes: + break; + } + } Parameter fCreate; fCreate.name = series.m_name; fCreate.encoding = iterationEncoding(); diff --git a/src/auxiliary/Filesystem.cpp b/src/auxiliary/Filesystem.cpp index 3e2c65f3af..38d8e209f8 100644 --- a/src/auxiliary/Filesystem.cpp +++ b/src/auxiliary/Filesystem.cpp @@ -19,6 +19,7 @@ * If not, see . */ #include "openPMD/auxiliary/Filesystem.hpp" +#include "openPMD/auxiliary/Mpi.hpp" #include "openPMD/auxiliary/StringManip.hpp" #ifdef _WIN32 @@ -175,41 +176,6 @@ bool remove_file(std::string const &path) #if openPMD_HAVE_MPI -namespace -{ - template - struct MPI_Types; - - template <> - struct MPI_Types - { - static MPI_Datatype const value; - }; - - template <> - struct MPI_Types - { - static MPI_Datatype const value; - }; - - template <> - struct MPI_Types - { - static MPI_Datatype const value; - }; - - /* - * Only some of these are actually instanciated, - * so suppress warnings for the others. - */ - [[maybe_unused]] MPI_Datatype const MPI_Types::value = - MPI_UNSIGNED; - [[maybe_unused]] MPI_Datatype const MPI_Types::value = - MPI_UNSIGNED_LONG; - [[maybe_unused]] MPI_Datatype const MPI_Types::value = - MPI_UNSIGNED_LONG_LONG; -} // namespace - std::string collective_file_read(std::string const &path, MPI_Comm comm) { int rank, size; @@ -232,7 +198,7 @@ std::string collective_file_read(std::string const &path, MPI_Comm comm) } stringLength = res.size() + 1; } - MPI_Datatype datatype = MPI_Types::value; + MPI_Datatype datatype = openPMD_MPI_type(); int err = MPI_Bcast(&stringLength, 1, datatype, 0, comm); if (err) { diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 1ca922c022..aaf1fd4378 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -40,6 +40,18 @@ std::vector getBackends() auto const backends = getBackends(); +std::vector testedFileExtensions() +{ + auto allExtensions = getFileExtensions(); + auto newEnd = std::remove_if( + allExtensions.begin(), allExtensions.end(), [](std::string const &ext) { + // sst and ssc need a receiver for testing + // bp4 is already tested via bp + return ext == "sst" || ext == "ssc" || ext == "bp4" | ext == "json"; + }); + return {allExtensions.begin(), newEnd}; +} + #else TEST_CASE("none", "[parallel]") @@ -1368,4 +1380,213 @@ TEST_CASE("adios2_ssc", "[parallel][adios2]") { adios2_ssc(); } + +void append_mode( + std::string const &extension, + bool variableBased, + std::string jsonConfig = "{}") +{ + std::string filename = + (variableBased ? "../samples/append/append_variablebased." + : "../samples/append/append_groupbased.") + + extension; + MPI_Barrier(MPI_COMM_WORLD); + if (auxiliary::directory_exists("../samples/append")) + { + auxiliary::remove_directory("../samples/append"); + } + MPI_Barrier(MPI_COMM_WORLD); + std::vector data(10, 0); + auto writeSomeIterations = [&data]( + WriteIterations &&writeIterations, + std::vector indices) { + for (auto index : indices) + { + auto it = writeIterations[index]; + auto dataset = it.meshes["E"]["x"]; + dataset.resetDataset({Datatype::INT, {10}}); + dataset.storeChunk(data, {0}, {10}); + // test that it works without closing too + it.close(); + } + }; + { + Series write(filename, Access::APPEND, MPI_COMM_WORLD, jsonConfig); + if (variableBased) + { + if (write.backend() != "ADIOS2") + { + return; + } + write.setIterationEncoding(IterationEncoding::variableBased); + } + writeSomeIterations( + write.writeIterations(), std::vector{0, 1}); + } + { + Series write(filename, Access::APPEND, MPI_COMM_WORLD, jsonConfig); + if (variableBased) + { + write.setIterationEncoding(IterationEncoding::variableBased); + } + if (write.backend() == "MPI_ADIOS1") + { + REQUIRE_THROWS_WITH( + write.flush(), + Catch::Equals( + "Operation unsupported in ADIOS1: Appending to existing " + "file on disk (use Access::CREATE to overwrite)")); + // destructor will be noisy now + return; + } + + writeSomeIterations( + write.writeIterations(), std::vector{2, 3}); + write.flush(); + } + { + using namespace std::chrono_literals; + /* + * Put a little sleep here to trigger writing of a different /date + * attribute. ADIOS2 v2.7 does not like that so this test ensures that + * we deal with it. + */ + std::this_thread::sleep_for(1s); + Series write(filename, Access::APPEND, MPI_COMM_WORLD, jsonConfig); + if (variableBased) + { + write.setIterationEncoding(IterationEncoding::variableBased); + } + if (write.backend() == "MPI_ADIOS1") + { + REQUIRE_THROWS_WITH( + write.flush(), + Catch::Equals( + "Operation unsupported in ADIOS1: Appending to existing " + "file on disk (use Access::CREATE to overwrite)")); + // destructor will be noisy now + return; + } + + writeSomeIterations( + write.writeIterations(), std::vector{4, 3}); + write.flush(); + } + { + Series read(filename, Access::READ_ONLY, MPI_COMM_WORLD); + if (variableBased || extension == "bp5") + { + // in variable-based encodings, iterations are not parsed ahead of + // time but as they go + unsigned counter = 0; + for (auto const &iteration : read.readIterations()) + { + REQUIRE(iteration.iterationIndex == counter); + ++counter; + } + REQUIRE(counter == 5); + } + else + { + REQUIRE(read.iterations.size() == 5); + } + /* + * Roadmap: for now, reading this should work by ignoring the last + * duplicate iteration. + * After merging https://github.com/openPMD/openPMD-api/pull/949, we + * should see both instances when reading. + * Final goal: Read only the last instance. + */ + helper::listSeries(read); + } +#if 100000000 * ADIOS2_VERSION_MAJOR + 1000000 * ADIOS2_VERSION_MINOR + \ + 10000 * ADIOS2_VERSION_PATCH + 100 * ADIOS2_VERSION_TWEAK >= \ + 208002700 + // AppendAfterSteps has a bug before that version + if (extension == "bp5") + { + { + Series write( + filename, + Access::APPEND, + MPI_COMM_WORLD, + json::merge( + jsonConfig, + R"({"adios2":{"engine":{"parameters":{"AppendAfterSteps":-3}}}})")); + if (variableBased) + { + write.setIterationEncoding(IterationEncoding::variableBased); + } + if (write.backend() == "ADIOS1") + { + REQUIRE_THROWS_WITH( + write.flush(), + Catch::Equals( + "Operation unsupported in ADIOS1: Appending to " + "existing " + "file on disk (use Access::CREATE to overwrite)")); + // destructor will be noisy now + return; + } + + writeSomeIterations( + write.writeIterations(), std::vector{4, 5}); + write.flush(); + } + { + Series read(filename, Access::READ_ONLY, MPI_COMM_WORLD); + // in variable-based encodings, iterations are not parsed ahead of + // time but as they go + unsigned counter = 0; + for (auto const &iteration : read.readIterations()) + { + REQUIRE(iteration.iterationIndex == counter); + ++counter; + } + REQUIRE(counter == 6); + helper::listSeries(read); + } + } +#endif +} + +TEST_CASE("append_mode", "[parallel]") +{ + for (auto const &t : testedFileExtensions()) + { + if (t == "bp" || t == "bp4" || t == "bp5") + { + std::string jsonConfigOld = R"END( +{ + "adios2": + { + "schema": 0, + "engine": + { + "usesteps" : true + } + } +})END"; + std::string jsonConfigNew = R"END( +{ + "adios2": + { + "schema": 20210209, + "engine": + { + "usesteps" : true + } + } +})END"; + append_mode(t, false, jsonConfigOld); + append_mode(t, false, jsonConfigNew); + append_mode(t, true, jsonConfigOld); + append_mode(t, true, jsonConfigNew); + } + else + { + append_mode(t, false); + } + } +} #endif diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index eac05b0214..20d20d6a71 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -27,6 +28,7 @@ #include #include #include +#include #include #include @@ -6320,9 +6322,14 @@ void append_mode( std::string jsonConfig = "{}") { - std::string filename = (variableBased ? "../samples/append_variablebased." - : "../samples/append_groupbased.") + + std::string filename = + (variableBased ? "../samples/append/append_variablebased." + : "../samples/append/append_groupbased.") + extension; + if (auxiliary::directory_exists("../samples/append")) + { + auxiliary::remove_directory("../samples/append"); + } std::vector data(10, 0); auto writeSomeIterations = [&data]( WriteIterations &&writeIterations, @@ -6338,7 +6345,7 @@ void append_mode( } }; { - Series write(filename, Access::CREATE, jsonConfig); + Series write(filename, Access::APPEND, jsonConfig); if (variableBased) { if (write.backend() != "ADIOS2") @@ -6372,6 +6379,13 @@ void append_mode( write.flush(); } { + using namespace std::chrono_literals; + /* + * Put a little sleep here to trigger writing of a different /date + * attribute. ADIOS2 v2.7 does not like that so this test ensures that + * we deal with it. + */ + std::this_thread::sleep_for(1s); Series write(filename, Access::APPEND, jsonConfig); if (variableBased) { @@ -6540,8 +6554,8 @@ void append_mode_filebased(std::string const &extension) } { Series write( - "../samples/append/append_%T." + extension, - Access::CREATE, + "../samples/append/append_%06T." + extension, + Access::APPEND, jsonConfig); writeSomeIterations( write.writeIterations(), std::vector{0, 1});