Fix append mode double attributes

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -31,6 +31,7 @@
 #include "openPMD/IO/InvalidatableFile.hpp"
 #include "openPMD/IterationEncoding.hpp"
 #include "openPMD/auxiliary/JSON_internal.hpp"
+#include "openPMD/auxiliary/Mpi.hpp"
 #include "openPMD/backend/Writable.hpp"
 #include "openPMD/config.hpp"
 
@@ -150,6 +151,11 @@ class ADIOS2IOHandlerImpl
     void
     createFile(Writable *, Parameter<Operation::CREATE_FILE> const &) override;
 
+    void checkFile(Writable *, Parameter<Operation::CHECK_FILE> &) override;
+
+    // MPI Collective
+    bool checkFile(std::string fullFilePath) const;
+
     void
     createPath(Writable *, Parameter<Operation::CREATE_PATH> const &) override;
 
@@ -226,6 +232,9 @@ class ADIOS2IOHandlerImpl
 
 private:
     adios2::ADIOS m_ADIOS;
+#if openPMD_HAVE_MPI
+    std::optional<MPI_Comm> m_communicator;
+#endif
     /*
      * If the iteration encoding is variableBased, we default to using the
      * 2021_02_09 schema since it allows mutable attributes.
@@ -329,7 +338,7 @@ class ADIOS2IOHandlerImpl
     // use m_config
     std::optional<std::vector<ParameterizedOperator> > getOperators();
 
-    std::string fileSuffix() const;
+    std::string fileSuffix(bool verbose = true) const;
 
     /*
      * We need to give names to IO objects. These names are irrelevant

include/openPMD/IO/ADIOS/CommonADIOS1IOHandler.hpp

@@ -52,6 +52,7 @@ class CommonADIOS1IOHandlerImpl : public AbstractIOHandlerImpl
 public:
     void
     createFile(Writable *, Parameter<Operation::CREATE_FILE> const &) override;
+    void checkFile(Writable *, Parameter<Operation::CHECK_FILE> &) override;
     void
     createPath(Writable *, Parameter<Operation::CREATE_PATH> const &) override;
     void createDataset(

include/openPMD/IO/AbstractIOHandlerImpl.hpp

@@ -58,6 +58,12 @@ class AbstractIOHandlerImpl
                         deref_dynamic_cast<Parameter<Operation::CREATE_FILE> >(
                             i.parameter.get()));
                     break;
+                case O::CHECK_FILE:
+                    checkFile(
+                        i.writable,
+                        deref_dynamic_cast<Parameter<Operation::CHECK_FILE> >(
+                            i.parameter.get()));
+                    break;
                 case O::CREATE_PATH:
                     createPath(
                         i.writable,
@@ -220,6 +226,17 @@ class AbstractIOHandlerImpl
     virtual void
     closeFile(Writable *, Parameter<Operation::CLOSE_FILE> const &) = 0;
 
+    /**
+     * Check if the file specified by the parameter is already present on disk.
+     * The Writable is irrelevant for this method.
+     * A backend can choose to ignore this task and specify FileExists::DontKnow
+     * in the out parameter.
+     * The consequence will be that some top-level attributes might be defined
+     * a second time when appending to an existing file, because the frontend
+     * cannot be sure that the file already has these attributes.
+     */
+    virtual void checkFile(Writable *, Parameter<Operation::CHECK_FILE> &) = 0;
+
     /** Advance the file/stream that this writable belongs to.
      *
      * If the backend is based around usage of IO steps (especially streaming

include/openPMD/IO/HDF5/HDF5IOHandlerImpl.hpp

@@ -25,6 +25,7 @@
 #include "openPMD/IO/AbstractIOHandlerImpl.hpp"
 
 #include "openPMD/auxiliary/JSON_internal.hpp"
+#include "openPMD/auxiliary/Mpi.hpp"
 
 #include <hdf5.h>
 #include <optional>
@@ -43,6 +44,7 @@ class HDF5IOHandlerImpl : public AbstractIOHandlerImpl
 
     void
     createFile(Writable *, Parameter<Operation::CREATE_FILE> const &) override;
+    void checkFile(Writable *, Parameter<Operation::CHECK_FILE> &) override;
     void
     createPath(Writable *, Parameter<Operation::CREATE_PATH> const &) override;
     void createDataset(
@@ -92,6 +94,15 @@ class HDF5IOHandlerImpl : public AbstractIOHandlerImpl
     hid_t m_H5T_CDOUBLE;
     hid_t m_H5T_CLONG_DOUBLE;
 
+protected:
+#if openPMD_HAVE_MPI
+    /*
+     * Not defined in ParallelHDF5IOHandlerImpl, so we don't have to write
+     * some methods twice.
+     */
+    std::optional<MPI_Comm> m_communicator;
+#endif
+
 private:
     json::TracingJSON m_config;
     std::string m_chunks = "auto";

include/openPMD/IO/IOTask.hpp

@@ -44,7 +44,8 @@ Writable *getWritable(Attributable *);
 /** Type of IO operation between logical and persistent data.
  */
 OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){
-    CREATE_FILE,      OPEN_FILE,      CLOSE_FILE,    DELETE_FILE,
+    CREATE_FILE,      CHECK_FILE,     OPEN_FILE,     CLOSE_FILE,
+    DELETE_FILE,
 
     CREATE_PATH,      CLOSE_PATH,     OPEN_PATH,     DELETE_PATH,
     LIST_PATHS,
@@ -118,6 +119,32 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::CREATE_FILE>
     IterationEncoding encoding = IterationEncoding::groupBased;
 };
 
+template <>
+struct OPENPMDAPI_EXPORT Parameter<Operation::CHECK_FILE>
+    : public AbstractParameter
+{
+    Parameter() = default;
+    Parameter(Parameter const &p)
+        : AbstractParameter(), name(p.name), fileExists(p.fileExists)
+    {}
+
+    std::unique_ptr<AbstractParameter> clone() const override
+    {
+        return std::unique_ptr<AbstractParameter>(
+            new Parameter<Operation::CHECK_FILE>(*this));
+    }
+
+    std::string name = "";
+    enum class FileExists
+    {
+        DontKnow,
+        Yes,
+        No
+    };
+    std::shared_ptr<FileExists> fileExists =
+        std::make_shared<FileExists>(FileExists::DontKnow);
+};
+
 template <>
 struct OPENPMDAPI_EXPORT Parameter<Operation::OPEN_FILE>
     : public AbstractParameter

include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp

@@ -160,6 +160,8 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl
     void
     createFile(Writable *, Parameter<Operation::CREATE_FILE> const &) override;
 
+    void checkFile(Writable *, Parameter<Operation::CHECK_FILE> &) override;
+
     void
     createPath(Writable *, Parameter<Operation::CREATE_PATH> const &) override;
 

include/openPMD/Series.hpp

@@ -82,6 +82,8 @@ namespace internal
          * the same instance.
          */
         std::optional<WriteIterations> m_writeIterations;
+
+        Attributable mockedDefaults;
         /**
          * Needed if reading a single iteration of a file-based series.
          * Users may specify the concrete filename of one iteration instead of
@@ -534,7 +536,7 @@ OPENPMD_private
     bool hasExpansionPattern(std::string filenameWithExtension);
     bool reparseExpansionPattern(std::string filenameWithExtension);
     void init(std::shared_ptr<AbstractIOHandler>, std::unique_ptr<ParsedInput>);
-    void initDefaults(IterationEncoding);
+    void initDefaults(IterationEncoding, bool initAll = false);
     /**
      * @brief Internal call for flushing a Series.
      *

include/openPMD/auxiliary/Mpi.hpp

@@ -0,0 +1,83 @@
+/* Copyright 2022 Franz Poeschel
+ *
+ * This file is part of openPMD-api.
+ *
+ * openPMD-api is free software: you can redistribute it and/or modify
+ * it under the terms of of either the GNU General Public License or
+ * the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * openPMD-api is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License and the GNU Lesser General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * and the GNU Lesser General Public License along with openPMD-api.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+#pragma once
+
+#include "openPMD/config.hpp"
+
+#include <optional>
+#include <stdexcept>
+#include <type_traits>
+#include <utility>
+
+#if openPMD_HAVE_MPI
+#include <mpi.h>
+#endif
+
+namespace openPMD::auxiliary
+{
+#if openPMD_HAVE_MPI
+
+using Mock_MPI_Comm = MPI_Comm;
+
+namespace
+{
+    template <typename>
+    struct MPI_Types;
+
+    template <>
+    struct MPI_Types<unsigned long>
+    {
+        static MPI_Datatype const value;
+    };
+
+    template <>
+    struct MPI_Types<unsigned long long>
+    {
+        static MPI_Datatype const value;
+    };
+
+    template <>
+    struct MPI_Types<unsigned>
+    {
+        static MPI_Datatype const value;
+    };
+
+    template <>
+    struct MPI_Types<char>
+    {
+        static MPI_Datatype const value;
+    };
+
+    /*
+     * Only some of these are actually instanciated,
+     * so suppress warnings for the others.
+     */
+    [[maybe_unused]] MPI_Datatype const MPI_Types<unsigned>::value =
+        MPI_UNSIGNED;
+    [[maybe_unused]] MPI_Datatype const MPI_Types<unsigned long>::value =
+        MPI_UNSIGNED_LONG;
+    [[maybe_unused]] MPI_Datatype const MPI_Types<unsigned long long>::value =
+        MPI_UNSIGNED_LONG_LONG;
+    [[maybe_unused]] MPI_Datatype const MPI_Types<char>::value = MPI_CHAR;
+} // namespace
+
+#endif
+} // namespace openPMD::auxiliary

src/IO/ADIOS/ADIOS1IOHandler.cpp

@@ -104,6 +104,12 @@ std::future<void> ADIOS1IOHandlerImpl::flush()
                     deref_dynamic_cast<Parameter<Operation::CREATE_FILE> >(
                         i.parameter.get()));
                 break;
+            case O::CHECK_FILE:
+                checkFile(
+                    i.writable,
+                    deref_dynamic_cast<Parameter<Operation::CHECK_FILE> >(
+                        i.parameter.get()));
+                break;
             case O::CREATE_PATH:
                 createPath(
                     i.writable,
@@ -346,6 +352,7 @@ void ADIOS1IOHandler::enqueue(IOTask const &i)
     switch (i.operation)
     {
     case Operation::CREATE_FILE:
+    case Operation::CHECK_FILE:
     case Operation::CREATE_PATH:
     case Operation::OPEN_PATH:
     case Operation::CREATE_DATASET: