Mapping between ADIOS steps and openPMD iterations

.github/workflows/intel.yml

@@ -17,7 +17,12 @@ jobs:
       run: |
         sudo .github/workflows/dependencies/install_icc
     - name: Build
-      env: {CXXFLAGS: -Werror}
+      # Due to compiler bugs in Intel compiler, we need to disable warning 1011
+      # (missing return value), otherwise `if constexpr` functions
+      # don't compile.
+      # See https://community.intel.com/t5/Intel-C-Compiler/quot-if-constexpr-quot-and-quot-missing-return-statement-quot-in/td-p/1154551
+      # Using a local pragma does not work due to the reasons stated there.
+      env: {CXXFLAGS: -Werror -wd1011}
       run: |
         set +e; source /opt/intel/oneapi/setvars.sh; set -e
         share/openPMD/download_samples.sh build

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -1155,13 +1155,6 @@ namespace detail
          */
         void invalidateVariablesMap();
 
-    private:
-        ADIOS2IOHandlerImpl *m_impl;
-        std::optional<adios2::Engine> m_engine; //! ADIOS engine
-        /**
-         * The ADIOS2 engine type, to be passed to adios2::IO::SetEngine
-         */
-        std::string m_engineType;
         /*
          * streamStatus is NoStream for file-based ADIOS engines.
          * This is relevant for the method BufferedActions::requireActiveStep,
@@ -1253,6 +1246,14 @@ namespace detail
         };
         StreamStatus streamStatus = StreamStatus::OutsideOfStep;
 
+    private:
+        ADIOS2IOHandlerImpl *m_impl;
+        std::optional<adios2::Engine> m_engine; //! ADIOS engine
+        /**
+         * The ADIOS2 engine type, to be passed to adios2::IO::SetEngine
+         */
+        std::string m_engineType;
+
         /**
          * See documentation for StreamStatus::Parsing.
          * Will be set true under the circumstance described there in order to

include/openPMD/IO/AbstractIOHandler.hpp

@@ -121,6 +121,28 @@ namespace internal
     FlushParams const defaultFlushParams{};
 
     struct ParsedFlushParams;
+
+    /**
+     * Some parts of the openPMD object model are read-only when accessing
+     * a Series in Access::READ_ONLY mode, notably Containers and Attributes.
+     * They are filled at parse time and not modified afterwards.
+     * Such state-changing operations are hence allowed under either of two
+     * conditions:
+     * 1) The Series is opened in an open mode that allows writing in any way.
+     *    (Currently any but Access::READ_ONLY).
+     * 2) The Series is in Parsing state. This way, modifying the open mode
+     *    during parsing can be avoided.
+     */
+    enum class SeriesStatus : unsigned char
+    {
+        Default, ///< Mutability of objects in the openPMD object model is
+                 ///< determined by the open mode (Access enum), normal state in
+                 ///< which the user interacts with the Series.
+        Parsing ///< All objects in the openPMD object model are temporarily
+                ///< mutable to allow inserting newly-parsed data.
+                ///< Special state only active while internal routines are
+                ///< running.
+    };
 } // namespace internal
 
 /** Interface for communicating between logical and physically persistent data.
@@ -192,6 +214,7 @@ class AbstractIOHandler
     // why do these need to be separate?
     Access const m_backendAccess;
     Access const m_frontendAccess;
+    internal::SeriesStatus m_seriesStatus = internal::SeriesStatus::Default;
     std::queue<IOTask> m_work;
 }; // AbstractIOHandler
 

include/openPMD/IO/AbstractIOHandlerImpl.hpp

@@ -252,8 +252,10 @@ class AbstractIOHandlerImpl
      * The advance mode is determined by parameters.mode.
      * The return status code shall be stored as parameters.status.
      */
-    virtual void advance(Writable *, Parameter<Operation::ADVANCE> &)
-    {}
+    virtual void advance(Writable *, Parameter<Operation::ADVANCE> &parameters)
+    {
+        *parameters.status = AdvanceStatus::RANDOMACCESS;
+    }
 
     /** Close an openPMD group.
      *
@@ -488,8 +490,13 @@ class AbstractIOHandlerImpl
      * datatype parameters.dtype. Any existing attribute with the same name
      * should be overwritten. If possible, only the value should be changed if
      * the datatype stays the same. The attribute should be written to physical
-     * storage after the operation completes successfully. All datatypes of
-     * Datatype should be supported in a type-safe way.
+     * storage after the operation completes successfully. If the parameter
+     * changesOverSteps is true, then the attribute must be able to hold
+     * different values across IO steps. If the backend does not support IO
+     * steps in such a way, the attribute should not be written. (IO steps are
+     * an optional backend feature and the frontend must implement fallback
+     * measures in such a case) All datatypes of Datatype should be supported in
+     * a type-safe way.
      */
     virtual void
     writeAttribute(Writable *, Parameter<Operation::WRITE_ATT> const &) = 0;

include/openPMD/IO/IOTask.hpp

@@ -537,6 +537,7 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::WRITE_ATT>
         : AbstractParameter()
         , name(p.name)
         , dtype(p.dtype)
+        , changesOverSteps(p.changesOverSteps)
         , resource(p.resource)
     {}
 
@@ -548,6 +549,13 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::WRITE_ATT>
 
     std::string name = "";
     Datatype dtype = Datatype::UNDEFINED;
+    /*
+     * If true, this attribute changes across IO steps.
+     * It should only be written in backends that support IO steps,
+     * otherwise writing should be skipped.
+     * The frontend is responsible for handling both situations.
+     */
+    bool changesOverSteps = false;
     Attribute::resource resource;
 };
 

include/openPMD/Iteration.hpp

@@ -28,7 +28,10 @@
 #include "openPMD/backend/Attributable.hpp"
 #include "openPMD/backend/Container.hpp"
 
+#include <cstdint>
+#include <deque>
 #include <optional>
+#include <tuple>
 
 namespace openPMD
 {
@@ -282,14 +285,57 @@ class Iteration : public Attributable
     void readGorVBased(std::string const &groupPath, bool beginStep);
     void read_impl(std::string const &groupPath);
 
+    /**
+     * Status after beginning an IO step. Currently includes:
+     * * The advance status (OK, OVER, RANDOMACCESS)
+     * * The opened iterations, in case the snapshot attribute is found
+     */
+    struct BeginStepStatus
+    {
+        using AvailableIterations_t = std::optional<std::deque<uint64_t> >;
+
+        AdvanceStatus stepStatus{};
+        /*
+         * If the iteration attribute `snapshot` is present, the value of that
+         * attribute. Otherwise empty.
+         */
+        AvailableIterations_t iterationsInOpenedStep;
+
+        /*
+         * Most of the time, the AdvanceStatus part of this struct is what we
+         * need, so let's make it easy to access.
+         */
+        inline operator AdvanceStatus() const
+        {
+            return stepStatus;
+        }
+
+        /*
+         * Support for std::tie()
+         */
+        inline operator std::tuple<AdvanceStatus &, AvailableIterations_t &>()
+        {
+            return std::tuple<AdvanceStatus &, AvailableIterations_t &>{
+                stepStatus, iterationsInOpenedStep};
+        }
+    };
+
     /**
      * @brief Begin an IO step on the IO file (or file-like object)
      *        containing this iteration. In case of group-based iteration
      *        layout, this will be the complete Series.
      *
-     * @return AdvanceStatus
+     * @return BeginStepStatus
+     */
+    BeginStepStatus beginStep(bool reread);
+
+    /*
+     * Iteration-independent variant for beginStep().
+     * Useful in group-based iteration encoding where the Iteration will only
+     * be known after opening the step.
      */
-    AdvanceStatus beginStep(bool reread);
+    static BeginStepStatus
+    beginStep(std::optional<Iteration> thisObject, Series &series, bool reread);
 
     /**
      * @brief End an IO step on the IO file (or file-like object)

include/openPMD/ReadIterations.hpp

@@ -23,6 +23,8 @@
 #include "openPMD/Iteration.hpp"
 #include "openPMD/Series.hpp"
 
+#include <deque>
+#include <iostream>
 #include <optional>
 
 namespace openPMD
@@ -54,7 +56,8 @@ class SeriesIterator
     using maybe_series_t = std::optional<Series>;
 
     maybe_series_t m_series;
-    iteration_index_t m_currentIteration = 0;
+    std::deque<iteration_index_t> m_iterationsInCurrentStep;
+    uint64_t m_currentIteration{};
 
 public:
     //! construct the end() iterator
@@ -71,6 +74,39 @@ class SeriesIterator
     bool operator!=(SeriesIterator const &other) const;
 
     static SeriesIterator end();
+
+private:
+    inline bool setCurrentIteration()
+    {
+        if (m_iterationsInCurrentStep.empty())
+        {
+            std::cerr << "[ReadIterations] Encountered a step without "
+                         "iterations. Closing the Series."
+                      << std::endl;
+            *this = end();
+            return false;
+        }
+        m_currentIteration = *m_iterationsInCurrentStep.begin();
+        return true;
+    }
+
+    inline std::optional<uint64_t> peekCurrentIteration()
+    {
+        if (m_iterationsInCurrentStep.empty())
+        {
+            return std::nullopt;
+        }
+        else
+        {
+            return {*m_iterationsInCurrentStep.begin()};
+        }
+    }
+
+    std::optional<SeriesIterator *> nextIterationInStep();
+
+    std::optional<SeriesIterator *> nextStep();
+
+    std::optional<SeriesIterator *> loopBody();
 };
 
 /**

include/openPMD/Series.hpp

@@ -37,8 +37,11 @@
 #include <mpi.h>
 #endif
 
+#include <cstdint>
+#include <deque>
 #include <map>
 #include <optional>
+#include <set>
 #include <string>
 
 // expose private and protected members for invasive testing
@@ -82,6 +85,12 @@ namespace internal
          * the same instance.
          */
         std::optional<WriteIterations> m_writeIterations;
+        /**
+         * For writing: Remember which iterations have been written in the
+         * currently active output step. Use this later when writing the
+         * snapshot attribute.
+         */
+        std::set<uint64_t> m_currentlyActiveIterations;
         /**
          * Needed if reading a single iteration of a file-based series.
          * Users may specify the concrete filename of one iteration instead of
@@ -576,8 +585,10 @@ OPENPMD_private
      * Note on re-parsing of a Series:
      * If init == false, the parsing process will seek for new
      * Iterations/Records/Record Components etc.
+     * If series.iterations contains the attribute `snapshot`, returns its
+     * value.
      */
-    void readGorVBased(bool init = true);
+    std::optional<std::deque<uint64_t> > readGorVBased(bool init = true);
     void readBase();
     std::string iterationFilename(uint64_t i);
 
@@ -627,6 +638,22 @@ OPENPMD_private
         internal::AttributableData &file,
         iterations_iterator it,
         Iteration &iteration);
+
+    AdvanceStatus advance(AdvanceMode mode);
+
+    /**
+     * @brief Called at the end of an IO step to store the iterations defined
+     *        in the IO step to the snapshot attribute.
+     *
+     * @param doFlush If true, flush the IO handler.
+     */
+    void flushStep(bool doFlush);
+
+    /*
+     * Returns the current content of the /data/snapshot attribute.
+     * (We could also add this to the public API some time)
+     */
+    std::optional<std::vector<uint64_t> > currentSnapshot() const;
 }; // Series
 } // namespace openPMD
 

include/openPMD/Streaming.hpp

@@ -19,8 +19,9 @@ namespace openPMD
  */
 enum class AdvanceStatus : unsigned char
 {
-    OK, /* stream goes on */
-    OVER /* stream is over */
+    OK, ///< stream goes on
+    OVER, ///< stream is over
+    RANDOMACCESS ///< there is no stream, it will never be over
 };
 
 /**

include/openPMD/backend/Attributable.hpp

@@ -463,7 +463,9 @@ inline bool Attributable::setAttributeImpl(
     internal::attr_value_check(key, value, setAttributeMode);
 
     auto &attri = get();
-    if (IOHandler() && Access::READ_ONLY == IOHandler()->m_frontendAccess)
+    if (IOHandler() &&
+        IOHandler()->m_seriesStatus == internal::SeriesStatus::Default &&
+        Access::READ_ONLY == IOHandler()->m_frontendAccess)
     {
         auxiliary::OutOfRangeMsg const out_of_range_msg(
             "Attribute", "can not be set (read-only).");

include/openPMD/backend/Container.hpp

@@ -288,7 +288,9 @@ class Container : public Attributable
             return it->second;
         else
         {
-            if (Access::READ_ONLY == IOHandler()->m_frontendAccess)
+            if (IOHandler()->m_seriesStatus !=
+                    internal::SeriesStatus::Parsing &&
+                Access::READ_ONLY == IOHandler()->m_frontendAccess)
             {
                 auxiliary::OutOfRangeMsg const out_of_range_msg;
                 throw std::out_of_range(out_of_range_msg(key));
@@ -321,7 +323,9 @@ class Container : public Attributable
             return it->second;
         else
         {
-            if (Access::READ_ONLY == IOHandler()->m_frontendAccess)
+            if (IOHandler()->m_seriesStatus !=
+                    internal::SeriesStatus::Parsing &&
+                Access::READ_ONLY == IOHandler()->m_frontendAccess)
             {
                 auxiliary::OutOfRangeMsg out_of_range_msg;
                 throw std::out_of_range(out_of_range_msg(key));