From abccffdcf30a6d708da1bf44b9cd8cf2af8bed86 Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Fri, 5 Aug 2022 12:40:00 -0400
Subject: [PATCH 01/17] Rework BP5 Attribute handling

---
 source/adios2/core/Engine.cpp                 |   7 +
 source/adios2/core/Engine.h                   |   2 +
 source/adios2/core/IO.tcc                     |  88 +++++----
 source/adios2/engine/bp5/BP5Writer.cpp        |  20 +-
 source/adios2/engine/bp5/BP5Writer.h          |   2 +
 source/adios2/toolkit/format/bp5/BP5Base.h    |  87 +++++++++
 .../toolkit/format/bp5/BP5Deserializer.cpp    | 178 ++++++++++++++----
 .../toolkit/format/bp5/BP5Deserializer.h      |   9 +-
 .../toolkit/format/bp5/BP5Serializer.cpp      |  99 ++++++++++
 .../adios2/toolkit/format/bp5/BP5Serializer.h |   6 +
 10 files changed, 427 insertions(+), 71 deletions(-)

diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp
index 5ea591a6f3..ab9c1706d8 100644
--- a/source/adios2/core/Engine.cpp
+++ b/source/adios2/core/Engine.cpp
@@ -172,6 +172,13 @@ void Engine::InitTransports() {}
 
 void Engine::NotifyEngineAttribute(std::string name, DataType type) noexcept {}
 
+// if not overriden, default to name/type version
+void Engine::NotifyEngineAttribute(std::string name, AttributeBase *attr,
+                                   void *Data) noexcept
+{
+    NotifyEngineAttribute(name, attr->m_Type);
+}
+
 void Engine::NotifyEngineNoVarsQuery() {}
 
 // DoPut*
diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h
index cba7365da8..c102ef64ba 100644
--- a/source/adios2/core/Engine.h
+++ b/source/adios2/core/Engine.h
@@ -501,6 +501,8 @@ class Engine
      */
     virtual void NotifyEngineAttribute(std::string name,
                                        DataType type) noexcept;
+    virtual void NotifyEngineAttribute(std::string name, AttributeBase *attr,
+                                       void *Data) noexcept;
 
     /** Notify the engine when InquireVariable is called when the IO is empty.
      * Called from IO.tcc
diff --git a/source/adios2/core/IO.tcc b/source/adios2/core/IO.tcc
index 7eecb8b081..5763ca432f 100644
--- a/source/adios2/core/IO.tcc
+++ b/source/adios2/core/IO.tcc
@@ -126,31 +126,34 @@ Attribute<T> &IO::DefineAttribute(const std::string &name, const T &value,
     auto itExistingAttribute = m_Attributes.find(globalName);
     if (itExistingAttribute != m_Attributes.end())
     {
-        if (itExistingAttribute->second->m_Type == helper::GetDataType<T>())
+        if (helper::ValueToString(value) !=
+            itExistingAttribute->second->GetInfo()["Value"])
         {
-            if (!itExistingAttribute->second->Equals(
-                    static_cast<const void *>(&value), 1))
+            if (itExistingAttribute->second->m_Type == helper::GetDataType<T>())
             {
-
                 Attribute<T> &a =
                     static_cast<Attribute<T> &>(*itExistingAttribute->second);
+
                 a.Modify(value);
+                void *Data = &a.m_DataSingleValue;
+                if (a.m_DataArray.size() != 0)
+                    Data = a.m_DataArray.data();
                 for (auto &e : m_Engines)
                 {
                     e.second->NotifyEngineAttribute(
-                        globalName, itExistingAttribute->second->m_Type);
+                        globalName, itExistingAttribute->second.get(), Data);
                 }
             }
-        }
-        else
-        {
-            helper::Throw<std::invalid_argument>(
-                "Core", "IO", "DefineAttribute",
-                "modifiable attribute " + globalName +
-                    " has been defined with type " +
-                    ToString(itExistingAttribute->second->m_Type) +
-                    ". Type cannot be changed to " +
-                    ToString(helper::GetDataType<T>()));
+            else
+            {
+                helper::Throw<std::invalid_argument>(
+                    "Core", "IO", "DefineAttribute",
+                    "modifiable attribute " + globalName +
+                        " has been defined with type " +
+                        ToString(itExistingAttribute->second->m_Type) +
+                        ". Type cannot be changed to " +
+                        ToString(helper::GetDataType<T>()));
+            }
         }
         return static_cast<Attribute<T> &>(*itExistingAttribute->second);
     }
@@ -161,8 +164,16 @@ Attribute<T> &IO::DefineAttribute(const std::string &name, const T &value,
                             globalName, value, allowModification)));
         for (auto &e : m_Engines)
         {
+            Attribute<T> &a =
+                static_cast<Attribute<T> &>(*itAttributePair.first->second);
+            void *Data = &a.m_DataSingleValue;
+            if (a.m_DataArray.size() != 0)
+                Data = a.m_DataArray.data();
+
             e.second->NotifyEngineAttribute(
-                globalName, itAttributePair.first->second->m_Type);
+                globalName, itAttributePair.first->second.get(), Data);
+            std::cout << "In Notify First element of array is " << *(T *)Data
+                      << std::endl;
         }
         return static_cast<Attribute<T> &>(*itAttributePair.first->second);
     }
@@ -191,31 +202,36 @@ IO::DefineAttribute(const std::string &name, const T *array,
     auto itExistingAttribute = m_Attributes.find(globalName);
     if (itExistingAttribute != m_Attributes.end())
     {
-        if (itExistingAttribute->second->m_Type == helper::GetDataType<T>())
+        const std::string arrayValues(
+            "{ " +
+            helper::VectorToCSV(std::vector<T>(array, array + elements)) +
+            " }");
+
+        if (itExistingAttribute->second->GetInfo()["Value"] != arrayValues)
         {
-            if (!itExistingAttribute->second->Equals(
-                    static_cast<const void *>(array), elements))
+            if (itExistingAttribute->second->m_Type == helper::GetDataType<T>())
             {
-
                 Attribute<T> &a =
                     static_cast<Attribute<T> &>(*itExistingAttribute->second);
                 a.Modify(array, elements);
+                void *Data = &a.m_DataSingleValue;
+                if (a.m_DataArray.size() != 0)
+                    Data = a.m_DataArray.data();
                 for (auto &e : m_Engines)
                 {
-                    e.second->NotifyEngineAttribute(
-                        globalName, itExistingAttribute->second->m_Type);
+                    e.second->NotifyEngineAttribute(globalName, &a, Data);
                 }
             }
-        }
-        else
-        {
-            helper::Throw<std::invalid_argument>(
-                "Core", "IO", "DefineAttribute",
-                "modifiable attribute " + globalName +
-                    " has been defined with type " +
-                    ToString(itExistingAttribute->second->m_Type) +
-                    ". Type cannot be changed to " +
-                    ToString(helper::GetDataType<T>()));
+            else
+            {
+                helper::Throw<std::invalid_argument>(
+                    "Core", "IO", "DefineAttribute",
+                    "modifiable attribute " + globalName +
+                        " has been defined with type " +
+                        ToString(itExistingAttribute->second->m_Type) +
+                        ". Type cannot be changed to " +
+                        ToString(helper::GetDataType<T>()));
+            }
         }
         return static_cast<Attribute<T> &>(*itExistingAttribute->second);
     }
@@ -224,10 +240,14 @@ IO::DefineAttribute(const std::string &name, const T *array,
         auto itAttributePair = m_Attributes.emplace(
             globalName, std::unique_ptr<AttributeBase>(new Attribute<T>(
                             globalName, array, elements, allowModification)));
+        Attribute<T> &a =
+            static_cast<Attribute<T> &>(*itAttributePair.first->second);
+        void *Data = (void *)array;
         for (auto &e : m_Engines)
         {
-            e.second->NotifyEngineAttribute(
-                globalName, itAttributePair.first->second->m_Type);
+            e.second->NotifyEngineAttribute(globalName, &a, Data);
+            std::cout << "In Notify First element of array is " << *(T *)Data
+                      << std::endl;
         }
         return static_cast<Attribute<T> &>(*itAttributePair.first->second);
     }
diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp
index 2a1d878cfa..561d97edac 100644
--- a/source/adios2/engine/bp5/BP5Writer.cpp
+++ b/source/adios2/engine/bp5/BP5Writer.cpp
@@ -422,7 +422,25 @@ void BP5Writer::WriteMetadataFileIndex(uint64_t MetaDataPos,
 
 void BP5Writer::NotifyEngineAttribute(std::string name, DataType type) noexcept
 {
-    m_MarshalAttributesNecessary = true;
+    helper::Throw<std::invalid_argument>(
+        "BP5Writer", "Engine", "ThrowUp",
+        "Engine does not support NotifyEngineAttribute");
+}
+
+void BP5Writer::NotifyEngineAttribute(std::string name, AttributeBase *Attr,
+                                      void *data) noexcept
+{
+    if (Attr->m_IsSingleValue)
+    {
+        m_BP5Serializer.SoloSerializeAttribute(name.c_str(), Attr->m_Type,
+                                               (size_t)-1, data);
+    }
+    else
+    {
+        m_BP5Serializer.SoloSerializeAttribute(name.c_str(), Attr->m_Type,
+                                               Attr->m_Elements, data);
+    }
+    m_MarshalAttributesNecessary = false;
 }
 
 void BP5Writer::MarshalAttributes()
diff --git a/source/adios2/engine/bp5/BP5Writer.h b/source/adios2/engine/bp5/BP5Writer.h
index 027a89bab2..75538ca61a 100644
--- a/source/adios2/engine/bp5/BP5Writer.h
+++ b/source/adios2/engine/bp5/BP5Writer.h
@@ -115,6 +115,8 @@ class BP5Writer : public BP5Engine, public core::Engine
     /** Notify the engine when a new attribute is defined or modified. Called
      * from IO.tcc
      */
+    void NotifyEngineAttribute(std::string name, AttributeBase *Attr,
+                               void *data) noexcept;
 
     void EnterComputationBlock() noexcept;
     /** Inform about computation block through User->ADIOS->IO */
diff --git a/source/adios2/toolkit/format/bp5/BP5Base.h b/source/adios2/toolkit/format/bp5/BP5Base.h
index d9b2d17785..0e556b5090 100644
--- a/source/adios2/toolkit/format/bp5/BP5Base.h
+++ b/source/adios2/toolkit/format/bp5/BP5Base.h
@@ -80,6 +80,93 @@ class BP5Base
         size_t DataBlockSize;
     };
 
+    struct PrimitiveTypeAttr
+    {
+        const char *Name = NULL;
+        size_t TotalElementSize = 0;
+        char *Values;
+        ~PrimitiveTypeAttr()
+        {
+            free((void *)Name);
+            free((void *)Values);
+        }
+    };
+
+    struct StringArrayAttr
+    {
+        const char *Name = NULL;
+        size_t ElementCount = 0;
+        const char **Values = NULL;
+        ~StringArrayAttr()
+        {
+            free((void *)Name);
+            for (size_t i = 0; i < ElementCount; i++)
+                free((void *)Values[i]);
+        };
+    };
+
+    struct BP5AttrStruct
+    {
+        size_t PrimAttrCount = 0;
+        struct PrimitiveTypeAttr *PrimAttrs =
+            (struct PrimitiveTypeAttr *)malloc(1);
+        size_t StrAttrCount = 0;
+        struct StringArrayAttr *StrAttrs = (struct StringArrayAttr *)malloc(1);
+    };
+
+    size_t DataTypeSize[(int)DataType::Struct + 1] = {
+        0,                   // None
+        1,                   // Int8
+        2,                   // Int16
+        4,                   // Int32
+        8,                   // Int64
+        1,                   // UInt8
+        2,                   // UInt16
+        4,                   // UInt32
+        8,                   // UInt64
+        sizeof(float),       // Float
+        sizeof(double),      // Double
+        sizeof(long double), // LongDouble
+        sizeof(cfloat),      // FloatComplex
+        sizeof(cdouble),     // DoubleComplex
+        0,                   // String
+        1,                   // Char
+        0,                   // Struct
+    };
+
+    FMField prim_attr_field_list[4] = {
+        {"name", "string", sizeof(char *), FMOffset(PrimitiveTypeAttr *, Name)},
+        {"TotalElementSize", "integer", sizeof(size_t),
+         FMOffset(PrimitiveTypeAttr *, TotalElementSize)},
+        {"Values", "char[TotalElementSize]", 16,
+         FMOffset(PrimitiveTypeAttr *, Values)},
+        {NULL, NULL, 0, 0}};
+
+    FMField string_attr_field_list[4] = {
+        {"name", "string", sizeof(char *), FMOffset(StringArrayAttr *, Name)},
+        {"ElementCount", "integer", sizeof(size_t),
+         FMOffset(StringArrayAttr *, ElementCount)},
+        {"Values", "string[ElementCount]", sizeof(char *),
+         FMOffset(StringArrayAttr *, Values)},
+        {NULL, NULL, 0, 0}};
+
+    FMField bp5_attr_field_list[5] = {
+        {"PrimAttrCount", "integer", sizeof(size_t),
+         FMOffset(BP5AttrStruct *, PrimAttrCount)},
+        {"PrimAttrs", "PrimAttr[PrimAttrCount]", sizeof(PrimitiveTypeAttr),
+         FMOffset(BP5AttrStruct *, PrimAttrs)},
+        {"StrAttrCount", "integer", sizeof(size_t),
+         FMOffset(BP5AttrStruct *, StrAttrCount)},
+        {"StrAttrs", "StrAttr[StrAttrCount]", sizeof(StringArrayAttr),
+         FMOffset(BP5AttrStruct *, StrAttrs)},
+        {NULL, NULL, 0, 0}};
+
+    FMStructDescRec attr_struct_list[4] = {
+        {"GenericAttributes", bp5_attr_field_list, sizeof(BP5AttrStruct), NULL},
+        {"PrimAttr", prim_attr_field_list, sizeof(PrimitiveTypeAttr), NULL},
+        {"StrAttr", string_attr_field_list, sizeof(StringArrayAttr), NULL},
+        {NULL, NULL, 0, NULL}};
+
     void BP5BitfieldSet(struct BP5MetadataInfoStruct *MBase, int Bit) const;
     int BP5BitfieldTest(struct BP5MetadataInfoStruct *MBase, int Bit) const;
     FMField *MetaArrayRecListPtr;
diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
index d2bcf51f2f..71a448ce86 100644
--- a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
+++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
@@ -158,22 +158,17 @@ DataType BP5Deserializer::TranslateFFSType2ADIOS(const char *Type, int size)
     return DataType::None;
 }
 
-const char *BP5Deserializer::BreakdownVarName(const char *Name,
-                                              DataType *type_p,
-                                              int *element_size_p)
+void BP5Deserializer::BreakdownVarName(const char *Name, char **base_name_p,
+                                       DataType *type_p, int *element_size_p)
 {
-    // const char *NameStart = strchr(strchr(Name + 4, '_') + 1, '_') + 1;
-    // sscanf(Name + 4, "%d_%d", &ElementSize, &Type);
-    /* string formatted as bp5_%d_%d_actualname */
-    char *p;
-    // + 3 to skip BP5_ or bp5_ prefix
-    long n = strtol(Name + 4, &p, 10);
-    *element_size_p = static_cast<int>(n);
-    ++p; // skip '_'
-    long Type = strtol(p, &p, 10);
+    int Type;
+    int ElementSize;
+    const char *NameStart = strchr(strchr(Name + 4, '_') + 1, '_') + 1;
+    // + 4 to skip BP5_ or bp5_ prefix
+    sscanf(Name + 4, "%d_%d_", &ElementSize, &Type);
+    *element_size_p = ElementSize;
     *type_p = (DataType)Type;
-    ++p; // skip '_'
-    return p;
+    *base_name_p = strdup(NameStart);
 }
 
 void BP5Deserializer::BreakdownFieldType(const char *FieldType, bool &Operator,
@@ -237,16 +232,14 @@ void BP5Deserializer::BreakdownV1ArrayName(const char *Name, char **base_name_p,
 void BP5Deserializer::BreakdownArrayName(const char *Name, char **base_name_p,
                                          DataType *type_p, int *element_size_p)
 {
-    /* string formatted as bp5_%d_%d_actualname */
-    char *p;
+    int Type;
+    int ElementSize;
+    const char *NameStart = strchr(strchr(Name + 4, '_') + 1, '_') + 1;
     // + 3 to skip BP5_ or bp5_ prefix
-    long n = strtol(Name + 4, &p, 10);
-    *element_size_p = static_cast<int>(n);
-    ++p; // skip '_'
-    long Type = strtol(p, &p, 10);
+    sscanf(Name + 4, "%d_%d", &ElementSize, &Type);
+    *element_size_p = ElementSize;
     *type_p = (DataType)Type;
-    ++p; // skip '_'
-    *base_name_p = strdup(p);
+    *base_name_p = strdup(NameStart);
 }
 
 BP5Deserializer::BP5VarRec *BP5Deserializer::LookupVarByKey(void *Key) const
@@ -778,11 +771,6 @@ void BP5Deserializer::InstallAttributeData(void *AttributeBlock,
     if (BlockLen == 0)
         return;
 
-    if (Step != m_LastAttrStep)
-    {
-        m_Engine->m_IO.RemoveAllAttributes();
-        m_LastAttrStep = Step;
-    }
     FFSformat =
         FFSTypeHandle_from_encode(ReaderFFSContext, (char *)AttributeBlock);
     if (!FFSformat)
@@ -828,20 +816,50 @@ void BP5Deserializer::InstallAttributeData(void *AttributeBlock,
         FMdump_data(FMFormat_of_original(FFSformat), BaseData, 1024000);
         printf("\n\n");
     }
+    if (strcmp(name_of_FMformat(FMFormat_of_original(FFSformat)),
+               "GenericAttributes") == 0)
+    {
+        InstallAttributesV2(FFSformat, BaseData, Step);
+    }
+    else if (strcmp(name_of_FMformat(FMFormat_of_original(FFSformat)),
+                    "Attributes") == 0)
+    {
+        InstallAttributesV1(FFSformat, BaseData, Step);
+    }
+    else
+    {
+        helper::Throw<std::logic_error>("Toolkit", "format::BP5Deserializer",
+                                        "InstallAttributeData",
+                                        "Internal error or file corruption, "
+                                        "not able to install this format");
+    }
+}
+
+void BP5Deserializer::InstallAttributesV1(FFSTypeHandle FFSformat,
+                                          void *BaseData, size_t Step)
+{
+    FMFieldList FieldList;
+    FMStructDescList FormatList;
+
+    if (Step != m_LastAttrStep)
+    {
+        m_Engine->m_IO.RemoveAllAttributes();
+        m_LastAttrStep = Step;
+    }
     FormatList = format_list_of_FMFormat(FMFormat_of_original(FFSformat));
     FieldList = FormatList[0].field_list;
     int i = 0;
     while (FieldList[i].field_name)
     {
-
+        char *FieldName;
         void *field_data = (char *)BaseData + FieldList[i].field_offset;
 
         if (!NameIndicatesAttrArray(FieldList[i].field_name))
         {
             DataType Type;
             int ElemSize;
-            const char *FieldName =
-                BreakdownVarName(FieldList[i].field_name, &Type, &ElemSize);
+            BreakdownVarName(FieldList[i].field_name, &FieldName, &Type,
+                             &ElemSize);
             if (Type == adios2::DataType::Struct)
             {
                 return;
@@ -865,6 +883,7 @@ void BP5Deserializer::InstallAttributeData(void *AttributeBlock,
                 std::cout << "Loading attribute matched no type "
                           << ToString(Type) << std::endl;
             }
+            free(FieldName);
             i++;
         }
         else
@@ -915,6 +934,101 @@ void BP5Deserializer::InstallAttributeData(void *AttributeBlock,
     }
 }
 
+void BP5Deserializer::InstallAttributesV2(FFSTypeHandle FFSformat,
+                                          void *BaseData, size_t Step)
+{
+    BP5AttrStruct *Attrs = (BP5AttrStruct *)BaseData;
+
+    auto lf_BreakdownTypeArray = [](const char *Name, DataType &Type,
+                                    bool &Array) {
+        Type = (DataType)(Name[0] - '0');
+        Array = false;
+        if ((int)Type > 18)
+        {
+            Type = (DataType)((int)Type - 18);
+            Array = true;
+        }
+    };
+
+    for (size_t i = 0; i < Attrs->PrimAttrCount; i++)
+    {
+        PrimitiveTypeAttr *ThisAttr = &Attrs->PrimAttrs[i];
+        bool Array;
+        DataType Type;
+        lf_BreakdownTypeArray(ThisAttr->Name, Type, Array);
+        const char *Name = &ThisAttr->Name[1];
+        if (Array)
+        {
+            if (Type == adios2::DataType::Struct)
+            {
+                return;
+            }
+#define declare_type(T)                                                        \
+    else if (Type == helper::GetDataType<T>())                                 \
+    {                                                                          \
+        m_Engine->m_IO.DefineAttribute<T>(Name, (T *)ThisAttr->Values,         \
+                                          ThisAttr->TotalElementSize /         \
+                                              DataTypeSize[(int)Type],         \
+                                          "", "/", true);                      \
+    }
+
+            ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type)
+#undef declare_type
+            else
+            {
+                std::cout << "Loading attribute matched no type "
+                          << ToString(Type) << std::endl;
+            }
+        }
+        else
+        {
+            if (Type == adios2::DataType::Struct)
+            {
+                return;
+            }
+#define declare_type(T)                                                        \
+    else if (Type == helper::GetDataType<T>())                                 \
+    {                                                                          \
+        m_Engine->m_IO.DefineAttribute<T>(Name, *(T *)ThisAttr->Values, "",    \
+                                          "/", true);                          \
+    }
+
+            ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type)
+#undef declare_type
+            else
+            {
+                std::cout << "Loading attribute matched no type "
+                          << ToString(Type) << std::endl;
+            }
+        }
+    }
+    for (size_t i = 0; i < Attrs->StrAttrCount; i++)
+    {
+        auto *ThisAttr = &Attrs->StrAttrs[i];
+        bool Array;
+        DataType Type;
+        lf_BreakdownTypeArray(ThisAttr->Name, Type, Array);
+        const char *Name = &ThisAttr->Name[1];
+        if (Array)
+        {
+            std::vector<std::string> array;
+            array.resize(ThisAttr->ElementCount);
+            const char **str_array = ThisAttr->Values;
+            for (size_t i = 0; i < ThisAttr->ElementCount; i++)
+            {
+                array[i].assign(str_array[i]);
+            }
+            m_Engine->m_IO.DefineAttribute<std::string>(
+                Name, array.data(), array.size(), "", "/", true);
+        }
+        else
+        {
+            m_Engine->m_IO.DefineAttribute<std::string>(
+                Name, ThisAttr->Values[0], "", "/", true);
+        }
+    }
+}
+
 bool BP5Deserializer::QueueGet(core::VariableBase &variable, void *DestData)
 {
     if (!m_RandomAccessMode)
@@ -1302,10 +1416,6 @@ BP5Deserializer::GenerateReadRequests(const bool doAllocTempBuffers,
 void BP5Deserializer::FinalizeGet(const ReadRequest &Read, const bool freeAddr)
 {
     auto Req = PendingRequests[Read.ReqIndex];
-    /*std::cout << "    Req: block = " << Req.BlockID << " step = " << Req.Step
-              << " var = " << Req.VarRec->VarName << " start = " << Req.Start
-              << " count = " << Req.Count << " dest "
-              << reinterpret_cast<size_t>(Req.Data) << std::endl;*/
     int ElementSize = Req.VarRec->ElementSize;
     MetaArrayRec *writer_meta_base =
         (MetaArrayRec *)GetMetadataBase(Req.VarRec, Req.Step, Read.WriterRank);
diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.h b/source/adios2/toolkit/format/bp5/BP5Deserializer.h
index c6f082d014..5c8c2c5d50 100644
--- a/source/adios2/toolkit/format/bp5/BP5Deserializer.h
+++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.h
@@ -57,6 +57,11 @@ class BP5Deserializer : virtual public BP5Base
                          size_t WriterRank, size_t Step = SIZE_MAX);
     void InstallAttributeData(void *AttributeBlock, size_t BlockLen,
                               size_t Step = SIZE_MAX);
+    void InstallAttributesV1(FFSTypeHandle FFSformat, void *BaseData,
+                             size_t Step);
+    void InstallAttributesV2(FFSTypeHandle FFSformat, void *BaseData,
+                             size_t Step);
+
     void SetupForStep(size_t Step, size_t WriterCount);
     // return from QueueGet is true if a sync is needed to fill the data
     bool QueueGet(core::VariableBase &variable, void *DestData);
@@ -180,8 +185,8 @@ class BP5Deserializer : virtual public BP5Base
     BP5VarRec *LookupVarByName(const char *Name);
     BP5VarRec *CreateVarRec(const char *ArrayName);
     void ReverseDimensions(size_t *Dimensions, int count, int times);
-    const char *BreakdownVarName(const char *Name, DataType *type_p,
-                                 int *element_size_p);
+    void BreakdownVarName(const char *Name, char **base_name_p,
+                          DataType *type_p, int *element_size_p);
     void BreakdownFieldType(const char *FieldType, bool &Operator,
                             bool &MinMax);
     void BreakdownArrayName(const char *Name, char **base_name_p,
diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
index 3838ad5c94..db976c69db 100644
--- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
+++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
@@ -865,6 +865,76 @@ void BP5Serializer::MarshalAttribute(const char *Name, const DataType Type,
     }
 }
 
+void BP5Serializer::SoloSerializeAttribute(const char *Name,
+                                           const DataType Type,
+                                           size_t ElemCount, const void *Data)
+{
+    if (!PendingAttrs)
+        PendingAttrs = new (BP5AttrStruct);
+    char *TmpName = (char *)malloc(strlen(Name) + 2);
+    TmpName[0] = '0' + (int)Type;
+    if (ElemCount != (size_t)-1)
+        TmpName[0] += 18; // indicates an array
+    strcpy(&TmpName[1], Name);
+    if (Type == DataType::String)
+    {
+        PendingAttrs->StrAttrCount++;
+        PendingAttrs->StrAttrs = (struct StringArrayAttr *)realloc(
+            PendingAttrs->StrAttrs,
+            sizeof(StringArrayAttr) * PendingAttrs->StrAttrCount);
+        StringArrayAttr *ThisAttr =
+            &PendingAttrs->StrAttrs[PendingAttrs->StrAttrCount - 1];
+        memset(ThisAttr, 0, sizeof(*ThisAttr));
+        ThisAttr->Name = TmpName;
+        if (ElemCount == (size_t)-1)
+        {
+            std::string *Str = (std::string *)Data;
+            ThisAttr->ElementCount = 1;
+            ThisAttr->Values = (const char **)malloc(sizeof(char *));
+            ThisAttr->Values[0] = Str->c_str();
+        }
+        else
+        {
+            std::string *StrArray = (std::string *)Data;
+            ThisAttr->ElementCount = ElemCount;
+            ThisAttr->Values =
+                (const char **)malloc(sizeof(char *) * ElemCount);
+            for (size_t i = 0; i < ElemCount; i++)
+            {
+                ThisAttr->Values[i] = StrArray[i].c_str();
+            }
+        }
+    }
+    else
+    {
+        if ((Type == DataType::None) || (Type == DataType::Struct))
+        {
+            helper::Throw<std::logic_error>(
+                "Toolkit", "format::BP5Serializer",
+                "doesn't support this type of Attribute", ToString(Type));
+        }
+        char *Array = (char *)Data;
+        PendingAttrs->PrimAttrCount++;
+        PendingAttrs->PrimAttrs = (struct PrimitiveTypeAttr *)realloc(
+            PendingAttrs->PrimAttrs,
+            sizeof(PrimitiveTypeAttr) * PendingAttrs->PrimAttrCount);
+        PrimitiveTypeAttr *ThisAttr =
+            &PendingAttrs->PrimAttrs[PendingAttrs->PrimAttrCount - 1];
+        if (ElemCount == (size_t)-1)
+        {
+            ElemCount = 1;
+        }
+        memset(ThisAttr, 0, sizeof(*ThisAttr));
+        ThisAttr->Name = TmpName;
+        ThisAttr->TotalElementSize = ElemCount * DataTypeSize[(int)Type];
+        ThisAttr->Values = (char *)malloc(ThisAttr->TotalElementSize);
+        std::memcpy((void *)ThisAttr->Values, (void *)Array,
+                    ThisAttr->TotalElementSize);
+        std::cout << "Setting First element of array is "
+                  << *(double *)ThisAttr->Values << std::endl;
+    }
+}
+
 void BP5Serializer::InitStep(BufferV *DataBuffer)
 {
     if (CurDataBuffer != NULL)
@@ -1023,6 +1093,8 @@ BP5Serializer::TimestepInfo BP5Serializer::CloseTimestep(int timestep,
         new BufferFFS(MetaEncodeBuffer, MetaDataBlock, MetaDataSize);
 
     BufferFFS *AttrData = NULL;
+
+    // old way of doing attributes
     if (NewAttribute && Info.AttributeFields)
     {
         AttributeEncodeBuffer = create_FFSBuffer();
@@ -1033,6 +1105,33 @@ BP5Serializer::TimestepInfo BP5Serializer::CloseTimestep(int timestep,
             new BufferFFS(AttributeEncodeBuffer, AttributeBlock, AttributeSize);
     }
 
+    if (PendingAttrs)
+    {
+        if (!GenericAttributeFormat)
+        {
+            MetaMetaInfoBlock Block;
+            GenericAttributeFormat =
+                register_data_format(Info.LocalFMContext, &attr_struct_list[0]);
+            Info.AttributeFormat = GenericAttributeFormat;
+            int size;
+            Block.MetaMetaInfo =
+                get_server_rep_FMformat(GenericAttributeFormat, &size);
+            Block.MetaMetaInfoLen = size;
+            Block.MetaMetaID =
+                get_server_ID_FMformat(GenericAttributeFormat, &size);
+            Block.MetaMetaIDLen = size;
+            Formats.push_back(Block);
+        }
+        AttributeEncodeBuffer = create_FFSBuffer();
+        void *AttributeBlock =
+            FFSencode(AttributeEncodeBuffer, GenericAttributeFormat,
+                      PendingAttrs, &AttributeSize);
+        AttrData =
+            new BufferFFS(AttributeEncodeBuffer, AttributeBlock, AttributeSize);
+        //	FMdump_encoded_data(GenericAttributeFormat, AttributeBlock,
+        // 1024000);
+    }
+
     // FMdump_encoded_data(Info.MetaFormat, MetaDataBlock, 1024000);
     /* free all those copied dimensions, etc */
     MBase = (struct BP5MetadataInfoStruct *)Metadata;
diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.h b/source/adios2/toolkit/format/bp5/BP5Serializer.h
index e59558972b..b52104d11a 100644
--- a/source/adios2/toolkit/format/bp5/BP5Serializer.h
+++ b/source/adios2/toolkit/format/bp5/BP5Serializer.h
@@ -66,6 +66,8 @@ class BP5Serializer : virtual public BP5Base
                  bool Sync, BufferV::BufferPos *span);
     void MarshalAttribute(const char *Name, const DataType Type,
                           size_t ElemSize, size_t ElemCount, const void *Data);
+    void SoloSerializeAttribute(const char *Name, const DataType Type,
+                                size_t ElemCount, const void *Data);
 
     /*
      *  InitStep must be called with an appropriate BufferV subtype before a
@@ -153,6 +155,8 @@ class BP5Serializer : virtual public BP5Base
         int AttributeSize = 0;
     };
 
+    FMFormat GenericAttributeFormat = NULL;
+
     struct DeferredExtern
     {
         size_t MetaOffset;
@@ -175,6 +179,8 @@ class BP5Serializer : virtual public BP5Base
     };
     std::vector<DeferredSpanMinMax> DefSpanMinMax;
 
+    BP5AttrStruct *PendingAttrs = nullptr;
+
     FFSWriterMarshalBase Info;
     void *MetadataBuf = NULL;
     bool NewAttribute = false;

From d1f911b9d89e683115fe7e5b0fed8ab5f205f780 Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Fri, 5 Aug 2022 14:08:20 -0400
Subject: [PATCH 02/17] Warnings

---
 source/adios2/engine/bp4/BP4Writer.cpp               | 6 ++++++
 source/adios2/engine/bp4/BP4Writer.h                 | 2 ++
 source/adios2/toolkit/format/bp5/BP5Deserializer.cpp | 2 --
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/source/adios2/engine/bp4/BP4Writer.cpp b/source/adios2/engine/bp4/BP4Writer.cpp
index b13ea5f636..bcb86792b6 100644
--- a/source/adios2/engine/bp4/BP4Writer.cpp
+++ b/source/adios2/engine/bp4/BP4Writer.cpp
@@ -880,6 +880,12 @@ void BP4Writer::NotifyEngineAttribute(std::string name, DataType type) noexcept
     m_BP4Serializer.m_SerializedAttributes.erase(name);
 }
 
+void BP4Writer::NotifyEngineAttribute(std::string name, AttributeBase *attr,
+                                      void *Data) noexcept
+{
+    NotifyEngineAttribute(name, attr->m_Type);
+}
+
 } // end namespace engine
 } // end namespace core
 } // end namespace adios2
diff --git a/source/adios2/engine/bp4/BP4Writer.h b/source/adios2/engine/bp4/BP4Writer.h
index ebb5b52b43..056d8d4fa6 100644
--- a/source/adios2/engine/bp4/BP4Writer.h
+++ b/source/adios2/engine/bp4/BP4Writer.h
@@ -173,6 +173,8 @@ class BP4Writer : public core::Engine
     void PerformPutCommon(Variable<T> &variable);
 
     void NotifyEngineAttribute(std::string name, DataType type) noexcept;
+    virtual void NotifyEngineAttribute(std::string name, AttributeBase *attr,
+                                       void *Data) noexcept;
 };
 
 } // end namespace engine
diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
index 71a448ce86..517908373a 100644
--- a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
+++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
@@ -763,8 +763,6 @@ void BP5Deserializer::InstallAttributeData(void *AttributeBlock,
                                            size_t BlockLen, size_t Step)
 {
     static int DumpMetadata = -1;
-    FMFieldList FieldList;
-    FMStructDescList FormatList;
     void *BaseData;
     FFSTypeHandle FFSformat;
 

From f32bb989c5a5d0b754e6e37307b821c161cfa01e Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Fri, 5 Aug 2022 14:35:11 -0400
Subject: [PATCH 03/17] Warnings

---
 source/adios2/toolkit/format/bp5/BP5Base.h         | 11 -----------
 source/adios2/toolkit/format/bp5/BP5Serializer.cpp |  3 +++
 2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/source/adios2/toolkit/format/bp5/BP5Base.h b/source/adios2/toolkit/format/bp5/BP5Base.h
index 0e556b5090..6c5f99171b 100644
--- a/source/adios2/toolkit/format/bp5/BP5Base.h
+++ b/source/adios2/toolkit/format/bp5/BP5Base.h
@@ -85,11 +85,6 @@ class BP5Base
         const char *Name = NULL;
         size_t TotalElementSize = 0;
         char *Values;
-        ~PrimitiveTypeAttr()
-        {
-            free((void *)Name);
-            free((void *)Values);
-        }
     };
 
     struct StringArrayAttr
@@ -97,12 +92,6 @@ class BP5Base
         const char *Name = NULL;
         size_t ElementCount = 0;
         const char **Values = NULL;
-        ~StringArrayAttr()
-        {
-            free((void *)Name);
-            for (size_t i = 0; i < ElementCount; i++)
-                free((void *)Values[i]);
-        };
     };
 
     struct BP5AttrStruct
diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
index db976c69db..d36e92bff0 100644
--- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
+++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
@@ -1130,6 +1130,9 @@ BP5Serializer::TimestepInfo BP5Serializer::CloseTimestep(int timestep,
             new BufferFFS(AttributeEncodeBuffer, AttributeBlock, AttributeSize);
         //	FMdump_encoded_data(GenericAttributeFormat, AttributeBlock,
         // 1024000);
+        FMfree_var_rec_elements(GenericAttributeFormat, PendingAttrs);
+        delete (PendingAttrs);
+        PendingAttrs = nullptr;
     }
 
     // FMdump_encoded_data(Info.MetaFormat, MetaDataBlock, 1024000);

From a0f6c75a227c88cbc4943131831fd0eb582a2c99 Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Fri, 5 Aug 2022 16:15:50 -0400
Subject: [PATCH 04/17] Spurious output

---
 source/adios2/core/IO.tcc                          | 4 ----
 source/adios2/toolkit/format/bp5/BP5Serializer.cpp | 2 --
 2 files changed, 6 deletions(-)

diff --git a/source/adios2/core/IO.tcc b/source/adios2/core/IO.tcc
index 5763ca432f..1b07ee1f19 100644
--- a/source/adios2/core/IO.tcc
+++ b/source/adios2/core/IO.tcc
@@ -172,8 +172,6 @@ Attribute<T> &IO::DefineAttribute(const std::string &name, const T &value,
 
             e.second->NotifyEngineAttribute(
                 globalName, itAttributePair.first->second.get(), Data);
-            std::cout << "In Notify First element of array is " << *(T *)Data
-                      << std::endl;
         }
         return static_cast<Attribute<T> &>(*itAttributePair.first->second);
     }
@@ -246,8 +244,6 @@ IO::DefineAttribute(const std::string &name, const T *array,
         for (auto &e : m_Engines)
         {
             e.second->NotifyEngineAttribute(globalName, &a, Data);
-            std::cout << "In Notify First element of array is " << *(T *)Data
-                      << std::endl;
         }
         return static_cast<Attribute<T> &>(*itAttributePair.first->second);
     }
diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
index d36e92bff0..4a92bccbc3 100644
--- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
+++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
@@ -930,8 +930,6 @@ void BP5Serializer::SoloSerializeAttribute(const char *Name,
         ThisAttr->Values = (char *)malloc(ThisAttr->TotalElementSize);
         std::memcpy((void *)ThisAttr->Values, (void *)Array,
                     ThisAttr->TotalElementSize);
-        std::cout << "Setting First element of array is "
-                  << *(double *)ThisAttr->Values << std::endl;
     }
 }
 

From 581ca16a8a7edf885d7ec9702c313bc551cd581c Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Wed, 10 Aug 2022 17:47:15 -0400
Subject: [PATCH 05/17] Fix element len

---
 source/adios2/toolkit/format/bp5/BP5Base.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/adios2/toolkit/format/bp5/BP5Base.h b/source/adios2/toolkit/format/bp5/BP5Base.h
index 6c5f99171b..00bb31f9aa 100644
--- a/source/adios2/toolkit/format/bp5/BP5Base.h
+++ b/source/adios2/toolkit/format/bp5/BP5Base.h
@@ -127,7 +127,7 @@ class BP5Base
         {"name", "string", sizeof(char *), FMOffset(PrimitiveTypeAttr *, Name)},
         {"TotalElementSize", "integer", sizeof(size_t),
          FMOffset(PrimitiveTypeAttr *, TotalElementSize)},
-        {"Values", "char[TotalElementSize]", 16,
+        {"Values", "char[TotalElementSize]", 1,
          FMOffset(PrimitiveTypeAttr *, Values)},
         {NULL, NULL, 0, 0}};
 

From eec4f7af37f0ea6a5f54df0db2507532b58de987 Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Thu, 11 Aug 2022 09:57:49 -0400
Subject: [PATCH 06/17] msan

---
 source/adios2/toolkit/format/bp5/BP5Serializer.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
index 4a92bccbc3..ca13d92097 100644
--- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
+++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
@@ -891,7 +891,7 @@ void BP5Serializer::SoloSerializeAttribute(const char *Name,
             std::string *Str = (std::string *)Data;
             ThisAttr->ElementCount = 1;
             ThisAttr->Values = (const char **)malloc(sizeof(char *));
-            ThisAttr->Values[0] = Str->c_str();
+            ThisAttr->Values[0] = strdup(Str->c_str());
         }
         else
         {
@@ -901,7 +901,7 @@ void BP5Serializer::SoloSerializeAttribute(const char *Name,
                 (const char **)malloc(sizeof(char *) * ElemCount);
             for (size_t i = 0; i < ElemCount; i++)
             {
-                ThisAttr->Values[i] = StrArray[i].c_str();
+	        ThisAttr->Values[i] = strdup(StrArray[i].c_str());
             }
         }
     }

From 0bf8ac536d02b0940382b4b6665e75cdd06f75d7 Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Thu, 11 Aug 2022 11:09:47 -0400
Subject: [PATCH 07/17] clang-format

---
 source/adios2/toolkit/format/bp5/BP5Serializer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
index ca13d92097..97abf0c108 100644
--- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
+++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
@@ -901,7 +901,7 @@ void BP5Serializer::SoloSerializeAttribute(const char *Name,
                 (const char **)malloc(sizeof(char *) * ElemCount);
             for (size_t i = 0; i < ElemCount; i++)
             {
-	        ThisAttr->Values[i] = strdup(StrArray[i].c_str());
+                ThisAttr->Values[i] = strdup(StrArray[i].c_str());
             }
         }
     }

From bb80d85f869ab52b0eac411da54f8f0b70980cd0 Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Thu, 11 Aug 2022 11:51:18 -0400
Subject: [PATCH 08/17] avoid warning?

---
 source/adios2/toolkit/format/bp5/BP5Serializer.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
index 97abf0c108..a0668b5cbf 100644
--- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
+++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
@@ -884,7 +884,7 @@ void BP5Serializer::SoloSerializeAttribute(const char *Name,
             sizeof(StringArrayAttr) * PendingAttrs->StrAttrCount);
         StringArrayAttr *ThisAttr =
             &PendingAttrs->StrAttrs[PendingAttrs->StrAttrCount - 1];
-        memset(ThisAttr, 0, sizeof(*ThisAttr));
+        memset((void *)ThisAttr, 0, sizeof(*ThisAttr));
         ThisAttr->Name = TmpName;
         if (ElemCount == (size_t)-1)
         {
@@ -924,7 +924,7 @@ void BP5Serializer::SoloSerializeAttribute(const char *Name,
         {
             ElemCount = 1;
         }
-        memset(ThisAttr, 0, sizeof(*ThisAttr));
+        memset((void *)ThisAttr, 0, sizeof(*ThisAttr));
         ThisAttr->Name = TmpName;
         ThisAttr->TotalElementSize = ElemCount * DataTypeSize[(int)Type];
         ThisAttr->Values = (char *)malloc(ThisAttr->TotalElementSize);

From 4c4b8387d8926a655b16c225f34729c77fcc2b1f Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Thu, 11 Aug 2022 21:27:24 -0400
Subject: [PATCH 09/17] Add tests, handle pre-existing attrs

---
 source/adios2/engine/bp5/BP5Writer.cpp        | 61 +++++++++++++++++++
 testing/adios2/engine/common/CMakeLists.txt   |  8 +++
 .../engine/staging-common/CMakeLists.txt      |  2 +-
 3 files changed, 70 insertions(+), 1 deletion(-)

diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp
index 561d97edac..ba3f57ba93 100644
--- a/source/adios2/engine/bp5/BP5Writer.cpp
+++ b/source/adios2/engine/bp5/BP5Writer.cpp
@@ -72,6 +72,62 @@ StepStatus BP5Writer::BeginStep(StepMode mode, const float timeoutSeconds)
         }
     }
 
+    if ((m_WriterStep == 0) && !getenv("OldAttr"))
+    {
+	const auto &attributes = m_IO.GetAttributes();
+
+	for (const auto &attributePair : attributes)
+	{
+	    const std::string name(attributePair.first);
+	    auto baseAttr = &attributePair.second;
+	    const DataType type((*baseAttr)->m_Type);
+	    int element_count = -1;
+
+	    if (!attributePair.second->m_IsSingleValue)
+	    {
+		element_count = (*baseAttr)->m_Elements;
+	    }
+
+	    if (type == DataType::None)
+	    {
+	    }
+	    else if (type == helper::GetDataType<std::string>())
+	    {
+		core::Attribute<std::string> &attribute =
+		    *m_IO.InquireAttribute<std::string>(name);
+		void *data_addr;
+		if (attribute.m_IsSingleValue)
+		{
+		    data_addr = (void *)&attribute.m_DataSingleValue;
+		}
+		else
+		{
+		    data_addr = &attribute.m_DataArray[0];
+		}
+		m_BP5Serializer.SoloSerializeAttribute(name.c_str(), attribute.m_Type,
+						   (size_t)-1, data_addr);
+	    }
+#define declare_type(T)                                                        \
+	    else if (type == helper::GetDataType<T>())			\
+	    {								\
+		core::Attribute<T> &attribute = *m_IO.InquireAttribute<T>(name); \
+		int element_count = -1;					\
+		void *data_addr = &attribute.m_DataSingleValue;		\
+		if (!attribute.m_IsSingleValue)				\
+		{							\
+		    element_count = attribute.m_Elements;		\
+		    data_addr = attribute.m_DataArray.data();		\
+		}							\
+		m_BP5Serializer.SoloSerializeAttribute(name.c_str(), attribute.m_Type, \
+						       element_count, data_addr); \
+	    }
+
+        ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type)
+#undef declare_type
+	    }
+    }
+
+	
     if (m_Parameters.AsyncWrite)
     {
         m_AsyncWriteLock.lock();
@@ -430,6 +486,11 @@ void BP5Writer::NotifyEngineAttribute(std::string name, DataType type) noexcept
 void BP5Writer::NotifyEngineAttribute(std::string name, AttributeBase *Attr,
                                       void *data) noexcept
 {
+    if (getenv("OldAttr")) {
+	    m_MarshalAttributesNecessary = true;
+	    return;
+    }
+
     if (Attr->m_IsSingleValue)
     {
         m_BP5Serializer.SoloSerializeAttribute(name.c_str(), Attr->m_Type,
diff --git a/testing/adios2/engine/common/CMakeLists.txt b/testing/adios2/engine/common/CMakeLists.txt
index de4580db4e..72db89c08d 100644
--- a/testing/adios2/engine/common/CMakeLists.txt
+++ b/testing/adios2/engine/common/CMakeLists.txt
@@ -43,6 +43,14 @@ if(NOT MSVC)
     )
 endif()
 
+if (ADIOS2_HAVE_BP5)
+  gtest_add_tests_helper(Common MPI_ONLY "" Engine. .BP5
+    EXTRA_ARGS "BP5" "1")
+endif()
+
+  gtest_add_tests_helper(Common MPI_ONLY "" Engine. .BP3
+    EXTRA_ARGS "BP3" "1")
+
 #if(ADIOS2_HAVE_DataMan)
 #  gtest_add_tests_helper(Common MPI_ONLY "" Engine. .DataMan
 #    EXTRA_ARGS "DataMan" "0"
diff --git a/testing/adios2/engine/staging-common/CMakeLists.txt b/testing/adios2/engine/staging-common/CMakeLists.txt
index 8ce04ab9ab..7e9daee3cf 100644
--- a/testing/adios2/engine/staging-common/CMakeLists.txt
+++ b/testing/adios2/engine/staging-common/CMakeLists.txt
@@ -107,7 +107,7 @@ if(ADIOS2_HAVE_MPI AND MPIEXEC_EXECUTABLE)
    endforeach()
 endif()
 
-set (SIMPLE_TESTS "1x1;1x1DefSync;1x1VarDestruction;1x1.SpanMinMax")
+set (SIMPLE_TESTS "1x1;1x1.Attrs;1x1DefSync;1x1VarDestruction;1x1.SpanMinMax")
 set (BP5File_ONLY_TESTS "1x1DataWrite")
 
 set (SIMPLE_FORTRAN_TESTS "")

From 9a4121c26244f1416f2555a280bc0221df86de5a Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Thu, 11 Aug 2022 22:20:01 -0400
Subject: [PATCH 10/17] clang-format

---
 source/adios2/engine/bp5/BP5Writer.cpp | 104 ++++++++++++-------------
 1 file changed, 52 insertions(+), 52 deletions(-)

diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp
index ba3f57ba93..1239497f13 100644
--- a/source/adios2/engine/bp5/BP5Writer.cpp
+++ b/source/adios2/engine/bp5/BP5Writer.cpp
@@ -74,60 +74,59 @@ StepStatus BP5Writer::BeginStep(StepMode mode, const float timeoutSeconds)
 
     if ((m_WriterStep == 0) && !getenv("OldAttr"))
     {
-	const auto &attributes = m_IO.GetAttributes();
-
-	for (const auto &attributePair : attributes)
-	{
-	    const std::string name(attributePair.first);
-	    auto baseAttr = &attributePair.second;
-	    const DataType type((*baseAttr)->m_Type);
-	    int element_count = -1;
-
-	    if (!attributePair.second->m_IsSingleValue)
-	    {
-		element_count = (*baseAttr)->m_Elements;
-	    }
-
-	    if (type == DataType::None)
-	    {
-	    }
-	    else if (type == helper::GetDataType<std::string>())
-	    {
-		core::Attribute<std::string> &attribute =
-		    *m_IO.InquireAttribute<std::string>(name);
-		void *data_addr;
-		if (attribute.m_IsSingleValue)
-		{
-		    data_addr = (void *)&attribute.m_DataSingleValue;
-		}
-		else
-		{
-		    data_addr = &attribute.m_DataArray[0];
-		}
-		m_BP5Serializer.SoloSerializeAttribute(name.c_str(), attribute.m_Type,
-						   (size_t)-1, data_addr);
-	    }
+        const auto &attributes = m_IO.GetAttributes();
+
+        for (const auto &attributePair : attributes)
+        {
+            const std::string name(attributePair.first);
+            auto baseAttr = &attributePair.second;
+            const DataType type((*baseAttr)->m_Type);
+            int element_count = -1;
+
+            if (!attributePair.second->m_IsSingleValue)
+            {
+                element_count = (*baseAttr)->m_Elements;
+            }
+
+            if (type == DataType::None)
+            {
+            }
+            else if (type == helper::GetDataType<std::string>())
+            {
+                core::Attribute<std::string> &attribute =
+                    *m_IO.InquireAttribute<std::string>(name);
+                void *data_addr;
+                if (attribute.m_IsSingleValue)
+                {
+                    data_addr = (void *)&attribute.m_DataSingleValue;
+                }
+                else
+                {
+                    data_addr = &attribute.m_DataArray[0];
+                }
+                m_BP5Serializer.SoloSerializeAttribute(
+                    name.c_str(), attribute.m_Type, (size_t)-1, data_addr);
+            }
 #define declare_type(T)                                                        \
-	    else if (type == helper::GetDataType<T>())			\
-	    {								\
-		core::Attribute<T> &attribute = *m_IO.InquireAttribute<T>(name); \
-		int element_count = -1;					\
-		void *data_addr = &attribute.m_DataSingleValue;		\
-		if (!attribute.m_IsSingleValue)				\
-		{							\
-		    element_count = attribute.m_Elements;		\
-		    data_addr = attribute.m_DataArray.data();		\
-		}							\
-		m_BP5Serializer.SoloSerializeAttribute(name.c_str(), attribute.m_Type, \
-						       element_count, data_addr); \
-	    }
+    else if (type == helper::GetDataType<T>())                                 \
+    {                                                                          \
+        core::Attribute<T> &attribute = *m_IO.InquireAttribute<T>(name);       \
+        int element_count = -1;                                                \
+        void *data_addr = &attribute.m_DataSingleValue;                        \
+        if (!attribute.m_IsSingleValue)                                        \
+        {                                                                      \
+            element_count = attribute.m_Elements;                              \
+            data_addr = attribute.m_DataArray.data();                          \
+        }                                                                      \
+        m_BP5Serializer.SoloSerializeAttribute(name.c_str(), attribute.m_Type, \
+                                               element_count, data_addr);      \
+    }
 
-        ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type)
+            ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type)
 #undef declare_type
-	    }
+        }
     }
 
-	
     if (m_Parameters.AsyncWrite)
     {
         m_AsyncWriteLock.lock();
@@ -486,9 +485,10 @@ void BP5Writer::NotifyEngineAttribute(std::string name, DataType type) noexcept
 void BP5Writer::NotifyEngineAttribute(std::string name, AttributeBase *Attr,
                                       void *data) noexcept
 {
-    if (getenv("OldAttr")) {
-	    m_MarshalAttributesNecessary = true;
-	    return;
+    if (getenv("OldAttr"))
+    {
+        m_MarshalAttributesNecessary = true;
+        return;
     }
 
     if (Attr->m_IsSingleValue)

From 50ae4737147e880a81017fdfe7d8fbdea46c2642 Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Thu, 11 Aug 2022 22:42:05 -0400
Subject: [PATCH 11/17] Kill warning

---
 source/adios2/engine/bp5/BP5Writer.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp
index 1239497f13..c0de0de180 100644
--- a/source/adios2/engine/bp5/BP5Writer.cpp
+++ b/source/adios2/engine/bp5/BP5Writer.cpp
@@ -81,7 +81,6 @@ StepStatus BP5Writer::BeginStep(StepMode mode, const float timeoutSeconds)
             const std::string name(attributePair.first);
             auto baseAttr = &attributePair.second;
             const DataType type((*baseAttr)->m_Type);
-            int element_count = -1;
 
             if (!attributePair.second->m_IsSingleValue)
             {

From d3309b3a0223c2b90b5ec21ef7bf4b79b112fa0b Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Thu, 11 Aug 2022 22:55:38 -0400
Subject: [PATCH 12/17] Kill warning

---
 source/adios2/engine/bp5/BP5Writer.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp
index c0de0de180..aac5c7c6a8 100644
--- a/source/adios2/engine/bp5/BP5Writer.cpp
+++ b/source/adios2/engine/bp5/BP5Writer.cpp
@@ -81,6 +81,7 @@ StepStatus BP5Writer::BeginStep(StepMode mode, const float timeoutSeconds)
             const std::string name(attributePair.first);
             auto baseAttr = &attributePair.second;
             const DataType type((*baseAttr)->m_Type);
+            int element_count = -1;
 
             if (!attributePair.second->m_IsSingleValue)
             {
@@ -104,7 +105,8 @@ StepStatus BP5Writer::BeginStep(StepMode mode, const float timeoutSeconds)
                     data_addr = &attribute.m_DataArray[0];
                 }
                 m_BP5Serializer.SoloSerializeAttribute(
-                    name.c_str(), attribute.m_Type, (size_t)-1, data_addr);
+                    name.c_str(), attribute.m_Type, (size_t)element_count,
+                    data_addr);
             }
 #define declare_type(T)                                                        \
     else if (type == helper::GetDataType<T>())                                 \

From 5e03434c2d5d940f51183d07fc607aa8b85801a1 Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Sat, 13 Aug 2022 15:04:40 -0400
Subject: [PATCH 13/17] Refactor to avoid code duplication

---
 source/adios2/engine/bp5/BP5Writer.cpp        | 61 +------------------
 .../toolkit/format/bp5/BP5Serializer.cpp      | 42 ++++++++++++-
 .../adios2/toolkit/format/bp5/BP5Serializer.h |  3 +-
 3 files changed, 46 insertions(+), 60 deletions(-)

diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp
index aac5c7c6a8..edf05f1391 100644
--- a/source/adios2/engine/bp5/BP5Writer.cpp
+++ b/source/adios2/engine/bp5/BP5Writer.cpp
@@ -78,54 +78,8 @@ StepStatus BP5Writer::BeginStep(StepMode mode, const float timeoutSeconds)
 
         for (const auto &attributePair : attributes)
         {
-            const std::string name(attributePair.first);
-            auto baseAttr = &attributePair.second;
-            const DataType type((*baseAttr)->m_Type);
-            int element_count = -1;
-
-            if (!attributePair.second->m_IsSingleValue)
-            {
-                element_count = (*baseAttr)->m_Elements;
-            }
-
-            if (type == DataType::None)
-            {
-            }
-            else if (type == helper::GetDataType<std::string>())
-            {
-                core::Attribute<std::string> &attribute =
-                    *m_IO.InquireAttribute<std::string>(name);
-                void *data_addr;
-                if (attribute.m_IsSingleValue)
-                {
-                    data_addr = (void *)&attribute.m_DataSingleValue;
-                }
-                else
-                {
-                    data_addr = &attribute.m_DataArray[0];
-                }
-                m_BP5Serializer.SoloSerializeAttribute(
-                    name.c_str(), attribute.m_Type, (size_t)element_count,
-                    data_addr);
-            }
-#define declare_type(T)                                                        \
-    else if (type == helper::GetDataType<T>())                                 \
-    {                                                                          \
-        core::Attribute<T> &attribute = *m_IO.InquireAttribute<T>(name);       \
-        int element_count = -1;                                                \
-        void *data_addr = &attribute.m_DataSingleValue;                        \
-        if (!attribute.m_IsSingleValue)                                        \
-        {                                                                      \
-            element_count = attribute.m_Elements;                              \
-            data_addr = attribute.m_DataArray.data();                          \
-        }                                                                      \
-        m_BP5Serializer.SoloSerializeAttribute(name.c_str(), attribute.m_Type, \
-                                               element_count, data_addr);      \
-    }
-
-            ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type)
-#undef declare_type
-        }
+	    m_BP5Serializer.NewSerializeAttribute(*(attributePair.second));
+	}
     }
 
     if (m_Parameters.AsyncWrite)
@@ -492,16 +446,7 @@ void BP5Writer::NotifyEngineAttribute(std::string name, AttributeBase *Attr,
         return;
     }
 
-    if (Attr->m_IsSingleValue)
-    {
-        m_BP5Serializer.SoloSerializeAttribute(name.c_str(), Attr->m_Type,
-                                               (size_t)-1, data);
-    }
-    else
-    {
-        m_BP5Serializer.SoloSerializeAttribute(name.c_str(), Attr->m_Type,
-                                               Attr->m_Elements, data);
-    }
+    m_BP5Serializer.NewSerializeAttribute(*Attr);
     m_MarshalAttributesNecessary = false;
 }
 
diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
index a0668b5cbf..b1780cfdf9 100644
--- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
+++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
@@ -865,7 +865,47 @@ void BP5Serializer::MarshalAttribute(const char *Name, const DataType Type,
     }
 }
 
-void BP5Serializer::SoloSerializeAttribute(const char *Name,
+void BP5Serializer::NewSerializeAttribute(const core::AttributeBase &baseAttr)
+{
+    const char *Name = baseAttr.m_Name.c_str();
+    const DataType Type = baseAttr.m_Type;
+    size_t ElemCount = baseAttr.m_Elements;
+    const void *Data = nullptr;
+    if (baseAttr.m_IsSingleValue) ElemCount = (size_t) -1;
+    if (Type == DataType::None)
+    {
+	return;
+    }
+    else if (Type == helper::GetDataType<std::string>())
+    {
+	const core::Attribute<std::string> *attribute = dynamic_cast<const core::Attribute<std::string>*>(&baseAttr);
+	if (attribute->m_IsSingleValue)
+	{
+	    Data = (void *)&attribute->m_DataSingleValue;
+	}
+	else
+	{
+	    Data = &(attribute->m_DataArray[0]);
+	}
+    }
+#define per_type_code(T)                                                        \
+    else if (Type == helper::GetDataType<T>())				\
+    {								\
+	const core::Attribute<T> *attribute = dynamic_cast<const core::Attribute<T>*>(&baseAttr);\
+	Data = (void*)(&attribute->m_DataSingleValue);		\
+	if (!attribute->m_IsSingleValue)				\
+	{								\
+	    Data = (void*)attribute->m_DataArray.data();		\
+	}								\
+    }
+
+    ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(per_type_code)
+#undef per_type_code
+
+    NewSerializeAttribute(Name, Type, ElemCount, Data);
+}
+    
+void BP5Serializer::NewSerializeAttribute(const char *Name,
                                            const DataType Type,
                                            size_t ElemCount, const void *Data)
 {
diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.h b/source/adios2/toolkit/format/bp5/BP5Serializer.h
index b52104d11a..61d849d987 100644
--- a/source/adios2/toolkit/format/bp5/BP5Serializer.h
+++ b/source/adios2/toolkit/format/bp5/BP5Serializer.h
@@ -66,7 +66,8 @@ class BP5Serializer : virtual public BP5Base
                  bool Sync, BufferV::BufferPos *span);
     void MarshalAttribute(const char *Name, const DataType Type,
                           size_t ElemSize, size_t ElemCount, const void *Data);
-    void SoloSerializeAttribute(const char *Name, const DataType Type,
+    void NewSerializeAttribute(const core::AttributeBase &baseAttr);
+    void NewSerializeAttribute(const char *Name, const DataType Type,
                                 size_t ElemCount, const void *Data);
 
     /*

From aea6d5da4f0c942a5c693a4c8e9e7a261804d2f2 Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Sat, 13 Aug 2022 15:14:52 -0400
Subject: [PATCH 14/17] clang-format

---
 source/adios2/engine/bp5/BP5Writer.cpp        |  4 +-
 .../toolkit/format/bp5/BP5Serializer.cpp      | 50 ++++++++++---------
 .../adios2/toolkit/format/bp5/BP5Serializer.h |  2 +-
 3 files changed, 29 insertions(+), 27 deletions(-)

diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp
index edf05f1391..85b787709f 100644
--- a/source/adios2/engine/bp5/BP5Writer.cpp
+++ b/source/adios2/engine/bp5/BP5Writer.cpp
@@ -78,8 +78,8 @@ StepStatus BP5Writer::BeginStep(StepMode mode, const float timeoutSeconds)
 
         for (const auto &attributePair : attributes)
         {
-	    m_BP5Serializer.NewSerializeAttribute(*(attributePair.second));
-	}
+            m_BP5Serializer.NewSerializeAttribute(*(attributePair.second));
+        }
     }
 
     if (m_Parameters.AsyncWrite)
diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
index b1780cfdf9..3694c96394 100644
--- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
+++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
@@ -871,32 +871,35 @@ void BP5Serializer::NewSerializeAttribute(const core::AttributeBase &baseAttr)
     const DataType Type = baseAttr.m_Type;
     size_t ElemCount = baseAttr.m_Elements;
     const void *Data = nullptr;
-    if (baseAttr.m_IsSingleValue) ElemCount = (size_t) -1;
+    if (baseAttr.m_IsSingleValue)
+        ElemCount = (size_t)-1;
     if (Type == DataType::None)
     {
-	return;
+        return;
     }
     else if (Type == helper::GetDataType<std::string>())
     {
-	const core::Attribute<std::string> *attribute = dynamic_cast<const core::Attribute<std::string>*>(&baseAttr);
-	if (attribute->m_IsSingleValue)
-	{
-	    Data = (void *)&attribute->m_DataSingleValue;
-	}
-	else
-	{
-	    Data = &(attribute->m_DataArray[0]);
-	}
+        const core::Attribute<std::string> *attribute =
+            dynamic_cast<const core::Attribute<std::string> *>(&baseAttr);
+        if (attribute->m_IsSingleValue)
+        {
+            Data = (void *)&attribute->m_DataSingleValue;
+        }
+        else
+        {
+            Data = &(attribute->m_DataArray[0]);
+        }
     }
-#define per_type_code(T)                                                        \
-    else if (Type == helper::GetDataType<T>())				\
-    {								\
-	const core::Attribute<T> *attribute = dynamic_cast<const core::Attribute<T>*>(&baseAttr);\
-	Data = (void*)(&attribute->m_DataSingleValue);		\
-	if (!attribute->m_IsSingleValue)				\
-	{								\
-	    Data = (void*)attribute->m_DataArray.data();		\
-	}								\
+#define per_type_code(T)                                                       \
+    else if (Type == helper::GetDataType<T>())                                 \
+    {                                                                          \
+        const core::Attribute<T> *attribute =                                  \
+            dynamic_cast<const core::Attribute<T> *>(&baseAttr);               \
+        Data = (void *)(&attribute->m_DataSingleValue);                        \
+        if (!attribute->m_IsSingleValue)                                       \
+        {                                                                      \
+            Data = (void *)attribute->m_DataArray.data();                      \
+        }                                                                      \
     }
 
     ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(per_type_code)
@@ -904,10 +907,9 @@ void BP5Serializer::NewSerializeAttribute(const core::AttributeBase &baseAttr)
 
     NewSerializeAttribute(Name, Type, ElemCount, Data);
 }
-    
-void BP5Serializer::NewSerializeAttribute(const char *Name,
-                                           const DataType Type,
-                                           size_t ElemCount, const void *Data)
+
+void BP5Serializer::NewSerializeAttribute(const char *Name, const DataType Type,
+                                          size_t ElemCount, const void *Data)
 {
     if (!PendingAttrs)
         PendingAttrs = new (BP5AttrStruct);
diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.h b/source/adios2/toolkit/format/bp5/BP5Serializer.h
index 61d849d987..5e3bbf6508 100644
--- a/source/adios2/toolkit/format/bp5/BP5Serializer.h
+++ b/source/adios2/toolkit/format/bp5/BP5Serializer.h
@@ -68,7 +68,7 @@ class BP5Serializer : virtual public BP5Base
                           size_t ElemSize, size_t ElemCount, const void *Data);
     void NewSerializeAttribute(const core::AttributeBase &baseAttr);
     void NewSerializeAttribute(const char *Name, const DataType Type,
-                                size_t ElemCount, const void *Data);
+                               size_t ElemCount, const void *Data);
 
     /*
      *  InitStep must be called with an appropriate BufferV subtype before a

From 9d3aba28d2bd7530c76dcfd19ad626c12994c1e3 Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Sat, 13 Aug 2022 16:01:28 -0400
Subject: [PATCH 15/17] rename API and add description

---
 source/adios2/engine/bp5/BP5Writer.cpp        |  4 +--
 .../toolkit/format/bp5/BP5Serializer.cpp      |  9 ++++---
 .../adios2/toolkit/format/bp5/BP5Serializer.h | 26 ++++++++++++++++---
 3 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp
index 85b787709f..d2cbfddc6b 100644
--- a/source/adios2/engine/bp5/BP5Writer.cpp
+++ b/source/adios2/engine/bp5/BP5Writer.cpp
@@ -78,7 +78,7 @@ StepStatus BP5Writer::BeginStep(StepMode mode, const float timeoutSeconds)
 
         for (const auto &attributePair : attributes)
         {
-            m_BP5Serializer.NewSerializeAttribute(*(attributePair.second));
+            m_BP5Serializer.OnetimeMarshalAttribute(*(attributePair.second));
         }
     }
 
@@ -446,7 +446,7 @@ void BP5Writer::NotifyEngineAttribute(std::string name, AttributeBase *Attr,
         return;
     }
 
-    m_BP5Serializer.NewSerializeAttribute(*Attr);
+    m_BP5Serializer.OnetimeMarshalAttribute(*Attr);
     m_MarshalAttributesNecessary = false;
 }
 
diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
index 3694c96394..c9ac320191 100644
--- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
+++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
@@ -865,7 +865,7 @@ void BP5Serializer::MarshalAttribute(const char *Name, const DataType Type,
     }
 }
 
-void BP5Serializer::NewSerializeAttribute(const core::AttributeBase &baseAttr)
+void BP5Serializer::OnetimeMarshalAttribute(const core::AttributeBase &baseAttr)
 {
     const char *Name = baseAttr.m_Name.c_str();
     const DataType Type = baseAttr.m_Type;
@@ -905,11 +905,12 @@ void BP5Serializer::NewSerializeAttribute(const core::AttributeBase &baseAttr)
     ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(per_type_code)
 #undef per_type_code
 
-    NewSerializeAttribute(Name, Type, ElemCount, Data);
+    OnetimeMarshalAttribute(Name, Type, ElemCount, Data);
 }
 
-void BP5Serializer::NewSerializeAttribute(const char *Name, const DataType Type,
-                                          size_t ElemCount, const void *Data)
+void BP5Serializer::OnetimeMarshalAttribute(const char *Name,
+                                            const DataType Type,
+                                            size_t ElemCount, const void *Data)
 {
     if (!PendingAttrs)
         PendingAttrs = new (BP5AttrStruct);
diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.h b/source/adios2/toolkit/format/bp5/BP5Serializer.h
index 5e3bbf6508..c8b1a1baf7 100644
--- a/source/adios2/toolkit/format/bp5/BP5Serializer.h
+++ b/source/adios2/toolkit/format/bp5/BP5Serializer.h
@@ -64,11 +64,31 @@ class BP5Serializer : virtual public BP5Base
                  size_t ElemSize, size_t DimCount, const size_t *Shape,
                  const size_t *Count, const size_t *Offsets, const void *Data,
                  bool Sync, BufferV::BufferPos *span);
+    /*
+     * BP5 has two attribute marshalling methods.  The first,
+     * MarshallAttribute(), creates new MetaMetadata whenever a new
+     * attribute gets marshalled, and produces AttributeData that
+     * contains all extant attributes (so that only the most recent
+     * need be installed to the Deserializer).  The second,
+     * OnetimeMarshalAttribute(), produces MetaMetadata only on the
+     * first timestep, and produces AttributeData that contains only
+     * the attributes that were created or modified on that step (that
+     * is, each timesteps attribute data only contains the delta from
+     * the prior step).  Therefore for timestep X to have the right
+     * attributes *all* attribute data created prior to timestep X
+     * needs to be provided to the Deserializer (not just the most
+     * recent, as with the other approach).  The first approach is
+     * generally more space efficient and convenient for engines,
+     * provided that attributes are few and new attributes are rare.
+     * However, it performs very poorly if, for example, new
+     * attributes are produced on every timestep.  In the latter case,
+     * OnetimeMarshalAttribute() is by far better.
+     */
     void MarshalAttribute(const char *Name, const DataType Type,
                           size_t ElemSize, size_t ElemCount, const void *Data);
-    void NewSerializeAttribute(const core::AttributeBase &baseAttr);
-    void NewSerializeAttribute(const char *Name, const DataType Type,
-                               size_t ElemCount, const void *Data);
+    void OnetimeMarshalAttribute(const core::AttributeBase &baseAttr);
+    void OnetimeMarshalAttribute(const char *Name, const DataType Type,
+                                 size_t ElemCount, const void *Data);
 
     /*
      *  InitStep must be called with an appropriate BufferV subtype before a

From 4662ef2545ba44a5a063d4c5efb32fc2aaa416e3 Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Mon, 15 Aug 2022 15:57:30 -0400
Subject: [PATCH 16/17] Use engine params to control attribute marshalling

---
 source/adios2/engine/bp5/BP5Engine.h   |  1 +
 source/adios2/engine/bp5/BP5Writer.cpp |  4 ++--
 source/adios2/engine/sst/SstWriter.cpp | 27 ++++++++++++++++++++++----
 source/adios2/engine/sst/SstWriter.h   |  4 +++-
 source/adios2/toolkit/sst/sst_data.h   |  1 +
 5 files changed, 30 insertions(+), 7 deletions(-)

diff --git a/source/adios2/engine/bp5/BP5Engine.h b/source/adios2/engine/bp5/BP5Engine.h
index f836a4608f..d7b4bc8e92 100644
--- a/source/adios2/engine/bp5/BP5Engine.h
+++ b/source/adios2/engine/bp5/BP5Engine.h
@@ -160,6 +160,7 @@ class BP5Engine
     MACRO(StatsLevel, UInt, unsigned int, 1)                                   \
     MACRO(StatsBlockSize, SizeBytes, size_t, DefaultStatsBlockSize)            \
     MACRO(Threads, UInt, unsigned int, 0)                                      \
+    MACRO(UseOneTimeAttributes, Bool, bool, true)                              \
     MACRO(MaxOpenFilesAtOnce, UInt, unsigned int, UINT_MAX)
 
     struct BP5Params
diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp
index d2cbfddc6b..f7a2043fbd 100644
--- a/source/adios2/engine/bp5/BP5Writer.cpp
+++ b/source/adios2/engine/bp5/BP5Writer.cpp
@@ -72,7 +72,7 @@ StepStatus BP5Writer::BeginStep(StepMode mode, const float timeoutSeconds)
         }
     }
 
-    if ((m_WriterStep == 0) && !getenv("OldAttr"))
+    if ((m_WriterStep == 0) && m_Parameters.UseOneTimeAttributes)
     {
         const auto &attributes = m_IO.GetAttributes();
 
@@ -440,7 +440,7 @@ void BP5Writer::NotifyEngineAttribute(std::string name, DataType type) noexcept
 void BP5Writer::NotifyEngineAttribute(std::string name, AttributeBase *Attr,
                                       void *data) noexcept
 {
-    if (getenv("OldAttr"))
+    if (!m_Parameters.UseOneTimeAttributes)
     {
         m_MarshalAttributesNecessary = true;
         return;
diff --git a/source/adios2/engine/sst/SstWriter.cpp b/source/adios2/engine/sst/SstWriter.cpp
index c1fac36dc8..ee11c85b6e 100644
--- a/source/adios2/engine/sst/SstWriter.cpp
+++ b/source/adios2/engine/sst/SstWriter.cpp
@@ -2,7 +2,7 @@
  * Distributed under the OSI-approved Apache License, Version 2.0.  See
  * accompanying file Copyright.txt for details.
  *
- * Sst.cpp
+ * SstWriter.cpp
  *
  *  Created on: Aug 17, 2017
  *      Author: Greg Eisenhauer
@@ -187,10 +187,16 @@ void SstWriter::MarshalAttributes()
     PERFSTUBS_SCOPED_TIMER_FUNC();
     const auto &attributes = m_IO.GetAttributes();
 
-    const uint32_t attributesCount = static_cast<uint32_t>(attributes.size());
+    if ((m_WriterStep == 0) && Params.UseOneTimeAttributes)
+    {
+        for (const auto &attributePair : attributes)
+        {
+            m_BP5Serializer->OnetimeMarshalAttribute(*(attributePair.second));
+        }
+    }
 
     // if there are no new attributes, nothing to do
-    if (attributesCount == m_MarshaledAttributesCount)
+    if (!m_MarshalAttributesNecessary)
         return;
 
     for (const auto &attributePair : attributes)
@@ -245,7 +251,20 @@ void SstWriter::MarshalAttributes()
         ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type)
 #undef declare_type
     }
-    m_MarshaledAttributesCount = attributesCount;
+    m_MarshalAttributesNecessary = false;
+}
+
+void SstWriter::NotifyEngineAttribute(std::string name, AttributeBase *Attr,
+                                      void *data) noexcept
+{
+    if (!Params.UseOneTimeAttributes)
+    {
+        m_MarshalAttributesNecessary = true;
+        return;
+    }
+
+    m_BP5Serializer->OnetimeMarshalAttribute(*Attr);
+    m_MarshalAttributesNecessary = false;
 }
 
 void SstWriter::EndStep()
diff --git a/source/adios2/engine/sst/SstWriter.h b/source/adios2/engine/sst/SstWriter.h
index 9f6d984fdb..c28673066c 100644
--- a/source/adios2/engine/sst/SstWriter.h
+++ b/source/adios2/engine/sst/SstWriter.h
@@ -41,6 +41,8 @@ class SstWriter : public Engine
     void PerformPuts() final;
     void EndStep() final;
     void Flush(const int transportIndex = -1) final;
+    void NotifyEngineAttribute(std::string name, AttributeBase *Attr,
+                               void *data) noexcept;
 
     /**
      * Called if destructor is called on an open engine.  Should warn or take
@@ -85,7 +87,7 @@ class SstWriter : public Engine
     SstStream m_Output;
     long m_WriterStep = -1;
     bool m_DefinitionsNotified = false;
-    size_t m_MarshaledAttributesCount = 0;
+    bool m_MarshalAttributesNecessary = true; // first time through, marshal
     struct _SstParams Params;
 
     void MarshalAttributes();
diff --git a/source/adios2/toolkit/sst/sst_data.h b/source/adios2/toolkit/sst/sst_data.h
index 2494fa35f0..b613108770 100644
--- a/source/adios2/toolkit/sst/sst_data.h
+++ b/source/adios2/toolkit/sst/sst_data.h
@@ -77,6 +77,7 @@ typedef struct _SstStats
     MACRO(SpeculativePreloadMode, SpecPreloadMode, int, SpecPreloadAuto)       \
     MACRO(SpecAutoNodeThreshold, Int, int, 1)                                  \
     MACRO(ReaderShortCircuitReads, Bool, int, 0)                               \
+    MACRO(UseOneTimeAttributes, Bool, int, 0)                                  \
     MACRO(ControlModule, String, char *, NULL)
 
 typedef enum

From a40987c1df9af1e246d015f97bb8dc6f8c90211c Mon Sep 17 00:00:00 2001
From: Greg Eisenhauer <eisen@cc.gatech.edu>
Date: Mon, 15 Aug 2022 16:12:35 -0400
Subject: [PATCH 17/17] Override both NotifyEngineAttribute functions

---
 source/adios2/engine/sst/SstWriter.cpp | 7 +++++++
 source/adios2/engine/sst/SstWriter.h   | 1 +
 2 files changed, 8 insertions(+)

diff --git a/source/adios2/engine/sst/SstWriter.cpp b/source/adios2/engine/sst/SstWriter.cpp
index ee11c85b6e..ccaef60f59 100644
--- a/source/adios2/engine/sst/SstWriter.cpp
+++ b/source/adios2/engine/sst/SstWriter.cpp
@@ -254,6 +254,13 @@ void SstWriter::MarshalAttributes()
     m_MarshalAttributesNecessary = false;
 }
 
+void SstWriter::NotifyEngineAttribute(std::string name, DataType type) noexcept
+{
+    helper::Throw<std::invalid_argument>(
+        "SstWriter", "Engine", "ThrowUp",
+        "Engine does not support NotifyEngineAttribute");
+}
+
 void SstWriter::NotifyEngineAttribute(std::string name, AttributeBase *Attr,
                                       void *data) noexcept
 {
diff --git a/source/adios2/engine/sst/SstWriter.h b/source/adios2/engine/sst/SstWriter.h
index c28673066c..37c084fbb6 100644
--- a/source/adios2/engine/sst/SstWriter.h
+++ b/source/adios2/engine/sst/SstWriter.h
@@ -41,6 +41,7 @@ class SstWriter : public Engine
     void PerformPuts() final;
     void EndStep() final;
     void Flush(const int transportIndex = -1) final;
+    void NotifyEngineAttribute(std::string name, DataType type) noexcept;
     void NotifyEngineAttribute(std::string name, AttributeBase *Attr,
                                void *data) noexcept;