Merge pull request #3591 from eisenhauer/BP5MemSel

Implement BP5 (and BP4) reader-side memory selection, do testing
ornladios · May 3, 2023 · 523d6a0 · 523d6a0
2 parents 50aa9a3 + 70f330c
commit 523d6a0
Show file tree

Hide file tree

Showing 6 changed files with 183 additions and 38 deletions.
diff --git a/source/adios2/helper/adiosType.h b/source/adios2/helper/adiosType.h
@@ -134,6 +134,17 @@ class CoreDims
     typedef size_t *iterator;
     iterator begin() { return &DimensSpan[0]; }
     iterator end() { return &DimensSpan[DimCount]; }
+    friend std::ostream &operator<<(std::ostream &os, const CoreDims &m)
+    {
+        os << "{";
+        for (size_t i = 0; i < m.size(); i++)
+        {
+            os << m[i];
+            if (i < m.size() - 1)
+                os << ", ";
+        }
+        return os << "}";
+    }
 };
 
 class DimsArray : public CoreDims

diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc
@@ -598,11 +598,55 @@ void BP4Deserializer::PostDataRead(
             ? Dims(blockInfo.Count.size(), 0)
             : blockInfo.Start;
 
-    helper::ClipContiguousMemory(
-        blockInfo.Data, blockInfoStart, blockInfo.Count,
-        m_ThreadBuffers[threadID][0].data(), subStreamBoxInfo.BlockBox,
-        subStreamBoxInfo.IntersectionBox, m_IsRowMajor, m_ReverseDimensions,
-        endianReverse, blockInfo.MemSpace);
+    if (!blockInfo.MemoryStart.empty())
+    {
+        if (endianReverse)
+        {
+            helper::Throw<std::invalid_argument>(
+                "Toolkit", "format::bp::BP4Deserializer", "PostDataRead",
+                "endianReverse "
+                "not supported with MemorySelection");
+        }
+
+        if (m_ReverseDimensions)
+        {
+            helper::Throw<std::invalid_argument>(
+                "Toolkit", "format::bp::BP4Deserializer", "PostDataRead",
+                "ReverseDimensions not supported with "
+                "MemorySelection");
+        }
+
+        helper::DimsArray intersectStart(
+            subStreamBoxInfo.IntersectionBox.first);
+        helper::DimsArray intersectCount(
+            subStreamBoxInfo.IntersectionBox.second);
+        helper::DimsArray blockStart(subStreamBoxInfo.BlockBox.first);
+        helper::DimsArray blockCount(subStreamBoxInfo.BlockBox.second);
+        helper::DimsArray memoryStart(blockInfoStart);
+        for (size_t d = 0; d < intersectStart.size(); d++)
+        {
+            // change {intersect,block}Count from [start, end] to {start, count}
+            intersectCount[d] -= (intersectStart[d] - 1);
+            blockCount[d] -= (blockStart[d] - 1);
+            // shift everything by MemoryStart
+            intersectStart[d] += blockInfo.MemoryStart[d];
+            blockStart[d] += blockInfo.MemoryStart[d];
+        }
+        helper::NdCopy(m_ThreadBuffers[threadID][0].data(), intersectStart,
+                       intersectCount, true, true,
+                       reinterpret_cast<char *>(blockInfo.Data), intersectStart,
+                       intersectCount, true, true, sizeof(T), intersectStart,
+                       blockCount, memoryStart,
+                       helper::DimsArray(blockInfo.MemoryCount), false);
+    }
+    else
+    {
+        helper::ClipContiguousMemory(
+            blockInfo.Data, blockInfoStart, blockInfo.Count,
+            m_ThreadBuffers[threadID][0].data(), subStreamBoxInfo.BlockBox,
+            subStreamBoxInfo.IntersectionBox, m_IsRowMajor, m_ReverseDimensions,
+            endianReverse, blockInfo.MemSpace);
+    }
 }
 
 void BP4Deserializer::BackCompatDecompress(

diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp
@@ -1473,7 +1473,9 @@ static bool IntersectionStartCount(const size_t dimensionsSize,
         outstart[d] = intersectionStart;
         outcount[d] = intersectionEnd - intersectionStart + 1;
         if (outcount[d] == 0)
+        {
             return false;
+        }
     }
     return true;
 }
@@ -1822,22 +1824,79 @@ void BP5Deserializer::FinalizeGet(const ReadRequest &Read, const bool freeAddr)
         }
     }
 
+    VariableBase *VB = static_cast<VariableBase *>(Req.VarRec->Variable);
     DimsArray inStart(DimCount, RankOffset);
     DimsArray inCount(DimCount, RankSize);
     DimsArray outStart(DimCount, SelOffset);
     DimsArray outCount(DimCount, SelSize);
+    DimsArray outMemStart(VB->m_MemoryStart);
+    DimsArray outMemCount(VB->m_MemoryCount);
     if (!m_ReaderIsRowMajor)
     {
         std::reverse(inStart.begin(), inStart.end());
         std::reverse(inCount.begin(), inCount.end());
         std::reverse(outStart.begin(), outStart.end());
         std::reverse(outCount.begin(), outCount.end());
+        std::reverse(outMemStart.begin(), outMemStart.end());
+        std::reverse(outMemCount.begin(), outMemCount.end());
     }
 
-    helper::NdCopy(VirtualIncomingData, inStart, inCount, true, true,
-                   (char *)Req.Data, outStart, outCount, true, true,
-                   ElementSize, CoreDims(), CoreDims(), CoreDims(), CoreDims(),
-                   false, Req.MemSpace);
+    if (VB->m_MemoryStart.size() > 0)
+    {
+#ifdef NOTDEF // haven't done endinness for BP5
+        if (endianReverse)
+        {
+            helper::Throw<std::invalid_argument>(
+                "Toolkit", "format::bp::BP5Deserializer", "PostDataRead",
+                "endianReverse "
+                "not supported with MemorySelection");
+        }
+        if (m_ReaderIsRowMajor)
+        {
+            helper::Throw<std::invalid_argument>(
+                "Toolkit", "format::bp::BP5Deserializer", "PostDataRead",
+                "ReverseDimensions not supported with "
+                "MemorySelection");
+        }
+#endif
+
+        const Box<Dims> selectionBox =
+            helper::StartEndBox(Dims(outStart.begin(), outStart.end()),
+                                Dims(outCount.begin(), outCount.end()));
+        const Box<Dims> BlockBox =
+            helper::StartEndBox(Dims(inStart.begin(), inStart.end()),
+                                Dims(inCount.begin(), inCount.end()));
+        const Box<Dims> IntersectionBox =
+            helper::IntersectionBox(selectionBox, BlockBox);
+        VirtualIncomingData =
+            Read.DestinationAddr; //  Don't do the fake offset thing
+        helper::DimsArray intersectStart(IntersectionBox.first);
+        helper::DimsArray intersectCount(IntersectionBox.second);
+        helper::DimsArray blockStart(BlockBox.first);
+        helper::DimsArray blockCount(BlockBox.second);
+        helper::DimsArray memoryStart(DimCount, SelOffset);
+        helper::DimsArray memoryCount(VB->m_MemoryCount);
+        for (size_t d = 0; d < intersectStart.size(); d++)
+        {
+            // change {intersect,block}Count from [start, end] to {start, count}
+            intersectCount[d] -= (intersectStart[d] - 1);
+            blockCount[d] -= (blockStart[d] - 1);
+            // shift everything by MemoryStart
+            intersectStart[d] += VB->m_MemoryStart[d];
+            blockStart[d] += VB->m_MemoryStart[d];
+        }
+        helper::NdCopy(VirtualIncomingData, intersectStart, intersectCount,
+                       true, true, (char *)Req.Data, intersectStart,
+                       intersectCount, true, true, ElementSize, intersectStart,
+                       blockCount, memoryStart, memoryCount, false);
+    }
+    else
+    {
+        helper::NdCopy(VirtualIncomingData, inStart, inCount, true, true,
+                       (char *)Req.Data, outStart, outCount, true, true,
+                       ElementSize, CoreDims(), CoreDims(), CoreDims(),
+                       CoreDims(), false, Req.MemSpace);
+    }
     if (freeAddr)
     {
         free((char *)Read.DestinationAddr);

diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp
@@ -894,7 +894,7 @@ void BP5Serializer::Marshal(void *Variable, const char *Name,
                     lf_QueueSpanMinMax(*Span, ElemCount, (DataType)Rec->Type,
                                        MemSpace, Rec->MetaOffset,
                                        Rec->MinMaxOffset,
-                                       MetaEntry->BlockCount /*BlockNum*/);
+                                       MetaEntry->BlockCount - 1 /*BlockNum*/);
                 }
             }
 

diff --git a/testing/adios2/interface/CMakeLists.txt b/testing/adios2/interface/CMakeLists.txt
@@ -6,9 +6,26 @@
 # These tests should be *very* fast
 set(CTEST_TEST_TIMEOUT 10)
 
-gtest_add_tests_helper(Interface MPI_ALLOW ADIOS Interface. "")
+set(BP3_DIR ${CMAKE_CURRENT_BINARY_DIR}/bp3)
+set(BP4_DIR ${CMAKE_CURRENT_BINARY_DIR}/bp4)
+set(BP5_DIR ${CMAKE_CURRENT_BINARY_DIR}/bp5)
+set(BPfile_DIR ${CMAKE_CURRENT_BINARY_DIR}/bpfile)
+set(FS_DIR ${CMAKE_CURRENT_BINARY_DIR}/filestream)
+file(MAKE_DIRECTORY ${BP3_DIR})
+file(MAKE_DIRECTORY ${BP4_DIR})
+file(MAKE_DIRECTORY ${BP5_DIR})
+file(MAKE_DIRECTORY ${BPfile_DIR})
+file(MAKE_DIRECTORY ${FS_DIR})
+
+gtest_add_tests_helper(Interface MPI_ALLOW ADIOS Interface.  .BP3
+    WORKING_DIRECTORY ${BP3_DIR} EXTRA_ARGS "BP3")
+gtest_add_tests_helper(Interface MPI_ALLOW ADIOS Interface.  .BPfile
+    WORKING_DIRECTORY ${BPfile_DIR} EXTRA_ARGS "BPfile")
 gtest_add_tests_helper(Write MPI_ALLOW ADIOSInterface Interface. "")
 gtest_add_tests_helper(DefineVariable MPI_ALLOW ADIOS Interface. "")
 gtest_add_tests_helper(DefineAttribute MPI_ALLOW ADIOS Interface. "")
-gtest_add_tests_helper(Selection MPI_NONE ADIOS Interface. "")
+gtest_add_tests_helper(Selection MPI_NONE ADIOS Interface.  .BP3
+    WORKING_DIRECTORY ${BP3_DIR} EXTRA_ARGS "BP3")
+gtest_add_tests_helper(Selection MPI_NONE ADIOS Interface.  .BPfile
+    WORKING_DIRECTORY ${BPfile_DIR} EXTRA_ARGS "BPfile")
 gtest_add_tests_helper(NoMpi MPI_NONE ADIOS Interface. "")