diff --git a/source/adios2/helper/adiosType.h b/source/adios2/helper/adiosType.h index 4a329a4b11..c7828e3098 100644 --- a/source/adios2/helper/adiosType.h +++ b/source/adios2/helper/adiosType.h @@ -134,6 +134,17 @@ class CoreDims typedef size_t *iterator; iterator begin() { return &DimensSpan[0]; } iterator end() { return &DimensSpan[DimCount]; } + friend std::ostream &operator<<(std::ostream &os, const CoreDims &m) + { + os << "{"; + for (size_t i = 0; i < m.size(); i++) + { + os << m[i]; + if (i < m.size() - 1) + os << ", "; + } + return os << "}"; + } }; class DimsArray : public CoreDims diff --git a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc index c1494afbfb..a9fcca30f0 100644 --- a/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc +++ b/source/adios2/toolkit/format/bp/bp4/BP4Deserializer.tcc @@ -598,11 +598,55 @@ void BP4Deserializer::PostDataRead( ? Dims(blockInfo.Count.size(), 0) : blockInfo.Start; - helper::ClipContiguousMemory( - blockInfo.Data, blockInfoStart, blockInfo.Count, - m_ThreadBuffers[threadID][0].data(), subStreamBoxInfo.BlockBox, - subStreamBoxInfo.IntersectionBox, m_IsRowMajor, m_ReverseDimensions, - endianReverse, blockInfo.MemSpace); + if (!blockInfo.MemoryStart.empty()) + { + if (endianReverse) + { + helper::Throw( + "Toolkit", "format::bp::BP4Deserializer", "PostDataRead", + "endianReverse " + "not supported with MemorySelection"); + } + + if (m_ReverseDimensions) + { + helper::Throw( + "Toolkit", "format::bp::BP4Deserializer", "PostDataRead", + "ReverseDimensions not supported with " + "MemorySelection"); + } + + helper::DimsArray intersectStart( + subStreamBoxInfo.IntersectionBox.first); + helper::DimsArray intersectCount( + subStreamBoxInfo.IntersectionBox.second); + helper::DimsArray blockStart(subStreamBoxInfo.BlockBox.first); + helper::DimsArray blockCount(subStreamBoxInfo.BlockBox.second); + helper::DimsArray memoryStart(blockInfoStart); + for (size_t d = 0; d < intersectStart.size(); d++) + { + // change {intersect,block}Count from [start, end] to {start, count} + intersectCount[d] -= (intersectStart[d] - 1); + blockCount[d] -= (blockStart[d] - 1); + // shift everything by MemoryStart + intersectStart[d] += blockInfo.MemoryStart[d]; + blockStart[d] += blockInfo.MemoryStart[d]; + } + helper::NdCopy(m_ThreadBuffers[threadID][0].data(), intersectStart, + intersectCount, true, true, + reinterpret_cast(blockInfo.Data), intersectStart, + intersectCount, true, true, sizeof(T), intersectStart, + blockCount, memoryStart, + helper::DimsArray(blockInfo.MemoryCount), false); + } + else + { + helper::ClipContiguousMemory( + blockInfo.Data, blockInfoStart, blockInfo.Count, + m_ThreadBuffers[threadID][0].data(), subStreamBoxInfo.BlockBox, + subStreamBoxInfo.IntersectionBox, m_IsRowMajor, m_ReverseDimensions, + endianReverse, blockInfo.MemSpace); + } } void BP4Deserializer::BackCompatDecompress( diff --git a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp index 3d648779ec..9a0c98c432 100644 --- a/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Deserializer.cpp @@ -1473,7 +1473,9 @@ static bool IntersectionStartCount(const size_t dimensionsSize, outstart[d] = intersectionStart; outcount[d] = intersectionEnd - intersectionStart + 1; if (outcount[d] == 0) + { return false; + } } return true; } @@ -1822,22 +1824,79 @@ void BP5Deserializer::FinalizeGet(const ReadRequest &Read, const bool freeAddr) } } + VariableBase *VB = static_cast(Req.VarRec->Variable); DimsArray inStart(DimCount, RankOffset); DimsArray inCount(DimCount, RankSize); DimsArray outStart(DimCount, SelOffset); DimsArray outCount(DimCount, SelSize); + DimsArray outMemStart(VB->m_MemoryStart); + DimsArray outMemCount(VB->m_MemoryCount); if (!m_ReaderIsRowMajor) { std::reverse(inStart.begin(), inStart.end()); std::reverse(inCount.begin(), inCount.end()); std::reverse(outStart.begin(), outStart.end()); std::reverse(outCount.begin(), outCount.end()); + std::reverse(outMemStart.begin(), outMemStart.end()); + std::reverse(outMemCount.begin(), outMemCount.end()); } - helper::NdCopy(VirtualIncomingData, inStart, inCount, true, true, - (char *)Req.Data, outStart, outCount, true, true, - ElementSize, CoreDims(), CoreDims(), CoreDims(), CoreDims(), - false, Req.MemSpace); + if (VB->m_MemoryStart.size() > 0) + { +#ifdef NOTDEF // haven't done endinness for BP5 + if (endianReverse) + { + helper::Throw( + "Toolkit", "format::bp::BP5Deserializer", "PostDataRead", + "endianReverse " + "not supported with MemorySelection"); + } + if (m_ReaderIsRowMajor) + { + helper::Throw( + "Toolkit", "format::bp::BP5Deserializer", "PostDataRead", + "ReverseDimensions not supported with " + "MemorySelection"); + } +#endif + + const Box selectionBox = + helper::StartEndBox(Dims(outStart.begin(), outStart.end()), + Dims(outCount.begin(), outCount.end())); + const Box BlockBox = + helper::StartEndBox(Dims(inStart.begin(), inStart.end()), + Dims(inCount.begin(), inCount.end())); + const Box IntersectionBox = + helper::IntersectionBox(selectionBox, BlockBox); + VirtualIncomingData = + Read.DestinationAddr; // Don't do the fake offset thing + helper::DimsArray intersectStart(IntersectionBox.first); + helper::DimsArray intersectCount(IntersectionBox.second); + helper::DimsArray blockStart(BlockBox.first); + helper::DimsArray blockCount(BlockBox.second); + helper::DimsArray memoryStart(DimCount, SelOffset); + helper::DimsArray memoryCount(VB->m_MemoryCount); + for (size_t d = 0; d < intersectStart.size(); d++) + { + // change {intersect,block}Count from [start, end] to {start, count} + intersectCount[d] -= (intersectStart[d] - 1); + blockCount[d] -= (blockStart[d] - 1); + // shift everything by MemoryStart + intersectStart[d] += VB->m_MemoryStart[d]; + blockStart[d] += VB->m_MemoryStart[d]; + } + helper::NdCopy(VirtualIncomingData, intersectStart, intersectCount, + true, true, (char *)Req.Data, intersectStart, + intersectCount, true, true, ElementSize, intersectStart, + blockCount, memoryStart, memoryCount, false); + } + else + { + helper::NdCopy(VirtualIncomingData, inStart, inCount, true, true, + (char *)Req.Data, outStart, outCount, true, true, + ElementSize, CoreDims(), CoreDims(), CoreDims(), + CoreDims(), false, Req.MemSpace); + } if (freeAddr) { free((char *)Read.DestinationAddr); diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp index 5f072d4a76..06d2176856 100644 --- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp @@ -894,7 +894,7 @@ void BP5Serializer::Marshal(void *Variable, const char *Name, lf_QueueSpanMinMax(*Span, ElemCount, (DataType)Rec->Type, MemSpace, Rec->MetaOffset, Rec->MinMaxOffset, - MetaEntry->BlockCount /*BlockNum*/); + MetaEntry->BlockCount - 1 /*BlockNum*/); } } diff --git a/testing/adios2/interface/CMakeLists.txt b/testing/adios2/interface/CMakeLists.txt index 7d3f26579c..050be805b9 100644 --- a/testing/adios2/interface/CMakeLists.txt +++ b/testing/adios2/interface/CMakeLists.txt @@ -6,9 +6,26 @@ # These tests should be *very* fast set(CTEST_TEST_TIMEOUT 10) -gtest_add_tests_helper(Interface MPI_ALLOW ADIOS Interface. "") +set(BP3_DIR ${CMAKE_CURRENT_BINARY_DIR}/bp3) +set(BP4_DIR ${CMAKE_CURRENT_BINARY_DIR}/bp4) +set(BP5_DIR ${CMAKE_CURRENT_BINARY_DIR}/bp5) +set(BPfile_DIR ${CMAKE_CURRENT_BINARY_DIR}/bpfile) +set(FS_DIR ${CMAKE_CURRENT_BINARY_DIR}/filestream) +file(MAKE_DIRECTORY ${BP3_DIR}) +file(MAKE_DIRECTORY ${BP4_DIR}) +file(MAKE_DIRECTORY ${BP5_DIR}) +file(MAKE_DIRECTORY ${BPfile_DIR}) +file(MAKE_DIRECTORY ${FS_DIR}) + +gtest_add_tests_helper(Interface MPI_ALLOW ADIOS Interface. .BP3 + WORKING_DIRECTORY ${BP3_DIR} EXTRA_ARGS "BP3") +gtest_add_tests_helper(Interface MPI_ALLOW ADIOS Interface. .BPfile + WORKING_DIRECTORY ${BPfile_DIR} EXTRA_ARGS "BPfile") gtest_add_tests_helper(Write MPI_ALLOW ADIOSInterface Interface. "") gtest_add_tests_helper(DefineVariable MPI_ALLOW ADIOS Interface. "") gtest_add_tests_helper(DefineAttribute MPI_ALLOW ADIOS Interface. "") -gtest_add_tests_helper(Selection MPI_NONE ADIOS Interface. "") +gtest_add_tests_helper(Selection MPI_NONE ADIOS Interface. .BP3 + WORKING_DIRECTORY ${BP3_DIR} EXTRA_ARGS "BP3") +gtest_add_tests_helper(Selection MPI_NONE ADIOS Interface. .BPfile + WORKING_DIRECTORY ${BPfile_DIR} EXTRA_ARGS "BPfile") gtest_add_tests_helper(NoMpi MPI_NONE ADIOS Interface. "") diff --git a/testing/adios2/interface/TestADIOSSelection.cpp b/testing/adios2/interface/TestADIOSSelection.cpp index 26dc829d16..8685eb1035 100644 --- a/testing/adios2/interface/TestADIOSSelection.cpp +++ b/testing/adios2/interface/TestADIOSSelection.cpp @@ -129,6 +129,8 @@ MultiArray makeArray(std::initializer_list> t) return arr; } +std::string engine = "BPfile"; // default if no argument + TEST(MultiArray, Constructor) { using MultiArrayT = MultiArray; @@ -247,7 +249,7 @@ TEST_F(ADIOS2_CXX11_API_Selection, SelectionNone) // clang-format on // write - m_IOWriter.SetEngine("BP3"); + m_IOWriter.SetEngine(engine); auto writer = m_IOWriter.Open("test_selection_none.bp", adios2::Mode::Write); auto var = m_IOWriter.DefineVariable("var", {3, 4}); @@ -257,11 +259,12 @@ TEST_F(ADIOS2_CXX11_API_Selection, SelectionNone) // read back auto arr_read = MultiArrayT(ref.dims()); - m_IOReader.SetEngine("BP3"); auto engine = m_IOReader.Open("test_selection_none.bp", adios2::Mode::Read); + engine.BeginStep(); var = m_IOReader.InquireVariable("var"); var.SetSelection({{0, 0}, {3, 4}}); engine.Get(var, arr_read.data()); + engine.EndStep(); engine.Close(); EXPECT_EQ(arr_read, ref); @@ -279,7 +282,7 @@ TEST_F(ADIOS2_CXX11_API_Selection, SelectionWrite) // clang-format on // write - m_IOWriter.SetEngine("BP3"); + m_IOWriter.SetEngine(engine); auto writer = m_IOWriter.Open("test_selection_write.bp", adios2::Mode::Write); auto var = m_IOWriter.DefineVariable("var", {3, 4}); @@ -289,12 +292,13 @@ TEST_F(ADIOS2_CXX11_API_Selection, SelectionWrite) // read back auto arr_read = MultiArrayT(ref.dims()); - m_IOReader.SetEngine("BP3"); auto engine = m_IOReader.Open("test_selection_write.bp", adios2::Mode::Read); + engine.BeginStep(); var = m_IOReader.InquireVariable("var"); var.SetSelection({{0, 0}, {3, 4}}); engine.Get(var, arr_read.data()); + engine.EndStep(); engine.Close(); EXPECT_EQ(arr_read, ref); @@ -312,7 +316,7 @@ TEST_F(ADIOS2_CXX11_API_Selection, SelectionWriteStart) // clang-format on // write - m_IOWriter.SetEngine("BP3"); + m_IOWriter.SetEngine(engine); auto writer = m_IOWriter.Open("test_selection_write_start.bp", adios2::Mode::Write); auto var = m_IOWriter.DefineVariable("var", {3, 4}); @@ -322,12 +326,13 @@ TEST_F(ADIOS2_CXX11_API_Selection, SelectionWriteStart) // read back auto arr_read = MultiArrayT(ref.dims()); - m_IOReader.SetEngine("BP3"); auto engine = m_IOReader.Open("test_selection_write_start.bp", adios2::Mode::Read); + engine.BeginStep(); var = m_IOReader.InquireVariable("var"); var.SetSelection({{0, 0}, {3, 4}}); engine.Get(var, arr_read.data()); + engine.EndStep(); engine.Close(); EXPECT_EQ(arr_read, ref); @@ -344,7 +349,7 @@ TEST_F(ADIOS2_CXX11_API_Selection, SelectionRead) {20., 21.}}); // clang-format on - m_IOWriter.SetEngine("BP3"); + m_IOWriter.SetEngine(engine); auto writer = m_IOWriter.Open("test_selection_read.bp", adios2::Mode::Write); auto var = m_IOWriter.DefineVariable("var", {3, 4}); @@ -354,11 +359,12 @@ TEST_F(ADIOS2_CXX11_API_Selection, SelectionRead) // read back auto arr_read = MultiArrayT(ref.dims()); - m_IOReader.SetEngine("BP3"); auto engine = m_IOReader.Open("test_selection_read.bp", adios2::Mode::Read); + engine.BeginStep(); var = m_IOReader.InquireVariable("var"); var.SetSelection({{0, 0}, {3, 2}}); engine.Get(var, arr_read.data()); + engine.EndStep(); engine.Close(); EXPECT_EQ(arr_read, ref); @@ -376,7 +382,7 @@ TEST_F(ADIOS2_CXX11_API_Selection, SelectionReadStart) // clang-format on // write - m_IOWriter.SetEngine("BP3"); + m_IOWriter.SetEngine(engine); auto writer = m_IOWriter.Open("test_selection_read_start.bp", adios2::Mode::Write); auto var = m_IOWriter.DefineVariable("var", {3, 4}); @@ -386,12 +392,13 @@ TEST_F(ADIOS2_CXX11_API_Selection, SelectionReadStart) // read back auto arr_read = MultiArrayT(ref.dims()); - m_IOReader.SetEngine("BP3"); auto engine = m_IOReader.Open("test_selection_read_start.bp", adios2::Mode::Read); + engine.BeginStep(); var = m_IOReader.InquireVariable("var"); var.SetSelection({{0, 2}, {3, 2}}); engine.Get(var, arr_read.data()); + engine.EndStep(); engine.Close(); EXPECT_EQ(arr_read, ref); @@ -411,7 +418,7 @@ TEST_F(ADIOS2_CXX11_API_Selection, MemorySelectionNone) // clang-format on // write - m_IOWriter.SetEngine("BP3"); + m_IOWriter.SetEngine(engine); auto writer = m_IOWriter.Open("test_mem_selection_none.bp", adios2::Mode::Write); auto var = m_IOWriter.DefineVariable("var", {3, 4}); @@ -422,12 +429,13 @@ TEST_F(ADIOS2_CXX11_API_Selection, MemorySelectionNone) // read back auto arr_read = MultiArrayT(ref.dims()); - m_IOReader.SetEngine("BP3"); auto engine = m_IOReader.Open("test_mem_selection_none.bp", adios2::Mode::Read); + engine.BeginStep(); var = m_IOReader.InquireVariable("var"); var.SetSelection({{0, 0}, {3, 4}}); engine.Get(var, arr_read.data()); + engine.EndStep(); engine.Close(); EXPECT_EQ(arr_read, ref); @@ -447,7 +455,7 @@ TEST_F(ADIOS2_CXX11_API_Selection, MemorySelectionWrite) // clang-format on // write - m_IOWriter.SetEngine("BP3"); + m_IOWriter.SetEngine(engine); auto writer = m_IOWriter.Open("test_mem_selection_write.bp", adios2::Mode::Write); auto var = m_IOWriter.DefineVariable("var", {3, 4}); @@ -458,12 +466,13 @@ TEST_F(ADIOS2_CXX11_API_Selection, MemorySelectionWrite) // read back auto arr_read = MultiArrayT(ref.dims()); - m_IOReader.SetEngine("BP3"); auto engine = m_IOReader.Open("test_mem_selection_write.bp", adios2::Mode::Read); + engine.BeginStep(); var = m_IOReader.InquireVariable("var"); var.SetSelection({{0, 0}, {3, 2}}); engine.Get(var, arr_read.data()); + engine.EndStep(); engine.Close(); EXPECT_EQ(arr_read, ref); @@ -483,7 +492,7 @@ TEST_F(ADIOS2_CXX11_API_Selection, MemorySelectionWriteStart) // clang-format on // write - m_IOWriter.SetEngine("BP3"); + m_IOWriter.SetEngine(engine); auto writer = m_IOWriter.Open("test_mem_selection_write_start.bp", adios2::Mode::Write); auto var = m_IOWriter.DefineVariable("var", {3, 4}); @@ -494,12 +503,13 @@ TEST_F(ADIOS2_CXX11_API_Selection, MemorySelectionWriteStart) // read back auto arr_read = MultiArrayT(ref.dims()); - m_IOReader.SetEngine("BP3"); auto engine = m_IOReader.Open("test_mem_selection_write_start.bp", adios2::Mode::Read); + engine.BeginStep(); var = m_IOReader.InquireVariable("var"); var.SetSelection({{0, 0}, {3, 2}}); engine.Get(var, arr_read.data()); + engine.EndStep(); engine.Close(); EXPECT_EQ(arr_read, ref); @@ -518,7 +528,7 @@ TEST_F(ADIOS2_CXX11_API_Selection, MemorySelectionRead) {0., 0., 0., 0.}}); // clang-format on - m_IOWriter.SetEngine("BP3"); + m_IOWriter.SetEngine(engine); auto writer = m_IOWriter.Open("test_mem_selection_read.bp", adios2::Mode::Write); auto var = m_IOWriter.DefineVariable("var", {3, 4}); @@ -528,13 +538,14 @@ TEST_F(ADIOS2_CXX11_API_Selection, MemorySelectionRead) // read back auto arr_read = MultiArrayT(ref.dims()); - m_IOReader.SetEngine("BP3"); auto engine = m_IOReader.Open("test_mem_selection_read.bp", adios2::Mode::Read); + engine.BeginStep(); var = m_IOReader.InquireVariable("var"); var.SetSelection({{0, 0}, {3, 2}}); var.SetMemorySelection({{1, 1}, {5, 4}}); engine.Get(var, arr_read.data()); + engine.EndStep(); engine.Close(); EXPECT_EQ(arr_read, ref); @@ -553,7 +564,7 @@ TEST_F(ADIOS2_CXX11_API_Selection, MemorySelectionReadStart) {0., 0., 0., 0.}}); // clang-format on - m_IOWriter.SetEngine("BP3"); + m_IOWriter.SetEngine(engine); auto writer = m_IOWriter.Open("test_mem_selection_read_start.bp", adios2::Mode::Write); auto var = m_IOWriter.DefineVariable("var", {3, 4}); @@ -563,9 +574,8 @@ TEST_F(ADIOS2_CXX11_API_Selection, MemorySelectionReadStart) // read back auto arr_read = MultiArrayT(ref.dims()); - m_IOReader.SetEngine("BP3"); - auto engine = - m_IOReader.Open("test_mem_selection_read_start.bp", adios2::Mode::Read); + auto engine = m_IOReader.Open("test_mem_selection_read_start.bp", + adios2::Mode::ReadRandomAccess); var = m_IOReader.InquireVariable("var"); var.SetSelection({{0, 2}, {3, 2}}); var.SetMemorySelection({{1, 1}, {5, 4}}); @@ -590,7 +600,7 @@ TEST_F(ADIOS2_CXX11_API_Selection, MemorySelectionComplex) {0., 0., 0., 0.}}); // clang-format on - m_IOWriter.SetEngine("BP3"); + m_IOWriter.SetEngine(engine); auto writer = m_IOWriter.Open("test_mem_selection_complex.bp", adios2::Mode::Write); auto var = m_IOWriter.DefineVariable("var", {4, 4}); @@ -611,9 +621,8 @@ TEST_F(ADIOS2_CXX11_API_Selection, MemorySelectionComplex) // read back center block, with bits from every block written auto arr_read = MultiArrayT(ref.dims()); - m_IOReader.SetEngine("BP3"); - auto engine = - m_IOReader.Open("test_mem_selection_complex.bp", adios2::Mode::Read); + auto engine = m_IOReader.Open("test_mem_selection_complex.bp", + adios2::Mode::ReadRandomAccess); var = m_IOReader.InquireVariable("var"); var.SetSelection({{1, 1}, {2, 2}}); var.SetMemorySelection({{1, 1}, {4, 4}}); @@ -626,5 +635,10 @@ TEST_F(ADIOS2_CXX11_API_Selection, MemorySelectionComplex) int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); + if (argc > 1) + { + engine = argv[1]; + std::cout << "Running with engine " << engine << std::endl; + } return RUN_ALL_TESTS(); }