diff --git a/source/adios2/engine/sst/SstWriter.tcc b/source/adios2/engine/sst/SstWriter.tcc index b2e850500f..237c902d8e 100644 --- a/source/adios2/engine/sst/SstWriter.tcc +++ b/source/adios2/engine/sst/SstWriter.tcc @@ -70,19 +70,61 @@ void SstWriter::PutSyncCommon(Variable &variable, const T *values) } else { - if (variable.m_Type == DataType::String) + if (!variable.m_MemoryCount.empty()) { - std::string &source = *(std::string *)values; - void *p = &(source[0]); + size_t ObjSize; + if (variable.m_Type == DataType::Struct) + { + ObjSize = variable.m_ElementSize; + } + else + { + ObjSize = helper::GetDataTypeSize(variable.m_Type); + } + + const bool sourceRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage); + helper::DimsArray MemoryStart(variable.m_MemoryStart); + helper::DimsArray MemoryCount(variable.m_MemoryCount); + helper::DimsArray varCount(variable.m_Count); + + int DimCount = (int)variable.m_Count.size(); + helper::DimsArray ZeroDims(DimCount, (size_t)0); + // get a temporary span then fill with memselection now + format::BufferV::BufferPos bp5span(0, 0, 0); + m_BP5Serializer->Marshal((void *)&variable, variable.m_Name.c_str(), variable.m_Type, variable.m_ElementSize, DimCount, Shape, - Count, Start, &p, true, nullptr); + Count, Start, nullptr, false, &bp5span); + void *ptr = m_BP5Serializer->GetPtr(bp5span.bufferIdx, bp5span.posInBuffer); + + if (!sourceRowMajor) + { + std::reverse(MemoryStart.begin(), MemoryStart.end()); + std::reverse(MemoryCount.begin(), MemoryCount.end()); + std::reverse(varCount.begin(), varCount.end()); + } + helper::NdCopy((const char *)values, helper::CoreDims(ZeroDims), MemoryCount, + sourceRowMajor, false, (char *)ptr, MemoryStart, varCount, + sourceRowMajor, false, (int)ObjSize, helper::CoreDims(), + helper::CoreDims(), helper::CoreDims(), helper::CoreDims(), + false /* safemode */, variable.m_MemSpace); } else { - m_BP5Serializer->Marshal((void *)&variable, variable.m_Name.c_str(), - variable.m_Type, variable.m_ElementSize, DimCount, Shape, - Count, Start, values, true, nullptr); + if (variable.m_Type == DataType::String) + { + std::string &source = *(std::string *)values; + void *p = &(source[0]); + m_BP5Serializer->Marshal((void *)&variable, variable.m_Name.c_str(), + variable.m_Type, variable.m_ElementSize, DimCount, + Shape, Count, Start, &p, true, nullptr); + } + else + { + m_BP5Serializer->Marshal((void *)&variable, variable.m_Name.c_str(), + variable.m_Type, variable.m_ElementSize, DimCount, + Shape, Count, Start, values, true, nullptr); + } } } } diff --git a/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp b/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp index ea52b5753d..aa8d516b21 100644 --- a/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp +++ b/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp @@ -15,6 +15,8 @@ #include "../SmallTestData.h" std::string engineName; // comes from command line +bool DoWrite = true; +bool DoRead = true; namespace { @@ -166,9 +168,13 @@ void AssignStep3D(const size_t step, std::vector> &vector, } // end anonymous namespace +#if ADIOS2_USE_MPI +MPI_Comm testComm; +#endif + void BPSteps1D(const size_t ghostCells) { - const std::string fname("BPSteps1D_" + std::to_string(ghostCells) + ".bp"); + const std::string fname("BPSteps1D_" + std::to_string(ghostCells)); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -178,15 +184,16 @@ void BPSteps1D(const size_t ghostCells) const size_t NSteps = 3; #if ADIOS2_USE_MPI - MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); - MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + MPI_Comm_rank(testComm, &mpiRank); + MPI_Comm_size(testComm, &mpiSize); #endif #if ADIOS2_USE_MPI - adios2::ADIOS adios(MPI_COMM_WORLD); + adios2::ADIOS adios(testComm); #else adios2::ADIOS adios; #endif + if (DoWrite) { adios2::IO io = adios.DeclareIO("WriteIO"); @@ -195,6 +202,7 @@ void BPSteps1D(const size_t ghostCells) io.SetEngine(engineName); } + io.SetParameters("StatsLevel=1"); const adios2::Dims shape{static_cast(Nx * mpiSize)}; const adios2::Dims start{static_cast(Nx * mpiRank)}; const adios2::Dims count{Nx}; @@ -256,9 +264,10 @@ void BPSteps1D(const size_t ghostCells) bpWriter.Close(); } #if ADIOS2_USE_MPI - MPI_Barrier(MPI_COMM_WORLD); + MPI_Barrier(testComm); #endif // Reader + if (DoRead) { adios2::IO io = adios.DeclareIO("ReadIO"); @@ -377,7 +386,7 @@ void BPSteps1D(const size_t ghostCells) void BPSteps2D4x2(const size_t ghostCells) { - const std::string fname("BPSteps2D4x2_" + std::to_string(ghostCells) + ".bp"); + const std::string fname("BPSteps2D4x2_" + std::to_string(ghostCells)); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -391,15 +400,16 @@ void BPSteps2D4x2(const size_t ghostCells) const size_t NSteps = 3; #if ADIOS2_USE_MPI - MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); - MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + MPI_Comm_rank(testComm, &mpiRank); + MPI_Comm_size(testComm, &mpiSize); #endif #if ADIOS2_USE_MPI - adios2::ADIOS adios(MPI_COMM_WORLD); + adios2::ADIOS adios(testComm); #else adios2::ADIOS adios; #endif + if (DoWrite) { adios2::IO io = adios.DeclareIO("WriteIO"); @@ -470,9 +480,10 @@ void BPSteps2D4x2(const size_t ghostCells) bpWriter.Close(); } #if ADIOS2_USE_MPI - MPI_Barrier(MPI_COMM_WORLD); + MPI_Barrier(testComm); #endif // Reader + if (DoRead) { adios2::IO io = adios.DeclareIO("ReadIO"); @@ -601,7 +612,7 @@ void BPSteps2D4x2(const size_t ghostCells) void BPSteps3D8x2x4(const size_t ghostCells) { - const std::string fname("BPSteps3D8x2x4_" + std::to_string(ghostCells) + ".bp"); + const std::string fname("BPSteps3D8x2x4_" + std::to_string(ghostCells)); int mpiRank = 0, mpiSize = 1; // Number of rows @@ -617,15 +628,16 @@ void BPSteps3D8x2x4(const size_t ghostCells) const size_t NSteps = 3; #if ADIOS2_USE_MPI - MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); - MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); + MPI_Comm_rank(testComm, &mpiRank); + MPI_Comm_size(testComm, &mpiSize); #endif #if ADIOS2_USE_MPI - adios2::ADIOS adios(MPI_COMM_WORLD); + adios2::ADIOS adios(testComm); #else adios2::ADIOS adios; #endif + if (DoWrite) { adios2::IO io = adios.DeclareIO("WriteIO"); @@ -698,9 +710,10 @@ void BPSteps3D8x2x4(const size_t ghostCells) bpWriter.Close(); } #if ADIOS2_USE_MPI - MPI_Barrier(MPI_COMM_WORLD); + MPI_Barrier(testComm); #endif // Reader + if (DoRead) { adios2::IO io = adios.DeclareIO("ReadIO"); @@ -878,25 +891,80 @@ INSTANTIATE_TEST_SUITE_P(ghostCells, BPWriteMemSelReadVector, ::testing::Values( int main(int argc, char **argv) { + int result; + ::testing::InitGoogleTest(&argc, argv); + int bare_arg = 0; + + for (int i = 1; i < argc; i++) + { + if (strcmp(argv[i], "-do_write") == 0) + { + DoWrite = true; + DoRead = false; + } + else if (strcmp(argv[i], "-do_read") == 0) + { + DoWrite = false; + DoRead = true; + } + else if (argv[i][0] == '-') + { + std::cerr << "Unknown argument: " << argv[i] << std::endl; + exit(1); + } + else + { + std::string fname; + std::string engineParams; + if (bare_arg == 0) + { + /* first arg without -- is engine */ + engineName = std::string(argv[1]); + bare_arg++; + } + else if (bare_arg == 1) + { + /* second arg without -- is filename */ + // fname = std::string(argv[1]); + bare_arg++; + } + else if (bare_arg == 2) + { + // engineParams = ParseEngineParams(argv[1]); + bare_arg++; + } + else + { + + throw std::invalid_argument("Unknown argument \"" + std::string(argv[1]) + "\""); + } + } + } + #if ADIOS2_USE_MPI int provided; + int thread_support_level = + (engineName == "SST" || engineName == "sst") ? MPI_THREAD_MULTIPLE : MPI_THREAD_SINGLE; // MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP - MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); -#endif + MPI_Init_thread(nullptr, nullptr, thread_support_level, &provided); - int result; - ::testing::InitGoogleTest(&argc, argv); + int key; + MPI_Comm_rank(MPI_COMM_WORLD, &key); - if (argc > 1) - { - engineName = std::string(argv[1]); - } + const unsigned int color = (DoRead & !DoWrite) ? 1 : 0; + + MPI_Comm_split(MPI_COMM_WORLD, color, key, &testComm); +#endif result = RUN_ALL_TESTS(); #if ADIOS2_USE_MPI +#ifdef CRAY_MPICH_VERSION + MPI_Barrier(MPI_COMM_WORLD); +#else MPI_Finalize(); +#endif #endif return result; diff --git a/testing/adios2/engine/staging-common/CMakeLists.txt b/testing/adios2/engine/staging-common/CMakeLists.txt index a5a3f09056..ee20d68b8c 100644 --- a/testing/adios2/engine/staging-common/CMakeLists.txt +++ b/testing/adios2/engine/staging-common/CMakeLists.txt @@ -149,8 +149,13 @@ set (ALL_SIMPLE_TESTS "") list (APPEND ALL_SIMPLE_TESTS ${SIMPLE_TESTS} ${SIMPLE_FORTRAN_TESTS} ${SIMPLE_MPI_TESTS} ${SIMPLE_ZFP_TESTS}) set (SST_SPECIFIC_TESTS "") +import_bp_test(WriteMemorySelectionRead 1 1) +list (APPEND SST_SPECIFIC_TESTS "WriteMemorySelectionRead.1x1") list (APPEND SST_SPECIFIC_TESTS "1x1.SstRUDP;1x1.LocalMultiblock;RoundRobinDistribution.1x1x3;AllToAllDistribution.1x1x3;OnDemandSingle.1x1") + if (ADIOS2_HAVE_MPI) + import_bp_test(WriteMemorySelectionRead 3 3) + list (APPEND SST_SPECIFIC_TESTS "WriteMemorySelectionRead.3x3") list (APPEND SST_SPECIFIC_TESTS "2x3.SstRUDP;2x1.LocalMultiblock;5x3.LocalMultiblock;") endif() diff --git a/testing/adios2/engine/staging-common/TestSupp.cmake b/testing/adios2/engine/staging-common/TestSupp.cmake index 13ce1765a4..ca63109a77 100644 --- a/testing/adios2/engine/staging-common/TestSupp.cmake +++ b/testing/adios2/engine/staging-common/TestSupp.cmake @@ -323,3 +323,15 @@ function(from_hex HEX DEC) set(${DEC} ${_res} PARENT_SCOPE) endfunction() +function(import_bp_test BASENAME WRITE_SCALE READ_SCALE) + set (WRITER_POSTFIX "Serial") + set (READER_POSTFIX "Serial") + if (${WRITE_SCALE} GREATER 1) + set (WRITER_POSTFIX "MPI") + endif() + if (${READ_SCALE} GREATER 1) + set (READER_POSTFIX "MPI") + endif() + set (${BASENAME}.${WRITE_SCALE}x${READ_SCALE}_CMD "run_test.py.$ -nw ${WRITE_SCALE} -nr ${READ_SCALE} --warg=-do_write --rarg=-do_read -w $ -r $" PARENT_SCOPE) + +endfunction()