Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add Memory Selection to SST, and prototype a mechanism for running st… #3823

Merged
merged 3 commits into from
Sep 27, 2023
Merged

Add Memory Selection to SST, and prototype a mechanism for running st… #3823

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2884f71
Import tests from bp to staging common, implement memory selection in…
eisenhauer Sep 22, 2023
51f7b6f
Try always using the MPI version
eisenhauer Sep 26, 2023
5bde490
Try always using the MPI version
eisenhauer Sep 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 49 additions & 7 deletions source/adios2/engine/sst/SstWriter.tcc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,19 +70,61 @@ void SstWriter::PutSyncCommon(Variable<T> &variable, const T *values)
}
else
{
if (variable.m_Type == DataType::String)
if (!variable.m_MemoryCount.empty())
{
std::string &source = *(std::string *)values;
void *p = &(source[0]);
size_t ObjSize;
if (variable.m_Type == DataType::Struct)
{
ObjSize = variable.m_ElementSize;
}
else
{
ObjSize = helper::GetDataTypeSize(variable.m_Type);
}

const bool sourceRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage);
helper::DimsArray MemoryStart(variable.m_MemoryStart);
helper::DimsArray MemoryCount(variable.m_MemoryCount);
helper::DimsArray varCount(variable.m_Count);

int DimCount = (int)variable.m_Count.size();
helper::DimsArray ZeroDims(DimCount, (size_t)0);
// get a temporary span then fill with memselection now
format::BufferV::BufferPos bp5span(0, 0, 0);

m_BP5Serializer->Marshal((void *)&variable, variable.m_Name.c_str(),
variable.m_Type, variable.m_ElementSize, DimCount, Shape,
Count, Start, &p, true, nullptr);
Count, Start, nullptr, false, &bp5span);
void *ptr = m_BP5Serializer->GetPtr(bp5span.bufferIdx, bp5span.posInBuffer);

if (!sourceRowMajor)
{
std::reverse(MemoryStart.begin(), MemoryStart.end());
std::reverse(MemoryCount.begin(), MemoryCount.end());
std::reverse(varCount.begin(), varCount.end());
}
helper::NdCopy((const char *)values, helper::CoreDims(ZeroDims), MemoryCount,
sourceRowMajor, false, (char *)ptr, MemoryStart, varCount,
sourceRowMajor, false, (int)ObjSize, helper::CoreDims(),
helper::CoreDims(), helper::CoreDims(), helper::CoreDims(),
false /* safemode */, variable.m_MemSpace);
}
else
{
m_BP5Serializer->Marshal((void *)&variable, variable.m_Name.c_str(),
variable.m_Type, variable.m_ElementSize, DimCount, Shape,
Count, Start, values, true, nullptr);
if (variable.m_Type == DataType::String)
{
std::string &source = *(std::string *)values;
void *p = &(source[0]);
m_BP5Serializer->Marshal((void *)&variable, variable.m_Name.c_str(),
variable.m_Type, variable.m_ElementSize, DimCount,
Shape, Count, Start, &p, true, nullptr);
}
else
{
m_BP5Serializer->Marshal((void *)&variable, variable.m_Name.c_str(),
variable.m_Type, variable.m_ElementSize, DimCount,
Shape, Count, Start, values, true, nullptr);
}
}
}
}
Expand Down
114 changes: 91 additions & 23 deletions testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
#include "../SmallTestData.h"

std::string engineName; // comes from command line
bool DoWrite = true;
bool DoRead = true;

namespace
{
Expand Down Expand Up @@ -166,9 +168,13 @@ void AssignStep3D(const size_t step, std::vector<std::complex<double>> &vector,

} // end anonymous namespace

#if ADIOS2_USE_MPI
MPI_Comm testComm;
#endif

void BPSteps1D(const size_t ghostCells)
{
const std::string fname("BPSteps1D_" + std::to_string(ghostCells) + ".bp");
const std::string fname("BPSteps1D_" + std::to_string(ghostCells));

int mpiRank = 0, mpiSize = 1;
// Number of rows
Expand All @@ -178,15 +184,16 @@ void BPSteps1D(const size_t ghostCells)
const size_t NSteps = 3;

#if ADIOS2_USE_MPI
MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
MPI_Comm_rank(testComm, &mpiRank);
MPI_Comm_size(testComm, &mpiSize);
#endif

#if ADIOS2_USE_MPI
adios2::ADIOS adios(MPI_COMM_WORLD);
adios2::ADIOS adios(testComm);
#else
adios2::ADIOS adios;
#endif
if (DoWrite)
{
adios2::IO io = adios.DeclareIO("WriteIO");

Expand All @@ -195,6 +202,7 @@ void BPSteps1D(const size_t ghostCells)
io.SetEngine(engineName);
}

io.SetParameters("StatsLevel=1");
const adios2::Dims shape{static_cast<size_t>(Nx * mpiSize)};
const adios2::Dims start{static_cast<size_t>(Nx * mpiRank)};
const adios2::Dims count{Nx};
Expand Down Expand Up @@ -256,9 +264,10 @@ void BPSteps1D(const size_t ghostCells)
bpWriter.Close();
}
#if ADIOS2_USE_MPI
MPI_Barrier(MPI_COMM_WORLD);
MPI_Barrier(testComm);
#endif
// Reader
if (DoRead)
{
adios2::IO io = adios.DeclareIO("ReadIO");

Expand Down Expand Up @@ -377,7 +386,7 @@ void BPSteps1D(const size_t ghostCells)

void BPSteps2D4x2(const size_t ghostCells)
{
const std::string fname("BPSteps2D4x2_" + std::to_string(ghostCells) + ".bp");
const std::string fname("BPSteps2D4x2_" + std::to_string(ghostCells));

int mpiRank = 0, mpiSize = 1;
// Number of rows
Expand All @@ -391,15 +400,16 @@ void BPSteps2D4x2(const size_t ghostCells)
const size_t NSteps = 3;

#if ADIOS2_USE_MPI
MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
MPI_Comm_rank(testComm, &mpiRank);
MPI_Comm_size(testComm, &mpiSize);
#endif

#if ADIOS2_USE_MPI
adios2::ADIOS adios(MPI_COMM_WORLD);
adios2::ADIOS adios(testComm);
#else
adios2::ADIOS adios;
#endif
if (DoWrite)
{
adios2::IO io = adios.DeclareIO("WriteIO");

Expand Down Expand Up @@ -470,9 +480,10 @@ void BPSteps2D4x2(const size_t ghostCells)
bpWriter.Close();
}
#if ADIOS2_USE_MPI
MPI_Barrier(MPI_COMM_WORLD);
MPI_Barrier(testComm);
#endif
// Reader
if (DoRead)
{
adios2::IO io = adios.DeclareIO("ReadIO");

Expand Down Expand Up @@ -601,7 +612,7 @@ void BPSteps2D4x2(const size_t ghostCells)

void BPSteps3D8x2x4(const size_t ghostCells)
{
const std::string fname("BPSteps3D8x2x4_" + std::to_string(ghostCells) + ".bp");
const std::string fname("BPSteps3D8x2x4_" + std::to_string(ghostCells));

int mpiRank = 0, mpiSize = 1;
// Number of rows
Expand All @@ -617,15 +628,16 @@ void BPSteps3D8x2x4(const size_t ghostCells)
const size_t NSteps = 3;

#if ADIOS2_USE_MPI
MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
MPI_Comm_rank(testComm, &mpiRank);
MPI_Comm_size(testComm, &mpiSize);
#endif

#if ADIOS2_USE_MPI
adios2::ADIOS adios(MPI_COMM_WORLD);
adios2::ADIOS adios(testComm);
#else
adios2::ADIOS adios;
#endif
if (DoWrite)
{
adios2::IO io = adios.DeclareIO("WriteIO");

Expand Down Expand Up @@ -698,9 +710,10 @@ void BPSteps3D8x2x4(const size_t ghostCells)
bpWriter.Close();
}
#if ADIOS2_USE_MPI
MPI_Barrier(MPI_COMM_WORLD);
MPI_Barrier(testComm);
#endif
// Reader
if (DoRead)
{
adios2::IO io = adios.DeclareIO("ReadIO");

Expand Down Expand Up @@ -878,25 +891,80 @@ INSTANTIATE_TEST_SUITE_P(ghostCells, BPWriteMemSelReadVector, ::testing::Values(

int main(int argc, char **argv)
{
int result;
::testing::InitGoogleTest(&argc, argv);
int bare_arg = 0;

for (int i = 1; i < argc; i++)
{
if (strcmp(argv[i], "-do_write") == 0)
{
DoWrite = true;
DoRead = false;
}
else if (strcmp(argv[i], "-do_read") == 0)
{
DoWrite = false;
DoRead = true;
}
else if (argv[i][0] == '-')
{
std::cerr << "Unknown argument: " << argv[i] << std::endl;
exit(1);
}
else
{
std::string fname;
std::string engineParams;
if (bare_arg == 0)
{
/* first arg without -- is engine */
engineName = std::string(argv[1]);
bare_arg++;
}
else if (bare_arg == 1)
{
/* second arg without -- is filename */
// fname = std::string(argv[1]);
bare_arg++;
}
else if (bare_arg == 2)
{
// engineParams = ParseEngineParams(argv[1]);
bare_arg++;
}
else
{

throw std::invalid_argument("Unknown argument \"" + std::string(argv[1]) + "\"");
}
}
}

#if ADIOS2_USE_MPI
int provided;
int thread_support_level =
(engineName == "SST" || engineName == "sst") ? MPI_THREAD_MULTIPLE : MPI_THREAD_SINGLE;

// MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP
MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided);
#endif
MPI_Init_thread(nullptr, nullptr, thread_support_level, &provided);

int result;
::testing::InitGoogleTest(&argc, argv);
int key;
MPI_Comm_rank(MPI_COMM_WORLD, &key);

if (argc > 1)
{
engineName = std::string(argv[1]);
}
const unsigned int color = (DoRead & !DoWrite) ? 1 : 0;

MPI_Comm_split(MPI_COMM_WORLD, color, key, &testComm);
#endif

result = RUN_ALL_TESTS();

#if ADIOS2_USE_MPI
#ifdef CRAY_MPICH_VERSION
MPI_Barrier(MPI_COMM_WORLD);
#else
MPI_Finalize();
#endif
#endif

return result;
Expand Down
5 changes: 5 additions & 0 deletions testing/adios2/engine/staging-common/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,8 +149,13 @@ set (ALL_SIMPLE_TESTS "")
list (APPEND ALL_SIMPLE_TESTS ${SIMPLE_TESTS} ${SIMPLE_FORTRAN_TESTS} ${SIMPLE_MPI_TESTS} ${SIMPLE_ZFP_TESTS})

set (SST_SPECIFIC_TESTS "")
import_bp_test(WriteMemorySelectionRead 1 1)
list (APPEND SST_SPECIFIC_TESTS "WriteMemorySelectionRead.1x1")
list (APPEND SST_SPECIFIC_TESTS "1x1.SstRUDP;1x1.LocalMultiblock;RoundRobinDistribution.1x1x3;AllToAllDistribution.1x1x3;OnDemandSingle.1x1")

if (ADIOS2_HAVE_MPI)
import_bp_test(WriteMemorySelectionRead 3 3)
list (APPEND SST_SPECIFIC_TESTS "WriteMemorySelectionRead.3x3")
list (APPEND SST_SPECIFIC_TESTS "2x3.SstRUDP;2x1.LocalMultiblock;5x3.LocalMultiblock;")
endif()

Expand Down
12 changes: 12 additions & 0 deletions testing/adios2/engine/staging-common/TestSupp.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,3 +323,15 @@ function(from_hex HEX DEC)
set(${DEC} ${_res} PARENT_SCOPE)
endfunction()

function(import_bp_test BASENAME WRITE_SCALE READ_SCALE)
set (WRITER_POSTFIX "Serial")
set (READER_POSTFIX "Serial")
if(ADIOS2_HAVE_MPI)
set (WRITER_POSTFIX "MPI")
endif()
if(ADIOS2_HAVE_MPI)
set (READER_POSTFIX "MPI")
endif()
set (${BASENAME}.${WRITE_SCALE}x${READ_SCALE}_CMD "run_test.py.$<CONFIG> -nw ${WRITE_SCALE} -nr ${READ_SCALE} --warg=-do_write --rarg=-do_read -w $<TARGET_FILE:Test.Engine.BP.${BASENAME}.${WRITER_POSTFIX}> -r $<TARGET_FILE:Test.Engine.BP.${BASENAME}.${READER_POSTFIX}>" PARENT_SCOPE)

endfunction()
Loading