Skip to content

Commit

Permalink
Merge pull request #1551 from eisenhauer/StagingCommonTests
Browse files Browse the repository at this point in the history
More common tests
  • Loading branch information
eisenhauer authored Jun 28, 2019
2 parents 277386f + c0e64a8 commit 33f6be0
Show file tree
Hide file tree
Showing 16 changed files with 951 additions and 950 deletions.
113 changes: 111 additions & 2 deletions source/adios2/toolkit/sst/cp/ffs_marshal.c
Original file line number Diff line number Diff line change
Expand Up @@ -894,6 +894,7 @@ static void DecodeAndPrepareData(SstStream Stream, int Writer)
FMFieldList FieldList;
FMStructDescList FormatList;
void *BaseData;
int DumpData = -1;

FFSformat = FFSTypeHandle_from_encode(Stream->ReaderFFSContext,
WriterInfo->RawBuffer);
Expand Down Expand Up @@ -925,8 +926,16 @@ static void DecodeAndPrepareData(SstStream Stream, int Writer)
FFSdecode_to_buffer(Stream->ReaderFFSContext, WriterInfo->RawBuffer,
decode_buf);
}
// printf("\nIncomingDatablock is %p :\n", BaseData);
// FMdump_data(FMFormat_of_original(FFSformat), BaseData, 1024000);
if (DumpData == -1)
{
DumpData = (getenv("SstDumpData") != NULL);
}
if (DumpData)
{
printf("\nOn Rank %d, IncomingDatablock from writer %d is %p :\n",
Stream->Rank, Writer, BaseData);
FMdump_data(FMFormat_of_original(FFSformat), BaseData, 1024000);
}
Info->DataBaseAddrs[Writer] = BaseData;
FormatList = format_list_of_FMFormat(FMFormat_of_original(FFSformat));
FieldList = FormatList[0].field_list;
Expand Down Expand Up @@ -1220,10 +1229,110 @@ void ExtractSelectionFromPartialCM(int ElementSize, size_t Dims,
free(FirstIndex);
}

typedef struct _range_list
{
size_t start;
size_t end;
struct _range_list *next;
} * range_list;

range_list static OneDCoverage(size_t start, size_t end,
range_list uncovered_list)
{
if (uncovered_list == NULL)
return NULL;

if ((start <= uncovered_list->start) && (end >= uncovered_list->end))
{
/* this uncovered element is covered now, recurse on next */
range_list next = uncovered_list->next;
free(uncovered_list);
return OneDCoverage(start, end, next);
}
else if ((end < uncovered_list->end) && (start > uncovered_list->start))
{
/* covering a bit in the middle */
range_list new = malloc(sizeof(*new));
new->next = uncovered_list->next;
new->end = uncovered_list->end;
new->start = end + 1;
uncovered_list->end = start - 1;
uncovered_list->next = new;
return (uncovered_list);
}
else if ((end < uncovered_list->start) || (start > uncovered_list->end))
{
uncovered_list->next = OneDCoverage(start, end, uncovered_list->next);
return uncovered_list;
}
else if (start <= uncovered_list->start)
{
/* we don't cover completely nor a middle portion, so this means we span
* the beginning */
uncovered_list->start = end + 1;
uncovered_list->next = OneDCoverage(start, end, uncovered_list->next);
return uncovered_list;
}
else if (end >= uncovered_list->end)
{
/* we don't cover completely nor a middle portion, so this means we span
* the end */
uncovered_list->end = start - 1;
uncovered_list->next = OneDCoverage(start, end, uncovered_list->next);
return uncovered_list;
}
return NULL;
}

static void DumpCoverageList(range_list list)
{
if (!list)
return;
printf("%ld - %ld", list->start, list->end);
if (list->next != NULL)
{
printf(", ");
DumpCoverageList(list->next);
}
}

static void ImplementGapWarning(SstStream Stream, FFSArrayRequest Req)
{
if (Req->RequestType == Local)
{
/* no analysis here */
return;
}
if (Req->VarRec->DimCount != 1)
{
/* at this point, multidimensional fill analysis is too much */
return;
}
struct _range_list *Required = malloc(sizeof(*Required));
Required->next = NULL;
Required->start = Req->Start[0];
Required->end = Req->Start[0] + Req->Count[0] - 1;
for (int i = 0; i < Stream->WriterCohortSize; i++)
{
size_t start = Req->VarRec->PerWriterStart[i][0];
size_t end = start + Req->VarRec->PerWriterCounts[i][0] - 1;
Required = OneDCoverage(start, end, Required);
}
if (Required != NULL)
{
printf("WARNING: Reader Rank %d requested elements %lu - %lu,\n\tbut "
"these elements were not written by any writer rank: \n",
Stream->Rank, (unsigned long)Req->Start[0],
(unsigned long)Req->Start[0] + Req->Count[0] - 1);
DumpCoverageList(Required);
}
}

static void FillReadRequests(SstStream Stream, FFSArrayRequest Reqs)
{
while (Reqs)
{
ImplementGapWarning(Stream, Reqs);
for (int i = 0; i < Stream->WriterCohortSize; i++)
{
if (NeedWriter(Reqs, i))
Expand Down
19 changes: 17 additions & 2 deletions testing/adios2/engine/staging-common/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,11 @@ add_executable(TestCommonWrite TestCommonWrite.cpp)
add_executable(TestCommonWriteAttrs TestCommonWriteAttrs.cpp)
add_executable(TestCommonWriteModes TestCommonWriteModes.cpp)
add_executable(TestCommonWriteLocal TestCommonWriteLocal.cpp)
add_executable(TestCommonWriteShared TestCommonWriteShared.cpp)
add_executable(TestCommonRead TestCommonRead.cpp)
add_executable(TestCommonReadAttrs TestCommonReadAttrs.cpp)
add_executable(TestCommonReadLocal TestCommonReadLocal.cpp)
add_executable(TestCommonReadShared TestCommonReadShared.cpp)
add_executable(TestCommonServer TestCommonServer.cpp)
add_executable(TestCommonClient TestCommonClient.cpp)
if(ADIOS2_HAVE_Fortran)
Expand All @@ -49,17 +51,21 @@ if(SST_INCLUDE_DIRS)
target_include_directories(TestCommonWriteModes PRIVATE ${SST_INCLUDE_DIRS})
target_include_directories(TestCommonWriteAttrs PRIVATE ${SST_INCLUDE_DIRS})
target_include_directories(TestCommonWriteLocal PRIVATE ${SST_INCLUDE_DIRS})
target_include_directories(TestCommonWriteShared PRIVATE ${SST_INCLUDE_DIRS})
target_include_directories(TestCommonRead PRIVATE ${SST_INCLUDE_DIRS})
target_include_directories(TestCommonReadLocal PRIVATE ${SST_INCLUDE_DIRS})
target_include_directories(TestCommonReadShared PRIVATE ${SST_INCLUDE_DIRS})
target_include_directories(TestCommonServer PRIVATE ${SST_INCLUDE_DIRS})
target_include_directories(TestCommonClient PRIVATE ${SST_INCLUDE_DIRS})
endif()
target_link_libraries(TestCommonWrite adios2 gtest_interface ${Sst_LIBRARY})
target_link_libraries(TestCommonWriteModes adios2 gtest_interface ${Sst_LIBRARY})
target_link_libraries(TestCommonWriteAttrs adios2 gtest_interface ${Sst_LIBRARY})
target_link_libraries(TestCommonWriteLocal adios2 gtest_interface ${Sst_LIBRARY})
target_link_libraries(TestCommonWriteShared adios2 gtest_interface ${Sst_LIBRARY})
target_link_libraries(TestCommonRead adios2 gtest_interface ${Sst_LIBRARY})
target_link_libraries(TestCommonReadLocal adios2 gtest_interface ${Sst_LIBRARY})
target_link_libraries(TestCommonReadShared adios2 gtest_interface ${Sst_LIBRARY})
target_link_libraries(TestCommonReadAttrs adios2 gtest_interface ${Sst_LIBRARY})
target_link_libraries(TestCommonServer adios2 gtest_interface ${Sst_LIBRARY})
target_link_libraries(TestCommonClient adios2 gtest_interface ${Sst_LIBRARY})
Expand All @@ -69,8 +75,10 @@ if(ADIOS2_HAVE_MPI)
target_link_libraries(TestCommonWriteModes MPI::MPI_C)
target_link_libraries(TestCommonWriteAttrs MPI::MPI_C)
target_link_libraries(TestCommonWriteLocal MPI::MPI_C)
target_link_libraries(TestCommonWriteShared MPI::MPI_C)
target_link_libraries(TestCommonRead MPI::MPI_C)
target_link_libraries(TestCommonReadLocal MPI::MPI_C)
target_link_libraries(TestCommonReadShared MPI::MPI_C)
target_link_libraries(TestCommonReadAttrs MPI::MPI_C)
target_link_libraries(TestCommonServer MPI::MPI_C)
target_link_libraries(TestCommonClient MPI::MPI_C)
Expand Down Expand Up @@ -109,7 +117,7 @@ if (ADIOS2_HAVE_MPI AND NOT ("${MPIEXEC_EXECUTABLE}" STREQUAL "MPIEXEC_EXECUTABL
endforeach()
endif()

set (SIMPLE_TESTS "1x1;NoReaderNoWait;TimeoutOnOpen;Modes;1x1.Attrs;1x1.Local")
set (SIMPLE_TESTS "1x1;NoReaderNoWait;TimeoutOnOpen;Modes;1x1.Attrs;1x1.Local;1x1.SharedNothing;1x1.SharedIO;1x1.SharedVar;1x1.SharedNothingSync;1x1.SharedIOSync;1x1.SharedVarSync;2x1.SharedNothing;2x1.SharedIO;2x1.SharedVar;2x1.SharedNothingSync;2x1.SharedIOSync;2x1.SharedVarSync")
set (SIMPLE_FORTRAN_TESTS "")
if(ADIOS2_HAVE_Fortran)
set (SIMPLE_FORTRAN_TESTS "FtoC.1x1;CtoF.1x1;FtoF.1x1")
Expand Down Expand Up @@ -175,6 +183,10 @@ if(ADIOS2_HAVE_MPI)
list (REMOVE_ITEM INSITU_TESTS "TimeoutOnOpen" "Modes" "1x1.Attrs")
# Local Vars don't work for InSitu
list (FILTER INSITU_TESTS EXCLUDE REGEX ".*Local$")
# Multiple streams don't work for InSitu
list (FILTER INSITU_TESTS EXCLUDE REGEX "x1.Shared")
# Fortran Destination doesn't work for InSitu
list (FILTER INSITU_TESTS EXCLUDE REGEX "toF")
# DelayedReader doesn't make sense for InSitu
list (FILTER INSITU_TESTS EXCLUDE REGEX "DelayedReader.*")
# Fortran scalar reads don't work for InSitu
Expand All @@ -188,9 +200,12 @@ endif()
# Setup tests for BP engine
#
if (NOT MSVC) # not on windows
# Local Vars don't work for InSitu
set (BP_TESTS ${ALL_SIMPLE_TESTS})
list (FILTER BP_TESTS EXCLUDE REGEX ".*SharedVar$")
foreach(test ${BP_TESTS})
add_common_test(${test} BP)
add_common_test(${test} BP3)
add_common_test(${test} BP4)
endforeach()
endif(NOT MSVC)

Expand Down
180 changes: 180 additions & 0 deletions testing/adios2/engine/staging-common/ParseArgs.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,180 @@
#ifndef _WIN32
#include "strings.h"
#else
#define strcasecmp _stricmp
#endif
std::string fname = "ADIOS2Common";
std::string engine = "sst";
adios2::Params engineParams = {}; // parsed from command line

bool SharedIO = false;
bool SharedVar = false;
int DurationSeconds = 60 * 60 * 24 * 365; // one year default
int DelayMS = 1000; // one step per sec default
int CompressSz = 0;
int CompressZfp = 0;
int TimeGapExpected = 0;
int IgnoreTimeGap = 1;
int ExpectOpenTimeout = 0;
std::string shutdown_name = "DieTest";
adios2::Mode GlobalWriteMode = adios2::Mode::Deferred;

static std::string Trim(std::string &str)
{
size_t first = str.find_first_not_of(' ');
size_t last = str.find_last_not_of(' ');
return str.substr(first, (last - first + 1));
}

/*
* Engine parameters spec is a poor-man's JSON. name:value pairs are separated
* by commas. White space is trimmed off front and back. No quotes or anything
* fancy allowed.
*/
static adios2::Params ParseEngineParams(std::string Input)
{
std::istringstream ss(Input);
std::string Param;
adios2::Params Ret = {};

while (std::getline(ss, Param, ','))
{
std::istringstream ss2(Param);
std::string ParamName;
std::string ParamValue;
std::getline(ss2, ParamName, ':');
if (!std::getline(ss2, ParamValue, ':'))
{
throw std::invalid_argument("Engine parameter \"" + Param +
"\" missing value");
}
Ret[Trim(ParamName)] = Trim(ParamValue);
}
return Ret;
}

static void ParseArgs(int argc, char **argv)
{
int bare_arg = 0;
while (argc > 1)
{
if (std::string(argv[1]) == "--expect_time_gap")
{

TimeGapExpected++;
IgnoreTimeGap = 0;
}
else if (std::string(argv[1]) == "--expect_contiguous_time")
{
TimeGapExpected = 0;
IgnoreTimeGap = 0;
}
else if (std::string(argv[1]) == "--compress_sz")
{
CompressSz++;
}
else if (std::string(argv[1]) == "--shared_io")
{
SharedIO = true;
}
else if (std::string(argv[1]) == "--shared_var")
{
SharedVar = true;
SharedIO = true;
}
else if (std::string(argv[1]) == "--compress_zfp")
{
CompressZfp++;
}
else if (std::string(argv[1]) == "--filename")
{
fname = std::string(argv[2]);
argv++;
argc--;
}
else if (std::string(argv[1]) == "--write_mode")
{
if (strcasecmp(argv[2], "sync") == 0)
{
GlobalWriteMode = adios2::Mode::Sync;
}
else if (strcasecmp(argv[2], "deferred") == 0)
{
GlobalWriteMode = adios2::Mode::Deferred;
}
else
{
std::cerr << "Invalid mode for --write_mode " << argv[2]
<< std::endl;
}
argv++;
argc--;
}
else if (std::string(argv[1]) == "--engine_params")
{
engineParams = ParseEngineParams(argv[2]);
argv++;
argc--;
}
else if (std::string(argv[1]) == "--engine")
{
engine = std::string(argv[2]);
argv++;
argc--;
}
else if (std::string(argv[1]) == "--expect_timeout")
{
ExpectOpenTimeout++;
}
else if (std::string(argv[1]) == "--duration")
{
std::istringstream ss(argv[2]);
if (!(ss >> DurationSeconds))
std::cerr << "Invalid number for duration " << argv[1] << '\n';
argv++;
argc--;
}
else if (std::string(argv[1]) == "--shutdown_filename")
{
shutdown_name = std::string(argv[2]);
argv++;
argc--;
}
else if (std::string(argv[1]) == "--ms_delay")
{
std::istringstream ss(argv[2]);
if (!(ss >> DelayMS))
std::cerr << "Invalid number for ms_delay " << argv[1] << '\n';
argv++;
argc--;
}
else
{
if (bare_arg == 0)
{
/* first arg without -- is engine */
engine = std::string(argv[1]);
bare_arg++;
}
else if (bare_arg == 1)
{
/* second arg without -- is filename */
fname = std::string(argv[1]);
bare_arg++;
}
else if (bare_arg == 2)
{
engineParams = ParseEngineParams(argv[1]);
bare_arg++;
}
else
{

throw std::invalid_argument("Unknown argument \"" +
std::string(argv[1]) + "\"");
}
}
argv++;
argc--;
}
}
Loading

0 comments on commit 33f6be0

Please sign in to comment.