Merge pull request #3095 from vicentebolea/add-mpi-dataplane

SST: Add MPI SST dataplane

.github/workflows/everything.yml

@@ -114,6 +114,9 @@ jobs:
             compiler: cuda
             parallel: serial
             constrains: build_only
+          - os: el8
+            compiler: gcc10
+            parallel: mpich
 
     steps:
       - uses: actions/checkout@v3

cmake/DetectOptions.cmake

@@ -369,6 +369,9 @@ if(ADIOS2_USE_SST AND NOT WIN32)
       set(ADIOS2_SST_HAVE_CRAY_DRC TRUE)
     endif()
   endif()
+  if(ADIOS2_HAVE_MPI)
+    set(ADIOS2_SST_HAVE_MPI TRUE)
+  endif()
 endif()
 
 # DAOS

docs/user_guide/source/advanced/ecp_hardware.rst

@@ -0,0 +1,59 @@
+######################
+ADIOS2 in ECP hardware
+######################
+
+ADIOS2 is widely used in ECP (Exascale Computing Project) HPC (high performance
+computing) systems, some particular ADIOS2 features needs from specifics
+workarounds to run successfully.
+
+OLCF CRUSHER
+============
+
+SST MPI Data Transport
+----------------------
+
+MPI Data Transport relies on client-server features of MPI which are currently
+supported in Cray-MPI implementations with some caveats. Here are some of the
+observed issues and what its workaround (if any) are:
+
+**MPI_Finalize** will block the system process in the "Writer/Producer" ADIOS2
+instance. The reason is that the Producer ADIOS instance internally calls
+`MPI_Open_port` which somehow even after calling `MPI_Close_port` `MPI_Finalize`
+still consider its port to be in used, hence blocking the process. The
+workaround is to use a `MPI_Barrier(MPI_COMM_WORLD)` instead of `MPI_Finalize()`
+call.
+
+**srun does not understand mpmd instructions** Simply disable them with the flag
+`-DADIOS2_RUN_MPI_MPMD_TESTS=OFF`
+
+**Tests timeout**  Since we launch every tests with srun the scheduling times
+can exceed the test default timeout. Use a large timeout (5mins) for running
+your tests.
+
+Examples of launching ADIOS2 SST unit tests using MPI DP:
+
+.. code-block:: bash
+ # We omit some of the srun (SLURM) arguments which are specific of the project
+ # you are working on. Note that you could avoid calling srun directly by
+ # setting the CMAKE variable `MPIEXEC_EXECUTABLE`.
+
+ # Launch simple writer test instance
+ srun {PROJFLAGS }-N 1 /gpfs/alpine/proj-shared/csc331/vbolea/ADIOS2-build/bin/TestCommonWrite SST mpi_dp_test CPCommPattern=Min,MarshalMethod=BP5'
+
+ # On another terminal launch multiple instances of the Reader test
+ srun {PROJFLAGS} -N 2 /gpfs/alpine/proj-shared/csc331/vbolea/ADIOS2-build/bin/TestCommonRead SST mpi_dp_test
+
+Alternatively, you can configure your CMake build to use srun directly:
+
+.. code-block:: bash
+ cmake . -DMPIEXEC_EXECUTABLE:FILEPATH="/usr/bin/srun" \
+         -DMPIEXEC_EXTRA_FLAGS:STRING="-A{YourProject} -pbatch -t10" \
+         -DMPIEXEC_NUMPROC_FLAG:STRING="-N" \
+         -DMPIEXEC_MAX_NUMPROCS:STRING="-8" \
+         -DADIOS2_RUN_MPI_MPMD_TESTS=OFF
+
+ cmake --build .
+ ctest
+
+ # monitor your jobs
+ watch -n1 squeue -l -u $USER

docs/user_guide/source/engines/sst.rst

@@ -157,7 +157,7 @@ the underlying network communication mechanism to use for exchanging
 data in SST.  Generally this is chosen by SST based upon what is
 available on the current platform.  However, specifying this engine
 parameter allows overriding SST's choice.  Current allowed values are
-**"RDMA"** and **"WAN"**.  (**ib** and **fabric** are accepted as
+**"MPI"**, **"RDMA"**, and **"WAN"**.  (**ib** and **fabric** are accepted as
 equivalent to **RDMA** and **evpath** is equivalent to **WAN**.)
 Generally both the reader and writer should be using the same network
 transport, and the network transport chosen may be dictated by the
@@ -288,7 +288,7 @@ BeginStep timeouts) and writer-side rules (like queue limit behavior) apply.
  QueueLimit                      integer             **0** (no queue limits)
  QueueFullPolicy                 string              **Block**, Discard
  ReserveQueueLimit               integer             **0** (no queue limits)
- DataTransport                   string              **default varies by platform**, RDMA, WAN
+ DataTransport                   string              **default varies by platform**, MPI, RDMA, WAN
  WANDataTransport                string              **sockets**, enet, ib
  ControlTransport                string              **TCP**, Scalable
  NetworkInterface                string              **NULL**

docs/user_guide/source/index.rst

@@ -38,6 +38,7 @@ Funded by the `Exascale Computing Project (ECP) <https://www.exascaleproject.org
    advanced/memory_management
    advanced/gpu_aware
    advanced/plugins
+   advanced/ecp_hardware
 
 .. toctree::
    :caption: Ecosystem Tools

examples/basics/globalArray/globalArray_write.cpp

@@ -42,7 +42,8 @@ int main(int argc, char *argv[])
 {
     int rank = 0, nproc = 1;
 #if ADIOS2_USE_MPI
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &nproc);
 #endif

examples/basics/joinedArray/joinedArray_write.cpp

@@ -42,7 +42,8 @@ int main(int argc, char *argv[])
     int rank = 0;
 #if ADIOS2_USE_MPI
     int nproc = 1;
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &nproc);
 #endif

examples/basics/localArray/localArray_write.cpp

@@ -47,7 +47,8 @@ int main(int argc, char *argv[])
     int rank = 0;
 #if ADIOS2_USE_MPI
     int nproc = 1;
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &nproc);
 #endif

examples/basics/values/values_write.cpp

@@ -30,7 +30,8 @@ int main(int argc, char *argv[])
 {
     int rank = 0, nproc = 1;
 #if ADIOS2_USE_MPI
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &nproc);
 #endif

examples/experimental/runtimeconfig/hello/helloBPWriterXML.cpp

@@ -27,7 +27,8 @@
 
 int main(int argc, char *argv[])
 {
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     int rank, size;
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);

examples/heatTransfer/read/heatRead.cpp

@@ -61,7 +61,16 @@ void Compute(const std::vector<double> &Tin, std::vector<double> &Tout,
 
 int main(int argc, char *argv[])
 {
-    MPI_Init(&argc, &argv);
+    int provided;
+    std::string engineName = std::string(argv[argc - 1]);
+
+    int threadSupportLevel = MPI_THREAD_SINGLE;
+    if (engineName == "SST")
+    {
+        threadSupportLevel = MPI_THREAD_MULTIPLE;
+    }
+
+    MPI_Init_thread(&argc, &argv, threadSupportLevel, &provided);
 
     /* When writer and reader is launched together with a single mpirun command,
        the world comm spans all applications. We have to split and create the

examples/heatTransfer/write/main.cpp

@@ -37,7 +37,17 @@ void printUsage()
 
 int main(int argc, char *argv[])
 {
-    MPI_Init(&argc, &argv);
+    int provided;
+
+    std::string engineName = std::string(argv[argc - 1]);
+
+    int threadSupportLevel = MPI_THREAD_SINGLE;
+    if (engineName == "SST")
+    {
+        threadSupportLevel = MPI_THREAD_MULTIPLE;
+    }
+
+    MPI_Init_thread(&argc, &argv, threadSupportLevel, &provided);
 
     /* When writer and reader is launched together with a single mpirun command,
        the world comm spans all applications. We have to split and create the

examples/hello/bpAttributeWriter/helloBPAttributeWriter.cpp

@@ -20,7 +20,8 @@
 
 int main(int argc, char *argv[])
 {
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     int rank, size;
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);

examples/hello/bpFWriteCRead/CppReader.cpp

@@ -15,7 +15,8 @@
 
 int main(int argc, char *argv[])
 {
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     int rank, size;
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);

examples/hello/bpFWriteCRead/CppWriter.cpp

@@ -15,7 +15,8 @@
 
 int main(int argc, char *argv[])
 {
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     int rank, size;
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);

examples/hello/bpFlushWriter/helloBPFlushWriter.cpp

@@ -20,7 +20,8 @@
 
 int main(int argc, char *argv[])
 {
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     int rank, size;
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);

examples/hello/bpReader/helloBPReader.cpp

@@ -21,7 +21,8 @@
 
 int main(int argc, char *argv[])
 {
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     int rank, size;
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);

examples/hello/bpReader/helloBPReaderHeatMap2D.cpp

@@ -30,7 +30,8 @@
 
 int main(int argc, char *argv[])
 {
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     int rank, size;
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);

examples/hello/bpReader/helloBPReaderHeatMap3D.cpp

@@ -34,7 +34,8 @@
 
 int main(int argc, char *argv[])
 {
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     int rank, size;
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);

examples/hello/bpTimeWriter/helloBPTimeWriter.cpp

@@ -21,7 +21,8 @@
 
 int main(int argc, char *argv[])
 {
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     int rank, size;
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);

examples/hello/bpWriter/helloBPPutDeferred.cpp

@@ -35,7 +35,8 @@ int main(int argc, char *argv[])
     int rank, size;
 
 #if ADIOS2_USE_MPI
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);
 #else

examples/hello/bpWriter/helloBPSZ.cpp

@@ -34,7 +34,8 @@ int main(int argc, char *argv[])
     int rank, size;
 
 #if ADIOS2_USE_MPI
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);
 #else

examples/hello/bpWriter/helloBPSubStreams.cpp

@@ -22,7 +22,8 @@ int main(int argc, char *argv[])
 {
     int rank, size;
 #if ADIOS2_USE_MPI
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);
 #else

examples/hello/bpWriter/helloBPWriter.c

@@ -39,7 +39,8 @@ int main(int argc, char *argv[])
     int rank, size;
 
 #if ADIOS2_USE_MPI
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);
 #else

examples/hello/bpWriter/helloBPWriter.cpp

@@ -23,7 +23,8 @@ int main(int argc, char *argv[])
 {
     int rank, size;
 #if ADIOS2_USE_MPI
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);
 #else

examples/hello/datamanReader/helloDataManReader.cpp

@@ -32,7 +32,8 @@ void PrintData(std::vector<T> &data, size_t step)
 int main(int argc, char *argv[])
 {
     // initialize MPI
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
     MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
 

examples/hello/datamanWriter/helloDataManWriter.cpp

@@ -51,7 +51,8 @@ std::vector<T> GenerateData(const size_t step)
 int main(int argc, char *argv[])
 {
     // initialize MPI
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank);
     MPI_Comm_size(MPI_COMM_WORLD, &mpiSize);
 

examples/hello/dataspacesReader/helloDataSpacesReader.cpp

@@ -28,7 +28,8 @@ int main(int argc, char *argv[])
     int size;
 
 #if ADIOS2_USE_MPI
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);
 #else

examples/hello/dataspacesWriter/helloDataSpacesWriter.cpp

@@ -25,7 +25,8 @@ int main(int argc, char *argv[])
     int size;
 
 #if ADIOS2_USE_MPI
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);
 #else

examples/hello/hdf5Reader/helloHDF5Reader.cpp

@@ -66,7 +66,8 @@ void ReadData(adios2::IO h5IO, adios2::Engine &h5Reader,
 
 int main(int argc, char *argv[])
 {
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     int rank, size;
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);

examples/hello/hdf5Writer/helloHDF5Writer.cpp

@@ -19,7 +19,8 @@
 
 int main(int argc, char *argv[])
 {
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     int rank, size;
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);

examples/hello/inlineReaderWriter/helloInlineReaderWriter.cpp

@@ -83,7 +83,8 @@ int main(int argc, char *argv[])
 {
     int rank = 0, size = 1;
 #if ADIOS2_USE_MPI
-    MPI_Init(&argc, &argv);
+    int provided;
+    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
     MPI_Comm_rank(MPI_COMM_WORLD, &rank);
     MPI_Comm_size(MPI_COMM_WORLD, &size);
     adios2::ADIOS adios(MPI_COMM_WORLD);