diff --git a/.github/workflows/everything.yml b/.github/workflows/everything.yml index ce23bd295f..89c304de60 100644 --- a/.github/workflows/everything.yml +++ b/.github/workflows/everything.yml @@ -114,6 +114,9 @@ jobs: compiler: cuda parallel: serial constrains: build_only + - os: el8 + compiler: gcc10 + parallel: mpich steps: - uses: actions/checkout@v3 diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index d4e2e5dbf3..9a41f19bb2 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -369,6 +369,9 @@ if(ADIOS2_USE_SST AND NOT WIN32) set(ADIOS2_SST_HAVE_CRAY_DRC TRUE) endif() endif() + if(ADIOS2_HAVE_MPI) + set(ADIOS2_SST_HAVE_MPI TRUE) + endif() endif() # DAOS diff --git a/docs/user_guide/source/advanced/ecp_hardware.rst b/docs/user_guide/source/advanced/ecp_hardware.rst new file mode 100644 index 0000000000..5f79efb49b --- /dev/null +++ b/docs/user_guide/source/advanced/ecp_hardware.rst @@ -0,0 +1,59 @@ +###################### +ADIOS2 in ECP hardware +###################### + +ADIOS2 is widely used in ECP (Exascale Computing Project) HPC (high performance +computing) systems, some particular ADIOS2 features needs from specifics +workarounds to run successfully. + +OLCF CRUSHER +============ + +SST MPI Data Transport +---------------------- + +MPI Data Transport relies on client-server features of MPI which are currently +supported in Cray-MPI implementations with some caveats. Here are some of the +observed issues and what its workaround (if any) are: + +**MPI_Finalize** will block the system process in the "Writer/Producer" ADIOS2 +instance. The reason is that the Producer ADIOS instance internally calls +`MPI_Open_port` which somehow even after calling `MPI_Close_port` `MPI_Finalize` +still consider its port to be in used, hence blocking the process. The +workaround is to use a `MPI_Barrier(MPI_COMM_WORLD)` instead of `MPI_Finalize()` +call. + +**srun does not understand mpmd instructions** Simply disable them with the flag +`-DADIOS2_RUN_MPI_MPMD_TESTS=OFF` + +**Tests timeout** Since we launch every tests with srun the scheduling times +can exceed the test default timeout. Use a large timeout (5mins) for running +your tests. + +Examples of launching ADIOS2 SST unit tests using MPI DP: + +.. code-block:: bash + # We omit some of the srun (SLURM) arguments which are specific of the project + # you are working on. Note that you could avoid calling srun directly by + # setting the CMAKE variable `MPIEXEC_EXECUTABLE`. + + # Launch simple writer test instance + srun {PROJFLAGS }-N 1 /gpfs/alpine/proj-shared/csc331/vbolea/ADIOS2-build/bin/TestCommonWrite SST mpi_dp_test CPCommPattern=Min,MarshalMethod=BP5' + + # On another terminal launch multiple instances of the Reader test + srun {PROJFLAGS} -N 2 /gpfs/alpine/proj-shared/csc331/vbolea/ADIOS2-build/bin/TestCommonRead SST mpi_dp_test + +Alternatively, you can configure your CMake build to use srun directly: + +.. code-block:: bash + cmake . -DMPIEXEC_EXECUTABLE:FILEPATH="/usr/bin/srun" \ + -DMPIEXEC_EXTRA_FLAGS:STRING="-A{YourProject} -pbatch -t10" \ + -DMPIEXEC_NUMPROC_FLAG:STRING="-N" \ + -DMPIEXEC_MAX_NUMPROCS:STRING="-8" \ + -DADIOS2_RUN_MPI_MPMD_TESTS=OFF + + cmake --build . + ctest + + # monitor your jobs + watch -n1 squeue -l -u $USER diff --git a/docs/user_guide/source/engines/sst.rst b/docs/user_guide/source/engines/sst.rst index 67f7fa995c..0ba0a80664 100644 --- a/docs/user_guide/source/engines/sst.rst +++ b/docs/user_guide/source/engines/sst.rst @@ -157,7 +157,7 @@ the underlying network communication mechanism to use for exchanging data in SST. Generally this is chosen by SST based upon what is available on the current platform. However, specifying this engine parameter allows overriding SST's choice. Current allowed values are -**"RDMA"** and **"WAN"**. (**ib** and **fabric** are accepted as +**"MPI"**, **"RDMA"**, and **"WAN"**. (**ib** and **fabric** are accepted as equivalent to **RDMA** and **evpath** is equivalent to **WAN**.) Generally both the reader and writer should be using the same network transport, and the network transport chosen may be dictated by the @@ -288,7 +288,7 @@ BeginStep timeouts) and writer-side rules (like queue limit behavior) apply. QueueLimit integer **0** (no queue limits) QueueFullPolicy string **Block**, Discard ReserveQueueLimit integer **0** (no queue limits) - DataTransport string **default varies by platform**, RDMA, WAN + DataTransport string **default varies by platform**, MPI, RDMA, WAN WANDataTransport string **sockets**, enet, ib ControlTransport string **TCP**, Scalable NetworkInterface string **NULL** diff --git a/docs/user_guide/source/index.rst b/docs/user_guide/source/index.rst index a18698fbda..6fd178f524 100644 --- a/docs/user_guide/source/index.rst +++ b/docs/user_guide/source/index.rst @@ -38,6 +38,7 @@ Funded by the `Exascale Computing Project (ECP) &Tin, std::vector &Tout, int main(int argc, char *argv[]) { - MPI_Init(&argc, &argv); + int provided; + std::string engineName = std::string(argv[argc - 1]); + + int threadSupportLevel = MPI_THREAD_SINGLE; + if (engineName == "SST") + { + threadSupportLevel = MPI_THREAD_MULTIPLE; + } + + MPI_Init_thread(&argc, &argv, threadSupportLevel, &provided); /* When writer and reader is launched together with a single mpirun command, the world comm spans all applications. We have to split and create the diff --git a/examples/heatTransfer/write/main.cpp b/examples/heatTransfer/write/main.cpp index d8ec64ad6d..2f9566b7fc 100644 --- a/examples/heatTransfer/write/main.cpp +++ b/examples/heatTransfer/write/main.cpp @@ -37,7 +37,17 @@ void printUsage() int main(int argc, char *argv[]) { - MPI_Init(&argc, &argv); + int provided; + + std::string engineName = std::string(argv[argc - 1]); + + int threadSupportLevel = MPI_THREAD_SINGLE; + if (engineName == "SST") + { + threadSupportLevel = MPI_THREAD_MULTIPLE; + } + + MPI_Init_thread(&argc, &argv, threadSupportLevel, &provided); /* When writer and reader is launched together with a single mpirun command, the world comm spans all applications. We have to split and create the diff --git a/examples/hello/bpAttributeWriter/helloBPAttributeWriter.cpp b/examples/hello/bpAttributeWriter/helloBPAttributeWriter.cpp index 7ada5e9148..a47c8e8670 100644 --- a/examples/hello/bpAttributeWriter/helloBPAttributeWriter.cpp +++ b/examples/hello/bpAttributeWriter/helloBPAttributeWriter.cpp @@ -20,7 +20,8 @@ int main(int argc, char *argv[]) { - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); diff --git a/examples/hello/bpFWriteCRead/CppReader.cpp b/examples/hello/bpFWriteCRead/CppReader.cpp index 7858c0b4a2..9804115226 100644 --- a/examples/hello/bpFWriteCRead/CppReader.cpp +++ b/examples/hello/bpFWriteCRead/CppReader.cpp @@ -15,7 +15,8 @@ int main(int argc, char *argv[]) { - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); diff --git a/examples/hello/bpFWriteCRead/CppWriter.cpp b/examples/hello/bpFWriteCRead/CppWriter.cpp index ff658de531..efb54214cf 100644 --- a/examples/hello/bpFWriteCRead/CppWriter.cpp +++ b/examples/hello/bpFWriteCRead/CppWriter.cpp @@ -15,7 +15,8 @@ int main(int argc, char *argv[]) { - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); diff --git a/examples/hello/bpFlushWriter/helloBPFlushWriter.cpp b/examples/hello/bpFlushWriter/helloBPFlushWriter.cpp index e88e5253fe..6dcb86c85a 100644 --- a/examples/hello/bpFlushWriter/helloBPFlushWriter.cpp +++ b/examples/hello/bpFlushWriter/helloBPFlushWriter.cpp @@ -20,7 +20,8 @@ int main(int argc, char *argv[]) { - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); diff --git a/examples/hello/bpReader/helloBPReader.cpp b/examples/hello/bpReader/helloBPReader.cpp index 1e1cb483fa..80d4606197 100644 --- a/examples/hello/bpReader/helloBPReader.cpp +++ b/examples/hello/bpReader/helloBPReader.cpp @@ -21,7 +21,8 @@ int main(int argc, char *argv[]) { - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); diff --git a/examples/hello/bpReader/helloBPReaderHeatMap2D.cpp b/examples/hello/bpReader/helloBPReaderHeatMap2D.cpp index ceebdb8e91..36ef12cce3 100644 --- a/examples/hello/bpReader/helloBPReaderHeatMap2D.cpp +++ b/examples/hello/bpReader/helloBPReaderHeatMap2D.cpp @@ -30,7 +30,8 @@ int main(int argc, char *argv[]) { - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); diff --git a/examples/hello/bpReader/helloBPReaderHeatMap3D.cpp b/examples/hello/bpReader/helloBPReaderHeatMap3D.cpp index 81ed5f4941..5068498a64 100644 --- a/examples/hello/bpReader/helloBPReaderHeatMap3D.cpp +++ b/examples/hello/bpReader/helloBPReaderHeatMap3D.cpp @@ -34,7 +34,8 @@ int main(int argc, char *argv[]) { - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); diff --git a/examples/hello/bpTimeWriter/helloBPTimeWriter.cpp b/examples/hello/bpTimeWriter/helloBPTimeWriter.cpp index 84468aa354..cf30c590b0 100644 --- a/examples/hello/bpTimeWriter/helloBPTimeWriter.cpp +++ b/examples/hello/bpTimeWriter/helloBPTimeWriter.cpp @@ -21,7 +21,8 @@ int main(int argc, char *argv[]) { - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); diff --git a/examples/hello/bpWriter/helloBPPutDeferred.cpp b/examples/hello/bpWriter/helloBPPutDeferred.cpp index a455a56751..65574f3f71 100644 --- a/examples/hello/bpWriter/helloBPPutDeferred.cpp +++ b/examples/hello/bpWriter/helloBPPutDeferred.cpp @@ -35,7 +35,8 @@ int main(int argc, char *argv[]) int rank, size; #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); #else diff --git a/examples/hello/bpWriter/helloBPSZ.cpp b/examples/hello/bpWriter/helloBPSZ.cpp index a9943557ed..f173cc7dfe 100644 --- a/examples/hello/bpWriter/helloBPSZ.cpp +++ b/examples/hello/bpWriter/helloBPSZ.cpp @@ -34,7 +34,8 @@ int main(int argc, char *argv[]) int rank, size; #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); #else diff --git a/examples/hello/bpWriter/helloBPSubStreams.cpp b/examples/hello/bpWriter/helloBPSubStreams.cpp index 59ed618eef..168008422d 100644 --- a/examples/hello/bpWriter/helloBPSubStreams.cpp +++ b/examples/hello/bpWriter/helloBPSubStreams.cpp @@ -22,7 +22,8 @@ int main(int argc, char *argv[]) { int rank, size; #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); #else diff --git a/examples/hello/bpWriter/helloBPWriter.c b/examples/hello/bpWriter/helloBPWriter.c index 789f21e034..40b1f1788c 100644 --- a/examples/hello/bpWriter/helloBPWriter.c +++ b/examples/hello/bpWriter/helloBPWriter.c @@ -39,7 +39,8 @@ int main(int argc, char *argv[]) int rank, size; #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); #else diff --git a/examples/hello/bpWriter/helloBPWriter.cpp b/examples/hello/bpWriter/helloBPWriter.cpp index 6c5714b8e7..9307b7d80e 100644 --- a/examples/hello/bpWriter/helloBPWriter.cpp +++ b/examples/hello/bpWriter/helloBPWriter.cpp @@ -23,7 +23,8 @@ int main(int argc, char *argv[]) { int rank, size; #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); #else diff --git a/examples/hello/datamanReader/helloDataManReader.cpp b/examples/hello/datamanReader/helloDataManReader.cpp index bb5ad24699..3ffb8bfa60 100644 --- a/examples/hello/datamanReader/helloDataManReader.cpp +++ b/examples/hello/datamanReader/helloDataManReader.cpp @@ -32,7 +32,8 @@ void PrintData(std::vector &data, size_t step) int main(int argc, char *argv[]) { // initialize MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); diff --git a/examples/hello/datamanWriter/helloDataManWriter.cpp b/examples/hello/datamanWriter/helloDataManWriter.cpp index d116908939..530bf52a91 100644 --- a/examples/hello/datamanWriter/helloDataManWriter.cpp +++ b/examples/hello/datamanWriter/helloDataManWriter.cpp @@ -51,7 +51,8 @@ std::vector GenerateData(const size_t step) int main(int argc, char *argv[]) { // initialize MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); diff --git a/examples/hello/dataspacesReader/helloDataSpacesReader.cpp b/examples/hello/dataspacesReader/helloDataSpacesReader.cpp index a68fb21702..278f89ccd4 100644 --- a/examples/hello/dataspacesReader/helloDataSpacesReader.cpp +++ b/examples/hello/dataspacesReader/helloDataSpacesReader.cpp @@ -28,7 +28,8 @@ int main(int argc, char *argv[]) int size; #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); #else diff --git a/examples/hello/dataspacesWriter/helloDataSpacesWriter.cpp b/examples/hello/dataspacesWriter/helloDataSpacesWriter.cpp index 74fb19317b..754e706600 100644 --- a/examples/hello/dataspacesWriter/helloDataSpacesWriter.cpp +++ b/examples/hello/dataspacesWriter/helloDataSpacesWriter.cpp @@ -25,7 +25,8 @@ int main(int argc, char *argv[]) int size; #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); #else diff --git a/examples/hello/hdf5Reader/helloHDF5Reader.cpp b/examples/hello/hdf5Reader/helloHDF5Reader.cpp index 8031c66ec4..bbc9c75a1f 100644 --- a/examples/hello/hdf5Reader/helloHDF5Reader.cpp +++ b/examples/hello/hdf5Reader/helloHDF5Reader.cpp @@ -66,7 +66,8 @@ void ReadData(adios2::IO h5IO, adios2::Engine &h5Reader, int main(int argc, char *argv[]) { - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); diff --git a/examples/hello/hdf5Writer/helloHDF5Writer.cpp b/examples/hello/hdf5Writer/helloHDF5Writer.cpp index 90a0c549ca..ac290da756 100644 --- a/examples/hello/hdf5Writer/helloHDF5Writer.cpp +++ b/examples/hello/hdf5Writer/helloHDF5Writer.cpp @@ -19,7 +19,8 @@ int main(int argc, char *argv[]) { - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); diff --git a/examples/hello/inlineReaderWriter/helloInlineReaderWriter.cpp b/examples/hello/inlineReaderWriter/helloInlineReaderWriter.cpp index ae0a54cdf7..82a3c10857 100644 --- a/examples/hello/inlineReaderWriter/helloInlineReaderWriter.cpp +++ b/examples/hello/inlineReaderWriter/helloInlineReaderWriter.cpp @@ -83,7 +83,8 @@ int main(int argc, char *argv[]) { int rank = 0, size = 1; #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); adios2::ADIOS adios(MPI_COMM_WORLD); diff --git a/examples/hello/skeleton/helloSkeletonReader.cpp b/examples/hello/skeleton/helloSkeletonReader.cpp index 7f5f847fbc..4beec54796 100644 --- a/examples/hello/skeleton/helloSkeletonReader.cpp +++ b/examples/hello/skeleton/helloSkeletonReader.cpp @@ -16,7 +16,8 @@ int main(int argc, char *argv[]) #if ADIOS2_USE_MPI int wrank = 0, wnproc = 1; MPI_Comm mpiReaderComm; - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &wrank); MPI_Comm_size(MPI_COMM_WORLD, &wnproc); diff --git a/examples/hello/skeleton/helloSkeletonWriter.cpp b/examples/hello/skeleton/helloSkeletonWriter.cpp index 2c18524d00..40623692ab 100644 --- a/examples/hello/skeleton/helloSkeletonWriter.cpp +++ b/examples/hello/skeleton/helloSkeletonWriter.cpp @@ -24,7 +24,8 @@ int main(int argc, char *argv[]) #if ADIOS2_USE_MPI int wrank = 0, wnproc = 1; MPI_Comm mpiWriterComm; - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &wrank); MPI_Comm_size(MPI_COMM_WORLD, &wnproc); diff --git a/examples/hello/sstReader/helloSstReader.cpp b/examples/hello/sstReader/helloSstReader.cpp index 2003518fae..b762c42eac 100644 --- a/examples/hello/sstReader/helloSstReader.cpp +++ b/examples/hello/sstReader/helloSstReader.cpp @@ -27,7 +27,8 @@ int main(int argc, char *argv[]) int size; #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provide; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provide); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); #else diff --git a/examples/hello/sstWriter/helloSstWriter.cpp b/examples/hello/sstWriter/helloSstWriter.cpp index 5cf47aceb5..5cdc147b5b 100644 --- a/examples/hello/sstWriter/helloSstWriter.cpp +++ b/examples/hello/sstWriter/helloSstWriter.cpp @@ -24,7 +24,8 @@ int main(int argc, char *argv[]) int size; #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provide; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provide); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); #else diff --git a/examples/query/test.cpp b/examples/query/test.cpp index 927d3347d7..8452f1d551 100644 --- a/examples/query/test.cpp +++ b/examples/query/test.cpp @@ -14,7 +14,8 @@ int main(int argc, char *argv[]) { - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); diff --git a/examples/useCases/insituGlobalArrays/insituGlobalArraysReaderNxN.cpp b/examples/useCases/insituGlobalArrays/insituGlobalArraysReaderNxN.cpp index 6cff5b88d1..6ef2bb6bb3 100644 --- a/examples/useCases/insituGlobalArrays/insituGlobalArraysReaderNxN.cpp +++ b/examples/useCases/insituGlobalArrays/insituGlobalArraysReaderNxN.cpp @@ -214,7 +214,8 @@ int main(int argc, char *argv[]) { int rank = 0, nproc = 1; #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); int wrank, wnproc; MPI_Comm_rank(MPI_COMM_WORLD, &wrank); MPI_Comm_size(MPI_COMM_WORLD, &wnproc); diff --git a/examples/useCases/insituGlobalArrays/insituGlobalArraysWriter.cpp b/examples/useCases/insituGlobalArrays/insituGlobalArraysWriter.cpp index 01681f1d5f..11321f92e5 100644 --- a/examples/useCases/insituGlobalArrays/insituGlobalArraysWriter.cpp +++ b/examples/useCases/insituGlobalArrays/insituGlobalArraysWriter.cpp @@ -89,7 +89,8 @@ int main(int argc, char *argv[]) { int rank = 0, nproc = 1; #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); int wrank, wnproc; MPI_Comm_rank(MPI_COMM_WORLD, &wrank); MPI_Comm_size(MPI_COMM_WORLD, &wnproc); diff --git a/scripts/ci/cmake-v2/ci-el8-gcc10-mpich.cmake b/scripts/ci/cmake-v2/ci-el8-gcc10-mpich.cmake new file mode 100644 index 0000000000..b0c5fc870f --- /dev/null +++ b/scripts/ci/cmake-v2/ci-el8-gcc10-mpich.cmake @@ -0,0 +1,33 @@ +# Client maintainer: chuck.atkins@kitware.com + +include(ProcessorCount) +ProcessorCount(NCPUS) +math(EXPR N2CPUS "${NCPUS}*2") + +set(ENV{CC} gcc) +set(ENV{CXX} g++) +set(ENV{FC} gfortran) + +set(dashboard_cache " +ADIOS2_USE_BZip2:BOOL=ON +ADIOS2_USE_Blosc:BOOL=ON +ADIOS2_USE_DataMan:BOOL=ON +ADIOS2_USE_Fortran:BOOL=ON +ADIOS2_USE_HDF5:BOOL=ON +ADIOS2_USE_MPI:BOOL=ON +ADIOS2_USE_Python:BOOL=ON +ADIOS2_USE_SZ:BOOL=ON +ADIOS2_USE_ZeroMQ:STRING=ON +ADIOS2_USE_ZFP:BOOL=ON + +CMAKE_C_FLAGS:STRING=-Wall +CMAKE_CXX_FLAGS:STRING=-Wall +CMAKE_Fortran_FLAGS:STRING=-Wall + +MPIEXEC_MAX_NUMPROCS:STRING=${N2CPUS} +") + +set(CTEST_TEST_ARGS PARALLEL_LEVEL 1) +set(CTEST_CMAKE_GENERATOR "Unix Makefiles") +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/images-v2/Dockerfile.ci-spack-el8-leaf b/scripts/ci/images-v2/Dockerfile.ci-spack-el8-leaf index 8b8365cad8..641b0c99cd 100644 --- a/scripts/ci/images-v2/Dockerfile.ci-spack-el8-leaf +++ b/scripts/ci/images-v2/Dockerfile.ci-spack-el8-leaf @@ -1,6 +1,10 @@ ARG COMPILER_IMG_BASE FROM ornladios/adios2:ci-spack-el8-${COMPILER_IMG_BASE}-base +ARG MPI_FLAVOR=openmpi +RUN sed "s|^packages:$|&\n mpich:\n version: [3.3.2]\n variants: device=ch3 netmod=tcp|" \ + -i /etc/spack/packages.yaml + ARG EXTRA_VARIANTS ARG CUDA_VARIANT RUN sed \ @@ -9,6 +13,11 @@ RUN sed \ -e "s|variants: api=|variants: ${EXTRA_VARIANTS} api=|" \ -i /etc/spack/packages.yaml +ARG MPI_FLAVOR=openmpi +RUN sed "s|^\(\s\+\)all:.*|&\n\1 providers:\n\1 mpi: [${MPI_FLAVOR}]|" \ + -i /etc/spack/packages.yaml && \ + cat /etc/spack/packages.yaml + # Build dependencies ARG COMPILER_SPACK_ID RUN . /etc/profile.d/modules.sh && \ diff --git a/scripts/ci/images-v2/build-mpich-image.sh b/scripts/ci/images-v2/build-mpich-image.sh new file mode 100755 index 0000000000..134e71b2a2 --- /dev/null +++ b/scripts/ci/images-v2/build-mpich-image.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +docker build -t "ornladios/adios2:ci-spack-el8-gcc10-mpich" \ + -f Dockerfile.ci-spack-el8-leaf \ + --build-arg COMPILER_IMG_BASE=gcc10 \ + --build-arg EXTRA_VARIANTS=+mpi \ + --build-arg MPI_FLAVOR=mpich \ + --build-arg COMPILER_SPACK_ID=gcc \ + . diff --git a/scripts/docker/packages.yaml b/scripts/docker/packages.yaml index 2fe6ba28e7..e3f52d6f3a 100644 --- a/scripts/docker/packages.yaml +++ b/scripts/docker/packages.yaml @@ -1,5 +1,10 @@ packages: all: target: [haswell] + provides: + mpi: [openmpi] adios2: variants: +mpi+blosc+bzip2+zfp+sz+png+sst+dataman+ssc+hdf5+python+fortran + mpich: + version: [3.3.2] + variants: device=ch3 netmod=tcp diff --git a/source/adios2/toolkit/sst/CMakeLists.txt b/source/adios2/toolkit/sst/CMakeLists.txt index 7f3911b960..be3ae8b761 100644 --- a/source/adios2/toolkit/sst/CMakeLists.txt +++ b/source/adios2/toolkit/sst/CMakeLists.txt @@ -39,6 +39,11 @@ if(ADIOS2_HAVE_ZFP) target_link_libraries(sst PRIVATE zfp::zfp) endif() +if(ADIOS2_HAVE_MPI) + target_sources(sst PRIVATE dp/mpi_dp.c) + target_link_libraries(sst PRIVATE MPI::MPI_C) +endif() + # Set library version information set_target_properties(sst PROPERTIES OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_sst @@ -54,6 +59,7 @@ set(SST_CONFIG_OPTS FI_GNI CRAY_DRC NVStream + MPI ) include(SSTFunctions) GenerateSSTHeaderConfig(${SST_CONFIG_OPTS}) diff --git a/source/adios2/toolkit/sst/cp/cp_common.c b/source/adios2/toolkit/sst/cp/cp_common.c index 8c9c6835fb..a4c3a51c10 100644 --- a/source/adios2/toolkit/sst/cp/cp_common.c +++ b/source/adios2/toolkit/sst/cp/cp_common.c @@ -131,6 +131,10 @@ void CP_validateParams(SstStream Stream, SstParams Params, int Writer) { Params->DataTransport = strdup("rdma"); } + else + { + Params->DataTransport = strdup(SelectedTransport); + } free(SelectedTransport); } if (Params->ControlTransport == NULL) @@ -1204,6 +1208,7 @@ extern void SstStreamDestroy(SstStream Stream) free(Stream->Timesteps); Stream->Timesteps = Next; } + if (Stream->DP_Stream) { STREAM_MUTEX_UNLOCK(Stream); @@ -1215,8 +1220,10 @@ extern void SstStreamDestroy(SstStream Stream) { Stream->DP_Interface->destroyWriter(&Svcs, Stream->DP_Stream); } + Stream->DP_Stream = NULL; STREAM_MUTEX_LOCK(Stream); } + if (Stream->Readers) { for (int i = 0; i < Stream->ReaderCount; i++) diff --git a/source/adios2/toolkit/sst/cp/cp_writer.c b/source/adios2/toolkit/sst/cp/cp_writer.c index 7c6fe73375..5b02434deb 100644 --- a/source/adios2/toolkit/sst/cp/cp_writer.c +++ b/source/adios2/toolkit/sst/cp/cp_writer.c @@ -1475,6 +1475,14 @@ static void CloseWSRStream(CManager cm, void *WSR_Stream_v) "Delayed task Moving Reader stream %p to status %s\n", CP_WSR_Stream, SSTStreamStatusStr[PeerClosed]); CP_PeerFailCloseWSReader(CP_WSR_Stream, PeerClosed); + + if (strncmp("mpi", ParentStream->ConfigParams->DataTransport, 3) == 0 && + CP_WSR_Stream->DP_WSR_Stream) + { + CP_WSR_Stream->ParentStream->DP_Interface->destroyWriterPerReader( + &Svcs, CP_WSR_Stream->DP_WSR_Stream); + CP_WSR_Stream->DP_WSR_Stream = NULL; + } STREAM_MUTEX_UNLOCK(ParentStream); } @@ -1526,6 +1534,18 @@ static void CP_PeerFailCloseWSReader(WS_ReaderInfo CP_WSR_Stream, CMfree(CMadd_delayed_task(ParentStream->CPInfo->SharedCM->cm, 2, 0, CloseWSRStream, CP_WSR_Stream)); } + else + { + if (strncmp("mpi", ParentStream->ConfigParams->DataTransport, 3) == + 0 && + CP_WSR_Stream->DP_WSR_Stream) + { + CP_WSR_Stream->ParentStream->DP_Interface + ->destroyWriterPerReader(&Svcs, + CP_WSR_Stream->DP_WSR_Stream); + CP_WSR_Stream->DP_WSR_Stream = NULL; + } + } } CP_verbose(ParentStream, PerStepVerbose, "Moving Reader stream %p to status %s\n", CP_WSR_Stream, diff --git a/source/adios2/toolkit/sst/dp/dp.c b/source/adios2/toolkit/sst/dp/dp.c index 755c6400a2..37f6ab45ab 100644 --- a/source/adios2/toolkit/sst/dp/dp.c +++ b/source/adios2/toolkit/sst/dp/dp.c @@ -16,6 +16,9 @@ extern CP_DP_Interface LoadRdmaDP(); #ifdef SST_HAVE_DAOS extern CP_DP_Interface LoadDaosDP(); #endif /* SST_HAVE_LIBFABRIC */ +#ifdef SST_HAVE_MPI +extern CP_DP_Interface LoadMpiDP(); +#endif /* SST_HAVE_MPI*/ extern CP_DP_Interface LoadEVpathDP(); typedef struct _DPElement @@ -68,6 +71,10 @@ CP_DP_Interface SelectDP(CP_Services Svcs, void *CP_Stream, AddDPPossibility(Svcs, CP_Stream, List, LoadDaosDP(), "daos", Params); #endif /* SST_HAVE_DAOS */ +#ifdef SST_HAVE_MPI + List = AddDPPossibility(Svcs, CP_Stream, List, LoadMpiDP(), "mpi", Params); +#endif /* SST_HAVE_MPI */ + int SelectedDP = -1; int BestPriority = -1; int BestPrioDP = -1; diff --git a/source/adios2/toolkit/sst/dp/mpi_dp.c b/source/adios2/toolkit/sst/dp/mpi_dp.c new file mode 100644 index 0000000000..c692558e7a --- /dev/null +++ b/source/adios2/toolkit/sst/dp/mpi_dp.c @@ -0,0 +1,992 @@ +#include "dp_interface.h" +#include "sst_data.h" +#include + +#include +#include +#include +#include +#include +#include + +#define MPI_DP_CONTACT_STRING_LEN 64 + +static pthread_rwlock_t LockRS = PTHREAD_RWLOCK_INITIALIZER; +static pthread_rwlock_t LockTS = PTHREAD_RWLOCK_INITIALIZER; +static pthread_once_t OnceMpiInitializer = PTHREAD_ONCE_INIT; + +/*****Stream Basic Structures ***********************************************/ + +/* Base Stream class, used implicitly */ +typedef struct _MpiStream +{ + void *CP_Stream; + int Rank; + int PID; +} MpiStream; + +/* Link Stream class, used implicitly */ +typedef struct _MpiStreamLink +{ + int CohortSize; + CP_PeerCohort PeerCohort; + SstStats Stats; +} MpiStreamLink; + +/** + * Readers Stream. + * + * It contains the needed data to communicate with a single Writer. + */ +typedef struct _MpiStreamRD +{ + MpiStream Stream; + MpiStreamLink Link; + + CMFormat ReadRequestFormat; + struct _MpiWriterContactInfo *WriterContactInfo; +} * MpiStreamRD; + +/** + * Writers Stream. + * + * It does not directly contains data related to each of the connected Readers. + * Instead it contains a collection of MpiStreamWSR that represent the Stream + * used for interacting with each (one/multiple) of the Readers. + */ +typedef struct _MpiStreamWS +{ + MpiStream Stream; + + struct _TimeStepEntry *TimeSteps; + CMFormat ReadReplyFormat; + int ReaderCount; + struct _MpiStreamWSR **Readers; + pthread_mutex_t MutexReaders; +} * MpiStreamWS; + +/** + * WritersPerReader streams. + * + * It is used in the Writer side to represent the Stream used for communicated + * with a single Reader. + */ +typedef struct _MpiStreamWSR +{ + MpiStreamLink Link; + + struct _MpiStreamWS *StreamWS; + struct _MpiReaderContactInfo *ReaderContactInfo; + char MpiPortName[MPI_MAX_PORT_NAME]; +} * MpiStreamWSR; + +typedef struct _MpiPerTimeStepInfo +{ + char *CheckString; +} * MpiPerTimeStepInfo; + +typedef struct _TimeStepEntry +{ + long TimeStep; + struct _SstData *Data; + struct _MpiPerTimeStepInfo *DP_TimeStepInfo; + struct _TimeStepEntry *Next; +} * TimeStepList; + +typedef struct _MpiReaderContactInfo +{ + char *ContactString; + void *StreamRS; + MPI_Comm MpiComm; +} * MpiReaderContactInfo; + +typedef struct _MpiWriterContactInfo +{ + char *ContactString; + void *StreamWS; + MPI_Comm MpiComm; + int PID; +} * MpiWriterContactInfo; + +/*****Message Data Structures ***********************************************/ + +/** + * Represents where the connection of two streams takes places: + * Remotly|Locally. + */ +enum MPI_DP_COMM_TYPE +{ + MPI_DP_REMOTE = 0, + MPI_DP_LOCAL = 1, +}; + +typedef struct _MpiReadRequestMsg +{ + long TimeStep; + size_t Offset; + size_t Length; + void *StreamWS; + void *StreamRS; + int RequestingRank; + int NotifyCondition; + enum MPI_DP_COMM_TYPE CommType; +} * MpiReadRequestMsg; + +typedef struct _MpiReadReplyMsg +{ + long TimeStep; + size_t DataLength; + void *StreamRS; + int NotifyCondition; + char *MpiPortName; + char *Data; +} * MpiReadReplyMsg; + +static FMField MpiReadRequestList[] = { + {"TimeStep", "integer", sizeof(long), + FMOffset(MpiReadRequestMsg, TimeStep)}, + {"Offset", "integer", sizeof(size_t), FMOffset(MpiReadRequestMsg, Offset)}, + {"Length", "integer", sizeof(size_t), FMOffset(MpiReadRequestMsg, Length)}, + {"StreamWS", "integer", sizeof(void *), + FMOffset(MpiReadRequestMsg, StreamWS)}, + {"StreamRS", "integer", sizeof(void *), + FMOffset(MpiReadRequestMsg, StreamRS)}, + {"RequestingRank", "integer", sizeof(int), + FMOffset(MpiReadRequestMsg, RequestingRank)}, + {"NotifyCondition", "integer", sizeof(int), + FMOffset(MpiReadRequestMsg, NotifyCondition)}, + {"CommType", "integer", sizeof(int), FMOffset(MpiReadRequestMsg, CommType)}, + {NULL, NULL, 0, 0}}; + +static FMStructDescRec MpiReadRequestStructs[] = { + {"MpiReadRequest", MpiReadRequestList, sizeof(struct _MpiReadRequestMsg), + NULL}, + {NULL, NULL, 0, NULL}}; + +static FMField MpiReadReplyList[] = { + {"TimeStep", "integer", sizeof(long), FMOffset(MpiReadReplyMsg, TimeStep)}, + {"StreamRS", "integer", sizeof(void *), + FMOffset(MpiReadReplyMsg, StreamRS)}, + {"DataLength", "integer", sizeof(size_t), + FMOffset(MpiReadReplyMsg, DataLength)}, + {"NotifyCondition", "integer", sizeof(int), + FMOffset(MpiReadReplyMsg, NotifyCondition)}, + {"MpiPortName", "string", sizeof(char *), + FMOffset(MpiReadReplyMsg, MpiPortName)}, + {"Data", "char[DataLength]", sizeof(char), FMOffset(MpiReadReplyMsg, Data)}, + {NULL, NULL, 0, 0}}; + +static FMStructDescRec MpiReadReplyStructs[] = { + {"MpiReadReply", MpiReadReplyList, sizeof(struct _MpiReadReplyMsg), NULL}, + {NULL, NULL, 0, NULL}}; + +static FMField MpiReaderContactList[] = { + {"ContactString", "string", sizeof(char *), + FMOffset(MpiReaderContactInfo, ContactString)}, + {"reader_ID", "integer", sizeof(void *), + FMOffset(MpiReaderContactInfo, StreamRS)}, + {NULL, NULL, 0, 0}}; + +static FMStructDescRec MpiReaderContactStructs[] = { + {"MpiReaderContactInfo", MpiReaderContactList, + sizeof(struct _MpiReaderContactInfo), NULL}, + {NULL, NULL, 0, NULL}}; + +static FMField MpiWriterContactList[] = { + {"ContactString", "string", sizeof(char *), + FMOffset(MpiWriterContactInfo, ContactString)}, + {"writer_ID", "integer", sizeof(void *), + FMOffset(MpiWriterContactInfo, StreamWS)}, + {"MpiComm", "integer", sizeof(int), + FMOffset(MpiWriterContactInfo, MpiComm)}, + {"PID", "integer", sizeof(int), FMOffset(MpiWriterContactInfo, PID)}, + {NULL, NULL, 0, 0}}; + +static FMStructDescRec MpiWriterContactStructs[] = { + {"MpiWriterContactInfo", MpiWriterContactList, + sizeof(struct _MpiWriterContactInfo), NULL}, + {NULL, NULL, 0, NULL}}; + +/*****Internal functions*****************************************************/ + +static void MpiReadReplyHandler(CManager cm, CMConnection conn, void *msg_v, + void *client_Data, attr_list attrs); + +/** + * Initialize MPI in the mode that it is required for MPI_DP to work. + * + * It can be called multiple times. + */ +static void MpiInitialize() +{ + int IsInitialized = 0; + int provided; + + MPI_Initialized(&IsInitialized); + if (!IsInitialized) + { + MPI_Init_thread(NULL, NULL, MPI_THREAD_MULTIPLE, &provided); + } + else + { + MPI_Query_thread(&provided); + } + + if (provided != MPI_THREAD_MULTIPLE) + { + fprintf(stderr, + "MPI init without MPI_THREAD_MULTIPLE (Externally " + "initialized:%s)\n", + IsInitialized ? "true" : "false"); + } +} + +/*****Public accessible functions********************************************/ + +/** + * InitReader. + * + * Called by the control plane collectively during the early stages of Open on + * the reader side. It should do whatever is necessary to initialize a new + * reader-side data plane. A pointer to per-reader-rank contact information + * should be placed in *ReaderContactInfoPtr. The structure of that + * information should be described by DPInterface.ReaderContactFormats. (This + * is an FFS format description. See + * https://www.cc.gatech.edu/systems/projects/FFS/.) + */ +static DP_RS_Stream MpiInitReader(CP_Services Svcs, void *CP_Stream, + void **ReaderContactInfoPtr, + struct _SstParams *Params, + attr_list WriterContact, SstStats Stats) +{ + pthread_once(&OnceMpiInitializer, MpiInitialize); + + MpiStreamRD Stream = calloc(sizeof(struct _MpiStreamRD), 1); + MpiReaderContactInfo Contact = + calloc(sizeof(struct _MpiReaderContactInfo), 1); + CManager cm = Svcs->getCManager(CP_Stream); + char *MpiContactString = calloc(sizeof(char), MPI_DP_CONTACT_STRING_LEN); + SMPI_Comm comm = Svcs->getMPIComm(CP_Stream); + CMFormat F; + + Stream->Stream.CP_Stream = CP_Stream; + Stream->Stream.PID = getpid(); + Stream->Link.Stats = Stats; + + SMPI_Comm_rank(comm, &Stream->Stream.Rank); + + snprintf(MpiContactString, MPI_DP_CONTACT_STRING_LEN, "Reader Rank %d", + Stream->Stream.Rank); + + /* + * add a handler for read reply messages + */ + Stream->ReadRequestFormat = CMregister_format(cm, MpiReadRequestStructs); + F = CMregister_format(cm, MpiReadReplyStructs); + CMregister_handler(F, MpiReadReplyHandler, Svcs); + + Contact->ContactString = MpiContactString; + Contact->StreamRS = Stream; + + *ReaderContactInfoPtr = Contact; + + Svcs->verbose(Stream->Stream.CP_Stream, DPTraceVerbose, + "MPI dataplane reader initialized, reader rank %d", + Stream->Stream.Rank); + + return Stream; +} + +static char *FetchTimeStep(TimeStepList timesteps, long timestep, long offset, + long length) +{ + TimeStepList ts = timesteps; + + pthread_rwlock_rdlock(&LockTS); + + // Find the requested timestep + while (ts != NULL && ts->TimeStep != timestep) + { + ts = ts->Next; + } + + if (ts == NULL) + { + fprintf(stderr, "Failed to read TimeStep %ld, not found\n", timestep); + return NULL; + } + + char *outboundBuffer = malloc(sizeof(char) * length); + memcpy(outboundBuffer, ts->Data->block + offset, length); + + pthread_rwlock_unlock(&LockTS); + + return outboundBuffer; +} + +static void MpiReadRequestHandler(CManager cm, CMConnection conn, void *msg_v, + void *client_Data, attr_list attrs) +{ + MpiReadRequestMsg ReadRequestMsg = (MpiReadRequestMsg)msg_v; + MpiStreamWSR StreamWSR = ReadRequestMsg->StreamWS; + MpiStreamWS StreamWS = StreamWSR->StreamWS; + CP_Services Svcs = (CP_Services)client_Data; + + Svcs->verbose(StreamWS->Stream.CP_Stream, DPTraceVerbose, + "MpiReadRequestHandler:" + "read request from reader=%d,ts=%d,off=%d,len=%d\n", + ReadRequestMsg->RequestingRank, ReadRequestMsg->TimeStep, + ReadRequestMsg->Offset, ReadRequestMsg->Length); + + PERFSTUBS_TIMER_START_FUNC(timer); + + char *outboundBuffer = NULL; + if (NULL == (outboundBuffer = FetchTimeStep( + StreamWS->TimeSteps, ReadRequestMsg->TimeStep, + ReadRequestMsg->Offset, ReadRequestMsg->Length))) + { + PERFSTUBS_TIMER_STOP_FUNC(timer); + return; + } + + struct _MpiReadReplyMsg ReadReplyMsg = { + .TimeStep = ReadRequestMsg->TimeStep, + .DataLength = ReadRequestMsg->Length, + .StreamRS = ReadRequestMsg->StreamRS, + .NotifyCondition = ReadRequestMsg->NotifyCondition, + .MpiPortName = StreamWSR->MpiPortName, + }; + + if (MPI_DP_LOCAL == ReadRequestMsg->CommType) + { + ReadReplyMsg.Data = outboundBuffer; + } + + Svcs->verbose( + StreamWS->Stream.CP_Stream, DPTraceVerbose, + "MpiReadRequestHandler: Replying reader=%d with MPI port name=%s\n", + ReadRequestMsg->RequestingRank, StreamWSR->MpiPortName); + + Svcs->sendToPeer(StreamWS->Stream.CP_Stream, StreamWSR->Link.PeerCohort, + ReadRequestMsg->RequestingRank, StreamWS->ReadReplyFormat, + &ReadReplyMsg); + + if (MPI_DP_REMOTE == ReadRequestMsg->CommType) + { + // Send the actual Data using MPI + MPI_Comm *comm = + &StreamWSR->ReaderContactInfo[ReadRequestMsg->RequestingRank] + .MpiComm; + MPI_Errhandler worldErrHandler; + MPI_Comm_get_errhandler(MPI_COMM_WORLD, &worldErrHandler); + MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); + int ret = MPI_Send(outboundBuffer, ReadRequestMsg->Length, MPI_CHAR, 0, + ReadRequestMsg->NotifyCondition, *comm); + MPI_Comm_set_errhandler(MPI_COMM_WORLD, worldErrHandler); + + if (ret != MPI_SUCCESS) + { + MPI_Comm_accept(StreamWSR->MpiPortName, MPI_INFO_NULL, 0, + MPI_COMM_SELF, comm); + Svcs->verbose( + StreamWS->Stream.CP_Stream, DPTraceVerbose, + "MpiReadRequestHandler: Accepted client, Link.CohortSize=%d\n", + StreamWSR->Link.CohortSize); + MPI_Send(outboundBuffer, ReadRequestMsg->Length, MPI_CHAR, 0, + ReadRequestMsg->NotifyCondition, *comm); + } + } + + free(outboundBuffer); + + PERFSTUBS_TIMER_STOP_FUNC(timer); +} + +typedef struct _MpiCompletionHandle +{ + int CMcondition; + CManager cm; + void *CPStream; + void *Buffer; + int Rank; + enum MPI_DP_COMM_TYPE CommType; +} * MpiCompletionHandle; + +/** + * This is invoked at the Reader side when a reply is ready to be read + * + */ +static void MpiReadReplyHandler(CManager cm, CMConnection conn, void *msg_v, + void *client_Data, attr_list attrs) +{ + PERFSTUBS_TIMER_START_FUNC(timer); + MpiReadReplyMsg ReadReplyMsg = (MpiReadReplyMsg)msg_v; + MpiStreamRD StreamRS = ReadReplyMsg->StreamRS; + CP_Services Svcs = (CP_Services)client_Data; + MpiCompletionHandle Handle = + CMCondition_get_client_data(cm, ReadReplyMsg->NotifyCondition); + + Svcs->verbose( + StreamRS->Stream.CP_Stream, DPTraceVerbose, + "MpiReadReplyHandler: Read recv from rank=%d,condition=%d,size=%d\n", + Handle->Rank, ReadReplyMsg->NotifyCondition, ReadReplyMsg->DataLength); + + if (MPI_DP_LOCAL == Handle->CommType) + { + memcpy(Handle->Buffer, ReadReplyMsg->Data, ReadReplyMsg->DataLength); + } + else + { + pthread_rwlock_rdlock(&LockRS); + MPI_Comm comm = StreamRS->WriterContactInfo[Handle->Rank].MpiComm; + pthread_rwlock_unlock(&LockRS); + + MPI_Errhandler worldErrHandler; + MPI_Comm_get_errhandler(MPI_COMM_WORLD, &worldErrHandler); + MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); + int ret = + MPI_Recv(Handle->Buffer, ReadReplyMsg->DataLength, MPI_CHAR, 0, + ReadReplyMsg->NotifyCondition, comm, MPI_STATUS_IGNORE); + MPI_Comm_set_errhandler(MPI_COMM_WORLD, worldErrHandler); + + if (ret != MPI_SUCCESS) + { + MPI_Comm_connect(ReadReplyMsg->MpiPortName, MPI_INFO_NULL, 0, + MPI_COMM_SELF, &comm); + + Svcs->verbose(StreamRS->Stream.CP_Stream, DPTraceVerbose, + "MpiReadReplyHandler: Connecting to MPI Server\n"); + MPI_Recv(Handle->Buffer, ReadReplyMsg->DataLength, MPI_CHAR, 0, + ReadReplyMsg->NotifyCondition, comm, MPI_STATUS_IGNORE); + } + + pthread_rwlock_wrlock(&LockRS); + StreamRS->WriterContactInfo[Handle->Rank].MpiComm = comm; + pthread_rwlock_unlock(&LockRS); + } + StreamRS->Link.Stats->DataBytesReceived += ReadReplyMsg->DataLength; + + /* + * Signal the condition to wake the reader if they are waiting. + */ + CMCondition_signal(cm, ReadReplyMsg->NotifyCondition); + PERFSTUBS_TIMER_STOP_FUNC(timer); +} + +/* + * + * InitWriter. Called by the control plane collectively during the early + * stages of Open on the writer side. It should do whatever is necessary to + * initialize a new writer-side data plane. This does *not* include creating + * contact information per se. That can be put off until InitWriterPerReader(). + * + */ +static DP_WS_Stream MpiInitWriter(CP_Services Svcs, void *CP_Stream, + struct _SstParams *Params, attr_list DPAttrs, + SstStats Stats) +{ + pthread_once(&OnceMpiInitializer, MpiInitialize); + + /* Make MutexReaders to be recursive */ + + MpiStreamWS Stream = calloc(sizeof(struct _MpiStreamWS), 1); + CManager cm = Svcs->getCManager(CP_Stream); + SMPI_Comm comm = Svcs->getMPIComm(CP_Stream); + CMFormat F; + + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&Stream->MutexReaders, &attr); + + SMPI_Comm_rank(comm, &Stream->Stream.Rank); + + Stream->Stream.CP_Stream = CP_Stream; + Stream->Stream.PID = getpid(); + + /* + * add a handler for read request messages + */ + F = CMregister_format(cm, MpiReadRequestStructs); + CMregister_handler(F, MpiReadRequestHandler, Svcs); + + /* + * register read reply message structure so we can send later + */ + Stream->ReadReplyFormat = CMregister_format(cm, MpiReadReplyStructs); + + return (void *)Stream; +} + +/** + * InitWriterPerReader. + * + * Called by the control plane collectively when accepting a new reader + * connection. It receives the per-rank reader contact information (as created + * on the connecting peer in InitReader) and should create its own + * per-writer-rank contact information and place it in *writerContactInfoPtr. + * The structure of that information should be described by + * DPInterface.WriterContactFormats. (This is an FFS format description. See + * https://www.cc.gatech.edu/systems/projects/FFS/.) + * + */ +static DP_WSR_Stream +MpiInitWriterPerReader(CP_Services Svcs, DP_WS_Stream WS_Stream_v, + int readerCohortSize, CP_PeerCohort PeerCohort, + void **providedReaderInfo_v, void **WriterContactInfoPtr) +{ + MpiStreamWS StreamWS = (MpiStreamWS)WS_Stream_v; + MpiStreamWSR StreamWSR = calloc(sizeof(struct _MpiStreamWSR), 1); + MpiWriterContactInfo ContactInfo; + SMPI_Comm comm = Svcs->getMPIComm(StreamWS->Stream.CP_Stream); + MpiReaderContactInfo *providedReaderInfo = + (MpiReaderContactInfo *)providedReaderInfo_v; + char *MpiContactString = calloc(sizeof(char), MPI_DP_CONTACT_STRING_LEN); + + int Rank; + SMPI_Comm_rank(comm, &Rank); + snprintf(MpiContactString, MPI_DP_CONTACT_STRING_LEN, + "Writer Rank %d, test contact", Rank); + + MPI_Open_port(MPI_INFO_NULL, StreamWSR->MpiPortName); + + StreamWSR->StreamWS = StreamWS; /* pointer to writer struct */ + StreamWSR->Link.PeerCohort = PeerCohort; + StreamWSR->Link.CohortSize = readerCohortSize; + + Svcs->verbose(StreamWS->Stream.CP_Stream, DPTraceVerbose, + "MPI dataplane WriterPerReader to be initialized\n"); + + /* + * make a copy of writer contact information (original will not be + * preserved) + */ + StreamWSR->ReaderContactInfo = + calloc(sizeof(struct _MpiReaderContactInfo), readerCohortSize); + for (int i = 0; i < readerCohortSize; i++) + { + StreamWSR->ReaderContactInfo[i].ContactString = + strdup(providedReaderInfo[i]->ContactString); + StreamWSR->ReaderContactInfo[i].StreamRS = + providedReaderInfo[i]->StreamRS; + StreamWSR->ReaderContactInfo[i].MpiComm = MPI_COMM_NULL; + Svcs->verbose( + StreamWS->Stream.CP_Stream, DPTraceVerbose, + "Received contact info \"%s\", RD_Stream %p for Reader Rank %d\n", + StreamWSR->ReaderContactInfo[i].ContactString, + StreamWSR->ReaderContactInfo[i].StreamRS, i); + } + + /* + * add this writer-side reader-specific stream to the parent writer stream + * structure + */ + pthread_mutex_lock(&StreamWS->MutexReaders); + StreamWS->Readers = realloc( + StreamWS->Readers, sizeof(*StreamWSR) * (StreamWS->ReaderCount + 1)); + StreamWS->Readers[StreamWS->ReaderCount] = StreamWSR; + StreamWS->ReaderCount++; + pthread_mutex_unlock(&StreamWS->MutexReaders); + + ContactInfo = calloc(sizeof(struct _MpiWriterContactInfo), 1); + ContactInfo->ContactString = MpiContactString; + ContactInfo->StreamWS = StreamWSR; + ContactInfo->PID = StreamWS->Stream.PID; + *WriterContactInfoPtr = ContactInfo; + + return StreamWSR; +} + +static void MpiProvideWriterDataToReader(CP_Services Svcs, + DP_RS_Stream RS_Stream_v, + int writerCohortSize, + CP_PeerCohort PeerCohort, + void **providedWriterInfo_v) +{ + MpiStreamRD StreamRS = (MpiStreamRD)RS_Stream_v; + MpiWriterContactInfo *providedWriterInfo = + (MpiWriterContactInfo *)providedWriterInfo_v; + + StreamRS->Link.PeerCohort = PeerCohort; + StreamRS->Link.CohortSize = writerCohortSize; + + /* + * make a copy of writer contact information (original will not be + * preserved) + */ + struct _MpiWriterContactInfo *tmp = + calloc(sizeof(struct _MpiWriterContactInfo), writerCohortSize); + for (int i = 0; i < writerCohortSize; i++) + { + tmp[i].ContactString = strdup(providedWriterInfo[i]->ContactString); + tmp[i].StreamWS = providedWriterInfo[i]->StreamWS; + tmp[i].MpiComm = MPI_COMM_NULL; + tmp[i].PID = providedWriterInfo[i]->PID; + + if (StreamRS->WriterContactInfo && + StreamRS->WriterContactInfo[i].MpiComm != MPI_COMM_NULL) + { + MPI_Comm_disconnect(&StreamRS->WriterContactInfo[i].MpiComm); + } + + Svcs->verbose(StreamRS->Stream.CP_Stream, DPTraceVerbose, + "Received contact info \"%s\", WS_stream %p for WSR Rank " + "%d\n", + tmp[i].ContactString, tmp[i].StreamWS, i); + } + + StreamRS->WriterContactInfo = tmp; +} + +/** + * ReadRemoteMemory. + * + * Called by the control plane on the reader side to request that timestep data + * from the writer side, identified by Rank, TimeStep, starting at a particular + * Offset and continuing for Length, be placed into a local Buffer. The + * DP_TimeStepInfo value will be the per-rank per-timestep information that was + * created during ProvideTimeStep by that writer rank for that timestep. + * This is an asyncronous request in that it need not be completed before this + * call returns. The void* return value will later be passed to a + * WaitForCompletion call and should represent a completion handle. + * + */ +static void *MpiReadRemoteMemory(CP_Services Svcs, DP_RS_Stream Stream_v, + int Rank, long TimeStep, size_t Offset, + size_t Length, void *Buffer, + void *DP_TimeStepInfo) +{ + /* DP_RS_Stream is the return from InitReader */ + MpiStreamRD Stream = (MpiStreamRD)Stream_v; + CManager cm = Svcs->getCManager(Stream->Stream.CP_Stream); + MpiCompletionHandle ret = calloc(sizeof(struct _MpiCompletionHandle), 1); + + pthread_rwlock_rdlock(&LockRS); + + ret->CMcondition = CMCondition_get(cm, NULL); + ret->CPStream = Stream->Stream.CP_Stream; + ret->cm = cm; + ret->Buffer = Buffer; + ret->Rank = Rank; + ret->CommType = (Stream->WriterContactInfo[Rank].PID == Stream->Stream.PID) + ? MPI_DP_LOCAL + : MPI_DP_REMOTE; + + /* + * set the completion handle as client Data on the condition so that + * handler has access to it. + */ + CMCondition_set_client_data(cm, ret->CMcondition, ret); + + Svcs->verbose( + Stream->Stream.CP_Stream, DPTraceVerbose, + "Reader (rank %d) requesting to read remote memory for TimeStep %d " + "from Rank %d, StreamWSR = %p, Offset=%d, Length=%d\n", + Stream->Stream.Rank, TimeStep, Rank, + Stream->WriterContactInfo[Rank].StreamWS, Offset, Length); + + /* send request to appropriate writer */ + struct _MpiReadRequestMsg ReadRequestMsg = { + .TimeStep = TimeStep, + .Offset = Offset, + .Length = Length, + .StreamWS = Stream->WriterContactInfo[Rank].StreamWS, + .StreamRS = Stream, + .RequestingRank = Stream->Stream.Rank, + .NotifyCondition = ret->CMcondition, + .CommType = ret->CommType}; + + pthread_rwlock_unlock(&LockRS); + + Svcs->sendToPeer(Stream->Stream.CP_Stream, Stream->Link.PeerCohort, Rank, + Stream->ReadRequestFormat, &ReadRequestMsg); + return ret; +} + +/** + * WaitForCompletion. + * + * Called by the control plane on the reader side with a Handle that is the + * return value of a prior ReadRemoteMemory call. This call should not return + * until that particular remote memory read is complete and the buffer is full. + * A zero return means that the read failed and will result in a (hopefully + * orderly) shutdown of the stream. + */ +static int MpiWaitForCompletion(CP_Services Svcs, void *Handle_v) +{ + MpiCompletionHandle Handle = (MpiCompletionHandle)Handle_v; + Svcs->verbose( + Handle->CPStream, DPTraceVerbose, + "Waiting for completion of memory read to rank %d, condition %d\n", + Handle->Rank, Handle->CMcondition); + + /* + * Wait for the CM condition to be signalled. If it has been already, + * this returns immediately. Copying the incoming data to the waiting + * buffer has been done by the reply handler. + */ + int Ret = CMCondition_wait(Handle->cm, Handle->CMcondition); + if (!Ret) + { + Svcs->verbose(Handle->CPStream, DPTraceVerbose, + "Remote memory read to rank %d with " + "condition %d has FAILED because of " + "writer failure\n", + Handle->Rank, Handle->CMcondition); + } + else + { + if (Handle->CMcondition != -1) + Svcs->verbose(Handle->CPStream, DPTraceVerbose, + "Remote memory read to rank %d with condition %d has " + "completed\n", + Handle->Rank, Handle->CMcondition); + } + free(Handle); + return Ret; +} + +/** + * ProvideTimeStep. + * + * Called by the control plane collectively on the writer side to "give" the + * data plane new data that is should then serve to the readers. DP must do + * everything necessary here to allow future service (until ReleaseTimeStep is + * called). The DP can create per-timestep per-rank identifying information + * that will be placed in the timestep metadata and provided to the reader + * during remote read requests. A pointer to this contact information should + * be placed in *TimeStepInfoPtr. This structure should be described by + * DPInterface.TimeStepInfoFormats. + * + */ +static void MpiProvideTimeStep(CP_Services Svcs, DP_WS_Stream Stream_v, + struct _SstData *Data, + struct _SstData *LocalMetadata, long TimeStep, + void **TimeStepInfoPtr) +{ + MpiStreamWS Stream = (MpiStreamWS)Stream_v; + TimeStepList Entry = calloc(sizeof(struct _TimeStepEntry), 1); + struct _MpiPerTimeStepInfo *Info = + calloc(sizeof(struct _MpiPerTimeStepInfo), 1); + + Info->CheckString = calloc(sizeof(char), MPI_DP_CONTACT_STRING_LEN); + snprintf(Info->CheckString, MPI_DP_CONTACT_STRING_LEN, + "Mpi info for timestep %ld from rank %d", TimeStep, + Stream->Stream.Rank); + + Entry->Data = malloc(sizeof(struct _SstData)); + memcpy(Entry->Data, Data, sizeof(struct _SstData)); + Entry->TimeStep = TimeStep; + Entry->DP_TimeStepInfo = Info; + + pthread_rwlock_wrlock(&LockTS); + Entry->Next = Stream->TimeSteps; + Stream->TimeSteps = Entry; + pthread_rwlock_unlock(&LockTS); + *TimeStepInfoPtr = Info; +} + +/** + * ReleaseTimeStep. + * + * Called by the control plane collectively on the writer side to tell the data + * plane that a particular timestep is no longer required and any resources + * devoted to serving it can be released. + */ +static void MpiReleaseTimeStep(CP_Services Svcs, DP_WS_Stream Stream_v, + long TimeStep) +{ + MpiStreamWS Stream = (MpiStreamWS)Stream_v; + TimeStepList List = Stream->TimeSteps; + + Svcs->verbose(Stream->Stream.CP_Stream, DPTraceVerbose, + "Releasing timestep %ld\n", TimeStep); + + pthread_rwlock_wrlock(&LockTS); + if (Stream->TimeSteps->TimeStep == TimeStep) + { + Stream->TimeSteps = List->Next; + free(List->Data); + free(List); + } + else + { + TimeStepList last = List; + List = List->Next; + while (List != NULL) + { + if (List->TimeStep == TimeStep) + { + last->Next = List->Next; + free(List->Data); + free(List); + pthread_rwlock_unlock(&LockTS); + return; + } + last = List; + List = List->Next; + } + /* + * Shouldn't ever get here because we should never release a + * timestep that we don't have. + */ + fprintf(stderr, "Failed to release TimeStep %ld, not found\n", + TimeStep); + } + pthread_rwlock_unlock(&LockTS); +} + +static int MpiGetPriority(CP_Services Svcs, void *CP_Stream, + struct _SstParams *Params) +{ +#if defined(MPICH) + // Only enabled when MPI_THREAD_MULTIPLE and using MPICH + int provided = 0; + pthread_once(&OnceMpiInitializer, MpiInitialize); + MPI_Query_thread(&provided); + if (provided == MPI_THREAD_MULTIPLE) + { + return 100; + } +#endif + return -100; +} + +static void MpiNotifyConnFailure(CP_Services Svcs, DP_RS_Stream Stream_v, + int FailedPeerRank) +{ + /* DP_RS_Stream is the return from InitReader */ + MpiStreamRD Stream = (MpiStreamRD)Stream_v; + Svcs->verbose(Stream->Stream.CP_Stream, DPTraceVerbose, + "received notification that writer peer " + "%d has failed, failing any pending " + "requests\n", + FailedPeerRank); +} + +static void MpiDestroyWriterPerReader(CP_Services Svcs, + DP_WSR_Stream WSR_Stream_v) +{ + MpiStreamWSR StreamWSR = (MpiStreamWSR)WSR_Stream_v; + MpiStreamWS StreamWS = StreamWSR->StreamWS; + + char MpiPortName[MPI_MAX_PORT_NAME] = {0}; + const int CohortSize = StreamWSR->Link.CohortSize; + MPI_Comm *Comms_to_disconnect = calloc(sizeof(MPI_Comm), CohortSize); + + pthread_mutex_lock(&StreamWS->MutexReaders); + strncpy(MpiPortName, StreamWSR->MpiPortName, MPI_MAX_PORT_NAME); + + for (int i = 0; i < CohortSize; i++) + { + Comms_to_disconnect[i] = StreamWSR->ReaderContactInfo[i].MpiComm; + if (StreamWSR->ReaderContactInfo[i].ContactString) + { + free(StreamWSR->ReaderContactInfo[i].ContactString); + } + } + + if (StreamWSR->ReaderContactInfo) + { + free(StreamWSR->ReaderContactInfo); + } + + StreamWSR->Link.CohortSize = 0; + + for (int i = 0; i < StreamWS->ReaderCount; i++) + { + if (StreamWS->Readers[i] == StreamWSR) + { + StreamWS->Readers[i] = StreamWS->Readers[StreamWS->ReaderCount - 1]; + + StreamWS->Readers = + realloc(StreamWS->Readers, + sizeof(*StreamWSR) * (StreamWS->ReaderCount - 1)); + StreamWS->ReaderCount--; + break; + } + } + + free(StreamWSR); + pthread_mutex_unlock(&StreamWS->MutexReaders); + + // MPI routines must be called outsie of a critical region + for (int i = 0; i < CohortSize; i++) + { + if (Comms_to_disconnect[i] != MPI_COMM_NULL) + { + MPI_Comm_disconnect(&Comms_to_disconnect[i]); + } + } + free(Comms_to_disconnect); + + MPI_Close_port(MpiPortName); +} + +static void MpiDestroyWriter(CP_Services Svcs, DP_WS_Stream WS_Stream_v) +{ + MpiStreamWS StreamWS = (MpiStreamWS)WS_Stream_v; + + pthread_mutex_lock(&StreamWS->MutexReaders); + while (StreamWS->ReaderCount) + { + MpiDestroyWriterPerReader(Svcs, + StreamWS->Readers[StreamWS->ReaderCount - 1]); + } + + pthread_mutex_t *mutex_to_delete = &StreamWS->MutexReaders; + free(StreamWS->Readers); + free(StreamWS); + pthread_mutex_unlock(mutex_to_delete); +} + +static void MpiDestroyReader(CP_Services Svcs, DP_RS_Stream RS_Stream_v) +{ + MpiStreamRD StreamRS = (MpiStreamRD)RS_Stream_v; + const int CohortSize = StreamRS->Link.CohortSize; + MPI_Comm *MpiComms = calloc(sizeof(MPI_Comm), CohortSize); + + pthread_rwlock_wrlock(&LockRS); + + for (int i = 0; i < CohortSize; i++) + { + MpiComms[i] = StreamRS->WriterContactInfo[i].MpiComm; + free(StreamRS->WriterContactInfo[i].ContactString); + } + free(StreamRS->WriterContactInfo); + free(StreamRS); + + pthread_rwlock_unlock(&LockRS); + + for (int i = 0; i < CohortSize; i++) + { + if (MpiComms[i] != MPI_COMM_NULL) + { + MPI_Comm_disconnect(&MpiComms[i]); + } + } + free(MpiComms); +} + +extern CP_DP_Interface LoadMpiDP() +{ + static struct _CP_DP_Interface mpiDPInterface = { + .ReaderContactFormats = MpiReaderContactStructs, + .WriterContactFormats = MpiWriterContactStructs, + .initReader = MpiInitReader, + .initWriter = MpiInitWriter, + .initWriterPerReader = MpiInitWriterPerReader, + .provideWriterDataToReader = MpiProvideWriterDataToReader, + .readRemoteMemory = MpiReadRemoteMemory, + .waitForCompletion = MpiWaitForCompletion, + .provideTimestep = MpiProvideTimeStep, + .releaseTimestep = MpiReleaseTimeStep, + .getPriority = MpiGetPriority, + .destroyReader = MpiDestroyReader, + .destroyWriter = MpiDestroyWriter, + .destroyWriterPerReader = MpiDestroyWriterPerReader, + .notifyConnFailure = MpiNotifyConnFailure, + }; + + return &mpiDPInterface; +} diff --git a/source/utils/adios_iotest/adios_iotest.cpp b/source/utils/adios_iotest/adios_iotest.cpp index 8bbed46b71..27a743fcff 100644 --- a/source/utils/adios_iotest/adios_iotest.cpp +++ b/source/utils/adios_iotest/adios_iotest.cpp @@ -18,7 +18,15 @@ int main(int argc, char *argv[]) { - MPI_Init(&argc, &argv); + int provided; + int threadSupportLevel = MPI_THREAD_SINGLE; + + if (std::string(argv[1]) == "SST") + { + threadSupportLevel = MPI_THREAD_MULTIPLE; + } + + MPI_Init_thread(&argc, &argv, threadSupportLevel, &provided); Settings settings; diff --git a/source/utils/adios_reorganize/main.cpp b/source/utils/adios_reorganize/main.cpp index 79c4aadf2c..b6369be5f2 100644 --- a/source/utils/adios_reorganize/main.cpp +++ b/source/utils/adios_reorganize/main.cpp @@ -20,7 +20,8 @@ int main(int argc, char *argv[]) { #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); #endif try diff --git a/testing/adios2/bindings/C/TestBPAvailableVariablesAttribites.cpp b/testing/adios2/bindings/C/TestBPAvailableVariablesAttribites.cpp index 357ed70f2b..0c5cc4b0d2 100644 --- a/testing/adios2/bindings/C/TestBPAvailableVariablesAttribites.cpp +++ b/testing/adios2/bindings/C/TestBPAvailableVariablesAttribites.cpp @@ -268,7 +268,8 @@ TEST_F(BPAvailableVariablesAttributes, AvailableVariablesAttributes) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/bindings/C/TestBPWriteAggregateReadLocal.cpp b/testing/adios2/bindings/C/TestBPWriteAggregateReadLocal.cpp index 790ea42580..9bdfa6c6cb 100644 --- a/testing/adios2/bindings/C/TestBPWriteAggregateReadLocal.cpp +++ b/testing/adios2/bindings/C/TestBPWriteAggregateReadLocal.cpp @@ -368,7 +368,8 @@ INSTANTIATE_TEST_SUITE_P(Substreams, BPWriteAggregateReadLocalTest, int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/bindings/C/TestBPWriteReadMultiblock.cpp b/testing/adios2/bindings/C/TestBPWriteReadMultiblock.cpp index 339d6b8711..761b3b9651 100644 --- a/testing/adios2/bindings/C/TestBPWriteReadMultiblock.cpp +++ b/testing/adios2/bindings/C/TestBPWriteReadMultiblock.cpp @@ -297,7 +297,8 @@ TEST_F(BPWriteReadMultiblockCC, ZeroSizeBlocks) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/bindings/C/TestBPWriteTypes.cpp b/testing/adios2/bindings/C/TestBPWriteTypes.cpp index f2911360f5..f29e4932ac 100644 --- a/testing/adios2/bindings/C/TestBPWriteTypes.cpp +++ b/testing/adios2/bindings/C/TestBPWriteTypes.cpp @@ -519,7 +519,8 @@ TEST_F(ADIOS2_C_API_IO, ReturnedStrings) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/bindings/C/TestNullWriteRead.cpp b/testing/adios2/bindings/C/TestNullWriteRead.cpp index f5800c8f33..56ab2bf457 100644 --- a/testing/adios2/bindings/C/TestNullWriteRead.cpp +++ b/testing/adios2/bindings/C/TestNullWriteRead.cpp @@ -115,7 +115,8 @@ TEST_F(NullWriteReadTests_C_API, NullWriteRead1D8) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/bindings/fortran/TestAdios2BindingsFortranIO.F90 b/testing/adios2/bindings/fortran/TestAdios2BindingsFortranIO.F90 index 17d69d4fda..2d0fa4b0e0 100644 --- a/testing/adios2/bindings/fortran/TestAdios2BindingsFortranIO.F90 +++ b/testing/adios2/bindings/fortran/TestAdios2BindingsFortranIO.F90 @@ -151,7 +151,8 @@ program main external testing_adios_io_engine external testing_adios_io_engine_default - call MPI_Init(ierr) + INTEGER provided + call MPI_Init_thread(MPI_THREAD_MULTIPLE, provided, ierr) call testing_adios_io_engine() call testing_adios_io_engine_default() diff --git a/testing/adios2/bindings/fortran/TestBPReadGlobalsByName.F90 b/testing/adios2/bindings/fortran/TestBPReadGlobalsByName.F90 index 7703096f78..58d8f5d6fe 100644 --- a/testing/adios2/bindings/fortran/TestBPReadGlobalsByName.F90 +++ b/testing/adios2/bindings/fortran/TestBPReadGlobalsByName.F90 @@ -14,7 +14,8 @@ program TestBPReadGlobalsByName type(adios2_engine):: writer, reader ! Launch MPI - call MPI_Init(ierr) + INTEGER provided + call MPI_Init_thread(MPI_THREAD_MULTIPLE, provided, ierr) call MPI_Comm_rank(MPI_COMM_WORLD, irank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, isize, ierr) diff --git a/testing/adios2/bindings/fortran/TestBPWriteReadAttributes.F90 b/testing/adios2/bindings/fortran/TestBPWriteReadAttributes.F90 index 7499017818..ae89d06ac7 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteReadAttributes.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteReadAttributes.F90 @@ -32,7 +32,8 @@ program TestBPWriteAttributes ! Launch MPI - call MPI_Init(ierr) + INTEGER provided + call MPI_Init_thread(MPI_THREAD_MULTIPLE, provided, ierr) ! Create adios handler passing the communicator, debug mode and error flag call adios2_init(adios, MPI_COMM_WORLD, ierr) diff --git a/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 b/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 index b50dad7d29..18ad4c6954 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteTypes.F90 @@ -31,7 +31,8 @@ program TestBPWriteTypes #if ADIOS2_USE_MPI ! Launch MPI - call MPI_Init(ierr) + INTEGER provided + call MPI_Init_thread(MPI_THREAD_MULTIPLE, provided, ierr) call MPI_Comm_rank(MPI_COMM_WORLD, irank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, isize, ierr) #else diff --git a/testing/adios2/bindings/fortran/TestBPWriteTypesByName.F90 b/testing/adios2/bindings/fortran/TestBPWriteTypesByName.F90 index f3e3b4190d..63e49a990a 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteTypesByName.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteTypesByName.F90 @@ -17,7 +17,8 @@ program TestBPWriteTypes integer(kind=8), dimension(:), allocatable :: shape_in ! Launch MPI - call MPI_Init(ierr) + INTEGER provided + call MPI_Init_thread(MPI_THREAD_MULTIPLE, provided, ierr) call MPI_Comm_rank(MPI_COMM_WORLD, irank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, isize, ierr) diff --git a/testing/adios2/bindings/fortran/TestBPWriteTypesLocal.F90 b/testing/adios2/bindings/fortran/TestBPWriteTypesLocal.F90 index aaa599be53..1309352506 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteTypesLocal.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteTypesLocal.F90 @@ -27,7 +27,8 @@ program TestBPWriteTypes ! Program starts ! Launch MPI - call MPI_Init(ierr) + INTEGER provided + call MPI_Init_thread(MPI_THREAD_MULTIPLE, provided, ierr) call MPI_Comm_rank(MPI_COMM_WORLD, irank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, isize, ierr) diff --git a/testing/adios2/bindings/fortran/TestBPWriteVariableAttributes.F90 b/testing/adios2/bindings/fortran/TestBPWriteVariableAttributes.F90 index a8604295e5..c7a63e6c16 100644 --- a/testing/adios2/bindings/fortran/TestBPWriteVariableAttributes.F90 +++ b/testing/adios2/bindings/fortran/TestBPWriteVariableAttributes.F90 @@ -15,7 +15,8 @@ program TestBPWriteVariableAttributes integer :: ierr, i ! Launch MPI - call MPI_Init(ierr) + INTEGER provided + call MPI_Init_thread(MPI_THREAD_MULTIPLE, provided, ierr) ! Create adios handler passing the communicator, debug mode and error flag call adios2_init(adios, MPI_COMM_WORLD, ierr) diff --git a/testing/adios2/bindings/fortran/TestF2C_BPReadFBlocks.cpp b/testing/adios2/bindings/fortran/TestF2C_BPReadFBlocks.cpp index 8caffacb2f..559e4d3952 100644 --- a/testing/adios2/bindings/fortran/TestF2C_BPReadFBlocks.cpp +++ b/testing/adios2/bindings/fortran/TestF2C_BPReadFBlocks.cpp @@ -126,7 +126,8 @@ TEST_F(BPReadFBlocks, FHeatMap3D) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/bindings/fortran/TestNullEngine.F90 b/testing/adios2/bindings/fortran/TestNullEngine.F90 index 1599f88e14..0c79e956f1 100644 --- a/testing/adios2/bindings/fortran/TestNullEngine.F90 +++ b/testing/adios2/bindings/fortran/TestNullEngine.F90 @@ -17,7 +17,8 @@ program TestNullEngine ! Program starts ! Launch MPI - call MPI_Init(ierr) + INTEGER provided + call MPI_Init_thread(MPI_THREAD_MULTIPLE, provided, ierr) call MPI_Comm_rank(MPI_COMM_WORLD, irank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, isize, ierr) diff --git a/testing/adios2/bindings/fortran/TestRemove.F90 b/testing/adios2/bindings/fortran/TestRemove.F90 index 7059c599dd..b49a2b9506 100644 --- a/testing/adios2/bindings/fortran/TestRemove.F90 +++ b/testing/adios2/bindings/fortran/TestRemove.F90 @@ -17,7 +17,8 @@ program TestRemove #if ADIOS2_USE_MPI ! Launch MPI - call MPI_Init(ierr) + INTEGER provided + call MPI_Init_thread(MPI_THREAD_MULTIPLE, provided, ierr) call MPI_Comm_rank(MPI_COMM_WORLD, irank, ierr) call MPI_Comm_size(MPI_COMM_WORLD, isize, ierr) #else diff --git a/testing/adios2/engine/bp/TestBPBufferSize.cpp b/testing/adios2/engine/bp/TestBPBufferSize.cpp index 03060da0b8..0bec2c87e3 100644 --- a/testing/adios2/engine/bp/TestBPBufferSize.cpp +++ b/testing/adios2/engine/bp/TestBPBufferSize.cpp @@ -266,7 +266,8 @@ TEST_F(BPBufferSizeTest, SyncDeferredIdenticalUsage) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPChangingShape.cpp b/testing/adios2/engine/bp/TestBPChangingShape.cpp index daae74e86e..44615d0020 100644 --- a/testing/adios2/engine/bp/TestBPChangingShape.cpp +++ b/testing/adios2/engine/bp/TestBPChangingShape.cpp @@ -364,7 +364,8 @@ TEST_F(BPChangingShape, MultiBlock) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPDirectIO.cpp b/testing/adios2/engine/bp/TestBPDirectIO.cpp index 3ce4eefa16..2289b599fe 100644 --- a/testing/adios2/engine/bp/TestBPDirectIO.cpp +++ b/testing/adios2/engine/bp/TestBPDirectIO.cpp @@ -122,7 +122,8 @@ TEST_F(ADIOSReadDirectIOTest, BufferResize) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif ::testing::InitGoogleTest(&argc, argv); diff --git a/testing/adios2/engine/bp/TestBPFStreamWriteReadHighLevelAPI.cpp b/testing/adios2/engine/bp/TestBPFStreamWriteReadHighLevelAPI.cpp index 9b5ed33efa..6c007a30f2 100644 --- a/testing/adios2/engine/bp/TestBPFStreamWriteReadHighLevelAPI.cpp +++ b/testing/adios2/engine/bp/TestBPFStreamWriteReadHighLevelAPI.cpp @@ -722,7 +722,8 @@ TEST_F(StreamWriteReadHighLevelAPI, DoubleOpenException) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPInquireDefine.cpp b/testing/adios2/engine/bp/TestBPInquireDefine.cpp index f3f21921de..1e2af856d5 100644 --- a/testing/adios2/engine/bp/TestBPInquireDefine.cpp +++ b/testing/adios2/engine/bp/TestBPInquireDefine.cpp @@ -224,7 +224,8 @@ TEST_F(ADIOSInquireDefineTest, Read) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif ::testing::InitGoogleTest(&argc, argv); int result = RUN_ALL_TESTS(); diff --git a/testing/adios2/engine/bp/TestBPInquireVariableException.cpp b/testing/adios2/engine/bp/TestBPInquireVariableException.cpp index 3dca82bbb0..65a054f9f7 100644 --- a/testing/adios2/engine/bp/TestBPInquireVariableException.cpp +++ b/testing/adios2/engine/bp/TestBPInquireVariableException.cpp @@ -75,7 +75,8 @@ TEST_F(ADIOSInquireVariableException, Read) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif ::testing::InitGoogleTest(&argc, argv); int result = RUN_ALL_TESTS(); diff --git a/testing/adios2/engine/bp/TestBPLargeMetadata.cpp b/testing/adios2/engine/bp/TestBPLargeMetadata.cpp index 785ccb1781..ec80577a22 100644 --- a/testing/adios2/engine/bp/TestBPLargeMetadata.cpp +++ b/testing/adios2/engine/bp/TestBPLargeMetadata.cpp @@ -159,7 +159,8 @@ TEST_F(BPLargeMetadata, ManyLongStrings) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPNoXMLRecovery.cpp b/testing/adios2/engine/bp/TestBPNoXMLRecovery.cpp index f29d5ef87d..b5407747cc 100644 --- a/testing/adios2/engine/bp/TestBPNoXMLRecovery.cpp +++ b/testing/adios2/engine/bp/TestBPNoXMLRecovery.cpp @@ -10,7 +10,8 @@ int main(int argc, char *argv[]) { int rank = 0; #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &rank); #endif adios2::ADIOS ad; diff --git a/testing/adios2/engine/bp/TestBPParameterSelectSteps.cpp b/testing/adios2/engine/bp/TestBPParameterSelectSteps.cpp index 3e88c8978a..a97dcf5371 100644 --- a/testing/adios2/engine/bp/TestBPParameterSelectSteps.cpp +++ b/testing/adios2/engine/bp/TestBPParameterSelectSteps.cpp @@ -300,7 +300,8 @@ INSTANTIATE_TEST_SUITE_P(BPParameterSelectSteps, BPParameterSelectStepsP, int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPSelectSteps.cpp b/testing/adios2/engine/bp/TestBPSelectSteps.cpp index 89c66d7fea..759d09e428 100644 --- a/testing/adios2/engine/bp/TestBPSelectSteps.cpp +++ b/testing/adios2/engine/bp/TestBPSelectSteps.cpp @@ -316,7 +316,8 @@ TEST_F(ADIOSReadSelectionStepsTest, Read) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif ::testing::InitGoogleTest(&argc, argv); int result = RUN_ALL_TESTS(); diff --git a/testing/adios2/engine/bp/TestBPStepsFileGlobalArray.cpp b/testing/adios2/engine/bp/TestBPStepsFileGlobalArray.cpp index f5ca3d1355..f752810b5f 100644 --- a/testing/adios2/engine/bp/TestBPStepsFileGlobalArray.cpp +++ b/testing/adios2/engine/bp/TestBPStepsFileGlobalArray.cpp @@ -959,7 +959,8 @@ INSTANTIATE_TEST_SUITE_P( int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPStepsFileLocalArray.cpp b/testing/adios2/engine/bp/TestBPStepsFileLocalArray.cpp index 43e95ba5a1..54da200c72 100644 --- a/testing/adios2/engine/bp/TestBPStepsFileLocalArray.cpp +++ b/testing/adios2/engine/bp/TestBPStepsFileLocalArray.cpp @@ -602,7 +602,8 @@ INSTANTIATE_TEST_SUITE_P( int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPStepsInSituGlobalArray.cpp b/testing/adios2/engine/bp/TestBPStepsInSituGlobalArray.cpp index 865618452b..14da8a6487 100644 --- a/testing/adios2/engine/bp/TestBPStepsInSituGlobalArray.cpp +++ b/testing/adios2/engine/bp/TestBPStepsInSituGlobalArray.cpp @@ -686,7 +686,8 @@ INSTANTIATE_TEST_SUITE_P( int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPStepsInSituLocalArray.cpp b/testing/adios2/engine/bp/TestBPStepsInSituLocalArray.cpp index 2ee9c9148c..b72a720a7f 100644 --- a/testing/adios2/engine/bp/TestBPStepsInSituLocalArray.cpp +++ b/testing/adios2/engine/bp/TestBPStepsInSituLocalArray.cpp @@ -537,7 +537,8 @@ INSTANTIATE_TEST_SUITE_P( int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPTimeAggregation.cpp b/testing/adios2/engine/bp/TestBPTimeAggregation.cpp index 9a77284d72..44220e84e5 100644 --- a/testing/adios2/engine/bp/TestBPTimeAggregation.cpp +++ b/testing/adios2/engine/bp/TestBPTimeAggregation.cpp @@ -714,7 +714,8 @@ INSTANTIATE_TEST_SUITE_P(FlushStepsCount, BPTestTimeAggregation, int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteAggregateRead.cpp b/testing/adios2/engine/bp/TestBPWriteAggregateRead.cpp index 3429006b25..832511e537 100644 --- a/testing/adios2/engine/bp/TestBPWriteAggregateRead.cpp +++ b/testing/adios2/engine/bp/TestBPWriteAggregateRead.cpp @@ -1002,7 +1002,8 @@ INSTANTIATE_TEST_SUITE_P(Substreams, BPWriteAggregateReadTest, int main(int argc, char **argv) { - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); int result; ::testing::InitGoogleTest(&argc, argv); diff --git a/testing/adios2/engine/bp/TestBPWriteAppendReadADIOS2.cpp b/testing/adios2/engine/bp/TestBPWriteAppendReadADIOS2.cpp index fcf1dedde7..e4d45c5cc9 100644 --- a/testing/adios2/engine/bp/TestBPWriteAppendReadADIOS2.cpp +++ b/testing/adios2/engine/bp/TestBPWriteAppendReadADIOS2.cpp @@ -912,7 +912,8 @@ TEST_F(BPWriteAppendReadTestADIOS2, ADIOS2BPWriteAppendReadVaryingAggregation) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteFlushRead.cpp b/testing/adios2/engine/bp/TestBPWriteFlushRead.cpp index 994b9644e2..037ae79161 100644 --- a/testing/adios2/engine/bp/TestBPWriteFlushRead.cpp +++ b/testing/adios2/engine/bp/TestBPWriteFlushRead.cpp @@ -1441,7 +1441,8 @@ TEST_F(BPWriteFlushRead, ADIOS2BPWrite1D2Dfstream) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp b/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp index ef53cefe53..507a14caa4 100644 --- a/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp +++ b/testing/adios2/engine/bp/TestBPWriteMemorySelectionRead.cpp @@ -950,7 +950,8 @@ INSTANTIATE_TEST_SUITE_P(ghostCells, BPWriteMemSelReadVector, int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteMultiblockRead.cpp b/testing/adios2/engine/bp/TestBPWriteMultiblockRead.cpp index 867bde2f7b..89c6df9872 100644 --- a/testing/adios2/engine/bp/TestBPWriteMultiblockRead.cpp +++ b/testing/adios2/engine/bp/TestBPWriteMultiblockRead.cpp @@ -1413,7 +1413,8 @@ TEST_F(BPWriteMultiblockReadTest, ADIOS2BPWriteRead1D8ZeroBlock) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteNull.cpp b/testing/adios2/engine/bp/TestBPWriteNull.cpp index c73ed1575b..399c201672 100644 --- a/testing/adios2/engine/bp/TestBPWriteNull.cpp +++ b/testing/adios2/engine/bp/TestBPWriteNull.cpp @@ -618,7 +618,8 @@ TEST_F(BPWriteNullTest, BPWrite2D4x2_MultiSteps) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteProfilingJSON.cpp b/testing/adios2/engine/bp/TestBPWriteProfilingJSON.cpp index 389fb1d06a..f19db6f924 100644 --- a/testing/adios2/engine/bp/TestBPWriteProfilingJSON.cpp +++ b/testing/adios2/engine/bp/TestBPWriteProfilingJSON.cpp @@ -319,7 +319,8 @@ TEST_F(BPWriteProfilingJSONTest, ADIOS2BPWriteProfilingJSON_Off) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteReadADIOS2.cpp b/testing/adios2/engine/bp/TestBPWriteReadADIOS2.cpp index 43e16d0eba..80212b5196 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadADIOS2.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadADIOS2.cpp @@ -2300,7 +2300,8 @@ TEST_F(BPWriteReadTestADIOS2, GetDeferredWithoutEndStep) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteReadADIOS2fstream.cpp b/testing/adios2/engine/bp/TestBPWriteReadADIOS2fstream.cpp index 7b18dd55e0..5852665e9d 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadADIOS2fstream.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadADIOS2fstream.cpp @@ -1651,7 +1651,8 @@ TEST_F(BPWriteReadTestADIOS2fstream, OpenEngineTwice) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteReadADIOS2stdio.cpp b/testing/adios2/engine/bp/TestBPWriteReadADIOS2stdio.cpp index 0bacc47865..9d5201058b 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadADIOS2stdio.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadADIOS2stdio.cpp @@ -1650,7 +1650,8 @@ TEST_F(BPWriteReadTestADIOS2stdio, OpenEngineTwice) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2.cpp b/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2.cpp index d9d41f6b29..6fc647ffee 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2.cpp @@ -1019,7 +1019,8 @@ TEST_F(BPWriteReadAsStreamTestADIOS2, ReaderWriterDefineVariable) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2_Threads.cpp b/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2_Threads.cpp index 397fac4600..c2dbfa4fff 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2_Threads.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadAsStreamADIOS2_Threads.cpp @@ -1592,7 +1592,8 @@ TEST_F(BPWriteReadAsStreamTestADIOS2_Threads, int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteReadAttributes.cpp b/testing/adios2/engine/bp/TestBPWriteReadAttributes.cpp index 7e8ceb73ff..b39ff6177b 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadAttributes.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadAttributes.cpp @@ -1233,7 +1233,8 @@ TEST_F(BPWriteReadAttributes, WriteReadStreamModifiable) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteReadAttributesMultirank.cpp b/testing/adios2/engine/bp/TestBPWriteReadAttributesMultirank.cpp index 13824f3ab5..8a3b2f91c4 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadAttributesMultirank.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadAttributesMultirank.cpp @@ -122,7 +122,8 @@ TEST_F(BPWriteReadAttributeTestMultirank, ADIOS2BPWriteReadArrayTypes) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteReadBlockInfo.cpp b/testing/adios2/engine/bp/TestBPWriteReadBlockInfo.cpp index 387086fc73..0a5459637e 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadBlockInfo.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadBlockInfo.cpp @@ -923,7 +923,8 @@ TEST_F(BPWriteReadBlockInfo, BPWriteReadBlockInfo1D8_C) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteReadCuda.cpp b/testing/adios2/engine/bp/TestBPWriteReadCuda.cpp index 1d278acd5c..b00776cf91 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadCuda.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadCuda.cpp @@ -170,7 +170,8 @@ INSTANTIATE_TEST_SUITE_P(Rate, BPWRCUDA, ::testing::Values("deferred", "sync")); int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteReadLocalVariables.cpp b/testing/adios2/engine/bp/TestBPWriteReadLocalVariables.cpp index eb50719715..f0d0013252 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadLocalVariables.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadLocalVariables.cpp @@ -1916,7 +1916,8 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal2DChangeCount) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteReadLocalVariablesSel.cpp b/testing/adios2/engine/bp/TestBPWriteReadLocalVariablesSel.cpp index 83281ab37c..2cd4aa84cd 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadLocalVariablesSel.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadLocalVariablesSel.cpp @@ -1833,7 +1833,8 @@ TEST_F(BPWriteReadLocalVariablesSel, BPWriteReadLocal1DAllStepsSel) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteReadLocalVariablesSelHighLevel.cpp b/testing/adios2/engine/bp/TestBPWriteReadLocalVariablesSelHighLevel.cpp index c9ca471089..dcf62024ea 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadLocalVariablesSelHighLevel.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadLocalVariablesSelHighLevel.cpp @@ -594,7 +594,8 @@ TEST_F(BPWriteReadLocalVariablesSelHighLevel, BPWriteReadLocal1DAllStepsSel) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteReadMultiblock.cpp b/testing/adios2/engine/bp/TestBPWriteReadMultiblock.cpp index bf92f732ac..e5f7a0729a 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadMultiblock.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadMultiblock.cpp @@ -2235,7 +2235,8 @@ TEST_F(BPWriteReadMultiblockTest, MultiblockPerformDataWrite) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteReadVariableSpan.cpp b/testing/adios2/engine/bp/TestBPWriteReadVariableSpan.cpp index 3d8ba56d73..ace640eaf6 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadVariableSpan.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadVariableSpan.cpp @@ -1596,7 +1596,8 @@ TEST_F(BPWriteReadSpan, BPWriteSpanOperatorException) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/TestBPWriteReadVector.cpp b/testing/adios2/engine/bp/TestBPWriteReadVector.cpp index a0816204ed..73d93ddc7e 100644 --- a/testing/adios2/engine/bp/TestBPWriteReadVector.cpp +++ b/testing/adios2/engine/bp/TestBPWriteReadVector.cpp @@ -1289,7 +1289,8 @@ TEST_F(BPWriteReadVector, ADIOS2BPWriteReadVector2D4x2_MultiSteps) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp index c2fe7b12ee..2fd620ccc5 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadBZIP2.cpp @@ -893,7 +893,8 @@ INSTANTIATE_TEST_SUITE_P( int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc.cpp index 2b49ac40e8..43c5a6ce53 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadBlosc.cpp @@ -946,7 +946,8 @@ INSTANTIATE_TEST_SUITE_P( int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadLocalVariables.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadLocalVariables.cpp index be2b0ea666..fcc8bd2557 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadLocalVariables.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadLocalVariables.cpp @@ -1602,7 +1602,8 @@ TEST_F(BPWriteReadLocalVariables, ADIOS2BPWriteReadLocal1DBlockInfo) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadMGARD.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadMGARD.cpp index db495dd0ff..d3d755c56d 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadMGARD.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadMGARD.cpp @@ -1006,7 +1006,8 @@ INSTANTIATE_TEST_SUITE_P(MGARDAccuracy, BPWriteReadMGARD, int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadPNG.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadPNG.cpp index ec1a81bc76..b65ea1a419 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadPNG.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadPNG.cpp @@ -449,7 +449,8 @@ INSTANTIATE_TEST_SUITE_P( int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadSZ.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadSZ.cpp index e583560ed5..97f0e042b0 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadSZ.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadSZ.cpp @@ -1043,7 +1043,8 @@ INSTANTIATE_TEST_SUITE_P(SZAccuracy, BPWriteReadSZ, int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadSzComplex.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadSzComplex.cpp index f531f7bdd2..587e17f889 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadSzComplex.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadSzComplex.cpp @@ -366,7 +366,8 @@ TEST_F(BPEngineTest, SzComplex) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; ::testing::InitGoogleTest(&argc, argv); diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadZfp.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadZfp.cpp index e7e81dcc02..377e338ef5 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadZfp.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadZfp.cpp @@ -975,7 +975,8 @@ INSTANTIATE_TEST_SUITE_P(ZFPRate, BPWRZFP, ::testing::Values("8", "9", "10")); int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpComplex.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpComplex.cpp index ffb1aad7c7..b3ae43dde2 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpComplex.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpComplex.cpp @@ -366,7 +366,8 @@ TEST_F(BPEngineTest, ZfpComplex) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; ::testing::InitGoogleTest(&argc, argv); diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpConfig.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpConfig.cpp index 1f470a81cd..8987b5a68f 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpConfig.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpConfig.cpp @@ -910,7 +910,8 @@ INSTANTIATE_TEST_SUITE_P( int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpCuda.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpCuda.cpp index ba404f6a07..8b207d4272 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpCuda.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpCuda.cpp @@ -156,7 +156,8 @@ INSTANTIATE_TEST_SUITE_P(ZFPRate, BPWRZFPCUDA, int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpHighLevelAPI.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpHighLevelAPI.cpp index c5ce9d7858..5a8dd482aa 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpHighLevelAPI.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpHighLevelAPI.cpp @@ -728,7 +728,8 @@ INSTANTIATE_TEST_SUITE_P(ZfpRate, BPWriteReadZfpHighLevelAPI, int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpRemoveOperations.cpp b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpRemoveOperations.cpp index 6bd936b6f2..7d98fb183a 100644 --- a/testing/adios2/engine/bp/operations/TestBPWriteReadZfpRemoveOperations.cpp +++ b/testing/adios2/engine/bp/operations/TestBPWriteReadZfpRemoveOperations.cpp @@ -497,7 +497,8 @@ INSTANTIATE_TEST_SUITE_P(ZFPRate, BPWRZFPODD, int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/common/TestEngineCommon.cpp b/testing/adios2/engine/common/TestEngineCommon.cpp index 0a4829b661..01684b0420 100644 --- a/testing/adios2/engine/common/TestEngineCommon.cpp +++ b/testing/adios2/engine/common/TestEngineCommon.cpp @@ -290,13 +290,21 @@ int main(int argc, char **argv) std::thread threadTimeout(threadTimeoutRun, 300); threadTimeout.detach(); - MPI_Init(&argc, &argv); + int provided; ::testing::InitGoogleTest(&argc, argv); + engineName = std::string(argv[1]); + + int threadSupportLevel = MPI_THREAD_SINGLE; + if (engineName == "SST") + { + threadSupportLevel = MPI_THREAD_MULTIPLE; + } + + MPI_Init_thread(&argc, &argv, threadSupportLevel, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &wrank); MPI_Comm_size(MPI_COMM_WORLD, &numprocs); - engineName = std::string(argv[1]); unsigned int p = ParseUintParam("serializeWriterReader", argv[2]); serializeWriterReader = (p != 0); diff --git a/testing/adios2/engine/hdf5/TestHDF5Append.cpp b/testing/adios2/engine/hdf5/TestHDF5Append.cpp index 141a048e20..159db7869d 100644 --- a/testing/adios2/engine/hdf5/TestHDF5Append.cpp +++ b/testing/adios2/engine/hdf5/TestHDF5Append.cpp @@ -382,7 +382,8 @@ TEST_F(AppendTimeStepTest, ADIOS2HDF5WriteAppendRead) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/hdf5/TestHDF5StreamWriteReadHighLevelAPI.cpp b/testing/adios2/engine/hdf5/TestHDF5StreamWriteReadHighLevelAPI.cpp index 599c174246..b9a4e41d42 100644 --- a/testing/adios2/engine/hdf5/TestHDF5StreamWriteReadHighLevelAPI.cpp +++ b/testing/adios2/engine/hdf5/TestHDF5StreamWriteReadHighLevelAPI.cpp @@ -520,7 +520,8 @@ TEST_F(StreamWriteReadHighLevelAPI_HDF5, DoubleOpenException) int main(int argc, char **argv) { #ifdef TEST_HDF5_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/hdf5/TestHDF5WriteMemorySelectionRead.cpp b/testing/adios2/engine/hdf5/TestHDF5WriteMemorySelectionRead.cpp index 7ff41fe6e5..48ff09dd3b 100644 --- a/testing/adios2/engine/hdf5/TestHDF5WriteMemorySelectionRead.cpp +++ b/testing/adios2/engine/hdf5/TestHDF5WriteMemorySelectionRead.cpp @@ -930,7 +930,8 @@ INSTANTIATE_TEST_SUITE_P(ghostCells, HDF5WriteMemSelReadVector, int main(int argc, char **argv) { #ifdef TEST_HDF5_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/hdf5/TestHDF5WriteReadAsStream.cpp b/testing/adios2/engine/hdf5/TestHDF5WriteReadAsStream.cpp index 27e33670ed..75dff2e4d6 100644 --- a/testing/adios2/engine/hdf5/TestHDF5WriteReadAsStream.cpp +++ b/testing/adios2/engine/hdf5/TestHDF5WriteReadAsStream.cpp @@ -1016,7 +1016,8 @@ TEST_F(HDF5WriteReadAsStreamTestADIOS2, ReaderWriterDefineVariable) int main(int argc, char **argv) { #ifdef TEST_HDF5_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/hdf5/TestHDF5WriteReadAttributesADIOS2.cpp b/testing/adios2/engine/hdf5/TestHDF5WriteReadAttributesADIOS2.cpp index a7d851401b..c2ae64a861 100644 --- a/testing/adios2/engine/hdf5/TestHDF5WriteReadAttributesADIOS2.cpp +++ b/testing/adios2/engine/hdf5/TestHDF5WriteReadAttributesADIOS2.cpp @@ -725,7 +725,8 @@ TEST_F(BPWriteReadAttributeTestADIOS2, ADIOS2BPWriteReadArrayTypesVar) int main(int argc, char **argv) { #ifdef TEST_HDF5_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/hdf5/TestNativeHDF5WriteRead.cpp b/testing/adios2/engine/hdf5/TestNativeHDF5WriteRead.cpp index 606a32c75a..ec5c7bc720 100644 --- a/testing/adios2/engine/hdf5/TestNativeHDF5WriteRead.cpp +++ b/testing/adios2/engine/hdf5/TestNativeHDF5WriteRead.cpp @@ -3212,7 +3212,8 @@ TEST_F(HDF5WriteReadTest, /*DISABLE_*/ ATTRTESTADIOS2vsHDF5) int main(int argc, char **argv) { #ifdef TEST_HDF5_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/inline/TestInlineWriteRead.cpp b/testing/adios2/engine/inline/TestInlineWriteRead.cpp index 66577d9d0f..80bfd3aee6 100644 --- a/testing/adios2/engine/inline/TestInlineWriteRead.cpp +++ b/testing/adios2/engine/inline/TestInlineWriteRead.cpp @@ -949,7 +949,8 @@ TEST_F(InlineWriteRead, PointerArithmetic) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/mhs/TestMhsMultiRank.cpp b/testing/adios2/engine/mhs/TestMhsMultiRank.cpp index f763f89de5..4a2632a64c 100644 --- a/testing/adios2/engine/mhs/TestMhsMultiRank.cpp +++ b/testing/adios2/engine/mhs/TestMhsMultiRank.cpp @@ -240,7 +240,8 @@ TEST_F(MhsEngineTest, TestMhsMultiRank) int main(int argc, char **argv) { - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); int result; diff --git a/testing/adios2/engine/mhs/TestMhsMultiReader.cpp b/testing/adios2/engine/mhs/TestMhsMultiReader.cpp index 53ef64edf9..e91f0d02ea 100644 --- a/testing/adios2/engine/mhs/TestMhsMultiReader.cpp +++ b/testing/adios2/engine/mhs/TestMhsMultiReader.cpp @@ -234,7 +234,8 @@ TEST_F(MhsEngineTest, TestMhsMultiReader) int main(int argc, char **argv) { - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &mpiRank); MPI_Comm_size(MPI_COMM_WORLD, &mpiSize); int result; diff --git a/testing/adios2/engine/mhs/TestMhsSingleRank.cpp b/testing/adios2/engine/mhs/TestMhsSingleRank.cpp index bd79559738..26f49abc08 100644 --- a/testing/adios2/engine/mhs/TestMhsSingleRank.cpp +++ b/testing/adios2/engine/mhs/TestMhsSingleRank.cpp @@ -277,7 +277,8 @@ TEST_F(MhsEngineTest, TestMhsSingleRank) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); #endif int result; ::testing::InitGoogleTest(&argc, argv); diff --git a/testing/adios2/engine/null/TestNullWriteRead.cpp b/testing/adios2/engine/null/TestNullWriteRead.cpp index 0c985eb23a..accef76f7b 100644 --- a/testing/adios2/engine/null/TestNullWriteRead.cpp +++ b/testing/adios2/engine/null/TestNullWriteRead.cpp @@ -181,7 +181,8 @@ TEST_F(NullWriteReadTests, NullWriteRead1D8) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/nullcore/TestNullCoreWrite.cpp b/testing/adios2/engine/nullcore/TestNullCoreWrite.cpp index 389d94ebe6..c6c72c2227 100644 --- a/testing/adios2/engine/nullcore/TestNullCoreWrite.cpp +++ b/testing/adios2/engine/nullcore/TestNullCoreWrite.cpp @@ -160,7 +160,8 @@ TEST_F(NullCoreWriteTest, ADIOS2NullCoreWrite1D8) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/skeleton/TestSkeletonReader.cpp b/testing/adios2/engine/skeleton/TestSkeletonReader.cpp index 05136746a9..2ac35d6b8a 100644 --- a/testing/adios2/engine/skeleton/TestSkeletonReader.cpp +++ b/testing/adios2/engine/skeleton/TestSkeletonReader.cpp @@ -62,7 +62,8 @@ int main(int argc, char *argv[]) #if ADIOS2_USE_MPI int wrank = 0, wnproc = 1; MPI_Comm mpiReaderComm; - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &wrank); MPI_Comm_size(MPI_COMM_WORLD, &wnproc); diff --git a/testing/adios2/engine/skeleton/TestSkeletonWriter.cpp b/testing/adios2/engine/skeleton/TestSkeletonWriter.cpp index 8966c739ae..b768d4d706 100644 --- a/testing/adios2/engine/skeleton/TestSkeletonWriter.cpp +++ b/testing/adios2/engine/skeleton/TestSkeletonWriter.cpp @@ -34,7 +34,8 @@ int main(int argc, char *argv[]) #if ADIOS2_USE_MPI int wrank = 0, wnproc = 1; MPI_Comm mpiWriterComm; - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &wrank); MPI_Comm_size(MPI_COMM_WORLD, &wnproc); diff --git a/testing/adios2/engine/sst/TestSstParamFails.cpp b/testing/adios2/engine/sst/TestSstParamFails.cpp index 79806b64a0..8d4c53b015 100644 --- a/testing/adios2/engine/sst/TestSstParamFails.cpp +++ b/testing/adios2/engine/sst/TestSstParamFails.cpp @@ -225,7 +225,8 @@ TEST_F(SstParamFailTest, PreciousFirstTimestepParamTest) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/sst/TestSstWriterFails.cpp b/testing/adios2/engine/sst/TestSstWriterFails.cpp index 9dc57d5c5d..b6308e71ba 100644 --- a/testing/adios2/engine/sst/TestSstWriterFails.cpp +++ b/testing/adios2/engine/sst/TestSstWriterFails.cpp @@ -126,7 +126,8 @@ TEST_F(SstWriteFails, InvalidBeginStep) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/engine/staging-common/TestCommonClient.cpp b/testing/adios2/engine/staging-common/TestCommonClient.cpp index 7b5c408437..116e1db8bb 100644 --- a/testing/adios2/engine/staging-common/TestCommonClient.cpp +++ b/testing/adios2/engine/staging-common/TestCommonClient.cpp @@ -388,18 +388,25 @@ TEST_F(SstReadTest, ADIOS2SstRead) int main(int argc, char **argv) { -#if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); -#endif int result; ::testing::InitGoogleTest(&argc, argv); DelayMS = 500; // smaller default for common client ParseArgs(argc, argv); +#if ADIOS2_USE_MPI + int provided; + int thread_support_level = + (engine == "SST") ? MPI_THREAD_MULTIPLE : MPI_THREAD_SINGLE; + MPI_Init_thread(nullptr, nullptr, thread_support_level, &provided); +#endif result = RUN_ALL_TESTS(); #if ADIOS2_USE_MPI +#ifdef CRAY_MPICH_VERSION + MPI_Barrier(MPI_COMM_WORLD); +#else MPI_Finalize(); +#endif #endif return result; diff --git a/testing/adios2/engine/staging-common/TestCommonRead.cpp b/testing/adios2/engine/staging-common/TestCommonRead.cpp index b5c462eb15..06c17feb14 100644 --- a/testing/adios2/engine/staging-common/TestCommonRead.cpp +++ b/testing/adios2/engine/staging-common/TestCommonRead.cpp @@ -476,8 +476,16 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8) int main(int argc, char **argv) { + int result; + ::testing::InitGoogleTest(&argc, argv); + ParseArgs(argc, argv); + #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + int thread_support_level = (engine == "SST" || engine == "sst") + ? MPI_THREAD_MULTIPLE + : MPI_THREAD_SINGLE; + MPI_Init_thread(nullptr, nullptr, thread_support_level, &provided); int key; MPI_Comm_rank(MPI_COMM_WORLD, &key); @@ -486,14 +494,14 @@ int main(int argc, char **argv) MPI_Comm_split(MPI_COMM_WORLD, color, key, &testComm); #endif - int result; - ::testing::InitGoogleTest(&argc, argv); - ParseArgs(argc, argv); - result = RUN_ALL_TESTS(); #if ADIOS2_USE_MPI +#ifdef CRAY_MPICH_VERSION + MPI_Barrier(MPI_COMM_WORLD); +#else MPI_Finalize(); +#endif #endif return result; diff --git a/testing/adios2/engine/staging-common/TestCommonReadAttrs.cpp b/testing/adios2/engine/staging-common/TestCommonReadAttrs.cpp index 7a98ca88f7..230eae8318 100644 --- a/testing/adios2/engine/staging-common/TestCommonReadAttrs.cpp +++ b/testing/adios2/engine/staging-common/TestCommonReadAttrs.cpp @@ -354,8 +354,17 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8) int main(int argc, char **argv) { + int result; + ::testing::InitGoogleTest(&argc, argv); + + ParseArgs(argc, argv); + #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + int thread_support_level = (engine == "SST" || engine == "sst") + ? MPI_THREAD_MULTIPLE + : MPI_THREAD_SINGLE; + MPI_Init_thread(nullptr, nullptr, thread_support_level, &provided); int key; MPI_Comm_rank(MPI_COMM_WORLD, &key); @@ -364,15 +373,14 @@ int main(int argc, char **argv) MPI_Comm_split(MPI_COMM_WORLD, color, key, &testComm); #endif - int result; - ::testing::InitGoogleTest(&argc, argv); - - ParseArgs(argc, argv); - result = RUN_ALL_TESTS(); #if ADIOS2_USE_MPI +#ifdef CRAY_MPICH_VERSION + MPI_Barrier(MPI_COMM_WORLD); +#else MPI_Finalize(); +#endif #endif return result; diff --git a/testing/adios2/engine/staging-common/TestCommonReadF.F90 b/testing/adios2/engine/staging-common/TestCommonReadF.F90 index b8f2637921..90d7d5fc61 100644 --- a/testing/adios2/engine/staging-common/TestCommonReadF.F90 +++ b/testing/adios2/engine/staging-common/TestCommonReadF.F90 @@ -45,7 +45,7 @@ program TestSstRead integer(kind = 8), dimension(:), allocatable::shape_in #if ADIOS2_USE_MPI - integer::key, color, testComm + integer::key, color, testComm, provided, threadSupportLevel #endif allocate(variables(20)) @@ -67,15 +67,20 @@ program TestSstRead insteps = 0; #if ADIOS2_USE_MPI + threadSupportLevel = MPI_THREAD_SINGLE; + if (engine == "SST") then + threadSupportLevel = MPI_THREAD_MULTIPLE; + endif + !Launch MPI - call MPI_Init(ierr) + call MPI_Init_thread(threadSupportLevel, provided, ierr) call MPI_Comm_rank(MPI_COMM_WORLD, key, ierr); color = 2 call MPI_Comm_split(MPI_COMM_WORLD, color, key, testComm, ierr); - call MPI_Comm_rank(testComm, irank, ierr) + call MPI_Comm_rank(testComm, irank, ierr) call MPI_Comm_size(testComm, isize, ierr) !Create adios handler passing the communicator, debug mode and error flag @@ -271,8 +276,12 @@ program TestSstRead #if ADIOS2_USE_MPI +#ifdef CRAY_MPICH_VERSION + call MPI_Barrier(MPI_COMM_WORLD) +#else call MPI_Finalize(ierr) #endif +#endif end program TestSstRead diff --git a/testing/adios2/engine/staging-common/TestCommonReadLocal.cpp b/testing/adios2/engine/staging-common/TestCommonReadLocal.cpp index 0b3fc6767c..717baa1b0e 100644 --- a/testing/adios2/engine/staging-common/TestCommonReadLocal.cpp +++ b/testing/adios2/engine/staging-common/TestCommonReadLocal.cpp @@ -168,8 +168,17 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8) int main(int argc, char **argv) { + int result; + ::testing::InitGoogleTest(&argc, argv); + + ParseArgs(argc, argv); + #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + int thread_support_level = (engine == "SST" || engine == "sst") + ? MPI_THREAD_MULTIPLE + : MPI_THREAD_SINGLE; + MPI_Init_thread(nullptr, nullptr, thread_support_level, &provided); int key; MPI_Comm_rank(MPI_COMM_WORLD, &key); @@ -178,15 +187,14 @@ int main(int argc, char **argv) MPI_Comm_split(MPI_COMM_WORLD, color, key, &testComm); #endif - int result; - ::testing::InitGoogleTest(&argc, argv); - - ParseArgs(argc, argv); - result = RUN_ALL_TESTS(); #if ADIOS2_USE_MPI +#ifdef CRAY_MPICH_VERSION + MPI_Barrier(MPI_COMM_WORLD); +#else MPI_Finalize(); +#endif #endif return result; diff --git a/testing/adios2/engine/staging-common/TestCommonReadR64.cpp b/testing/adios2/engine/staging-common/TestCommonReadR64.cpp index 5b1a42c48f..0d5f15d2cb 100644 --- a/testing/adios2/engine/staging-common/TestCommonReadR64.cpp +++ b/testing/adios2/engine/staging-common/TestCommonReadR64.cpp @@ -142,8 +142,16 @@ TEST_F(CommonReadTest, ADIOS2CommonRead) int main(int argc, char **argv) { + int result; + ::testing::InitGoogleTest(&argc, argv); + ParseArgs(argc, argv); + #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + int thread_support_level = (engine == "SST" || engine == "sst") + ? MPI_THREAD_MULTIPLE + : MPI_THREAD_SINGLE; + MPI_Init_thread(nullptr, nullptr, thread_support_level, &provided); int key; MPI_Comm_rank(MPI_COMM_WORLD, &key); @@ -152,14 +160,14 @@ int main(int argc, char **argv) MPI_Comm_split(MPI_COMM_WORLD, color, key, &testComm); #endif - int result; - ::testing::InitGoogleTest(&argc, argv); - ParseArgs(argc, argv); - result = RUN_ALL_TESTS(); #if ADIOS2_USE_MPI +#ifdef CRAY_MPICH_VERSION + MPI_Barrier(MPI_COMM_WORLD); +#else MPI_Finalize(); +#endif #endif return result; diff --git a/testing/adios2/engine/staging-common/TestCommonReadShared.cpp b/testing/adios2/engine/staging-common/TestCommonReadShared.cpp index 8c7d099576..cfb0b6c354 100644 --- a/testing/adios2/engine/staging-common/TestCommonReadShared.cpp +++ b/testing/adios2/engine/staging-common/TestCommonReadShared.cpp @@ -142,8 +142,17 @@ TEST_F(CommonReadTest, ADIOS2CommonRead1D8) int main(int argc, char **argv) { + int result; + ::testing::InitGoogleTest(&argc, argv); + + ParseArgs(argc, argv); + #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + int thread_support_level = (engine == "SST" || engine == "sst") + ? MPI_THREAD_MULTIPLE + : MPI_THREAD_SINGLE; + MPI_Init_thread(nullptr, nullptr, thread_support_level, &provided); int key; MPI_Comm_rank(MPI_COMM_WORLD, &key); @@ -152,15 +161,14 @@ int main(int argc, char **argv) MPI_Comm_split(MPI_COMM_WORLD, color, key, &testComm); #endif - int result; - ::testing::InitGoogleTest(&argc, argv); - - ParseArgs(argc, argv); - result = RUN_ALL_TESTS(); #if ADIOS2_USE_MPI +#ifdef CRAY_MPICH_VERSION + MPI_Barrier(MPI_COMM_WORLD); +#else MPI_Finalize(); +#endif #endif return result; diff --git a/testing/adios2/engine/staging-common/TestCommonServer.cpp b/testing/adios2/engine/staging-common/TestCommonServer.cpp index a85bddbc04..9053e347e4 100644 --- a/testing/adios2/engine/staging-common/TestCommonServer.cpp +++ b/testing/adios2/engine/staging-common/TestCommonServer.cpp @@ -195,19 +195,27 @@ TEST_F(CommonServerTest, ADIOS2CommonServer) int main(int argc, char **argv) { -#if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); -#endif - int result; ::testing::InitGoogleTest(&argc, argv); ParseArgs(argc, argv); +#if ADIOS2_USE_MPI + int provided; + int thread_support_level = (engine == "SST" || engine == "sst") + ? MPI_THREAD_MULTIPLE + : MPI_THREAD_SINGLE; + MPI_Init_thread(nullptr, nullptr, thread_support_level, &provided); +#endif + result = RUN_ALL_TESTS(); #if ADIOS2_USE_MPI +#ifdef CRAY_MPICH_VERSION + MPI_Barrier(MPI_COMM_WORLD); +#else MPI_Finalize(); +#endif #endif return result; diff --git a/testing/adios2/engine/staging-common/TestCommonWrite.cpp b/testing/adios2/engine/staging-common/TestCommonWrite.cpp index 514901eeb1..7431fa5724 100644 --- a/testing/adios2/engine/staging-common/TestCommonWrite.cpp +++ b/testing/adios2/engine/staging-common/TestCommonWrite.cpp @@ -245,8 +245,19 @@ TEST_F(CommonWriteTest, ADIOS2CommonWrite) int main(int argc, char **argv) { + int result; + ::testing::InitGoogleTest(&argc, argv); + + DelayMS = 0; // zero for common writer + + ParseArgs(argc, argv); + #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + int thread_support_level = (engine == "SST" || engine == "sst") + ? MPI_THREAD_MULTIPLE + : MPI_THREAD_SINGLE; + MPI_Init_thread(nullptr, nullptr, thread_support_level, &provided); int key; MPI_Comm_rank(MPI_COMM_WORLD, &key); @@ -255,17 +266,14 @@ int main(int argc, char **argv) MPI_Comm_split(MPI_COMM_WORLD, color, key, &testComm); #endif - int result; - ::testing::InitGoogleTest(&argc, argv); - - DelayMS = 0; // zero for common writer - - ParseArgs(argc, argv); - result = RUN_ALL_TESTS(); #if ADIOS2_USE_MPI +#ifdef CRAY_MPICH_VERSION + MPI_Barrier(MPI_COMM_WORLD); +#else MPI_Finalize(); +#endif #endif return result; diff --git a/testing/adios2/engine/staging-common/TestCommonWriteAttrs.cpp b/testing/adios2/engine/staging-common/TestCommonWriteAttrs.cpp index fffb642380..0f7738a750 100644 --- a/testing/adios2/engine/staging-common/TestCommonWriteAttrs.cpp +++ b/testing/adios2/engine/staging-common/TestCommonWriteAttrs.cpp @@ -194,8 +194,17 @@ TEST_F(CommonWriteTest, ADIOS2CommonWrite) int main(int argc, char **argv) { + int result; + ::testing::InitGoogleTest(&argc, argv); + + ParseArgs(argc, argv); + #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + int thread_support_level = (engine == "SST" || engine == "sst") + ? MPI_THREAD_MULTIPLE + : MPI_THREAD_SINGLE; + MPI_Init_thread(nullptr, nullptr, thread_support_level, &provided); int key; MPI_Comm_rank(MPI_COMM_WORLD, &key); @@ -204,15 +213,14 @@ int main(int argc, char **argv) MPI_Comm_split(MPI_COMM_WORLD, color, key, &testComm); #endif - int result; - ::testing::InitGoogleTest(&argc, argv); - - ParseArgs(argc, argv); - result = RUN_ALL_TESTS(); #if ADIOS2_USE_MPI +#ifdef CRAY_MPICH_VERSION + MPI_Barrier(MPI_COMM_WORLD); +#else MPI_Finalize(); +#endif #endif return result; diff --git a/testing/adios2/engine/staging-common/TestCommonWriteF.F90 b/testing/adios2/engine/staging-common/TestCommonWriteF.F90 index 1ff2821938..6c8ebc003a 100644 --- a/testing/adios2/engine/staging-common/TestCommonWriteF.F90 +++ b/testing/adios2/engine/staging-common/TestCommonWriteF.F90 @@ -42,7 +42,7 @@ program TestSstWrite integer(kind = 8)::localtime #if ADIOS2_USE_MPI - integer::testComm, color, key + integer::testComm, color, key, provided, threadSupportLevel #endif allocate(variables(20)) @@ -61,8 +61,13 @@ program TestSstWrite endif #if ADIOS2_USE_MPI + threadSupportLevel = MPI_THREAD_SINGLE; + if (engine == "SST") then + threadSupportLevel = MPI_THREAD_MULTIPLE; + endif + !Launch MPI - call MPI_Init(ierr) + call MPI_Init_thread(threadSupportLevel, provided, ierr) call MPI_Comm_rank(MPI_COMM_WORLD, key, ierr); @@ -205,7 +210,11 @@ program TestSstWrite call adios2_finalize(adios, ierr) #if ADIOS2_USE_MPI +#ifdef CRAY_MPICH_VERSION + call MPI_Barrier(MPI_COMM_WORLD) +#else call MPI_Finalize(ierr) +#endif #endif end program TestSstWrite diff --git a/testing/adios2/engine/staging-common/TestCommonWriteLocal.cpp b/testing/adios2/engine/staging-common/TestCommonWriteLocal.cpp index bcefb2126c..f3c0ab0e6d 100644 --- a/testing/adios2/engine/staging-common/TestCommonWriteLocal.cpp +++ b/testing/adios2/engine/staging-common/TestCommonWriteLocal.cpp @@ -109,8 +109,19 @@ TEST_F(CommonWriteTest, ADIOS2CommonWrite) int main(int argc, char **argv) { + int result; + ::testing::InitGoogleTest(&argc, argv); + + DelayMS = 0; // zero default for common writer + + ParseArgs(argc, argv); + #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + int thread_support_level = (engine == "SST" || engine == "sst") + ? MPI_THREAD_MULTIPLE + : MPI_THREAD_SINGLE; + MPI_Init_thread(nullptr, nullptr, thread_support_level, &provided); int key; MPI_Comm_rank(MPI_COMM_WORLD, &key); @@ -119,17 +130,14 @@ int main(int argc, char **argv) MPI_Comm_split(MPI_COMM_WORLD, color, key, &testComm); #endif - int result; - ::testing::InitGoogleTest(&argc, argv); - - DelayMS = 0; // zero default for common writer - - ParseArgs(argc, argv); - result = RUN_ALL_TESTS(); #if ADIOS2_USE_MPI +#ifdef CRAY_MPICH_VERSION + MPI_Barrier(MPI_COMM_WORLD); +#else MPI_Finalize(); +#endif #endif return result; diff --git a/testing/adios2/engine/staging-common/TestCommonWriteModes.cpp b/testing/adios2/engine/staging-common/TestCommonWriteModes.cpp index 7b6c4d1eb4..ad3f775182 100644 --- a/testing/adios2/engine/staging-common/TestCommonWriteModes.cpp +++ b/testing/adios2/engine/staging-common/TestCommonWriteModes.cpp @@ -183,8 +183,17 @@ TEST_F(CommonWriteTest, ADIOS2CommonWrite) int main(int argc, char **argv) { + int result; + ::testing::InitGoogleTest(&argc, argv); + + ParseArgs(argc, argv); + #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + int thread_support_level = (engine == "SST" || engine == "sst") + ? MPI_THREAD_MULTIPLE + : MPI_THREAD_SINGLE; + MPI_Init_thread(nullptr, nullptr, thread_support_level, &provided); int key; MPI_Comm_rank(MPI_COMM_WORLD, &key); @@ -193,15 +202,14 @@ int main(int argc, char **argv) MPI_Comm_split(MPI_COMM_WORLD, color, key, &testComm); #endif - int result; - ::testing::InitGoogleTest(&argc, argv); - - ParseArgs(argc, argv); - result = RUN_ALL_TESTS(); #if ADIOS2_USE_MPI +#ifdef CRAY_MPICH_VERSION + MPI_Barrier(MPI_COMM_WORLD); +#else MPI_Finalize(); +#endif #endif return result; diff --git a/testing/adios2/engine/staging-common/TestCommonWriteShared.cpp b/testing/adios2/engine/staging-common/TestCommonWriteShared.cpp index 90219964ea..c2f09ad8be 100644 --- a/testing/adios2/engine/staging-common/TestCommonWriteShared.cpp +++ b/testing/adios2/engine/staging-common/TestCommonWriteShared.cpp @@ -162,8 +162,17 @@ TEST_F(CommonWriteTest, ADIOS2CommonWrite) int main(int argc, char **argv) { + int result; + ::testing::InitGoogleTest(&argc, argv); + + ParseArgs(argc, argv); + #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + int thread_support_level = (engine == "SST" || engine == "sst") + ? MPI_THREAD_MULTIPLE + : MPI_THREAD_SINGLE; + MPI_Init_thread(nullptr, nullptr, thread_support_level, &provided); int key; MPI_Comm_rank(MPI_COMM_WORLD, &key); @@ -172,15 +181,14 @@ int main(int argc, char **argv) MPI_Comm_split(MPI_COMM_WORLD, color, key, &testComm); #endif - int result; - ::testing::InitGoogleTest(&argc, argv); - - ParseArgs(argc, argv); - result = RUN_ALL_TESTS(); #if ADIOS2_USE_MPI +#ifdef CRAY_MPICH_VERSION + MPI_Barrier(MPI_COMM_WORLD); +#else MPI_Finalize(); +#endif #endif return result; diff --git a/testing/adios2/engine/staging-common/TestDistributionRead.cpp b/testing/adios2/engine/staging-common/TestDistributionRead.cpp index 6df2cd0ad6..556204e67e 100644 --- a/testing/adios2/engine/staging-common/TestDistributionRead.cpp +++ b/testing/adios2/engine/staging-common/TestDistributionRead.cpp @@ -252,7 +252,11 @@ int main(int argc, char **argv) result = RUN_ALL_TESTS(); #if ADIOS2_USE_MPI +#ifdef CRAY_MPICH_VERSION + MPI_Barrier(MPI_COMM_WORLD); +#else MPI_Finalize(); +#endif #endif return result; diff --git a/testing/adios2/engine/staging-common/TestDistributionWrite.cpp b/testing/adios2/engine/staging-common/TestDistributionWrite.cpp index e23449950a..8e7b040ef9 100644 --- a/testing/adios2/engine/staging-common/TestDistributionWrite.cpp +++ b/testing/adios2/engine/staging-common/TestDistributionWrite.cpp @@ -154,7 +154,11 @@ int main(int argc, char **argv) result = RUN_ALL_TESTS(); #if ADIOS2_USE_MPI +#ifdef CRAY_MPICH_VERSION + MPI_Barrier(MPI_COMM_WORLD); +#else MPI_Finalize(); +#endif #endif return result; diff --git a/testing/adios2/engine/staging-common/TestStagingMPMD.cpp b/testing/adios2/engine/staging-common/TestStagingMPMD.cpp index 8c16ff9a02..9661dcb369 100644 --- a/testing/adios2/engine/staging-common/TestStagingMPMD.cpp +++ b/testing/adios2/engine/staging-common/TestStagingMPMD.cpp @@ -364,13 +364,21 @@ int main(int argc, char **argv) std::thread threadTimeout(threadTimeoutRun, 300); threadTimeout.detach(); - MPI_Init(&argc, &argv); + int provided; ::testing::InitGoogleTest(&argc, argv); + engineName = std::string(argv[1]); + + int threadSupportLevel = MPI_THREAD_SINGLE; + if (engineName == "SST") + { + threadSupportLevel = MPI_THREAD_MULTIPLE; + } + + MPI_Init_thread(&argc, &argv, threadSupportLevel, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &wrank); MPI_Comm_size(MPI_COMM_WORLD, &numprocs); - engineName = std::string(argv[1]); if (argc > 2) { engineParams = ParseEngineParams(argv[2]); @@ -385,6 +393,11 @@ int main(int argc, char **argv) int result; result = RUN_ALL_TESTS(); +#ifdef CRAY_MPICH_VERSION + MPI_Barrier(MPI_COMM_WORLD); +#else MPI_Finalize(); +#endif + return result; } diff --git a/testing/adios2/engine/staging-common/TestSupp.cmake b/testing/adios2/engine/staging-common/TestSupp.cmake index 26466c5cc7..a5d8b76e42 100644 --- a/testing/adios2/engine/staging-common/TestSupp.cmake +++ b/testing/adios2/engine/staging-common/TestSupp.cmake @@ -270,12 +270,15 @@ function(add_common_test basename engine) endif() set (timeout "${${basename}_TIMEOUT}") if ("${timeout}" STREQUAL "") - set (timeout "60") + if (DEFINED MPIEXEC_EXECUTABLE AND "${MPIEXEC_EXECUTABLE}" MATCHES "srun") + set (timeout "1000") + else() + set (timeout "60") + endif() endif() set_tests_properties(${testname} PROPERTIES TIMEOUT ${timeout} ${${basename}_PROPERTIES} - RUN_SERIAL TRUE ) endfunction() diff --git a/testing/adios2/engine/staging-common/TestThreads.cpp b/testing/adios2/engine/staging-common/TestThreads.cpp index 0a4568c2cb..1c09fc4a12 100644 --- a/testing/adios2/engine/staging-common/TestThreads.cpp +++ b/testing/adios2/engine/staging-common/TestThreads.cpp @@ -150,6 +150,7 @@ TEST_F(TestThreads, Basic) auto read_fut = std::async(std::launch::async, Read, BaseName, 0); auto write_fut = std::async(std::launch::async, Write, BaseName, 0); bool reader_success = read_fut.get(); + sleep(1); bool writer_success = write_fut.get(); EXPECT_TRUE(reader_success); EXPECT_TRUE(writer_success); diff --git a/testing/adios2/interface/TestADIOSDefineAttribute.cpp b/testing/adios2/interface/TestADIOSDefineAttribute.cpp index 9b90e2c67e..9f0a50bbd4 100644 --- a/testing/adios2/interface/TestADIOSDefineAttribute.cpp +++ b/testing/adios2/interface/TestADIOSDefineAttribute.cpp @@ -800,7 +800,8 @@ TEST_F(ADIOSDefineAttributeTest, VariableException) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/interface/TestADIOSDefineVariable.cpp b/testing/adios2/interface/TestADIOSDefineVariable.cpp index 24f00fa54e..3c6bb3759a 100644 --- a/testing/adios2/interface/TestADIOSDefineVariable.cpp +++ b/testing/adios2/interface/TestADIOSDefineVariable.cpp @@ -715,7 +715,8 @@ TEST_F(ADIOSDefineVariableTest, DefineStructVariable) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/interface/TestADIOSInterface.cpp b/testing/adios2/interface/TestADIOSInterface.cpp index 73245ba047..3af95cbbba 100644 --- a/testing/adios2/interface/TestADIOSInterface.cpp +++ b/testing/adios2/interface/TestADIOSInterface.cpp @@ -503,7 +503,8 @@ TYPED_TEST(ADIOS2_CXX11_API_MultiBlock, Put2Writers) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/interface/TestADIOSInterfaceWrite.cpp b/testing/adios2/interface/TestADIOSInterfaceWrite.cpp index 3d23262a88..c5f8b89b72 100644 --- a/testing/adios2/interface/TestADIOSInterfaceWrite.cpp +++ b/testing/adios2/interface/TestADIOSInterfaceWrite.cpp @@ -621,7 +621,8 @@ TEST_F(ADIOSInterfaceWriteTest, Exceptions) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/performance/manyvars/PerfManyVars.c b/testing/adios2/performance/manyvars/PerfManyVars.c index ff33676473..ce88079804 100644 --- a/testing/adios2/performance/manyvars/PerfManyVars.c +++ b/testing/adios2/performance/manyvars/PerfManyVars.c @@ -156,7 +156,8 @@ int main(int argc, char **argv) { int err, i; - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(comm, &rank); MPI_Comm_size(comm, &size); diff --git a/testing/adios2/performance/manyvars/TestManyVars.cpp b/testing/adios2/performance/manyvars/TestManyVars.cpp index 9e6cc7874b..e50cf65047 100644 --- a/testing/adios2/performance/manyvars/TestManyVars.cpp +++ b/testing/adios2/performance/manyvars/TestManyVars.cpp @@ -483,7 +483,8 @@ INSTANTIATE_TEST_SUITE_P(NxM, TestManyVars, int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); #endif ::testing::InitGoogleTest(&argc, argv); diff --git a/testing/adios2/performance/metadata/PerfMetaData.cpp b/testing/adios2/performance/metadata/PerfMetaData.cpp index 1cea17c0c0..15add1e16a 100644 --- a/testing/adios2/performance/metadata/PerfMetaData.cpp +++ b/testing/adios2/performance/metadata/PerfMetaData.cpp @@ -522,7 +522,8 @@ int main(int argc, char **argv) if (UseMPI) { - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &key); MPI_Comm_size(MPI_COMM_WORLD, &WriterSize); diff --git a/testing/adios2/performance/query/TestBPQuery.cpp b/testing/adios2/performance/query/TestBPQuery.cpp index e0d4be30e7..72a499aaaa 100644 --- a/testing/adios2/performance/query/TestBPQuery.cpp +++ b/testing/adios2/performance/query/TestBPQuery.cpp @@ -294,8 +294,8 @@ TEST_F(BPQueryTest, BP4) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - // MPI_Init(nullptr, nullptr); - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/xml/TestXMLConfig.cpp b/testing/adios2/xml/TestXMLConfig.cpp index 3667284e46..58e06e2ee3 100644 --- a/testing/adios2/xml/TestXMLConfig.cpp +++ b/testing/adios2/xml/TestXMLConfig.cpp @@ -130,7 +130,8 @@ TEST_F(XMLConfigTest, OpNoneException) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/adios2/yaml/TestYAMLConfig.cpp b/testing/adios2/yaml/TestYAMLConfig.cpp index 0d37c16b51..5380d1ff38 100644 --- a/testing/adios2/yaml/TestYAMLConfig.cpp +++ b/testing/adios2/yaml/TestYAMLConfig.cpp @@ -100,7 +100,8 @@ TEST_F(YAMLConfigTest, OpNullException) int main(int argc, char **argv) { #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/examples/heatTransfer/TestSSTBPMx1.cmake b/testing/examples/heatTransfer/TestSSTBPMx1.cmake index e240d50474..7da75f9cb4 100644 --- a/testing/examples/heatTransfer/TestSSTBPMx1.cmake +++ b/testing/examples/heatTransfer/TestSSTBPMx1.cmake @@ -10,12 +10,12 @@ add_test(NAME HeatTransfer.SST.BP.Mx1 ${MPIEXEC_NUMPROC_FLAG} 4 $ ${PROJECT_SOURCE_DIR}/examples/heatTransfer/heat_sst_bp.xml - Write.bp 2 2 10 10 10 10 + Write.bp 2 2 10 10 10 10 SST : ${MPIEXEC_NUMPROC_FLAG} 1 $ ${PROJECT_SOURCE_DIR}/examples/heatTransfer/heat_sst_bp.xml - Write.bp Read.bp 1 1 + Write.bp Read.bp 1 1 SST ) set_tests_properties(HeatTransfer.SST.BP.Mx1 PROPERTIES PROCESSORS 5) diff --git a/testing/examples/heatTransfer/TestSSTBPMxM.cmake b/testing/examples/heatTransfer/TestSSTBPMxM.cmake index 8f398ac1e8..286b707eb7 100644 --- a/testing/examples/heatTransfer/TestSSTBPMxM.cmake +++ b/testing/examples/heatTransfer/TestSSTBPMxM.cmake @@ -10,12 +10,12 @@ add_test(NAME HeatTransfer.SST.BP.MxM ${MPIEXEC_NUMPROC_FLAG} 4 $ ${PROJECT_SOURCE_DIR}/examples/heatTransfer/heat_sst_bp.xml - Write.bp 2 2 10 10 10 10 + Write.bp 2 2 10 10 10 10 SST : ${MPIEXEC_NUMPROC_FLAG} 4 $ ${PROJECT_SOURCE_DIR}/examples/heatTransfer/heat_sst_bp.xml - Write.bp Read.bp 2 2 + Write.bp Read.bp 2 2 SST ) set_tests_properties(HeatTransfer.SST.BP.MxM PROPERTIES PROCESSORS 8) diff --git a/testing/examples/heatTransfer/TestSSTBPMxN.cmake b/testing/examples/heatTransfer/TestSSTBPMxN.cmake index cc2c98ac89..1860244786 100644 --- a/testing/examples/heatTransfer/TestSSTBPMxN.cmake +++ b/testing/examples/heatTransfer/TestSSTBPMxN.cmake @@ -10,12 +10,12 @@ add_test(NAME HeatTransfer.SST.BP.MxN ${MPIEXEC_NUMPROC_FLAG} 4 $ ${PROJECT_SOURCE_DIR}/examples/heatTransfer/heat_sst_bp.xml - Write.bp 2 2 10 10 10 10 + Write.bp 2 2 10 10 10 10 SST : ${MPIEXEC_NUMPROC_FLAG} 3 $ ${PROJECT_SOURCE_DIR}/examples/heatTransfer/heat_sst_bp.xml - Write.bp Read.bp 1 3 + Write.bp Read.bp 1 3 SST ) set_tests_properties(HeatTransfer.SST.BP.MxN PROPERTIES PROCESSORS 7) diff --git a/testing/examples/heatTransfer/TestSSTBPRDMAMxN.cmake b/testing/examples/heatTransfer/TestSSTBPRDMAMxN.cmake index 1153b9e729..70dbb99a7e 100644 --- a/testing/examples/heatTransfer/TestSSTBPRDMAMxN.cmake +++ b/testing/examples/heatTransfer/TestSSTBPRDMAMxN.cmake @@ -10,12 +10,12 @@ add_test(NAME HeatTransfer.SST.BP.RDMA.MxN ${MPIEXEC_NUMPROC_FLAG} 4 $ ${PROJECT_SOURCE_DIR}/examples/heatTransfer/heat_sst_bp_rdma.xml - Write.bp 2 2 10 10 10 10 + Write.bp 2 2 10 10 10 10 SST : ${MPIEXEC_NUMPROC_FLAG} 3 $ ${PROJECT_SOURCE_DIR}/examples/heatTransfer/heat_sst_bp_rdma.xml - Write.bp Read.bp 1 3 + Write.bp Read.bp 1 3 SST ) set_tests_properties(HeatTransfer.SST.BP.RDMA.MxN PROPERTIES PROCESSORS 7) diff --git a/testing/examples/heatTransfer/TestSSTFFSMx1.cmake b/testing/examples/heatTransfer/TestSSTFFSMx1.cmake index 81c624cd4d..1c261b0f0f 100644 --- a/testing/examples/heatTransfer/TestSSTFFSMx1.cmake +++ b/testing/examples/heatTransfer/TestSSTFFSMx1.cmake @@ -10,12 +10,12 @@ add_test(NAME HeatTransfer.SST.FFS.Mx1 ${MPIEXEC_NUMPROC_FLAG} 4 $ ${PROJECT_SOURCE_DIR}/examples/heatTransfer/heat_sst_ffs.xml - Write.bp 2 2 10 10 10 10 + Write.bp 2 2 10 10 10 10 SST : ${MPIEXEC_NUMPROC_FLAG} 1 $ ${PROJECT_SOURCE_DIR}/examples/heatTransfer/heat_sst_ffs.xml - Write.bp Read.bp 1 1 + Write.bp Read.bp 1 1 SST ) set_tests_properties(HeatTransfer.SST.FFS.Mx1 PROPERTIES PROCESSORS 5) diff --git a/testing/examples/heatTransfer/TestSSTFFSMxM.cmake b/testing/examples/heatTransfer/TestSSTFFSMxM.cmake index c57905b83a..500869e30c 100644 --- a/testing/examples/heatTransfer/TestSSTFFSMxM.cmake +++ b/testing/examples/heatTransfer/TestSSTFFSMxM.cmake @@ -10,12 +10,12 @@ add_test(NAME HeatTransfer.SST.FFS.MxM ${MPIEXEC_NUMPROC_FLAG} 4 $ ${PROJECT_SOURCE_DIR}/examples/heatTransfer/heat_sst_ffs.xml - Write.bp 2 2 10 10 10 10 + Write.bp 2 2 10 10 10 10 SST : ${MPIEXEC_NUMPROC_FLAG} 4 $ ${PROJECT_SOURCE_DIR}/examples/heatTransfer/heat_sst_ffs.xml - Write.bp Read.bp 2 2 + Write.bp Read.bp 2 2 SST ) set_tests_properties(HeatTransfer.SST.FFS.MxM PROPERTIES PROCESSORS 8) diff --git a/testing/examples/heatTransfer/TestSSTFFSMxN.cmake b/testing/examples/heatTransfer/TestSSTFFSMxN.cmake index 58097a4a07..774a1ce7ff 100644 --- a/testing/examples/heatTransfer/TestSSTFFSMxN.cmake +++ b/testing/examples/heatTransfer/TestSSTFFSMxN.cmake @@ -10,12 +10,12 @@ add_test(NAME HeatTransfer.SST.FFS.MxN ${MPIEXEC_NUMPROC_FLAG} 4 $ ${PROJECT_SOURCE_DIR}/examples/heatTransfer/heat_sst_ffs.xml - Write.bp 2 2 10 10 10 10 + Write.bp 2 2 10 10 10 10 SST : ${MPIEXEC_NUMPROC_FLAG} 3 $ ${PROJECT_SOURCE_DIR}/examples/heatTransfer/heat_sst_ffs.xml - Write.bp Read.bp 1 3 + Write.bp Read.bp 1 3 SST ) set_tests_properties(HeatTransfer.SST.FFS.MxN PROPERTIES PROCESSORS 7) diff --git a/testing/examples/heatTransfer/TestSSTFFSRDMAMxN.cmake b/testing/examples/heatTransfer/TestSSTFFSRDMAMxN.cmake index 597c95048f..70c933e375 100644 --- a/testing/examples/heatTransfer/TestSSTFFSRDMAMxN.cmake +++ b/testing/examples/heatTransfer/TestSSTFFSRDMAMxN.cmake @@ -10,12 +10,12 @@ add_test(NAME HeatTransfer.SST.FFS.MxN ${MPIEXEC_NUMPROC_FLAG} 4 $ ${PROJECT_SOURCE_DIR}/examples/heatTransfer/heat_sst_ffs_rdma.xml - Write.bp 2 2 10 10 10 10 + Write.bp 2 2 10 10 10 10 SST : ${MPIEXEC_NUMPROC_FLAG} 3 $ ${PROJECT_SOURCE_DIR}/examples/heatTransfer/heat_sst_ffs_rdma.xml - Write.bp Read.bp 1 3 + Write.bp Read.bp 1 3 SST ) set_tests_properties(HeatTransfer.SST.FFS.MxN PROPERTIES PROCESSORS 7) diff --git a/testing/h5vol/TestH5VolWriteReadBPFile.cpp b/testing/h5vol/TestH5VolWriteReadBPFile.cpp index 59008ea761..5aefd76c9c 100644 --- a/testing/h5vol/TestH5VolWriteReadBPFile.cpp +++ b/testing/h5vol/TestH5VolWriteReadBPFile.cpp @@ -1115,7 +1115,8 @@ TEST_F(H5VolWriteReadTest, H5VolWriteHDF5Read2D4x2) int main(int argc, char **argv) { #ifdef TEST_HDF5_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); #endif int result; diff --git a/testing/install/C/main_mpi.c b/testing/install/C/main_mpi.c index 9341094fe9..7e7479ed96 100644 --- a/testing/install/C/main_mpi.c +++ b/testing/install/C/main_mpi.c @@ -15,7 +15,8 @@ int main(int argc, char **argv) { - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); adios2_adios *adios = adios2_init_mpi(MPI_COMM_WORLD); if (!adios) diff --git a/testing/install/CXX11/main_mpi.cxx b/testing/install/CXX11/main_mpi.cxx index 417954fffc..69a735c11c 100644 --- a/testing/install/CXX11/main_mpi.cxx +++ b/testing/install/CXX11/main_mpi.cxx @@ -13,7 +13,8 @@ int main(int argc, char **argv) { - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); adios2::ADIOS adios(MPI_COMM_WORLD); diff --git a/testing/utils/changingshape/TestUtilsChangingShape.cpp b/testing/utils/changingshape/TestUtilsChangingShape.cpp index 23c893986f..aab4faf00d 100644 --- a/testing/utils/changingshape/TestUtilsChangingShape.cpp +++ b/testing/utils/changingshape/TestUtilsChangingShape.cpp @@ -31,7 +31,8 @@ int main(int argc, char **argv) int rank = 0, nproc = 1; #if ADIOS2_USE_MPI - MPI_Init(nullptr, nullptr); + int provided; + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nproc); adios2::ADIOS adios(MPI_COMM_WORLD); diff --git a/testing/utils/cwriter/TestUtilsCWriter.c b/testing/utils/cwriter/TestUtilsCWriter.c index fc1779f2d1..9431f964c1 100644 --- a/testing/utils/cwriter/TestUtilsCWriter.c +++ b/testing/utils/cwriter/TestUtilsCWriter.c @@ -24,7 +24,8 @@ int main(int argc, char *argv[]) int rank = 0; int nproc = 1; #if ADIOS2_USE_MPI - MPI_Init(&argc, &argv); + int provided; + MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nproc); adios2_adios *adiosH = adios2_init_mpi(MPI_COMM_WORLD);