Merge pull request #2408 from JasonRuonanWang/ssc-mpi-join

refined mpi handshake in preparation for adding mpi join for ssc
ornladios · Aug 4, 2020 · bce597a · bce597a
2 parents 5607d31 + c058098
commit bce597a
Show file tree

Hide file tree

Showing 4 changed files with 50 additions and 30 deletions.
diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp
@@ -240,19 +240,14 @@ void SscReader::SyncMpiPattern()
 {
     TAU_SCOPED_TIMER_FUNC();
 
-    auto appRankMaps =
-        helper::Handshake(m_Name, 'r', m_OpenTimeoutSecs, CommAsMPI(m_Comm));
-
-    MPI_Group worldGroup;
     MPI_Group streamGroup;
+    MPI_Group readerGroup;
+    MPI_Comm writerComm;
+    MPI_Comm readerComm;
 
-    MPI_Comm_group(MPI_COMM_WORLD, &worldGroup);
-    MPI_Group_incl(worldGroup, appRankMaps[1].size(), appRankMaps[1].data(),
-                   &m_MpiAllWritersGroup);
-    MPI_Group_incl(worldGroup, appRankMaps[0].size(), appRankMaps[0].data(),
-                   &streamGroup);
-
-    MPI_Comm_create_group(MPI_COMM_WORLD, streamGroup, 0, &m_StreamComm);
+    helper::HandshakeComm(m_Name, 'r', m_OpenTimeoutSecs, CommAsMPI(m_Comm),
+                          streamGroup, m_MpiAllWritersGroup, readerGroup,
+                          m_StreamComm, writerComm, readerComm);
 }
 
 bool SscReader::SyncWritePattern()

diff --git a/source/adios2/engine/ssc/SscWriter.cpp b/source/adios2/engine/ssc/SscWriter.cpp
@@ -186,19 +186,14 @@ void SscWriter::SyncMpiPattern()
 {
     TAU_SCOPED_TIMER_FUNC();
 
-    auto appRankMaps =
-        helper::Handshake(m_Name, 'w', m_OpenTimeoutSecs, CommAsMPI(m_Comm));
-
-    MPI_Group worldGroup;
     MPI_Group streamGroup;
+    MPI_Group writerGroup;
+    MPI_Comm writerComm;
+    MPI_Comm readerComm;
 
-    MPI_Comm_group(MPI_COMM_WORLD, &worldGroup);
-    MPI_Group_incl(worldGroup, appRankMaps[2].size(), appRankMaps[2].data(),
-                   &m_MpiAllReadersGroup);
-    MPI_Group_incl(worldGroup, appRankMaps[0].size(), appRankMaps[0].data(),
-                   &streamGroup);
-
-    MPI_Comm_create_group(MPI_COMM_WORLD, streamGroup, 0, &m_StreamComm);
+    helper::HandshakeComm(m_Name, 'w', m_OpenTimeoutSecs, CommAsMPI(m_Comm),
+                          streamGroup, writerGroup, m_MpiAllReadersGroup,
+                          m_StreamComm, writerComm, readerComm);
 }
 
 void SscWriter::SyncWritePattern(bool finalStep)

diff --git a/source/adios2/helper/adiosMpiHandshake.cpp b/source/adios2/helper/adiosMpiHandshake.cpp
@@ -21,10 +21,32 @@ namespace adios2
 namespace helper
 {
 
-const std::vector<std::vector<int>> Handshake(const std::string &filename,
-                                              const char mode,
-                                              const int timeoutSeconds,
-                                              MPI_Comm localComm)
+#ifndef _WIN32
+void HandshakeComm(const std::string &filename, const char mode,
+                   const int timeoutSeconds, MPI_Comm localComm,
+                   MPI_Group &streamGroup, MPI_Group &writerGroup,
+                   MPI_Group &readerGroup, MPI_Comm &streamComm,
+                   MPI_Comm &writerComm, MPI_Comm &readerComm)
+{
+    auto appRankMaps = HandshakeRank(filename, mode, timeoutSeconds, localComm);
+    MPI_Group worldGroup;
+    MPI_Comm_group(MPI_COMM_WORLD, &worldGroup);
+    MPI_Group_incl(worldGroup, static_cast<int>(appRankMaps[0].size()),
+                   appRankMaps[0].data(), &streamGroup);
+    MPI_Group_incl(worldGroup, static_cast<int>(appRankMaps[1].size()),
+                   appRankMaps[1].data(), &writerGroup);
+    MPI_Group_incl(worldGroup, static_cast<int>(appRankMaps[2].size()),
+                   appRankMaps[2].data(), &readerGroup);
+    MPI_Comm_create_group(MPI_COMM_WORLD, streamGroup, 0, &streamComm);
+    MPI_Comm_create_group(MPI_COMM_WORLD, writerGroup, 0, &writerComm);
+    MPI_Comm_create_group(MPI_COMM_WORLD, readerGroup, 0, &readerComm);
+}
+#endif
+
+const std::vector<std::vector<int>> HandshakeRank(const std::string &filename,
+                                                  const char mode,
+                                                  const int timeoutSeconds,
+                                                  MPI_Comm localComm)
 {
     std::vector<std::vector<int>> ret(3);
 

diff --git a/source/adios2/helper/adiosMpiHandshake.h b/source/adios2/helper/adiosMpiHandshake.h
@@ -48,10 +48,18 @@ namespace helper
  * for stream *filename*. [1] is the vector of all writer ranks for stream
  * *filename*. [2] is the vector of all reader ranks for stream *filename*.
  */
-const std::vector<std::vector<int>> Handshake(const std::string &filename,
-                                              const char mode,
-                                              const int timeoutSeconds,
-                                              MPI_Comm localComm);
+const std::vector<std::vector<int>> HandshakeRank(const std::string &filename,
+                                                  const char mode,
+                                                  const int timeoutSeconds,
+                                                  MPI_Comm localComm);
+
+#ifndef _WIN32
+void HandshakeComm(const std::string &filename, const char mode,
+                   const int timeoutSeconds, MPI_Comm localComm,
+                   MPI_Group &streamGroup, MPI_Group &writerGroup,
+                   MPI_Group &readerGroup, MPI_Comm &streamComm,
+                   MPI_Comm &writerComm, MPI_Comm &readerComm);
+#endif
 
 } // end namespace helper
 } // end namespace adios2