From b0e35043196baa993e6d67d06812a5edfbf58e8a Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Wed, 14 Aug 2019 11:48:58 -0400 Subject: [PATCH 1/7] start refining dataman --- .../adios2/engine/dataman/DataManCommon.cpp | 27 ++------ source/adios2/engine/dataman/DataManCommon.h | 19 ++---- .../adios2/engine/dataman/DataManReader.cpp | 32 +++------- source/adios2/engine/dataman/DataManReader.h | 3 +- .../adios2/engine/dataman/DataManWriter.cpp | 62 +++---------------- source/adios2/engine/dataman/DataManWriter.h | 2 - .../adios2/engine/dataman/DataManWriter.tcc | 50 +++++---------- source/adios2/engine/ssc/SscReader.cpp | 2 +- source/adios2/engine/ssc/SscWriter.cpp | 3 +- source/adios2/engine/table/TableWriter.cpp | 7 ++- .../format/dataman/DataManSerializer.cpp | 5 +- .../format/dataman/DataManSerializer.h | 3 +- .../toolkit/transportman/wanman/WANMan.cpp | 57 +++++------------ .../toolkit/transportman/wanman/WANMan.h | 8 +-- 14 files changed, 74 insertions(+), 206 deletions(-) diff --git a/source/adios2/engine/dataman/DataManCommon.cpp b/source/adios2/engine/dataman/DataManCommon.cpp index a0227e965f..258ed7adb1 100644 --- a/source/adios2/engine/dataman/DataManCommon.cpp +++ b/source/adios2/engine/dataman/DataManCommon.cpp @@ -20,35 +20,20 @@ namespace engine DataManCommon::DataManCommon(const std::string engineType, IO &io, const std::string &name, const Mode mode, MPI_Comm mpiComm) -: Engine(engineType, io, name, mode, mpiComm), - m_FileTransport(mpiComm, m_DebugMode) +: Engine(engineType, io, name, mode, mpiComm), m_WANMan(m_MPIComm, m_DebugMode), + m_IsRowMajor(helper::IsRowMajor(io.m_HostLanguage)), + m_DataManSerializer(mpiComm, m_IsRowMajor) { // initialize parameters MPI_Comm_rank(mpiComm, &m_MpiRank); MPI_Comm_size(mpiComm, &m_MpiSize); - m_IsLittleEndian = helper::IsLittleEndian(); - m_IsRowMajor = helper::IsRowMajor(io.m_HostLanguage); - GetStringParameter(m_IO.m_Parameters, "WorkflowMode", m_WorkflowMode); GetBoolParameter(m_IO.m_Parameters, "AlwaysProvideLatestTimestep", m_ProvideLatest); - if (m_WorkflowMode != "file" && m_WorkflowMode != "stream") + if (m_IO.m_TransportsParameters.empty()) { - throw(std::invalid_argument( - "WorkflowMode parameter for DataMan must be File or Stream")); - } - m_Channels = m_IO.m_TransportsParameters.size(); - if (m_Channels == 0) - { - m_Channels = 1; - m_IO.m_TransportsParameters.push_back({{"Library", "ZMQ"}, - {"IPAddress", "127.0.0.1"}, - {"Port", "12306"}, - {"Name", m_Name}}); - } - for (size_t i = 0; i < m_Channels; ++i) - { - m_IO.m_TransportsParameters[i]["Name"] = m_Name + std::to_string(i); + m_IO.m_TransportsParameters.push_back( + {{"IPAddress", "127.0.0.1"}, {"Port", "12306"}}); } } diff --git a/source/adios2/engine/dataman/DataManCommon.h b/source/adios2/engine/dataman/DataManCommon.h index 9f31bf0b74..efe911de45 100644 --- a/source/adios2/engine/dataman/DataManCommon.h +++ b/source/adios2/engine/dataman/DataManCommon.h @@ -37,27 +37,20 @@ class DataManCommon : public Engine virtual ~DataManCommon() = default; protected: + int m_Verbosity = 0; + int m_MpiRank; int m_MpiSize; - int m_Channels; - std::string m_WorkflowMode = "stream"; bool m_ProvideLatest = false; - size_t m_BufferSize = 1024 * 1024 * 1024; - bool m_DoMonitor = false; + size_t m_BufferSize = 128 * 1024 * 1024; int64_t m_CurrentStep = -1; - bool m_IsLittleEndian; bool m_IsRowMajor; bool m_ContiguousMajor = true; - int m_Verbosity = 0; - - transport::FileFStream m_FileTransport; - - std::vector m_StreamNames; - - std::shared_ptr m_WANMan; - std::shared_ptr m_DataThread; + format::DataManSerializer m_DataManSerializer; + transportman::WANMan m_WANMan; + std::thread m_DataThread; bool GetStringParameter(Params ¶ms, std::string key, std::string &value); diff --git a/source/adios2/engine/dataman/DataManReader.cpp b/source/adios2/engine/dataman/DataManReader.cpp index 8d3cb2c746..788e49cdfe 100644 --- a/source/adios2/engine/dataman/DataManReader.cpp +++ b/source/adios2/engine/dataman/DataManReader.cpp @@ -24,8 +24,7 @@ namespace engine DataManReader::DataManReader(IO &io, const std::string &name, const Mode mode, MPI_Comm mpiComm) -: DataManCommon("DataManReader", io, name, mode, mpiComm), - m_DataManSerializer(mpiComm, 0, m_IsRowMajor) +: DataManCommon("DataManReader", io, name, mode, mpiComm) { m_EndMessage = " in call to IO Open DataManReader " + m_Name + "\n"; Init(); @@ -167,28 +166,20 @@ void DataManReader::Flush(const int transportIndex) {} void DataManReader::Init() { - if (m_WorkflowMode == "file") - { - m_FileTransport.Open(m_Name, Mode::Read); - return; - } // initialize transports - m_WANMan = std::make_shared(m_MPIComm, m_DebugMode); - m_WANMan->OpenTransports(m_IO.m_TransportsParameters, Mode::Read, - m_WorkflowMode, true); + m_WANMan.OpenTransports(m_IO.m_TransportsParameters, Mode::Read, true); // start threads m_Listening = true; - m_DataThread = - std::make_shared(&DataManReader::IOThread, this, m_WANMan); + m_DataThread = std::thread(&DataManReader::IOThread, this); } -void DataManReader::IOThread(std::shared_ptr man) +void DataManReader::IOThread() { while (m_Listening) { - std::shared_ptr> buffer = man->Read(0); + std::shared_ptr> buffer = m_WANMan.Read(0); if (buffer != nullptr) { int ret = m_DataManSerializer.PutPack(buffer); @@ -224,18 +215,11 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) void DataManReader::DoClose(const int transportIndex) { - if (transportIndex == -1) + m_Listening = false; + if (m_DataThread.joinable()) { - m_Listening = false; - if (m_DataThread != nullptr) - { - if (m_DataThread->joinable()) - { - m_DataThread->join(); - } - } + m_DataThread.join(); } - m_WANMan = nullptr; } } // end namespace engine diff --git a/source/adios2/engine/dataman/DataManReader.h b/source/adios2/engine/dataman/DataManReader.h index 330bfc6f47..54af81afed 100644 --- a/source/adios2/engine/dataman/DataManReader.h +++ b/source/adios2/engine/dataman/DataManReader.h @@ -39,11 +39,10 @@ class DataManReader : public DataManCommon bool m_Listening = false; size_t m_FinalStep = std::numeric_limits::max(); - format::DataManSerializer m_DataManSerializer; format::DmvVecPtrMap m_MetaDataMap; void Init(); - void IOThread(std::shared_ptr man); + void IOThread(); void DoClose(const int transportIndex = -1) final; #define declare_type(T) \ diff --git a/source/adios2/engine/dataman/DataManWriter.cpp b/source/adios2/engine/dataman/DataManWriter.cpp index 0eb1ad94fd..8466f40249 100644 --- a/source/adios2/engine/dataman/DataManWriter.cpp +++ b/source/adios2/engine/dataman/DataManWriter.cpp @@ -40,10 +40,7 @@ StepStatus DataManWriter::BeginStep(StepMode mode, const float timeout_sec) } ++m_CurrentStep; - for (size_t i = 0; i < m_Channels; ++i) - { - m_DataManSerializer[i]->NewWriterBuffer(m_BufferSize); - } + m_DataManSerializer.NewWriterBuffer(m_BufferSize); if (m_Verbosity >= 5) { @@ -60,37 +57,22 @@ void DataManWriter::PerformPuts() {} void DataManWriter::EndStep() { - for (auto &serializer : m_DataManSerializer) - { - serializer->PutAttributes(m_IO); - } + m_DataManSerializer.PutAttributes(m_IO); if (m_CurrentStep == 0) { - m_DataManSerializer[0]->AggregateMetadata(); + m_DataManSerializer.AggregateMetadata(); m_AggregatedMetadataMutex.lock(); int64_t stepProvided; m_AggregatedMetadata = - m_DataManSerializer[0]->GetAggregatedMetadataPack(0, stepProvided, - -1); + m_DataManSerializer.GetAggregatedMetadataPack(0, stepProvided, -1); m_AggregatedMetadataMutex.unlock(); } - if (m_WorkflowMode == "file") - { - const auto buf = m_DataManSerializer[0]->GetLocalPack(); - m_FileTransport.Write(buf->data(), buf->size()); - } - else if (m_WorkflowMode == "stream") - { - for (size_t i = 0; i < m_Channels; ++i) - { - m_DataManSerializer[i]->AttachAttributes(); - const auto buf = m_DataManSerializer[i]->GetLocalPack(); - m_BufferSize = buf->size(); - m_WANMan->Write(buf, i); - } - } + m_DataManSerializer.AttachAttributes(); + const auto buf = m_DataManSerializer.GetLocalPack(); + m_BufferSize = buf->size(); + m_WANMan.Write(buf); } void DataManWriter::Flush(const int transportIndex) {} @@ -99,25 +81,7 @@ void DataManWriter::Flush(const int transportIndex) {} void DataManWriter::Init() { - - if (m_WorkflowMode == "file") - { - m_FileTransport.Open(m_Name, Mode::Write); - return; - } - - // initialize transports - m_WANMan = std::make_shared(m_MPIComm, m_DebugMode); - m_WANMan->OpenTransports(m_IO.m_TransportsParameters, Mode::Write, - m_WorkflowMode, true); - - // initialize serializer - for (size_t i = 0; i < m_Channels; ++i) - { - m_DataManSerializer.push_back( - std::make_shared(m_MPIComm, m_BufferSize, - m_IsRowMajor)); - } + m_WANMan.OpenTransports(m_IO.m_TransportsParameters, Mode::Write, true); } #define declare_type(T) \ @@ -134,13 +98,7 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) void DataManWriter::DoClose(const int transportIndex) { - if (m_WorkflowMode == "file") - { - m_FileTransport.Close(); - return; - } - - m_WANMan->Write(format::DataManSerializer::EndSignal(CurrentStep()), 0); + m_WANMan.Write(format::DataManSerializer::EndSignal(CurrentStep()), 0); } void DataManWriter::MetadataThread(const std::string &address) diff --git a/source/adios2/engine/dataman/DataManWriter.h b/source/adios2/engine/dataman/DataManWriter.h index 2eb88a08cd..e0d9100deb 100644 --- a/source/adios2/engine/dataman/DataManWriter.h +++ b/source/adios2/engine/dataman/DataManWriter.h @@ -46,8 +46,6 @@ class DataManWriter : public DataManCommon int m_Port = 12307; std::vector m_FullAddresses; - std::vector> m_DataManSerializer; - void Init(); void MetadataThread(const std::string &address); std::thread m_MetadataThread; diff --git a/source/adios2/engine/dataman/DataManWriter.tcc b/source/adios2/engine/dataman/DataManWriter.tcc index 066c17d477..712515821a 100644 --- a/source/adios2/engine/dataman/DataManWriter.tcc +++ b/source/adios2/engine/dataman/DataManWriter.tcc @@ -53,44 +53,24 @@ void DataManWriter::PutDeferredCommon(Variable &variable, const T *values) if (m_IsRowMajor) { - for (size_t i = 0; i < m_Channels; ++i) - { - m_DataManSerializer[i]->PutVar(variable, m_Name, CurrentStep(), - m_MpiRank, "", - m_IO.m_TransportsParameters[i]); - } + m_DataManSerializer.PutVar(variable, m_Name, CurrentStep(), m_MpiRank, + "", Params()); } else { - if (m_ContiguousMajor) - { - Dims start = variable.m_Start; - Dims count = variable.m_Count; - Dims shape = variable.m_Shape; - Dims memstart = variable.m_MemoryStart; - Dims memcount = variable.m_MemoryCount; - std::reverse(start.begin(), start.end()); - std::reverse(count.begin(), count.end()); - std::reverse(shape.begin(), shape.end()); - std::reverse(memstart.begin(), memstart.end()); - std::reverse(memcount.begin(), memcount.end()); - for (size_t i = 0; i < m_Channels; ++i) - { - m_DataManSerializer[i]->PutVar( - variable.m_Data, variable.m_Name, shape, start, count, - memstart, memcount, m_Name, CurrentStep(), m_MpiRank, "", - m_IO.m_TransportsParameters[i]); - } - } - else - { - for (size_t i = 0; i < m_Channels; ++i) - { - m_DataManSerializer[i]->PutVar(variable, m_Name, CurrentStep(), - m_MpiRank, "", - m_IO.m_TransportsParameters[i]); - } - } + Dims start = variable.m_Start; + Dims count = variable.m_Count; + Dims shape = variable.m_Shape; + Dims memstart = variable.m_MemoryStart; + Dims memcount = variable.m_MemoryCount; + std::reverse(start.begin(), start.end()); + std::reverse(count.begin(), count.end()); + std::reverse(shape.begin(), shape.end()); + std::reverse(memstart.begin(), memstart.end()); + std::reverse(memcount.begin(), memcount.end()); + m_DataManSerializer.PutVar(variable.m_Data, variable.m_Name, shape, + start, count, memstart, memcount, m_Name, + CurrentStep(), m_MpiRank, "", Params()); } } diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp index 6b94047123..1280d50735 100644 --- a/source/adios2/engine/ssc/SscReader.cpp +++ b/source/adios2/engine/ssc/SscReader.cpp @@ -28,7 +28,7 @@ namespace engine SscReader::SscReader(IO &io, const std::string &name, const Mode mode, MPI_Comm mpiComm) : Engine("SscReader", io, name, mode, mpiComm), - m_DataManSerializer(mpiComm, 0, helper::IsRowMajor(io.m_HostLanguage)), + m_DataManSerializer(mpiComm, helper::IsRowMajor(io.m_HostLanguage)), m_RepliedMetadata(std::make_shared>()) { TAU_SCOPED_TIMER_FUNC(); diff --git a/source/adios2/engine/ssc/SscWriter.cpp b/source/adios2/engine/ssc/SscWriter.cpp index ff235c033b..f0b1b6b3a0 100644 --- a/source/adios2/engine/ssc/SscWriter.cpp +++ b/source/adios2/engine/ssc/SscWriter.cpp @@ -28,8 +28,7 @@ namespace engine SscWriter::SscWriter(IO &io, const std::string &name, const Mode mode, MPI_Comm mpiComm) : Engine("SscWriter", io, name, mode, mpiComm), - m_DataManSerializer(mpiComm, m_DefaultBufferSize, - helper::IsRowMajor(io.m_HostLanguage)) + m_DataManSerializer(mpiComm, helper::IsRowMajor(io.m_HostLanguage)) { TAU_SCOPED_TIMER_FUNC(); Init(); diff --git a/source/adios2/engine/table/TableWriter.cpp b/source/adios2/engine/table/TableWriter.cpp index d66bcf3de3..45446e8a7b 100644 --- a/source/adios2/engine/table/TableWriter.cpp +++ b/source/adios2/engine/table/TableWriter.cpp @@ -27,7 +27,7 @@ TableWriter::TableWriter(IO &io, const std::string &name, const Mode mode, MPI_Comm mpiComm) : Engine("TableWriter", io, name, mode, mpiComm), m_IsRowMajor(helper::IsRowMajor(m_IO.m_HostLanguage)), - m_Deserializer(m_MPIComm, 0, m_IsRowMajor), + m_Deserializer(m_MPIComm, m_IsRowMajor), m_SendStagingMan(mpiComm, Mode::Read, m_Timeout, 128), m_SubAdios(MPI_COMM_WORLD, adios2::DebugOFF), m_SubIO(m_SubAdios.DeclareIO("SubIO")) @@ -228,8 +228,9 @@ void TableWriter::InitTransports() for (int i = 0; i < m_Aggregators; ++i) { - auto s = std::make_shared( - m_MPIComm, m_BufferSize, m_IsRowMajor); + auto s = std::make_shared(m_MPIComm, + m_IsRowMajor); + s->NewWriterBuffer(m_BufferSize); s->SetDestination(m_AllAddresses[i]); m_Serializers.push_back(s); } diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp index 04249d5cb5..8e97b133e7 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp @@ -21,16 +21,13 @@ namespace adios2 namespace format { -DataManSerializer::DataManSerializer(MPI_Comm mpiComm, - const size_t writerBufferSize, - const bool isRowMajor) +DataManSerializer::DataManSerializer(MPI_Comm mpiComm, const bool isRowMajor) : m_MpiComm(mpiComm), m_IsRowMajor(isRowMajor), m_IsLittleEndian(helper::IsLittleEndian()), m_DeferredRequestsToSend(std::make_shared()) { MPI_Comm_size(m_MpiComm, &m_MpiSize); MPI_Comm_rank(m_MpiComm, &m_MpiRank); - NewWriterBuffer(writerBufferSize); } void DataManSerializer::NewWriterBuffer(size_t bufferSize) diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.h b/source/adios2/toolkit/format/dataman/DataManSerializer.h index d4c8fe967f..de79d03754 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.h +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.h @@ -83,8 +83,7 @@ using DeferredRequestMapPtr = std::shared_ptr; class DataManSerializer { public: - DataManSerializer(MPI_Comm mpiComm, const size_t writerBufferSize, - const bool isRowMajor); + DataManSerializer(MPI_Comm mpiComm, const bool isRowMajor); // clear and allocate new buffer for writer void NewWriterBuffer(size_t size); diff --git a/source/adios2/toolkit/transportman/wanman/WANMan.cpp b/source/adios2/toolkit/transportman/wanman/WANMan.cpp index 48ddce5cc7..025a2044b6 100644 --- a/source/adios2/toolkit/transportman/wanman/WANMan.cpp +++ b/source/adios2/toolkit/transportman/wanman/WANMan.cpp @@ -76,8 +76,7 @@ WANMan::~WANMan() void WANMan::SetMaxReceiveBuffer(size_t size) { m_MaxReceiveBuffer = size; } void WANMan::OpenTransports(const std::vector ¶msVector, - const Mode mode, const std::string &workflowMode, - const bool profile) + const Mode mode, const bool profile) { m_TransportsParameters = paramsVector; m_BufferQueue.resize(paramsVector.size()); @@ -85,15 +84,11 @@ void WANMan::OpenTransports(const std::vector ¶msVector, for (size_t i = 0; i < paramsVector.size(); ++i) { // Get parameters - std::string library; - GetStringParameter(paramsVector[i], "Library", library); std::string ip; GetStringParameter(paramsVector[i], "IPAddress", ip); std::string port; GetStringParameter(paramsVector[i], "Port", port); GetIntParameter(paramsVector[i], "Timeout", m_Timeout); - std::string name; - GetStringParameter(paramsVector[i], "Name", name); // Calculate port number int mpiRank, mpiSize; @@ -105,45 +100,25 @@ void WANMan::OpenTransports(const std::vector ¶msVector, } port = std::to_string(stoi(port) + mpiRank); - if (library == "zmq" || library == "ZMQ") - { -#ifdef ADIOS2_HAVE_ZEROMQ - std::shared_ptr wanTransport; - wanTransport = - std::make_shared(m_Timeout); + std::shared_ptr wanTransport; + wanTransport = std::make_shared(m_Timeout); - std::string fullIP = "tcp://" + ip + ":" + port; - wanTransport->Open(fullIP, mode); - m_Transports.emplace(i, wanTransport); + std::string fullIP = "tcp://" + ip + ":" + port; + wanTransport->Open(fullIP, mode); + m_Transports.emplace(i, wanTransport); - // launch thread - if (mode == Mode::Read) - { - m_Reading = true; - m_ReadThreads.emplace_back( - std::thread(&WANMan::ReadThread, this, wanTransport)); - } - else if (mode == Mode::Write) - { - m_Writing = true; - m_WriteThreads.emplace_back( - std::thread(&WANMan::WriteThread, this, wanTransport, i)); - } -#else - throw std::invalid_argument( - "ERROR: this version of ADIOS2 didn't compile with " - "ZMQ library, in call to Open\n"); -#endif + // launch thread + if (mode == Mode::Read) + { + m_Reading = true; + m_ReadThreads.emplace_back( + std::thread(&WANMan::ReadThread, this, wanTransport)); } - else + else if (mode == Mode::Write) { - if (m_DebugMode) - { - throw std::invalid_argument("ERROR: wan transport " + library + - " not supported or not " - "provided in IO AddTransport, " - "in call to Open\n"); - } + m_Writing = true; + m_WriteThreads.emplace_back( + std::thread(&WANMan::WriteThread, this, wanTransport, i)); } } } diff --git a/source/adios2/toolkit/transportman/wanman/WANMan.h b/source/adios2/toolkit/transportman/wanman/WANMan.h index ddef5ad9d9..9b1645b3eb 100644 --- a/source/adios2/toolkit/transportman/wanman/WANMan.h +++ b/source/adios2/toolkit/transportman/wanman/WANMan.h @@ -33,11 +33,11 @@ class WANMan ~WANMan(); void OpenTransports(const std::vector ¶metersVector, - const Mode openMode, const std::string &workflowMode, - const bool profile); + const Mode openMode, const bool profile); - void Write(const std::vector &buffer, size_t transportId); - void Write(std::shared_ptr> buffer, size_t transportId); + void Write(const std::vector &buffer, size_t transportId = 0); + void Write(std::shared_ptr> buffer, + size_t transportId = 0); std::shared_ptr> Read(size_t id); From 93cbdce527bdf9ade8f178f38fa50bbda7b58107 Mon Sep 17 00:00:00 2001 From: Ruonan Wang Date: Mon, 19 Aug 2019 01:28:16 -0400 Subject: [PATCH 2/7] finished refining dataman --- source/adios2/CMakeLists.txt | 3 - .../adios2/engine/dataman/DataManCommon.cpp | 39 +-- source/adios2/engine/dataman/DataManCommon.h | 28 +- .../adios2/engine/dataman/DataManReader.cpp | 133 ++++++---- source/adios2/engine/dataman/DataManReader.h | 13 +- .../adios2/engine/dataman/DataManReader.tcc | 47 ++-- .../adios2/engine/dataman/DataManWriter.cpp | 120 ++++++--- source/adios2/engine/dataman/DataManWriter.h | 19 +- .../adios2/engine/dataman/DataManWriter.tcc | 13 - source/adios2/engine/ssc/SscReader.cpp | 109 ++++---- source/adios2/engine/ssc/SscReader.h | 9 +- source/adios2/engine/ssc/SscWriter.cpp | 6 +- source/adios2/engine/ssc/SscWriter.h | 3 +- source/adios2/engine/table/TableWriter.cpp | 23 +- source/adios2/engine/table/TableWriter.h | 3 +- source/adios2/engine/table/TableWriter.tcc | 9 +- .../format/dataman/DataManSerializer.cpp | 11 - .../format/dataman/DataManSerializer.h | 2 - .../toolkit/transport/socket/SocketZmq.cpp | 39 --- .../toolkit/transport/socket/SocketZmq.h | 41 --- .../transport/socket/SocketZmqPubSub.cpp | 94 ------- .../transport/socket/SocketZmqPubSub.h | 36 --- .../transport/socket/SocketZmqReqRep.cpp | 102 -------- .../transport/socket/SocketZmqReqRep.h | 36 --- .../transportman/stagingman/StagingMan.cpp | 125 ++++++--- .../transportman/stagingman/StagingMan.h | 37 +-- .../toolkit/transportman/wanman/WANMan.cpp | 247 ++++++------------ .../toolkit/transportman/wanman/WANMan.h | 57 ++-- testing/adios2/engine/dataman/TestDataMan.h | 66 ++--- .../engine/dataman/TestDataManP2P1D.cpp | 15 +- .../dataman/TestDataManP2P2DMemSelect.cpp | 29 +- .../dataman/TestDataManP2P3DMemSelect.cpp | 29 +- .../engine/dataman/TestDataManP2PBzip2D.cpp | 17 +- .../engine/dataman/TestDataManP2PSz2D.cpp | 19 +- .../engine/dataman/TestDataManP2PZfp2D.cpp | 17 +- .../engine/dataman/TestDataManSubscribe1D.cpp | 11 +- 36 files changed, 570 insertions(+), 1037 deletions(-) delete mode 100644 source/adios2/toolkit/transport/socket/SocketZmq.cpp delete mode 100644 source/adios2/toolkit/transport/socket/SocketZmq.h delete mode 100644 source/adios2/toolkit/transport/socket/SocketZmqPubSub.cpp delete mode 100644 source/adios2/toolkit/transport/socket/SocketZmqPubSub.h delete mode 100644 source/adios2/toolkit/transport/socket/SocketZmqReqRep.cpp delete mode 100644 source/adios2/toolkit/transport/socket/SocketZmqReqRep.h diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 0a7d2e90fd..27c79119cd 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -112,9 +112,6 @@ endif() if(ADIOS2_HAVE_ZeroMQ) target_sources(adios2 PRIVATE - toolkit/transport/socket/SocketZmq.cpp - toolkit/transport/socket/SocketZmqReqRep.cpp - toolkit/transport/socket/SocketZmqPubSub.cpp toolkit/transportman/wanman/WANMan.cpp toolkit/transportman/stagingman/StagingMan.cpp ) diff --git a/source/adios2/engine/dataman/DataManCommon.cpp b/source/adios2/engine/dataman/DataManCommon.cpp index 258ed7adb1..aad0899e5e 100644 --- a/source/adios2/engine/dataman/DataManCommon.cpp +++ b/source/adios2/engine/dataman/DataManCommon.cpp @@ -2,7 +2,7 @@ * Distributed under the OSI-approved Apache License, Version 2.0. See * accompanying file Copyright.txt for details. * - * DataManReader.cpp + * DataManCommon.cpp * * Created on: Feb 12, 2018 * Author: Jason Wang @@ -20,25 +20,22 @@ namespace engine DataManCommon::DataManCommon(const std::string engineType, IO &io, const std::string &name, const Mode mode, MPI_Comm mpiComm) -: Engine(engineType, io, name, mode, mpiComm), m_WANMan(m_MPIComm, m_DebugMode), +: Engine(engineType, io, name, mode, mpiComm), m_IsRowMajor(helper::IsRowMajor(io.m_HostLanguage)), m_DataManSerializer(mpiComm, m_IsRowMajor) { - - // initialize parameters MPI_Comm_rank(mpiComm, &m_MpiRank); MPI_Comm_size(mpiComm, &m_MpiSize); - GetBoolParameter(m_IO.m_Parameters, "AlwaysProvideLatestTimestep", - m_ProvideLatest); - if (m_IO.m_TransportsParameters.empty()) - { - m_IO.m_TransportsParameters.push_back( - {{"IPAddress", "127.0.0.1"}, {"Port", "12306"}}); - } + GetParameter(m_IO.m_Parameters, "IPAddress", m_IPAddress); + GetParameter(m_IO.m_Parameters, "Port", m_Port); + GetParameter(m_IO.m_Parameters, "StagingMode", m_StagingMode); + GetParameter(m_IO.m_Parameters, "Timeout", m_Timeout); } -bool DataManCommon::GetStringParameter(Params ¶ms, std::string key, - std::string &value) +DataManCommon::~DataManCommon() {} + +bool DataManCommon::GetParameter(const Params ¶ms, const std::string &key, + std::string &value) { auto it = params.find(key); if (it != params.end()) @@ -50,8 +47,20 @@ bool DataManCommon::GetStringParameter(Params ¶ms, std::string key, return false; } -bool DataManCommon::GetBoolParameter(Params ¶ms, std::string key, - bool &value) +bool DataManCommon::GetParameter(const Params ¶ms, const std::string &key, + int &value) +{ + auto it = params.find(key); + if (it != params.end()) + { + value = stoi(it->second); + return true; + } + return false; +} + +bool DataManCommon::GetParameter(const Params ¶ms, const std::string &key, + bool &value) { auto it = params.find(key); if (it != params.end()) diff --git a/source/adios2/engine/dataman/DataManCommon.h b/source/adios2/engine/dataman/DataManCommon.h index efe911de45..7dac1ba5c0 100644 --- a/source/adios2/engine/dataman/DataManCommon.h +++ b/source/adios2/engine/dataman/DataManCommon.h @@ -17,7 +17,7 @@ #include "adios2/helper/adiosSystem.h" #include "adios2/toolkit/format/dataman/DataManSerializer.h" #include "adios2/toolkit/format/dataman/DataManSerializer.tcc" -#include "adios2/toolkit/transport/file/FileFStream.h" +#include "adios2/toolkit/transportman/stagingman/StagingMan.h" #include "adios2/toolkit/transportman/wanman/WANMan.h" namespace adios2 @@ -33,28 +33,32 @@ class DataManCommon : public Engine public: DataManCommon(const std::string engineType, IO &io, const std::string &name, const Mode mode, MPI_Comm mpiComm); - - virtual ~DataManCommon() = default; + virtual ~DataManCommon(); protected: + // external paremeters int m_Verbosity = 0; + size_t m_SerializerBufferSize = 128 * 1024 * 1024; + size_t m_ReceiverBufferSize = 128 * 1024 * 1024; + std::string m_StagingMode = "wide"; + int m_Timeout = 5; + // internal variables int m_MpiRank; int m_MpiSize; - bool m_ProvideLatest = false; - size_t m_BufferSize = 128 * 1024 * 1024; int64_t m_CurrentStep = -1; - + bool m_ThreadActive = true; bool m_IsRowMajor; - bool m_ContiguousMajor = true; + std::string m_IPAddress; + int m_Port = 50001; format::DataManSerializer m_DataManSerializer; - transportman::WANMan m_WANMan; - std::thread m_DataThread; - bool GetStringParameter(Params ¶ms, std::string key, - std::string &value); - bool GetBoolParameter(Params ¶ms, std::string key, bool &value); + bool GetParameter(const Params ¶ms, const std::string &key, + bool &value); + bool GetParameter(const Params ¶ms, const std::string &key, int &value); + bool GetParameter(const Params ¶ms, const std::string &key, + std::string &value); }; // end class DataManCommon diff --git a/source/adios2/engine/dataman/DataManReader.cpp b/source/adios2/engine/dataman/DataManReader.cpp index 788e49cdfe..b239d9078c 100644 --- a/source/adios2/engine/dataman/DataManReader.cpp +++ b/source/adios2/engine/dataman/DataManReader.cpp @@ -26,13 +26,55 @@ DataManReader::DataManReader(IO &io, const std::string &name, const Mode mode, MPI_Comm mpiComm) : DataManCommon("DataManReader", io, name, mode, mpiComm) { - m_EndMessage = " in call to IO Open DataManReader " + m_Name + "\n"; - Init(); + GetParameter(m_IO.m_Parameters, "AlwaysProvideLatestTimestep", + m_ProvideLatest); + m_ZmqRequester.OpenRequester(m_Timeout, m_ReceiverBufferSize); + + if (m_StagingMode == "wide") + { + if (m_IPAddress.empty()) + { + throw(std::invalid_argument( + "IP address not specified in wide area staging")); + } + std::string address = + "tcp://" + m_IPAddress + ":" + std::to_string(m_Port); + std::string request = "Address"; + auto reply = + m_ZmqRequester.Request(request.data(), request.size(), address); + while (reply == nullptr or reply->empty()) + { + reply = + m_ZmqRequester.Request(request.data(), request.size(), address); + } + + for (auto i : *reply) + { + std::cout << i; + } + auto addJson = nlohmann::json::parse(*reply); + m_DataAddresses = + addJson["DataAddresses"].get>(); + m_ControlAddresses = + addJson["ControlAddresses"].get>(); + } + else if (m_StagingMode == "local") + { + } + + for (const auto &address : m_DataAddresses) + { + auto dataZmq = std::make_shared(); + dataZmq->OpenSubscriber(address, m_Timeout, m_ReceiverBufferSize); + m_ZmqSubscriberVec.push_back(dataZmq); + } + + m_SubscriberThread = std::thread(&DataManReader::SubscriberThread, this); } DataManReader::~DataManReader() { - if (m_IsClosed == false) + if (not m_IsClosed) { DoClose(); } @@ -57,50 +99,45 @@ StepStatus DataManReader::BeginStep(StepMode stepMode, while (vars == nullptr) { - auto now_time = std::chrono::system_clock::now(); - auto duration = std::chrono::duration_cast( - now_time - start_time); - // timeout == std::numeric_limits::max() means there is no - // timeout, and it should block - // forever until it receives something. - if (timeoutSeconds >= 0.0) - { - if (duration.count() > timeoutSeconds) - { - return StepStatus::NotReady; - } - } - m_MetaDataMap = m_DataManSerializer.GetMetaData(); - if (!m_ProvideLatest) + if (m_ProvideLatest) { - size_t minStep = std::numeric_limits::max(); - ; + size_t maxStep = 0; for (const auto &i : m_MetaDataMap) { - if (minStep > i.first) + if (maxStep < i.first) { - minStep = i.first; + maxStep = i.first; } } - m_CurrentStep = minStep; + m_CurrentStep = maxStep; } else { - size_t maxStep = 0; + size_t minStep = std::numeric_limits::max(); for (const auto &i : m_MetaDataMap) { - if (maxStep < i.first) + if (minStep > i.first) { - maxStep = i.first; + minStep = i.first; } } - m_CurrentStep = maxStep; + m_CurrentStep = minStep; } auto currentStepIt = m_MetaDataMap.find(m_CurrentStep); - if (currentStepIt != m_MetaDataMap.end()) + if (currentStepIt == m_MetaDataMap.end()) + { + auto now_time = std::chrono::system_clock::now(); + auto duration = std::chrono::duration_cast( + now_time - start_time); + if (duration.count() > timeoutSeconds) + { + return StepStatus::NotReady; + } + } + else { vars = currentStepIt->second; } @@ -146,12 +183,6 @@ void DataManReader::PerformGets() {} void DataManReader::EndStep() { - - if (m_Verbosity >= 5) - { - std::cout << "DataManReader::EndStep() start. Current step " - << m_CurrentStep << std::endl; - } m_DataManSerializer.Erase(m_CurrentStep); if (m_Verbosity >= 5) { @@ -164,28 +195,20 @@ void DataManReader::Flush(const int transportIndex) {} // PRIVATE -void DataManReader::Init() +void DataManReader::SubscriberThread() { - - // initialize transports - m_WANMan.OpenTransports(m_IO.m_TransportsParameters, Mode::Read, true); - - // start threads - m_Listening = true; - m_DataThread = std::thread(&DataManReader::IOThread, this); -} - -void DataManReader::IOThread() -{ - while (m_Listening) + while (m_ThreadActive) { - std::shared_ptr> buffer = m_WANMan.Read(0); - if (buffer != nullptr) + for (auto &z : m_ZmqSubscriberVec) { - int ret = m_DataManSerializer.PutPack(buffer); - if (ret > 0) + auto buffer = z->PopBufferQueue(); + if (buffer != nullptr && buffer->size() > 0) { - m_FinalStep = ret; + int ret = m_DataManSerializer.PutPack(buffer); + if (ret > 0) + { + m_FinalStep = ret; + } } } } @@ -215,10 +238,10 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) void DataManReader::DoClose(const int transportIndex) { - m_Listening = false; - if (m_DataThread.joinable()) + m_ThreadActive = false; + if (m_SubscriberThread.joinable()) { - m_DataThread.join(); + m_SubscriberThread.join(); } } diff --git a/source/adios2/engine/dataman/DataManReader.h b/source/adios2/engine/dataman/DataManReader.h index 54af81afed..aba93a7596 100644 --- a/source/adios2/engine/dataman/DataManReader.h +++ b/source/adios2/engine/dataman/DataManReader.h @@ -28,21 +28,24 @@ class DataManReader : public DataManCommon DataManReader(IO &io, const std::string &name, const Mode mode, MPI_Comm mpiComm); virtual ~DataManReader(); - StepStatus BeginStep(StepMode stepMode, - const float timeoutSeconds = -1.0) final; + StepStatus BeginStep(StepMode stepMode, const float timeoutSeconds) final; size_t CurrentStep() const final; void PerformGets() final; void EndStep() final; void Flush(const int transportIndex = -1) final; private: - bool m_Listening = false; + bool m_ProvideLatest = false; size_t m_FinalStep = std::numeric_limits::max(); + transportman::StagingMan m_ZmqRequester; + std::vector m_DataAddresses; + std::vector m_ControlAddresses; + std::vector> m_ZmqSubscriberVec; + std::thread m_SubscriberThread; format::DmvVecPtrMap m_MetaDataMap; - void Init(); - void IOThread(); + void SubscriberThread(); void DoClose(const int transportIndex = -1) final; #define declare_type(T) \ diff --git a/source/adios2/engine/dataman/DataManReader.tcc b/source/adios2/engine/dataman/DataManReader.tcc index 1a22bdde3d..e11deecd61 100644 --- a/source/adios2/engine/dataman/DataManReader.tcc +++ b/source/adios2/engine/dataman/DataManReader.tcc @@ -34,38 +34,35 @@ void DataManReader::GetDeferredCommon(Variable &variable, T *data) { if (m_IsRowMajor) { - while (m_DataManSerializer.GetVar(data, variable.m_Name, - variable.m_Start, variable.m_Count, - m_CurrentStep, variable.m_MemoryStart, - variable.m_MemoryCount) != 0) + while (true) { + int ret = m_DataManSerializer.GetVar( + data, variable.m_Name, variable.m_Start, variable.m_Count, + m_CurrentStep, variable.m_MemoryStart, variable.m_MemoryCount); + if (ret == 0) + { + break; + } } } else { - if (m_ContiguousMajor) - { - Dims start = variable.m_Start; - Dims count = variable.m_Count; - Dims memstart = variable.m_MemoryStart; - Dims memcount = variable.m_MemoryCount; - std::reverse(start.begin(), start.end()); - std::reverse(count.begin(), count.end()); - std::reverse(memstart.begin(), memstart.end()); - std::reverse(memcount.begin(), memcount.end()); - while (m_DataManSerializer.GetVar(data, variable.m_Name, start, - count, m_CurrentStep, memstart, - memcount) != 0) - { - } - } - else + Dims start = variable.m_Start; + Dims count = variable.m_Count; + Dims memstart = variable.m_MemoryStart; + Dims memcount = variable.m_MemoryCount; + std::reverse(start.begin(), start.end()); + std::reverse(count.begin(), count.end()); + std::reverse(memstart.begin(), memstart.end()); + std::reverse(memcount.begin(), memcount.end()); + while (true) { - while (m_DataManSerializer.GetVar( - data, variable.m_Name, variable.m_Start, - variable.m_Count, m_CurrentStep, variable.m_MemoryStart, - variable.m_MemoryCount) != 0) + int ret = + m_DataManSerializer.GetVar(data, variable.m_Name, start, count, + m_CurrentStep, memstart, memcount); + if (ret == 0) { + break; } } } diff --git a/source/adios2/engine/dataman/DataManWriter.cpp b/source/adios2/engine/dataman/DataManWriter.cpp index 8466f40249..0e818ed6a8 100644 --- a/source/adios2/engine/dataman/DataManWriter.cpp +++ b/source/adios2/engine/dataman/DataManWriter.cpp @@ -23,29 +23,75 @@ namespace core namespace engine { -DataManWriter::DataManWriter(IO &io, const std::string &name, const Mode mode, - MPI_Comm mpiComm) -: DataManCommon("DataManWriter", io, name, mode, mpiComm) +DataManWriter::DataManWriter(IO &io, const std::string &name, + const Mode openMode, MPI_Comm mpiComm) +: DataManCommon("DataManWriter", io, name, openMode, mpiComm) { - m_EndMessage = ", in call to Open DataManWriter\n"; - Init(); + if (m_StagingMode == "wide") + { + if (m_IPAddress.empty()) + { + throw(std::invalid_argument( + "IP address not specified in wide area staging")); + } + m_Port += m_MpiRank; + m_ControlAddress = + "tcp://" + m_IPAddress + ":" + std::to_string(m_Port); + m_DataAddress = + "tcp://" + m_IPAddress + ":" + std::to_string(m_Port + m_MpiSize); + + std::vector allDaVec(32 * m_MpiSize, '\0'); + std::vector allCaVec(32 * m_MpiSize, '\0'); + + MPI_Allgather(m_DataAddress.data(), m_DataAddress.size(), MPI_CHAR, + allDaVec.data(), 32, MPI_CHAR, m_MPIComm); + MPI_Allgather(m_ControlAddress.data(), m_ControlAddress.size(), + MPI_CHAR, allCaVec.data(), 32, MPI_CHAR, m_MPIComm); + + std::vector daVec; + std::vector caVec; + + for (int i = 0; i < m_MpiSize; ++i) + { + daVec.push_back(std::string(allDaVec.begin() + i * 32, + allDaVec.begin() + (i + 1) * 32)); + caVec.push_back(std::string(allCaVec.begin() + i * 32, + allCaVec.begin() + (i + 1) * 32)); + } + + nlohmann::json addJson; + addJson["DataAddresses"] = daVec; + addJson["ControlAddresses"] = caVec; + + m_AllAddresses = addJson.dump() + '\0'; + } + else if (m_StagingMode == "local") + { + } + + m_DataPublisher.OpenPublisher(m_DataAddress, m_Timeout); + + m_ControlThread = + std::thread(&DataManWriter::ControlThread, this, m_ControlAddress); } -StepStatus DataManWriter::BeginStep(StepMode mode, const float timeout_sec) +DataManWriter::~DataManWriter() { - if (m_Verbosity >= 5) + if (not m_IsClosed) { - std::cout << "DataManWriter::BeginStep() begin. Last step " - << m_CurrentStep << std::endl; + DoClose(); } - ++m_CurrentStep; +} - m_DataManSerializer.NewWriterBuffer(m_BufferSize); +StepStatus DataManWriter::BeginStep(StepMode mode, const float timeout_sec) +{ + ++m_CurrentStep; + m_DataManSerializer.NewWriterBuffer(m_SerializerBufferSize); if (m_Verbosity >= 5) { - std::cout << "DataManWriter::BeginStep() end. Current step " - << m_CurrentStep << std::endl; + std::cout << "DataManWriter::BeginStep() Rank " << m_MpiRank + << ", Step " << m_CurrentStep << std::endl; } return StepStatus::OK; @@ -57,33 +103,20 @@ void DataManWriter::PerformPuts() {} void DataManWriter::EndStep() { - m_DataManSerializer.PutAttributes(m_IO); - if (m_CurrentStep == 0) { - m_DataManSerializer.AggregateMetadata(); - m_AggregatedMetadataMutex.lock(); - int64_t stepProvided; - m_AggregatedMetadata = - m_DataManSerializer.GetAggregatedMetadataPack(0, stepProvided, -1); - m_AggregatedMetadataMutex.unlock(); + m_DataManSerializer.PutAttributes(m_IO); } - m_DataManSerializer.AttachAttributes(); const auto buf = m_DataManSerializer.GetLocalPack(); - m_BufferSize = buf->size(); - m_WANMan.Write(buf); + m_SerializerBufferSize = buf->size(); + m_DataPublisher.PushBufferQueue(buf); } void DataManWriter::Flush(const int transportIndex) {} // PRIVATE functions below -void DataManWriter::Init() -{ - m_WANMan.OpenTransports(m_IO.m_TransportsParameters, Mode::Write, true); -} - #define declare_type(T) \ void DataManWriter::DoPutSync(Variable &variable, const T *values) \ { \ @@ -98,20 +131,35 @@ ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) void DataManWriter::DoClose(const int transportIndex) { - m_WANMan.Write(format::DataManSerializer::EndSignal(CurrentStep()), 0); + nlohmann::json endSignal; + endSignal["FinalStep"] = m_CurrentStep; + std::string s = endSignal.dump() + '\0'; + auto cvp = std::make_shared>(s.size()); + std::memcpy(cvp->data(), s.c_str(), s.size()); + m_DataPublisher.PushBufferQueue(cvp); + + m_ThreadActive = false; + if (m_ControlThread.joinable()) + { + m_ControlThread.join(); + } } -void DataManWriter::MetadataThread(const std::string &address) +void DataManWriter::ControlThread(const std::string &address) { - transportman::StagingMan tpm(m_MPIComm, Mode::Write, 0, 1e7); - tpm.OpenTransport(address); - while (m_Listening) + transportman::StagingMan tpm; + tpm.OpenReplier(address, m_Timeout, 8192); + while (m_ThreadActive) { auto request = tpm.ReceiveRequest(); if (request && request->size() > 0) { - std::lock_guard lck(m_AggregatedMetadataMutex); - tpm.SendReply(m_AggregatedMetadata); + std::cout << "ControlThread received package ============= \n"; + std::string r(request->begin(), request->end()); + if (r == "Address") + { + tpm.SendReply(m_AllAddresses.data(), m_AllAddresses.size()); + } } } } diff --git a/source/adios2/engine/dataman/DataManWriter.h b/source/adios2/engine/dataman/DataManWriter.h index e0d9100deb..428341f460 100644 --- a/source/adios2/engine/dataman/DataManWriter.h +++ b/source/adios2/engine/dataman/DataManWriter.h @@ -13,7 +13,6 @@ #define ADIOS2_ENGINE_DATAMAN_DATAMAN_WRITER_H_ #include "DataManCommon.h" -#include "adios2/toolkit/transportman/stagingman/StagingMan.h" namespace adios2 { @@ -28,7 +27,7 @@ class DataManWriter : public DataManCommon public: DataManWriter(IO &io, const std::string &name, const Mode mode, MPI_Comm mpiComm); - ~DataManWriter() = default; + virtual ~DataManWriter(); StepStatus BeginStep(StepMode mode, const float timeoutSeconds = -1.0) final; @@ -38,17 +37,17 @@ class DataManWriter : public DataManCommon void Flush(const int transportIndex = -1) final; private: - size_t m_BufferSize = 1024 * 1024 * 1024; - bool m_Listening = true; + std::string m_DataAddress; + std::string m_ControlAddress; + std::string m_AllAddresses; + + transportman::WANMan m_DataPublisher; + format::VecPtr m_AggregatedMetadata = nullptr; std::mutex m_AggregatedMetadataMutex; - int m_AppID = 0; - int m_Port = 12307; - std::vector m_FullAddresses; - void Init(); - void MetadataThread(const std::string &address); - std::thread m_MetadataThread; + void ControlThread(const std::string &address); + std::thread m_ControlThread; #define declare_type(T) \ void DoPutSync(Variable &, const T *) final; \ diff --git a/source/adios2/engine/dataman/DataManWriter.tcc b/source/adios2/engine/dataman/DataManWriter.tcc index 712515821a..26ad3aa0dd 100644 --- a/source/adios2/engine/dataman/DataManWriter.tcc +++ b/source/adios2/engine/dataman/DataManWriter.tcc @@ -38,19 +38,6 @@ void DataManWriter::PutDeferredCommon(Variable &variable, const T *values) variable.SetData(values); - if (variable.m_Shape.empty()) - { - variable.m_Shape = variable.m_Count; - } - if (variable.m_Count.empty()) - { - variable.m_Count = variable.m_Shape; - } - if (variable.m_Start.empty()) - { - variable.m_Start.assign(variable.m_Count.size(), 0); - } - if (m_IsRowMajor) { m_DataManSerializer.PutVar(variable, m_Name, CurrentStep(), m_MpiRank, diff --git a/source/adios2/engine/ssc/SscReader.cpp b/source/adios2/engine/ssc/SscReader.cpp index 1280d50735..cb097e339d 100644 --- a/source/adios2/engine/ssc/SscReader.cpp +++ b/source/adios2/engine/ssc/SscReader.cpp @@ -32,16 +32,50 @@ SscReader::SscReader(IO &io, const std::string &name, const Mode mode, m_RepliedMetadata(std::make_shared>()) { TAU_SCOPED_TIMER_FUNC(); - m_DataTransport = std::make_shared( - mpiComm, Mode::Read, m_Timeout, 1e9); - m_MetadataTransport = std::make_shared( - mpiComm, Mode::Read, m_Timeout, 1e8); - m_EndMessage = " in call to IO Open SscReader " + m_Name + "\n"; + MPI_Comm_rank(mpiComm, &m_MpiRank); - Init(); + srand(time(NULL)); + + // initialize parameters from IO + for (const auto &pair : m_IO.m_Parameters) + { + std::string key(pair.first); + std::transform(key.begin(), key.end(), key.begin(), ::tolower); + std::string value(pair.second); + std::transform(value.begin(), value.end(), value.begin(), ::tolower); + if (key == "verbose") + { + m_Verbosity = std::stoi(value); + } + } + + // retrieve all writer addresses + helper::HandshakeReader(m_MPIComm, m_AppID, m_FullAddresses, m_Name, "ssc"); + + // initialize transports + m_DataTransport.OpenRequester(m_Timeout, m_DataReceiverBufferSize); + m_MetadataTransport.OpenRequester(m_Timeout, m_MetadataReceiverBufferSize); + + // request attributes from writer + auto reply = std::make_shared>(); + if (m_MpiRank == 0) + { + std::vector request(2 * sizeof(int64_t)); + reinterpret_cast(request.data())[0] = m_AppID; + reinterpret_cast(request.data())[1] = -3; + std::string address = m_FullAddresses[rand() % m_FullAddresses.size()]; + while (reply->size() < 6) + { + reply = m_MetadataTransport.Request(request.data(), request.size(), + address); + } + } + m_DataManSerializer.PutAggregatedMetadata(reply, m_MPIComm); + m_DataManSerializer.GetAttributes(m_IO); + if (m_Verbosity >= 5) { - std::cout << "Staging Reader " << m_MpiRank << " Open(" << m_Name + std::cout << "SscReader " << m_MpiRank << " Open(" << m_Name << ") in constructor." << std::endl; } } @@ -54,10 +88,6 @@ SscReader::~SscReader() std::cout << "Staging Reader " << m_MpiRank << " destructor on " << m_Name << "\n"; } - m_MetadataTransport->CloseTransport(); - m_DataTransport->CloseTransport(); - m_DataTransport = nullptr; - m_MetadataTransport = nullptr; } StepStatus SscReader::BeginStepIterator(StepMode stepMode, @@ -224,7 +254,8 @@ void SscReader::PerformGets() for (const auto &i : *requests) { - auto reply = m_DataTransport->Request(*i.second, i.first); + auto reply = m_DataTransport.Request(i.second->data(), i.second->size(), + i.first); if (reply->empty()) { Log(1, @@ -334,57 +365,6 @@ void SscReader::EndStep() ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) #undef declare_type -void SscReader::Init() -{ - TAU_SCOPED_TIMER_FUNC(); - srand(time(NULL)); - InitParameters(); - helper::HandshakeReader(m_MPIComm, m_AppID, m_FullAddresses, m_Name, "ssc"); - - format::VecPtr reply = std::make_shared>(); - if (m_MpiRank == 0) - { - std::vector request(2 * sizeof(int64_t)); - reinterpret_cast(request.data())[0] = m_AppID; - reinterpret_cast(request.data())[1] = -3; - std::string address = m_FullAddresses[rand() % m_FullAddresses.size()]; - while (reply->size() < 6) - { - reply = m_MetadataTransport->Request(request, address); - } - } - m_DataManSerializer.PutAggregatedMetadata(reply, m_MPIComm); - m_DataManSerializer.GetAttributes(m_IO); - - if (m_Verbosity >= 5) - { - for (const auto &i : m_FullAddresses) - { - std::cout << i << std::endl; - } - } -} - -void SscReader::InitParameters() -{ - TAU_SCOPED_TIMER_FUNC(); - for (const auto &pair : m_IO.m_Parameters) - { - std::string key(pair.first); - std::transform(key.begin(), key.end(), key.begin(), ::tolower); - - std::string value(pair.second); - std::transform(value.begin(), value.end(), value.begin(), ::tolower); - - if (key == "verbose") - { - m_Verbosity = std::stoi(value); - } - } -} - -void SscReader::InitTransports() {} - void SscReader::RequestMetadata(const int64_t step) { TAU_SCOPED_TIMER_FUNC(); @@ -395,7 +375,8 @@ void SscReader::RequestMetadata(const int64_t step) reinterpret_cast(request.data())[0] = m_AppID; reinterpret_cast(request.data())[1] = step; std::string address = m_FullAddresses[rand() % m_FullAddresses.size()]; - reply = m_MetadataTransport->Request(request, address); + reply = m_MetadataTransport.Request(request.data(), request.size(), + address); } m_DataManSerializer.PutAggregatedMetadata(reply, m_MPIComm); } diff --git a/source/adios2/engine/ssc/SscReader.h b/source/adios2/engine/ssc/SscReader.h index 42d572a684..f9ff489764 100644 --- a/source/adios2/engine/ssc/SscReader.h +++ b/source/adios2/engine/ssc/SscReader.h @@ -47,8 +47,8 @@ class SscReader : public Engine private: bool m_Tolerance = false; format::DataManSerializer m_DataManSerializer; - std::shared_ptr m_DataTransport; - std::shared_ptr m_MetadataTransport; + transportman::StagingMan m_DataTransport; + transportman::StagingMan m_MetadataTransport; format::DmvVecPtrMap m_MetaDataMap; int64_t m_CurrentStep = -1; int m_MpiRank; @@ -57,6 +57,8 @@ class SscReader : public Engine int m_RetryMax = 128; size_t m_AppID; bool m_ConnectionLost = false; + size_t m_DataReceiverBufferSize = 128 * 1024 * 1024; + size_t m_MetadataReceiverBufferSize = 128 * 1024 * 1024; struct Request { @@ -74,9 +76,6 @@ class SscReader : public Engine void RequestMetadata(const int64_t step = -5); - void Init() final; - void InitParameters() final; - void InitTransports() final; template void CheckIOVariable(const std::string &name, const Dims &shape); diff --git a/source/adios2/engine/ssc/SscWriter.cpp b/source/adios2/engine/ssc/SscWriter.cpp index f0b1b6b3a0..593f8f6fcb 100644 --- a/source/adios2/engine/ssc/SscWriter.cpp +++ b/source/adios2/engine/ssc/SscWriter.cpp @@ -47,7 +47,7 @@ StepStatus SscWriter::BeginStep(StepMode mode, const float timeoutSeconds) if (m_CurrentStep % m_StepsPerAggregation == 0) { m_CurrentStepActive = true; - m_DataManSerializer.NewWriterBuffer(m_DefaultBufferSize); + m_DataManSerializer.NewWriterBuffer(m_SerializationBufferSize); if (not m_AttributesSet) { m_DataManSerializer.PutAttributes(m_IO); @@ -156,8 +156,8 @@ void SscWriter::InitTransports() void SscWriter::ReplyThread(const std::string &address) { - transportman::StagingMan tpm(m_MPIComm, Mode::Write, m_Timeout, 1e9); - tpm.OpenTransport(address); + transportman::StagingMan tpm; + tpm.OpenReplier(address, m_Timeout, m_ReceiverBufferSize); while (m_Listening) { auto request = tpm.ReceiveRequest(); diff --git a/source/adios2/engine/ssc/SscWriter.h b/source/adios2/engine/ssc/SscWriter.h index 08386c5083..93751d3ed2 100644 --- a/source/adios2/engine/ssc/SscWriter.h +++ b/source/adios2/engine/ssc/SscWriter.h @@ -45,7 +45,8 @@ class SscWriter : public Engine private: int m_Channels = 1; - size_t m_DefaultBufferSize = 1024; + size_t m_SerializationBufferSize = 1024; + size_t m_ReceiverBufferSize = 1e8; int m_Port = 12307; int m_MaxRanksPerNode = 200; int m_MaxAppsPerNode = 10; diff --git a/source/adios2/engine/table/TableWriter.cpp b/source/adios2/engine/table/TableWriter.cpp index 45446e8a7b..be151e94a7 100644 --- a/source/adios2/engine/table/TableWriter.cpp +++ b/source/adios2/engine/table/TableWriter.cpp @@ -28,7 +28,6 @@ TableWriter::TableWriter(IO &io, const std::string &name, const Mode mode, : Engine("TableWriter", io, name, mode, mpiComm), m_IsRowMajor(helper::IsRowMajor(m_IO.m_HostLanguage)), m_Deserializer(m_MPIComm, m_IsRowMajor), - m_SendStagingMan(mpiComm, Mode::Read, m_Timeout, 128), m_SubAdios(MPI_COMM_WORLD, adios2::DebugOFF), m_SubIO(m_SubAdios.DeclareIO("SubIO")) { @@ -72,8 +71,8 @@ void TableWriter::EndStep() for (auto serializer : m_Serializers) { auto localPack = serializer->GetLocalPack(); - auto reply = - m_SendStagingMan.Request(*localPack, serializer->GetDestination()); + auto reply = m_SendStagingMan.Request( + localPack->data(), localPack->size(), serializer->GetDestination()); if (m_Verbosity >= 1) { std::cout << "TableWriter::EndStep Rank " << m_MpiRank @@ -102,9 +101,9 @@ void TableWriter::EndStep() void TableWriter::ReplyThread() { - transportman::StagingMan receiveStagingMan(m_MPIComm, Mode::Write, - m_Timeout, 1e9); - receiveStagingMan.OpenTransport(m_AllAddresses[m_MpiRank]); + transportman::StagingMan receiveStagingMan; + receiveStagingMan.OpenReplier(m_AllAddresses[m_MpiRank], m_Timeout, + m_ReceiverBufferSize); while (m_Listening) { auto request = receiveStagingMan.ReceiveRequest(); @@ -159,10 +158,6 @@ void TableWriter::Init() TAU_SCOPED_TIMER_FUNC(); InitParameters(); InitTransports(); - - m_SubIO.SetEngine("bp4"); - m_SubEngine = std::make_shared( - m_SubIO.Open(m_Name, adios2::Mode::Write)); } void TableWriter::InitParameters() @@ -226,17 +221,23 @@ void TableWriter::InitTransports() { TAU_SCOPED_TIMER_FUNC(); + m_SendStagingMan.OpenRequester(m_Timeout, 32); + for (int i = 0; i < m_Aggregators; ++i) { auto s = std::make_shared(m_MPIComm, m_IsRowMajor); - s->NewWriterBuffer(m_BufferSize); + s->NewWriterBuffer(m_SerializerBufferSize); s->SetDestination(m_AllAddresses[i]); m_Serializers.push_back(s); } m_Listening = true; m_ReplyThread = std::thread(&TableWriter::ReplyThread, this); + + m_SubIO.SetEngine("bp4"); + m_SubEngine = std::make_shared( + m_SubIO.Open(m_Name, adios2::Mode::Write)); } void TableWriter::PutAggregatorBuffer() diff --git a/source/adios2/engine/table/TableWriter.h b/source/adios2/engine/table/TableWriter.h index e9ed38b218..e052428584 100644 --- a/source/adios2/engine/table/TableWriter.h +++ b/source/adios2/engine/table/TableWriter.h @@ -54,7 +54,8 @@ class TableWriter : public Engine int m_Port = 6789; int m_MaxRanksPerNode = 200; int m_Aggregators = 10; - size_t m_BufferSize = 1 * 1024 * 1024; + size_t m_SerializerBufferSize = 1 * 1024 * 1024; + size_t m_ReceiverBufferSize = 512 * 1024 * 1024; size_t m_RowsPerAggregatorBuffer = 400; std::unordered_map>> diff --git a/source/adios2/engine/table/TableWriter.tcc b/source/adios2/engine/table/TableWriter.tcc index 400b79816c..9011c6da11 100644 --- a/source/adios2/engine/table/TableWriter.tcc +++ b/source/adios2/engine/table/TableWriter.tcc @@ -65,12 +65,13 @@ void TableWriter::PutDeferredCommon(Variable &variable, const T *data) auto serializer = m_Serializers[i]; serializer->PutVar(variable, m_Name, CurrentStep(), m_MpiRank, "", Params()); - if (serializer->LocalBufferSize() > m_BufferSize / 2) + if (serializer->LocalBufferSize() > m_SerializerBufferSize / 2) { auto localPack = serializer->GetLocalPack(); - auto reply = m_SendStagingMan.Request(*localPack, - serializer->GetDestination()); - serializer->NewWriterBuffer(m_BufferSize); + auto reply = + m_SendStagingMan.Request(localPack->data(), localPack->size(), + serializer->GetDestination()); + serializer->NewWriterBuffer(m_SerializerBufferSize); if (m_Verbosity >= 1) { std::cout << "TableWriter::PutDeferredCommon Rank " << m_MpiRank diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp index 8e97b133e7..8638092568 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp @@ -287,17 +287,6 @@ void DataManSerializer::PutAggregatedMetadata(VecPtr input, MPI_Comm mpiComm) } } -VecPtr DataManSerializer::EndSignal(size_t step) -{ - TAU_SCOPED_TIMER_FUNC(); - nlohmann::json j; - j["FinalStep"] = step; - std::string s = j.dump() + '\0'; - auto c = std::make_shared>(s.size()); - std::memcpy(c->data(), s.c_str(), s.size()); - return c; -} - bool DataManSerializer::IsCompressionAvailable(const std::string &method, const std::string &type, const Dims &count) diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.h b/source/adios2/toolkit/format/dataman/DataManSerializer.h index de79d03754..e1009526be 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.h +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.h @@ -126,8 +126,6 @@ class DataManSerializer int64_t &stepProvided, const int64_t appID); - static VecPtr EndSignal(size_t step); - VecPtr GenerateReply( const std::vector &request, size_t &step, const std::unordered_map &compressionParams); diff --git a/source/adios2/toolkit/transport/socket/SocketZmq.cpp b/source/adios2/toolkit/transport/socket/SocketZmq.cpp deleted file mode 100644 index 1070fe7ed9..0000000000 --- a/source/adios2/toolkit/transport/socket/SocketZmq.cpp +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * SocketZmq.cpp - * - * Created on: Feb 2, 2019 - * Author: Jason Wang wangr1@ornl.gov - */ - -#include "SocketZmq.h" - -#include -#include - -namespace adios2 -{ -namespace transport -{ - -SocketZmq::SocketZmq(const int timeout) -{ - m_Context = zmq_ctx_new(); - if (!m_Context) - { - throw std::runtime_error("Creating ZeroMQ context failed."); - } -} - -SocketZmq::~SocketZmq() -{ - if (m_Context) - { - zmq_ctx_destroy(m_Context); - } -} - -} // end namespace transport -} // end namespace adios2 diff --git a/source/adios2/toolkit/transport/socket/SocketZmq.h b/source/adios2/toolkit/transport/socket/SocketZmq.h deleted file mode 100644 index b571e4e8d1..0000000000 --- a/source/adios2/toolkit/transport/socket/SocketZmq.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * SocketZmqP2P.h - * - * Created on: Nov 11, 2018 - * Author: Jason Wang wangr1@ornl.gov - */ - -#ifndef ADIOS2_TOOLKIT_TRANSPORT_WAN_WANZMQ_H_ -#define ADIOS2_TOOLKIT_TRANSPORT_WAN_WANZMQ_H_ - -#include "adios2/toolkit/transport/Transport.h" - -namespace adios2 -{ -namespace transport -{ - -class SocketZmq -{ -public: - SocketZmq(const int timeout); - virtual ~SocketZmq(); - virtual int Open(const std::string &address, const Mode openMode) = 0; - virtual int Write(const char *buffer, const size_t size) = 0; - virtual int Read(char *buffer, const size_t size) = 0; - virtual int Close() = 0; - -protected: - void *m_Context = nullptr; - void *m_Socket = nullptr; - const int m_Timeout = 3; - int m_Verbosity = 0; -}; - -} // end namespace transport -} // end namespace adios - -#endif /* ADIOS2_TOOLKIT_TRANSPORT_WAN_WANZMQ_H_ */ diff --git a/source/adios2/toolkit/transport/socket/SocketZmqPubSub.cpp b/source/adios2/toolkit/transport/socket/SocketZmqPubSub.cpp deleted file mode 100644 index b530676532..0000000000 --- a/source/adios2/toolkit/transport/socket/SocketZmqPubSub.cpp +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * SocketZmqPubSub.cpp - * - * Created on: May 26, 2017 - * Author: Jason Wang wangr1@ornl.gov - */ - -#include "SocketZmqPubSub.h" - -#include -#include - -namespace adios2 -{ -namespace transport -{ - -SocketZmqPubSub::SocketZmqPubSub(const int timeout) : SocketZmq(timeout) {} - -SocketZmqPubSub::~SocketZmqPubSub() -{ - if (m_Socket) - { - zmq_close(m_Socket); - } - if (m_Context) - { - zmq_ctx_destroy(m_Context); - } -} - -int SocketZmqPubSub::Open(const std::string &address, const Mode openMode) -{ - std::string openModeStr; - - int error = -1; - if (openMode == Mode::Write) - { - openModeStr = "Write"; - m_Socket = zmq_socket(m_Context, ZMQ_PUB); - error = zmq_bind(m_Socket, address.c_str()); - } - else if (openMode == Mode::Read) - { - openModeStr = "Read"; - m_Socket = zmq_socket(m_Context, ZMQ_SUB); - error = zmq_connect(m_Socket, address.c_str()); - zmq_setsockopt(m_Socket, ZMQ_SUBSCRIBE, "", 0); - } - else - { - throw std::invalid_argument( - "[SocketZmqPubSub::Open] received invalid OpenMode parameter"); - } - - if (m_Verbosity >= 5) - { - std::cout << "[SocketZmqPubSub Transport] "; - std::cout << "OpenMode: " << openModeStr << ", "; - std::cout << "Address: " << address << ", "; - std::cout << std::endl; - } - - return error; -} - -int SocketZmqPubSub::Write(const char *buffer, const size_t size) -{ - return zmq_send(m_Socket, buffer, size, ZMQ_DONTWAIT); -} - -int SocketZmqPubSub::Read(char *buffer, const size_t size) -{ - return zmq_recv(m_Socket, buffer, size, ZMQ_DONTWAIT); -} - -int SocketZmqPubSub::Close() -{ - if (m_Socket == nullptr) - { - return -1; - } - else - { - zmq_close(m_Socket); - } - return 0; -} - -} // end namespace transport -} // end namespace adios2 diff --git a/source/adios2/toolkit/transport/socket/SocketZmqPubSub.h b/source/adios2/toolkit/transport/socket/SocketZmqPubSub.h deleted file mode 100644 index 293dc701b9..0000000000 --- a/source/adios2/toolkit/transport/socket/SocketZmqPubSub.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * SocketZmqPubSub.h - * - * Created on: May 26, 2017 - * Author: Jason Wang wangr1@ornl.gov - */ - -#ifndef ADIOS2_TOOLKIT_TRANSPORT_SOCKET_SOCKETZMQPUBSUB_H_ -#define ADIOS2_TOOLKIT_TRANSPORT_SOCKET_SOCKETZMQPUBSUB_H_ - -#include "adios2/toolkit/transport/socket/SocketZmq.h" - -namespace adios2 -{ -namespace transport -{ - -class SocketZmqPubSub : public SocketZmq -{ - -public: - SocketZmqPubSub(const int timeout); - virtual ~SocketZmqPubSub(); - int Open(const std::string &address, const Mode openMode) final; - int Write(const char *buffer, const size_t size) final; - int Read(char *buffer, const size_t size) final; - int Close() final; -}; - -} // end namespace transport -} // end namespace adios - -#endif /* ADIOS2_TOOLKIT_TRANSPORT_SOCKET_SOCKETZMQPUBSUB_H_ */ diff --git a/source/adios2/toolkit/transport/socket/SocketZmqReqRep.cpp b/source/adios2/toolkit/transport/socket/SocketZmqReqRep.cpp deleted file mode 100644 index 86a1f930cc..0000000000 --- a/source/adios2/toolkit/transport/socket/SocketZmqReqRep.cpp +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * SocketZmqReqRep.cpp - * - * Created on: May 26, 2017 - * Author: Jason Wang wangr1@ornl.gov - */ - -#include "SocketZmqReqRep.h" - -#include -#include - -namespace adios2 -{ -namespace transport -{ - -SocketZmqReqRep::SocketZmqReqRep(const int timeout) : SocketZmq(timeout) -{ - if (m_Context == nullptr || m_Context == NULL) - { - m_Context = zmq_ctx_new(); - } - if (m_Context == nullptr || m_Context == NULL) - { - throw std::runtime_error("[SocketZmqReqRep::SocketZmqReqRep] Creating " - "ZeroMQ context failed."); - } -} - -SocketZmqReqRep::~SocketZmqReqRep() -{ - Close(); - if (m_Context) - { - zmq_ctx_destroy(m_Context); - } -} - -int SocketZmqReqRep::Open(const std::string &fullAddress, const Mode openMode) -{ - std::string openModeStr; - int error = -100; - if (openMode == Mode::Write) - { - openModeStr = "Write/ZMQ_REP"; - m_Socket = zmq_socket(m_Context, ZMQ_REP); - error = zmq_bind(m_Socket, fullAddress.c_str()); - zmq_setsockopt(m_Socket, ZMQ_RCVTIMEO, &m_Timeout, sizeof(m_Timeout)); - zmq_setsockopt(m_Socket, ZMQ_LINGER, &m_Timeout, sizeof(m_Timeout)); - } - else if (openMode == Mode::Read) - { - openModeStr = "Read/ZMQ_REQ"; - m_Socket = zmq_socket(m_Context, ZMQ_REQ); - error = zmq_connect(m_Socket, fullAddress.c_str()); - zmq_setsockopt(m_Socket, ZMQ_SNDTIMEO, &m_Timeout, sizeof(m_Timeout)); - zmq_setsockopt(m_Socket, ZMQ_RCVTIMEO, &m_Timeout, sizeof(m_Timeout)); - zmq_setsockopt(m_Socket, ZMQ_LINGER, &m_Timeout, sizeof(m_Timeout)); - } - else - { - throw std::invalid_argument( - "[SocketZmqReqRep::Open] invalid OpenMode parameter"); - } - - if (m_Verbosity >= 5) - { - std::cout << "[SocketZmqReqRep Transport] "; - std::cout << "OpenMode: " << openModeStr << ", "; - std::cout << "Address: " << fullAddress << ", "; - std::cout << "Timeout: " << m_Timeout << ", "; - std::cout << std::endl; - } - - return error; -} - -int SocketZmqReqRep::Write(const char *buffer, size_t size) -{ - return zmq_send(m_Socket, buffer, size, 0); -} - -int SocketZmqReqRep::Read(char *buffer, size_t size) -{ - return zmq_recv(m_Socket, buffer, size, 0); -} - -int SocketZmqReqRep::Close() -{ - if (m_Socket) - { - zmq_close(m_Socket); - } - return 0; -} - -} // end namespace transport -} // end namespace adios2 diff --git a/source/adios2/toolkit/transport/socket/SocketZmqReqRep.h b/source/adios2/toolkit/transport/socket/SocketZmqReqRep.h deleted file mode 100644 index c1d853e6c3..0000000000 --- a/source/adios2/toolkit/transport/socket/SocketZmqReqRep.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Distributed under the OSI-approved Apache License, Version 2.0. See - * accompanying file Copyright.txt for details. - * - * SocketZmqReqRep.h - * - * Created on: May 26, 2017 - * Author: Jason Wang wangr1@ornl.gov - */ - -#ifndef ADIOS2_TOOLKIT_TRANSPORT_SOCKET_SOCKETZMQREQREP_H_ -#define ADIOS2_TOOLKIT_TRANSPORT_SOCKET_SOCKETZMQREQREP_H_ - -#include "adios2/toolkit/transport/socket/SocketZmq.h" - -namespace adios2 -{ -namespace transport -{ - -class SocketZmqReqRep : public SocketZmq -{ - -public: - SocketZmqReqRep(const int timeout); - virtual ~SocketZmqReqRep(); - int Open(const std::string &fullAddress, const Mode openMode) final; - int Write(const char *buffer, size_t size) final; - int Read(char *buffer, size_t size) final; - int Close() final; -}; - -} // end namespace transport -} // end namespace adios - -#endif /* ADIOS2_TOOLKIT_TRANSPORT_SOCKET_SOCKETZMQREQREP_H_ */ diff --git a/source/adios2/toolkit/transportman/stagingman/StagingMan.cpp b/source/adios2/toolkit/transportman/stagingman/StagingMan.cpp index ed2ffd6970..0c64169e8f 100644 --- a/source/adios2/toolkit/transportman/stagingman/StagingMan.cpp +++ b/source/adios2/toolkit/transportman/stagingman/StagingMan.cpp @@ -17,98 +17,143 @@ namespace adios2 namespace transportman { -StagingMan::StagingMan(const MPI_Comm mpiComm, const Mode openMode, - const int timeout, const size_t maxBufferSize) -: m_MpiComm(mpiComm), m_Timeout(timeout), m_OpenMode(openMode), - m_Transport(timeout), m_MaxBufferSize(maxBufferSize) +StagingMan::StagingMan() { - m_Buffer.reserve(maxBufferSize); + m_Context = zmq_ctx_new(); + if (not m_Context) + { + throw std::runtime_error("creating zmq context failed"); + } +} + +StagingMan::~StagingMan() +{ + if (m_Socket) + { + zmq_close(m_Socket); + } + if (m_Context) + { + zmq_ctx_destroy(m_Context); + } +} + +void StagingMan::OpenRequester(const int timeout, + const size_t receiverBufferSize) +{ + m_Timeout = timeout; + m_ReceiverBuffer.reserve(receiverBufferSize); +} + +void StagingMan::OpenReplier(const std::string &address, const int timeout, + const size_t receiverBufferSize) +{ + m_Timeout = timeout; + m_ReceiverBuffer.reserve(receiverBufferSize); + + m_Socket = zmq_socket(m_Context, ZMQ_REP); + if (not m_Socket) + { + throw std::runtime_error("creating zmq socket failed"); + } + + int error = zmq_bind(m_Socket, address.c_str()); + if (error) + { + throw std::runtime_error("binding zmq socket failed"); + } + + zmq_setsockopt(m_Socket, ZMQ_RCVTIMEO, &m_Timeout, sizeof(m_Timeout)); + zmq_setsockopt(m_Socket, ZMQ_LINGER, &m_Timeout, sizeof(m_Timeout)); } -StagingMan::~StagingMan() {} +std::shared_ptr> StagingMan::ReceiveRequest() +{ + int bytes = zmq_recv(m_Socket, m_ReceiverBuffer.data(), + m_ReceiverBuffer.capacity(), 0); + if (bytes <= 0) + { + return nullptr; + } + auto request = std::make_shared>(bytes); + std::memcpy(request->data(), m_ReceiverBuffer.data(), bytes); + return request; +} -void StagingMan::OpenTransport(const std::string &fullAddress) +void StagingMan::SendReply(std::shared_ptr> reply) { - m_Transport.Open(fullAddress, m_OpenMode); + zmq_send(m_Socket, reply->data(), reply->size(), 0); } -void StagingMan::CloseTransport() { m_Transport.Close(); } +void StagingMan::SendReply(const void *reply, const size_t size) +{ + zmq_send(m_Socket, reply, size, 0); +} std::shared_ptr> -StagingMan::Request(const std::vector &request, +StagingMan::Request(const void *request, const size_t size, const std::string &address) { auto reply = std::make_shared>(); - int ret = m_Transport.Open(address, m_OpenMode); + void *socket = zmq_socket(m_Context, ZMQ_REQ); + ; + + int ret = 1; auto start_time = std::chrono::system_clock::now(); while (ret) { - ret = m_Transport.Open(address, m_OpenMode); + ret = zmq_connect(socket, address.c_str()); + zmq_setsockopt(socket, ZMQ_SNDTIMEO, &m_Timeout, sizeof(m_Timeout)); + zmq_setsockopt(socket, ZMQ_RCVTIMEO, &m_Timeout, sizeof(m_Timeout)); + zmq_setsockopt(socket, ZMQ_LINGER, &m_Timeout, sizeof(m_Timeout)); + auto now_time = std::chrono::system_clock::now(); auto duration = std::chrono::duration_cast( now_time - start_time); if (duration.count() > m_Timeout) { - m_Transport.Close(); + zmq_close(socket); return reply; } } - ret = m_Transport.Write(request.data(), request.size()); - + ret = -1; start_time = std::chrono::system_clock::now(); while (ret < 1) { - ret = m_Transport.Write(request.data(), request.size()); + ret = zmq_send(socket, request, size, 0); auto now_time = std::chrono::system_clock::now(); auto duration = std::chrono::duration_cast( now_time - start_time); if (duration.count() > m_Timeout) { - m_Transport.Close(); + zmq_close(socket); return reply; } } - ret = m_Transport.Read(m_Buffer.data(), m_MaxBufferSize); - + ret = -1; start_time = std::chrono::system_clock::now(); while (ret < 1) { - ret = m_Transport.Read(m_Buffer.data(), m_MaxBufferSize); + ret = zmq_recv(socket, m_ReceiverBuffer.data(), + m_ReceiverBuffer.capacity(), 0); auto now_time = std::chrono::system_clock::now(); auto duration = std::chrono::duration_cast( now_time - start_time); if (duration.count() > m_Timeout) { - m_Transport.Close(); + zmq_close(socket); return reply; } } reply->resize(ret); - std::memcpy(reply->data(), m_Buffer.data(), ret); - m_Transport.Close(); + std::memcpy(reply->data(), m_ReceiverBuffer.data(), ret); + zmq_close(socket); return reply; } -std::shared_ptr> StagingMan::ReceiveRequest() -{ - int bytes = m_Transport.Read(m_Buffer.data(), m_MaxBufferSize); - if (bytes <= 0) - { - return nullptr; - } - auto request = std::make_shared>(bytes); - std::memcpy(request->data(), m_Buffer.data(), bytes); - return request; -} - -void StagingMan::SendReply(std::shared_ptr> reply) -{ - m_Transport.Write(reply->data(), reply->size()); -} - } // end namespace transportman } // end namespace adios2 diff --git a/source/adios2/toolkit/transportman/stagingman/StagingMan.h b/source/adios2/toolkit/transportman/stagingman/StagingMan.h index ca545b00c4..61807d5b8e 100644 --- a/source/adios2/toolkit/transportman/stagingman/StagingMan.h +++ b/source/adios2/toolkit/transportman/stagingman/StagingMan.h @@ -13,12 +13,8 @@ #include "adios2/core/IO.h" #include "adios2/core/Operator.h" -#include "adios2/toolkit/format/bp3/BP3.h" -#include "adios2/toolkit/transportman/TransportMan.h" -#ifdef ADIOS2_HAVE_ZEROMQ -#include "adios2/toolkit/transport/socket/SocketZmqReqRep.h" -#endif +#include namespace adios2 { @@ -29,33 +25,28 @@ class StagingMan { public: - StagingMan(const MPI_Comm mpiComm, const Mode openMode, const int timeout, - const size_t maxBufferSize); - + StagingMan(); ~StagingMan(); - void OpenTransport(const std::string &fullAddress); - - void CloseTransport(); - - std::shared_ptr> Request(const std::vector &request, - const std::string &address); + // requester + void OpenRequester(const int timeout, const size_t receiverBufferSize); + std::shared_ptr> + Request(const void *request, const size_t size, const std::string &address); + // replier + void OpenReplier(const std::string &address, const int timeout, + const size_t receiverBufferSize); std::shared_ptr> ReceiveRequest(); - void SendReply(std::shared_ptr> reply); + void SendReply(const void *reply, const size_t size); private: - MPI_Comm m_MpiComm; - int m_Timeout; - Mode m_OpenMode; int m_Verbosity = 0; + int m_Timeout; - size_t m_MaxBufferSize; - - std::vector m_Buffer; - - transport::SocketZmqReqRep m_Transport; + std::vector m_ReceiverBuffer; + void *m_Context = nullptr; + void *m_Socket = nullptr; }; } // end namespace transportman diff --git a/source/adios2/toolkit/transportman/wanman/WANMan.cpp b/source/adios2/toolkit/transportman/wanman/WANMan.cpp index 025a2044b6..b35c92bb2e 100644 --- a/source/adios2/toolkit/transportman/wanman/WANMan.cpp +++ b/source/adios2/toolkit/transportman/wanman/WANMan.cpp @@ -8,43 +8,28 @@ * Author: Jason Wang wangr1@ornl.gov */ -#include //TODO go away -#include //TODO go away - #include "WANMan.h" #include "adios2/common/ADIOSMacros.h" #include "adios2/helper/adiosFunctions.h" -#ifdef ADIOS2_HAVE_ZEROMQ -#include "adios2/toolkit/transport/socket/SocketZmqPubSub.h" -#endif +#include namespace adios2 { namespace transportman { -WANMan::WANMan(MPI_Comm mpiComm, const bool debugMode) -: m_MpiComm(mpiComm), m_DebugMode(debugMode) -{ -} +WANMan::WANMan() {} WANMan::~WANMan() { auto start_time = std::chrono::system_clock::now(); while (true) { - int s = 0; - m_Mutex.lock(); - for (const auto &i : m_BufferQueue) - { - if (i.size() != 0) - { - ++s; - } - } - m_Mutex.unlock(); + m_BufferQueueMutex.lock(); + size_t s = m_BufferQueue.size(); + m_BufferQueueMutex.unlock(); if (s == 0) { break; @@ -56,198 +41,126 @@ WANMan::~WANMan() { break; } - // add a sleep here because this loop is occupying the buffer queue - // lock, and this sleep would make time for reader or writer thread and - // help it finish sooner. - std::this_thread::sleep_for(std::chrono::microseconds(1)); } - for (auto &readThread : m_ReadThreads) - { - m_Reading = false; - readThread.join(); - } - for (auto &writeThread : m_WriteThreads) + + m_ThreadActive = false; + if (m_Thread.joinable()) { - m_Writing = false; - writeThread.join(); + m_Thread.join(); } } -void WANMan::SetMaxReceiveBuffer(size_t size) { m_MaxReceiveBuffer = size; } - -void WANMan::OpenTransports(const std::vector ¶msVector, - const Mode mode, const bool profile) +void WANMan::OpenPublisher(const std::string &address, const int timeout) { - m_TransportsParameters = paramsVector; - m_BufferQueue.resize(paramsVector.size()); - - for (size_t i = 0; i < paramsVector.size(); ++i) - { - // Get parameters - std::string ip; - GetStringParameter(paramsVector[i], "IPAddress", ip); - std::string port; - GetStringParameter(paramsVector[i], "Port", port); - GetIntParameter(paramsVector[i], "Timeout", m_Timeout); - - // Calculate port number - int mpiRank, mpiSize; - MPI_Comm_rank(m_MpiComm, &mpiRank); - MPI_Comm_size(m_MpiComm, &mpiSize); - if (port.empty()) - { - port = std::to_string(stoi(port) + i * mpiSize); - } - port = std::to_string(stoi(port) + mpiRank); - - std::shared_ptr wanTransport; - wanTransport = std::make_shared(m_Timeout); - - std::string fullIP = "tcp://" + ip + ":" + port; - wanTransport->Open(fullIP, mode); - m_Transports.emplace(i, wanTransport); + m_Timeout = timeout; + m_Thread = std::thread(&WANMan::WriterThread, this, address); +} - // launch thread - if (mode == Mode::Read) - { - m_Reading = true; - m_ReadThreads.emplace_back( - std::thread(&WANMan::ReadThread, this, wanTransport)); - } - else if (mode == Mode::Write) - { - m_Writing = true; - m_WriteThreads.emplace_back( - std::thread(&WANMan::WriteThread, this, wanTransport, i)); - } - } +void WANMan::OpenSubscriber(const std::string &address, const int timeout, + const size_t bufferSize) +{ + m_Timeout = timeout; + m_Thread = + std::thread(&WANMan::ReaderThread, this, address, timeout, bufferSize); } -void WANMan::Write(std::shared_ptr> buffer, size_t id) +void WANMan::PushBufferQueue(std::shared_ptr> buffer) { - PushBufferQueue(buffer, id); + std::lock_guard l(m_BufferQueueMutex); + m_BufferQueue.push(buffer); } -void WANMan::Write(const std::vector &buffer, size_t transportId) +std::shared_ptr> WANMan::PopBufferQueue() { - if (transportId >= m_Transports.size()) + std::lock_guard l(m_BufferQueueMutex); + if (m_BufferQueue.empty()) { - throw std::runtime_error( - "ERROR: No valid transports found, from WANMan::WriteSocket()"); + return nullptr; + } + else + { + auto ret = m_BufferQueue.front(); + m_BufferQueue.pop(); + return ret; } - - m_Transports[transportId]->Write(buffer.data(), buffer.size()); } -std::shared_ptr> WANMan::Read(size_t id) +void WANMan::WriterThread(const std::string &address) { - return PopBufferQueue(id); -} + void *context = zmq_ctx_new(); + if (not context) + { + throw std::runtime_error("creating zmq context failed"); + } -void WANMan::PushBufferQueue(std::shared_ptr> v, size_t id) -{ - std::lock_guard l(m_Mutex); - m_BufferQueue[id].push(v); -} + void *socket = zmq_socket(context, ZMQ_PUB); + if (not socket) + { + throw std::runtime_error("creating zmq socket failed"); + } -std::shared_ptr> WANMan::PopBufferQueue(size_t id) -{ - std::lock_guard l(m_Mutex); - if (m_BufferQueue[id].size() > 0) + int error = zmq_bind(socket, address.c_str()); + if (error) { - std::shared_ptr> vec = m_BufferQueue[id].front(); - m_BufferQueue[id].pop(); - return vec; + throw std::runtime_error("binding zmq socket failed"); } - return nullptr; -} -void WANMan::WriteThread(std::shared_ptr transport, - size_t id) -{ - while (m_Writing) + while (m_ThreadActive) { - std::shared_ptr> buffer = PopBufferQueue(id); - if (buffer != nullptr) + auto buffer = PopBufferQueue(); + if (buffer != nullptr and buffer->size() > 0) { - if (buffer->size() > 0) + zmq_send(socket, buffer->data(), buffer->size(), ZMQ_DONTWAIT); + if (m_Verbosity >= 5) { - transport->Write(buffer->data(), buffer->size()); + std::cout << "WANMan::WriterThread sent package size " + << buffer->size() << std::endl; } } } } -void WANMan::ReadThread(std::shared_ptr transport) +void WANMan::ReaderThread(const std::string &address, const int timeout, + const size_t receiverBufferSize) { - std::vector buffer(m_MaxReceiveBuffer); - while (m_Reading) + void *context = zmq_ctx_new(); + if (not context) { - int ret = transport->Read(buffer.data(), m_MaxReceiveBuffer); - if (ret > 0) - { - std::shared_ptr> bufferQ = - std::make_shared>(ret); - std::memcpy(bufferQ->data(), buffer.data(), ret); - PushBufferQueue(bufferQ, 0); - } + throw std::runtime_error("creating zmq context failed"); } -} -bool WANMan::GetBoolParameter(const Params ¶ms, const std::string &key) -{ - auto itKey = params.find(key); - if (itKey != params.end()) + void *socket = zmq_socket(context, ZMQ_SUB); + if (not socket) { - std::string value = itKey->second; - std::transform(value.begin(), value.end(), value.begin(), ::tolower); - if (value == "yes" || value == "true") - { - return true; - } - else if (value == "no" || value == "false") - { - return false; - } + throw std::runtime_error("creating zmq socket failed"); } - return false; -} -bool WANMan::GetStringParameter(const Params ¶ms, const std::string &key, - std::string &value) -{ - auto it = params.find(key); - if (it != params.end()) + int error = zmq_connect(socket, address.c_str()); + if (error) { - value = it->second; - return true; + throw std::runtime_error("connecting zmq socket failed"); } - return false; -} -bool WANMan::GetIntParameter(const Params ¶ms, const std::string &key, - int &value) -{ - auto it = params.find(key); - if (it != params.end()) + zmq_setsockopt(socket, ZMQ_SUBSCRIBE, "", 0); + + std::vector receiverBuffer(receiverBufferSize); + + while (m_ThreadActive) { - try - { - value = std::stoi(it->second); - return true; - } - catch (std::exception &e) + int ret = zmq_recv(socket, receiverBuffer.data(), receiverBufferSize, + ZMQ_DONTWAIT); + if (ret > 0) { - std::cout << "Parameter " << key - << " should be an integer in string format. However, " - << e.what() - << " has been caught while trying to convert " - "the value to an integer." - << std::endl; - return false; + auto buff = std::make_shared>(ret); + std::memcpy(buff->data(), receiverBuffer.data(), ret); + PushBufferQueue(buff); + if (m_Verbosity >= 5) + { + std::cout << "WANMan::ReaderThread received package size " + << buff->size() << std::endl; + } } } - return false; } } // end namespace transportman diff --git a/source/adios2/toolkit/transportman/wanman/WANMan.h b/source/adios2/toolkit/transportman/wanman/WANMan.h index 9b1645b3eb..b7d435652e 100644 --- a/source/adios2/toolkit/transportman/wanman/WANMan.h +++ b/source/adios2/toolkit/transportman/wanman/WANMan.h @@ -17,7 +17,6 @@ #include "adios2/core/IO.h" #include "adios2/core/Operator.h" -#include "adios2/toolkit/transport/socket/SocketZmq.h" namespace adios2 { @@ -28,55 +27,31 @@ class WANMan { public: - WANMan(MPI_Comm mpiComm, const bool debugMode); - + WANMan(); ~WANMan(); - void OpenTransports(const std::vector ¶metersVector, - const Mode openMode, const bool profile); - - void Write(const std::vector &buffer, size_t transportId = 0); - void Write(std::shared_ptr> buffer, - size_t transportId = 0); - - std::shared_ptr> Read(size_t id); + void OpenPublisher(const std::string &address, const int timeout); + void OpenSubscriber(const std::string &address, const int timeout, + const size_t receiveBufferSize); - void SetMaxReceiveBuffer(size_t size); + void PushBufferQueue(std::shared_ptr> buffer); + std::shared_ptr> PopBufferQueue(); private: - std::unordered_map> - m_Transports; - MPI_Comm m_MpiComm; - bool m_DebugMode; - - // Objects for buffer queue - std::vector>>> m_BufferQueue; - void PushBufferQueue(std::shared_ptr> v, size_t id); - std::shared_ptr> PopBufferQueue(size_t id); - std::mutex m_Mutex; - - // Functions for parsing parameters - bool GetBoolParameter(const Params ¶ms, const std::string &key); - bool GetStringParameter(const Params ¶ms, const std::string &key, - std::string &value); - bool GetIntParameter(const Params ¶ms, const std::string &key, - int &value); - - // For read thread - void ReadThread(std::shared_ptr transport); - std::vector m_ReadThreads; - bool m_Reading = false; + // For buffer queue + std::queue>> m_BufferQueue; + std::mutex m_BufferQueueMutex; - // For write thread - void WriteThread(std::shared_ptr transport, - size_t id); - std::vector m_WriteThreads; - bool m_Writing = false; + // For threads + void WriterThread(const std::string &address); + void ReaderThread(const std::string &address, const int timeout, + const size_t receiveBufferSize); + std::thread m_Thread; + bool m_ThreadActive = true; // parameters - std::vector m_TransportsParameters; - size_t m_MaxReceiveBuffer = 256 * 1024 * 1024; int m_Timeout = 10; + int m_Verbosity = 0; }; } // end namespace transportman diff --git a/testing/adios2/engine/dataman/TestDataMan.h b/testing/adios2/engine/dataman/TestDataMan.h index 249b019220..49e65f22c6 100644 --- a/testing/adios2/engine/dataman/TestDataMan.h +++ b/testing/adios2/engine/dataman/TestDataMan.h @@ -59,8 +59,7 @@ void PrintData(const T *data, const size_t size, const size_t step) } template -void VerifyData(const std::complex *data, const size_t size, size_t step, - const std::vector &transParams) +void VerifyData(const std::complex *data, const size_t size, size_t step) { std::vector> tmpdata(size); GenData(tmpdata, step); @@ -76,26 +75,13 @@ void VerifyData(const std::complex *data, const size_t size, size_t step, } template -void VerifyData(const T *data, const size_t size, size_t step, - const std::vector &transParams) +void VerifyData(const T *data, const size_t size, size_t step) { - bool compressed = false; - for (const auto &i : transParams) - { - auto j = i.find("CompressionMethod"); - if (j != i.end()) - { - compressed = true; - } - } std::vector tmpdata(size); GenData(tmpdata, step); for (size_t i = 0; i < size; ++i) { - if (!compressed) - { - ASSERT_EQ(data[i], tmpdata[i]); - } + ASSERT_EQ(data[i], tmpdata[i]); } if (print_lines < 100) { @@ -105,15 +91,13 @@ void VerifyData(const T *data, const size_t size, size_t step, } template -void VerifyData(const std::vector &data, const size_t step, - const std::vector &transParams) +void VerifyData(const std::vector &data, const size_t step) { - VerifyData(data.data(), data.size(), step, transParams); + VerifyData(data.data(), data.size(), step); } void DataManWriter(const Dims &shape, const Dims &start, const Dims &count, - const size_t steps, const adios2::Params &engineParams, - const std::vector &transParams) + const size_t steps, const adios2::Params &engineParams) { size_t datasize = std::accumulate(count.begin(), count.end(), 1, std::multiplies()); @@ -125,10 +109,6 @@ void DataManWriter(const Dims &shape, const Dims &start, const Dims &count, adios2::IO dataManIO = adios.DeclareIO("WAN"); dataManIO.SetEngine("DataMan"); dataManIO.SetParameters(engineParams); - for (const auto ¶ms : transParams) - { - dataManIO.AddTransport("WAN", params); - } std::vector myChars(datasize); std::vector myUChars(datasize); std::vector myShorts(datasize); @@ -191,8 +171,7 @@ void DataManWriter(const Dims &shape, const Dims &start, const Dims &count, } void DataManReaderP2P(const Dims &shape, const Dims &start, const Dims &count, - const size_t steps, const adios2::Params &engineParams, - const std::vector &transParams) + const size_t steps, const adios2::Params &engineParams) { size_t datasize = std::accumulate(count.begin(), count.end(), 1, std::multiplies()); @@ -204,10 +183,6 @@ void DataManReaderP2P(const Dims &shape, const Dims &start, const Dims &count, adios2::IO dataManIO = adios.DeclareIO("WAN"); dataManIO.SetEngine("DataMan"); dataManIO.SetParameters(engineParams); - for (const auto ¶ms : transParams) - { - dataManIO.AddTransport("WAN", params); - } adios2::Engine dataManReader = dataManIO.Open("stream", adios2::Mode::Read); std::vector myChars(datasize); @@ -284,16 +259,16 @@ void DataManReaderP2P(const Dims &shape, const Dims &start, const Dims &count, adios2::Mode::Sync); dataManReader.Get(bpDComplexes, myDComplexes.data(), adios2::Mode::Sync); - VerifyData(myChars, currentStep, transParams); - VerifyData(myUChars, currentStep, transParams); - VerifyData(myShorts, currentStep, transParams); - VerifyData(myUShorts, currentStep, transParams); - VerifyData(myInts, currentStep, transParams); - VerifyData(myUInts, currentStep, transParams); - VerifyData(myFloats, currentStep, transParams); - VerifyData(myDoubles, currentStep, transParams); - VerifyData(myComplexes, currentStep, transParams); - VerifyData(myDComplexes, currentStep, transParams); + VerifyData(myChars, currentStep); + VerifyData(myUChars, currentStep); + VerifyData(myShorts, currentStep); + VerifyData(myUShorts, currentStep); + VerifyData(myInts, currentStep); + VerifyData(myUInts, currentStep); + VerifyData(myFloats, currentStep); + VerifyData(myDoubles, currentStep); + VerifyData(myComplexes, currentStep); + VerifyData(myDComplexes, currentStep); dataManReader.EndStep(); } else @@ -317,7 +292,6 @@ void DataManReaderP2P(const Dims &shape, const Dims &start, const Dims &count, void DataManReaderSubscribe(const Dims &shape, const Dims &start, const Dims &count, const size_t steps, const adios2::Params &engineParams, - const std::vector &transParams, const size_t timeout) { size_t datasize = std::accumulate(count.begin(), count.end(), 1, @@ -331,10 +305,6 @@ void DataManReaderSubscribe(const Dims &shape, const Dims &start, adios2::IO dataManIO = adios.DeclareIO("WAN"); dataManIO.SetEngine("DataMan"); dataManIO.SetParameters(engineParams); - for (const auto ¶ms : transParams) - { - dataManIO.AddTransport("WAN", params); - } adios2::Engine dataManReader = dataManIO.Open("stream", adios2::Mode::Read); adios2::Variable bpFloats; size_t i = 0; @@ -362,7 +332,7 @@ void DataManReaderSubscribe(const Dims &shape, const Dims &start, dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync); i = dataManReader.CurrentStep(); - VerifyData(myFloats, i, transParams); + VerifyData(myFloats, i); } else if (status == adios2::StepStatus::EndOfStream) { diff --git a/testing/adios2/engine/dataman/TestDataManP2P1D.cpp b/testing/adios2/engine/dataman/TestDataManP2P1D.cpp index 131e69299b..7200a1cd9f 100644 --- a/testing/adios2/engine/dataman/TestDataManP2P1D.cpp +++ b/testing/adios2/engine/dataman/TestDataManP2P1D.cpp @@ -24,18 +24,15 @@ TEST_F(DataManEngineTest, WriteRead_1D_P2P) Dims start = {0}; Dims count = {10}; size_t steps = 200; - adios2::Params engineParams = {{"WorkflowMode", "Stream"}}; - std::vector transportParams = {{{"Library", "ZMQ"}, - {"IPAddress", "127.0.0.1"}, - {"Port", "12306"}, - {"Timeout", "5"}}}; + adios2::Params engineParams = {{"IPAddress", "127.0.0.1"}, + {"Port", "12306"}}; // run workflow - auto r = std::thread(DataManReaderP2P, shape, start, count, steps, - engineParams, transportParams); + auto r = + std::thread(DataManReaderP2P, shape, start, count, steps, engineParams); std::cout << "Reader thread started" << std::endl; - auto w = std::thread(DataManWriter, shape, start, count, steps, - engineParams, transportParams); + auto w = + std::thread(DataManWriter, shape, start, count, steps, engineParams); std::cout << "Writer thread started" << std::endl; w.join(); std::cout << "Writer thread ended" << std::endl; diff --git a/testing/adios2/engine/dataman/TestDataManP2P2DMemSelect.cpp b/testing/adios2/engine/dataman/TestDataManP2P2DMemSelect.cpp index ea7899e3a7..e94c14d5be 100644 --- a/testing/adios2/engine/dataman/TestDataManP2P2DMemSelect.cpp +++ b/testing/adios2/engine/dataman/TestDataManP2P2DMemSelect.cpp @@ -117,8 +117,7 @@ void VerifyData(const T *data, size_t step, const Dims &start, void DataManWriterP2PMemSelect(const Dims &shape, const Dims &start, const Dims &count, const size_t steps, - const adios2::Params &engineParams, - const std::vector &transParams) + const adios2::Params &engineParams) { size_t datasize = std::accumulate(count.begin(), count.end(), 1, std::multiplies()); @@ -130,10 +129,6 @@ void DataManWriterP2PMemSelect(const Dims &shape, const Dims &start, adios2::IO dataManIO = adios.DeclareIO("WAN"); dataManIO.SetEngine("DataMan"); dataManIO.SetParameters(engineParams); - for (const auto ¶ms : transParams) - { - dataManIO.AddTransport("WAN", params); - } std::vector myChars(datasize); std::vector myUChars(datasize); std::vector myShorts(datasize); @@ -198,8 +193,7 @@ void DataManWriterP2PMemSelect(const Dims &shape, const Dims &start, void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start, const Dims &count, const Dims &memStart, const Dims &memCount, const size_t steps, - const adios2::Params &engineParams, - const std::vector &transParams) + const adios2::Params &engineParams) { #ifdef ADIOS2_HAVE_MPI adios2::ADIOS adios(MPI_COMM_SELF, adios2::DebugON); @@ -209,10 +203,6 @@ void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start, adios2::IO dataManIO = adios.DeclareIO("WAN"); dataManIO.SetEngine("DataMan"); dataManIO.SetParameters(engineParams); - for (const auto ¶ms : transParams) - { - dataManIO.AddTransport("WAN", params); - } adios2::Engine dataManReader = dataManIO.Open("stream", adios2::Mode::Read); size_t datasize = std::accumulate(memCount.begin(), memCount.end(), 1, @@ -363,20 +353,15 @@ TEST_F(DataManEngineTest, WriteRead_2D_MemSelect) memcount = {7, 9}; size_t steps = 1000; - adios2::Params engineParams = {{"WorkflowMode", "stream"}}; - std::vector transportParams = {{ - {"Library", "ZMQ"}, - {"IPAddress", "127.0.0.1"}, - {"Port", "12312"}, - }}; + adios2::Params engineParams = {{"IPAddress", "127.0.0.1"}, + {"Port", "12308"}}; // run workflow - auto r = - std::thread(DataManReaderP2PMemSelect, shape, start, count, memstart, - memcount, steps, engineParams, transportParams); + auto r = std::thread(DataManReaderP2PMemSelect, shape, start, count, + memstart, memcount, steps, engineParams); std::cout << "Reader thread started" << std::endl; auto w = std::thread(DataManWriterP2PMemSelect, shape, start, count, steps, - engineParams, transportParams); + engineParams); std::cout << "Writer thread started" << std::endl; w.join(); std::cout << "Writer thread ended" << std::endl; diff --git a/testing/adios2/engine/dataman/TestDataManP2P3DMemSelect.cpp b/testing/adios2/engine/dataman/TestDataManP2P3DMemSelect.cpp index e21655b26f..c0a7fa4ef0 100644 --- a/testing/adios2/engine/dataman/TestDataManP2P3DMemSelect.cpp +++ b/testing/adios2/engine/dataman/TestDataManP2P3DMemSelect.cpp @@ -103,8 +103,7 @@ void VerifyData(const int *data, size_t step, const Dims &start, void DataManWriterP2PMemSelect(const Dims &shape, const Dims &start, const Dims &count, const size_t steps, - const adios2::Params &engineParams, - const std::vector &transParams) + const adios2::Params &engineParams) { #ifdef ADIOS2_HAVE_MPI adios2::ADIOS adios(MPI_COMM_SELF, adios2::DebugON); @@ -114,10 +113,6 @@ void DataManWriterP2PMemSelect(const Dims &shape, const Dims &start, adios2::IO dataManIO = adios.DeclareIO("WAN"); dataManIO.SetEngine("DataMan"); dataManIO.SetParameters(engineParams); - for (const auto ¶ms : transParams) - { - dataManIO.AddTransport("WAN", params); - } auto bpInts = dataManIO.DefineVariable("bpInts", shape, start, count); adios2::Engine dataManWriter = dataManIO.Open("stream", adios2::Mode::Write); @@ -133,8 +128,7 @@ void DataManWriterP2PMemSelect(const Dims &shape, const Dims &start, void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start, const Dims &count, const Dims &memStart, const Dims &memCount, const size_t steps, - const adios2::Params &engineParams, - const std::vector &transParams) + const adios2::Params &engineParams) { #ifdef ADIOS2_HAVE_MPI adios2::ADIOS adios(MPI_COMM_SELF, adios2::DebugON); @@ -144,10 +138,6 @@ void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start, adios2::IO dataManIO = adios.DeclareIO("WAN"); dataManIO.SetEngine("DataMan"); dataManIO.SetParameters(engineParams); - for (const auto ¶ms : transParams) - { - dataManIO.AddTransport("WAN", params); - } adios2::Engine dataManReader = dataManIO.Open("stream", adios2::Mode::Read); std::vector myInts = reader_data; size_t i; @@ -197,20 +187,15 @@ TEST_F(DataManEngineTest, WriteRead_3D_MemSelect) { size_t steps = 10; - adios2::Params engineParams = {{"WorkflowMode", "stream"}}; - std::vector transportParams = {{ - {"Library", "ZMQ"}, - {"IPAddress", "127.0.0.1"}, - {"Port", "12313"}, - }}; + adios2::Params engineParams = {{"IPAddress", "127.0.0.1"}, + {"Port", "12310"}}; // run workflow - auto r = - std::thread(DataManReaderP2PMemSelect, shape, start, count, memstart, - memcount, steps, engineParams, transportParams); + auto r = std::thread(DataManReaderP2PMemSelect, shape, start, count, + memstart, memcount, steps, engineParams); std::cout << "Reader thread started" << std::endl; auto w = std::thread(DataManWriterP2PMemSelect, shape, start, count, steps, - engineParams, transportParams); + engineParams); std::cout << "Writer thread started" << std::endl; w.join(); std::cout << "Writer thread ended" << std::endl; diff --git a/testing/adios2/engine/dataman/TestDataManP2PBzip2D.cpp b/testing/adios2/engine/dataman/TestDataManP2PBzip2D.cpp index a76c03a1ca..18fbd8a1a7 100644 --- a/testing/adios2/engine/dataman/TestDataManP2PBzip2D.cpp +++ b/testing/adios2/engine/dataman/TestDataManP2PBzip2D.cpp @@ -25,20 +25,15 @@ TEST_F(DataManEngineTest, DISABLED_WriteRead_2D_P2P_BZip2) Dims start = {0, 0}; Dims count = {10, 10}; size_t steps = 200; - adios2::Params engineParams = {{"WorkflowMode", "stream"}}; - std::vector transportParams = {{ - {"Library", "ZMQ"}, - {"IPAddress", "127.0.0.1"}, - {"Port", "12309"}, - {"CompressionMethod", "bzip2"}, - }}; + adios2::Params engineParams = {{"IPAddress", "127.0.0.1"}, + {"Port", "12312"}}; // run workflow - auto r = std::thread(DataManReaderP2P, shape, start, count, steps, - engineParams, transportParams); + auto r = + std::thread(DataManReaderP2P, shape, start, count, steps, engineParams); std::cout << "Reader thread started" << std::endl; - auto w = std::thread(DataManWriter, shape, start, count, steps, - engineParams, transportParams); + auto w = + std::thread(DataManWriter, shape, start, count, steps, engineParams); std::cout << "Writer thread started" << std::endl; w.join(); std::cout << "Writer thread ended" << std::endl; diff --git a/testing/adios2/engine/dataman/TestDataManP2PSz2D.cpp b/testing/adios2/engine/dataman/TestDataManP2PSz2D.cpp index 89039ad8c8..7627aa7efe 100644 --- a/testing/adios2/engine/dataman/TestDataManP2PSz2D.cpp +++ b/testing/adios2/engine/dataman/TestDataManP2PSz2D.cpp @@ -25,21 +25,14 @@ TEST_F(DataManEngineTest, WriteRead_2D_P2P_SZ) Dims start = {0, 0}; Dims count = {6, 8}; size_t steps = 20; - adios2::Params engineParams = {{"WorkflowMode", "stream"}}; - std::vector transportParams = {{ - {"Library", "ZMQ"}, - {"IPAddress", "127.0.0.1"}, - {"Port", "12308"}, - {"CompressionMethod", "sz"}, - {"sz:accuracy", "0.01"}, - {"CompressionVariables", "bpFloats,bpDoubles"}, - }}; + adios2::Params engineParams = {{"IPAddress", "127.0.0.1"}, + {"Port", "12314"}}; // run workflow - auto r = std::thread(DataManReaderP2P, shape, start, count, steps, - engineParams, transportParams); + auto r = + std::thread(DataManReaderP2P, shape, start, count, steps, engineParams); std::cout << "Reader thread started" << std::endl; - auto w = std::thread(DataManWriter, shape, start, count, steps, - engineParams, transportParams); + auto w = + std::thread(DataManWriter, shape, start, count, steps, engineParams); std::cout << "Writer thread started" << std::endl; w.join(); std::cout << "Writer thread ended" << std::endl; diff --git a/testing/adios2/engine/dataman/TestDataManP2PZfp2D.cpp b/testing/adios2/engine/dataman/TestDataManP2PZfp2D.cpp index eda5699bde..7372bd1e41 100644 --- a/testing/adios2/engine/dataman/TestDataManP2PZfp2D.cpp +++ b/testing/adios2/engine/dataman/TestDataManP2PZfp2D.cpp @@ -25,20 +25,15 @@ TEST_F(DataManEngineTest, WriteRead_2D_P2P_Zfp) Dims start = {0, 0}; Dims count = {5, 5}; size_t steps = 200; - adios2::Params engineParams = {{"WorkflowMode", "stream"}}; - std::vector transportParams = { - {{"Library", "ZMQ"}, - {"IPAddress", "127.0.0.1"}, - {"Port", "12307"}, - {"CompressionMethod", "zfp"}, - {"zfp:precision", "0.001"}}}; + adios2::Params engineParams = {{"IPAddress", "127.0.0.1"}, + {"Port", "12316"}}; // run workflow - auto r = std::thread(DataManReaderP2P, shape, start, count, steps, - engineParams, transportParams); + auto r = + std::thread(DataManReaderP2P, shape, start, count, steps, engineParams); std::cout << "Reader thread started" << std::endl; - auto w = std::thread(DataManWriter, shape, start, count, steps, - engineParams, transportParams); + auto w = + std::thread(DataManWriter, shape, start, count, steps, engineParams); std::cout << "Writer thread started" << std::endl; w.join(); std::cout << "Writer thread ended" << std::endl; diff --git a/testing/adios2/engine/dataman/TestDataManSubscribe1D.cpp b/testing/adios2/engine/dataman/TestDataManSubscribe1D.cpp index 762fb3825c..fc23805cbe 100644 --- a/testing/adios2/engine/dataman/TestDataManSubscribe1D.cpp +++ b/testing/adios2/engine/dataman/TestDataManSubscribe1D.cpp @@ -25,16 +25,15 @@ TEST_F(DataManEngineTest, WriteRead_1D_Subscribe) Dims start = {0}; Dims count = {10}; size_t steps = 1000; - adios2::Params engineParams = {{"WorkflowMode", "stream"}}; - std::vector transportParams = { - {{"Library", "ZMQ"}, {"IPAddress", "127.0.0.1"}, {"Port", "12310"}}}; + adios2::Params engineParams = {{"IPAddress", "127.0.0.1"}, + {"Port", "12318"}}; // run workflow auto r = std::thread(DataManReaderSubscribe, shape, start, count, steps, - engineParams, transportParams, timeout); + engineParams, timeout); std::cout << "Reader thread started" << std::endl; - auto w = std::thread(DataManWriter, shape, start, count, steps, - engineParams, transportParams); + auto w = + std::thread(DataManWriter, shape, start, count, steps, engineParams); std::cout << "Writer thread started" << std::endl; w.join(); std::cout << "Writer thread ended" << std::endl; From 21609b3c306f28b0b9bd0f98ff3684d3d1d67cc1 Mon Sep 17 00:00:00 2001 From: Ruonan Wang Date: Mon, 19 Aug 2019 02:11:55 -0400 Subject: [PATCH 3/7] renamed ZMQ transports --- source/adios2/CMakeLists.txt | 4 +-- source/adios2/engine/dataman/DataManCommon.h | 4 +-- .../adios2/engine/dataman/DataManReader.cpp | 8 ++--- source/adios2/engine/dataman/DataManReader.h | 4 +-- .../adios2/engine/dataman/DataManWriter.cpp | 10 +++--- source/adios2/engine/dataman/DataManWriter.h | 2 +- source/adios2/engine/ssc/SscReader.h | 6 ++-- source/adios2/engine/ssc/SscWriter.cpp | 10 +++--- source/adios2/engine/ssc/SscWriter.h | 2 +- source/adios2/engine/table/TableWriter.cpp | 8 ++--- source/adios2/engine/table/TableWriter.h | 4 +-- .../zmqpubsub/ZmqPubSub.cpp} | 32 +++++++++---------- .../WANMan.h => zmq/zmqpubsub/ZmqPubSub.h} | 18 +++++------ .../zmqreqrep/ZmqReqRep.cpp} | 27 ++++++++-------- .../zmqreqrep/ZmqReqRep.h} | 18 +++++------ 15 files changed, 76 insertions(+), 81 deletions(-) rename source/adios2/toolkit/{transportman/wanman/WANMan.cpp => zmq/zmqpubsub/ZmqPubSub.cpp} (79%) rename source/adios2/toolkit/{transportman/wanman/WANMan.h => zmq/zmqpubsub/ZmqPubSub.h} (81%) rename source/adios2/toolkit/{transportman/stagingman/StagingMan.cpp => zmq/zmqreqrep/ZmqReqRep.cpp} (87%) rename source/adios2/toolkit/{transportman/stagingman/StagingMan.h => zmq/zmqreqrep/ZmqReqRep.h} (78%) diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 27c79119cd..2492f76a65 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -112,8 +112,8 @@ endif() if(ADIOS2_HAVE_ZeroMQ) target_sources(adios2 PRIVATE - toolkit/transportman/wanman/WANMan.cpp - toolkit/transportman/stagingman/StagingMan.cpp + toolkit/zmq/zmqreqrep/ZmqReqRep.cpp + toolkit/zmq/zmqpubsub/ZmqPubSub.cpp ) target_link_libraries(adios2 PRIVATE ZeroMQ::ZMQ) endif() diff --git a/source/adios2/engine/dataman/DataManCommon.h b/source/adios2/engine/dataman/DataManCommon.h index 7dac1ba5c0..d312f1973e 100644 --- a/source/adios2/engine/dataman/DataManCommon.h +++ b/source/adios2/engine/dataman/DataManCommon.h @@ -17,8 +17,8 @@ #include "adios2/helper/adiosSystem.h" #include "adios2/toolkit/format/dataman/DataManSerializer.h" #include "adios2/toolkit/format/dataman/DataManSerializer.tcc" -#include "adios2/toolkit/transportman/stagingman/StagingMan.h" -#include "adios2/toolkit/transportman/wanman/WANMan.h" +#include "adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.h" +#include "adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.h" namespace adios2 { diff --git a/source/adios2/engine/dataman/DataManReader.cpp b/source/adios2/engine/dataman/DataManReader.cpp index b239d9078c..66d36234f6 100644 --- a/source/adios2/engine/dataman/DataManReader.cpp +++ b/source/adios2/engine/dataman/DataManReader.cpp @@ -47,11 +47,6 @@ DataManReader::DataManReader(IO &io, const std::string &name, const Mode mode, reply = m_ZmqRequester.Request(request.data(), request.size(), address); } - - for (auto i : *reply) - { - std::cout << i; - } auto addJson = nlohmann::json::parse(*reply); m_DataAddresses = addJson["DataAddresses"].get>(); @@ -60,11 +55,12 @@ DataManReader::DataManReader(IO &io, const std::string &name, const Mode mode, } else if (m_StagingMode == "local") { + // TODO: Add filesystem based handshake } for (const auto &address : m_DataAddresses) { - auto dataZmq = std::make_shared(); + auto dataZmq = std::make_shared(); dataZmq->OpenSubscriber(address, m_Timeout, m_ReceiverBufferSize); m_ZmqSubscriberVec.push_back(dataZmq); } diff --git a/source/adios2/engine/dataman/DataManReader.h b/source/adios2/engine/dataman/DataManReader.h index aba93a7596..5c8fba2f88 100644 --- a/source/adios2/engine/dataman/DataManReader.h +++ b/source/adios2/engine/dataman/DataManReader.h @@ -37,10 +37,10 @@ class DataManReader : public DataManCommon private: bool m_ProvideLatest = false; size_t m_FinalStep = std::numeric_limits::max(); - transportman::StagingMan m_ZmqRequester; + adios2::zmq::ZmqReqRep m_ZmqRequester; std::vector m_DataAddresses; std::vector m_ControlAddresses; - std::vector> m_ZmqSubscriberVec; + std::vector> m_ZmqSubscriberVec; std::thread m_SubscriberThread; format::DmvVecPtrMap m_MetaDataMap; diff --git a/source/adios2/engine/dataman/DataManWriter.cpp b/source/adios2/engine/dataman/DataManWriter.cpp index 0e818ed6a8..bb4dd4ec00 100644 --- a/source/adios2/engine/dataman/DataManWriter.cpp +++ b/source/adios2/engine/dataman/DataManWriter.cpp @@ -67,6 +67,7 @@ DataManWriter::DataManWriter(IO &io, const std::string &name, } else if (m_StagingMode == "local") { + // TODO: Add filesystem based handshake } m_DataPublisher.OpenPublisher(m_DataAddress, m_Timeout); @@ -147,18 +148,17 @@ void DataManWriter::DoClose(const int transportIndex) void DataManWriter::ControlThread(const std::string &address) { - transportman::StagingMan tpm; - tpm.OpenReplier(address, m_Timeout, 8192); + adios2::zmq::ZmqReqRep replier; + replier.OpenReplier(address, m_Timeout, 8192); while (m_ThreadActive) { - auto request = tpm.ReceiveRequest(); + auto request = replier.ReceiveRequest(); if (request && request->size() > 0) { - std::cout << "ControlThread received package ============= \n"; std::string r(request->begin(), request->end()); if (r == "Address") { - tpm.SendReply(m_AllAddresses.data(), m_AllAddresses.size()); + replier.SendReply(m_AllAddresses.data(), m_AllAddresses.size()); } } } diff --git a/source/adios2/engine/dataman/DataManWriter.h b/source/adios2/engine/dataman/DataManWriter.h index 428341f460..fa05213a16 100644 --- a/source/adios2/engine/dataman/DataManWriter.h +++ b/source/adios2/engine/dataman/DataManWriter.h @@ -41,7 +41,7 @@ class DataManWriter : public DataManCommon std::string m_ControlAddress; std::string m_AllAddresses; - transportman::WANMan m_DataPublisher; + adios2::zmq::ZmqPubSub m_DataPublisher; format::VecPtr m_AggregatedMetadata = nullptr; std::mutex m_AggregatedMetadataMutex; diff --git a/source/adios2/engine/ssc/SscReader.h b/source/adios2/engine/ssc/SscReader.h index f9ff489764..70786a7525 100644 --- a/source/adios2/engine/ssc/SscReader.h +++ b/source/adios2/engine/ssc/SscReader.h @@ -20,7 +20,7 @@ #include "adios2/toolkit/format/dataman/DataManSerializer.h" #include "adios2/toolkit/format/dataman/DataManSerializer.tcc" #include "adios2/toolkit/profiling/taustubs/tautimer.hpp" -#include "adios2/toolkit/transportman/stagingman/StagingMan.h" +#include "adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.h" namespace adios2 { @@ -47,8 +47,8 @@ class SscReader : public Engine private: bool m_Tolerance = false; format::DataManSerializer m_DataManSerializer; - transportman::StagingMan m_DataTransport; - transportman::StagingMan m_MetadataTransport; + adios2::zmq::ZmqReqRep m_DataTransport; + adios2::zmq::ZmqReqRep m_MetadataTransport; format::DmvVecPtrMap m_MetaDataMap; int64_t m_CurrentStep = -1; int m_MpiRank; diff --git a/source/adios2/engine/ssc/SscWriter.cpp b/source/adios2/engine/ssc/SscWriter.cpp index 593f8f6fcb..bd0c9f500c 100644 --- a/source/adios2/engine/ssc/SscWriter.cpp +++ b/source/adios2/engine/ssc/SscWriter.cpp @@ -156,11 +156,11 @@ void SscWriter::InitTransports() void SscWriter::ReplyThread(const std::string &address) { - transportman::StagingMan tpm; - tpm.OpenReplier(address, m_Timeout, m_ReceiverBufferSize); + adios2::zmq::ZmqReqRep replier; + replier.OpenReplier(address, m_Timeout, m_ReceiverBufferSize); while (m_Listening) { - auto request = tpm.ReceiveRequest(); + auto request = replier.ReceiveRequest(); if (request == nullptr) { continue; @@ -185,7 +185,7 @@ void SscWriter::ReplyThread(const std::string &address) stepRequested, stepProvided, m_AppID); } } - tpm.SendReply(aggMetadata); + replier.SendReply(aggMetadata); if (m_Verbosity >= 100) { @@ -205,7 +205,7 @@ void SscWriter::ReplyThread(const std::string &address) std::unordered_map p = m_CompressionParams; m_CompressionParamsMutex.unlock(); auto reply = m_DataManSerializer.GenerateReply(*request, step, p); - tpm.SendReply(reply); + replier.SendReply(reply); if (reply->size() <= 16) { if (m_Tolerance) diff --git a/source/adios2/engine/ssc/SscWriter.h b/source/adios2/engine/ssc/SscWriter.h index 93751d3ed2..0b7abde9f6 100644 --- a/source/adios2/engine/ssc/SscWriter.h +++ b/source/adios2/engine/ssc/SscWriter.h @@ -17,7 +17,7 @@ #include "adios2/toolkit/format/dataman/DataManSerializer.h" #include "adios2/toolkit/format/dataman/DataManSerializer.tcc" #include "adios2/toolkit/profiling/taustubs/tautimer.hpp" -#include "adios2/toolkit/transportman/stagingman/StagingMan.h" +#include "adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.h" namespace adios2 { diff --git a/source/adios2/engine/table/TableWriter.cpp b/source/adios2/engine/table/TableWriter.cpp index be151e94a7..a45d2fe41a 100644 --- a/source/adios2/engine/table/TableWriter.cpp +++ b/source/adios2/engine/table/TableWriter.cpp @@ -101,12 +101,12 @@ void TableWriter::EndStep() void TableWriter::ReplyThread() { - transportman::StagingMan receiveStagingMan; - receiveStagingMan.OpenReplier(m_AllAddresses[m_MpiRank], m_Timeout, + adios2::zmq::ZmqReqRep replier; + replier.OpenReplier(m_AllAddresses[m_MpiRank], m_Timeout, m_ReceiverBufferSize); while (m_Listening) { - auto request = receiveStagingMan.ReceiveRequest(); + auto request = replier.ReceiveRequest(); if (request == nullptr or request->empty()) { if (m_Verbosity >= 20) @@ -118,7 +118,7 @@ void TableWriter::ReplyThread() } m_Deserializer.PutPack(request); format::VecPtr reply = std::make_shared>(1, 'K'); - receiveStagingMan.SendReply(reply); + replier.SendReply(reply); if (m_Verbosity >= 1) { std::cout << "TableWriter::ReplyThread " << m_MpiRank diff --git a/source/adios2/engine/table/TableWriter.h b/source/adios2/engine/table/TableWriter.h index e052428584..a16f0960fc 100644 --- a/source/adios2/engine/table/TableWriter.h +++ b/source/adios2/engine/table/TableWriter.h @@ -16,7 +16,7 @@ #include "adios2/engine/bp4/BP4Writer.h" #include "adios2/toolkit/format/dataman/DataManSerializer.h" #include "adios2/toolkit/format/dataman/DataManSerializer.tcc" -#include "adios2/toolkit/transportman/stagingman/StagingMan.h" +#include "adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.h" #include "../../bindings/CXX11/adios2.h" @@ -98,7 +98,7 @@ class TableWriter : public Engine std::vector> m_Serializers; format::DataManSerializer m_Deserializer; - transportman::StagingMan m_SendStagingMan; + adios2::zmq::ZmqReqRep m_SendStagingMan; #define declare_type(T) \ void DoPutSync(Variable &, const T *) final; \ diff --git a/source/adios2/toolkit/transportman/wanman/WANMan.cpp b/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.cpp similarity index 79% rename from source/adios2/toolkit/transportman/wanman/WANMan.cpp rename to source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.cpp index b35c92bb2e..80c26a940a 100644 --- a/source/adios2/toolkit/transportman/wanman/WANMan.cpp +++ b/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.cpp @@ -2,13 +2,13 @@ * Distributed under the OSI-approved Apache License, Version 2.0. See * accompanying file Copyright.txt for details. * - * WANMan.cpp + * ZmqPubSub.cpp * * Created on: Jun 1, 2017 * Author: Jason Wang wangr1@ornl.gov */ -#include "WANMan.h" +#include "ZmqPubSub.h" #include "adios2/common/ADIOSMacros.h" #include "adios2/helper/adiosFunctions.h" @@ -17,12 +17,12 @@ namespace adios2 { -namespace transportman +namespace zmq { -WANMan::WANMan() {} +ZmqPubSub::ZmqPubSub() {} -WANMan::~WANMan() +ZmqPubSub::~ZmqPubSub() { auto start_time = std::chrono::system_clock::now(); while (true) @@ -50,27 +50,27 @@ WANMan::~WANMan() } } -void WANMan::OpenPublisher(const std::string &address, const int timeout) +void ZmqPubSub::OpenPublisher(const std::string &address, const int timeout) { m_Timeout = timeout; - m_Thread = std::thread(&WANMan::WriterThread, this, address); + m_Thread = std::thread(&ZmqPubSub::WriterThread, this, address); } -void WANMan::OpenSubscriber(const std::string &address, const int timeout, +void ZmqPubSub::OpenSubscriber(const std::string &address, const int timeout, const size_t bufferSize) { m_Timeout = timeout; m_Thread = - std::thread(&WANMan::ReaderThread, this, address, timeout, bufferSize); + std::thread(&ZmqPubSub::ReaderThread, this, address, timeout, bufferSize); } -void WANMan::PushBufferQueue(std::shared_ptr> buffer) +void ZmqPubSub::PushBufferQueue(std::shared_ptr> buffer) { std::lock_guard l(m_BufferQueueMutex); m_BufferQueue.push(buffer); } -std::shared_ptr> WANMan::PopBufferQueue() +std::shared_ptr> ZmqPubSub::PopBufferQueue() { std::lock_guard l(m_BufferQueueMutex); if (m_BufferQueue.empty()) @@ -85,7 +85,7 @@ std::shared_ptr> WANMan::PopBufferQueue() } } -void WANMan::WriterThread(const std::string &address) +void ZmqPubSub::WriterThread(const std::string &address) { void *context = zmq_ctx_new(); if (not context) @@ -113,14 +113,14 @@ void WANMan::WriterThread(const std::string &address) zmq_send(socket, buffer->data(), buffer->size(), ZMQ_DONTWAIT); if (m_Verbosity >= 5) { - std::cout << "WANMan::WriterThread sent package size " + std::cout << "ZmqPubSub::WriterThread sent package size " << buffer->size() << std::endl; } } } } -void WANMan::ReaderThread(const std::string &address, const int timeout, +void ZmqPubSub::ReaderThread(const std::string &address, const int timeout, const size_t receiverBufferSize) { void *context = zmq_ctx_new(); @@ -156,12 +156,12 @@ void WANMan::ReaderThread(const std::string &address, const int timeout, PushBufferQueue(buff); if (m_Verbosity >= 5) { - std::cout << "WANMan::ReaderThread received package size " + std::cout << "ZmqPubSub::ReaderThread received package size " << buff->size() << std::endl; } } } } -} // end namespace transportman +} // end namespace zmq } // end namespace adios2 diff --git a/source/adios2/toolkit/transportman/wanman/WANMan.h b/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.h similarity index 81% rename from source/adios2/toolkit/transportman/wanman/WANMan.h rename to source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.h index b7d435652e..bfcb6a79bb 100644 --- a/source/adios2/toolkit/transportman/wanman/WANMan.h +++ b/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.h @@ -2,14 +2,14 @@ * Distributed under the OSI-approved Apache License, Version 2.0. See * accompanying file Copyright.txt for details. * - * WANMan.h + * ZmqPubSub.h * * Created on: Jun 1, 2017 * Author: Jason Wang wangr1@ornl.gov */ -#ifndef ADIOS2_TOOLKIT_TRANSPORTMAN_WANMAN_WANMAN_H_ -#define ADIOS2_TOOLKIT_TRANSPORTMAN_WANMAN_WANMAN_H_ +#ifndef ADIOS2_TOOLKIT_ZMQ_ZMQPUBSUB_H_ +#define ADIOS2_TOOLKIT_ZMQ_ZMQPUBSUB_H_ #include #include @@ -20,15 +20,15 @@ namespace adios2 { -namespace transportman +namespace zmq { -class WANMan +class ZmqPubSub { public: - WANMan(); - ~WANMan(); + ZmqPubSub(); + ~ZmqPubSub(); void OpenPublisher(const std::string &address, const int timeout); void OpenSubscriber(const std::string &address, const int timeout, @@ -54,7 +54,7 @@ class WANMan int m_Verbosity = 0; }; -} // end namespace transportman +} // end namespace zmq } // end namespace adios2 -#endif /* ADIOS2_TOOLKIT_TRANSPORTMAN_WANMAN_WANMAN_H_ */ +#endif /* ADIOS2_TOOLKIT_ZMQ_ZMQPUBSUB_H_ */ diff --git a/source/adios2/toolkit/transportman/stagingman/StagingMan.cpp b/source/adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.cpp similarity index 87% rename from source/adios2/toolkit/transportman/stagingman/StagingMan.cpp rename to source/adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.cpp index 0c64169e8f..4c247f388c 100644 --- a/source/adios2/toolkit/transportman/stagingman/StagingMan.cpp +++ b/source/adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.cpp @@ -2,22 +2,23 @@ * Distributed under the OSI-approved Apache License, Version 2.0. See * accompanying file Copyright.txt for details. * - * StagingMan.cpp + * ZmqReqRep.cpp * * Created on: Oct 1, 2018 * Author: Jason Wang wangr1@ornl.gov */ #include +#include -#include "StagingMan.h" +#include "ZmqReqRep.h" namespace adios2 { -namespace transportman +namespace zmq { -StagingMan::StagingMan() +ZmqReqRep::ZmqReqRep() { m_Context = zmq_ctx_new(); if (not m_Context) @@ -26,7 +27,7 @@ StagingMan::StagingMan() } } -StagingMan::~StagingMan() +ZmqReqRep::~ZmqReqRep() { if (m_Socket) { @@ -38,14 +39,14 @@ StagingMan::~StagingMan() } } -void StagingMan::OpenRequester(const int timeout, +void ZmqReqRep::OpenRequester(const int timeout, const size_t receiverBufferSize) { m_Timeout = timeout; m_ReceiverBuffer.reserve(receiverBufferSize); } -void StagingMan::OpenReplier(const std::string &address, const int timeout, +void ZmqReqRep::OpenReplier(const std::string &address, const int timeout, const size_t receiverBufferSize) { m_Timeout = timeout; @@ -67,7 +68,7 @@ void StagingMan::OpenReplier(const std::string &address, const int timeout, zmq_setsockopt(m_Socket, ZMQ_LINGER, &m_Timeout, sizeof(m_Timeout)); } -std::shared_ptr> StagingMan::ReceiveRequest() +std::shared_ptr> ZmqReqRep::ReceiveRequest() { int bytes = zmq_recv(m_Socket, m_ReceiverBuffer.data(), m_ReceiverBuffer.capacity(), 0); @@ -80,24 +81,22 @@ std::shared_ptr> StagingMan::ReceiveRequest() return request; } -void StagingMan::SendReply(std::shared_ptr> reply) +void ZmqReqRep::SendReply(std::shared_ptr> reply) { zmq_send(m_Socket, reply->data(), reply->size(), 0); } -void StagingMan::SendReply(const void *reply, const size_t size) +void ZmqReqRep::SendReply(const void *reply, const size_t size) { zmq_send(m_Socket, reply, size, 0); } std::shared_ptr> -StagingMan::Request(const void *request, const size_t size, +ZmqReqRep::Request(const void *request, const size_t size, const std::string &address) { auto reply = std::make_shared>(); - void *socket = zmq_socket(m_Context, ZMQ_REQ); - ; int ret = 1; auto start_time = std::chrono::system_clock::now(); @@ -155,5 +154,5 @@ StagingMan::Request(const void *request, const size_t size, return reply; } -} // end namespace transportman +} // end namespace zmq } // end namespace adios2 diff --git a/source/adios2/toolkit/transportman/stagingman/StagingMan.h b/source/adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.h similarity index 78% rename from source/adios2/toolkit/transportman/stagingman/StagingMan.h rename to source/adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.h index 61807d5b8e..41ae3de6be 100644 --- a/source/adios2/toolkit/transportman/stagingman/StagingMan.h +++ b/source/adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.h @@ -2,14 +2,14 @@ * Distributed under the OSI-approved Apache License, Version 2.0. See * accompanying file Copyright.txt for details. * - * StagingMan.h + * ZmqReqRep.h * * Created on: Oct 1, 2018 * Author: Jason Wang wangr1@ornl.gov */ -#ifndef ADIOS2_TOOLKIT_TRANSPORTMAN_STAGINGMAN_H_ -#define ADIOS2_TOOLKIT_TRANSPORTMAN_STAGINGMAN_H_ +#ifndef ADIOS2_TOOLKIT_ZMQ_ZMQREQREP_H_ +#define ADIOS2_TOOLKIT_ZMQ_ZMQREQREP_H_ #include "adios2/core/IO.h" #include "adios2/core/Operator.h" @@ -18,15 +18,15 @@ namespace adios2 { -namespace transportman +namespace zmq { -class StagingMan +class ZmqReqRep { public: - StagingMan(); - ~StagingMan(); + ZmqReqRep(); + ~ZmqReqRep(); // requester void OpenRequester(const int timeout, const size_t receiverBufferSize); @@ -49,7 +49,7 @@ class StagingMan void *m_Socket = nullptr; }; -} // end namespace transportman +} // end namespace zmq } // end namespace adios2 -#endif /* ADIOS2_TOOLKIT_TRANSPORTMAN_STAGINGMAN_H_ */ +#endif /* ADIOS2_TOOLKIT_ZMQ_ZMQREQREP_H_ */ From 9e5923fd05c789a066f1b268b9465e4168fe318b Mon Sep 17 00:00:00 2001 From: Ruonan Wang Date: Mon, 19 Aug 2019 10:26:17 -0400 Subject: [PATCH 4/7] added missing headers --- source/adios2/engine/dataman/DataManCommon.h | 2 +- source/adios2/engine/table/TableWriter.cpp | 2 +- .../adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.cpp | 16 ++++++++-------- source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.h | 3 --- .../adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.cpp | 9 +++++---- 5 files changed, 15 insertions(+), 17 deletions(-) diff --git a/source/adios2/engine/dataman/DataManCommon.h b/source/adios2/engine/dataman/DataManCommon.h index d312f1973e..78b09ccbbe 100644 --- a/source/adios2/engine/dataman/DataManCommon.h +++ b/source/adios2/engine/dataman/DataManCommon.h @@ -17,8 +17,8 @@ #include "adios2/helper/adiosSystem.h" #include "adios2/toolkit/format/dataman/DataManSerializer.h" #include "adios2/toolkit/format/dataman/DataManSerializer.tcc" -#include "adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.h" #include "adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.h" +#include "adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.h" namespace adios2 { diff --git a/source/adios2/engine/table/TableWriter.cpp b/source/adios2/engine/table/TableWriter.cpp index a45d2fe41a..69e0a28b3b 100644 --- a/source/adios2/engine/table/TableWriter.cpp +++ b/source/adios2/engine/table/TableWriter.cpp @@ -103,7 +103,7 @@ void TableWriter::ReplyThread() { adios2::zmq::ZmqReqRep replier; replier.OpenReplier(m_AllAddresses[m_MpiRank], m_Timeout, - m_ReceiverBufferSize); + m_ReceiverBufferSize); while (m_Listening) { auto request = replier.ReceiveRequest(); diff --git a/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.cpp b/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.cpp index 80c26a940a..b863ec658c 100644 --- a/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.cpp +++ b/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.cpp @@ -8,13 +8,13 @@ * Author: Jason Wang wangr1@ornl.gov */ -#include "ZmqPubSub.h" - -#include "adios2/common/ADIOSMacros.h" -#include "adios2/helper/adiosFunctions.h" +#include +#include #include +#include "ZmqPubSub.h" + namespace adios2 { namespace zmq @@ -57,11 +57,11 @@ void ZmqPubSub::OpenPublisher(const std::string &address, const int timeout) } void ZmqPubSub::OpenSubscriber(const std::string &address, const int timeout, - const size_t bufferSize) + const size_t bufferSize) { m_Timeout = timeout; - m_Thread = - std::thread(&ZmqPubSub::ReaderThread, this, address, timeout, bufferSize); + m_Thread = std::thread(&ZmqPubSub::ReaderThread, this, address, timeout, + bufferSize); } void ZmqPubSub::PushBufferQueue(std::shared_ptr> buffer) @@ -121,7 +121,7 @@ void ZmqPubSub::WriterThread(const std::string &address) } void ZmqPubSub::ReaderThread(const std::string &address, const int timeout, - const size_t receiverBufferSize) + const size_t receiverBufferSize) { void *context = zmq_ctx_new(); if (not context) diff --git a/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.h b/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.h index bfcb6a79bb..3adcac2def 100644 --- a/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.h +++ b/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.h @@ -15,9 +15,6 @@ #include #include -#include "adios2/core/IO.h" -#include "adios2/core/Operator.h" - namespace adios2 { namespace zmq diff --git a/source/adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.cpp b/source/adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.cpp index 4c247f388c..2b5ea36698 100644 --- a/source/adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.cpp +++ b/source/adios2/toolkit/zmq/zmqreqrep/ZmqReqRep.cpp @@ -8,8 +8,9 @@ * Author: Jason Wang wangr1@ornl.gov */ -#include +#include #include +#include #include "ZmqReqRep.h" @@ -40,14 +41,14 @@ ZmqReqRep::~ZmqReqRep() } void ZmqReqRep::OpenRequester(const int timeout, - const size_t receiverBufferSize) + const size_t receiverBufferSize) { m_Timeout = timeout; m_ReceiverBuffer.reserve(receiverBufferSize); } void ZmqReqRep::OpenReplier(const std::string &address, const int timeout, - const size_t receiverBufferSize) + const size_t receiverBufferSize) { m_Timeout = timeout; m_ReceiverBuffer.reserve(receiverBufferSize); @@ -93,7 +94,7 @@ void ZmqReqRep::SendReply(const void *reply, const size_t size) std::shared_ptr> ZmqReqRep::Request(const void *request, const size_t size, - const std::string &address) + const std::string &address) { auto reply = std::make_shared>(); void *socket = zmq_socket(m_Context, ZMQ_REQ); From 9ee9bf10e71e472af4a25c6a4358b9e5a3f4a1e9 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Mon, 19 Aug 2019 11:21:11 -0400 Subject: [PATCH 5/7] added more missing headers to pass CI --- source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.cpp b/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.cpp index b863ec658c..e918c6b5d5 100644 --- a/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.cpp +++ b/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.cpp @@ -9,6 +9,7 @@ */ #include +#include #include #include From d0094596e6f61fee8aa5fe1c311160f729365945 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 20 Aug 2019 15:04:41 -0400 Subject: [PATCH 6/7] modified DataMan tests to adapt to latest changes --- .../adios2/engine/dataman/DataManCommon.cpp | 1 + .../adios2/engine/dataman/DataManReader.cpp | 36 +++++++++++++-- source/adios2/engine/dataman/DataManReader.h | 1 + .../format/dataman/DataManSerializer.cpp | 18 -------- .../toolkit/zmq/zmqpubsub/ZmqPubSub.cpp | 18 ++++++++ testing/adios2/engine/dataman/TestDataMan.h | 45 ++++++++----------- .../engine/dataman/TestDataManP2P1D.cpp | 3 +- .../dataman/TestDataManP2P2DMemSelect.cpp | 44 ++++++++++-------- .../dataman/TestDataManP2P3DMemSelect.cpp | 20 +++++---- .../engine/dataman/TestDataManP2PBzip2D.cpp | 3 +- .../engine/dataman/TestDataManP2PSz2D.cpp | 3 +- .../engine/dataman/TestDataManP2PZfp2D.cpp | 3 +- .../engine/dataman/TestDataManSubscribe1D.cpp | 3 +- 13 files changed, 118 insertions(+), 80 deletions(-) diff --git a/source/adios2/engine/dataman/DataManCommon.cpp b/source/adios2/engine/dataman/DataManCommon.cpp index aad0899e5e..c54b2eb485 100644 --- a/source/adios2/engine/dataman/DataManCommon.cpp +++ b/source/adios2/engine/dataman/DataManCommon.cpp @@ -30,6 +30,7 @@ DataManCommon::DataManCommon(const std::string engineType, IO &io, GetParameter(m_IO.m_Parameters, "Port", m_Port); GetParameter(m_IO.m_Parameters, "StagingMode", m_StagingMode); GetParameter(m_IO.m_Parameters, "Timeout", m_Timeout); + GetParameter(m_IO.m_Parameters, "Verbose", m_Verbosity); } DataManCommon::~DataManCommon() {} diff --git a/source/adios2/engine/dataman/DataManReader.cpp b/source/adios2/engine/dataman/DataManReader.cpp index 66d36234f6..6c916f3057 100644 --- a/source/adios2/engine/dataman/DataManReader.cpp +++ b/source/adios2/engine/dataman/DataManReader.cpp @@ -28,8 +28,11 @@ DataManReader::DataManReader(IO &io, const std::string &name, const Mode mode, { GetParameter(m_IO.m_Parameters, "AlwaysProvideLatestTimestep", m_ProvideLatest); + m_ZmqRequester.OpenRequester(m_Timeout, m_ReceiverBufferSize); + m_Verbosity = 10; + if (m_StagingMode == "wide") { if (m_IPAddress.empty()) @@ -42,10 +45,20 @@ DataManReader::DataManReader(IO &io, const std::string &name, const Mode mode, std::string request = "Address"; auto reply = m_ZmqRequester.Request(request.data(), request.size(), address); + + auto start_time = std::chrono::system_clock::now(); while (reply == nullptr or reply->empty()) { reply = m_ZmqRequester.Request(request.data(), request.size(), address); + auto now_time = std::chrono::system_clock::now(); + auto duration = std::chrono::duration_cast( + now_time - start_time); + if (duration.count() > m_Timeout) + { + m_InitFailed = true; + return; + } } auto addJson = nlohmann::json::parse(*reply); m_DataAddresses = @@ -70,6 +83,11 @@ DataManReader::DataManReader(IO &io, const std::string &name, const Mode mode, DataManReader::~DataManReader() { + if (m_Verbosity >= 5) + { + std::cout << "DataManReader::~DataManReader() Step " << m_CurrentStep + << std::endl; + } if (not m_IsClosed) { DoClose(); @@ -85,7 +103,7 @@ StepStatus DataManReader::BeginStep(StepMode stepMode, << m_CurrentStep << std::endl; } - if (m_CurrentStep == m_FinalStep && m_CurrentStep > 0) + if (m_InitFailed or (m_CurrentStep == m_FinalStep && m_CurrentStep > 0)) { return StepStatus::EndOfStream; } @@ -200,11 +218,21 @@ void DataManReader::SubscriberThread() auto buffer = z->PopBufferQueue(); if (buffer != nullptr && buffer->size() > 0) { - int ret = m_DataManSerializer.PutPack(buffer); - if (ret > 0) + // check if is control signal + if (buffer->size() < 64) { - m_FinalStep = ret; + try + { + nlohmann::json jmsg = + nlohmann::json::parse(buffer->data()); + m_FinalStep = jmsg["FinalStep"].get(); + continue; + } + catch (std::exception) + { + } } + m_DataManSerializer.PutPack(buffer); } } } diff --git a/source/adios2/engine/dataman/DataManReader.h b/source/adios2/engine/dataman/DataManReader.h index 5c8fba2f88..32cc2f1ea5 100644 --- a/source/adios2/engine/dataman/DataManReader.h +++ b/source/adios2/engine/dataman/DataManReader.h @@ -36,6 +36,7 @@ class DataManReader : public DataManCommon private: bool m_ProvideLatest = false; + bool m_InitFailed = false; size_t m_FinalStep = std::numeric_limits::max(); adios2::zmq::ZmqReqRep m_ZmqRequester; std::vector m_DataAddresses; diff --git a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp index 8638092568..f9fa744c5e 100644 --- a/source/adios2/toolkit/format/dataman/DataManSerializer.cpp +++ b/source/adios2/toolkit/format/dataman/DataManSerializer.cpp @@ -555,29 +555,11 @@ int DataManSerializer::PutPack(const VecPtr data) { return -1; } - - // check if is control signal - if (data->size() < 128) - { - try - { - nlohmann::json metaj = nlohmann::json::parse(data->data()); - size_t finalStep = metaj["FinalStep"]; - return finalStep; - } - catch (std::exception) - { - } - } - - // if not control signal then go through standard deserialization uint64_t metaPosition = (reinterpret_cast(data->data()))[0]; uint64_t metaSize = (reinterpret_cast(data->data()))[1]; nlohmann::json j = DeserializeJson(data->data() + metaPosition, metaSize); - JsonToDataManVarMap(j, data); - return 0; } diff --git a/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.cpp b/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.cpp index e918c6b5d5..808a246b90 100644 --- a/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.cpp +++ b/source/adios2/toolkit/zmq/zmqpubsub/ZmqPubSub.cpp @@ -119,6 +119,15 @@ void ZmqPubSub::WriterThread(const std::string &address) } } } + + if (socket) + { + zmq_close(socket); + } + if (context) + { + zmq_ctx_destroy(context); + } } void ZmqPubSub::ReaderThread(const std::string &address, const int timeout, @@ -162,6 +171,15 @@ void ZmqPubSub::ReaderThread(const std::string &address, const int timeout, } } } + + if (socket) + { + zmq_close(socket); + } + if (context) + { + zmq_ctx_destroy(context); + } } } // end namespace zmq diff --git a/testing/adios2/engine/dataman/TestDataMan.h b/testing/adios2/engine/dataman/TestDataMan.h index 49e65f22c6..45317cb29b 100644 --- a/testing/adios2/engine/dataman/TestDataMan.h +++ b/testing/adios2/engine/dataman/TestDataMan.h @@ -196,8 +196,8 @@ void DataManReaderP2P(const Dims &shape, const Dims &start, const Dims &count, std::vector> myComplexes(datasize); std::vector> myDComplexes(datasize); bool received_steps = false; - size_t i; - for (i = 0; i < steps; ++i) + size_t currentStep; + while (true) { adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5); if (status == adios2::StepStatus::OK) @@ -214,8 +214,7 @@ void DataManReaderP2P(const Dims &shape, const Dims &start, const Dims &count, } std::cout << std::endl; } - size_t currentStep = dataManReader.CurrentStep(); - // ASSERT_EQ(i, currentStep); + currentStep = dataManReader.CurrentStep(); adios2::Variable bpChars = dataManIO.InquireVariable("bpChars"); adios2::Variable bpUChars = @@ -271,12 +270,16 @@ void DataManReaderP2P(const Dims &shape, const Dims &start, const Dims &count, VerifyData(myDComplexes, currentStep); dataManReader.EndStep(); } - else + else if (status == adios2::StepStatus::EndOfStream) { - std::cout << "DataManReader end of stream at Step " << i + std::cout << "DataManReader end of stream at Step " << currentStep << std::endl; break; } + else if (status == adios2::StepStatus::NotReady) + { + continue; + } } if (received_steps) { @@ -284,9 +287,8 @@ void DataManReaderP2P(const Dims &shape, const Dims &start, const Dims &count, ASSERT_EQ(110, attInt.Data()[0]); ASSERT_NE(111, attInt.Data()[0]); } - // ASSERT_EQ(i, steps); + ASSERT_EQ(currentStep + 1, steps); dataManReader.Close(); - print_lines = 0; } void DataManReaderSubscribe(const Dims &shape, const Dims &start, @@ -307,37 +309,28 @@ void DataManReaderSubscribe(const Dims &shape, const Dims &start, dataManIO.SetParameters(engineParams); adios2::Engine dataManReader = dataManIO.Open("stream", adios2::Mode::Read); adios2::Variable bpFloats; - size_t i = 0; auto start_time = std::chrono::system_clock::now(); - while (i < steps - 1) + size_t currentStep; + while (true) { - auto now_time = std::chrono::system_clock::now(); - auto duration = std::chrono::duration_cast( - now_time - start_time); - if (duration.count() > timeout) - { - std::cout << "DataMan Timeout. Last step received: " << i - << std::endl; - ASSERT_GT(i, 0); - break; - } adios2::StepStatus status = dataManReader.BeginStep(); if (status == adios2::StepStatus::OK) { - while (not bpFloats) - { - bpFloats = dataManIO.InquireVariable("bpFloats"); - } + bpFloats = dataManIO.InquireVariable("bpFloats"); bpFloats.SetSelection({start, count}); dataManReader.Get(bpFloats, myFloats.data(), adios2::Mode::Sync); - i = dataManReader.CurrentStep(); - VerifyData(myFloats, i); + currentStep = dataManReader.CurrentStep(); + VerifyData(myFloats, currentStep); } else if (status == adios2::StepStatus::EndOfStream) { break; } + else if (status == adios2::StepStatus::NotReady) + { + continue; + } dataManReader.EndStep(); } dataManReader.Close(); diff --git a/testing/adios2/engine/dataman/TestDataManP2P1D.cpp b/testing/adios2/engine/dataman/TestDataManP2P1D.cpp index 7200a1cd9f..2ed056f4dd 100644 --- a/testing/adios2/engine/dataman/TestDataManP2P1D.cpp +++ b/testing/adios2/engine/dataman/TestDataManP2P1D.cpp @@ -23,7 +23,7 @@ TEST_F(DataManEngineTest, WriteRead_1D_P2P) Dims shape = {10}; Dims start = {0}; Dims count = {10}; - size_t steps = 200; + size_t steps = 1000; adios2::Params engineParams = {{"IPAddress", "127.0.0.1"}, {"Port", "12306"}}; @@ -31,6 +31,7 @@ TEST_F(DataManEngineTest, WriteRead_1D_P2P) auto r = std::thread(DataManReaderP2P, shape, start, count, steps, engineParams); std::cout << "Reader thread started" << std::endl; + std::this_thread::sleep_for(std::chrono::seconds(1)); auto w = std::thread(DataManWriter, shape, start, count, steps, engineParams); std::cout << "Writer thread started" << std::endl; diff --git a/testing/adios2/engine/dataman/TestDataManP2P2DMemSelect.cpp b/testing/adios2/engine/dataman/TestDataManP2P2DMemSelect.cpp index e94c14d5be..df76f44eb8 100644 --- a/testing/adios2/engine/dataman/TestDataManP2P2DMemSelect.cpp +++ b/testing/adios2/engine/dataman/TestDataManP2P2DMemSelect.cpp @@ -218,20 +218,10 @@ void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start, std::vector> myComplexes(datasize); std::vector> myDComplexes(datasize); bool received_steps = false; - size_t i; - for (i = 0; i < steps; ++i) + size_t currentStep; + while (true) { - GenData(myChars, i, memStart, memCount, shape); - GenData(myUChars, i, memStart, memCount, shape); - GenData(myShorts, i, memStart, memCount, shape); - GenData(myUShorts, i, memStart, memCount, shape); - GenData(myInts, i, memStart, memCount, shape); - GenData(myUInts, i, memStart, memCount, shape); - GenData(myFloats, i, memStart, memCount, shape); - GenData(myDoubles, i, memStart, memCount, shape); - GenData(myComplexes, i, memStart, memCount, shape); - GenData(myDComplexes, i, memStart, memCount, shape); - adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5); + adios2::StepStatus status = dataManReader.BeginStep(); if (status == adios2::StepStatus::OK) { received_steps = true; @@ -246,8 +236,17 @@ void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start, } std::cout << std::endl; } - size_t currentStep = dataManReader.CurrentStep(); - // ASSERT_EQ(i, currentStep); + currentStep = dataManReader.CurrentStep(); + GenData(myChars, currentStep, memStart, memCount, shape); + GenData(myUChars, currentStep, memStart, memCount, shape); + GenData(myShorts, currentStep, memStart, memCount, shape); + GenData(myUShorts, currentStep, memStart, memCount, shape); + GenData(myInts, currentStep, memStart, memCount, shape); + GenData(myUInts, currentStep, memStart, memCount, shape); + GenData(myFloats, currentStep, memStart, memCount, shape); + GenData(myDoubles, currentStep, memStart, memCount, shape); + GenData(myComplexes, currentStep, memStart, memCount, shape); + GenData(myDComplexes, currentStep, memStart, memCount, shape); adios2::Variable bpChars = dataManIO.InquireVariable("bpChars"); adios2::Variable bpUChars = @@ -320,12 +319,14 @@ void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start, shape); dataManReader.EndStep(); } - else + else if (status == adios2::StepStatus::EndOfStream) { - std::cout << "DataManReader end of stream at Step " << i - << std::endl; break; } + else if (status == adios2::StepStatus::NotReady) + { + continue; + } } if (received_steps) { @@ -335,7 +336,7 @@ void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start, ASSERT_EQ(110, attInt.Data()[0]); ASSERT_NE(111, attInt.Data()[0]); } - // ASSERT_EQ(i, steps); + ASSERT_EQ(currentStep + 1, steps); dataManReader.Close(); print_lines = 0; } @@ -356,15 +357,20 @@ TEST_F(DataManEngineTest, WriteRead_2D_MemSelect) adios2::Params engineParams = {{"IPAddress", "127.0.0.1"}, {"Port", "12308"}}; // run workflow + auto r = std::thread(DataManReaderP2PMemSelect, shape, start, count, memstart, memcount, steps, engineParams); std::cout << "Reader thread started" << std::endl; + std::this_thread::sleep_for(std::chrono::seconds(1)); + auto w = std::thread(DataManWriterP2PMemSelect, shape, start, count, steps, engineParams); std::cout << "Writer thread started" << std::endl; + w.join(); std::cout << "Writer thread ended" << std::endl; + r.join(); std::cout << "Reader thread ended" << std::endl; } diff --git a/testing/adios2/engine/dataman/TestDataManP2P3DMemSelect.cpp b/testing/adios2/engine/dataman/TestDataManP2P3DMemSelect.cpp index c0a7fa4ef0..fe5f4dd4f8 100644 --- a/testing/adios2/engine/dataman/TestDataManP2P3DMemSelect.cpp +++ b/testing/adios2/engine/dataman/TestDataManP2P3DMemSelect.cpp @@ -141,7 +141,7 @@ void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start, adios2::Engine dataManReader = dataManIO.Open("stream", adios2::Mode::Read); std::vector myInts = reader_data; size_t i; - for (i = 0; i < steps; ++i) + while (true) { adios2::StepStatus status = dataManReader.BeginStep(StepMode::Read, 5); if (status == adios2::StepStatus::OK) @@ -158,26 +158,25 @@ void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start, std::cout << std::endl; } size_t currentStep = dataManReader.CurrentStep(); - // ASSERT_EQ(i, currentStep); adios2::Variable bpInts = dataManIO.InquireVariable("bpInts"); - bpInts.SetSelection({start, count}); - bpInts.SetMemorySelection({memStart, memCount}); - dataManReader.Get(bpInts, myInts.data(), adios2::Mode::Sync); VerifyData(myInts.data(), currentStep, memStart, memCount, shape); dataManReader.EndStep(); } - else + else if (status == adios2::StepStatus::EndOfStream) { std::cout << "DataManReader end of stream at Step " << i << std::endl; break; } + else if (status == adios2::StepStatus::NotReady) + { + continue; + } } - // ASSERT_EQ(i, steps); dataManReader.Close(); print_lines = 0; } @@ -186,19 +185,24 @@ void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start, TEST_F(DataManEngineTest, WriteRead_3D_MemSelect) { - size_t steps = 10; + size_t steps = 10000; adios2::Params engineParams = {{"IPAddress", "127.0.0.1"}, {"Port", "12310"}}; // run workflow + auto r = std::thread(DataManReaderP2PMemSelect, shape, start, count, memstart, memcount, steps, engineParams); std::cout << "Reader thread started" << std::endl; + std::this_thread::sleep_for(std::chrono::seconds(1)); + auto w = std::thread(DataManWriterP2PMemSelect, shape, start, count, steps, engineParams); std::cout << "Writer thread started" << std::endl; + w.join(); std::cout << "Writer thread ended" << std::endl; + r.join(); std::cout << "Reader thread ended" << std::endl; } diff --git a/testing/adios2/engine/dataman/TestDataManP2PBzip2D.cpp b/testing/adios2/engine/dataman/TestDataManP2PBzip2D.cpp index 18fbd8a1a7..7ca4fe7c33 100644 --- a/testing/adios2/engine/dataman/TestDataManP2PBzip2D.cpp +++ b/testing/adios2/engine/dataman/TestDataManP2PBzip2D.cpp @@ -24,7 +24,7 @@ TEST_F(DataManEngineTest, DISABLED_WriteRead_2D_P2P_BZip2) Dims shape = {10, 10}; Dims start = {0, 0}; Dims count = {10, 10}; - size_t steps = 200; + size_t steps = 10000; adios2::Params engineParams = {{"IPAddress", "127.0.0.1"}, {"Port", "12312"}}; @@ -32,6 +32,7 @@ TEST_F(DataManEngineTest, DISABLED_WriteRead_2D_P2P_BZip2) auto r = std::thread(DataManReaderP2P, shape, start, count, steps, engineParams); std::cout << "Reader thread started" << std::endl; + std::this_thread::sleep_for(std::chrono::seconds(1)); auto w = std::thread(DataManWriter, shape, start, count, steps, engineParams); std::cout << "Writer thread started" << std::endl; diff --git a/testing/adios2/engine/dataman/TestDataManP2PSz2D.cpp b/testing/adios2/engine/dataman/TestDataManP2PSz2D.cpp index 7627aa7efe..9b0869fac2 100644 --- a/testing/adios2/engine/dataman/TestDataManP2PSz2D.cpp +++ b/testing/adios2/engine/dataman/TestDataManP2PSz2D.cpp @@ -24,13 +24,14 @@ TEST_F(DataManEngineTest, WriteRead_2D_P2P_SZ) Dims shape = {10, 10}; Dims start = {0, 0}; Dims count = {6, 8}; - size_t steps = 20; + size_t steps = 1000; adios2::Params engineParams = {{"IPAddress", "127.0.0.1"}, {"Port", "12314"}}; // run workflow auto r = std::thread(DataManReaderP2P, shape, start, count, steps, engineParams); std::cout << "Reader thread started" << std::endl; + std::this_thread::sleep_for(std::chrono::seconds(1)); auto w = std::thread(DataManWriter, shape, start, count, steps, engineParams); std::cout << "Writer thread started" << std::endl; diff --git a/testing/adios2/engine/dataman/TestDataManP2PZfp2D.cpp b/testing/adios2/engine/dataman/TestDataManP2PZfp2D.cpp index 7372bd1e41..aa6d0a77e0 100644 --- a/testing/adios2/engine/dataman/TestDataManP2PZfp2D.cpp +++ b/testing/adios2/engine/dataman/TestDataManP2PZfp2D.cpp @@ -24,7 +24,7 @@ TEST_F(DataManEngineTest, WriteRead_2D_P2P_Zfp) Dims shape = {5, 5}; Dims start = {0, 0}; Dims count = {5, 5}; - size_t steps = 200; + size_t steps = 1000; adios2::Params engineParams = {{"IPAddress", "127.0.0.1"}, {"Port", "12316"}}; @@ -32,6 +32,7 @@ TEST_F(DataManEngineTest, WriteRead_2D_P2P_Zfp) auto r = std::thread(DataManReaderP2P, shape, start, count, steps, engineParams); std::cout << "Reader thread started" << std::endl; + std::this_thread::sleep_for(std::chrono::seconds(1)); auto w = std::thread(DataManWriter, shape, start, count, steps, engineParams); std::cout << "Writer thread started" << std::endl; diff --git a/testing/adios2/engine/dataman/TestDataManSubscribe1D.cpp b/testing/adios2/engine/dataman/TestDataManSubscribe1D.cpp index fc23805cbe..853ee2e87d 100644 --- a/testing/adios2/engine/dataman/TestDataManSubscribe1D.cpp +++ b/testing/adios2/engine/dataman/TestDataManSubscribe1D.cpp @@ -24,7 +24,7 @@ TEST_F(DataManEngineTest, WriteRead_1D_Subscribe) Dims shape = {10}; Dims start = {0}; Dims count = {10}; - size_t steps = 1000; + size_t steps = 10000; adios2::Params engineParams = {{"IPAddress", "127.0.0.1"}, {"Port", "12318"}}; @@ -32,6 +32,7 @@ TEST_F(DataManEngineTest, WriteRead_1D_Subscribe) auto r = std::thread(DataManReaderSubscribe, shape, start, count, steps, engineParams, timeout); std::cout << "Reader thread started" << std::endl; + std::this_thread::sleep_for(std::chrono::seconds(1)); auto w = std::thread(DataManWriter, shape, start, count, steps, engineParams); std::cout << "Writer thread started" << std::endl; From 2ed2b803583caf3487f36a7691fe95b2538cec41 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 20 Aug 2019 15:16:13 -0400 Subject: [PATCH 7/7] minor fix for dataman tests --- source/adios2/engine/dataman/DataManReader.cpp | 2 -- testing/adios2/engine/dataman/TestDataMan.h | 2 +- testing/adios2/engine/dataman/TestDataManP2P2DMemSelect.cpp | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/source/adios2/engine/dataman/DataManReader.cpp b/source/adios2/engine/dataman/DataManReader.cpp index 6c916f3057..7a24d5507e 100644 --- a/source/adios2/engine/dataman/DataManReader.cpp +++ b/source/adios2/engine/dataman/DataManReader.cpp @@ -31,8 +31,6 @@ DataManReader::DataManReader(IO &io, const std::string &name, const Mode mode, m_ZmqRequester.OpenRequester(m_Timeout, m_ReceiverBufferSize); - m_Verbosity = 10; - if (m_StagingMode == "wide") { if (m_IPAddress.empty()) diff --git a/testing/adios2/engine/dataman/TestDataMan.h b/testing/adios2/engine/dataman/TestDataMan.h index 45317cb29b..ba69748b0f 100644 --- a/testing/adios2/engine/dataman/TestDataMan.h +++ b/testing/adios2/engine/dataman/TestDataMan.h @@ -286,8 +286,8 @@ void DataManReaderP2P(const Dims &shape, const Dims &start, const Dims &count, auto attInt = dataManIO.InquireAttribute("AttInt"); ASSERT_EQ(110, attInt.Data()[0]); ASSERT_NE(111, attInt.Data()[0]); + ASSERT_EQ(currentStep + 1, steps); } - ASSERT_EQ(currentStep + 1, steps); dataManReader.Close(); } diff --git a/testing/adios2/engine/dataman/TestDataManP2P2DMemSelect.cpp b/testing/adios2/engine/dataman/TestDataManP2P2DMemSelect.cpp index df76f44eb8..c6ee223f5a 100644 --- a/testing/adios2/engine/dataman/TestDataManP2P2DMemSelect.cpp +++ b/testing/adios2/engine/dataman/TestDataManP2P2DMemSelect.cpp @@ -335,8 +335,8 @@ void DataManReaderP2PMemSelect(const Dims &shape, const Dims &start, << ", expected 110" << std::endl; ASSERT_EQ(110, attInt.Data()[0]); ASSERT_NE(111, attInt.Data()[0]); + ASSERT_EQ(currentStep + 1, steps); } - ASSERT_EQ(currentStep + 1, steps); dataManReader.Close(); print_lines = 0; }