diff --git a/bindings/CXX11/CMakeLists.txt b/bindings/CXX11/CMakeLists.txt index ac043ead3b..51361c3152 100644 --- a/bindings/CXX11/CMakeLists.txt +++ b/bindings/CXX11/CMakeLists.txt @@ -11,6 +11,7 @@ target_sources(adios2 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/adios2/cxx11/IO.cpp ${CMAKE_CURRENT_SOURCE_DIR}/adios2/cxx11/IO.tcc ${CMAKE_CURRENT_SOURCE_DIR}/adios2/cxx11/Operator.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/adios2/cxx11/Query.cpp ${CMAKE_CURRENT_SOURCE_DIR}/adios2/cxx11/Types.cpp ${CMAKE_CURRENT_SOURCE_DIR}/adios2/cxx11/Types.tcc ${CMAKE_CURRENT_SOURCE_DIR}/adios2/cxx11/Variable.cpp @@ -40,6 +41,7 @@ install( adios2/cxx11/Attribute.h adios2/cxx11/Engine.h adios2/cxx11/Operator.h + adios2/cxx11/Query.h adios2/cxx11/Types.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/adios2/cxx11 ) diff --git a/bindings/CXX11/adios2.h b/bindings/CXX11/adios2.h index d674c2f235..752f787605 100644 --- a/bindings/CXX11/adios2.h +++ b/bindings/CXX11/adios2.h @@ -20,6 +20,7 @@ #include "adios2/cxx11/Engine.h" #include "adios2/cxx11/IO.h" #include "adios2/cxx11/Operator.h" +#include "adios2/cxx11/Query.h" #include "adios2/cxx11/Types.h" #include "adios2/cxx11/Variable.h" #include "adios2/cxx11/fstream/ADIOS2fstream.h" diff --git a/bindings/CXX11/adios2/cxx11/Engine.h b/bindings/CXX11/adios2/cxx11/Engine.h index 1b76d4759d..7ef701ff7c 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.h +++ b/bindings/CXX11/adios2/cxx11/Engine.h @@ -23,7 +23,7 @@ namespace adios2 /// \cond EXCLUDE_FROM_DOXYGEN // forward declare class IO; // friend - +class Query; // friend namespace core { class Engine; // private implementation @@ -33,6 +33,7 @@ class Engine; // private implementation class Engine { friend class IO; + friend class QueryWorker; public: /** diff --git a/bindings/CXX11/adios2/cxx11/Query.cpp b/bindings/CXX11/adios2/cxx11/Query.cpp new file mode 100644 index 0000000000..21cb309efc --- /dev/null +++ b/bindings/CXX11/adios2/cxx11/Query.cpp @@ -0,0 +1,19 @@ +#include "Query.h" +#include "adios2/toolkit/query/Worker.h" + +namespace adios2 +{ +QueryWorker::QueryWorker(const std::string &configFile, adios2::Engine &reader) +{ + adios2::query::Worker *m = + adios2::query::GetWorker(configFile, reader.m_Engine); + m_Worker = std::make_shared(*m); +} + +void QueryWorker::GetResultCoverage( + adios2::Box &outputSelection, + std::vector> &touched_blocks) +{ + return m_Worker->GetResultCoverage(outputSelection, touched_blocks); +} +} diff --git a/bindings/CXX11/adios2/cxx11/Query.h b/bindings/CXX11/adios2/cxx11/Query.h new file mode 100644 index 0000000000..9299a50c9b --- /dev/null +++ b/bindings/CXX11/adios2/cxx11/Query.h @@ -0,0 +1,41 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * Query.h : provides type utilities for ADIOS2 C++11 bindings + * + * Created on: Aug 20, 2019 + * Author: Junmin Gu + */ + +#ifndef ADIOS2_BINDINGS_CXX11_QUERY_H_ +#define ADIOS2_BINDINGS_CXX11_QUERY_H_ + +#include // otherwise MSVC complains about std::shared_ptr + +#include "Engine.h" + +namespace adios2 +{ + +namespace query +{ +class Worker; +} // + +class QueryWorker +{ +public: + QueryWorker(const std::string &configFile, adios2::Engine &engine); + + void + GetResultCoverage(adios2::Box &, + std::vector> &touched_blocks); + +private: + // adios2::query::Worker* m_Worker; + std::shared_ptr m_Worker; +}; // class QueryWorker + +} // end namespace adios2 +#endif /* ADIOS2_BINDINGS_CXX11_QUERY_H_ */ diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 9e586c0932..2ac65113d3 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -5,6 +5,7 @@ add_subdirectory(basics) add_subdirectory(hello) +add_subdirectory(query) add_subdirectory(useCases) if(ADIOS2_HAVE_MPI) diff --git a/examples/query/CMakeLists.txt b/examples/query/CMakeLists.txt new file mode 100644 index 0000000000..05c80f8f6e --- /dev/null +++ b/examples/query/CMakeLists.txt @@ -0,0 +1,10 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# + +if(ADIOS2_HAVE_MPI) + add_executable(queryTest test_xml.cpp) + target_link_libraries(queryTest adios2 MPI::MPI_C pugixml) +endif() + diff --git a/examples/query/test_xml.cpp b/examples/query/test_xml.cpp new file mode 100644 index 0000000000..853e308f9f --- /dev/null +++ b/examples/query/test_xml.cpp @@ -0,0 +1,110 @@ +#include "adios2.h" +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +//#include "adios2/toolkit/query/Worker.h" + +int main(int argc, char *argv[]) +{ + MPI_Init(&argc, &argv); + int rank, size; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &size); + + const std::size_t Nx = 10; + + try + { + std::string configFileName = "query.xml"; + std::string dataFileName = "/tmp/heatbp4.bp"; + if (argc <= 2) + { + std::cout << "Usage: " << argv[0] << " configFileName dataFileName" + << std::endl; + return 0; + } + + configFileName = argv[1]; + dataFileName = argv[2]; + + if (rank == 0) + { + std::cout << " using config file = " << configFileName << std::endl; + std::cout << " data file = " << dataFileName << std::endl; + } + + adios2::ADIOS ad = + adios2::ADIOS(configFileName, MPI_COMM_WORLD, adios2::DebugON); + + adios2::IO queryIO = ad.DeclareIO("query"); + adios2::Engine reader = + queryIO.Open(dataFileName, adios2::Mode::Read, MPI_COMM_WORLD); + adios2::QueryWorker w = adios2::QueryWorker(configFileName, reader); + + std::vector> touched_blocks; + + while (reader.BeginStep() == adios2::StepStatus::OK) + { + adios2::Box empty; + w.GetResultCoverage(empty, touched_blocks); + std::cout << " ... now can read out touched blocks ... size=" + << touched_blocks.size() << std::endl; + reader.EndStep(); + } + + return 0; + } + catch (std::exception &e) + { + std::cout << "Exception, STOPPING PROGRAM from rank " << rank << "\n"; + std::cout << e.what() << "\n"; + } + + MPI_Finalize(); + + return 0; +} + +bool testMe(std::string &queryConfigFile, std::string const &doubleVarName, + MPI_Comm comm) +{ + adios2::ADIOS ad(queryConfigFile, comm, adios2::DebugON); + std::string dataFileName = "test.file"; + + // adios2::query::Worker w(queryConfigFile, comm); + + // w.SetSource(inIO, reader); + + { + // the config file should have info on idx method to use + + /* + // if wanting to build customized minmax idx + // instead of using the existing block stats or bp4 minmax arrays + bool overwrite = false; + size_t recommendedSize = 20000; + + if (!w.PrepareIdx(overwrite, recommendedSize, doubleVarName)) + return false; + */ + } + + adios2::IO inIO = ad.DeclareIO("query"); + adios2::Engine reader = inIO.Open(dataFileName, adios2::Mode::Read, comm); + + // std::vector dataOutput; + // std::vector coordinateOutput; + + return true; +} + +/* + */ diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 90b96b4e30..871c31b262 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -80,6 +80,12 @@ add_library(adios2 toolkit/profiling/iochrono/Timer.cpp + toolkit/query/Query.cpp + toolkit/query/Worker.cpp + toolkit/query/XmlUtil.cpp + toolkit/query/XmlWorker.cpp + toolkit/query/BlockIndex.cpp + toolkit/transport/Transport.cpp toolkit/transport/file/FileStdio.cpp toolkit/transport/file/FileFStream.cpp diff --git a/source/adios2/toolkit/query/BlockIndex.cpp b/source/adios2/toolkit/query/BlockIndex.cpp new file mode 100644 index 0000000000..88aec8b91a --- /dev/null +++ b/source/adios2/toolkit/query/BlockIndex.cpp @@ -0,0 +1,25 @@ +#include "BlockIndex.h" +#include "Query.h" +//#include "BlockIndex.tcc" + +/* +// optional +void BlockIndex::GenerateIdxFile() +{} + +void BlockIndex::EvaluateIdxFile() +{} + +void BlockIndex::EvaluateBP4() +{ + +} + +void BlockIndex::Evaluate() +{ + if (m_IdxFileName.size() > 0) + return EvaluateIdxFile(); + else + return EvaluateBP4(); // default +} +*/ diff --git a/source/adios2/toolkit/query/BlockIndex.h b/source/adios2/toolkit/query/BlockIndex.h new file mode 100644 index 0000000000..2721ac305e --- /dev/null +++ b/source/adios2/toolkit/query/BlockIndex.h @@ -0,0 +1,110 @@ +#ifndef ADIOS2_BLOCK_INDEX_H +#define ADIOS2_BLOCK_INDEX_H + +#include "Index.h" +#include "Query.h" + +namespace adios2 +{ +namespace query +{ + +template +class BlockIndex +{ + struct Tree + { + // + // ** no need to keep the original block. might be smaller than + // blockIndex typename Variable::info& m_BlockInfo; + // + std::vector::Info> m_SubBlockInfo; + }; + +public: + BlockIndex(adios2::core::Variable &var, adios2::core::IO &io, + adios2::core::Engine &reader) + : m_Var(var), m_IdxIO(io), m_IdxReader(reader) + { + } + + void Generate(std::string &fromBPFile, const adios2::Params &inputs) {} + + void Evaluate(const QueryVar &query, + std::vector> &resultSubBlocks) + { + std::size_t found = m_IdxReader.m_EngineType.find("BP4"); + if (found != std::string::npos) + // if (m_IdxReader.m_EngineType.find("BP4") >= 0) + { + RunBP4Stat(query, resultSubBlocks); + } + else if (m_IdxReader.m_EngineType.find("BP3") >= 0) + { + RunDefaultBPStat(query, resultSubBlocks); + } + } + + void RunBP4Stat(const QueryVar &query, + std::vector> &hitBlocks) + { + std::cout << "BlockIndexEval(BP4) using BP3 now" << std::endl; + RunDefaultBPStat(query, hitBlocks); + } + + void RunDefaultBPStat(const QueryVar &query, + std::vector> &hitBlocks) + { + size_t currStep = m_IdxReader.CurrentStep(); + adios2::Dims currShape = m_Var.Shape(); + if (!query.IsSelectionValid(currShape)) + return; + + std::vector::Info> varBlocksInfo = + m_IdxReader.BlocksInfo(m_Var, currStep); + + for (auto &blockInfo : varBlocksInfo) + { + if (!query.TouchSelection(blockInfo.Start, blockInfo.Count)) + continue; + + T min = blockInfo.Min; + T max = blockInfo.Max; + + // std::cout<<" min: "< box = {blockInfo.Start, + blockInfo.Count}; + hitBlocks.push_back(box); + } + } + } + /* + void Init (adios2::Engine& idxReader, const adios2::Variable& var, int + ts); void SetContent(const QueryVar& query, int ts); + + bool isBlockValid(adios2::Dims& blockStart, adios2::Dims& blockCount, + adios2::Dims& selStart, adios2::Dims& selCount); + + void WalkThrough(const QueryVar& query, std::vector& results); + */ + + Tree m_Content; + adios2::core::Variable m_Var; + +private: + // + // blockid <=> vector of subcontents + // + // std::string& m_DataFileName; + adios2::core::Engine &m_IdxReader; + adios2::core::IO &m_IdxIO; + +}; // class blockIndex + +}; // end namespace query +}; // end namespace adios2 + +#endif diff --git a/source/adios2/toolkit/query/Index.h b/source/adios2/toolkit/query/Index.h new file mode 100644 index 0000000000..f752780698 --- /dev/null +++ b/source/adios2/toolkit/query/Index.h @@ -0,0 +1,37 @@ +#ifndef ADIOS2_QUERY_INDEX_H +#define ADIOS2_QUERY_INDEX_H + +#include "Query.h" + +namespace adios2 +{ +namespace query +{ +struct IndexInfo +{ + std::string m_IdxType; // minmax (default) + adios2::Params m_SetupParameters; +}; + +template +class AbstractQueryIndex +{ +public: + void Generate(std::string &fromBPFile, const adios2::Params &inputs) = 0; + void Evaluate(const QueryVar &query, + std::vector> &touchedBlocks) = 0; +}; + +/* +template + class BlockIndex : public AbstractQueryIndex +{ +public: + void Generate(std::string& fromBPFile, const adios2::Params& inputs); + void Evaluate(const Query& query, std::vector>& touchedBlocks); +}; +*/ +}; // name space query +}; // name space adios2 + +#endif diff --git a/source/adios2/toolkit/query/Query.cpp b/source/adios2/toolkit/query/Query.cpp new file mode 100644 index 0000000000..6c940e7bf7 --- /dev/null +++ b/source/adios2/toolkit/query/Query.cpp @@ -0,0 +1,100 @@ +#include "Query.h" +#include "adios2/helper/adiosFunctions.h" +//#include +#include "BlockIndex.h" + +#include "Query.tcc" + +namespace adios2 +{ +namespace query +{ +bool QueryComposite::AddNode(QueryBase *var) +{ + if (adios2::query::Relation::NOT == m_Relation) + { + // if (m_Nodes.size() > 0) return false; + // don't want to support NOT for composite queries + return false; + } + m_Nodes.push_back(var); + return true; +} + +void QueryComposite::BlockIndexEvaluate(adios2::core::IO &io, + adios2::core::Engine &reader, + std::vector> &touchedBlocks) +{ + std::cout << " to do: QueryComposite::BlockIndexEvalute" << std::endl; + if (m_Nodes.size() == 0) + return; + // plan to shift all var results to regions start at 0, and find out the + // overlapped regions boxes can be different size especially if they are + // from BP3 +} + +bool QueryVar::IsSelectionValid(adios2::Dims &shape) const +{ + if (0 == m_Selection.first.size()) + return true; + + if (m_Selection.first.size() != shape.size()) + { + std::cerr << "ERROR: query selection dimension is different from " + "shape dimension" + << std::endl; + return false; // different dimension + } + + for (int i = 0; i < shape.size(); i++) + { + if ((m_Selection.first[i] > shape[i]) || + (m_Selection.second[i] > shape[i])) + return false; + } + return true; +} + +bool QueryVar::TouchSelection(adios2::Dims &start, adios2::Dims &count) const +{ + if (0 == m_Selection.first.size()) + return true; + + const size_t dimensionsSize = start.size(); + + for (size_t i = 0; i < dimensionsSize; i++) + { + size_t end = start[i] + count[i]; + size_t selEnd = m_Selection.first[i] + m_Selection.second[i]; + + if (end <= m_Selection.first[i]) + return false; + if (selEnd <= start[i]) + return false; + } + return true; +} + +void QueryVar::BlockIndexEvaluate(adios2::core::IO &io, + adios2::core::Engine &reader, + std::vector> &touchedBlocks) +{ + const std::string varType = io.InquireVariableType(m_VarName); + + // Variable var = io.InquireVariable(m_VarName); + // BlockIndex idx(io, reader); + + // var already exists when loading query. skipping validity checking +#define declare_type(T) \ + if (varType == adios2::helper::GetType()) \ + { \ + core::Variable *var = io.InquireVariable(m_VarName); \ + BlockIndex idx(*var, io, reader); \ + idx.Evaluate(*this, touchedBlocks); \ + } + // ADIOS2_FOREACH_ATTRIBUTE_TYPE_1ARG(declare_type) //skip complex types + ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type) +#undef declare_type +} +} // namespace query +} // namespace adios2 diff --git a/source/adios2/toolkit/query/Query.h b/source/adios2/toolkit/query/Query.h new file mode 100644 index 0000000000..84cbb10e95 --- /dev/null +++ b/source/adios2/toolkit/query/Query.h @@ -0,0 +1,220 @@ +#ifndef ADIOS2_QUERY_H +#define ADIOS2_QUERY_H + +#include //std::ios_base::failure +#include //std::cout + +#include // accumulate +#include //std::invalid_argument std::exception +#include + +//#include "adios2.h" +#include "adios2/common/ADIOSTypes.h" +#include "adios2/core/ADIOS.h" +#include "adios2/core/Engine.h" +#include "adios2/core/IO.h" +#include "adios2/core/Variable.h" +namespace adios2 +{ +namespace query +{ +enum Op +{ + GT, + LT, + GE, + LE, + NE, + EQ +}; + +enum Relation +{ + AND, + OR, + NOT +}; + +static adios2::query::Relation strToRelation(std::string relationStr) noexcept +{ + if ((relationStr.compare("or") == 0) || (relationStr.compare("OR") == 0)) + return adios2::query::Relation::OR; + + return adios2::query::Relation::AND; // default +} + +static adios2::query::Op strToQueryOp(std::string opStr) noexcept +{ + if ((opStr.compare("lt") == 0) || (opStr.compare("LT") == 0)) + return adios2::query::Op::LT; + if ((opStr.compare("gt") == 0) || (opStr.compare("GT") == 0)) + return adios2::query::Op::GT; + if ((opStr.compare("ge") == 0) || (opStr.compare("GE") == 0)) + return adios2::query::Op::GE; + if ((opStr.compare("le") == 0) || (opStr.compare("LE") == 0)) + return adios2::query::Op::LE; + if ((opStr.compare("eq") == 0) || (opStr.compare("EQ") == 0)) + return adios2::query::Op::EQ; + if ((opStr.compare("ne") == 0) || (opStr.compare("NE") == 0)) + return adios2::query::Op::NE; + + return adios2::query::Op::EQ; // default +} + +static adios2::Dims split(const std::string &s, char delim) +{ + adios2::Dims dim; + + std::stringstream ss(s); + std::string item; + + while (getline(ss, item, delim)) + { + std::stringstream curr(item); + size_t val; + curr >> val; + dim.push_back(val); + } + + return dim; +} + +// +// classes +// +class Range +{ +public: + adios2::query::Op m_Op; + std::string m_StrValue; + // void* m_Value = nullptr; + + // template bool Check(T val) const ; + + template + bool CheckInterval(T &min, T &max) const; + + void Print() { std::cout << "===> " << m_StrValue << std::endl; } +}; // class Range + +class RangeTree +{ +public: + void AddLeaf(adios2::query::Op op, std::string value) + { + Range range; + range.m_Op = op; + range.m_StrValue = value; + + m_Leaves.push_back(range); + } + + void AddNode(RangeTree &node) { m_SubNodes.push_back(node); } + + void SetRelation(adios2::query::Relation r) { m_Relation = r; } + + void Print() + { + for (auto leaf : m_Leaves) + leaf.Print(); + for (auto node : m_SubNodes) + node.Print(); + } + + // template bool Check(T value) const ; + + template + bool CheckInterval(T &min, T &max) const; + + adios2::query::Relation m_Relation = adios2::query::Relation::AND; + std::vector m_Leaves; + std::vector m_SubNodes; +}; // class RangeTree + +class QueryBase +{ +public: + virtual ~QueryBase(){}; + virtual bool IsCompatible(adios2::Box &box) = 0; + virtual void Print() = 0; + virtual void BlockIndexEvaluate(adios2::core::IO &, adios2::core::Engine &, + std::vector> &touchedBlocks) = 0; +}; + +class QueryVar : public QueryBase +{ +public: + QueryVar(const std::string &varName) : m_VarName(varName) {} + + std::string &GetVarName() { return m_VarName; } + void BlockIndexEvaluate(adios2::core::IO &, adios2::core::Engine &, + std::vector> &touchedBlocks); + + void Print() { m_RangeTree.Print(); } + + bool IsCompatible(adios2::Box &box) + { + if (box.first.size() != m_Selection.first.size()) + return false; + + for (int n = 0; n < box.second.size(); n++) + if (box.second[n] != m_Selection.second[n]) + return false; + + return true; + } + + void SetSelection(adios2::Dims &start, adios2::Dims &count) + { + m_Selection.first = start; + m_Selection.second = count; + } + + bool IsSelectionValid(adios2::Dims &varShape) const; + + bool TouchSelection(adios2::Dims &start, adios2::Dims &count) const; + + RangeTree m_RangeTree; + adios2::Box m_Selection; + +private: + std::string m_VarName; +}; // class QueryVar + +class QueryComposite : public QueryBase +{ +public: + QueryComposite(adios2::query::Relation relation) : m_Relation(relation) {} + + void BlockIndexEvaluate(adios2::core::IO &, adios2::core::Engine &, + std::vector> &touchedBlocks); + + bool AddNode(QueryBase *v); + + void Print() + { + std::cout << " Composite query" << std::endl; + for (auto n : m_Nodes) + n->Print(); + } + + bool IsCompatible(adios2::Box &box) + { + if (m_Nodes.size() == 0) + return false; + return (m_Nodes[0])->IsCompatible(box); + } + +private: + adios2::query::Relation m_Relation = adios2::query::Relation::AND; + + std::vector m_Nodes; +}; // class QueryComposite + +/* + */ + +} // namespace query +} // namespace adiso2 + +#endif // define diff --git a/source/adios2/toolkit/query/Query.tcc b/source/adios2/toolkit/query/Query.tcc new file mode 100644 index 0000000000..418a5da375 --- /dev/null +++ b/source/adios2/toolkit/query/Query.tcc @@ -0,0 +1,85 @@ +namespace adios2 +{ +namespace query +{ + +template +bool Range::CheckInterval(T &min, T &max) const +{ + bool isHit = false; + /* + if (m_Value == nullptr) + { + std::stringstream convert(m_StrValue); + T val; + convert >> val; + m_Value = new T; + *((T*)m_Value) = val; + } + T value = *((T*)m_Value); + */ + std::stringstream convert(m_StrValue); + T value; + convert >> value; + + switch (m_Op) + { + case adios2::query::Op::GT: + isHit = (max > value); + break; + case adios2::query::Op::LT: + isHit = (min < value); + break; + case adios2::query::Op::GE: + isHit = (max >= value); + break; + case adios2::query::Op::LE: + isHit = (min <= value); + break; + case adios2::query::Op::EQ: + isHit = (max >= value) && (min <= value); + break; + case adios2::query::Op::NE: + isHit = !((max == value) && (min == value)); + break; + default: + break; + } + return isHit; +} + +template +bool RangeTree::CheckInterval(T &min, T &max) const +{ + if (adios2::query::Relation::AND == m_Relation) + { + for (auto &range : m_Leaves) + if (!range.CheckInterval(min, max)) + return false; + + for (auto &node : m_SubNodes) + if (!node.CheckInterval(min, max)) + return false; + + return true; // even if no leaves or nodes + } + + if (adios2::query::Relation::OR == m_Relation) + { + for (auto &range : m_Leaves) + if (range.CheckInterval(min, max)) + return true; + + for (auto &node : m_SubNodes) + if (node.CheckInterval(min, max)) + return true; + + return false; + } + + // anything else are false + return false; +} + +} +} diff --git a/source/adios2/toolkit/query/Util.h b/source/adios2/toolkit/query/Util.h new file mode 100644 index 0000000000..6d224f1b46 --- /dev/null +++ b/source/adios2/toolkit/query/Util.h @@ -0,0 +1,73 @@ +#ifndef ADIOS2_QUERY_UTIL_H +#define ADIOS2_QUERY_UTIL_H + +#include +#include //std::ios_base::failure +#include //std::cout +#include +#include +#include + +namespace adios2 +{ +namespace query +{ +static size_t ToUIntValue(const adios2::Params ¶ms, const std::string &key, + size_t defaultValue) +{ + auto it = params.find(key); + if (it != params.end()) + { + try + { + auto value = (size_t)(std::stoul(it->second)); + return value; + } + catch (std::exception &e) + { + std::cout << e.what() << std::endl; + return defaultValue; + } + } + return defaultValue; +} + +static bool ToBoolValue(const adios2::Params ¶ms, const std::string &key) +{ + auto keyPair = params.find(key); + if (keyPair != params.end()) + { + std::string value = keyPair->second; + std::transform(value.begin(), value.end(), value.begin(), + [](unsigned char c) { return std::tolower(c); }); + if (value == "yes" || value == "true") + { + return true; + } + } + return false; +} + +static bool EndsWith(const std::string &hostStr, const std::string &fileTag) +{ + if (hostStr.size() >= fileTag.size() && + hostStr.compare(hostStr.size() - fileTag.size(), fileTag.size(), + fileTag) == 0) + return true; + else + return false; +} + +static bool IsFileNameXML(const std::string &filename) +{ + return EndsWith(filename, ".xml"); +} + +static bool IsFileNameJSON(const std::string &filename) +{ + return EndsWith(filename, ".json"); +} +}; // namespace query +}; // name space adios2 + +#endif // QUERY_WORKER_H diff --git a/source/adios2/toolkit/query/Worker.cpp b/source/adios2/toolkit/query/Worker.cpp new file mode 100644 index 0000000000..0915deadf3 --- /dev/null +++ b/source/adios2/toolkit/query/Worker.cpp @@ -0,0 +1,67 @@ +#include "Worker.h" +//#include "XmlWorker.cpp" + +namespace adios2 +{ +namespace query +{ + +Worker::Worker(const std::string &queryFile, adios2::core::Engine *adiosEngine) +: m_QueryFile(queryFile), m_SourceReader(adiosEngine) +{ +} + +Worker::~Worker() +{ + if (m_SourceReader) + m_SourceReader->Close(); +} +/* +template +void Worker::BuildIdxFile(adios2::IndexType&type, adios2::Params& input) +{ + if (type != adios2::IndexType::MIN_MAX) + throw std::invalid_argument("Unable to build index with given type."); + + { + // build min max + m_IndexTool = new BlockIndex(); + + bool doOverWrite = ToBoolValue(input, +adios2::BlockIndexBuilder.m_ParamOverWrite); size_t blockSize = +ToUIntValue(input, adios2::BlockIndexBuilder.m_ParamBlockSize, 2000000); + + adios2::IO idxWriteIO = +m_adios2.DeclareIO(std::string("BLOCKINDEX-Write-")+m_DataReader->Name()); + adios2::BlockIndexBuilder builder(m_IdxFileName, m_Comm, overwrite); + builder.GenerateIndexFrom(*m_DataIO, idxWriteIO, *m_DataReader, blockSize); + } +} +*/ + +void Worker::GetResultCoverage(const adios2::Box &outputRegion, + std::vector> &touchedBlocks) +{ + // std::cout << " will evaluate shifting in output region later .." + // << std::endl; + touchedBlocks.clear(); + + if (m_Query && m_SourceReader) + m_Query->BlockIndexEvaluate(m_SourceReader->m_IO, *m_SourceReader, + touchedBlocks); + + // Query simpleQ = parse(); + + /* + if (m_Idx == nullptr) { + throw std::invalid_argument("Unable to use index with given type."); + } + + { + // use min max + + } + */ +} +} // namespace query +} // namespace adios2 diff --git a/source/adios2/toolkit/query/Worker.h b/source/adios2/toolkit/query/Worker.h new file mode 100644 index 0000000000..128d057f1b --- /dev/null +++ b/source/adios2/toolkit/query/Worker.h @@ -0,0 +1,136 @@ +#ifndef QUERY_WORKER_H +#define QUERY_WORKER_H + +#include //std::ios_base::failure +#include //std::cout + +#include +#include +#include + +#include "adios2/common/ADIOSMPI.h" +#include "adios2/common/ADIOSTypes.h" +#include + +#include "Index.h" +#include "Query.h" +#include "Util.h" +#include + +namespace adios2 +{ +namespace query +{ +class Worker +{ +public: + // maybe useful? + // Worker(const pugi::xml_node queryNode, MPI_Comm comm); + // will add Worker(jsonNode&, MPI_Comm comm); + + virtual ~Worker(); + + adios2::core::Engine *GetSourceReader() { return m_SourceReader; } + + // unsigned long Estimate(); + void GetResultCoverage(const adios2::Box &, + std::vector> &); + + /* + template void GetBlockIndexCoverage(adios2::Engine& bp4Reader, + adios2::query::Query& query, + std::vector>&) + { + if (query.IsMultiVar()){ + std::cout<<" This query type Not supported yet"<Name()); + // adios2::BlockIndexBuilder builder(m_IdxFileName, m_Comm, overwrite); + // builder.GenerateIndexFrom(*m_DataIO, idxWriteIO, *m_DataReader, + // recommendedSize); + //} + + bool HasMoreSteps() { return false; /*tbd*/ } + +protected: + Worker(const std::string &configFile, adios2::core::Engine *adiosEngine); + + // template Query& GetQuery() {} + + MPI_Comm m_Comm; + std::string m_QueryFile; // e.g. xml file + // adios2::core::ADIOS& m_Adios2; + // adios2::core::IO* m_IO = nullptr; + adios2::core::Engine *m_SourceReader = nullptr; + adios2::query::QueryBase *m_Query = nullptr; + +private: + // adios2::IO* m_DataIO = nullptr; + + // std::string m_IdxFileName; + + // adios2::query::AbstractQueryIndex* m_Idx = nullptr; +}; // worker + +class XmlWorker : public Worker +{ +public: + XmlWorker(const std::string &configFile, adios2::core::Engine *adiosEngine) + : Worker(configFile, adiosEngine) + { + ParseMe(); + } + + ~XmlWorker() + { + if (m_Query != nullptr) + delete m_Query; + } + void ParseMe(); + +private: + void ConstructTree(RangeTree &host, const pugi::xml_node &node); + void ConstructQuery(QueryVar &q, const pugi::xml_node &node); + + void ParseIONode(const pugi::xml_node &ioNode); + adios2::query::QueryVar *ParseVarNode(const pugi::xml_node &node, + adios2::core::IO ¤tIO, + adios2::core::Engine &reader); + +}; // XmlWorker + +static Worker *GetWorker(const std::string &configFile, + adios2::core::Engine *adiosEngine) +{ + std::ifstream fileStream(configFile); + + if (!fileStream) + throw std::ios_base::failure("ERROR: file " + configFile + + " not found. "); + + if (adios2::query::IsFileNameXML(configFile)) + { + // return new XmlWorker (configFile, comm); + return new XmlWorker(configFile, adiosEngine); + } + return nullptr; +} + +/* +template + void XmlWorker::Parse() +{ + std::cout<<"Parsing xml"< + +namespace adios2 +{ +namespace query +{ +namespace XmlUtil +{ +pugi::xml_document XMLDocument(const std::string &xmlContents) +{ + pugi::xml_document document; + auto parse_result = document.load_buffer_inplace( + const_cast(xmlContents.data()), xmlContents.size()); + + if (!parse_result) + { + throw std::invalid_argument("ERROR: XML parse error: " + + std::string(parse_result.description())); + } + + return document; +} + +pugi::xml_node XMLNode(const std::string nodeName, + const pugi::xml_document &xmlDocument, + const bool isMandatory, const bool isUnique) +{ + const pugi::xml_node node = xmlDocument.child(nodeName.c_str()); + + if (isMandatory && !node) + { + throw std::invalid_argument("ERROR: XML: no <" + nodeName + + "> element found"); + } + + if (isUnique) + { + const size_t nodes = + std::distance(xmlDocument.children(nodeName.c_str()).begin(), + xmlDocument.children(nodeName.c_str()).end()); + if (nodes > 1) + { + throw std::invalid_argument("ERROR: XML only one <" + nodeName + + "> element can exist inside " + + std::string(xmlDocument.name())); + } + } + + return node; +} + +pugi::xml_node XMLNode(const std::string nodeName, + const pugi::xml_node &upperNode, const bool isMandatory, + const bool isUnique) +{ + const pugi::xml_node node = upperNode.child(nodeName.c_str()); + + if (isMandatory && !node) + { + throw std::invalid_argument( + "ERROR: XML: no <" + nodeName + "> element found, inside <" + + std::string(upperNode.name()) + "> element "); + } + + if (isUnique) + { + const size_t nodes = + std::distance(upperNode.children(nodeName.c_str()).begin(), + upperNode.children(nodeName.c_str()).end()); + if (nodes > 1) + throw std::invalid_argument("ERROR: XML only one <" + nodeName + + "> element can exist inside <" + + std::string(upperNode.name()) + + "> element. "); + } + return node; +} + +pugi::xml_attribute XMLAttribute(const std::string attributeName, + const pugi::xml_node &node, + const bool isMandatory) +{ + const pugi::xml_attribute attribute = node.attribute(attributeName.c_str()); + + if (isMandatory && !attribute) + { + const std::string nodeName(node.name()); + throw std::invalid_argument("ERROR: XML: No attribute " + + attributeName + " found on <" + nodeName + + "> element"); + } + return attribute; +} + +} // namespace XmlUtil +} // namespace query +} // namespace adios2 diff --git a/source/adios2/toolkit/query/XmlUtil.h b/source/adios2/toolkit/query/XmlUtil.h new file mode 100644 index 0000000000..405314e02c --- /dev/null +++ b/source/adios2/toolkit/query/XmlUtil.h @@ -0,0 +1,35 @@ +#ifndef ADIOS2_QUERY_XMLUTIL_H +#define ADIOS2_QUERY_XMLUTIL_H + +#include + +namespace adios2 +{ +namespace query +{ +namespace XmlUtil +{ + +pugi::xml_attribute XMLAttribute(const std::string attributeName, + const pugi::xml_node &node, + const bool isMandatory = true); + +// adios2::Dims split (const std::string &s, char delim); + +pugi::xml_document XMLDocument(const std::string &xmlContents); + +pugi::xml_node XMLNode(const std::string nodeName, + const pugi::xml_document &xmlDocument, + const bool isMandatory = true, + const bool isUnique = false); + +pugi::xml_node XMLNode(const std::string nodeName, + const pugi::xml_node &upperNode, + const bool isMandatory = true, + const bool isUnique = false); + +} // namespace xmlUtil +} +} + +#endif diff --git a/source/adios2/toolkit/query/XmlWorker.cpp b/source/adios2/toolkit/query/XmlWorker.cpp new file mode 100644 index 0000000000..ed73a6c0f7 --- /dev/null +++ b/source/adios2/toolkit/query/XmlWorker.cpp @@ -0,0 +1,260 @@ +#include "Worker.h" +#include "XmlUtil.h" + +namespace adios2 +{ +namespace query +{ +void XmlWorker::ParseMe() +{ + // std::cerr << "TODO: ... will ... build index file if indicated " << + // std::endl; + + auto lf_FileContents = [&](const std::string &configXML) -> std::string { + std::ifstream fileStream(configXML); + if (!fileStream) + throw std::ios_base::failure("ERROR: file " + configXML + + " not found. "); + + std::ostringstream fileSS; + fileSS << fileStream.rdbuf(); + fileStream.close(); + + if (fileSS.str().empty()) + throw std::invalid_argument("ERROR: config xml file is empty."); + + return fileSS.str(); + }; // local function lf_FileContents + + const std::string fileContents = lf_FileContents(m_QueryFile); + const pugi::xml_document document = + adios2::query::XmlUtil::XMLDocument(fileContents); + + const pugi::xml_node config = + adios2::query::XmlUtil::XMLNode("adios-query", document, true); + + const pugi::xml_node ioNode = config.child("io"); + ParseIONode(ioNode); + + // for (const pugi::xml_node &ioNode : config.children("io")) +} // Parse() + +void XmlWorker::ParseIONode(const pugi::xml_node &ioNode) +{ + auto lf_GetParametersXML = + [&](const pugi::xml_node &node) -> adios2::Params { + const std::string errorMessage("in node " + std::string(node.value())); + + adios2::Params parameters; + + for (const pugi::xml_node paramNode : node.children("parameter")) + { + const pugi::xml_attribute key = + adios2::query::XmlUtil::XMLAttribute("key", paramNode); + const pugi::xml_attribute value = + adios2::query::XmlUtil::XMLAttribute("value", paramNode); + parameters.emplace(key.value(), value.value()); + } + return parameters; + }; // local function lf_GetParamtersXML + +#ifdef PARSE_IO + const pugi::xml_attribute ioName = + adios2::query::XmlUtil::XMLAttribute("name", ioNode); + const pugi::xml_attribute fileName = + adios2::query::XmlUtil::XMLAttribute("file", ioNode); + + // must be unique per io + const pugi::xml_node &engineNode = + adios2::query::XmlUtil::XMLNode("engine", ioNode, false, true); + // adios2::ADIOS adios(m_Comm, adios2::DebugON); + // adios2::IO currIO = m_Adios2.DeclareIO(ioName.value()); + m_IO = &(m_Adios2.DeclareIO(ioName.value())); + + if (engineNode) + { + const pugi::xml_attribute type = + adios2::query::XmlUtil::XMLAttribute("type", engineNode); + m_IO->SetEngine(type.value()); + + const adios2::Params parameters = lf_GetParametersXML(engineNode); + m_IO->SetParameters(parameters); + } + else + { + m_IO->SetEngine("BPFile"); + } + // adios2::Engine reader = currIO.Open(fileName.value(), + // adios2::Mode::Read, m_Comm); + m_SourceReader = + &(m_IO->Open(fileName.value(), adios2::Mode::Read, m_Comm)); +#else + const pugi::xml_attribute ioName = + adios2::query::XmlUtil::XMLAttribute("name", ioNode); + if (m_SourceReader->m_IO.m_Name.compare(ioName.value()) != 0) + throw std::ios_base::failure( + "invalid query io. Expecting io name = query"); +#endif + // std::cout< subqueries; + + adios2::Box ref; + for (const pugi::xml_node &qNameNode : ioNode.children("name")) + { + const pugi::xml_attribute tag = + adios2::query::XmlUtil::XMLAttribute("tag", qNameNode); + const pugi::xml_node &variable = qNameNode.child("var"); + QueryVar *q = + ParseVarNode(variable, m_SourceReader->m_IO, *m_SourceReader); + if (ref.first.size() == 0) + ref = q->m_Selection; + else if (!q->IsCompatible(ref)) + throw std::ios_base::failure("impactible query found on var:" + + q->GetVarName()); + // std::cout<<" found sub query for: "<m_IO, *m_SourceReader); + } + else + { + const pugi::xml_attribute op = + adios2::query::XmlUtil::XMLAttribute("op", qNode); + QueryComposite *q = + new QueryComposite(adios2::query::strToRelation(op.value())); + for (const pugi::xml_node &sub : qNode.children()) + { + // std::cout<<" .. "<AddNode(subqueries[sub.name()]); + } + m_Query = q; + // m_Query->AddNode(); + } + + // if (m_Query) m_Query->Print(); +} // parse_io_node + +// node is the variable node +QueryVar *XmlWorker::ParseVarNode(const pugi::xml_node &node, + adios2::core::IO ¤tIO, + adios2::core::Engine &reader) + +{ + const std::string variableName = + std::string(adios2::query::XmlUtil::XMLAttribute("name", node).value()); + + // const std::string varType = currentIO.VariableType(variableName); + const std::string varType = currentIO.InquireVariableType(variableName); + if (varType.size() == 0) + { + std::cerr << "No such variable: " << variableName << std::endl; + return nullptr; + } +#define declare_type(T) \ + if (varType == helper::GetType()) \ + { \ + core::Variable *var = currentIO.InquireVariable(variableName); \ + if (var) \ + { \ + QueryVar *q = new QueryVar(variableName); \ + adios2::Dims zero(var->Shape().size(), 0); \ + adios2::Dims shape = var->Shape(); \ + q->SetSelection(zero, shape); \ + ConstructQuery(*q, node); \ + return q; \ + } \ + } + ADIOS2_FOREACH_ATTRIBUTE_PRIMITIVE_STDTYPE_1ARG(declare_type) +#undef declare_type + return nullptr; +} // parse_var_node + +void XmlWorker::ConstructTree(RangeTree &host, const pugi::xml_node &node) +{ + std::string relationStr = + adios2::query::XmlUtil::XMLAttribute("value", node).value(); + host.SetRelation(adios2::query::strToRelation(relationStr)); + for (const pugi::xml_node rangeNode : node.children("range")) + { + std::string valStr = + adios2::query::XmlUtil::XMLAttribute("value", rangeNode).value(); + std::string opStr = + adios2::query::XmlUtil::XMLAttribute("compare", rangeNode).value(); + /* + std::stringstream convert(valStr); + T val; + convert >> val; + */ + host.AddLeaf(adios2::query::strToQueryOp(opStr), valStr); + } + + for (const pugi::xml_node opNode : node.children("op")) + { + adios2::query::RangeTree subNode; + ConstructTree(subNode, opNode); + host.AddNode(subNode); + } +} + +void XmlWorker::ConstructQuery(QueryVar &simpleQ, const pugi::xml_node &node) +{ + // QueryVar* simpleQ = new QueryVar(variableName); + pugi::xml_node bbNode = node.child("boundingbox"); + + if (bbNode) + { + adios2::Box box = + adios2::Box({100, 100}, {200, 200}); + std::string startStr = + adios2::query::XmlUtil::XMLAttribute("start", bbNode).value(); + std::string countStr = + adios2::query::XmlUtil::XMLAttribute("count", bbNode).value(); + + adios2::Dims start = split(startStr, ','); + adios2::Dims count = split(countStr, ','); + + if (start.size() != count.size()) + { + throw std::ios_base::failure( + "dim of startcount does match in the bounding box definition"); + } + + // simpleQ.setSelection(box.first, box.second); + adios2::Dims shape = + simpleQ.m_Selection.second; // set at the creation for default + simpleQ.SetSelection(start, count); + if (!simpleQ.IsSelectionValid(shape)) + throw std::ios_base::failure( + "invalid selections for selection of var: " + + simpleQ.GetVarName()); + } + +#ifdef NEVER // don't know whether this is useful. + pugi::xml_node tsNode = node.child("tstep"); + if (tsNode) + { + std::string startStr = + adios2::query::XmlUtil::XMLAttribute("start", tsNode).value(); + std::string countStr = + adios2::query::XmlUtil::XMLAttribute("count", tsNode).value(); + + if ((startStr.size() > 0) && (countStr.size() > 0)) + { + std::stringstream ss(startStr), cc(countStr); + ss >> simpleQ.m_TimestepStart; + cc >> simpleQ.m_TimestepCount; + } + } +#endif + pugi::xml_node relationNode = node.child("op"); + ConstructTree(simpleQ.m_RangeTree, relationNode); +} + +} // namespace query +} // namespace adios2