added query component and its CXX binding

bindings/CXX11/CMakeLists.txt

@@ -11,6 +11,7 @@ target_sources(adios2 PRIVATE
 	${CMAKE_CURRENT_SOURCE_DIR}/adios2/cxx11/IO.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/adios2/cxx11/IO.tcc
 	${CMAKE_CURRENT_SOURCE_DIR}/adios2/cxx11/Operator.cpp
+	${CMAKE_CURRENT_SOURCE_DIR}/adios2/cxx11/Query.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/adios2/cxx11/Types.cpp
 	${CMAKE_CURRENT_SOURCE_DIR}/adios2/cxx11/Types.tcc
 	${CMAKE_CURRENT_SOURCE_DIR}/adios2/cxx11/Variable.cpp
@@ -40,6 +41,7 @@ install(
         adios2/cxx11/Attribute.h
         adios2/cxx11/Engine.h
         adios2/cxx11/Operator.h
+        adios2/cxx11/Query.h
         adios2/cxx11/Types.h
   DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/adios2/cxx11
 )

bindings/CXX11/adios2.h

@@ -20,6 +20,7 @@
 #include "adios2/cxx11/Engine.h"
 #include "adios2/cxx11/IO.h"
 #include "adios2/cxx11/Operator.h"
+#include "adios2/cxx11/Query.h"
 #include "adios2/cxx11/Types.h"
 #include "adios2/cxx11/Variable.h"
 #include "adios2/cxx11/fstream/ADIOS2fstream.h"

bindings/CXX11/adios2/cxx11/Engine.h

@@ -23,7 +23,7 @@ namespace adios2
 /// \cond EXCLUDE_FROM_DOXYGEN
 // forward declare
 class IO; // friend
-
+class Query; // friend
 namespace core
 {
 class Engine; // private implementation
@@ -33,6 +33,7 @@ class Engine; // private implementation
 class Engine
 {
     friend class IO;
+    friend class QueryWorker;
 
 public:
     /**

bindings/CXX11/adios2/cxx11/Query.cpp

@@ -0,0 +1,19 @@
+#include "Query.h"
+#include "adios2/toolkit/query/Worker.h"
+
+namespace adios2
+{
+QueryWorker::QueryWorker(const std::string &configFile, adios2::Engine &reader)
+{
+    adios2::query::Worker *m =
+        adios2::query::GetWorker(configFile, reader.m_Engine);
+    m_Worker = std::make_shared<adios2::query::Worker>(*m);
+}
+
+void QueryWorker::GetResultCoverage(
+    adios2::Box<adios2::Dims> &outputSelection,
+    std::vector<adios2::Box<adios2::Dims>> &touched_blocks)
+{
+    return m_Worker->GetResultCoverage(outputSelection, touched_blocks);
+}
+}

bindings/CXX11/adios2/cxx11/Query.h

@@ -0,0 +1,41 @@
+/*
+ * Distributed under the OSI-approved Apache License, Version 2.0.  See
+ * accompanying file Copyright.txt for details.
+ *
+ * Query.h : provides type utilities for ADIOS2 C++11 bindings
+ *
+ *  Created on: Aug 20, 2019
+ *      Author: Junmin Gu <jgu@lbl.gov>
+ */
+
+#ifndef ADIOS2_BINDINGS_CXX11_QUERY_H_
+#define ADIOS2_BINDINGS_CXX11_QUERY_H_
+
+#include <memory> // otherwise MSVC complains about std::shared_ptr
+
+#include "Engine.h"
+
+namespace adios2
+{
+
+namespace query
+{
+class Worker;
+} //
+
+class QueryWorker
+{
+public:
+    QueryWorker(const std::string &configFile, adios2::Engine &engine);
+
+    void
+    GetResultCoverage(adios2::Box<adios2::Dims> &,
+                      std::vector<adios2::Box<adios2::Dims>> &touched_blocks);
+
+private:
+    // adios2::query::Worker* m_Worker;
+    std::shared_ptr<adios2::query::Worker> m_Worker;
+}; // class QueryWorker
+
+} // end namespace adios2
+#endif /* ADIOS2_BINDINGS_CXX11_QUERY_H_ */

examples/CMakeLists.txt

@@ -5,6 +5,7 @@
 
 add_subdirectory(basics)
 add_subdirectory(hello)
+add_subdirectory(query)
 add_subdirectory(useCases)
 
 if(ADIOS2_HAVE_MPI)

examples/query/CMakeLists.txt

@@ -0,0 +1,10 @@
+#------------------------------------------------------------------------------#
+# Distributed under the OSI-approved Apache License, Version 2.0.  See
+# accompanying file Copyright.txt for details.
+#------------------------------------------------------------------------------#
+
+if(ADIOS2_HAVE_MPI)
+  add_executable(queryTest test_xml.cpp)
+  target_link_libraries(queryTest adios2 MPI::MPI_C pugixml)    
+endif()
+

examples/query/test_xml.cpp

@@ -0,0 +1,110 @@
+#include "adios2.h"
+#include <mpi.h>
+
+#include <cstdint>
+#include <iomanip>
+#include <iostream>
+#include <math.h>
+#include <memory>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+//#include "adios2/toolkit/query/Worker.h"
+
+int main(int argc, char *argv[])
+{
+    MPI_Init(&argc, &argv);
+    int rank, size;
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+    const std::size_t Nx = 10;
+
+    try
+    {
+        std::string configFileName = "query.xml";
+        std::string dataFileName = "/tmp/heatbp4.bp";
+        if (argc <= 2)
+        {
+            std::cout << "Usage: " << argv[0] << " configFileName  dataFileName"
+                      << std::endl;
+            return 0;
+        }
+
+        configFileName = argv[1];
+        dataFileName = argv[2];
+
+        if (rank == 0)
+        {
+            std::cout << " using config file = " << configFileName << std::endl;
+            std::cout << "        data file  = " << dataFileName << std::endl;
+        }
+
+        adios2::ADIOS ad =
+            adios2::ADIOS(configFileName, MPI_COMM_WORLD, adios2::DebugON);
+
+        adios2::IO queryIO = ad.DeclareIO("query");
+        adios2::Engine reader =
+            queryIO.Open(dataFileName, adios2::Mode::Read, MPI_COMM_WORLD);
+        adios2::QueryWorker w = adios2::QueryWorker(configFileName, reader);
+
+        std::vector<adios2::Box<adios2::Dims>> touched_blocks;
+
+        while (reader.BeginStep() == adios2::StepStatus::OK)
+        {
+            adios2::Box<adios2::Dims> empty;
+            w.GetResultCoverage(empty, touched_blocks);
+            std::cout << " ... now can read out touched blocks ... size="
+                      << touched_blocks.size() << std::endl;
+            reader.EndStep();
+        }
+
+        return 0;
+    }
+    catch (std::exception &e)
+    {
+        std::cout << "Exception, STOPPING PROGRAM from rank " << rank << "\n";
+        std::cout << e.what() << "\n";
+    }
+
+    MPI_Finalize();
+
+    return 0;
+}
+
+bool testMe(std::string &queryConfigFile, std::string const &doubleVarName,
+            MPI_Comm comm)
+{
+    adios2::ADIOS ad(queryConfigFile, comm, adios2::DebugON);
+    std::string dataFileName = "test.file";
+
+    // adios2::query::Worker w(queryConfigFile, comm);
+
+    // w.SetSource(inIO, reader);
+
+    {
+        // the config file should have info on idx method to use
+
+        /*
+        // if wanting to build customized minmax idx
+        // instead of using the existing block stats or bp4 minmax arrays
+        bool overwrite = false;
+        size_t recommendedSize = 20000;
+
+        if (!w.PrepareIdx(overwrite, recommendedSize,  doubleVarName))
+          return false;
+        */
+    }
+
+    adios2::IO inIO = ad.DeclareIO("query");
+    adios2::Engine reader = inIO.Open(dataFileName, adios2::Mode::Read, comm);
+
+    // std::vector<double> dataOutput;
+    // std::vector<adios2::Dims> coordinateOutput;
+
+    return true;
+}
+
+/*
+ */

source/adios2/CMakeLists.txt

@@ -80,6 +80,12 @@ add_library(adios2
 
   toolkit/profiling/iochrono/Timer.cpp
 
+  toolkit/query/Query.cpp
+  toolkit/query/Worker.cpp
+  toolkit/query/XmlUtil.cpp
+  toolkit/query/XmlWorker.cpp
+  toolkit/query/BlockIndex.cpp
+
   toolkit/transport/Transport.cpp
   toolkit/transport/file/FileStdio.cpp
   toolkit/transport/file/FileFStream.cpp

source/adios2/toolkit/query/BlockIndex.cpp

@@ -0,0 +1,25 @@
+#include "BlockIndex.h"
+#include "Query.h"
+//#include "BlockIndex.tcc"
+
+/*
+// optional
+void BlockIndex::GenerateIdxFile()
+{}
+
+void BlockIndex::EvaluateIdxFile()
+{}
+
+void BlockIndex::EvaluateBP4()
+{
+
+}
+
+void BlockIndex::Evaluate()
+{
+  if (m_IdxFileName.size() > 0)
+    return EvaluateIdxFile();
+  else
+    return EvaluateBP4(); // default
+}
+*/

source/adios2/toolkit/query/BlockIndex.h

@@ -0,0 +1,110 @@
+#ifndef ADIOS2_BLOCK_INDEX_H
+#define ADIOS2_BLOCK_INDEX_H
+
+#include "Index.h"
+#include "Query.h"
+
+namespace adios2
+{
+namespace query
+{
+
+template <class T>
+class BlockIndex
+{
+    struct Tree
+    {
+        //
+        // ** no need to keep the original block. might be smaller than
+        // blockIndex typename Variable<T>::info& m_BlockInfo;
+        //
+        std::vector<typename adios2::core::Variable<T>::Info> m_SubBlockInfo;
+    };
+
+public:
+    BlockIndex<T>(adios2::core::Variable<T> &var, adios2::core::IO &io,
+                  adios2::core::Engine &reader)
+    : m_Var(var), m_IdxIO(io), m_IdxReader(reader)
+    {
+    }
+
+    void Generate(std::string &fromBPFile, const adios2::Params &inputs) {}
+
+    void Evaluate(const QueryVar &query,
+                  std::vector<adios2::Box<adios2::Dims>> &resultSubBlocks)
+    {
+        std::size_t found = m_IdxReader.m_EngineType.find("BP4");
+        if (found != std::string::npos)
+        // if (m_IdxReader.m_EngineType.find("BP4") >= 0)
+        {
+            RunBP4Stat(query, resultSubBlocks);
+        }
+        else if (m_IdxReader.m_EngineType.find("BP3") >= 0)
+        {
+            RunDefaultBPStat(query, resultSubBlocks);
+        }
+    }
+
+    void RunBP4Stat(const QueryVar &query,
+                    std::vector<adios2::Box<adios2::Dims>> &hitBlocks)
+    {
+        std::cout << "BlockIndexEval(BP4) using BP3 now" << std::endl;
+        RunDefaultBPStat(query, hitBlocks);
+    }
+
+    void RunDefaultBPStat(const QueryVar &query,
+                          std::vector<adios2::Box<adios2::Dims>> &hitBlocks)
+    {
+        size_t currStep = m_IdxReader.CurrentStep();
+        adios2::Dims currShape = m_Var.Shape();
+        if (!query.IsSelectionValid(currShape))
+            return;
+
+        std::vector<typename adios2::core::Variable<T>::Info> varBlocksInfo =
+            m_IdxReader.BlocksInfo(m_Var, currStep);
+
+        for (auto &blockInfo : varBlocksInfo)
+        {
+            if (!query.TouchSelection(blockInfo.Start, blockInfo.Count))
+                continue;
+
+            T min = blockInfo.Min;
+            T max = blockInfo.Max;
+
+            // std::cout<<" min: "<<min<<"  max: "<<max<<std::endl;
+            bool isHit = query.m_RangeTree.CheckInterval(min, max);
+            if (isHit)
+            {
+                adios2::Box<adios2::Dims> box = {blockInfo.Start,
+                                                 blockInfo.Count};
+                hitBlocks.push_back(box);
+            }
+        }
+    }
+    /*
+    void Init (adios2::Engine& idxReader, const adios2::Variable<T>& var, int
+    ts); void SetContent(const QueryVar& query, int ts);
+
+    bool isBlockValid(adios2::Dims& blockStart, adios2::Dims& blockCount,
+                      adios2::Dims& selStart, adios2::Dims& selCount);
+
+    void WalkThrough(const QueryVar& query, std::vector<unsigned int>& results);
+    */
+
+    Tree m_Content;
+    adios2::core::Variable<T> m_Var;
+
+private:
+    //
+    // blockid <=> vector of subcontents
+    //
+    // std::string& m_DataFileName;
+    adios2::core::Engine &m_IdxReader;
+    adios2::core::IO &m_IdxIO;
+
+}; // class blockIndex
+
+}; // end namespace query
+}; // end namespace adios2
+
+#endif