Merge pull request #3526 from eisenhauer/BP5FileRelease

Change default file engine to BP5

bindings/C/adios2/c/adios2_c_variable.cpp

@@ -591,7 +591,7 @@ adios2_error adios2_variable_min(void *min, const adios2_variable *variable)
         T *minT = reinterpret_cast<T *>(min);                                  \
         const adios2::core::Variable<T> *variableT =                           \
             dynamic_cast<const adios2::core::Variable<T> *>(variableBase);     \
-        *minT = variableT->m_Min;                                              \
+        *minT = variableT->Min(adios2::EngineCurrentStep);                     \
     }
         ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation)
 #undef declare_template_instantiation
@@ -628,7 +628,7 @@ adios2_error adios2_variable_max(void *max, const adios2_variable *variable)
         T *maxT = reinterpret_cast<T *>(max);                                  \
         const adios2::core::Variable<T> *variableT =                           \
             dynamic_cast<const adios2::core::Variable<T> *>(variableBase);     \
-        *maxT = variableT->m_Max;                                              \
+        *maxT = variableT->Max(adios2::EngineCurrentStep);                     \
     }
         ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation)
 #undef declare_template_instantiation

bindings/CXX11/adios2/cxx11/Variable.h

@@ -263,13 +263,13 @@ class Variable
     adios2::Dims Count() const;
 
     /**
-     * For read mode, inspect the number of available steps
+     * For readRandomAccess mode, inspect the number of available steps
      * @return available steps
      */
     size_t Steps() const;
 
     /**
-     * For read mode, inspect the start step for available steps
+     * For readRandomAccess mode, inspect the start step for available steps
      * @return available start step
      */
     size_t StepsStart() const;

bindings/CXX11/adios2/cxx11/Variable.tcc

@@ -69,7 +69,7 @@ Variable<T>::DoAllStepsBlocksInfoMap() const
     MinVarInfo *minBlocksInfo = nullptr;
     minBlocksInfo = m_Variable->m_Engine->MinBlocksInfo(*m_Variable, 0);
     if (!minBlocksInfo)
-        throw std::logic_error("no implemented");
+        throw std::logic_error("not implemented");
     std::map<size_t, std::vector<typename Variable<T>::Info>>
         allStepsBlocksInfo;
 

bindings/Python/py11Engine.cpp

@@ -254,6 +254,93 @@ Engine::BlocksInfo(std::string &var_name, const size_t step) const
     // Grab the specified variable object and get its type string
     adios2::DataType var_type = m_Engine->GetIO().InquireVariableType(var_name);
 
+    MinVarInfo *minBlocksInfo = nullptr;
+    auto itVariable = m_Engine->m_IO.GetVariables().find(var_name);
+    auto Variable = itVariable->second.get();
+    minBlocksInfo = m_Engine->MinBlocksInfo(*Variable, 0);
+    if (minBlocksInfo)
+    {
+        for (auto &info : minBlocksInfo->BlocksInfo)
+        {
+            std::map<std::string, std::string> info_map;
+            std::stringstream start_ss;
+            std::cout << "Info loop" << std::endl;
+            for (size_t i = 0; i < (size_t)minBlocksInfo->Dims; ++i)
+            {
+                if (i != 0)
+                    start_ss << ",";
+                start_ss << info.Start[i];
+            }
+            info_map["Start"] = start_ss.str();
+            std::stringstream count_ss;
+            for (size_t i = 0; i < (size_t)minBlocksInfo->Dims; ++i)
+            {
+                if (i != 0)
+                    count_ss << ",";
+                count_ss << info.Count[i];
+            }
+            info_map["Count"] = count_ss.str();
+            info_map["WriterID"] = std::to_string(info.WriterID);
+            info_map["BlockID"] = std::to_string(info.BlockID);
+            info_map["IsValue"] = minBlocksInfo->IsValue ? "True" : "False";
+            std::ostringstream osMax, osMin;
+            switch (var_type)
+            {
+            case DataType::Int8:
+                osMax << info.MinMax.MaxUnion.field_int8;
+                osMin << info.MinMax.MinUnion.field_int8;
+                break;
+            case DataType::Int16:
+                osMax << info.MinMax.MaxUnion.field_int16;
+                osMin << info.MinMax.MinUnion.field_int16;
+                break;
+            case DataType::Int32:
+                osMax << info.MinMax.MaxUnion.field_int32;
+                osMin << info.MinMax.MinUnion.field_int32;
+                break;
+            case DataType::Int64:
+                osMax << info.MinMax.MaxUnion.field_int64;
+                osMin << info.MinMax.MinUnion.field_int64;
+                break;
+            case DataType::UInt8:
+                osMax << info.MinMax.MaxUnion.field_uint8;
+                osMin << info.MinMax.MinUnion.field_uint8;
+                break;
+            case DataType::UInt16:
+                osMax << info.MinMax.MaxUnion.field_uint16;
+                osMin << info.MinMax.MinUnion.field_uint16;
+                break;
+            case DataType::UInt32:
+                osMax << info.MinMax.MaxUnion.field_uint32;
+                osMin << info.MinMax.MinUnion.field_uint32;
+                break;
+            case DataType::UInt64:
+                osMax << info.MinMax.MaxUnion.field_uint64;
+                osMin << info.MinMax.MinUnion.field_uint64;
+                break;
+            case DataType::Float:
+                osMax << info.MinMax.MaxUnion.field_float;
+                osMin << info.MinMax.MinUnion.field_float;
+                break;
+            case DataType::Double:
+                osMax << info.MinMax.MaxUnion.field_double;
+                osMin << info.MinMax.MinUnion.field_double;
+                break;
+            case DataType::LongDouble:
+                osMax << info.MinMax.MaxUnion.field_ldouble;
+                osMin << info.MinMax.MinUnion.field_ldouble;
+                break;
+            default:
+                break;
+            }
+            info_map["Max"] = osMax.str();
+            info_map["Min"] = osMin.str();
+            info_map["IsReverseDims"] =
+                minBlocksInfo->IsReverseDims ? "True" : "False";
+            rv.push_back(info_map);
+        }
+        return rv;
+    }
     // Use the macro incantation to call the right instantiation of
     // core::BlocksInfo<>() Note that we are flatting the Dims type items, and
     // returning everything as a dictionary of strings.
@@ -296,6 +383,7 @@ Engine::BlocksInfo(std::string &var_name, const size_t step) const
             rv.push_back(info_map);                                            \
         }                                                                      \
     }
+
     ADIOS2_FOREACH_PYTHON_TYPE_1ARG(GET_BLOCKS_INFO)
 #undef GET_BLOCKS_INFO
     else

docs/user_guide/source/advanced/plugins.rst

@@ -239,7 +239,7 @@ If you'd like to load your operator plugin through an XML config file, the follo
                     <parameter key="SecretKeyFile" value="test-key" />
                 </operation>
             </variable>
-            <engine type="BP4">
+            <engine type="BP5">
             </engine>
         </io>
     </adios-config>

docs/user_guide/source/components/engine.rst

@@ -572,7 +572,7 @@ If the ``SetEngine`` function is not invoked the default engine is the ``BPFile`
 +-------------------------+---------+---------------------------------------------+
 | Application             | Engine  | Description                                 |
 +-------------------------+---------+---------------------------------------------+
-| File                    | BP4     | DEFAULT write/read ADIOS2 native bp files   |
+| File                    | BP5     | DEFAULT write/read ADIOS2 native bp files   |
 |                         |         |                                             |
 |                         | HDF5    | write/read interoperability with HDF5 files |
 +-------------------------+---------+---------------------------------------------+

docs/user_guide/source/components/runtime.rst

@@ -67,7 +67,7 @@ For an example file refer to `adios2 config file example in our repo. <https://g
      
      Engine:
         # If Type is missing or commented out, default Engine is picked up
-        Type: "BP4"
+        Type: "BP5"
         # optional engine parameters
         key1: value1
         key2: value2

docs/user_guide/source/ecosystem/h5vol/vol.rst

@@ -51,7 +51,7 @@ Internal
 
 
 
-..  To choose what ADIOS2 Engine to use, set env variable: ADIOS2_ENGINE (default is BP4)
+..  To choose what ADIOS2 Engine to use, set env variable: ADIOS2_ENGINE (default is BP5)
 
 
 

docs/user_guide/source/engines/bp4.rst

@@ -3,7 +3,7 @@ BP4
 ***
 
 The BP4 Engine writes and reads files in ADIOS2 native binary-pack (bp version 4) format. 
-This is a new format for ADIOS 2.x which improves on the metadata operations of the older BP3 format. 
+This was a new format for ADIOS 2.5 and improved on the metadata operations of the older BP3 format. 
 Compared to the older format, BP4 provides three main advantages:
 
   * Fast and safe **appending** of multiple output steps into the same file. Better performance than writing new files each step. 

docs/user_guide/source/engines/bp5.rst

@@ -3,8 +3,9 @@ BP5
 ***
 
 The BP5 Engine writes and reads files in ADIOS2 native binary-pack (bp version 5) format. 
-This is a new format for ADIOS 2.8 which improves on the metadata operations and the memory consumption 
-of the older BP4/BP3 formats. Compared to the older format, BP5 provides three main advantages:
+This was a new format for ADIOS 2.8, improving on the metadata operations and the memory consumption 
+of the older BP4/BP3 formats. BP5 is the default file format as of
+ADIOS 2.9.  As compared to the older format, BP5 provides three main advantages:
 
   * **Lower memory** consumption. Deferred Puts will use user buffer for I/O wherever possible thus saving on a memory copy. 
     Aggregation uses a fixed-size shared-memory segment on each compute node instead of using MPI to send data from one process to another. 
@@ -14,7 +15,7 @@ of the older BP4/BP3 formats. Compared to the older format, BP5 provides three m
     Restart can append to an existing series by truncating unwanted steps. Readers can filter out unwanted steps to only see and process a 
     limited set of steps. Just like as in BP4, existing steps cannot be corrupted by appending new steps.
 
-In 2.8 BP5 is a brand new file format and engine. It still does **NOT** support some functionality of BP4:
+In 2.8 BP5 was a brand new file format and engine. It still does **NOT** support some functionality of BP4:
 
   * **Burst buffer support** for writing data.
 

docs/user_guide/source/engines/virtual_engines.rst

@@ -8,7 +8,7 @@ The following I/O uses cases are supported by virtual engine names:
 
 1. ``File``: File I/O (Default engine).
 
-   This sets up the I/O for files. If the file name passed in Open() ends with ".bp", then the BP4 engine will be used starting in v2.5.0.
+   This sets up the I/O for files. If the file name passed in Open() ends with ".bp", then the BP5 engine will be used starting in v2.9.0.
    If it ends with ".h5", the HDF5 engine will be used. For old .bp files (BP version 3 format), the BP3 engine 
    will be used for reading (v2.4.0 and below). 
 
@@ -46,16 +46,16 @@ Virtual Engine Setups
 
 These are the actual settings in ADIOS when a virtual engine is selected. The parameters below can be modified before the Open call. 
 
-1. ``File``. Refer to the parameter settings for these engines of ``BP4``, ``BP3`` and ``HDF5`` engines earlier in this section. 
+1. ``File``. Refer to the parameter settings for these engines of
+   ``BP5``, ``BP4``, ``BP3`` and ``HDF5`` engines earlier in this section. 
 
-2. ``FileStream``. The engine is ``BP4``. The parameters are set to:
+2. ``FileStream``. The engine is ``BP5``. The parameters are set to:
 
 ============================== ===================== ===========================================================
  **Key**                       **Value Format**      **Default** and Examples
 ============================== ===================== ===========================================================
  OpenTimeoutSecs                float                 **3600**  (wait for up to an hour)
  BeginStepPollingFrequencySecs  float                 **1**     (poll the file system with 1 second frequency
- StreamReader                   bool                  **On**    (process metadata in streaming mode)
 ============================== ===================== ===========================================================
 
 3. ``InSituAnalysis``. The engine is ``SST``. The parameters are set to:

examples/cuda/CMakeLists.txt

@@ -3,10 +3,6 @@
 # accompanying file Copyright.txt for details.
 #------------------------------------------------------------------------------#
 
-add_executable(CudaBP5WriteRead_cuda cudaBP5WriteRead.cu)
-target_link_libraries(CudaBP5WriteRead_cuda PUBLIC adios2::cxx11 CUDA::cudart)
-set_target_properties(CudaBP5WriteRead_cuda PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
-
-add_executable(CudaBP4WriteRead_cuda cudaBP4WriteRead.cu)
-target_link_libraries(CudaBP4WriteRead_cuda PUBLIC adios2::cxx11 CUDA::cudart)
-set_target_properties(CudaBP4WriteRead_cuda PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
+add_executable(CudaBPWriteRead_cuda cudaBPWriteRead.cu)
+target_link_libraries(CudaBPWriteRead_cuda PUBLIC adios2::cxx11 CUDA::cudart)
+set_target_properties(CudaBPWriteRead_cuda PROPERTIES CUDA_SEPARABLE_COMPILATION ON)