Merge pull request #1552 from pnorbert/bp4dbg

* commit 'd93747d':
  remove debug prints in BP4 serializer
  Fix flake8 errors
  Install all .py files for bp4dbg
  bp4dbg: dump metadata too. Print values for characteristics and attributes. Fix: BP4 and BP3 attribute length in data was written as 8 bytes and the wrong value; now writing the correct length and on 4 bytes only.

source/adios2/toolkit/format/bp3/BP3Serializer.tcc

@@ -162,7 +162,8 @@ void BP3Serializer::PutAttributeLengthInData(
 
     // back to attribute length
     size_t backPosition = attributeLengthPosition;
-    helper::CopyToBuffer(buffer, backPosition, &attributeLengthPosition);
+    uint32_t len = static_cast<uint32_t>(position - attributeLengthPosition);
+    helper::CopyToBuffer(buffer, backPosition, &len);
 
     absolutePosition += position - attributeLengthPosition;
 }

source/adios2/toolkit/format/bp4/BP4Serializer.tcc

@@ -96,7 +96,7 @@ inline void BP4Serializer::PutVariablePayload(
         PutOperationPayloadInBuffer(variable, blockInfo);
     }
 
-    std::cout << " -- Var payload name=" << variable.m_Name
+    /*std::cout << " -- Var payload name=" << variable.m_Name
               << " startPos = " << std::to_string(startingPos)
               << " end position = " << std::to_string(m_Data.m_Position)
               << " end abs.position = "
@@ -108,7 +108,7 @@ inline void BP4Serializer::PutVariablePayload(
                      m_Data.m_Buffer.data() + m_Data.m_Position - 8))
               << " 4 bytes ahead = '"
               << std::string(m_Data.m_Buffer.data() + startingPos - 4, 4) << "'"
-              << std::endl;
+              << std::endl;*/
 
     ProfilerStop("buffering");
 }
@@ -147,10 +147,12 @@ void BP4Serializer::PutAttributeLengthInData(
     const size_t attributeLengthPosition) noexcept
 {
     auto &buffer = m_Data.m_Buffer;
+    auto &position = m_Data.m_Position;
 
     // back to attribute length
     size_t backPosition = attributeLengthPosition;
-    helper::CopyToBuffer(buffer, backPosition, &attributeLengthPosition);
+    uint32_t len = static_cast<uint32_t>(position - attributeLengthPosition);
+    helper::CopyToBuffer(buffer, backPosition, &len);
 }
 
 template <>
@@ -507,14 +509,12 @@ void BP4Serializer::PutVariableMetadataInData(
     // starting position in vmdEnd from where we copy to buffer
     // we don't copy the \0 from vmdEnd !
     const char *ptr = vmdEnd + (sizeof(vmdEnd) - 1 - vmdEndLen);
-    std::cout << " -- Pad metadata with " << std::to_string(padSize)
+    /*std::cout << " -- Pad metadata with " << std::to_string(padSize)
               << " bytes. var = " << variable.m_Name << " rank = " << m_RankMPI
               << " position = " << std::to_string(position) << " pad string = '"
-              << ptr << "'" << std::endl;
-    /*              << " buffer memory address = "
-                  <<
-       std::to_string(reinterpret_cast<std::uintptr_t>(buffer.data()))
-                  << std::endl;*/
+              << ptr << "'"
+              << std::to_string(reinterpret_cast<std::uintptr_t>(buffer.data()))
+              << std::endl;*/
     helper::CopyToBuffer(buffer, position, &vmdEndLen, 1);
     helper::CopyToBuffer(buffer, position, ptr, vmdEndLen);
 

source/utils/CMakeLists.txt

@@ -39,5 +39,6 @@ if(ADIOS2_HAVE_MPI)
 endif()
 
 # bp4dbg Python tool
-install(PROGRAMS bp4dbg/bp4dbg.py bp4dbg/bp4dbg_data.py bp4dbg/bp4dbg_utils.py
+install(PROGRAMS bp4dbg/bp4dbg.py bp4dbg/bp4dbg_data.py bp4dbg/bp4dbg_utils.py 
+        bp4dbg/bp4dbg_metadata.py bp4dbg/bp4dbg_idxtable.py
         DESTINATION ${CMAKE_INSTALL_BINDIR}/bp4dbg)

source/utils/bp4dbg/bp4dbg.py

@@ -6,6 +6,7 @@
 import glob
 from bp4dbg_data import DumpData
 from bp4dbg_idxtable import DumpIndexTable
+from bp4dbg_metadata import DumpMetaData
 
 
 def SetupArgs():
@@ -67,28 +68,36 @@ def CheckFileName(args):
 
 
 def DumpIndexTableFile(args):
-    DumpIndexTable(args.idxFileName)
+    indexFileList = glob.glob(args.idxFileName)
+    if len(indexFileList) > 0:
+        DumpIndexTable(indexFileList[0])
+    else:
+        print("There is  no BP4 Index Table file as " + args.idxFileName)
 
 
 def DumpMetadataFiles(args):
     mdFileList = glob.glob(args.metadataFileName)
-    for fname in mdFileList:
-        print("=== Metadata File: " + fname + " ====")
+    if len(mdFileList) > 0:
+        for fname in mdFileList:
+            DumpMetaData(fname)
+    else:
+        print("There are no BP4 Metadata files in   " + args.metadataFileName)
 
 
 def DumpDataFiles(args):
     dataFileList = glob.glob(args.dataFileName)
-    for fname in dataFileList:
-        DumpData(fname)
+    if len(dataFileList) > 0:
+        for fname in dataFileList:
+            DumpData(fname)
+    else:
+        print("There are no BP4 Data files in       " + args.dataFileName)
 
 
 if __name__ == "__main__":
 
     args = SetupArgs()
-#    print(args)
-
-#    print("Debug file {0}".format(args.infile), flush=True)
     CheckFileName(args)
+    # print(args)
 
     if (args.dumpIdx):
         DumpIndexTableFile(args)