Merge pull request #1562 from pnorbert/bp4dbg-array-attr

  fix travis errors
  Print content of array attributes, in metadata as well as in data file. Format idx table better.

source/utils/bp4dbg/bp4dbg_data.py

@@ -399,14 +399,18 @@ def ReadAMD(f, attridx, attrsStartPosition, attrsTotalLength):
             print("'" + strList[j] + "'", end="")
             if j < len(strList) - 1:
                 print(", ", end="")
-            print("]")
+        print("]")
     else:
         nBytes = np.fromfile(f, dtype=np.uint32, count=1)[0]
         typeSize = bp4dbg_utils.GetTypeSize(typeID)
         nElems = int(nBytes / typeSize)
         data = readDataToNumpyArray(f, typeName, nElems)
-        print("      Value           : {0}  ({1} elements)".format(
-            data[0], nElems))
+        print("      Value           : [", end="")
+        for j in range(nElems):
+            print("{0}".format(data[j]), end="")
+            if j < nElems - 1:
+                print(", ", end="")
+        print("]")
 
     # End TAG AMD]
     tag = f.read(4)
@@ -554,7 +558,8 @@ def ReadPG(f, fileSize, pgidx):
 
 
 def DumpData(fileName):
-    print("=== Data File: " + fileName + " ====")
+    print("========================================================")
+    print("    Data File: " + fileName)
     with open(fileName, "rb") as f:
         fileSize = fstat(f.fileno()).st_size
         status = True

source/utils/bp4dbg/bp4dbg_idxtable.py

@@ -57,18 +57,20 @@ def ReadIndexHeader(f, fileSize):
         status = False
 
     print("------------------------------------------------------------"
-          "-----------------------------------------------------")
-    print("|      Version string      | Major | Minor | Micro | N/A 1B "
-          "| Endian | Subfiles | BP version | N/A 24B | isLive |")
+          "----------------------------------------------------")
+    print("|      Version string      | Major | Minor | Micro | unused "
+          "| Endian | Subfiles | BP version | unused | isLive |")
+    print("|          24 bytes        |   1B  |   1B  |   1B  |   1B   "
+          "|   1B   |    2B    |     1B     |   24B  |   8B   |")
     print("+-----------------------------------------------------------"
-          "----------------------------------------------------+")
+          "---------------------------------------------------+")
     print(
         "| {0} |   {1}   |   {2}   |   {3}   |        |  {4}   |    {5}   |"
-        "      {6}     |         |   {7}  |"
+        "      {6}     |        |   {7}  |"
         .format(versionStr, major, minor, micro, endian, hasSubfiles,
                 bpversion, isLive))
     print("------------------------------------------------------------"
-          "-----------------------------------------------------")
+          "----------------------------------------------------")
     return status
 
 
@@ -79,27 +81,28 @@ def ReadIndex(f, fileSize):
     nRows = int(nBytes / 64)
     table = f.read(nBytes)
     print(" ")
-    print("-----------------------------------------------------"
-          "--------------------------")
-    print("| Step |  Rank  |   PGPtr   |  VarPtr   |  AttPtr   |"
-          "  EndPtr   |   Timestep  |")
-    print("+----------------------------------------------------"
-          "-------------------------+")
+    print("-----------------------------------------------------------------"
+          "------------------------------------------")
+    print("|   Step   |   Rank   |    PGPtr    |   VarPtr    |   AttPtr    |"
+          "   EndPtr    |  Timestamp  |    unused   |")
+    print("+----------------------------------------------------------------"
+          "-----------------------------------------+")
     for r in range(0, nRows):
         pos = r * 64
         data = np.frombuffer(table, dtype=np.uint64, count=8, offset=pos)
-        step = str(data[0]).rjust(5)
-        rank = str(data[1]).rjust(8)
-        pgptr = str(data[2]).center(10)
-        varptr = str(data[3]).center(10)
-        attptr = str(data[4]).center(10)
-        endptr = str(data[5]).center(10)
+        step = str(data[0]).rjust(9)
+        rank = str(data[1]).rjust(9)
+        pgptr = str(data[2]).rjust(12)
+        varptr = str(data[3]).rjust(12)
+        attptr = str(data[4]).rjust(12)
+        endptr = str(data[5]).rjust(12)
         time = str(data[6]).rjust(12)
-        print("|" + step + " |" + rank + "|" + pgptr + " |" + varptr + " |" +
-              attptr + " |" + endptr + " |" + time + " |")
+        unused = str(data[7]).rjust(12)
+        print("|" + step + " |" + rank + " |" + pgptr + " |" + varptr + " |" +
+              attptr + " |" + endptr + " |" + time + " |" + unused + " |")
 
-    print("-----------------------------------------------------"
-          "--------------------------")
+    print("-----------------------------------------------------------------"
+          "------------------------------------------")
 
     if fileSize - f.tell() > 1:
         print("ERROR: There are {0} bytes at the end of file"
@@ -110,7 +113,9 @@ def ReadIndex(f, fileSize):
 
 
 def DumpIndexTable(fileName):
-    print("=== Index Table File: " + fileName + " ====")
+    print("========================================================")
+    print("    Index Table File: " + fileName)
+    print("========================================================")
     status = False
     with open(fileName, "rb") as f:
         fileSize = fstat(f.fileno()).st_size

source/utils/bp4dbg/bp4dbg_metadata.py

@@ -94,7 +94,8 @@ def bDataToNumpyArray(cData, typeName, nElements):
         return np.zeros(1, dtype=np.uint32)
 
 
-def ReadCharacteristicsFromMetaData(buf, idx, pos, limit, typeID, fileOffset):
+def ReadCharacteristicsFromMetaData(buf, idx, pos, limit, typeID,
+                                    fileOffset, isVarCharacteristics):
     cStartPosition = pos
     dataTypeName = bp4dbg_utils.GetTypeName(typeID)
     print("        Block {0}: ".format(idx))
@@ -108,6 +109,11 @@ def ReadCharacteristicsFromMetaData(buf, idx, pos, limit, typeID, fileOffset):
     pos = pos + 4
     print("            Characteristics Length  : {0}".format(charLen))
 
+    # For attributes, we need to remember the dimensions and size
+    # when reading the value
+    ndim = 0
+    nElems = 1
+
     for i in range(nChars):
         print("            Characteristics[{0}]".format(i))
         # 1 byte TYPE
@@ -126,6 +132,7 @@ def ReadCharacteristicsFromMetaData(buf, idx, pos, limit, typeID, fileOffset):
                 print("                Dims (lgo)  : (", end="")
                 for d in range(ndim):
                     p = 3 * d
+                    nElems = int(nElems * lgo[p])  # need for value later
                     print("{0}:{1}:{2}".format(lgo[p], lgo[p + 1], lgo[p + 2]),
                           end="")
                     if d < ndim - 1:
@@ -156,11 +163,25 @@ def ReadCharacteristicsFromMetaData(buf, idx, pos, limit, typeID, fileOffset):
                 print("]")
 
             else:
-                cData = buf[pos:pos + cLen]
-                pos = pos + cLen
-                data = bDataToNumpyArray(cData, dataTypeName, 1)
-                print("                Value       : {0}  ({1} bytes)".format(
-                      data[0], cLen))
+                if (isVarCharacteristics):
+                    cData = buf[pos:pos + cLen]
+                    pos = pos + cLen
+                    data = bDataToNumpyArray(cData, dataTypeName, 1)
+                    print("                Value       : {0}"
+                          "  ({1} bytes)".format(data[0], cLen))
+                else:  # attribute value characteristics are different
+                    dataTypeSize = bp4dbg_utils.GetTypeSize(typeID)
+                    nBytes = int(nElems * dataTypeSize)
+                    cData = buf[pos:pos + nBytes]
+                    pos = pos + nBytes
+                    data = bDataToNumpyArray(cData, dataTypeName, nElems)
+                    print("                Value       : [", end="")
+                    for j in range(nElems):
+                        print("{0}".format(data[j]), end="")
+                        if j < nElems - 1:
+                            print(", ", end="")
+                    print("]")
+
         elif cName == 'offset' or cName == 'payload_offset':
             cData = buf[pos:pos + cLen]
             pos = pos + cLen
@@ -307,7 +328,7 @@ def ReadVarMD(buf, idx, pos, limit, varStartOffset):
         newlimit = limit - (pos - varStartPosition)
         fileOffset = varStartOffset + (pos - varStartPosition)
         status, pos = ReadCharacteristicsFromMetaData(
-            buf, i, pos, newlimit, typeID, fileOffset)
+            buf, i, pos, newlimit, typeID, fileOffset, True)
         if (not status):
             return False
 
@@ -374,7 +395,7 @@ def ReadAttrMD(buf, idx, pos, limit, attrStartOffset):
         newlimit = limit - (pos - attrStartPosition)
         fileOffset = attrStartOffset + (pos - attrStartPosition)
         status, pos = ReadCharacteristicsFromMetaData(
-            buf, i, pos, newlimit, typeID, fileOffset)
+            buf, i, pos, newlimit, typeID, fileOffset, False)
         if (not status):
             return False
 
@@ -469,7 +490,8 @@ def ReadMetadataStep(f, fileSize, step):
 
 
 def DumpMetaData(fileName):
-    print("=== Metadata File: " + fileName + " ====")
+    print("========================================================")
+    print("    Metadata File: " + fileName)
     with open(fileName, "rb") as f:
         fileSize = fstat(f.fileno()).st_size
         status = True