From 821987058facf1246476e078de1d162027b6bfb8 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 27 Jun 2019 17:53:17 -0400 Subject: [PATCH 1/2] Print content of array attributes, in metadata as well as in data file. Format idx table better. --- source/utils/bp4dbg/bp4dbg_data.py | 13 +++++-- source/utils/bp4dbg/bp4dbg_idxtable.py | 51 ++++++++++++++------------ source/utils/bp4dbg/bp4dbg_metadata.py | 40 +++++++++++++++----- 3 files changed, 68 insertions(+), 36 deletions(-) diff --git a/source/utils/bp4dbg/bp4dbg_data.py b/source/utils/bp4dbg/bp4dbg_data.py index 66b21e297a..41b256da9c 100644 --- a/source/utils/bp4dbg/bp4dbg_data.py +++ b/source/utils/bp4dbg/bp4dbg_data.py @@ -399,14 +399,18 @@ def ReadAMD(f, attridx, attrsStartPosition, attrsTotalLength): print("'" + strList[j] + "'", end="") if j < len(strList) - 1: print(", ", end="") - print("]") + print("]") else: nBytes = np.fromfile(f, dtype=np.uint32, count=1)[0] typeSize = bp4dbg_utils.GetTypeSize(typeID) nElems = int(nBytes / typeSize) data = readDataToNumpyArray(f, typeName, nElems) - print(" Value : {0} ({1} elements)".format( - data[0], nElems)) + print(" Value : [", end="") + for j in range(nElems): + print("{0}".format(data[j]), end="") + if j < nElems - 1: + print(", ", end="") + print("]") # End TAG AMD] tag = f.read(4) @@ -554,7 +558,8 @@ def ReadPG(f, fileSize, pgidx): def DumpData(fileName): - print("=== Data File: " + fileName + " ====") + print("========================================================") + print(" Data File: " + fileName) with open(fileName, "rb") as f: fileSize = fstat(f.fileno()).st_size status = True diff --git a/source/utils/bp4dbg/bp4dbg_idxtable.py b/source/utils/bp4dbg/bp4dbg_idxtable.py index 05f5fcd5e4..6b5cdadb66 100644 --- a/source/utils/bp4dbg/bp4dbg_idxtable.py +++ b/source/utils/bp4dbg/bp4dbg_idxtable.py @@ -57,18 +57,20 @@ def ReadIndexHeader(f, fileSize): status = False print("------------------------------------------------------------" - "-----------------------------------------------------") - print("| Version string | Major | Minor | Micro | N/A 1B " - "| Endian | Subfiles | BP version | N/A 24B | isLive |") + "----------------------------------------------------") + print("| Version string | Major | Minor | Micro | unused " + "| Endian | Subfiles | BP version | unused | isLive |") + print("| 24 bytes | 1B | 1B | 1B | 1B " + "| 1B | 2B | 1B | 24B | 8B |") print("+-----------------------------------------------------------" - "----------------------------------------------------+") + "---------------------------------------------------+") print( "| {0} | {1} | {2} | {3} | | {4} | {5} |" - " {6} | | {7} |" + " {6} | | {7} |" .format(versionStr, major, minor, micro, endian, hasSubfiles, bpversion, isLive)) print("------------------------------------------------------------" - "-----------------------------------------------------") + "----------------------------------------------------") return status @@ -79,27 +81,28 @@ def ReadIndex(f, fileSize): nRows = int(nBytes / 64) table = f.read(nBytes) print(" ") - print("-----------------------------------------------------" - "--------------------------") - print("| Step | Rank | PGPtr | VarPtr | AttPtr |" - " EndPtr | Timestep |") - print("+----------------------------------------------------" - "-------------------------+") + print("-----------------------------------------------------------------" + "-------------------------------------") + print("| Step | Rank | PGPtr | VarPtr | AttPtr |" + " EndPtr | Timestamp | unused |") + print("+----------------------------------------------------------------" + "------------------------------------+") for r in range(0, nRows): pos = r * 64 data = np.frombuffer(table, dtype=np.uint64, count=8, offset=pos) - step = str(data[0]).rjust(5) - rank = str(data[1]).rjust(8) - pgptr = str(data[2]).center(10) - varptr = str(data[3]).center(10) - attptr = str(data[4]).center(10) - endptr = str(data[5]).center(10) + step = str(data[0]).rjust(9) + rank = str(data[1]).rjust(9) + pgptr = str(data[2]).rjust(12) + varptr = str(data[3]).rjust(12) + attptr = str(data[4]).rjust(12) + endptr = str(data[5]).rjust(12) time = str(data[6]).rjust(12) - print("|" + step + " |" + rank + "|" + pgptr + " |" + varptr + " |" + - attptr + " |" + endptr + " |" + time + " |") + unused = str(data[6]).rjust(12) + print("|" + step + " |" + rank + " |" + pgptr + " |" + varptr + " |" + + attptr + " |" + endptr + " |" + time + " | |") - print("-----------------------------------------------------" - "--------------------------") + print("-----------------------------------------------------------------" + "-------------------------------------") if fileSize - f.tell() > 1: print("ERROR: There are {0} bytes at the end of file" @@ -110,7 +113,9 @@ def ReadIndex(f, fileSize): def DumpIndexTable(fileName): - print("=== Index Table File: " + fileName + " ====") + print("========================================================") + print(" Index Table File: " + fileName) + print("========================================================") status = False with open(fileName, "rb") as f: fileSize = fstat(f.fileno()).st_size diff --git a/source/utils/bp4dbg/bp4dbg_metadata.py b/source/utils/bp4dbg/bp4dbg_metadata.py index 9b2df15206..cf0836cc20 100644 --- a/source/utils/bp4dbg/bp4dbg_metadata.py +++ b/source/utils/bp4dbg/bp4dbg_metadata.py @@ -94,7 +94,8 @@ def bDataToNumpyArray(cData, typeName, nElements): return np.zeros(1, dtype=np.uint32) -def ReadCharacteristicsFromMetaData(buf, idx, pos, limit, typeID, fileOffset): +def ReadCharacteristicsFromMetaData(buf, idx, pos, limit, typeID, + fileOffset, isVarCharacteristics): cStartPosition = pos dataTypeName = bp4dbg_utils.GetTypeName(typeID) print(" Block {0}: ".format(idx)) @@ -108,6 +109,11 @@ def ReadCharacteristicsFromMetaData(buf, idx, pos, limit, typeID, fileOffset): pos = pos + 4 print(" Characteristics Length : {0}".format(charLen)) + # For attributes, we need to remember the dimensions and size + # when reading the value + ndim = 0 + nElems = 1 + for i in range(nChars): print(" Characteristics[{0}]".format(i)) # 1 byte TYPE @@ -126,6 +132,7 @@ def ReadCharacteristicsFromMetaData(buf, idx, pos, limit, typeID, fileOffset): print(" Dims (lgo) : (", end="") for d in range(ndim): p = 3 * d + nElems = int(nElems * lgo[p]) # need for value later print("{0}:{1}:{2}".format(lgo[p], lgo[p + 1], lgo[p + 2]), end="") if d < ndim - 1: @@ -156,11 +163,25 @@ def ReadCharacteristicsFromMetaData(buf, idx, pos, limit, typeID, fileOffset): print("]") else: - cData = buf[pos:pos + cLen] - pos = pos + cLen - data = bDataToNumpyArray(cData, dataTypeName, 1) - print(" Value : {0} ({1} bytes)".format( - data[0], cLen)) + if (isVarCharacteristics): + cData = buf[pos:pos + cLen] + pos = pos + cLen + data = bDataToNumpyArray(cData, dataTypeName, 1) + print(" Value : {0} ({1} bytes)".format( + data[0], cLen)) + else: # attribute value characteristics are different + dataTypeSize = bp4dbg_utils.GetTypeSize(typeID) + nBytes = int(nElems * dataTypeSize) + cData = buf[pos:pos + nBytes] + pos = pos + nBytes + data = bDataToNumpyArray(cData, dataTypeName, nElems) + print(" Value : [", end="") + for j in range(nElems): + print("{0}".format(data[j]), end="") + if j < nElems - 1: + print(", ", end="") + print("]") + elif cName == 'offset' or cName == 'payload_offset': cData = buf[pos:pos + cLen] pos = pos + cLen @@ -307,7 +328,7 @@ def ReadVarMD(buf, idx, pos, limit, varStartOffset): newlimit = limit - (pos - varStartPosition) fileOffset = varStartOffset + (pos - varStartPosition) status, pos = ReadCharacteristicsFromMetaData( - buf, i, pos, newlimit, typeID, fileOffset) + buf, i, pos, newlimit, typeID, fileOffset, True) if (not status): return False @@ -374,7 +395,7 @@ def ReadAttrMD(buf, idx, pos, limit, attrStartOffset): newlimit = limit - (pos - attrStartPosition) fileOffset = attrStartOffset + (pos - attrStartPosition) status, pos = ReadCharacteristicsFromMetaData( - buf, i, pos, newlimit, typeID, fileOffset) + buf, i, pos, newlimit, typeID, fileOffset, False) if (not status): return False @@ -469,7 +490,8 @@ def ReadMetadataStep(f, fileSize, step): def DumpMetaData(fileName): - print("=== Metadata File: " + fileName + " ====") + print("========================================================") + print(" Metadata File: " + fileName) with open(fileName, "rb") as f: fileSize = fstat(f.fileno()).st_size status = True From 608006b940b9a8bb5e494b796f9db5f845f74905 Mon Sep 17 00:00:00 2001 From: Norbert Podhorszki Date: Thu, 27 Jun 2019 19:09:01 -0400 Subject: [PATCH 2/2] fix travis errors --- source/utils/bp4dbg/bp4dbg_idxtable.py | 12 ++++++------ source/utils/bp4dbg/bp4dbg_metadata.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/utils/bp4dbg/bp4dbg_idxtable.py b/source/utils/bp4dbg/bp4dbg_idxtable.py index 6b5cdadb66..20b7428f84 100644 --- a/source/utils/bp4dbg/bp4dbg_idxtable.py +++ b/source/utils/bp4dbg/bp4dbg_idxtable.py @@ -82,11 +82,11 @@ def ReadIndex(f, fileSize): table = f.read(nBytes) print(" ") print("-----------------------------------------------------------------" - "-------------------------------------") + "------------------------------------------") print("| Step | Rank | PGPtr | VarPtr | AttPtr |" - " EndPtr | Timestamp | unused |") + " EndPtr | Timestamp | unused |") print("+----------------------------------------------------------------" - "------------------------------------+") + "-----------------------------------------+") for r in range(0, nRows): pos = r * 64 data = np.frombuffer(table, dtype=np.uint64, count=8, offset=pos) @@ -97,12 +97,12 @@ def ReadIndex(f, fileSize): attptr = str(data[4]).rjust(12) endptr = str(data[5]).rjust(12) time = str(data[6]).rjust(12) - unused = str(data[6]).rjust(12) + unused = str(data[7]).rjust(12) print("|" + step + " |" + rank + " |" + pgptr + " |" + varptr + " |" + - attptr + " |" + endptr + " |" + time + " | |") + attptr + " |" + endptr + " |" + time + " |" + unused + " |") print("-----------------------------------------------------------------" - "-------------------------------------") + "------------------------------------------") if fileSize - f.tell() > 1: print("ERROR: There are {0} bytes at the end of file" diff --git a/source/utils/bp4dbg/bp4dbg_metadata.py b/source/utils/bp4dbg/bp4dbg_metadata.py index cf0836cc20..e725a838cd 100644 --- a/source/utils/bp4dbg/bp4dbg_metadata.py +++ b/source/utils/bp4dbg/bp4dbg_metadata.py @@ -167,8 +167,8 @@ def ReadCharacteristicsFromMetaData(buf, idx, pos, limit, typeID, cData = buf[pos:pos + cLen] pos = pos + cLen data = bDataToNumpyArray(cData, dataTypeName, 1) - print(" Value : {0} ({1} bytes)".format( - data[0], cLen)) + print(" Value : {0}" + " ({1} bytes)".format(data[0], cLen)) else: # attribute value characteristics are different dataTypeSize = bp4dbg_utils.GetTypeSize(typeID) nBytes = int(nElems * dataTypeSize)