From 8f5ceb3f69a35387ca3f5dd14e09455d00e9e836 Mon Sep 17 00:00:00 2001 From: Becker Awqatty Date: Fri, 26 Jun 2020 13:02:42 -0400 Subject: [PATCH 1/3] added to `ChannelDataArrayBlock.parseWith` optimized parsing for homogeneous data - removed dead code path in heterogeneous data case --- idelib/parsers.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/idelib/parsers.py b/idelib/parsers.py index 8adb6aba..16f54fcd 100644 --- a/idelib/parsers.py +++ b/idelib/parsers.py @@ -922,12 +922,22 @@ def parseWith(self, parser, start=None, end=None, step=1, subchannel=None): self._parser = parser self._streamDtype = streamDtype + if ( + len(streamDtype) > 0 + and len({streamDtype[i] for i in xrange(len(streamDtype))}) == 1 + # ^ np.dtype isn't iterable :( + ): + rawData = np.frombuffer(self.payload, dtype=streamDtype[0]) + rawData = rawData.reshape(-1, len(streamDtype))[start:end:step].T + + if subchannel is not None: + return rawData[[subchannel]] + return rawData + rawData = np.frombuffer(self.payload, dtype=streamDtype)[start:end:step] # Special cases for single-channel outputs - if len(streamDtype) == 0: - return rawData[np.newaxis] - elif subchannel is not None: + if subchannel is not None: return rawData[streamDtype.names[subchannel]][np.newaxis] data = np.empty((len(streamDtype),) + rawData.shape, From fcb067ba3e0e55db1e5ce2329a14c3866c6b3aef Mon Sep 17 00:00:00 2001 From: Becker Awqatty Date: Fri, 26 Jun 2020 15:12:25 -0400 Subject: [PATCH 2/3] refactored `ChannelDataArrayBlock.parseWith` to cache item data type for homogeneous data arrays --- idelib/parsers.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/idelib/parsers.py b/idelib/parsers.py index 16f54fcd..c8983dd7 100644 --- a/idelib/parsers.py +++ b/idelib/parsers.py @@ -881,6 +881,7 @@ def __init__(self, element): self._parser = None self._streamDtype = None + self._commonDtype = None @property def payload(self): @@ -908,6 +909,7 @@ def parseWith(self, parser, start=None, end=None, step=1, subchannel=None): if parser is self._parser: streamDtype = self._streamDtype + commonDtype = self._commonDtype else: parser_format = parser.format if parser_format[0] in ['<', '>']: @@ -919,18 +921,21 @@ def parseWith(self, parser, start=None, end=None, step=1, subchannel=None): streamDtype = np.dtype(','.join([endian+typeId for typeId in parser_format])) + isHomogeneous = len(set(parser_format)) == 1 + if isHomogeneous: + commonDtype = np.dtype(endian + parser_format[0]) + else: + commonDtype = None + self._parser = parser self._streamDtype = streamDtype + self._commonDtype = commonDtype - if ( - len(streamDtype) > 0 - and len({streamDtype[i] for i in xrange(len(streamDtype))}) == 1 - # ^ np.dtype isn't iterable :( - ): - rawData = np.frombuffer(self.payload, dtype=streamDtype[0]) - rawData = rawData.reshape(-1, len(streamDtype))[start:end:step].T + if commonDtype is not None: + rawData = np.frombuffer(self.payload, dtype=commonDtype) + rawData = rawData.reshape(-1, len(streamDtype) or 1)[start:end:step].T - if subchannel is not None: + if len(streamDtype) > 0 and subchannel is not None: return rawData[[subchannel]] return rawData From 36d0e73c1e8ca2720a6958c1a4b02a5d3db3b466 Mon Sep 17 00:00:00 2001 From: Becker Awqatty Date: Wed, 15 Jul 2020 10:09:20 -0400 Subject: [PATCH 3/3] refactored `ChannelDataArrayBlock.parseWith` - reordered conditional to reduce indentation - removed whitespace lint on `streamDtype` assignment --- idelib/parsers.py | 86 +++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/idelib/parsers.py b/idelib/parsers.py index c8983dd7..19ca2fc0 100644 --- a/idelib/parsers.py +++ b/idelib/parsers.py @@ -905,59 +905,59 @@ def parseWith(self, parser, start=None, end=None, step=1, subchannel=None): # All numpy-compatible data streams use `struct.Struct` parsers isNumpyCompatibleFormat = isinstance(parser, struct.Struct) - if isNumpyCompatibleFormat: - - if parser is self._parser: - streamDtype = self._streamDtype - commonDtype = self._commonDtype + if not isNumpyCompatibleFormat: + # No parser format -> assume that data is from an old .ide file + # & should be handled safely using the parser object + blocks = list(ChannelDataBlock.parseWith( + self, parser, start, end, step, subchannel + )) + return np.array(blocks, dtype=np.float64).T + + if parser is self._parser: + streamDtype = self._streamDtype + commonDtype = self._commonDtype + else: + parser_format = parser.format + if parser_format[0] in ['<', '>']: + endian = parser_format[0] + parser_format = parser_format[1:] else: - parser_format = parser.format - if parser_format[0] in ['<', '>']: - endian = parser_format[0] - parser_format = parser_format[1:] - else: - endian = '>' + endian = '>' - streamDtype = np.dtype(','.join([endian+typeId - for typeId in parser_format])) - - isHomogeneous = len(set(parser_format)) == 1 - if isHomogeneous: - commonDtype = np.dtype(endian + parser_format[0]) - else: - commonDtype = None + streamDtype = np.dtype( + ','.join([endian+typeId for typeId in parser_format]) + ) - self._parser = parser - self._streamDtype = streamDtype - self._commonDtype = commonDtype + isHomogeneous = len(set(parser_format)) == 1 + if isHomogeneous: + commonDtype = np.dtype(endian + parser_format[0]) + else: + commonDtype = None - if commonDtype is not None: - rawData = np.frombuffer(self.payload, dtype=commonDtype) - rawData = rawData.reshape(-1, len(streamDtype) or 1)[start:end:step].T + self._parser = parser + self._streamDtype = streamDtype + self._commonDtype = commonDtype - if len(streamDtype) > 0 and subchannel is not None: - return rawData[[subchannel]] - return rawData + if commonDtype is not None: + rawData = np.frombuffer(self.payload, dtype=commonDtype) + rawData = rawData.reshape(-1, len(streamDtype) or 1)[start:end:step].T - rawData = np.frombuffer(self.payload, dtype=streamDtype)[start:end:step] + if len(streamDtype) > 0 and subchannel is not None: + return rawData[[subchannel]] + return rawData - # Special cases for single-channel outputs - if subchannel is not None: - return rawData[streamDtype.names[subchannel]][np.newaxis] + rawData = np.frombuffer(self.payload, dtype=streamDtype)[start:end:step] - data = np.empty((len(streamDtype),) + rawData.shape, - dtype=np.float64) - for i, chName in enumerate(streamDtype.names): - data[i] = rawData[chName] + # Special cases for single-channel outputs + if subchannel is not None: + return rawData[streamDtype.names[subchannel]][np.newaxis] - return data + data = np.empty((len(streamDtype),) + rawData.shape, + dtype=np.float64) + for i, chName in enumerate(streamDtype.names): + data[i] = rawData[chName] - # No parser format -> assume that data is from an old .ide file type - # & should be handled safely using the parser object - blocks = list(ChannelDataBlock.parseWith( - self, parser, start, end, step, subchannel - )) - return np.array(blocks, dtype=np.float64).T + return data def parseByIndexWith(self, parser, indices, subchannel=None): """ Parse an element's payload and get a specific set of samples. Used