From ad3177323c2013e4de8f509c2db260db3144d0ad Mon Sep 17 00:00:00 2001 From: Aryan Roy <50577809+aryan26roy@users.noreply.github.com> Date: Sun, 6 Nov 2022 14:19:36 +0530 Subject: [PATCH] refactor: final refactoring for Forth generation (#749) * rename GenHelper to ForthLevelStash and add docstring with better explanation * add error handling for Python class generation * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * change check procedure for Forth generation * fix identation in generated Python code * add forth generation cancellation * add forth generation cancellation to streamer generated classes * Rename @aryan26roy comments to 'AwkwardForth testing'. * make forth the default * fix some tests * fix more tests * chore: remove references to Identifier (once known as Identities). * Also remove 'uproot' parameters from Forms. * Carefully merged #710 and #749. * This PR requires Awkward 2.0.0rc2. * No TVector2. * Structured the generated code in a more readable way. * change parameters * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix bug * fix awkward form * add streamer name * fix type * fix typo * fix awkward form * fix code generation bug * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * linting * remove ._v2 Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: jpivarski --- src/uproot/_awkward_forth.py | 27 +- src/uproot/containers.py | 244 +++++---- src/uproot/deserialization.py | 5 +- src/uproot/interpretation/objects.py | 50 +- src/uproot/model.py | 42 +- src/uproot/models/TArray.py | 1 + src/uproot/models/TAtt.py | 1 + src/uproot/models/TBasket.py | 1 + src/uproot/models/TBranch.py | 1 + src/uproot/models/TClonesArray.py | 1 + src/uproot/models/TDatime.py | 12 +- src/uproot/models/TGraph.py | 1 + src/uproot/models/TH.py | 1 + src/uproot/models/THashList.py | 1 + src/uproot/models/TLeaf.py | 1 + src/uproot/models/TList.py | 1 + src/uproot/models/TMatrixT.py | 1 + src/uproot/models/TNamed.py | 1 + src/uproot/models/TObjArray.py | 1 + src/uproot/models/TObjString.py | 1 + src/uproot/models/TObject.py | 12 +- src/uproot/models/TRef.py | 26 +- src/uproot/models/TString.py | 18 +- src/uproot/models/TTable.py | 1 + src/uproot/models/TTree.py | 1 + src/uproot/streamers.py | 462 +++++++++++------- tests/test_0034-generic-objects-in-ttrees.py | 2 +- .../test_0053-parents-should-not-be-bases.py | 14 +- tests/test_0442-regular-TClonesArray.py | 2 +- 29 files changed, 525 insertions(+), 407 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index 9ffff7434..a8d0aae53 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -60,6 +60,10 @@ def replace_form_and_model(self, form, model): return temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form def get_code_recursive(self, node): + pre, post, init, header = self.tree_walk(node) + return pre, post, init, header + + def tree_walk(self, node): if "content" in node.keys(): if node["content"] is None: return ( @@ -69,14 +73,14 @@ def get_code_recursive(self, node): node["header_code"], ) else: - pre, post, init, header = self.get_code_recursive(node["content"]) + pre, post, init, header = self.tree_walk(node["content"]) pre2 = "".join(node["pre_code"]) pre2 = pre2 + pre post2 = "".join(node["post_code"]) post2 = post2 + post init = node["init_code"] + init header = node["header_code"] + header - return pre2, post2, init, header + return pre2 + post2, "", init, header elif self.var_set: return "", "", "", "" @@ -273,21 +277,28 @@ def add_to_init(self, code): self.final_init.append(code) -class GenHelper: +def forth_stash(context): + """ + Returns a ForthLevelStash object if ForthGeneration is to be done, else None. + """ + if hasattr(context.get("forth"), "gen"): + return ForthLevelStash(context["forth"].gen) + else: + return None + + +class ForthLevelStash: """ - Helper class to aid Forth code generation within one read/read_members function call. + Helper class to stash code at one level of Forth code generation. Keeps the code generation clean and maintains order for the code snippets. """ def __init__(self, context): - self.forth_present = False self._pre_code = [] self._post_code = [] self._header = "" self._init = "" self._form_key = [] - if hasattr(context.get("forth"), "gen"): - self.forth_present = True - self._gen_obj = context["forth"].gen + self._gen_obj = context def is_forth(self): return self.forth_present diff --git a/src/uproot/containers.py b/src/uproot/containers.py index 06461e895..3301df1d2 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -48,23 +48,23 @@ def _content_cache_key(content): def _read_nested( model, length, chunk, cursor, context, file, selffile, parent, header=True ): - helper_obj = uproot._awkward_forth.GenHelper(context) + forth_stash = uproot._awkward_forth.forth_stash(context) - if helper_obj.is_forth(): - forth_obj = helper_obj.get_gen_obj() + if forth_stash is not None: + forth_obj = forth_stash.get_gen_obj() if isinstance(model, numpy.dtype): symbol = uproot._awkward_forth.symbol_dict.get(model) - if symbol is None and helper_obj.is_forth(): + if symbol is None and forth_stash is not None: raise TypeError("Cannot be awkward") - if helper_obj.is_forth(): + if forth_stash is not None: key = forth_obj.get_keys(1) form_key = f"node{key}-data" - helper_obj.add_to_header( + forth_stash.add_to_header( f"output node{key}-data {uproot._awkward_forth.convert_dtype(symbol)}\n" ) - helper_obj.add_to_pre(f"stream #!{symbol}-> node{key}-data\n") + forth_stash.add_to_pre(f"stream #!{symbol}-> node{key}-data\n") if forth_obj.should_add_form(): forth_obj.add_form_key(form_key) forth_obj.add_form( @@ -76,7 +76,7 @@ def _read_nested( ) temp = forth_obj.add_node( f"node{key}", - helper_obj.get_attrs(), + forth_stash.get_attrs(), "i64", 1, None, @@ -87,26 +87,26 @@ def _read_nested( else: values = numpy.empty(length, dtype=_stl_object_type) if isinstance(model, AsContainer): - if helper_obj.is_forth(): + if forth_stash is not None: if length == 0: forth_obj.var_set = True temp_count = context["forth"].gen.count_obj for i in range(length): - if helper_obj.is_forth(): + if forth_stash is not None: context["forth"].gen.count_obj = temp_count values[i] = model.read( chunk, cursor, context, file, selffile, parent, header=header ) else: - if helper_obj.is_forth(): + if forth_stash is not None: temp_count = context["forth"].gen.count_obj for i in range(length): - if helper_obj.is_forth(): + if forth_stash is not None: if "temp_ref" in context.keys(): context["forth"].gen.go_to(context["temp_ref"]) context["forth"].gen.count_obj = temp_count values[i] = model.read(chunk, cursor, context, file, selffile, parent) - if helper_obj.is_forth(): + if forth_stash is not None: if "temp_ref" in context.keys(): del context["temp_ref"] return values @@ -340,8 +340,8 @@ def awkward_form(self, file, context): ) def read(self, chunk, cursor, context, file, selffile, parent, header=True): - # @aryan26roy: test_0637's tests aren't expected to enter here - + # AwkwardForth testing: test_0637's tests aren't expected to enter here + context["cancel_forth"] = True classname = cursor.string(chunk, context) cursor.skip(1) cls = file.class_named(classname) @@ -465,13 +465,13 @@ def awkward_form(self, file, context): ) def read(self, chunk, cursor, context, file, selffile, parent, header=True): - # @aryan26roy: test_0637's 00,03,25,27,30,33,35,36,38,39,45,47,51,56,57,58,60,61,63,65,68,70,71,72,73,74,75,78,79 + # AwkwardForth testing: test_0637's 00,03,25,27,30,33,35,36,38,39,45,47,51,56,57,58,60,61,63,65,68,70,71,72,73,74,75,78,79 - helper_obj = uproot._awkward_forth.GenHelper(context) + forth_stash = uproot._awkward_forth.forth_stash(context) - if helper_obj.is_forth(): + if forth_stash is not None: # raise NotImplementedError - forth_obj = helper_obj.get_gen_obj() + forth_obj = forth_stash.get_gen_obj() keys = forth_obj.get_keys(2) offsets_num = keys[0] data_num = keys[1] @@ -482,22 +482,22 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): instance_version, is_memberwise, ) = uproot.deserialization.numbytes_version(chunk, cursor, context) - if helper_obj.is_forth(): + if forth_stash is not None: temp_jump = cursor._index - start_cursor._index if temp_jump != 0: - helper_obj.add_to_pre(f"{temp_jump} stream skip\n") + forth_stash.add_to_pre(f"{temp_jump} stream skip\n") if self._length_bytes == "1-5": out = cursor.string(chunk, context) - if helper_obj.is_forth(): - helper_obj.add_to_pre( + if forth_stash is not None: + forth_stash.add_to_pre( f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup node{offsets_num}-offsets +<- stack stream #!B-> node{data_num}-data\n" ) elif self._length_bytes == "4": length = cursor.field(chunk, _stl_container_size, context) out = cursor.string_with_length(chunk, context, length) - if helper_obj.is_forth(): - helper_obj.add_to_pre( + if forth_stash is not None: + forth_stash.add_to_pre( f"stream !I-> stack dup node{offsets_num}-offsets +<- stack stream #B-> node{data_num}-data\n" ) else: @@ -514,7 +514,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): file.file_path, ) - if helper_obj.is_forth(): + if forth_stash is not None: if forth_obj.should_add_form(): temp_aform = f'{{"class": "ListOffsetArray", "offsets": "i64", "content": {{"class": "NumpyArray", "primitive": "uint8", "inner_shape": [], "parameters": {{"__array__": "char"}}, "form_key": "node{data_num}"}}, "parameters": {{"__array__": "string"}}, "form_key": "node{offsets_num}"}}' forth_obj.add_form(json.loads(temp_aform)) @@ -525,13 +525,13 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): ] for elem in form_keys: forth_obj.add_form_key(elem) - helper_obj.add_to_header( + forth_stash.add_to_header( f"output node{offsets_num}-offsets int64\noutput node{data_num}-data uint8\n" ) - helper_obj.add_to_init(f"0 node{offsets_num}-offsets <- stack\n") + forth_stash.add_to_init(f"0 node{offsets_num}-offsets <- stack\n") temp_form = forth_obj.add_node( f"node{offsets_num}", - helper_obj.get_attrs(), + forth_stash.get_attrs(), "i64", 0, None, @@ -605,7 +605,7 @@ def awkward_form(self, file, context): raise uproot.interpretation.objects.CannotBeAwkward("arbitrary pointer") def read(self, chunk, cursor, context, file, selffile, parent, header=True): - # @aryan26roy: test_0637's 29,45,46,49,50 (Awkward Form discovered at read-time) + # AwkwardForth testing: test_0637's 29,45,46,49,50 (Awkward Form discovered at read-time) return uproot.deserialization.read_object_any( chunk, cursor, context, file, selffile, parent @@ -694,12 +694,12 @@ def awkward_form(self, file, context): return awkward.forms.ListOffsetForm(context["index_format"], values_form) def read(self, chunk, cursor, context, file, selffile, parent, header=True): - # @aryan26roy: test_0637's 01,02,23,24,25,26,27,28,30,51,52 + # AwkwardForth testing: test_0637's 01,02,23,24,25,26,27,28,30,51,52 - helper_obj = uproot._awkward_forth.GenHelper(context) + forth_stash = uproot._awkward_forth.forth_stash(context) - if helper_obj.is_forth(): - forth_obj = helper_obj.get_gen_obj() + if forth_stash is not None: + forth_obj = forth_stash.get_gen_obj() offsets_num = forth_obj.get_keys(1) if self._header and header: @@ -709,10 +709,10 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): instance_version, is_memberwise, ) = uproot.deserialization.numbytes_version(chunk, cursor, context) - if helper_obj.is_forth(): + if forth_stash is not None: temp_jump = cursor._index - start_cursor._index if temp_jump != 0: - helper_obj.add_to_pre(f"{temp_jump} stream skip\n") + forth_stash.add_to_pre(f"{temp_jump} stream skip\n") if is_memberwise: raise NotImplementedError( """memberwise serialization of {} @@ -728,21 +728,21 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): return remainder.view(self._values).reshape(-1, *self.inner_shape) else: - if helper_obj.is_forth(): - helper_obj.add_to_header( + if forth_stash is not None: + forth_stash.add_to_header( f"output node{offsets_num}-offsets int64\n" ) form_key = f"node{offsets_num}-offsets" - helper_obj.add_to_init(f"0 node{offsets_num}-offsets <- stack\n") - helper_obj.add_to_pre( + forth_stash.add_to_init(f"0 node{offsets_num}-offsets <- stack\n") + forth_stash.add_to_pre( "0 bytestops I-> stack \nbegin\ndup stream pos <>\nwhile\nswap 1 + swap\n" ) if len(self.inner_shape) > 0: - helper_obj.add_to_post( + forth_stash.add_to_post( f"repeat\nswap {self.inner_shape[0]} / node{offsets_num}-offsets +<- stack drop\n" ) else: - helper_obj.add_to_post( + forth_stash.add_to_post( f"repeat\nswap node{offsets_num}-offsets +<- stack drop\n" ) if forth_obj.should_add_form(): @@ -755,26 +755,26 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): forth_obj.add_form(json.loads(temp_aform)) temp = forth_obj.add_node( f"node{offsets_num}", - helper_obj.get_attrs(), + forth_stash.get_attrs(), "i64", 1, {}, ) - if cursor.index >= chunk.stop and helper_obj.is_forth(): + if cursor.index >= chunk.stop and forth_stash is not None: forth_obj.var_set = True out = [] - if helper_obj.is_forth(): + if forth_stash is not None: temp_count = forth_obj.count_obj while cursor.displacement(start_cursor) < num_bytes: - if helper_obj.is_forth(): + if forth_stash is not None: forth_obj.count_obj = temp_count out.append( self._values.read( chunk, cursor, context, file, selffile, parent ) ) - if helper_obj.is_forth(): + if forth_stash is not None: forth_obj.go_to(temp["content"]) if self._header and header: @@ -792,8 +792,8 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): else: if self._speedbump: - if helper_obj.is_forth(): - helper_obj.add_to_pre("1 stream skip\n") + if forth_stash is not None: + forth_stash.add_to_pre("1 stream skip\n") cursor.skip(1) if isinstance(self._values, numpy.dtype): @@ -802,21 +802,21 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): else: - if helper_obj.is_forth(): - helper_obj.add_to_header( + if forth_stash is not None: + forth_stash.add_to_header( f"output node{offsets_num}-offsets int64\n" ) form_key = f"node{offsets_num}-offsets" - helper_obj.add_to_init(f"0 node{offsets_num}-offsets <- stack\n") - helper_obj.add_to_pre( + forth_stash.add_to_init(f"0 node{offsets_num}-offsets <- stack\n") + forth_stash.add_to_pre( "0 bytestops I-> stack \nbegin\ndup stream pos <>\nwhile\nswap 1 + swap\n" ) if len(self.inner_shape) > 0: - helper_obj.add_to_post( + forth_stash.add_to_post( f"repeat\nswap {self.inner_shape[0]} / node{offsets_num}-offsets +<- stack drop\n" ) else: - helper_obj.add_to_post( + forth_stash.add_to_post( f"repeat\nswap node{offsets_num}-offsets +<- stack drop\n" ) if forth_obj.should_add_form(): @@ -829,29 +829,29 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): forth_obj.add_form(json.loads(temp_aform)) temp = forth_obj.add_node( f"node{offsets_num}", - helper_obj.get_attrs(), + forth_stash.get_attrs(), "i64", 1, {}, ) - if cursor.index >= chunk.stop and helper_obj.is_forth(): + if cursor.index >= chunk.stop and forth_stash is not None: forth_obj.var_set = True out = [] - if helper_obj.is_forth(): + if forth_stash is not None: temp_count = forth_obj.count_obj while cursor.index < chunk.stop: - if helper_obj.is_forth(): + if forth_stash is not None: forth_obj.count_obj = temp_count out.append( self._values.read( chunk, cursor, context, file, selffile, parent ) ) - if helper_obj.is_forth(): + if forth_stash is not None: forth_obj.go_to(temp) - # if helper_obj.is_forth(): + # if forth_stash is not None: # forth_obj.go_to(temp) return uproot._util.objectarray1d(out).reshape(-1, *self.inner_shape) @@ -910,8 +910,8 @@ def awkward_form(self, file, context): ) def read(self, chunk, cursor, context, file, selffile, parent, header=True): - # @aryan26roy: test_0637's (none! untested! but it's just like AsVector) - + # AwkwardForth testing: test_0637's (none! untested! but it's just like AsVector) + context["cancel_forth"] = True if self._header and header: start_cursor = cursor.copy() ( @@ -1056,10 +1056,10 @@ def awkward_form(self, file, context): ) def read(self, chunk, cursor, context, file, selffile, parent, header=True): - # @aryan26roy: test_0637's 00,03,04,06,07,08,09,10,11,12,13,14,15,16,17,23,24,26,27,28,31,33,36,38,41,42,43,44,45,46,49,50,55,56,57,58,59,60,61,62,63,67,68,72,73,76,77,80 - helper_obj = uproot._awkward_forth.GenHelper(context) - if helper_obj.is_forth(): - forth_obj = helper_obj.get_gen_obj() + # AwkwardForth testing: test_0637's 00,03,04,06,07,08,09,10,11,12,13,14,15,16,17,23,24,26,27,28,31,33,36,38,41,42,43,44,45,46,49,50,55,56,57,58,59,60,61,62,63,67,68,72,73,76,77,80 + forth_stash = uproot._awkward_forth.forth_stash(context) + if forth_stash is not None: + forth_obj = forth_stash.get_gen_obj() if self._header and header: start_cursor = cursor.copy() @@ -1068,10 +1068,10 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): instance_version, is_memberwise, ) = uproot.deserialization.numbytes_version(chunk, cursor, context) - if helper_obj.is_forth(): + if forth_stash is not None: temp_jump = cursor._index - start_cursor._index if temp_jump != 0: - helper_obj.add_to_pre(f"{temp_jump} stream skip\n") + forth_stash.add_to_pre(f"{temp_jump} stream skip\n") else: is_memberwise = False @@ -1080,10 +1080,8 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): _value_typename = _content_typename(self._values) if is_memberwise: - if helper_obj.is_forth(): - raise NotImplementedError( - "Forth Reader does not support memberwise serialization." - ) + if forth_stash is not None: + context["cancel_forth"] = True # let's hard-code in logic for std::pair for now if not _value_typename.startswith("pair"): raise NotImplementedError( @@ -1131,39 +1129,39 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): ) else: length = cursor.field(chunk, _stl_container_size, context) - if helper_obj.is_forth(): + if forth_stash is not None: key = forth_obj.get_keys(1) form_key = f"node{key}-offsets" - helper_obj.add_to_header(f"output node{key}-offsets int64\n") - helper_obj.add_to_init(f"0 node{key}-offsets <- stack\n") - helper_obj.add_to_pre( + forth_stash.add_to_header(f"output node{key}-offsets int64\n") + forth_stash.add_to_init(f"0 node{key}-offsets <- stack\n") + forth_stash.add_to_pre( f"stream !I-> stack\n dup node{key}-offsets +<- stack\n" ) - # helper_obj.add_to_post("loop\n") + # forth_stash.add_to_post("loop\n") if forth_obj.should_add_form(): forth_obj.add_form_key(form_key) temp_aform = f'{{ "class":"ListOffsetArray", "offsets":"i64", "content": "NULL", "parameters": {{}}, "form_key": "node{key}"}}' forth_obj.add_form(json.loads(temp_aform)) if not isinstance(self._values, numpy.dtype): - helper_obj.add_to_pre("0 do\n") - helper_obj.add_to_post("loop\n") + forth_stash.add_to_pre("0 do\n") + forth_stash.add_to_post("loop\n") temp = forth_obj.add_node( f"node{key}", - helper_obj.get_attrs(), + forth_stash.get_attrs(), "i64", 1, {}, ) context["temp_ref"] = temp - if length == 0 and helper_obj.is_forth(): + if length == 0 and forth_stash is not None: forth_obj.var_set = True values = _read_nested( self._values, length, chunk, cursor, context, file, selffile, parent ) - if helper_obj.is_forth(): + if forth_stash is not None and not context["cancel_forth"]: forth_obj.go_to(temp) @@ -1256,12 +1254,12 @@ def awkward_form(self, file, context): ) def read(self, chunk, cursor, context, file, selffile, parent, header=True): - # @aryan26roy: test_0637's 62,63,64,65,69,70,74,75,77 + # AwkwardForth testing: test_0637's 62,63,64,65,69,70,74,75,77 - helper_obj = uproot._awkward_forth.GenHelper(context) + forth_stash = uproot._awkward_forth.forth_stash(context) - if helper_obj.is_forth(): - forth_obj = helper_obj.get_gen_obj() + if forth_stash is not None: + forth_obj = forth_stash.get_gen_obj() if self._header and header: start_cursor = cursor.copy() @@ -1270,10 +1268,10 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): instance_version, is_memberwise, ) = uproot.deserialization.numbytes_version(chunk, cursor, context) - if helper_obj.is_forth(): + if forth_stash is not None: temp_jump = cursor._index - start_cursor._index if temp_jump != 0: - helper_obj.add_to_pre(f"{temp_jump} stream skip\n") + forth_stash.add_to_pre(f"{temp_jump} stream skip\n") else: is_memberwise = False @@ -1286,36 +1284,36 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): ) length = cursor.field(chunk, _stl_container_size, context) - if helper_obj.is_forth(): + if forth_stash is not None: key = forth_obj.get_keys(1) form_key = f"node{key}-offsets" - helper_obj.add_to_header(f"output node{key}-offsets int64\n") - helper_obj.add_to_init(f"0 node{key}-offsets <- stack\n") - helper_obj.add_to_pre( + forth_stash.add_to_header(f"output node{key}-offsets int64\n") + forth_stash.add_to_init(f"0 node{key}-offsets <- stack\n") + forth_stash.add_to_pre( f"stream !I-> stack\ndup node{key}-offsets +<- stack\n" ) - # helper_obj.add_to_post("loop\n") + # forth_stash.add_to_post("loop\n") if forth_obj.should_add_form(): forth_obj.add_form_key(form_key) temp_aform = f'{{ "class":"ListOffsetArray", "offsets":"i64", "content": "NULL", "parameters": {{"__array__": "set"}}, "form_key": "node{key}"}}' forth_obj.add_form(json.loads(temp_aform)) if not isinstance(self._keys, numpy.dtype): - helper_obj.add_to_pre("0 do\n") - helper_obj.add_to_post("loop\n") + forth_stash.add_to_pre("0 do\n") + forth_stash.add_to_post("loop\n") temp = forth_obj.add_node( f"node{key}", - helper_obj.get_attrs(), + forth_stash.get_attrs(), "i64", 1, {}, ) - if length == 0 and helper_obj.is_forth(): + if length == 0 and forth_stash is not None: forth_obj.var_set = True keys = _read_nested( self._keys, length, chunk, cursor, context, file, selffile, parent ) - if helper_obj.is_forth(): + if forth_stash is not None: forth_obj.go_to(temp) out = STLSet(keys) @@ -1441,16 +1439,16 @@ def awkward_form(self, file, context): uproot._util.awkward_form(self._values, file, context), ), None, + parameters={"__array__": "sorted_map"}, ), - parameters={"__array__": "sorted_map"}, ) def read(self, chunk, cursor, context, file, selffile, parent, header=True): - # @aryan26roy: test_0637's 00,33,35,39,47,48,66,67,68,69,70,71,72,73,74,75,76,77,78,79 - helper_obj = uproot._awkward_forth.GenHelper(context) + # AwkwardForth testing: test_0637's 00,33,35,39,47,48,66,67,68,69,70,71,72,73,74,75,76,77,78,79 + forth_stash = uproot._awkward_forth.forth_stash(context) - if helper_obj.is_forth(): - forth_obj = helper_obj.get_gen_obj() + if forth_stash is not None: + forth_obj = forth_stash.get_gen_obj() if self._header and header: start_cursor = cursor.copy() ( @@ -1464,28 +1462,28 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if is_memberwise: if self._header and header: cursor.skip(6) - if helper_obj.is_forth(): + if forth_stash is not None: temp_jump = cursor._index - start_cursor._index - helper_obj.add_to_pre(f"{temp_jump} stream skip\n") + forth_stash.add_to_pre(f"{temp_jump} stream skip\n") length = cursor.field(chunk, _stl_container_size, context) - if helper_obj.is_forth(): + if forth_stash is not None: key = forth_obj.get_keys(1) form_key = f"node{key}-offsets" - helper_obj.add_to_header(f"output node{key}-offsets int64\n") - helper_obj.add_to_init(f"0 node{key}-offsets <- stack\n") - helper_obj.add_to_pre( + forth_stash.add_to_header(f"output node{key}-offsets int64\n") + forth_stash.add_to_init(f"0 node{key}-offsets <- stack\n") + forth_stash.add_to_pre( f"stream !I-> stack\n dup node{key}-offsets +<- stack\n" ) if _has_nested_header(self._keys) and header: - if helper_obj.is_forth(): - helper_obj.add_to_pre("6 stream skip\n") + if forth_stash is not None: + forth_stash.add_to_pre("6 stream skip\n") cursor.skip(6) - if helper_obj.is_forth(): + if forth_stash is not None: if not isinstance(self._keys, numpy.dtype): - helper_obj.add_to_pre("dup 0 do\n") + forth_stash.add_to_pre("dup 0 do\n") else: - helper_obj.add_to_pre("dup\n") - if helper_obj.is_forth(): + forth_stash.add_to_pre("dup\n") + if forth_stash is not None: temp = {"name": "TOP", "content": {}} ( temp_node, @@ -1506,7 +1504,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): parent, header=False, ) - if helper_obj.is_forth(): + if forth_stash is not None: temp = {"name": "TOP", "content": {}} keys_form = forth_obj.top_form keys_model = forth_obj.top_node @@ -1518,14 +1516,14 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): temp_prev_form1, ) = forth_obj.replace_form_and_model(None, temp) context["temp_ref"] = temp - if helper_obj.is_forth(): + if forth_stash is not None: if not isinstance(self._keys, numpy.dtype): keys_model["content"]["post_code"].append("loop\n") if _has_nested_header(self._values) and header: cursor.skip(6) - if helper_obj.is_forth(): + if forth_stash is not None: keys_model["content"]["post_code"].append("6 stream skip\n") - if helper_obj.is_forth(): + if forth_stash is not None: if not isinstance(self._values, numpy.dtype): keys_model["content"]["post_code"].append("0 do\n") values = _read_nested( @@ -1539,7 +1537,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): parent, header=False, ) - if helper_obj.is_forth(): + if forth_stash is not None: values_form = forth_obj.top_form values_model = forth_obj.top_node if not isinstance(self._values, numpy.dtype): @@ -1564,7 +1562,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): forth_obj.add_form(aform) temp = forth_obj.add_node( "nodeMap", - helper_obj.get_attrs(), + forth_stash.get_attrs(), "i64", 1, [keys_model["content"], values_model["content"]], @@ -1585,7 +1583,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): return out else: - if helper_obj.is_forth(): + if forth_stash is not None: raise NotImplementedError( """non-memberwise serialization of {} in file {}""".format( diff --git a/src/uproot/deserialization.py b/src/uproot/deserialization.py index 042efae48..dcc525c0b 100644 --- a/src/uproot/deserialization.py +++ b/src/uproot/deserialization.py @@ -68,7 +68,10 @@ def c(name, version=None): new_scope["c"] = c - _actually_compile(class_code, new_scope) + try: + _actually_compile(class_code, new_scope) + except SyntaxError as err: + raise SyntaxError(class_code + "\n\n" + str(err)) from err out = new_scope[class_name] out.class_code = class_code diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index 92c3f8d08..95fe9edec 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -48,7 +48,7 @@ class AsObjects(uproot.interpretation.Interpretation): def __init__(self, model, branch=None): self._model = model self._branch = branch - self._forth = False + self._forth = True self._forth_lock = threading.Lock() self._forth_form_keys = None self._complete_forth_code = None @@ -244,7 +244,7 @@ def basket_array_forth( def _discover_forth(self, data, byte_offsets, branch, context, cursor_offset): output = numpy.empty(len(byte_offsets) - 1, dtype=numpy.dtype(object)) - + context["cancel_forth"] = False for i in range(len(byte_offsets) - 1): byte_start = byte_offsets[i] byte_stop = byte_offsets[i + 1] @@ -253,7 +253,8 @@ def _discover_forth(self, data, byte_offsets, branch, context, cursor_offset): cursor = uproot.source.cursor.Cursor( 0, origin=-(byte_start + cursor_offset) ) - context["forth"].gen.var_set = False + if "forth" in context.keys(): + context["forth"].gen.var_set = False output[i] = self._model.read( chunk, cursor, @@ -262,26 +263,29 @@ def _discover_forth(self, data, byte_offsets, branch, context, cursor_offset): branch.file.detached, branch, ) - context["forth"].gen.awkward_model = context["forth"].gen.top_node - if not context["forth"].gen.var_set: - context["forth"].prereaddone = True - self._assemble_forth( - context["forth"].gen, context["forth"].gen.top_node["content"] - ) - self._complete_forth_code = f"""input stream -input byteoffsets -input bytestops -{"".join(context["forth"].gen.final_header)} -{"".join(context["forth"].gen.final_init)} -0 do -byteoffsets I-> stack -stream seek -{"".join(context["forth"].gen.final_code)} -loop -""" - self._forth_form_keys = tuple(context["forth"].gen.form_keys) - self._form = context["forth"].gen.top_form - return None # we should re-read all the data with Forth + if context["cancel_forth"] and "forth" in context.keys(): + del context["forth"] + if "forth" in context.keys(): + context["forth"].gen.awkward_model = context["forth"].gen.top_node + if not context["forth"].gen.var_set: + context["forth"].prereaddone = True + self._assemble_forth( + context["forth"].gen, context["forth"].gen.top_node["content"] + ) + self._complete_forth_code = f"""input stream + input byteoffsets + input bytestops + {"".join(context["forth"].gen.final_header)} + {"".join(context["forth"].gen.final_init)} + 0 do + byteoffsets I-> stack + stream seek + {"".join(context["forth"].gen.final_code)} + loop + """ + self._forth_form_keys = tuple(context["forth"].gen.form_keys) + self._form = context["forth"].gen.top_form + return None # we should re-read all the data with Forth return output # Forth-generation was unsuccessful: this is Python output def _assemble_forth(self, forth_obj, awkward_model): diff --git a/src/uproot/model.py b/src/uproot/model.py index 90dfa5a5d..5c4a644d5 100644 --- a/src/uproot/model.py +++ b/src/uproot/model.py @@ -796,8 +796,8 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): context["breadcrumbs"] = old_breadcrumbs + (self,) self.hook_before_read(chunk=chunk, cursor=cursor, context=context, file=file) - helper_obj = uproot._awkward_forth.GenHelper(context) - if helper_obj.is_forth(): + forth_stash = uproot._awkward_forth.forth_stash(context) + if forth_stash is not None: forth_obj = context["forth"].gen if context.get("reading", True): @@ -805,7 +805,8 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): self.read_numbytes_version(chunk, cursor, context) length = cursor._index - temp_index if length != 0: - helper_obj.add_to_pre(f"{length} stream skip\n") + if forth_stash is not None: + forth_stash.add_to_pre(f"{length} stream skip\n") if ( issubclass(cls, VersionedModel) and self._instance_version != classname_version(cls.__name__) @@ -815,10 +816,10 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): if classname_version(correct_cls.__name__) != classname_version( cls.__name__ ): - if helper_obj.is_forth(): + if forth_stash is not None: forth_obj.add_node( "pass", - helper_obj.get_attrs(), + forth_stash.get_attrs(), "i64", 1, {}, @@ -834,30 +835,31 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): parent, concrete=concrete, ) - # if helper_obj.is_forth(): + # if forth_stash is not None: # forth_obj.go_to(temp) return temp_var if context.get("in_TBranch", False): - # @aryan26roy: test_0637's 01,02,05,08,09,11,12,13,15,16,29,35,38,39,44,45,46,47,49,50,52,56 + # AwkwardForth testing: test_0637's 01,02,05,08,09,11,12,13,15,16,29,35,38,39,44,45,46,47,49,50,52,56 if self._num_bytes is None and self._instance_version != self.class_version: self._instance_version = None cursor = self._cursor - if helper_obj.is_forth(): - helper_obj._pre_code.pop(-1) + if forth_stash is not None and not context["cancel_forth"]: + forth_stash._pre_code.pop(-1) elif self._instance_version == 0: - helper_obj.add_to_pre("4 stream skip\n") + if forth_stash is not None: + forth_stash.add_to_pre("4 stream skip\n") cursor.skip(4) if context.get("reading", True): self.hook_before_read_members( chunk=chunk, cursor=cursor, context=context, file=file ) - if helper_obj.is_forth(): + if forth_stash is not None: forth_obj.add_node( "model828", - helper_obj.get_attrs(), + forth_stash.get_attrs(), "i64", 1, {}, @@ -1321,10 +1323,10 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): """ import uproot.deserialization - helper_obj = uproot._awkward_forth.GenHelper(context) + forth_stash = uproot._awkward_forth.forth_stash(context) - if helper_obj.is_forth(): - forth_obj = helper_obj.get_gen_obj() + if forth_stash is not None: + forth_obj = forth_stash.get_gen_obj() # Ignores context["reading"], because otherwise, there would be nothing to do. start_index = cursor._index ( @@ -1335,12 +1337,12 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): versioned_cls = cls.class_of_version(version) bytes_skipped = cursor._index - start_index - if helper_obj.is_forth(): + if forth_stash is not None: # raise NotImplementedError - helper_obj.add_to_pre(f"{bytes_skipped} stream skip \n") + forth_stash.add_to_pre(f"{bytes_skipped} stream skip \n") forth_obj.add_node( "Model1319", - helper_obj.get_attrs(), + forth_stash.get_attrs(), "i64", 1, {}, @@ -1366,7 +1368,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): ) # versioned_cls.read starts with numbytes_version again because move=False (above) - # if helper_obj.is_forth(): + # if forth_stash is not None: temp_var = cls.postprocess( versioned_cls.read( chunk, cursor, context, file, selffile, parent, concrete=concrete @@ -1376,7 +1378,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): context, file, ) - # if helper_obj.is_forth(): + # if forth_stash is not None: # if "no_go_to" not in context.keys(): # raise NotImplementedError # forth_obj.go_to(temp_node) diff --git a/src/uproot/models/TArray.py b/src/uproot/models/TArray.py index 422f03125..3f0e3b360 100644 --- a/src/uproot/models/TArray.py +++ b/src/uproot/models/TArray.py @@ -26,6 +26,7 @@ def read_numbytes_version(self, chunk, cursor, context): pass def read_members(self, chunk, cursor, context, file): + context["cancel_forth"] = True if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} diff --git a/src/uproot/models/TAtt.py b/src/uproot/models/TAtt.py index ce1e58cd0..c54a79c8d 100644 --- a/src/uproot/models/TAtt.py +++ b/src/uproot/models/TAtt.py @@ -20,6 +20,7 @@ class Model_TAttLine_v1(uproot.model.VersionedModel): """ def read_members(self, chunk, cursor, context, file): + context["cancel_forth"] = True if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} diff --git a/src/uproot/models/TBasket.py b/src/uproot/models/TBasket.py index 9cc1d9d3f..134ac3028 100644 --- a/src/uproot/models/TBasket.py +++ b/src/uproot/models/TBasket.py @@ -210,6 +210,7 @@ def read_numbytes_version(self, chunk, cursor, context): pass def read_members(self, chunk, cursor, context, file): + context["cancel_forth"] = True assert isinstance(self._parent, uproot.behaviors.TBranch.TBranch) self._basket_num = context.get("basket_num") diff --git a/src/uproot/models/TBranch.py b/src/uproot/models/TBranch.py index 2f667ecec..dbedc7303 100644 --- a/src/uproot/models/TBranch.py +++ b/src/uproot/models/TBranch.py @@ -42,6 +42,7 @@ class Model_TBranch_v10(uproot.behaviors.TBranch.TBranch, uproot.model.Versioned behaviors = (uproot.behaviors.TBranch.TBranch,) def read_members(self, chunk, cursor, context, file): + context["cancel_forth"] = True if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} diff --git a/src/uproot/models/TClonesArray.py b/src/uproot/models/TClonesArray.py index 6da21d572..123ba4ff7 100644 --- a/src/uproot/models/TClonesArray.py +++ b/src/uproot/models/TClonesArray.py @@ -18,6 +18,7 @@ class Model_TClonesArray(uproot.model.VersionedModel, Sequence): """ def read_members(self, chunk, cursor, context, file): + context["cancel_forth"] = True if self.is_memberwise: raise NotImplementedError( "memberwise serialization of {}\nin file {}".format( diff --git a/src/uproot/models/TDatime.py b/src/uproot/models/TDatime.py index 9a8f7e576..c137d02cc 100644 --- a/src/uproot/models/TDatime.py +++ b/src/uproot/models/TDatime.py @@ -25,13 +25,13 @@ def read_numbytes_version(self, chunk, cursor, context): pass def read_members(self, chunk, cursor, context, file): - helper_obj = uproot._awkward_forth.GenHelper(context) - if helper_obj.is_forth(): - forth_obj = helper_obj.get_gen_obj() + forth_stash = uproot._awkward_forth.forth_stash(context) + if forth_stash is not None: + forth_obj = forth_stash.get_gen_obj() key = forth_obj.get_keys(1) form_key = f"node{key}-data" - helper_obj.add_to_header(f"output node{key}-data int32\n") - helper_obj.add_to_pre(f"stream !I-> node{key}-data\n") + forth_stash.add_to_header(f"output node{key}-data int32\n") + forth_stash.add_to_pre(f"stream !I-> node{key}-data\n") form_key = f"node{key}-data" if forth_obj.should_add_form(): forth_obj.add_form_key(form_key) @@ -49,7 +49,7 @@ def read_members(self, chunk, cursor, context, file): forth_obj.add_form(temp_aform) temp_form = forth_obj.add_node( f"node{key}", - helper_obj.get_attrs(), + forth_stash.get_attrs(), "i64", 0, None, diff --git a/src/uproot/models/TGraph.py b/src/uproot/models/TGraph.py index 9c554ef82..98cbbbb65 100644 --- a/src/uproot/models/TGraph.py +++ b/src/uproot/models/TGraph.py @@ -35,6 +35,7 @@ class Model_TGraph_v4(uproot.behaviors.TGraph.TGraph, uproot.model.VersionedMode behaviors = (uproot.behaviors.TGraph.TGraph,) def read_members(self, chunk, cursor, context, file): + context["cancel_forth"] = True if self.is_memberwise: raise NotImplementedError( "memberwise serialization of {}\nin file {}".format( diff --git a/src/uproot/models/TH.py b/src/uproot/models/TH.py index 8ba0fd21d..7cdcd7bcc 100644 --- a/src/uproot/models/TH.py +++ b/src/uproot/models/TH.py @@ -150,6 +150,7 @@ class Model_TAxis_v10(uproot.behaviors.TAxis.TAxis, uproot.model.VersionedModel) """ def read_members(self, chunk, cursor, context, file): + context["cancel_forth"] = True if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} diff --git a/src/uproot/models/THashList.py b/src/uproot/models/THashList.py index 1262e6559..929c1140c 100644 --- a/src/uproot/models/THashList.py +++ b/src/uproot/models/THashList.py @@ -17,6 +17,7 @@ def read_numbytes_version(self, chunk, cursor, context): pass def read_members(self, chunk, cursor, context, file): + context["cancel_forth"] = True if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} diff --git a/src/uproot/models/TLeaf.py b/src/uproot/models/TLeaf.py index f976ebfbe..e5d2e06e7 100644 --- a/src/uproot/models/TLeaf.py +++ b/src/uproot/models/TLeaf.py @@ -25,6 +25,7 @@ class Model_TLeaf_v2(uproot.model.VersionedModel): """ def read_members(self, chunk, cursor, context, file): + context["cancel_forth"] = True if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} diff --git a/src/uproot/models/TList.py b/src/uproot/models/TList.py index 769893a19..bb8bb8357 100644 --- a/src/uproot/models/TList.py +++ b/src/uproot/models/TList.py @@ -19,6 +19,7 @@ class Model_TList(uproot.model.Model, Sequence): """ def read_members(self, chunk, cursor, context, file): + context["cancel_forth"] = True if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} diff --git a/src/uproot/models/TMatrixT.py b/src/uproot/models/TMatrixT.py index 64d983429..a17b1b258 100644 --- a/src/uproot/models/TMatrixT.py +++ b/src/uproot/models/TMatrixT.py @@ -22,6 +22,7 @@ class Model_TMatrixTSym_3c_double_3e__v5(uproot.model.VersionedModel): """ def read_members(self, chunk, cursor, context, file): + context["cancel_forth"] = True if self.is_memberwise: raise NotImplementedError( "memberwise serialization of {}\nin file {}".format( diff --git a/src/uproot/models/TNamed.py b/src/uproot/models/TNamed.py index 98f427afb..40355f13a 100644 --- a/src/uproot/models/TNamed.py +++ b/src/uproot/models/TNamed.py @@ -16,6 +16,7 @@ class Model_TNamed(uproot.model.Model): """ def read_members(self, chunk, cursor, context, file): + context["cancel_forth"] = True if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} diff --git a/src/uproot/models/TObjArray.py b/src/uproot/models/TObjArray.py index 3b2a1aa4b..e31078aac 100644 --- a/src/uproot/models/TObjArray.py +++ b/src/uproot/models/TObjArray.py @@ -28,6 +28,7 @@ class Model_TObjArray(uproot.model.Model, Sequence): """ def read_members(self, chunk, cursor, context, file): + context["cancel_forth"] = True if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} diff --git a/src/uproot/models/TObjString.py b/src/uproot/models/TObjString.py index 63bc06364..ef4ba2452 100644 --- a/src/uproot/models/TObjString.py +++ b/src/uproot/models/TObjString.py @@ -27,6 +27,7 @@ class Model_TObjString(uproot.model.Model, str): writable = True def read_members(self, chunk, cursor, context, file): + context["cancel_forth"] = True if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} diff --git a/src/uproot/models/TObject.py b/src/uproot/models/TObject.py index be9b6a66b..34d281bba 100644 --- a/src/uproot/models/TObject.py +++ b/src/uproot/models/TObject.py @@ -25,10 +25,10 @@ def read_numbytes_version(self, chunk, cursor, context): pass def read_members(self, chunk, cursor, context, file): - helper_obj = uproot._awkward_forth.GenHelper(context) + forth_stash = uproot._awkward_forth.forth_stash(context) start_index = cursor._index - if helper_obj.is_forth(): - forth_obj = helper_obj.get_gen_obj() + if forth_stash is not None: + forth_obj = forth_stash.get_gen_obj() # raise NotImplementedError if self.is_memberwise: raise NotImplementedError( @@ -49,15 +49,15 @@ def read_members(self, chunk, cursor, context, file): if self._members["@fBits"] & uproot.const.kIsReferenced: cursor.skip(2) self._members["@fBits"] = int(self._members["@fBits"]) - if helper_obj.is_forth(): + if forth_stash is not None: skip_length = cursor._index - start_index - helper_obj.add_to_pre(f"{skip_length} stream skip \n") + forth_stash.add_to_pre(f"{skip_length} stream skip \n") if forth_obj.should_add_form(): temp_aform = '{"class": "RecordArray", "contents":[], "parameters": {"__record__": "TObject"}}' forth_obj.add_form(json.loads(temp_aform)) forth_obj.add_node( "TObjext", - helper_obj.get_attrs(), + forth_stash.get_attrs(), "i64", 0, {}, diff --git a/src/uproot/models/TRef.py b/src/uproot/models/TRef.py index 34ab8cfb6..1b6fa36ed 100644 --- a/src/uproot/models/TRef.py +++ b/src/uproot/models/TRef.py @@ -120,10 +120,10 @@ def name(self): return self._members["fName"] def read_members(self, chunk, cursor, context, file): - helper_obj = uproot._awkward_forth.GenHelper(context) - if helper_obj.is_forth(): + forth_stash = uproot._awkward_forth.forth_stash(context) + if forth_stash is not None: awkward = uproot.extras.awkward() # noqa:F841 - forth_obj = helper_obj.get_gen_obj() + forth_obj = forth_stash.get_gen_obj() # raise NotImplementedError if self.is_memberwise: raise NotImplementedError( @@ -132,18 +132,18 @@ def read_members(self, chunk, cursor, context, file): type(self).__name__, self.file.file_path ) ) - if helper_obj.is_forth(): + if forth_stash is not None: form_keys = forth_obj.get_keys(6) - helper_obj.add_to_pre("10 stream skip\n") - helper_obj.add_to_pre( + forth_stash.add_to_pre("10 stream skip\n") + forth_stash.add_to_pre( f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup node{form_keys[1]}-offsets +<- stack stream #!B-> node{form_keys[2]}-data\n" ) - helper_obj.add_to_pre( + forth_stash.add_to_pre( f"stream !I-> stack dup node{form_keys[3]}-data <- stack\n" ) - helper_obj.add_to_pre("6 stream skip\n") - helper_obj.add_to_pre( + forth_stash.add_to_pre("6 stream skip\n") + forth_stash.add_to_pre( f"dup node{form_keys[4]}-offsets +<- stack stream #!I-> node{form_keys[5]}-data\n" ) keys = [ @@ -156,17 +156,17 @@ def read_members(self, chunk, cursor, context, file): if forth_obj.should_add_form(): for elem in keys: forth_obj.add_form_key(elem) - temp_aform = f'{{"class": "RecordArray", "contents": {{"fname": {{"class": "ListOffsetArray", "offsets": "i64", "content": {{"class": "NumpyArray", "primitive": "uint8", "inner_shape": [], "parameters": {{"__array__": "char"}}, "form_key": "node{form_keys[2]}"}}, "parameters": {{}}, "form_key": "node{form_keys[1]}"}}, "fSize": {{"class": "NumpyArray", "primitive": "int64", "inner_shape": [], "parameters": {{}}, "form_key": "node{form_keys[3]}"}}, "refs": {{"class": "ListOffsetArray", "offsets": "i64", "content": {{"class": "NumpyArray", "primitive": "int64", "inner_shape": [], "parameters": {{}}, "form_key": "node{form_keys[5]}"}}, "parameters": {{}}, "form_key": "node{form_keys[4]}"}}}}, "parameters": {{}}, "form_key": "node{form_keys[0]}"}}' + temp_aform = f'{{"class": "RecordArray", "contents": {{"fName": {{"class": "ListOffsetArray", "offsets": "i64", "content": {{"class": "NumpyArray", "primitive": "uint8", "inner_shape": [], "parameters": {{"__array__": "char"}}, "form_key": "node{form_keys[2]}"}}, "parameters": {{}}, "form_key": "node{form_keys[1]}"}}, "fSize": {{"class": "NumpyArray", "primitive": "int64", "inner_shape": [], "parameters": {{}}, "form_key": "node{form_keys[3]}"}}, "refs": {{"class": "ListOffsetArray", "offsets": "i64", "content": {{"class": "NumpyArray", "primitive": "int64", "inner_shape": [], "parameters": {{}}, "form_key": "node{form_keys[5]}"}}, "parameters": {{}}, "form_key": "node{form_keys[4]}"}}}}, "parameters": {{}}, "form_key": "node{form_keys[0]}"}}' forth_obj.add_form(json.loads(temp_aform)) - helper_obj.add_to_header( + forth_stash.add_to_header( f"output node{form_keys[1]}-offsets int64\noutput node{form_keys[2]}-data uint8\noutput node{form_keys[3]}-data int64\noutput node{form_keys[4]}-offsets int64\noutput node{form_keys[5]}-data int64\n" ) - helper_obj.add_to_init( + forth_stash.add_to_init( f"0 node{form_keys[1]}-offsets <- stack\n0 node{form_keys[4]}-offsets <- stack\n" ) forth_obj.add_node( f"node{form_keys[0]}", - helper_obj.get_attrs(), + forth_stash.get_attrs(), "i64", 1, None, diff --git a/src/uproot/models/TString.py b/src/uproot/models/TString.py index 77033d4a9..320ffdcca 100644 --- a/src/uproot/models/TString.py +++ b/src/uproot/models/TString.py @@ -20,9 +20,9 @@ def read_numbytes_version(self, chunk, cursor, context): pass def read_members(self, chunk, cursor, context, file): - helper_obj = uproot._awkward_forth.GenHelper(context) - if helper_obj.is_forth(): - forth_obj = helper_obj.get_gen_obj() + forth_stash = uproot._awkward_forth.forth_stash(context) + if forth_stash is not None: + forth_obj = forth_stash.get_gen_obj() keys = forth_obj.get_keys(2) offsets_num = keys[0] data_num = keys[1] @@ -33,9 +33,9 @@ def read_members(self, chunk, cursor, context, file): type(self).__name__, self.file.file_path ) ) - if helper_obj.is_forth(): - helper_obj.add_to_pre( - f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup node{offsets_num}-offsets +<- stack stream #!B-> node{data_num}-data\n" + if forth_stash is not None: + forth_stash.add_to_pre( + f" stream !B-> stack dup 255 = if drop stream !I-> stack then dup node{offsets_num}-offsets +<- stack stream #!B-> node{data_num}-data\n" ) if forth_obj.should_add_form(): temp_aform = { @@ -59,13 +59,13 @@ def read_members(self, chunk, cursor, context, file): ] for elem in form_keys: forth_obj.add_form_key(elem) - helper_obj.add_to_header( + forth_stash.add_to_header( f"output node{offsets_num}-offsets int64\noutput node{data_num}-data uint8\n" ) - helper_obj.add_to_init(f"0 node{offsets_num}-offsets <- stack\n") + forth_stash.add_to_init(f"0 node{offsets_num}-offsets <- stack\n") temp_form = forth_obj.add_node( f"node{offsets_num}", - helper_obj.get_attrs(), + forth_stash.get_attrs(), "i64", 0, None, diff --git a/src/uproot/models/TTable.py b/src/uproot/models/TTable.py index e87ec63f8..ddd48a695 100644 --- a/src/uproot/models/TTable.py +++ b/src/uproot/models/TTable.py @@ -25,6 +25,7 @@ class Model_TDataSet(uproot.model.Model): """ def read_members(self, chunk, cursor, context, file): + context["cancel_forth"] = True if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} diff --git a/src/uproot/models/TTree.py b/src/uproot/models/TTree.py index ed33cf31a..0926415a1 100644 --- a/src/uproot/models/TTree.py +++ b/src/uproot/models/TTree.py @@ -46,6 +46,7 @@ class Model_TTree_v16(uproot.behaviors.TTree.TTree, uproot.model.VersionedModel) behaviors = (uproot.behaviors.TTree.TTree,) def read_members(self, chunk, cursor, context, file): + context["cancel_forth"] = True if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index 80b230199..9609ff707 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -220,9 +220,9 @@ def class_code(self): ] read_members.append( """ - helper_obj = uproot._awkward_forth.GenHelper(context) - if helper_obj.is_forth(): - forth_obj = helper_obj.get_gen_obj() + forth_stash = uproot._awkward_forth.forth_stash(context) + if forth_stash is not None: + forth_obj = forth_stash.get_gen_obj() content = {}""" ) read_member_n = [ @@ -277,8 +277,13 @@ def class_code(self): member_names, class_flags, ) - read_members.append( - " if helper_obj.is_forth():\n if forth_obj.should_add_form():\n forth_obj.add_form({'class': 'RecordArray', 'contents': content, 'parameters': {'record': 'TVector2'}}, len(content))\n temp = forth_obj.add_node('dynamic',helper_obj.get_attrs(),\"i64\",0,None)\n" + read_members.extend( + [ + " if forth_stash is not None:", + " if forth_obj.should_add_form():", + f" forth_obj.add_form({{'class': 'RecordArray', 'contents': content, 'parameters': {{'__record__': {self.name!r}}}}}, len(content))", + " temp = forth_obj.add_node('dynamic', forth_stash.get_attrs(), \"i64\", 0, None)", + ] ) if len(read_members) == 1: # untested as of PR #629 @@ -409,6 +414,7 @@ def read_members(self, chunk, cursor, context, file): concrete=self.concrete, ) ) + context["cancel_forth"] = True self._bases[0]._members["fName"] = _canonical_typename( self._bases[0]._members["fName"] ) @@ -528,7 +534,7 @@ def read_members(self, chunk, cursor, context, file): concrete=self.concrete, ) ) - + context["cancel_forth"] = True ( self._members["fType"], self._members["fSize"], @@ -627,10 +633,16 @@ def class_code( read_member_n.append(f" if member_index == {i}:") # untested as of PR #629 - read_members.append( - f" raise uproot.deserialization.DeserializationError('not implemented: class members defined by {type(self).__name__} of type {self.typename} in member {self.name} of class {streamerinfo.name}', chunk, cursor, context, file.file_path)" + read_members.extend( + [ + ' context["cancel_forth"] = True', + f" raise uproot.deserialization.DeserializationError('not implemented: class members defined by {type(self).__name__} of type {self.typename} in member {self.name} of class {streamerinfo.name}', chunk, cursor, context, file.file_path)", + ] + ) + + read_member_n.append( + f" raise uproot.deserialization.DeserializationError('not implemented: class members defined by {type(self).__name__} of type {self.typename} in member {self.name} of class {streamerinfo.name}', chunk, cursor, context, file.file_path)" ) - read_member_n.append(" " + read_members[-1]) strided_interpretation.append( f" raise uproot.interpretation.objects.CannotBeStrided('not implemented: class members defined by {type(self).__name__} of type {self.typename} in member {self.name} of class {streamerinfo.name}')" @@ -694,28 +706,46 @@ def class_code( ): read_member_n.append(f" if member_index == {i}:") - # @aryan26roy: test_0637's 01,02,08,09,11,12,13,15,16,29,38,45,46,49,50 - read_members.append( - " if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form = forth_obj.replace_form_and_model(None, {'name': 'TOP', 'content': {}})\n" - ) - read_members.append( - f" self._bases.append(c({self.name!r}, {self.base_version!r}).read(chunk, cursor, context, file, self._file, self._parent, concrete=self.concrete))" - ) - read_member_n.append(" " + read_members[-1]) - read_members.append( - " if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj.top_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj.top_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.prev_form = temp_prev_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" + # AwkwardForth testing: test_0637's 01,02,08,09,11,12,13,15,16,29,38,45,46,49,50 + read_members.extend( + [ + " if forth_stash is not None:", + " temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form = forth_obj.replace_form_and_model(None, {'name': 'TOP', 'content': {}})", + f" self._bases.append(c({self.name!r}, {self.base_version!r}).read(chunk, cursor, context, file, self._file, self._parent, concrete=self.concrete))", + " if forth_stash is not None and not context['cancel_forth']:", + " temp_prev_form1 = forth_obj.prev_form", + " temp_form1 = forth_obj.top_form", + " temp_model1 = forth_obj.top_node", + " temp_model_ref = forth_obj.awkward_model", + " forth_obj.awkward_model = temp_node", + " forth_obj.top_node = temp_node_top", + " forth_obj.aform = temp_form", + " forth_obj.prev_form = temp_prev_form", + " forth_obj.top_form = temp_form_top", + " temp_model1 = temp_model1['content']", + " forth_obj.add_node_whole(temp_model1, temp_model_ref)", + " content.update(temp_form1['contents'])", + " forth_obj.enable_adding()", + ] ) + + ### FIXME: what is this commented-out code for? # read_members.append( - # " if helper_obj.is_forth():\n temp_form = forth_obj.get_temp_form_top()\n content.update(temp_form['contents'])\n forth_obj.set_dummy_none(temp_top_dummy, temp_dummy, temp_top_flag)\n" + # " if forth_stash is not None:\n temp_form = forth_obj.get_temp_form_top()\n content.update(temp_form['contents'])\n forth_obj.set_dummy_none(temp_top_dummy, temp_dummy, temp_top_flag)\n" # ) + + read_member_n.append( + f" self._bases.append(c({self.name!r}, {self.base_version!r}).read(chunk, cursor, context, file, self._file, self._parent, concrete=self.concrete))" + ) + strided_interpretation.append( f" members.extend(file.class_named({self.name!r}, {self.base_version!r}).strided_interpretation(file, header, tobject_header, breadcrumbs).members)" ) - awkward_form.append( - f" tmp_awkward_form = file.class_named({self.name!r}, {self.base_version!r}).awkward_form(file, context)" - ) - awkward_form.append( - " contents.update(zip(tmp_awkward_form.fields, tmp_awkward_form.contents))" + awkward_form.extend( + [ + f" tmp_awkward_form = file.class_named({self.name!r}, {self.base_version!r}).awkward_form(file, context)", + " contents.update(zip(tmp_awkward_form.fields, tmp_awkward_form.contents))", + ] ) base_names_versions.append((self.name, self.base_version)) @@ -798,68 +828,63 @@ def class_code( read_member_n.append(f" if member_index == {i}:") read_members.append(f" tmp = self._dtype{len(dtypes)}") - read_member_n.append(" " + read_members[-1]) + read_member_n.append(f" tmp = self._dtype{len(dtypes)}") if streamerinfo.name == "TBranch" and self.name == "fBasketSeek": # untested as of PR #629 - read_members.append( - """ - if context.get('speedbump', True): - if cursor.bytes(chunk, 1, context)[0] == 2: - tmp = numpy.dtype('>i8') -""".strip( - "\n" - ) + read_members.extend( + [ + " if context.get('speedbump', True):", + " if cursor.bytes(chunk, 1, context)[0] == 2:", + " tmp = numpy.dtype('>i8')", + ] ) - read_member_n.append(" " + read_members[-1].replace("\n", "\n ")) - - else: - read_members.append( - """ - if context.get('speedbump', True): - cursor.skip(1) - if helper_obj.is_forth(): - helper_obj.add_to_pre('1 stream skip \\n') -""".strip( - "\n" - ) + read_member_n.extend( + [ + " if context.get('speedbump', True):", + " if cursor.bytes(chunk, 1, context)[0] == 2:", + " tmp = numpy.dtype('>i8')", + ] ) - read_member_n.append(" " + read_members[-1].replace("\n", "\n ")) - - # @aryan26roy: test_0637's 29,44,56 - read_members.append(" if helper_obj.is_forth():") - read_members.append(" key = forth_obj.get_keys(1)") - read_members.append(" key2 = forth_obj.get_keys(1)") - read_members.append(' form_key = f"node{key}-data"') - read_members.append(' form_key2 = f"node{key2}-offsets"') - read_members.append( - f' helper_obj.add_to_header(f"output node{{key}}-data {{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}\\n")' - ) - read_members.append( - ' helper_obj.add_to_header(f"output node{key2}-offsets int64\\n")' - ) - read_members.append( - ' helper_obj.add_to_init(f"0 node{key2}-offsets <- stack\\n")' - ) - read_members.append( - f' content[{self.name!r}] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' - ) - read_members.append( - f' helper_obj.add_to_pre(f" var_{self.count_name} @ dup node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> node{{key}}-data\\n")' - ) - read_members.append(" if forth_obj.should_add_form():") - read_members.append( - " forth_obj.add_form_key(form_key)" + else: + # AwkwardForth testing: test_0637's 29,44,56 + read_members.extend( + [ + " if context.get('speedbump', True):", + " cursor.skip(1)", + " if forth_stash is not None:", + " forth_stash.add_to_pre('1 stream skip \\n')", + " if forth_stash is not None:", + " key = forth_obj.get_keys(1)", + " key2 = forth_obj.get_keys(1)", + ' form_key = f"node{key}-data"', + ' form_key2 = f"node{key2}-offsets"', + f' forth_stash.add_to_header(f"output node{{key}}-data {{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}\\n")', + ' forth_stash.add_to_header(f"output node{key2}-offsets int64\\n")', + ' forth_stash.add_to_init(f"0 node{key2}-offsets <- stack\\n")', + f' content[{self.name!r}] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}', + f' forth_stash.add_to_pre(f" var_{self.count_name} @ dup node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> node{{key}}-data\\n")', + " if forth_obj.should_add_form():", + " forth_obj.add_form_key(form_key)", + " forth_obj.add_form_key(form_key2)", + ] ) - read_members.append( - " forth_obj.add_form_key(form_key2)" + read_member_n.extend( + [ + " if context.get('speedbump', True):", + " cursor.skip(1)", + " if forth_stash is not None:", + " forth_stash.add_to_pre('1 stream skip \\n')", + ] ) read_members.append( f" self._members[{self.name!r}] = cursor.array(chunk, self.member({self.count_name!r}), tmp, context);\n" ) - read_member_n.append(" " + read_members[-1]) + read_member_n.append( + f" self._members[{self.name!r}] = cursor.array(chunk, self.member({self.count_name!r}), tmp, context);\n" + ) strided_interpretation.append( f" raise uproot.interpretation.objects.CannotBeStrided('class members defined by {type(self).__name__} of type {self.typename} in member {self.name} of class {streamerinfo.name}')" @@ -925,20 +950,29 @@ def class_code( read_member_n.append(f" if member_index == {i}:") if self.typename == "Double32_t": # untested as of PR #629 - read_members.append( - f" self._members[{self.name!r}] = cursor.double32(chunk, context)" + read_members.extend( + [ + ' context["cancel_forth"] = True', + f" self._members[{self.name!r}] = cursor.double32(chunk, context)", + ] + ) + read_member_n.append( + f" self._members[{self.name!r}] = cursor.double32(chunk, context)" ) - read_member_n.append(" " + read_members[-1]) elif self.typename == "Float16_t": # untested as of PR #629 - read_members.append( - f" self._members[{self.name!r}] = cursor.float16(chunk, 12, context)" + read_members.extend( + [ + ' context["cancel_forth"] = True', + f" self._members[{self.name!r}] = cursor.float16(chunk, 12, context)", + ] + ) + read_member_n.append( + f" self._members[{self.name!r}] = cursor.float16(chunk, 12, context)" ) - read_member_n.append(" " + read_members[-1]) elif self.array_length == 0: - # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# if ( i == 0 or not isinstance(elements[i - 1], Model_TStreamerBasicType) @@ -957,64 +991,58 @@ def class_code( or elements[i + 1].array_length != 0 ): if len(fields[-1]) == 1: - # @aryan26roy: test_0637's 01,02,29,38,44,56 - read_members.append(" if helper_obj.is_forth():") - read_members.append(" key = forth_obj.get_keys(1)") - read_members.append(' form_key = f"node{key}-data"') - read_members.append( - f' helper_obj.add_to_header(f"output node{{key}}-data {uproot._awkward_forth.convert_dtype(formats[-1][0])}\\n")' - ) - read_members.append( - f' content[{fields[-1][0]!r}] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[-1][0])}", "inner_shape": [], "parameters": {{}}, "form_key": f"node{{key}}"}}' + # AwkwardForth testing: test_0637's 01,02,29,38,44,56 + read_members.extend( + [ + " if forth_stash is not None:", + " key = forth_obj.get_keys(1)", + ' form_key = f"node{key}-data"', + f' forth_stash.add_to_header(f"output node{{key}}-data {uproot._awkward_forth.convert_dtype(formats[-1][0])}\\n")', + f' content[{fields[-1][0]!r}] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[-1][0])}", "inner_shape": [], "parameters": {{}}, "form_key": f"node{{key}}"}}', + ] ) if fields[-1][0] in COUNT_NAMES: - read_members.append( - f' helper_obj.add_to_init(f"variable var_{fields[-1][0]}\\n")' - ) - read_members.append( - f' helper_obj.add_to_pre(f"stream !{formats[-1][0]}-> stack dup var_{fields[-1][0]} ! node{{key}}-data <- stack\\n")' + read_members.extend( + [ + f' forth_stash.add_to_init(f"variable var_{fields[-1][0]}\\n")', + f' forth_stash.add_to_pre(f"stream !{formats[-1][0]}-> stack dup var_{fields[-1][0]} ! node{{key}}-data <- stack\\n")', + ] ) else: read_members.append( - f' helper_obj.add_to_pre(f"stream !{formats[-1][0]}-> node{{key}}-data\\n")' + f' forth_stash.add_to_pre(f"stream !{formats[-1][0]}-> node{{key}}-data\\n")' ) - read_members.append( - " if forth_obj.should_add_form():" - ) - read_members.append( - " forth_obj.add_form_key(form_key)" - ) - read_members.append( - f" self._members[{fields[-1][0]!r}] = cursor.field(chunk, self._format{len(formats) - 1}, context)" + read_members.extend( + [ + " if forth_obj.should_add_form():", + " forth_obj.add_form_key(form_key)", + f" self._members[{fields[-1][0]!r}] = cursor.field(chunk, self._format{len(formats) - 1}, context)", + ] ) else: - read_members.append(" if helper_obj.is_forth():") + read_members.append(" if forth_stash is not None:") for i in range(len(formats[0])): - read_members.append(" key = forth_obj.get_keys(1)") - read_members.append(' form_key = f"node{key}-data"') - read_members.append( - f' helper_obj.add_to_header(f"output node{{key}}-data {uproot._awkward_forth.convert_dtype(formats[0][i])}\\n")' - ) - # read_members.append(' helper_obj.add_to_init(f"0 node{key}-offsets <- stack\\n")') - read_members.append( - f' content[{fields[0][i]!r}] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[0][i])}", "inner_shape": [], "parameters": {{}}, "form_key": f"node{{key}}"}}' - ) - read_members.append( - f' helper_obj.add_to_pre(f"stream !{formats[0][i]}-> node{{key}}-data\\n")' - ) - read_members.append( - " if forth_obj.should_add_form():" - ) - read_members.append( - " forth_obj.add_form_key(form_key)" + read_members.extend( + [ + " key = forth_obj.get_keys(1)", + ' form_key = f"node{key}-data"', + f' forth_stash.add_to_header(f"output node{{key}}-data {uproot._awkward_forth.convert_dtype(formats[0][i])}\\n")', + ### FIXME: what is this commented-out code? + # ' forth_stash.add_to_init(f"0 node{key}-offsets <- stack\\n")', + f' content[{fields[0][i]!r}] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[0][i])}", "inner_shape": [], "parameters": {{}}, "form_key": f"node{{key}}"}}', + f' forth_stash.add_to_pre(f"stream !{formats[0][i]}-> node{{key}}-data\\n")', + " if forth_obj.should_add_form():", + " forth_obj.add_form_key(form_key)", + ] ) + assign_members = ", ".join( f"self._members[{x!r}]" for x in fields[-1] ) - # @aryan26roy: test_0637's 01,02,05,08,09,11,12,13,15,16,29,35,39,45,46,47,49,50,56 + # AwkwardForth testing: test_0637's 01,02,05,08,09,11,12,13,15,16,29,35,39,45,46,47,49,50,56 read_members.append( f"\n {assign_members} = cursor.fields(chunk, self._format{len(formats) - 1}, context)" @@ -1025,40 +1053,46 @@ def class_code( ) else: - # @aryan26roy: test_0637's 44,56 - read_members.append(" if helper_obj.is_forth():") - read_members.append(" key = forth_obj.get_keys(1)") - read_members.append(" key2 = forth_obj.get_keys(1)") - read_members.append(' form_key = f"node{key}-data"') - read_members.append(' form_key2 = f"node{key2}-offsets"') - read_members.append( - f' helper_obj.add_to_header(f"output node{{key}}-data {{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}\\n")' - ) - read_members.append( - ' helper_obj.add_to_header(f"output node{key2}-offsets int64\\n")' - ) - read_members.append( - ' helper_obj.add_to_init(f"0 node{key2}-offsets <- stack\\n")' - ) - read_members.append( - f' content[{self.name!r}] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' - ) - read_members.append( - f' helper_obj.add_to_pre(f"{self.array_length} dup node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> node{{key}}-data\\n")\n' - ) - read_members.append(" if forth_obj.should_add_form():") - read_members.append( - " forth_obj.add_form_key(form_key)" - ) - read_members.append( - " forth_obj.add_form_key(form_key2)" - ) - read_members.append( - f" self._members[{self.name!r}] = cursor.array(chunk, {self.array_length}, self._dtype{len(dtypes)}, context)" + # AwkwardForth testing: test_0637's 44,56 + read_members.extend( + [ + " if forth_stash is not None:", + " key = forth_obj.get_keys(1)", + " key2 = forth_obj.get_keys(1)", + ' form_key = f"node{key}-data"', + ' form_key2 = f"node{key2}-offsets"', + f' forth_stash.add_to_header(f"output node{{key}}-data {{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}\\n")', + ' forth_stash.add_to_header(f"output node{key2}-offsets int64\\n")', + ' forth_stash.add_to_init(f"0 node{key2}-offsets <- stack\\n")', + f' content[{self.name!r}] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}', + f' forth_stash.add_to_pre(f"{self.array_length} dup node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> node{{key}}-data\\n")\n', + " if forth_obj.should_add_form():", + " forth_obj.add_form_key(form_key)", + " forth_obj.add_form_key(form_key2)", + f" self._members[{self.name!r}] = cursor.array(chunk, {self.array_length}, self._dtype{len(dtypes)}, context)", + ] ) + dtypes.append(_ftype_to_dtype(self.fType)) - read_member_n.append(" " + read_members[-1]) + read_member_n.extend( + [ + " if forth_stash is not None:", + " key = forth_obj.get_keys(1)", + " key2 = forth_obj.get_keys(1)", + ' form_key = f"node{key}-data"', + ' form_key2 = f"node{key2}-offsets"', + f' forth_stash.add_to_header(f"output node{{key}}-data {{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}\\n")', + ' forth_stash.add_to_header(f"output node{key2}-offsets int64\\n")', + ' forth_stash.add_to_init(f"0 node{key2}-offsets <- stack\\n")', + f' content[{self.name!r}] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}', + f' forth_stash.add_to_pre(f"{self.array_length} dup node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> node{{key}}-data\\n")\n', + " if forth_obj.should_add_form():", + " forth_obj.add_form_key(form_key)", + " forth_obj.add_form_key(form_key2)", + f" self._members[{self.name!r}] = cursor.array(chunk, {self.array_length}, self._dtype{len(dtypes)}, context)", + ] + ) if self.array_length == 0 and self.typename not in ("Double32_t", "Float16_t"): strided_interpretation.append( @@ -1210,23 +1244,24 @@ def class_code( class_flags, ): # untested as of PR #629 - read_members.append(" cursor.skip(6)") - read_members.append( - f" for tmp in range(self.member({self.count_name!r})):" - ) - read_members.append( - f" self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)" + read_members.extend( + [ + ' context["cancel_forth"] = True', + " cursor.skip(6)", + f" for tmp in range(self.member({self.count_name!r})):", + f" self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)", + ] ) strided_interpretation.append( f" raise uproot.interpretation.objects.CannotBeStrided('class members defined by {type(self).__name__} of type {self.typename} in member {self.name} of class {streamerinfo.name}')" ) - awkward_form.append( - f" tmp = file.class_named({self.typename.rstrip('*')!r}, 'max').awkward_form(file, context)" - ) - awkward_form.append( - f" contents[{self.name!r}] = ListOffsetForm(context['index_format'], tmp)" + awkward_form.extend( + [ + f" tmp = file.class_named({self.typename.rstrip('*')!r}, 'max').awkward_form(file, context)", + f" contents[{self.name!r}] = ListOffsetForm(context['index_format'], tmp)", + ] ) member_names.append(self.name) @@ -1315,17 +1350,36 @@ def class_code( string_header=True, ) - # @aryan26roy: test_0637's 35,38,39,44,45,47,50,56 - read_members.append( - " if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form = forth_obj.replace_form_and_model(None, {'name': 'TOP', 'content': {}})\n" - ) - read_members.append( - f" self._members[{self.name!r}] = self._stl_container{len(containers)}.read(chunk, cursor, context, file, self._file, self.concrete)" + # AwkwardForth testing: test_0637's 35,38,39,44,45,47,50,56 + read_members.extend( + [ + " if forth_stash is not None:", + " temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form = forth_obj.replace_form_and_model(None, {'name': 'TOP', 'content': {}})", + f" self._members[{self.name!r}] = self._stl_container{len(containers)}.read(chunk, cursor, context, file, self._file, self.concrete)", + " if forth_stash is not None:", + " temp_prev_form1 = forth_obj.prev_form", + " temp_form1 = forth_obj.top_form", + " temp_model1 = forth_obj.top_node", + " temp_model_ref = forth_obj.awkward_model", + " forth_obj.awkward_model = temp_node", + " forth_obj.prev_form = temp_prev_form", + " forth_obj.top_node = temp_node_top", + " forth_obj.aform = temp_form", + " forth_obj.top_form = temp_form_top", + " temp_model1 = temp_model1['content']", + f" content[{self.name!r}] = temp_form1", + " pre,post,init,header = forth_obj.get_code_recursive(temp_model1)", + " forth_stash.add_to_header(header)", + " forth_stash.add_to_pre(pre)", + " forth_stash.add_to_post(post)", + " forth_stash.add_to_init(init)", + ] ) - read_member_n.append(" " + read_members[-1]) - read_members.append( - f" if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj.top_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj.prev_form = temp_prev_form\n forth_obj.top_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n content[{self.name!r}] = temp_form1\n pre,post,init,header = forth_obj.get_code_recursive(temp_model1)\n helper_obj.add_to_header(header)\n helper_obj.add_to_pre(pre)\n helper_obj.add_to_post(post)\n helper_obj.add_to_init(init)" + + read_member_n.append( + f" self._members[{self.name!r}] = self._stl_container{len(containers)}.read(chunk, cursor, context, file, self._file, self.concrete)" ) + strided_interpretation.append( f" members.append(({self.name!r}, cls._stl_container{len(containers)}.strided_interpretation(file, header, tobject_header, breadcrumbs)))" ) @@ -1339,7 +1393,6 @@ def class_code( def read_members(self, chunk, cursor, context, file): start = cursor.index - self._bases.append( Model_TStreamerElement.read( chunk, @@ -1427,27 +1480,36 @@ def class_code( read_member_n.append(f" if member_index == {i}:") if self.fType == uproot.const.kObjectp or self.fType == uproot.const.kAnyp: - # @aryan26roy: test_0637's (none! untested!) + # AwkwardForth testing: test_0637's (none! untested!) - read_members.append( - f" self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)" + read_members.extend( + [ + ' context["cancel_forth"] = True', + f" self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)", + ] + ) + + read_member_n.append( + f" self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)" ) - read_member_n.append(" " + read_members[-1]) strided_interpretation.append( f" members.append(({self.name!r}, file.class_named({self.typename.rstrip('*')!r}, 'max').strided_interpretation(file, header, tobject_header, breadcrumbs)))" ) + awkward_form.append( f" contents[{self.name!r}] = file.class_named({self.typename.rstrip('*')!r}, 'max').awkward_form(file, context)" ) elif self.fType == uproot.const.kObjectP or self.fType == uproot.const.kAnyP: - # @aryan26roy: test_0637's (none! untested!) + # AwkwardForth testing: test_0637's (none! untested!) read_members.append( f" self._members[{self.name!r}] = read_object_any(chunk, cursor, context, file, self._file, self)" ) - read_member_n.append(" " + read_members[-1]) + read_member_n.append( + f" self._members[{self.name!r}] = read_object_any(chunk, cursor, context, file, self._file, self)" + ) strided_interpretation.append( f" raise uproot.interpretation.objects.CannotBeStrided('class members defined by {type(self).__name__} of type {self.typename} in member {self.name} of class {streamerinfo.name}')" ) @@ -1458,7 +1520,9 @@ def class_code( read_members.append( f" raise uproot.deserialization.DeserializationError('not implemented: class members defined by {type(self).__name__} with fType {self.fType}', chunk, cursor, context, file.file_path)" ) - read_member_n.append(" " + read_members[-1]) + read_member_n.append( + f" raise uproot.deserialization.DeserializationError('not implemented: class members defined by {type(self).__name__} with fType {self.fType}', chunk, cursor, context, file.file_path)" + ) member_names.append(self.name) @@ -1552,14 +1616,34 @@ def class_code( ): read_member_n.append(f" if member_index == {i}:") - # @aryan26roy: test_0637's 01,02,29,45,46,49,50,56 - - read_members.append( - f" if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form = forth_obj.replace_form_and_model(None, {{'name': 'TOP', 'content': {{}}}})\n self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)\n if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj.top_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj.prev_form = temp_prev_form\n forth_obj.top_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n content[{self.name!r}] = temp_form1\n pre,post,init,header = forth_obj.get_code_recursive(temp_model1)\n helper_obj.add_to_header(header)\n helper_obj.add_to_pre(pre)\n helper_obj.add_to_post(post)\n helper_obj.add_to_init(init)" + # AwkwardForth testing: test_0637's 01,02,29,45,46,49,50,56 + read_members.extend( + [ + " if forth_stash is not None:", + " temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form = forth_obj.replace_form_and_model(None, {'name': 'TOP', 'content': {}})", + f" self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)", + " if forth_stash is not None:", + " temp_prev_form1 = forth_obj.prev_form", + " temp_form1 = forth_obj.top_form", + " temp_model1 = forth_obj.top_node", + " temp_model_ref = forth_obj.awkward_model", + " forth_obj.awkward_model = temp_node", + " forth_obj.prev_form = temp_prev_form", + " forth_obj.top_node = temp_node_top", + " forth_obj.aform = temp_form", + " forth_obj.top_form = temp_form_top", + " temp_model1 = temp_model1['content']", + f" content[{self.name!r}] = temp_form1", + " pre,post,init,header = forth_obj.get_code_recursive(temp_model1)", + " forth_stash.add_to_header(header)", + " forth_stash.add_to_pre(pre)", + " forth_stash.add_to_post(post)", + " forth_stash.add_to_init(init)", + ] ) + read_member_n.append( - " " - + f" self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)" + f" self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)" ) strided_interpretation.append( diff --git a/tests/test_0034-generic-objects-in-ttrees.py b/tests/test_0034-generic-objects-in-ttrees.py index 620817418..6a93e35b7 100644 --- a/tests/test_0034-generic-objects-in-ttrees.py +++ b/tests/test_0034-generic-objects-in-ttrees.py @@ -1237,7 +1237,7 @@ def test_awkward_map_int_struct(): result = branch.array(library="ak") assert ( str(awkward.type(result)) - == '1 * [var * (int64, struct[{name: string, charge: int64, mass: float64}, parameters={"__record__": "BDSOutputROOTGeant4Data::ParticleInfo"}]), parameters={"__array__": "sorted_map"}]' + == '1 * var * tuple[[int32, struct[{name: string, charge: int32, mass: float64}, parameters={"__record__": "BDSOutputROOTGeant4Data::ParticleInfo"}]], parameters={"__array__": "sorted_map"}]' ) assert awkward.to_list(result[0, "0"]) == [ -1000020040, diff --git a/tests/test_0053-parents-should-not-be-bases.py b/tests/test_0053-parents-should-not-be-bases.py index c9bb866d9..d4e133240 100644 --- a/tests/test_0053-parents-should-not-be-bases.py +++ b/tests/test_0053-parents-should-not-be-bases.py @@ -87,8 +87,8 @@ def test_awkward_TRefArray(): with uproot.open(skhep_testdata.data_path("uproot-issue513.root"))["Delphes"] as t: assert t["GenJet.Particles"].array(entry_stop=1)[0].tolist() == [ { - "fName": "", "fSize": 45, + "fName": "", "refs": [ 2396, 2320, @@ -138,8 +138,8 @@ def test_awkward_TRefArray(): ], }, { - "fName": "", "fSize": 20, + "fName": "", "refs": [ 2370, 1830, @@ -164,8 +164,8 @@ def test_awkward_TRefArray(): ], }, { - "fName": "", "fSize": 21, + "fName": "", "refs": [ 2284, 1883, @@ -191,8 +191,8 @@ def test_awkward_TRefArray(): ], }, { - "fName": "", "fSize": 31, + "fName": "", "refs": [ 1379, 2123, @@ -228,8 +228,8 @@ def test_awkward_TRefArray(): ], }, { - "fName": "", "fSize": 26, + "fName": "", "refs": [ 1989, 1911, @@ -260,8 +260,8 @@ def test_awkward_TRefArray(): ], }, { - "fName": "", "fSize": 32, + "fName": "", "refs": [ 2082, 2445, @@ -298,8 +298,8 @@ def test_awkward_TRefArray(): ], }, { - "fName": "", "fSize": 26, + "fName": "", "refs": [ 2266, 1847, diff --git a/tests/test_0442-regular-TClonesArray.py b/tests/test_0442-regular-TClonesArray.py index 84bd240d0..007bb7ffe 100644 --- a/tests/test_0442-regular-TClonesArray.py +++ b/tests/test_0442-regular-TClonesArray.py @@ -35,4 +35,4 @@ def test_read_delphes_ak(delphes_tree): branch = delphes_tree["GenJet04/GenJet04.Constituents"] array = branch.array(library="ak") - assert set(array.fields) == {"fName", "fSize", "refs"} + assert set(array.fields) == {"fSize", "fName", "refs"}