From ac6ba633822b3d001548b5decba117da02151828 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Fri, 1 Jul 2022 12:02:17 -0500 Subject: [PATCH 01/41] added some forth generation --- src/uproot/streamers.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index 1fe9aa84e..8759b6d0f 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -671,17 +671,12 @@ def class_code( member_names, class_flags, ): - read_member_n.append( + read_members.append( """ - forth = False - forth_obj = None - fcode_pre = [] - fcode_post = [] - temp = None - if "forth" in context.keys(): - forth = True - forth_obj = context["forth"] - key = forth_obj.get_key()\n""" + helper_obj = uproot._awkward_forth.GenHelper(context) + if helper_obj.is_forth(): + forth_obj = helper_obj.get_gen_obj() + key = forth_obj.get_keys()\n""" ) read_member_n.append(f" if member_index == {i}:") read_members.append( @@ -1444,6 +1439,12 @@ class Model_TStreamerObject(TStreamerObjectTypes, Model_TStreamerElement): """ def read_members(self, chunk, cursor, context, file): + + helper_obj = uproot._awkward_forth.GenHelper(context) + + if helper_obj.is_forth(): + forth_obj = helper_obj.get_gen_obj() + start = cursor.index self._bases.append( @@ -1457,7 +1458,6 @@ def read_members(self, chunk, cursor, context, file): concrete=self.concrete, ) ) - self._serialization = _copy_bytes(chunk, start, cursor.index, cursor, context) From 60438e9abb6bb22bba0d13e0230636ebd9294075 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Fri, 1 Jul 2022 16:39:22 -0500 Subject: [PATCH 02/41] Added some forth generation to TStreamerBasicType --- src/uproot/_awkward_forth.py | 12 +- src/uproot/containers.py | 1 - src/uproot/interpretation/objects.py | 1 + src/uproot/model.py | 26 ++- src/uproot/models/TObject.py | 1 + src/uproot/streamers.py | 20 ++- .../test_0637-setup-tests-for-AwkwardForth.py | 160 ++++++++++-------- 7 files changed, 136 insertions(+), 85 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index ebb3f9632..2b704189b 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -34,7 +34,6 @@ def add_form(self, aform): if self.aform is None: self.aform = aform self.top_form = self.aform - else: if "content" in self.aform.keys(): if self.aform["content"] == "NULL": @@ -42,6 +41,12 @@ def add_form(self, aform): self.aform = self.aform["content"] else: raise ValueError + elif "contents" in self.aform.keys(): + if len(self.aform["content"]) < self.aform["parameters"]["lencon"]: + self.aform["contents"].append(aform) + self.aform = self.aform["contents"][-1] + else: + raise ValueError def get_keys(self, num_keys): if num_keys == 1: @@ -192,3 +197,8 @@ def add_to_init(self, code): def get_init(self): return self._init + + +def convert_dtype(format): + if format == 'd': + return "float64" diff --git a/src/uproot/containers.py b/src/uproot/containers.py index 6534e3722..43a34c0c3 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -1100,7 +1100,6 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if length == 0 and helper_obj.is_forth(): forth_obj.var_set = True - values = _read_nested( self._values, length, chunk, cursor, context, file, selffile, parent ) diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index eaeea8815..e1348f546 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -257,6 +257,7 @@ def basket_array_forth( ) if not context["forth"].var_set: self._prereaddone = True + print(context["forth"].awkward_model) self.assemble_forth( context["forth"], context["forth"].awkward_model ) diff --git a/src/uproot/model.py b/src/uproot/model.py index d4f57edc1..cd91f438e 100644 --- a/src/uproot/model.py +++ b/src/uproot/model.py @@ -760,7 +760,6 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): self._num_bytes = None self._instance_version = None self._is_memberwise = False - old_breadcrumbs = context.get("breadcrumbs", ()) context["breadcrumbs"] = old_breadcrumbs + (self,) @@ -773,7 +772,8 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): temp_index = cursor._index self.read_numbytes_version(chunk, cursor, context) length = cursor._index - temp_index - helper_obj.add_to_pre(f"{length} stream skip \n") + if length != 0: + helper_obj.add_to_pre(f"{length} stream skip \n") if ( issubclass(cls, VersionedModel) and self._instance_version != classname_version(cls.__name__) @@ -783,9 +783,20 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): if classname_version(correct_cls.__name__) != classname_version( cls.__name__ ): + if helper_obj.is_forth(): + temp = forth_obj.add_node( + "pass", + helper_obj.get_pre(), + helper_obj.get_post(), + helper_obj.get_init(), + helper_obj.get_header(), + "i64", + 1, + {}, + ) cursor.move_to(self._cursor.index) context["breadcrumbs"] = old_breadcrumbs - return correct_cls.read( + temp_var = correct_cls.read( chunk, cursor, context, @@ -794,10 +805,12 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): parent, concrete=concrete, ) + if helper_obj.is_forth(): + forth_obj.go_to(temp) + return temp_var if context.get("in_TBranch", False): # @aryan26roy: test_0637's 01,02,05,08,09,11,12,13,15,16,29,35,38,39,44,45,46,47,49,50,52,56 - if self._num_bytes is None and self._instance_version != self.class_version: self._instance_version = None cursor = self._cursor @@ -824,6 +837,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): self.read_members(chunk, cursor, context, file) if helper_obj.is_forth(): forth_obj.go_to(temp) + raise NotImplementedError self.hook_after_read_members( chunk=chunk, cursor=cursor, context=context, file=file ) @@ -1281,6 +1295,8 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): """ import uproot.deserialization + helper_obj = uproot._awkward_forth.GenHelper(context) + # Ignores context["reading"], because otherwise, there would be nothing to do. ( @@ -1311,6 +1327,8 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): ) # versioned_cls.read starts with numbytes_version again because move=False (above) + if helper_obj.is_forth(): + print(versioned_cls) return cls.postprocess( versioned_cls.read( chunk, cursor, context, file, selffile, parent, concrete=concrete diff --git a/src/uproot/models/TObject.py b/src/uproot/models/TObject.py index de9be1b7a..6e88bedac 100644 --- a/src/uproot/models/TObject.py +++ b/src/uproot/models/TObject.py @@ -24,6 +24,7 @@ def read_numbytes_version(self, chunk, cursor, context): pass def read_members(self, chunk, cursor, context, file): + #raise NotImplementedError if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index b590aeddb..a753f4a2f 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -14,6 +14,8 @@ import uproot +import uproot._awkward_forth + _canonical_typename_patterns = [ (re.compile(r"\bChar_t\b"), "char"), (re.compile(r"\bUChar_t\b"), "unsigned char"), @@ -246,7 +248,8 @@ def class_code(self): base_names_versions = [] member_names = [] class_flags = {} - + for i in range(len(self._members["fElements"])): + print(self._members["fElements"][i]) for i in range(len(self._members["fElements"])): self._members["fElements"][i].class_code( self, @@ -676,7 +679,7 @@ def class_code( helper_obj = uproot._awkward_forth.GenHelper(context) if helper_obj.is_forth(): forth_obj = helper_obj.get_gen_obj() - key = forth_obj.get_keys()\n""" + key = forth_obj.get_keys(1)\n""" ) read_member_n.append(f" if member_index == {i}:") @@ -871,7 +874,6 @@ def class_code( class_flags, ): read_member_n.append(f" if member_index == {i}:") - if self.typename == "Double32_t": # untested as of PR #629 read_members.append( @@ -887,6 +889,7 @@ def class_code( read_member_n.append(" " + read_members[-1]) elif self.array_length == 0: + #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# if ( i == 0 or not isinstance(elements[i - 1], Model_TStreamerBasicType) @@ -898,7 +901,6 @@ def class_code( fields[-1].append(self.name) formats[-1].append(_ftype_to_struct(self.fType)) - formats_memberwise.append(_ftype_to_struct(self.fType)) if ( @@ -906,6 +908,7 @@ def class_code( or not isinstance(elements[i + 1], Model_TStreamerBasicType) or elements[i + 1].array_length != 0 ): + if len(fields[-1]) == 1: # @aryan26roy: test_0637's 01,02,29,38,44,56 @@ -914,6 +917,15 @@ def class_code( ) else: + read_members.append(' if helper_obj.is_forth():') + for elem in formats[0]: + read_members.append(' key = forth_obj.get_keys(1)') + read_members.append(' form_key = f"part0-node{key}-data"') + read_members.append(f' helper_obj.add_to_header(f"output part0-node{{key}}-data {uproot._awkward_forth.convert_dtype(elem)}\\n")') + read_members.append(' helper_obj.add_to_init(f"0 part0-node{key}-offsets <- stack\\n")') + read_members.append(f' helper_obj.add_to_pre("stream !{elem}-> output")') + read_members.append(' if forth_obj.should_add_form():') + read_members.append(' forth_obj.add_form_key(form_key)') assign_members = ", ".join( f"self._members[{x!r}]" for x in fields[-1] ) diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index 514c1c3c4..a1d16bcfb 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -61,7 +61,7 @@ def test_06(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["CollectionTree/AnalysisJetsAuxDyn.NumTrkPt500"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -69,7 +69,7 @@ def test_07(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["CollectionTree/AnalysisJetsAuxDyn.SumPtTrkPt500"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -77,7 +77,7 @@ def test_08(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["CollectionTree/AnalysisJetsAuxDyn.GhostTrack"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == > (version 1) at 0x7fc6a08f2f70>, ...], ...] at 0x7fc6a08f2f10> @@ -87,7 +87,7 @@ def test_09(): "AntiKt10UFOCSSKJetsAuxDyn.GhostVR30Rmax4Rmin02TrackJet_BTagging201903" ] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == > (version 1) at 0x7febbf1b2fa0>], ...] at 0x7febbf1b2f40> @@ -95,7 +95,7 @@ def test_10(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["CollectionTree/CaloCalTopoClustersAuxDyn.e_sampl"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=3) + py = branch.array(interp, library="np", entry_stop=3) # py[-1] == @@ -105,7 +105,7 @@ def test_11(): "TruthBosonsWithDecayVerticesAuxDyn.incomingParticleLinks" ] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == > (version 1) at 0x7f636a9484c0>], ...] at 0x7f636a948eb0> @@ -113,7 +113,7 @@ def test_12(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["CollectionTree/TruthBottomAuxDyn.parentLinks"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == > (version 1) at 0x7fc259ae37c0>], ...] at 0x7fc259ae3f10> @@ -121,7 +121,7 @@ def test_13(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["CollectionTree/egammaClustersAuxDyn.constituentClusterLinks"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=3) + py = branch.array(interp, library="np", entry_stop=3) # py[-1] == > (version 1) at 0x7fa94e968c10>], ...] at 0x7fa94e968c70> @@ -129,7 +129,7 @@ def test_14(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["CollectionTree/egammaClustersAuxDyn.eta_sampl"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=3) + py = branch.array(interp, library="np", entry_stop=3) # py[-1] == @@ -137,7 +137,7 @@ def test_15(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["CollectionTree/AnalysisHLT_mu24_ilooseAuxDyn.TrigMatchedObjects"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == > (version 1) at 0x7fb9b9d24c10>], ...] at 0x7fb9b9d29250> @@ -145,7 +145,7 @@ def test_16(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["CollectionTree/AnalysisHLT_mu40AuxDyn.TrigMatchedObjects"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == > (version 1) at 0x7f6e29be5cd0>], ...] at 0x7f6e29bea250> @@ -153,7 +153,7 @@ def test_17(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: branch = file["E/Evt/AAObject/usr_names"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=None) + py = branch.array(interp, library="np", entry_stop=None) # py[-1] == @@ -161,7 +161,7 @@ def test_18(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: branch = file["E/Evt/hits/hits.t"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == array([80860738., 80861689., 80862014., 80861709., 80861737., 80861158., 80862362., 80860821., 80862271., 80862273., 80861294., 80861860., 80862548., 80861733., 80861605., 80860467., 80860408., 80861562., 80862012., 80862350., 80861491., 80860384., 80860930., 80861541., 80861461., 80861749., 80862352., 80861813., 80861822., 80861871., 80862000., 80862255., 80862253., 80862249., 80862266., 80862248., 80862246., 80862847., 80863032., 80861952., 80861954., 80861953., 80861957., 80861951., 80861961., 80861959., 80861955., 80861994., 80862060., 80861971., 80862004., 80862002., 80862059., 80861695., 80861813., 80861967., 80862919., 80862043., 80862054., 80862044., 80862044., 80862040., 80862043., 80862037., 80862040., 80862039., 80862070., 80862042., 80862322., 80861605., 80861865., 80863034., 80862987., 80861545., 80860392., 80861003., 80861564., 80862109., 80861821., 80862083., 80861121., 80862513., 80862513., 80862731., 80861604., 80862003., 80861910., 80861854., 80862297., 80860989., 80862948., 80862075., 80862141., 80862117., 80862039., 80862114., 80862075., 80862042., 80862072., 80862439., 80862481., 80861656., 80862096., 80862215., 80862215., 80862195., 80862458., 80862432., 80861915., 80861012., 80862208., 80861885., 80861888., 80861994., 80861883., 80862194., 80861812., 80862184., 80862309., 80862297., 80862840., 80862400., 80861565., 80862226., 80862149.]) @@ -169,7 +169,7 @@ def test_19(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: branch = file["E/Evt/hits/hits.a"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]) @@ -177,7 +177,7 @@ def test_20(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: branch = file["E/Evt/hits/hits.trig"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == array([0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0], dtype=int32) @@ -185,7 +185,7 @@ def test_21(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: branch = file["E/Evt/hits/hits.tot"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == array([29, 26, 22, 18, 22, 28, 28, 28, 21, 24, 28, 30, 25, 24, 30, 28, 29, 4, 21, 25, 26, 22, 23, 22, 23, 29, 23, 30, 24, 29, 31, 27, 32, 28, 30, 33, 33, 31, 29, 18, 23, 34, 21, 33, 33, 29, 37, 23, 21, 40, 25, 29, 22, 17, 31, 25, 28, 26, 21, 20, 25, 51, 38, 64, 42, 28, 29, 26, 21, 31, 22, 18, 41, 28, 29, 28, 29, 15, 25, 27, 24, 28, 28, 34, 28, 21, 19, 21, 20, 24, 26, 24, 13, 22, 30, 25, 17, 27, 24, 16, 31, 27, 29, 23, 26, 25, 26, 28, 12, 18, 30, 27, 48, 16, 25, 24, 27, 10, 21, 25, 30, 26, 26, 28, 24], dtype=uint32) @@ -193,7 +193,7 @@ def test_22(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: branch = file["E/Evt/hits/hits.pos.x"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]) @@ -201,7 +201,7 @@ def test_23(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: branch = file["E/Evt/trks/trks.usr_names"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == array([, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ], dtype=object) @@ -209,7 +209,7 @@ def test_24(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: branch = file["E/Evt/trks/trks.rec_stages"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == array([, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ], dtype=object) @@ -217,7 +217,7 @@ def test_25(): with uproot.open(skhep_testdata.data_path("uproot-issue-208.root")) as file: branch = file["config/VERSION/VERSION._name"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == array(['psychePolicy', 'psycheEventModel', 'psycheCore', 'psycheUtils', 'psycheND280Utils', 'psycheIO', 'psycheSelections', 'psycheSystematics', 'highlandEventModel', 'highlandTools', 'highlandCore', 'highlandCorrections', 'highlandIO', 'baseAnalysis', 'baseTrackerAnalysis', 'numuCCAnalysis'], dtype=object) @@ -225,7 +225,7 @@ def test_26(): with uproot.open(skhep_testdata.data_path("uproot-issue-208.root")) as file: branch = file["config/SEL/SEL._firstSteps"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == array([], dtype=object) @@ -233,7 +233,7 @@ def test_27(): with uproot.open(skhep_testdata.data_path("uproot-issue-208.root")) as file: branch = file["config/SEL/SEL._branchAlias"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == array([], dtype=object) @@ -241,7 +241,7 @@ def test_28(): with uproot.open(skhep_testdata.data_path("uproot-issue-208.root")) as file: branch = file["config/SEL/SEL._nCutsInBranch"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == array([], dtype=object) @@ -249,7 +249,7 @@ def test_29(): with uproot.open(skhep_testdata.data_path("uproot-issue213.root")) as file: branch = file["T/eventPack/fGenInfo"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -257,7 +257,7 @@ def test_30(): with uproot.open(skhep_testdata.data_path("uproot-issue243-new.root")) as file: branch = file["triggerList/triggerMap/triggerMap.first"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=16) + py = branch.array(interp, library="np", entry_stop=16) # py[-1] == array(['HLT_2j35_bmv2c2060_split_2j35_L14J15.0ETA25', 'HLT_j100_2j55_bmv2c2060_split'], dtype=object) @@ -265,7 +265,7 @@ def test_31(): with uproot.open(skhep_testdata.data_path("uproot-issue-308.root")) as file: branch = file["MetaData/BranchIDLists"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -273,7 +273,7 @@ def test_32(): with uproot.open(skhep_testdata.data_path("uproot-issue31.root")) as file: branch = file["T/data/name"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == "two" @@ -281,7 +281,7 @@ def test_33(): with uproot.open(skhep_testdata.data_path("uproot-issue367b.root")) as file: branch = file["tree/weights"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -289,7 +289,7 @@ def test_34(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: branch = file["Header/Header./Header.geant4Version"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == '$Name: geant4-10-05-patch-01 $' @@ -297,7 +297,7 @@ def test_35(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: branch = file["Geant4Data/Geant4Data./Geant4Data.particles"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == , ...} at 0x7fb557a012e0> @@ -305,7 +305,7 @@ def test_36(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: branch = file["Model/Model./Model.samplerNamesUnique"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -313,7 +313,7 @@ def test_37(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: branch = file["Model/Model./Model.staPos"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == array([, , , ..., , , ], dtype=object) @@ -321,7 +321,7 @@ def test_38(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: branch = file["Event/PRBHF_46."] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == (version 4) at 0x7f9be2f7b2e0> @@ -329,7 +329,7 @@ def test_39(): with uproot.open(skhep_testdata.data_path("uproot-issue399.root")) as file: branch = file["Geant4Data/Geant4Data./Geant4Data.particles"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == , ...} at 0x7f6d057397f0> @@ -337,7 +337,7 @@ def test_40(): with uproot.open(skhep_testdata.data_path("uproot-issue-407.root")) as file: branch = file["tree/branch"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -345,7 +345,7 @@ def test_41(): with uproot.open(skhep_testdata.data_path("uproot-issue468.root")) as file: branch = file["Event/Trajectory./Trajectory.XYZ"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -353,7 +353,7 @@ def test_42(): with uproot.open(skhep_testdata.data_path("uproot-issue468.root")) as file: branch = file["Event/Trajectory./Trajectory.energyDeposit"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -361,7 +361,7 @@ def test_43(): with uproot.open(skhep_testdata.data_path("uproot-issue468.root")) as file: branch = file["Event/Trajectory./Trajectory.ionA"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -369,7 +369,7 @@ def test_44(): with uproot.open(skhep_testdata.data_path("uproot-issue46.root")) as file: branch = file["tree/evt"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -377,7 +377,7 @@ def test_45(): with uproot.open(skhep_testdata.data_path("uproot-issue485.root")) as file: branch = file["MCTrack/global"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == , ...] at 0x7f88b9a8ad00> @@ -385,7 +385,7 @@ def test_46(): with uproot.open(skhep_testdata.data_path("uproot-issue485.root")) as file: branch = file["MCParticle/detector1"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == , ...] at 0x7f94bc223550> @@ -393,7 +393,7 @@ def test_47(): with uproot.open(skhep_testdata.data_path("uproot-issue494.root")) as file: branch = file["Geant4Data/Geant4Data./Geant4Data.particles"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == , ...} at 0x7f53a44278b0> @@ -401,7 +401,7 @@ def test_48(): with uproot.open(skhep_testdata.data_path("uproot-issue494.root")) as file: branch = file["Geant4Data/Geant4Data./Geant4Data.ions"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -409,7 +409,7 @@ def test_49(): with uproot.open(skhep_testdata.data_path("uproot-issue498.root")) as file: branch = file["MCParticle/timepix"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == ] at 0x7f0697bf1a00> @@ -417,7 +417,7 @@ def test_50(): with uproot.open(skhep_testdata.data_path("uproot-issue498.root")) as file: branch = file["PixelHit/timepix"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -425,7 +425,7 @@ def test_51(): with uproot.open(skhep_testdata.data_path("uproot-issue510b.root")) as file: branch = file["EDepSimEvents/Event/Primaries/Primaries.GeneratorName"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == array(['GENIE:fixed@density-fixed'], dtype=object) @@ -433,7 +433,7 @@ def test_52(): with uproot.open(skhep_testdata.data_path("uproot-issue513.root")) as file: branch = file["Delphes/EFlowPhoton/EFlowPhoton.Particles"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == array([, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ], dtype=object) @@ -441,7 +441,7 @@ def test_53(): with uproot.open(skhep_testdata.data_path("uproot-issue513.root")) as file: branch = file["Delphes/Jet/Jet.SoftDroppedSubJet2"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == array([, , , , , ], dtype=object) @@ -449,7 +449,7 @@ def test_54(): with uproot.open(skhep_testdata.data_path("uproot-issue513.root")) as file: branch = file["Delphes/Jet/Jet.TrimmedP4[5]"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == array([[, , , , ], [, , , , ], [, , , , ], [, , , , ], [, , , , ], [, , , , ]], dtype=object) @@ -457,7 +457,7 @@ def test_55(): with uproot.open(skhep_testdata.data_path("uproot-issue519.root")) as file: branch = file["testtree/testbranch"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -467,7 +467,7 @@ def test_56(): ) as file: branch = file["tree/evt"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -475,7 +475,7 @@ def test_57(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/vector_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -483,7 +483,7 @@ def test_58(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/vector_tstring"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -491,7 +491,7 @@ def test_59(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/vector_vector_int32"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -499,7 +499,7 @@ def test_60(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/vector_vector_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -507,7 +507,7 @@ def test_61(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/vector_vector_tstring"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -515,7 +515,7 @@ def test_62(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/vector_set_int32"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -523,7 +523,7 @@ def test_63(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/vector_set_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -531,7 +531,7 @@ def test_64(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/set_int32"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -539,7 +539,7 @@ def test_65(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/set_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -547,7 +547,7 @@ def test_66(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_int32_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -555,7 +555,7 @@ def test_67(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_int32_vector_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -563,7 +563,7 @@ def test_68(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_int32_vector_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -571,7 +571,7 @@ def test_69(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_int32_set_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -579,7 +579,7 @@ def test_70(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_int32_set_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -587,7 +587,7 @@ def test_71(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_string_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -595,7 +595,7 @@ def test_72(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_string_vector_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -603,7 +603,7 @@ def test_73(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_string_vector_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -611,7 +611,7 @@ def test_74(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_string_set_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -619,7 +619,7 @@ def test_75(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_string_set_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -627,7 +627,7 @@ def test_76(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_int32_vector_vector_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -635,7 +635,7 @@ def test_77(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_int32_vector_set_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -643,7 +643,7 @@ def test_78(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_string_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -651,7 +651,7 @@ def test_79(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_string_tstring"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -661,5 +661,15 @@ def test_80(): ) as file: branch = file["t/x"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(library="np", entry_stop=2) + py = branch.array(interp, library="np", entry_stop=2) + # py[-1] == + + +def test_81(): + with uproot.open( + skhep_testdata.data_path("uproot-HZZ-objects.root") + ) as file: + branch = file["events/MET"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + py = branch.array(interp, library="np", entry_stop=2) # py[-1] == From a27494e63a12fdf23cd8336ffe6cc5f8c3ee9e10 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Wed, 6 Jul 2022 13:41:42 -0500 Subject: [PATCH 03/41] TVector2_V3 works! --- src/uproot/_awkward_forth.py | 92 +++++++++++++++++++++------- src/uproot/containers.py | 27 ++++---- src/uproot/interpretation/objects.py | 12 ++-- src/uproot/model.py | 39 +++++++++--- src/uproot/models/TObject.py | 28 ++++++++- src/uproot/models/TRef.py | 17 ++--- src/uproot/streamers.py | 29 ++++----- tests/test_610-awkward-form.py | 16 +++++ 8 files changed, 184 insertions(+), 76 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index 2b704189b..53206b073 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -5,15 +5,23 @@ """ +from pytz import NonExistentTimeError + + class ForthGenerator: """ This class is passed through the Forth code generation, collecting Forth snippets and concatenating them at the end. """ def __init__(self, aform=None, count_obj=0, var_set=False): + self.dummy_form = False + self.top_dummy = None + self.dummy_aform = None self.aform = aform self.top_form = None - self.awkward_model = {} + self.awkward_model = {"name": "TOP", "content": {}} + self._prev_node = self.awkward_model + self.ref_list = [] self.forth_code = {} self.forth_keys = {} self.final_code = [] @@ -28,25 +36,53 @@ def traverse_aform(self): self.aform = self.aform.content def should_add_form(self): - return not bool(self.awkward_model) + if "content" in self.awkward_model.keys(): + if self.awkward_model["content"] is None: + return False + else: + return not bool(self.awkward_model["content"]) + + def get_temp_form_top(self): + return self.top_dummy - def add_form(self, aform): - if self.aform is None: - self.aform = aform - self.top_form = self.aform + def set_dummy_none(self): + self.top_dummy = None + self.dummy_aform = None + self.dummy_form = False + + def add_form(self, aform, conlen=0): + if self.dummy_form: + if self.dummy_aform is None: + self.dummy_aform = aform + self.top_dummy = aform + else: + if "content" in self.dummy_aform.keys(): + if self.dummy_aform["content"] == "NULL": + self.dummy_aform["content"] = aform + self.dummy_aform = self.dummy_aform["content"] + else: + raise ValueError + elif "contents" in self.dummy_aform.keys(): + if len(self.dummy_aform["content"]) < self.dummy_aform["parameters"]["lencon"]: + self.dummy_aform["contents"].append(aform) + else: + raise ValueError else: - if "content" in self.aform.keys(): - if self.aform["content"] == "NULL": - self.aform["content"] = aform - self.aform = self.aform["content"] - else: - raise ValueError - elif "contents" in self.aform.keys(): - if len(self.aform["content"]) < self.aform["parameters"]["lencon"]: - self.aform["contents"].append(aform) - self.aform = self.aform["contents"][-1] - else: - raise ValueError + if self.aform is None: + self.aform = aform + self.top_form = self.aform + else: + if "content" in self.aform.keys(): + if self.aform["content"] == "NULL": + self.aform["content"] = aform + self.aform = self.aform["content"] + else: + raise ValueError + elif "contents" in self.aform.keys(): + if len(self.aform["content"]) == conlen: + return + else: + raise ValueError def get_keys(self, num_keys): if num_keys == 1: @@ -66,18 +102,28 @@ def add_form_key(self, key): self.form_keys.append(key) def go_to(self, aform): - aform["content"] = self.awkward_model + #aform["content"] = self.awkward_model + self.awkward_model = aform + + def become(self, aform): self.awkward_model = aform def check_model(self): return bool(self.awkward_model) + def get_current_node(self): + self.ref_list.append(self._prev_node) + return len(self.ref_list) - 1 + + def get_ref(self, index): + return self.ref_list[index] + def add_node( self, name, precode, postcode, initcode, headercode, dtype, num_child, content ): if isinstance(self.awkward_model, dict): - if not bool(self.awkward_model): - self.awkward_model = { + if not bool(self.awkward_model["content"]) and self.awkward_model["content"] is not None: + temp_obj = { "name": name, "type": dtype, "pre_code": precode, @@ -87,9 +133,9 @@ def add_node( "num_child": num_child, "content": content, } - temp_node = self.awkward_model + self.awkward_model["content"] = temp_obj self.awkward_model = self.awkward_model["content"] - return temp_node + return temp_obj else: temp_node = self.awkward_model self.awkward_model = self.awkward_model["content"] diff --git a/src/uproot/containers.py b/src/uproot/containers.py index 43a34c0c3..dc1eebec9 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -761,16 +761,16 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): temp_bool = "false" temp_aform = f'{{ "class":"ListOffsetArray", "offsets":"i64", "content": "NULL", "has_identifier": false, "parameters": {{"uproot": {{"as": "vector", "header": {temp_bool}}}}}, "form_key": "node{offsets_num}"}}' forth_obj.add_form(json.loads(temp_aform)) - temp = forth_obj.add_node( - f"node{offsets_num}", - helper_obj.get_pre(), - helper_obj.get_post(), - helper_obj.get_init(), - helper_obj.get_header(), - "i64", - 1, - {}, - ) + temp = forth_obj.add_node( + f"node{offsets_num}", + helper_obj.get_pre(), + helper_obj.get_post(), + helper_obj.get_init(), + helper_obj.get_header(), + "i64", + 1, + {}, + ) if cursor.index >= chunk.stop and helper_obj.is_forth(): forth_obj.var_set = True @@ -782,9 +782,12 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): chunk, cursor, context, file, selffile, parent ) ) + if helper_obj.is_forth(): + forth_obj.go_to(temp['content']) + #print(forth_obj._prev_node, "UOUOUO") - if helper_obj.is_forth(): - forth_obj.go_to(temp) + # if helper_obj.is_forth(): + # forth_obj.go_to(temp) return uproot._util.objectarray1d(out).reshape(-1, *self.inner_shape) diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index e1348f546..be506ddf2 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -69,6 +69,7 @@ def __init__(self, model, branch=None): self._forth_vm = threading.local() self._complete_forth_code = [] self._forth_vm_set = False + self._form = None @property def model(self): @@ -207,7 +208,7 @@ def basket_array_forth( context["forth"] = forth_gen if isinstance(library, uproot.interpretation.library.Awkward): - self._form = self.awkward_form( + form = self.awkward_form( branch.file, { "index_format": "i64", @@ -216,7 +217,7 @@ def basket_array_forth( "breadcrumbs": (), }, ) - if awkward_can_optimize(self, self._form): + if awkward_can_optimize(self, form): import awkward._connect._uproot extra = { @@ -227,9 +228,8 @@ def basket_array_forth( "cursor_offset": cursor_offset, } output = awkward._connect._uproot.basket_array( - self._form, data, byte_offsets, extra + form, data, byte_offsets, extra ) - if not self._forth_vm_set: if not self._prereaddone: if output is None: @@ -257,9 +257,8 @@ def basket_array_forth( ) if not context["forth"].var_set: self._prereaddone = True - print(context["forth"].awkward_model) self.assemble_forth( - context["forth"], context["forth"].awkward_model + context["forth"], context["forth"]._prev_node["content"] ) self._complete_forth_code = f'input stream\ninput byteoffsets\ninput bytestops\n{"".join(context["forth"].final_header)}\n{"".join(context["forth"].final_init)}\n0 do\nbyteoffsets I-> stack\nstream seek \n{"".join(context["forth"].final_code)}\nloop' self._forth_vm.vm = awkward.forth.ForthMachine64( @@ -273,7 +272,6 @@ def basket_array_forth( self._complete_forth_code ) self._forth_vm_set = True - if self._forth_vm_set: byte_start = byte_offsets[0] byte_stop = byte_offsets[-1] diff --git a/src/uproot/model.py b/src/uproot/model.py index cd91f438e..a167fd2a0 100644 --- a/src/uproot/model.py +++ b/src/uproot/model.py @@ -805,8 +805,8 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): parent, concrete=concrete, ) - if helper_obj.is_forth(): - forth_obj.go_to(temp) + # if helper_obj.is_forth(): + # forth_obj.go_to(temp) return temp_var if context.get("in_TBranch", False): @@ -825,7 +825,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): ) if helper_obj.is_forth(): temp = forth_obj.add_node( - "pass", + "model828", helper_obj.get_pre(), helper_obj.get_post(), helper_obj.get_init(), @@ -834,10 +834,8 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): 1, {}, ) + self.read_members(chunk, cursor, context, file) - if helper_obj.is_forth(): - forth_obj.go_to(temp) - raise NotImplementedError self.hook_after_read_members( chunk=chunk, cursor=cursor, context=context, file=file ) @@ -1294,11 +1292,14 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): :doc:`uproot.model.UnknownClassVersion` is created instead. """ import uproot.deserialization + temp_node = None helper_obj = uproot._awkward_forth.GenHelper(context) + if helper_obj.is_forth(): + forth_obj = helper_obj.get_gen_obj() # Ignores context["reading"], because otherwise, there would be nothing to do. - + start_index = cursor._index ( num_bytes, version, @@ -1306,6 +1307,19 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): ) = uproot.deserialization.numbytes_version(chunk, cursor, context, move=False) versioned_cls = cls.class_of_version(version) + bytes_skipped = cursor._index - start_index + if helper_obj.is_forth(): + helper_obj.add_to_pre(f"{bytes_skipped} stream skip \n") + temp_node = forth_obj.add_node( + "Model1319", + helper_obj.get_pre(), + helper_obj.get_post(), + helper_obj.get_init(), + helper_obj.get_header(), + "i64", + 1, + {}, + ) if versioned_cls is not None: pass @@ -1327,9 +1341,9 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): ) # versioned_cls.read starts with numbytes_version again because move=False (above) - if helper_obj.is_forth(): - print(versioned_cls) - return cls.postprocess( + # if helper_obj.is_forth(): + # print(versioned_cls) + temp_var = cls.postprocess( versioned_cls.read( chunk, cursor, context, file, selffile, parent, concrete=concrete ), @@ -1338,6 +1352,11 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): context, file, ) + # if helper_obj.is_forth(): + # if "no_go_to" not in context.keys(): + #raise NotImplementedError + # forth_obj.go_to(temp_node) + return temp_var @classmethod def postprocess(cls, self, chunk, cursor, context, file): diff --git a/src/uproot/models/TObject.py b/src/uproot/models/TObject.py index 6e88bedac..ee41a1ec5 100644 --- a/src/uproot/models/TObject.py +++ b/src/uproot/models/TObject.py @@ -11,6 +11,8 @@ import uproot +import json + _tobject_format1 = struct.Struct(">h") _tobject_format2 = struct.Struct(">II") @@ -24,7 +26,12 @@ def read_numbytes_version(self, chunk, cursor, context): pass def read_members(self, chunk, cursor, context, file): - #raise NotImplementedError + # raise NotImplementedError + helper_obj = uproot._awkward_forth.GenHelper(context) + start_index = cursor._index + if helper_obj.is_forth(): + forth_obj = helper_obj.get_gen_obj() + #raise NotImplementedError if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} @@ -44,13 +51,28 @@ def read_members(self, chunk, cursor, context, file): if self._members["@fBits"] & uproot.const.kIsReferenced: cursor.skip(2) self._members["@fBits"] = int(self._members["@fBits"]) - + if helper_obj.is_forth(): + skip_length = cursor._index - start_index + helper_obj.add_to_pre(f"{skip_length} stream skip \n") + if forth_obj.should_add_form(): + temp_aform = '{"class": "RecordArray", "contents":{}, "parameters": {"__record__": "TObject"}}' + forth_obj.add_form(json.loads(temp_aform)) + temp = forth_obj.add_node( + "TObjext", + helper_obj.get_pre(), + helper_obj.get_post(), + helper_obj.get_init(), + helper_obj.get_header(), + "i64", + 0, + {}, + ) writable = True def _serialize(self, out, header, name, tobject_flags): out.append(b"\x00\x01" + _tobject_format2.pack(0, tobject_flags)) - @classmethod + @ classmethod def strided_interpretation( cls, file, header=False, tobject_header=True, breadcrumbs=(), original=None ): diff --git a/src/uproot/models/TRef.py b/src/uproot/models/TRef.py index 924ce60e4..48f7b3e17 100644 --- a/src/uproot/models/TRef.py +++ b/src/uproot/models/TRef.py @@ -120,6 +120,7 @@ def read_members(self, chunk, cursor, context, file): if helper_obj.is_forth(): awkward = uproot.extras.awkward() # noqa:F841 forth_obj = helper_obj.get_gen_obj() + #raise NotImplementedError if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} @@ -148,18 +149,21 @@ def read_members(self, chunk, cursor, context, file): f"part0-node{form_keys[2]}-data", f"part0-node{form_keys[5]}-data", ] + # print("===========================================") + # print(not bool(forth_obj.awkward_model["content"]), "YUYIYIIy") + # print("===========================================") if forth_obj.should_add_form(): for elem in keys: forth_obj.add_form_key(elem) temp_bool = "false" temp_aform = f'{{"class": "RecordArray", "contents": {{"fname": {{"class": "ListOffsetArray", "offsets": "i64", "content": {{"class": "NumpyArray", "primitive": "uint8", "inner_shape": [], "has_identifier": false, "parameters": {{"__array__": "char"}}, "form_key": "node{form_keys[2]}"}}, "has_identifier": false, "parameters": {{"uproot": {{"as": "vector", "header": {temp_bool}}}}}, "form_key": "node{form_keys[1]}"}}, "fSize": {{"class": "NumpyArray", "primitive": "int64", "inner_shape": [], "has_identifier": false, "parameters": {{}}, "form_key": "node{form_keys[3]}"}}, "refs": {{"class": "ListOffsetArray", "offsets": "i64", "content": {{"class": "NumpyArray", "primitive": "int64", "inner_shape": [], "has_identifier": false, "parameters": {{}}, "form_key": "node{form_keys[5]}"}}, "has_identifier": false, "parameters": {{}}, "form_key": "node{form_keys[4]}"}}}}, "has_identifier": false, "parameters": {{}}, "form_key": "node{form_keys[0]}"}}' forth_obj.add_form(json.loads(temp_aform)) - helper_obj.add_to_header( - f"output part0-node{form_keys[1]}-offsets int64\noutput part0-node{form_keys[2]}-data uint8\noutput part0-node{form_keys[3]}-data int64\noutput part0-node{form_keys[4]}-offsets int64\noutput part0-node{form_keys[5]}-data int64\n" - ) - helper_obj.add_to_init( - f"0 part0-node{form_keys[1]}-offsets <- stack\n0 part0-node{form_keys[4]}-offsets <- stack\n" - ) + helper_obj.add_to_header( + f"output part0-node{form_keys[1]}-offsets int64\noutput part0-node{form_keys[2]}-data uint8\noutput part0-node{form_keys[3]}-data int64\noutput part0-node{form_keys[4]}-offsets int64\noutput part0-node{form_keys[5]}-data int64\n" + ) + helper_obj.add_to_init( + f"0 part0-node{form_keys[1]}-offsets <- stack\n0 part0-node{form_keys[4]}-offsets <- stack\n" + ) temp = forth_obj.add_node( f"node{form_keys[0]}", helper_obj.get_pre(), @@ -170,7 +174,6 @@ def read_members(self, chunk, cursor, context, file): 1, None, ) - forth_obj.go_to(temp) cursor.skip(10) self._members["fName"] = cursor.string(chunk, context) diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index a753f4a2f..a9d96951c 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -214,6 +214,13 @@ def class_code(self): ' f"memberwise serialization of {type(self).__name__}\\nin file {self.file.file_path}"', " )", ] + read_members.append( + """ + helper_obj = uproot._awkward_forth.GenHelper(context) + if helper_obj.is_forth(): + forth_obj = helper_obj.get_gen_obj() + content = {}\n""" + ) read_member_n = [ " def read_member_n(self, chunk, cursor, context, file, member_index):" ] @@ -248,8 +255,6 @@ def class_code(self): base_names_versions = [] member_names = [] class_flags = {} - for i in range(len(self._members["fElements"])): - print(self._members["fElements"][i]) for i in range(len(self._members["fElements"])): self._members["fElements"][i].class_code( self, @@ -268,6 +273,7 @@ def class_code(self): member_names, class_flags, ) + read_members.append(" if helper_obj.is_forth():\n if forth_obj.should_add_form():\n forth_obj.add_form({'class': 'RecordArray', 'contents': content, 'parameters': {'record': 'TVector2'}})\n temp = forth_obj.add_node(f\"node{{forth_obj.get_keys()}}\",helper_obj.get_pre(),helper_obj.get_post(),helper_obj.get_init(),helper_obj.get_header(),\"i64\",0,None)") if len(read_members) == 1: # untested as of PR #629 read_members.append(" pass") @@ -674,20 +680,14 @@ def class_code( member_names, class_flags, ): - read_members.append( - """ - helper_obj = uproot._awkward_forth.GenHelper(context) - if helper_obj.is_forth(): - forth_obj = helper_obj.get_gen_obj() - key = forth_obj.get_keys(1)\n""" - ) read_member_n.append(f" if member_index == {i}:") # @aryan26roy: test_0637's 01,02,08,09,11,12,13,15,16,29,38,45,46,49,50 - + read_members.append(" if helper_obj.is_forth():\n forth_obj.dummy_form = True") read_members.append( f" self._bases.append(c({self.name!r}, {self.base_version!r}).read(chunk, cursor, context, file, self._file, self._parent, concrete=self.concrete))" ) + read_members.append(" if helper_obj.is_forth():\n temp_form = forth_obj.get_temp_form_top()\n content.update(temp_form['contents'])\n forth_obj.set_dummy_none()\n") read_member_n.append(" " + read_members[-1]) strided_interpretation.append( f" members.extend(file.class_named({self.name!r}, {self.base_version!r}).strided_interpretation(file, header, tobject_header, breadcrumbs).members)" @@ -918,12 +918,13 @@ def class_code( else: read_members.append(' if helper_obj.is_forth():') - for elem in formats[0]: + for i in range(len(formats[0])): read_members.append(' key = forth_obj.get_keys(1)') read_members.append(' form_key = f"part0-node{key}-data"') - read_members.append(f' helper_obj.add_to_header(f"output part0-node{{key}}-data {uproot._awkward_forth.convert_dtype(elem)}\\n")') - read_members.append(' helper_obj.add_to_init(f"0 part0-node{key}-offsets <- stack\\n")') - read_members.append(f' helper_obj.add_to_pre("stream !{elem}-> output")') + read_members.append(f' helper_obj.add_to_header(f"output part0-node{{key}}-data {uproot._awkward_forth.convert_dtype(formats[0][i])}\\n")') + #read_members.append(' helper_obj.add_to_init(f"0 part0-node{key}-offsets <- stack\\n")') + read_members.append(f' content["{fields[0][i]}"] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[0][i])}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f\"node{{key}}\"}}') + read_members.append(f' helper_obj.add_to_pre(f"stream !{formats[0][i]}-> part0-node{{key}}-data\\n")') read_members.append(' if forth_obj.should_add_form():') read_members.append(' forth_obj.add_form_key(form_key)') assign_members = ", ".join( diff --git a/tests/test_610-awkward-form.py b/tests/test_610-awkward-form.py index 5e64fde6f..6e425ed09 100644 --- a/tests/test_610-awkward-form.py +++ b/tests/test_610-awkward-form.py @@ -2,6 +2,7 @@ import json import sys +from matplotlib.style import library import numpy import pytest @@ -100,6 +101,21 @@ def test_awkward_array_tref_array_forth(): assert awk_data[6][2]["refs"][-1] == 676 +def test_awkward_array_tvector2_array_forth(): + + awk_data = None + + with uproot.open(skhep_testdata.data_path("uproot-HZZ-objects.root"))[ + "events/MET" + ] as tree: + interp = uproot.interpretation.identify.interpretation_of(tree, {}, False) + interp._forth = True + awk_data = tree.array(interp, library="ak") + assert awk_data[0]["fX"] == 5.912771224975586 + assert awk_data[4]["fY"] == -1.3100523948669434 + assert awk_data[1200]["fX"] == 1.9457910060882568 + + def test_awkward_vector_tstring(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root"))[ From ab56efd62c9a3cc339adf02554057a9056356268 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 6 Jul 2022 18:43:05 +0000 Subject: [PATCH 04/41] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/uproot/_awkward_forth.py | 14 ++++-- src/uproot/containers.py | 4 +- src/uproot/model.py | 5 ++- src/uproot/models/TObject.py | 8 ++-- src/uproot/models/TRef.py | 2 +- src/uproot/streamers.py | 45 +++++++++++++------ .../test_0637-setup-tests-for-AwkwardForth.py | 4 +- tests/test_610-awkward-form.py | 2 +- 8 files changed, 53 insertions(+), 31 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index 53206b073..d1fa47bbd 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -63,7 +63,10 @@ def add_form(self, aform, conlen=0): else: raise ValueError elif "contents" in self.dummy_aform.keys(): - if len(self.dummy_aform["content"]) < self.dummy_aform["parameters"]["lencon"]: + if ( + len(self.dummy_aform["content"]) + < self.dummy_aform["parameters"]["lencon"] + ): self.dummy_aform["contents"].append(aform) else: raise ValueError @@ -102,7 +105,7 @@ def add_form_key(self, key): self.form_keys.append(key) def go_to(self, aform): - #aform["content"] = self.awkward_model + # aform["content"] = self.awkward_model self.awkward_model = aform def become(self, aform): @@ -122,7 +125,10 @@ def add_node( self, name, precode, postcode, initcode, headercode, dtype, num_child, content ): if isinstance(self.awkward_model, dict): - if not bool(self.awkward_model["content"]) and self.awkward_model["content"] is not None: + if ( + not bool(self.awkward_model["content"]) + and self.awkward_model["content"] is not None + ): temp_obj = { "name": name, "type": dtype, @@ -246,5 +252,5 @@ def get_init(self): def convert_dtype(format): - if format == 'd': + if format == "d": return "float64" diff --git a/src/uproot/containers.py b/src/uproot/containers.py index dc1eebec9..34537efe4 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -783,8 +783,8 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): ) ) if helper_obj.is_forth(): - forth_obj.go_to(temp['content']) - #print(forth_obj._prev_node, "UOUOUO") + forth_obj.go_to(temp["content"]) + # print(forth_obj._prev_node, "UOUOUO") # if helper_obj.is_forth(): # forth_obj.go_to(temp) diff --git a/src/uproot/model.py b/src/uproot/model.py index a167fd2a0..0420355e5 100644 --- a/src/uproot/model.py +++ b/src/uproot/model.py @@ -1292,6 +1292,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): :doc:`uproot.model.UnknownClassVersion` is created instead. """ import uproot.deserialization + temp_node = None helper_obj = uproot._awkward_forth.GenHelper(context) @@ -1342,7 +1343,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): # versioned_cls.read starts with numbytes_version again because move=False (above) # if helper_obj.is_forth(): - # print(versioned_cls) + # print(versioned_cls) temp_var = cls.postprocess( versioned_cls.read( chunk, cursor, context, file, selffile, parent, concrete=concrete @@ -1354,7 +1355,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): ) # if helper_obj.is_forth(): # if "no_go_to" not in context.keys(): - #raise NotImplementedError + # raise NotImplementedError # forth_obj.go_to(temp_node) return temp_var diff --git a/src/uproot/models/TObject.py b/src/uproot/models/TObject.py index ee41a1ec5..530045fcc 100644 --- a/src/uproot/models/TObject.py +++ b/src/uproot/models/TObject.py @@ -5,14 +5,13 @@ """ +import json import struct import numpy import uproot -import json - _tobject_format1 = struct.Struct(">h") _tobject_format2 = struct.Struct(">II") @@ -31,7 +30,7 @@ def read_members(self, chunk, cursor, context, file): start_index = cursor._index if helper_obj.is_forth(): forth_obj = helper_obj.get_gen_obj() - #raise NotImplementedError + # raise NotImplementedError if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} @@ -67,12 +66,13 @@ def read_members(self, chunk, cursor, context, file): 0, {}, ) + writable = True def _serialize(self, out, header, name, tobject_flags): out.append(b"\x00\x01" + _tobject_format2.pack(0, tobject_flags)) - @ classmethod + @classmethod def strided_interpretation( cls, file, header=False, tobject_header=True, breadcrumbs=(), original=None ): diff --git a/src/uproot/models/TRef.py b/src/uproot/models/TRef.py index 48f7b3e17..44d83e4d6 100644 --- a/src/uproot/models/TRef.py +++ b/src/uproot/models/TRef.py @@ -120,7 +120,7 @@ def read_members(self, chunk, cursor, context, file): if helper_obj.is_forth(): awkward = uproot.extras.awkward() # noqa:F841 forth_obj = helper_obj.get_gen_obj() - #raise NotImplementedError + # raise NotImplementedError if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index a9d96951c..739592eec 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -13,7 +13,6 @@ import numpy import uproot - import uproot._awkward_forth _canonical_typename_patterns = [ @@ -273,7 +272,9 @@ def class_code(self): member_names, class_flags, ) - read_members.append(" if helper_obj.is_forth():\n if forth_obj.should_add_form():\n forth_obj.add_form({'class': 'RecordArray', 'contents': content, 'parameters': {'record': 'TVector2'}})\n temp = forth_obj.add_node(f\"node{{forth_obj.get_keys()}}\",helper_obj.get_pre(),helper_obj.get_post(),helper_obj.get_init(),helper_obj.get_header(),\"i64\",0,None)") + read_members.append( + " if helper_obj.is_forth():\n if forth_obj.should_add_form():\n forth_obj.add_form({'class': 'RecordArray', 'contents': content, 'parameters': {'record': 'TVector2'}})\n temp = forth_obj.add_node(f\"node{{forth_obj.get_keys()}}\",helper_obj.get_pre(),helper_obj.get_post(),helper_obj.get_init(),helper_obj.get_header(),\"i64\",0,None)" + ) if len(read_members) == 1: # untested as of PR #629 read_members.append(" pass") @@ -683,11 +684,15 @@ def class_code( read_member_n.append(f" if member_index == {i}:") # @aryan26roy: test_0637's 01,02,08,09,11,12,13,15,16,29,38,45,46,49,50 - read_members.append(" if helper_obj.is_forth():\n forth_obj.dummy_form = True") + read_members.append( + " if helper_obj.is_forth():\n forth_obj.dummy_form = True" + ) read_members.append( f" self._bases.append(c({self.name!r}, {self.base_version!r}).read(chunk, cursor, context, file, self._file, self._parent, concrete=self.concrete))" ) - read_members.append(" if helper_obj.is_forth():\n temp_form = forth_obj.get_temp_form_top()\n content.update(temp_form['contents'])\n forth_obj.set_dummy_none()\n") + read_members.append( + " if helper_obj.is_forth():\n temp_form = forth_obj.get_temp_form_top()\n content.update(temp_form['contents'])\n forth_obj.set_dummy_none()\n" + ) read_member_n.append(" " + read_members[-1]) strided_interpretation.append( f" members.extend(file.class_named({self.name!r}, {self.base_version!r}).strided_interpretation(file, header, tobject_header, breadcrumbs).members)" @@ -889,7 +894,7 @@ def class_code( read_member_n.append(" " + read_members[-1]) elif self.array_length == 0: - #++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# + # ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# if ( i == 0 or not isinstance(elements[i - 1], Model_TStreamerBasicType) @@ -917,16 +922,28 @@ def class_code( ) else: - read_members.append(' if helper_obj.is_forth():') + read_members.append(" if helper_obj.is_forth():") for i in range(len(formats[0])): - read_members.append(' key = forth_obj.get_keys(1)') - read_members.append(' form_key = f"part0-node{key}-data"') - read_members.append(f' helper_obj.add_to_header(f"output part0-node{{key}}-data {uproot._awkward_forth.convert_dtype(formats[0][i])}\\n")') - #read_members.append(' helper_obj.add_to_init(f"0 part0-node{key}-offsets <- stack\\n")') - read_members.append(f' content["{fields[0][i]}"] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[0][i])}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f\"node{{key}}\"}}') - read_members.append(f' helper_obj.add_to_pre(f"stream !{formats[0][i]}-> part0-node{{key}}-data\\n")') - read_members.append(' if forth_obj.should_add_form():') - read_members.append(' forth_obj.add_form_key(form_key)') + read_members.append(" key = forth_obj.get_keys(1)") + read_members.append( + ' form_key = f"part0-node{key}-data"' + ) + read_members.append( + f' helper_obj.add_to_header(f"output part0-node{{key}}-data {uproot._awkward_forth.convert_dtype(formats[0][i])}\\n")' + ) + # read_members.append(' helper_obj.add_to_init(f"0 part0-node{key}-offsets <- stack\\n")') + read_members.append( + f' content["{fields[0][i]}"] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[0][i])}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}' + ) + read_members.append( + f' helper_obj.add_to_pre(f"stream !{formats[0][i]}-> part0-node{{key}}-data\\n")' + ) + read_members.append( + " if forth_obj.should_add_form():" + ) + read_members.append( + " forth_obj.add_form_key(form_key)" + ) assign_members = ", ".join( f"self._members[{x!r}]" for x in fields[-1] ) diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index a1d16bcfb..88bd486ae 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -666,9 +666,7 @@ def test_80(): def test_81(): - with uproot.open( - skhep_testdata.data_path("uproot-HZZ-objects.root") - ) as file: + with uproot.open(skhep_testdata.data_path("uproot-HZZ-objects.root")) as file: branch = file["events/MET"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) py = branch.array(interp, library="np", entry_stop=2) diff --git a/tests/test_610-awkward-form.py b/tests/test_610-awkward-form.py index 6e425ed09..68ff10158 100644 --- a/tests/test_610-awkward-form.py +++ b/tests/test_610-awkward-form.py @@ -2,11 +2,11 @@ import json import sys -from matplotlib.style import library import numpy import pytest import skhep_testdata +from matplotlib.style import library import uproot From 989463fa0aac3212de280bf180aed1583eba991d Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Wed, 6 Jul 2022 13:44:07 -0500 Subject: [PATCH 05/41] added pytest approx for big float tests --- tests/test_610-awkward-form.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_610-awkward-form.py b/tests/test_610-awkward-form.py index 6e425ed09..a8884e475 100644 --- a/tests/test_610-awkward-form.py +++ b/tests/test_610-awkward-form.py @@ -111,9 +111,9 @@ def test_awkward_array_tvector2_array_forth(): interp = uproot.interpretation.identify.interpretation_of(tree, {}, False) interp._forth = True awk_data = tree.array(interp, library="ak") - assert awk_data[0]["fX"] == 5.912771224975586 - assert awk_data[4]["fY"] == -1.3100523948669434 - assert awk_data[1200]["fX"] == 1.9457910060882568 + assert awk_data[0]["fX"] == pytest.approx(5.912771224975586) + assert awk_data[4]["fY"] == pytest.approx(-1.3100523948669434) + assert awk_data[1200]["fX"] == pytest.approx(1.9457910060882568) def test_awkward_vector_tstring(): From 1c00b279e3500573f8b09cfe29db9a1d72d68d01 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Wed, 6 Jul 2022 13:52:35 -0500 Subject: [PATCH 06/41] linting --- src/uproot/_awkward_forth.py | 11 ++++++++--- src/uproot/model.py | 8 +++----- src/uproot/models/TObject.py | 2 +- src/uproot/models/TRef.py | 2 +- src/uproot/streamers.py | 5 ----- tests/test_610-awkward-form.py | 1 - 6 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index d1fa47bbd..f2e53f2f1 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -5,9 +5,6 @@ """ -from pytz import NonExistentTimeError - - class ForthGenerator: """ This class is passed through the Forth code generation, collecting Forth snippets and concatenating them at the end. @@ -252,5 +249,13 @@ def get_init(self): def convert_dtype(format): + """Takes datatype codes from classses and returns the full datatype name. + + Args: + format (string): The datatype in the dynamic class + + Returns: + string: The datatype in words. + """ if format == "d": return "float64" diff --git a/src/uproot/model.py b/src/uproot/model.py index 0420355e5..1c447f6ac 100644 --- a/src/uproot/model.py +++ b/src/uproot/model.py @@ -784,7 +784,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): cls.__name__ ): if helper_obj.is_forth(): - temp = forth_obj.add_node( + forth_obj.add_node( "pass", helper_obj.get_pre(), helper_obj.get_post(), @@ -824,7 +824,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): chunk=chunk, cursor=cursor, context=context, file=file ) if helper_obj.is_forth(): - temp = forth_obj.add_node( + forth_obj.add_node( "model828", helper_obj.get_pre(), helper_obj.get_post(), @@ -1293,8 +1293,6 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): """ import uproot.deserialization - temp_node = None - helper_obj = uproot._awkward_forth.GenHelper(context) if helper_obj.is_forth(): @@ -1311,7 +1309,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): bytes_skipped = cursor._index - start_index if helper_obj.is_forth(): helper_obj.add_to_pre(f"{bytes_skipped} stream skip \n") - temp_node = forth_obj.add_node( + forth_obj.add_node( "Model1319", helper_obj.get_pre(), helper_obj.get_post(), diff --git a/src/uproot/models/TObject.py b/src/uproot/models/TObject.py index 530045fcc..d14a4f6fa 100644 --- a/src/uproot/models/TObject.py +++ b/src/uproot/models/TObject.py @@ -56,7 +56,7 @@ def read_members(self, chunk, cursor, context, file): if forth_obj.should_add_form(): temp_aform = '{"class": "RecordArray", "contents":{}, "parameters": {"__record__": "TObject"}}' forth_obj.add_form(json.loads(temp_aform)) - temp = forth_obj.add_node( + forth_obj.add_node( "TObjext", helper_obj.get_pre(), helper_obj.get_post(), diff --git a/src/uproot/models/TRef.py b/src/uproot/models/TRef.py index 44d83e4d6..6ae1e557c 100644 --- a/src/uproot/models/TRef.py +++ b/src/uproot/models/TRef.py @@ -164,7 +164,7 @@ def read_members(self, chunk, cursor, context, file): helper_obj.add_to_init( f"0 part0-node{form_keys[1]}-offsets <- stack\n0 part0-node{form_keys[4]}-offsets <- stack\n" ) - temp = forth_obj.add_node( + forth_obj.add_node( f"node{form_keys[0]}", helper_obj.get_pre(), helper_obj.get_post(), diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index 739592eec..d8e9f44ad 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -1491,11 +1491,6 @@ class Model_TStreamerObject(TStreamerObjectTypes, Model_TStreamerElement): def read_members(self, chunk, cursor, context, file): - helper_obj = uproot._awkward_forth.GenHelper(context) - - if helper_obj.is_forth(): - forth_obj = helper_obj.get_gen_obj() - start = cursor.index self._bases.append( diff --git a/tests/test_610-awkward-form.py b/tests/test_610-awkward-form.py index 8f9afeb7b..4a0c91fff 100644 --- a/tests/test_610-awkward-form.py +++ b/tests/test_610-awkward-form.py @@ -6,7 +6,6 @@ import numpy import pytest import skhep_testdata -from matplotlib.style import library import uproot From 0519abff0df9fc3fec731150d5ecac73455a1f19 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Thu, 7 Jul 2022 08:55:06 -0500 Subject: [PATCH 07/41] vectors of arbitrary lengths work --- src/uproot/_awkward_forth.py | 26 +++++++++ src/uproot/containers.py | 80 ++++++++++++++++++++++++++-- src/uproot/interpretation/objects.py | 1 + src/uproot/models/TObject.py | 1 - src/uproot/streamers.py | 1 + 5 files changed, 103 insertions(+), 6 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index f2e53f2f1..3832cc1fc 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -4,6 +4,15 @@ This module defines utilities for adding components to the forth reader. """ +import numpy as np + +symbol_dict = { + np.dtype(">f4"): "f", + np.dtype(">f8"): "d", + np.dtype(">i8"): "q", + np.dtype(">i4"): "i", +} + class ForthGenerator: """ @@ -32,6 +41,17 @@ def __init__(self, aform=None, count_obj=0, var_set=False): def traverse_aform(self): self.aform = self.aform.content + def replace_form_and_model(self, form, model): + temp_node = self.awkward_model + temp_node_top = self._prev_node + self.awkward_model = model + self._prev_node = self.awkward_model + temp_form = self.aform + temp_form_top = self.top_form + self.top_form = None + self.aform = form + return temp_node, temp_node_top, temp_form, temp_form_top + def should_add_form(self): if "content" in self.awkward_model.keys(): if self.awkward_model["content"] is None: @@ -259,3 +279,9 @@ def convert_dtype(format): """ if format == "d": return "float64" + elif format == "f": + return "float32" + elif format == "q": + return "int64" + elif format == "i": + return "int32" diff --git a/src/uproot/containers.py b/src/uproot/containers.py index 34537efe4..420f26ef8 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -48,11 +48,45 @@ def _content_cache_key(content): def _read_nested( model, length, chunk, cursor, context, file, selffile, parent, header=True ): + helper_obj = uproot._awkward_forth.GenHelper(context) + + if helper_obj.is_forth(): + forth_obj = helper_obj.get_gen_obj() + if isinstance(model, numpy.dtype): + symbol = uproot._awkward_forth.symbol_dict.get(model) + if symbol is None and helper_obj.is_forth(): + raise TypeError("Cannot be awkward") + if helper_obj.is_forth(): + key = forth_obj.get_keys(1) + form_key = f"part0-node{key}-data" + helper_obj.add_to_header( + f"output part0-node{key}-data {uproot._awkward_forth.convert_dtype(symbol)}\n" + ) + helper_obj.add_to_pre(f"stream #!{symbol}-> part0-node{key}-data\n") + if forth_obj.should_add_form(): + forth_obj.add_form_key(form_key) + forth_obj.add_form( + { + "class": "NumpyArray", + "primitive": f"{uproot._awkward_forth.convert_dtype(symbol)}", + "form_key": f"node{key}", + } + ) + temp = forth_obj.add_node( + f"node{key}", + helper_obj.get_pre(), + helper_obj.get_post(), + helper_obj.get_init(), + helper_obj.get_header(), + "i64", + 1, + None, + ) + forth_obj.go_to(temp) return cursor.array(chunk, length, model, context) else: - values = numpy.empty(length, dtype=_stl_object_type) if isinstance(model, AsContainer): for i in range(length): @@ -1079,9 +1113,9 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): helper_obj.add_to_header(f"output part0-node{key}-offsets int64\n") helper_obj.add_to_init(f"0 part0-node{key}-offsets <- stack\n") helper_obj.add_to_pre( - f"stream !I-> stack\ndup part0-node{key}-offsets +<- stack\n0 do \n" + f"stream !I-> stack\ndup part0-node{key}-offsets +<- stack\n" ) - helper_obj.add_to_post("loop\n") + # helper_obj.add_to_post("loop\n") if forth_obj.should_add_form(): forth_obj.add_form_key(form_key) if self._header: @@ -1090,6 +1124,9 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): temp_bool = "false" temp_aform = f'{{ "class":"ListOffsetArray", "offsets":"i64", "content": "NULL", "has_identifier": false, "parameters": {{"uproot": {{"as": "vector", "header": {temp_bool}}}}}, "form_key": "node{key}"}}' forth_obj.add_form(json.loads(temp_aform)) + if not isinstance(self._values, numpy.dtype): + helper_obj.add_to_pre("0 do\n") + helper_obj.add_to_post("loop\n") temp = forth_obj.add_node( f"node{key}", helper_obj.get_pre(), @@ -1359,7 +1396,10 @@ def awkward_form(self, file, context): def read(self, chunk, cursor, context, file, selffile, parent, header=True): # @aryan26roy: test_0637's 00,33,35,39,47,48,66,67,68,69,70,71,72,73,74,75,76,77,78,79 + # helper_obj = uproot._awkward_forth.GenHelper(context) + # if helper_obj.is_forth(): + # forth_obj = helper_obj.get_gen_obj() if self._header and header: start_cursor = cursor.copy() ( @@ -1368,6 +1408,9 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): is_memberwise, ) = uproot.deserialization.numbytes_version(chunk, cursor, context) cursor.skip(6) + # if helper_obj.is_forth(): + # temp_jump = cursor._index - start_cursor._index + # helper_obj.add_to_pre(f"{temp_jump+6} stream skip\n") else: is_memberwise = False @@ -1375,7 +1418,10 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): length = cursor.field(chunk, _stl_container_size, context) if _has_nested_header(self._keys) and header: + # helper_obj.add_to_pre(f"6 stream skip\n") cursor.skip(6) + # if helper_obj.is_forth(): + # temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {"name": "TOP", "content": {}}) keys = _read_nested( self._keys, length, @@ -1387,8 +1433,13 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): parent, header=False, ) - + # if helper_obj.is_forth(): + # keys_form = forth_obj.top_form + # keys_model = forth_obj._prev_node + # print(keys_form, keys_model, "OOKKOKOKOk") + # temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {"name": "TOP", "content": {}}) if _has_nested_header(self._values) and header: + # keys_model["content"]["pre_code"].append(f"6 stream skip\n") cursor.skip(6) values = _read_nested( self._values, @@ -1401,7 +1452,26 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): parent, header=False, ) - + # if helper_obj.is_forth(): + # values_form = forth_obj.top_form + # values_model = forth_obj._prev_node + # forth_obj.awkward_model = temp_node + # forth_obj._prev_node = temp_node_top + # forth_obj.aform = temp_form + # forth_obj.top_form = temp_form_top + # aform = {"class": "RecordArray", "contents": {"key": keys_form, "value": values_form}} + # if forth_obj.should_add_form(): + # forth_obj.add_form(aform) + # temp = forth_obj.add_node( + # f"nodeMap", + # helper_obj.get_pre(), + # helper_obj.get_post(), + # helper_obj.get_init(), + # helper_obj.get_header(), + # "i64", + # 1, + # [keys_model, values_model], + # ) out = STLMap(keys, values) if self._header and header: diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index be506ddf2..2c3a6b48a 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -306,6 +306,7 @@ def basket_array_forth( return output def assemble_forth(self, forth_obj, awkward_model): + # print(awkward_model) forth_obj.add_to_header(awkward_model["header_code"]) forth_obj.add_to_init(awkward_model["init_code"]) forth_obj.add_to_final(awkward_model["pre_code"]) diff --git a/src/uproot/models/TObject.py b/src/uproot/models/TObject.py index d14a4f6fa..d56523343 100644 --- a/src/uproot/models/TObject.py +++ b/src/uproot/models/TObject.py @@ -25,7 +25,6 @@ def read_numbytes_version(self, chunk, cursor, context): pass def read_members(self, chunk, cursor, context, file): - # raise NotImplementedError helper_obj = uproot._awkward_forth.GenHelper(context) start_index = cursor._index if helper_obj.is_forth(): diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index d8e9f44ad..9e826e20f 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -908,6 +908,7 @@ def class_code( formats[-1].append(_ftype_to_struct(self.fType)) formats_memberwise.append(_ftype_to_struct(self.fType)) + # print("AHAHAHA") if ( i + 1 == len(elements) or not isinstance(elements[i + 1], Model_TStreamerBasicType) From 736b33b3a0415fe2f4cd0159890bd4dee5a6aa47 Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Thu, 7 Jul 2022 13:40:41 -0500 Subject: [PATCH 08/41] Turn off fail-fast to learn more about this error. --- .github/workflows/build-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 46a1e076a..1c553c14e 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -19,6 +19,7 @@ jobs: python-version: "3.6" - platform: "windows-latest" python-version: "3.7" + fail-fast: false runs-on: "${{ matrix.platform }}" From 972bdddfd94eaa723c1849abaec01ace9921309d Mon Sep 17 00:00:00 2001 From: Jim Pivarski Date: Thu, 7 Jul 2022 18:09:51 -0500 Subject: [PATCH 09/41] Made the Forth control flow thread-safe. --- src/uproot/_awkward_forth.py | 4 +- src/uproot/behaviors/TBranch.py | 8 +- src/uproot/interpretation/objects.py | 207 ++++++++++-------- src/uproot/model.py | 2 +- ...ward-form.py => test_0610-awkward-form.py} | 0 5 files changed, 124 insertions(+), 97 deletions(-) rename tests/{test_610-awkward-form.py => test_0610-awkward-form.py} (100%) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index 3832cc1fc..d2df8b490 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -233,9 +233,9 @@ def __init__(self, context): self._header = "" self._init = "" self._form_key = [] - if "forth" in context.keys(): + if hasattr(context.get("forth"), "gen"): self.forth_present = True - self._gen_obj = context["forth"] + self._gen_obj = context["forth"].gen def is_forth(self): return self.forth_present diff --git a/src/uproot/behaviors/TBranch.py b/src/uproot/behaviors/TBranch.py index 16ca07e45..e6fd3bef1 100644 --- a/src/uproot/behaviors/TBranch.py +++ b/src/uproot/behaviors/TBranch.py @@ -3244,6 +3244,9 @@ def chunk_to_basket(chunk, branch, basket_num): else: notifications.put(basket) + # all threads (if multithreaded) share a thread-local context for Forth + forth_context = threading.local() + def basket_to_array(basket): try: assert basket.basket_num is not None @@ -3251,12 +3254,15 @@ def basket_to_array(basket): interpretation = branchid_interpretation[branch.cache_key] basket_arrays = branchid_arrays[branch.cache_key] + context = dict(branch.context) + context["forth"] = forth_context + basket_arrays[basket.basket_num] = interpretation.basket_array( basket.data, basket.byte_offsets, basket, branch, - branch.context, + context, basket.member("fKeylen"), library, ) diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index 2c3a6b48a..f649fa3bd 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -64,11 +64,10 @@ class AsObjects(uproot.interpretation.Interpretation): def __init__(self, model, branch=None): self._model = model self._branch = branch - self._prereaddone = False self._forth = False - self._forth_vm = threading.local() - self._complete_forth_code = [] - self._forth_vm_set = False + self._forth_lock = threading.Lock() + self._forth_form_keys = None + self._complete_forth_code = None self._form = None @property @@ -202,96 +201,74 @@ def basket_array_forth( ) assert basket.byte_offsets is not None - output = None - if "forth" not in context.keys(): - forth_gen = uproot._awkward_forth.ForthGenerator() - context["forth"] = forth_gen + if not hasattr(context["forth"], "vm"): + # context["forth"] is a threading.local() + context["forth"].vm = None + context["forth"].gen = uproot._awkward_forth.ForthGenerator() - if isinstance(library, uproot.interpretation.library.Awkward): - form = self.awkward_form( - branch.file, - { - "index_format": "i64", - "header": False, - "tobject_header": True, - "breadcrumbs": (), - }, - ) - if awkward_can_optimize(self, form): - import awkward._connect._uproot - - extra = { - "interpretation": self, - "basket": basket, - "branch": branch, - "context": context, - "cursor_offset": cursor_offset, - } - output = awkward._connect._uproot.basket_array( - form, data, byte_offsets, extra - ) - if not self._forth_vm_set: - if not self._prereaddone: - if output is None: - output = numpy.empty( - len(byte_offsets) - 1, dtype=numpy.dtype(object) + if self._complete_forth_code is None: + with self._forth_lock: + # all threads have to wait until complete_forth_code is ready + + if self._complete_forth_code is None: + # another thread didn't make it while this thread waited + # this thread tries to make it now + output = self._discover_forth( + data, byte_offsets, branch, context, cursor_offset ) - for i in range(len(byte_offsets) - 1): - byte_start = byte_offsets[i] - byte_stop = byte_offsets[i + 1] - temp_data = data[byte_start:byte_stop] - chunk = uproot.source.chunk.Chunk.wrap( - branch.file.source, temp_data - ) - cursor = uproot.source.cursor.Cursor( - 0, origin=-(byte_start + cursor_offset) - ) - context["forth"].var_set = False - output[i] = self._model.read( - chunk, - cursor, - context, - branch.file, - branch.file.detached, - branch, + + if output is not None: + # Forth discovery was unsuccessful; return Python-derived + # output and maybe another basket will be more fruitful + self.hook_after_basket_array( + data=data, + byte_offsets=byte_offsets, + basket=basket, + branch=branch, + context=context, + output=output, + cursor_offset=cursor_offset, + library=library, ) - if not context["forth"].var_set: - self._prereaddone = True - self.assemble_forth( - context["forth"], context["forth"]._prev_node["content"] - ) - self._complete_forth_code = f'input stream\ninput byteoffsets\ninput bytestops\n{"".join(context["forth"].final_header)}\n{"".join(context["forth"].final_init)}\n0 do\nbyteoffsets I-> stack\nstream seek \n{"".join(context["forth"].final_code)}\nloop' - self._forth_vm.vm = awkward.forth.ForthMachine64( - self._complete_forth_code - ) - self._form = context["forth"].top_form - self._forth_vm_set = True - break - else: - self._forth_vm.vm = awkward.forth.ForthMachine64( + return output + + # if we didn't return already, either this thread generated the + # complete_forth_code or another did + assert self._complete_forth_code is not None + + if context["forth"].vm is None: + try: + context["forth"].vm = awkward.forth.ForthMachine64( self._complete_forth_code ) - self._forth_vm_set = True - if self._forth_vm_set: - byte_start = byte_offsets[0] - byte_stop = byte_offsets[-1] - temp_data = data[byte_start:byte_stop] - self._forth_vm.vm.begin( - { - "stream": numpy.array(temp_data), - "byteoffsets": numpy.array(byte_offsets[:-1]), - "bytestops": numpy.array(byte_offsets[1:]), - } - ) - self._forth_vm.vm.stack_push(len(byte_offsets) - 1) - self._forth_vm.vm.resume() - container = {} - for elem in context["forth"].form_keys: - if "offsets" in elem: - container[elem] = self._forth_vm.vm.output_Index64(elem) - else: - container[elem] = self._forth_vm.vm.output_NumpyArray(elem) - output = awkward.from_buffers(self._form, len(byte_offsets) - 1, container) + except Exception as err: + raise type(err)( + str(err) + + "\n\nForth code generated for this data type:\n\n" + + self._complete_forth_code + ) from err + + # get data using Forth + byte_start = byte_offsets[0] + byte_stop = byte_offsets[-1] + temp_data = data[byte_start:byte_stop] + context["forth"].vm.begin( + { + "stream": numpy.array(temp_data), + "byteoffsets": numpy.array(byte_offsets[:-1]), + "bytestops": numpy.array(byte_offsets[1:]), + } + ) + context["forth"].vm.stack_push(len(byte_offsets) - 1) + context["forth"].vm.resume() + container = {} + for elem in self._forth_form_keys: + if "offsets" in elem: + container[elem] = context["forth"].vm.output_Index64(elem) + else: + container[elem] = context["forth"].vm.output_NumpyArray(elem) + output = awkward.from_buffers(self._form, len(byte_offsets) - 1, container) + self.hook_after_basket_array( data=data, byte_offsets=byte_offsets, @@ -305,8 +282,52 @@ def basket_array_forth( return output - def assemble_forth(self, forth_obj, awkward_model): - # print(awkward_model) + def _discover_forth(self, data, byte_offsets, branch, context, cursor_offset): + output = numpy.empty(len(byte_offsets) - 1, dtype=numpy.dtype(object)) + + for i in range(len(byte_offsets) - 1): + byte_start = byte_offsets[i] + byte_stop = byte_offsets[i + 1] + temp_data = data[byte_start:byte_stop] + chunk = uproot.source.chunk.Chunk.wrap(branch.file.source, temp_data) + cursor = uproot.source.cursor.Cursor( + 0, origin=-(byte_start + cursor_offset) + ) + context["forth"].gen.var_set = False + output[i] = self._model.read( + chunk, + cursor, + context, + branch.file, + branch.file.detached, + branch, + ) + + if not context["forth"].gen.var_set: + context["forth"].prereaddone = True + self._assemble_forth( + context["forth"].gen, context["forth"].gen._prev_node["content"] + ) + + self._complete_forth_code = f"""input stream +input byteoffsets +input bytestops +{"".join(context["forth"].gen.final_header)} +{"".join(context["forth"].gen.final_init)} +0 do +byteoffsets I-> stack +stream seek +{"".join(context["forth"].gen.final_code)} +loop +""" + self._forth_form_keys = tuple(context["forth"].gen.form_keys) + self._form = context["forth"].gen.top_form + + return None # we should re-read all the data with Forth + + return output # Forth-generation was unsuccessful: this is Python output + + def _assemble_forth(self, forth_obj, awkward_model): forth_obj.add_to_header(awkward_model["header_code"]) forth_obj.add_to_init(awkward_model["init_code"]) forth_obj.add_to_final(awkward_model["pre_code"]) @@ -314,9 +335,9 @@ def assemble_forth(self, forth_obj, awkward_model): temp_content = awkward_model["content"] if isinstance(temp_content, list): for elem in temp_content: - self.assemble_forth(forth_obj, elem) + self._assemble_forth(forth_obj, elem) elif isinstance(temp_content, dict): - self.assemble_forth(forth_obj, awkward_model["content"]) + self._assemble_forth(forth_obj, awkward_model["content"]) else: pass forth_obj.add_to_final(awkward_model["post_code"]) diff --git a/src/uproot/model.py b/src/uproot/model.py index 1c447f6ac..b733cbc62 100644 --- a/src/uproot/model.py +++ b/src/uproot/model.py @@ -766,7 +766,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): self.hook_before_read(chunk=chunk, cursor=cursor, context=context, file=file) helper_obj = uproot._awkward_forth.GenHelper(context) if helper_obj.is_forth(): - forth_obj = context["forth"] + forth_obj = context["forth"].gen if context.get("reading", True): temp_index = cursor._index diff --git a/tests/test_610-awkward-form.py b/tests/test_0610-awkward-form.py similarity index 100% rename from tests/test_610-awkward-form.py rename to tests/test_0610-awkward-form.py From 081b09ed5260fbf969fd25970a8ce5e2d358474f Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Mon, 18 Jul 2022 09:11:29 -0500 Subject: [PATCH 10/41] fixed code generation bug --- src/uproot/containers.py | 70 ++++++++++++++-------------- src/uproot/interpretation/objects.py | 33 +++++++------ 2 files changed, 53 insertions(+), 50 deletions(-) diff --git a/src/uproot/containers.py b/src/uproot/containers.py index 420f26ef8..3de5d230b 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -1396,10 +1396,10 @@ def awkward_form(self, file, context): def read(self, chunk, cursor, context, file, selffile, parent, header=True): # @aryan26roy: test_0637's 00,33,35,39,47,48,66,67,68,69,70,71,72,73,74,75,76,77,78,79 - # helper_obj = uproot._awkward_forth.GenHelper(context) + helper_obj = uproot._awkward_forth.GenHelper(context) - # if helper_obj.is_forth(): - # forth_obj = helper_obj.get_gen_obj() + if helper_obj.is_forth(): + forth_obj = helper_obj.get_gen_obj() if self._header and header: start_cursor = cursor.copy() ( @@ -1408,9 +1408,9 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): is_memberwise, ) = uproot.deserialization.numbytes_version(chunk, cursor, context) cursor.skip(6) - # if helper_obj.is_forth(): - # temp_jump = cursor._index - start_cursor._index - # helper_obj.add_to_pre(f"{temp_jump+6} stream skip\n") + if helper_obj.is_forth(): + temp_jump = cursor._index - start_cursor._index + helper_obj.add_to_pre(f"{temp_jump+6} stream skip\n") else: is_memberwise = False @@ -1418,10 +1418,10 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): length = cursor.field(chunk, _stl_container_size, context) if _has_nested_header(self._keys) and header: - # helper_obj.add_to_pre(f"6 stream skip\n") + helper_obj.add_to_pre(f"6 stream skip\n") cursor.skip(6) - # if helper_obj.is_forth(): - # temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {"name": "TOP", "content": {}}) + if helper_obj.is_forth(): + temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {"name": "TOP", "content": {}}) keys = _read_nested( self._keys, length, @@ -1433,13 +1433,13 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): parent, header=False, ) - # if helper_obj.is_forth(): - # keys_form = forth_obj.top_form - # keys_model = forth_obj._prev_node - # print(keys_form, keys_model, "OOKKOKOKOk") - # temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {"name": "TOP", "content": {}}) + if helper_obj.is_forth(): + keys_form = forth_obj.top_form + keys_model = forth_obj._prev_node + print(keys_form, keys_model, "OOKKOKOKOk") + temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {"name": "TOP", "content": {}}) if _has_nested_header(self._values) and header: - # keys_model["content"]["pre_code"].append(f"6 stream skip\n") + keys_model["content"]["post_code"].append(f"6 stream skip\n") cursor.skip(6) values = _read_nested( self._values, @@ -1452,26 +1452,26 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): parent, header=False, ) - # if helper_obj.is_forth(): - # values_form = forth_obj.top_form - # values_model = forth_obj._prev_node - # forth_obj.awkward_model = temp_node - # forth_obj._prev_node = temp_node_top - # forth_obj.aform = temp_form - # forth_obj.top_form = temp_form_top - # aform = {"class": "RecordArray", "contents": {"key": keys_form, "value": values_form}} - # if forth_obj.should_add_form(): - # forth_obj.add_form(aform) - # temp = forth_obj.add_node( - # f"nodeMap", - # helper_obj.get_pre(), - # helper_obj.get_post(), - # helper_obj.get_init(), - # helper_obj.get_header(), - # "i64", - # 1, - # [keys_model, values_model], - # ) + if helper_obj.is_forth(): + values_form = forth_obj.top_form + values_model = forth_obj._prev_node + forth_obj.awkward_model = temp_node + forth_obj._prev_node = temp_node_top + forth_obj.aform = temp_form + forth_obj.top_form = temp_form_top + aform = {"class": "RecordArray", "contents": {"key": keys_form, "value": values_form}} + if forth_obj.should_add_form(): + forth_obj.add_form(aform) + temp = forth_obj.add_node( + f"nodeMap", + helper_obj.get_pre(), + helper_obj.get_post(), + helper_obj.get_init(), + helper_obj.get_header(), + "i64", + 1, + [keys_model, values_model], + ) out = STLMap(keys, values) if self._header and header: diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index 2c3a6b48a..de26b7b17 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -285,7 +285,10 @@ def basket_array_forth( ) self._forth_vm.vm.stack_push(len(byte_offsets) - 1) self._forth_vm.vm.resume() + print(self._forth_vm.vm.outputs) + print("==================================================") container = {} + print(self._form, "OPPOPO") for elem in context["forth"].form_keys: if "offsets" in elem: container[elem] = self._forth_vm.vm.output_Index64(elem) @@ -540,7 +543,7 @@ def __init__(self, model, members, original=None): self._original = original super().__init__(_unravel_members(members)) - @property + @ property def model(self): """ The full Uproot deserialization model for the data @@ -548,7 +551,7 @@ def model(self): """ return self._model - @property + @ property def members(self): """ The name (str) and fixed-width @@ -557,7 +560,7 @@ def members(self): """ return self._members - @property + @ property def original(self): """ If not None, this was the original @@ -578,7 +581,7 @@ def __repr__(self): def __eq__(self, other): return isinstance(other, AsStridedObjects) and self._model == other._model - @property + @ property def numpy_dtype(self): return numpy.dtype(object) @@ -601,11 +604,11 @@ def awkward_form( form = awkward.forms.RegularForm(form, dim) return form - @property + @ property def cache_key(self): return f"{type(self).__name__}({self._model.__name__})" - @property + @ property def typename(self): return uproot.model.classname_decode(self._model.__name__)[0] @@ -681,7 +684,7 @@ def __repr__(self): self._cursor_offset, ) - @property + @ property def model(self): """ The full Uproot deserialization model for the data @@ -689,28 +692,28 @@ def model(self): """ return self._model - @property + @ property def branch(self): """ The ``TBranch`` from which the data are drawn. """ return self._branch - @property + @ property def context(self): """ Auxiliary data used in deserialization (dict). """ return self._context - @property + @ property def byte_offsets(self): """ Index where each entry of the ``byte_content`` starts and stops. """ return self._byte_offsets - @property + @ property def byte_content(self): """ Raw but uncompressed data, directly from @@ -718,7 +721,7 @@ def byte_content(self): """ return self._byte_content - @property + @ property def cursor_offset(self): """ Correction to the integer keys used in @@ -820,21 +823,21 @@ def __init__(self, interpretation, array): "non-numpy temporary arrays require changes in library interpretations" ) - @property + @ property def interpretation(self): """ The interpretation that produced this array. """ return self._interpretation - @property + @ property def array(self): """ Underlying array object, which may be NumPy or another temporary array. """ return self._array - @property + @ property def shape(self): assert self._array.shape == (len(self),) + self._interpretation.inner_shape return self._array.shape From 92b0b08552fa6f5797fb8b373ee8b660388e9e58 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Mon, 18 Jul 2022 13:50:52 -0500 Subject: [PATCH 11/41] STL MAP works now! --- src/uproot/containers.py | 46 +++++++++++++------ src/uproot/model.py | 1 - src/uproot/streamers.py | 2 - .../test_0637-setup-tests-for-AwkwardForth.py | 4 +- 4 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/uproot/containers.py b/src/uproot/containers.py index 3de5d230b..fb59f1603 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -89,12 +89,20 @@ def _read_nested( else: values = numpy.empty(length, dtype=_stl_object_type) if isinstance(model, AsContainer): + if helper_obj.is_forth(): + temp_count = context["forth"].gen.count_obj for i in range(length): + if helper_obj.is_forth(): + context["forth"].gen.count_obj = temp_count values[i] = model.read( chunk, cursor, context, file, selffile, parent, header=header ) else: + if helper_obj.is_forth(): + temp_count = context["forth"].gen.count_obj for i in range(length): + if helper_obj.is_forth(): + context["forth"].gen.count_obj = temp_count values[i] = model.read(chunk, cursor, context, file, selffile, parent) return values @@ -469,7 +477,6 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): keys = forth_obj.get_keys(2) offsets_num = keys[0] data_num = keys[1] - if self._header and header: start_cursor = cursor.copy() ( @@ -480,13 +487,13 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if helper_obj.is_forth(): temp_jump = cursor._index - start_cursor._index if temp_jump != 0: - helper_obj.add_to_pre(f"{temp_jump} stream skip") + helper_obj.add_to_pre(f"{temp_jump} stream skip\n") if self._length_bytes == "1-5": out = cursor.string(chunk, context) if helper_obj.is_forth(): helper_obj.add_to_pre( - f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup part0-node{offsets_num}-offsets +<- stack stream #!B-> part0-node{data_num}-data \n" + f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup part0-node{offsets_num}-offsets +<- stack stream #!B-> part0-node{data_num}-data\n" ) elif self._length_bytes == "4": length = cursor.field(chunk, _stl_container_size, context) @@ -818,7 +825,6 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): ) if helper_obj.is_forth(): forth_obj.go_to(temp["content"]) - # print(forth_obj._prev_node, "UOUOUO") # if helper_obj.is_forth(): # forth_obj.go_to(temp) @@ -1410,16 +1416,24 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): cursor.skip(6) if helper_obj.is_forth(): temp_jump = cursor._index - start_cursor._index - helper_obj.add_to_pre(f"{temp_jump+6} stream skip\n") + helper_obj.add_to_pre(f"{temp_jump} stream skip\n") else: is_memberwise = False - + #raise NotImplementedError if is_memberwise: length = cursor.field(chunk, _stl_container_size, context) - + if helper_obj.is_forth(): + key = forth_obj.get_keys(1) + form_key = f"part0-node{key}-offsets" + helper_obj.add_to_header(f"output part0-node{key}-offsets int64\n") + helper_obj.add_to_init(f"0 part0-node{key}-offsets <- stack\n") + helper_obj.add_to_pre(f"stream !I-> stack\n dup part0-node{key}-offsets +<- stack\n") if _has_nested_header(self._keys) and header: - helper_obj.add_to_pre(f"6 stream skip\n") + if helper_obj.is_forth(): + helper_obj.add_to_pre("6 stream skip\n") cursor.skip(6) + if helper_obj.is_forth(): + helper_obj.add_to_pre("dup 0 do\n") if helper_obj.is_forth(): temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {"name": "TOP", "content": {}}) keys = _read_nested( @@ -1436,11 +1450,15 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if helper_obj.is_forth(): keys_form = forth_obj.top_form keys_model = forth_obj._prev_node - print(keys_form, keys_model, "OOKKOKOKOk") - temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {"name": "TOP", "content": {}}) + temp_node1, temp_node_top1, temp_form1, temp_form_top1 = forth_obj.replace_form_and_model(None, {"name": "TOP", "content": {}}) + if helper_obj.is_forth(): + keys_model["content"]["post_code"].append("loop\n") if _has_nested_header(self._values) and header: - keys_model["content"]["post_code"].append(f"6 stream skip\n") cursor.skip(6) + if helper_obj.is_forth(): + keys_model["content"]["post_code"].append("6 stream skip\n") + if helper_obj.is_forth(): + keys_model["content"]["post_code"].append("0 do\n") values = _read_nested( self._values, length, @@ -1455,12 +1473,14 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if helper_obj.is_forth(): values_form = forth_obj.top_form values_model = forth_obj._prev_node + values_model["content"]["post_code"].append("loop \n") forth_obj.awkward_model = temp_node forth_obj._prev_node = temp_node_top forth_obj.aform = temp_form forth_obj.top_form = temp_form_top - aform = {"class": "RecordArray", "contents": {"key": keys_form, "value": values_form}} + aform = {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "RecordArray", "contents": [keys_form, values_form], "parameters": {"__array__": "sorted_map"}, }, "form_key": f"node{key}"} if forth_obj.should_add_form(): + forth_obj.add_form_key(form_key) forth_obj.add_form(aform) temp = forth_obj.add_node( f"nodeMap", @@ -1470,7 +1490,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): helper_obj.get_header(), "i64", 1, - [keys_model, values_model], + [keys_model["content"], values_model["content"]], ) out = STLMap(keys, values) diff --git a/src/uproot/model.py b/src/uproot/model.py index b733cbc62..82ce7d1bc 100644 --- a/src/uproot/model.py +++ b/src/uproot/model.py @@ -1341,7 +1341,6 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): # versioned_cls.read starts with numbytes_version again because move=False (above) # if helper_obj.is_forth(): - # print(versioned_cls) temp_var = cls.postprocess( versioned_cls.read( chunk, cursor, context, file, selffile, parent, concrete=concrete diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index 9e826e20f..2b18656d2 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -907,8 +907,6 @@ def class_code( fields[-1].append(self.name) formats[-1].append(_ftype_to_struct(self.fType)) formats_memberwise.append(_ftype_to_struct(self.fType)) - - # print("AHAHAHA") if ( i + 1 == len(elements) or not isinstance(elements[i + 1], Model_TStreamerBasicType) diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index 88bd486ae..83ed3cc74 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -13,7 +13,9 @@ def test_00(): with uproot.open(skhep_testdata.data_path("issue367b.root")) as file: branch = file["tree/weights"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, entry_stop=2) + assert py[0]['0'][0] == 'expskin_FluxUnisim' # py[-1] == From 7370cca6406752135833c39ac7f7d18a83ec4307 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Tue, 19 Jul 2022 14:43:12 -0500 Subject: [PATCH 12/41] TLorentzVector works! --- src/uproot/_awkward_forth.py | 20 ++++++-- src/uproot/containers.py | 35 ++++++++++++-- src/uproot/interpretation/objects.py | 31 ++++++------ src/uproot/model.py | 2 + src/uproot/models/TObject.py | 2 +- src/uproot/streamers.py | 47 +++++++++++++------ .../test_0637-setup-tests-for-AwkwardForth.py | 7 ++- 7 files changed, 103 insertions(+), 41 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index d2df8b490..4a898d47f 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -62,10 +62,10 @@ def should_add_form(self): def get_temp_form_top(self): return self.top_dummy - def set_dummy_none(self): - self.top_dummy = None - self.dummy_aform = None - self.dummy_form = False + def set_dummy_none(self, temp_top, temp_form, temp_flag): + self.top_dummy = temp_top + self.dummy_aform = temp_form + self.dummy_form = temp_flag def add_form(self, aform, conlen=0): if self.dummy_form: @@ -138,6 +138,18 @@ def get_current_node(self): def get_ref(self, index): return self.ref_list[index] + def enable_adding(self): + if "content" in self.awkward_model.keys(): + if self.awkward_model["content"] == None: + self.awkward_model["content"] = {} + + def add_node_whole(self, new_node, ref_latest): + if "content" in self.awkward_model.keys(): + self.awkward_model["content"] = new_node + else: + self.awkward_model["contents"].append(new_node) + self.awkward_model = ref_latest + def add_node( self, name, precode, postcode, initcode, headercode, dtype, num_child, content ): diff --git a/src/uproot/containers.py b/src/uproot/containers.py index fb59f1603..c831c60a2 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -1419,7 +1419,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): helper_obj.add_to_pre(f"{temp_jump} stream skip\n") else: is_memberwise = False - #raise NotImplementedError + # raise NotImplementedError if is_memberwise: length = cursor.field(chunk, _stl_container_size, context) if helper_obj.is_forth(): @@ -1427,7 +1427,9 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): form_key = f"part0-node{key}-offsets" helper_obj.add_to_header(f"output part0-node{key}-offsets int64\n") helper_obj.add_to_init(f"0 part0-node{key}-offsets <- stack\n") - helper_obj.add_to_pre(f"stream !I-> stack\n dup part0-node{key}-offsets +<- stack\n") + helper_obj.add_to_pre( + f"stream !I-> stack\n dup part0-node{key}-offsets +<- stack\n" + ) if _has_nested_header(self._keys) and header: if helper_obj.is_forth(): helper_obj.add_to_pre("6 stream skip\n") @@ -1435,7 +1437,14 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if helper_obj.is_forth(): helper_obj.add_to_pre("dup 0 do\n") if helper_obj.is_forth(): - temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {"name": "TOP", "content": {}}) + ( + temp_node, + temp_node_top, + temp_form, + temp_form_top, + ) = forth_obj.replace_form_and_model( + None, {"name": "TOP", "content": {}} + ) keys = _read_nested( self._keys, length, @@ -1450,7 +1459,14 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if helper_obj.is_forth(): keys_form = forth_obj.top_form keys_model = forth_obj._prev_node - temp_node1, temp_node_top1, temp_form1, temp_form_top1 = forth_obj.replace_form_and_model(None, {"name": "TOP", "content": {}}) + ( + temp_node1, + temp_node_top1, + temp_form1, + temp_form_top1, + ) = forth_obj.replace_form_and_model( + None, {"name": "TOP", "content": {}} + ) if helper_obj.is_forth(): keys_model["content"]["post_code"].append("loop\n") if _has_nested_header(self._values) and header: @@ -1478,7 +1494,16 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): forth_obj._prev_node = temp_node_top forth_obj.aform = temp_form forth_obj.top_form = temp_form_top - aform = {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "RecordArray", "contents": [keys_form, values_form], "parameters": {"__array__": "sorted_map"}, }, "form_key": f"node{key}"} + aform = { + "class": "ListOffsetArray", + "offsets": "i64", + "content": { + "class": "RecordArray", + "contents": [keys_form, values_form], + "parameters": {"__array__": "sorted_map"}, + }, + "form_key": f"node{key}", + } if forth_obj.should_add_form(): forth_obj.add_form_key(form_key) forth_obj.add_form(aform) diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index 14d5bed5a..002d87a6d 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -267,6 +267,7 @@ def basket_array_forth( container[elem] = context["forth"].vm.output_Index64(elem) else: container[elem] = context["forth"].vm.output_NumpyArray(elem) + print(self._form) output = awkward.from_buffers(self._form, len(byte_offsets) - 1, container) self.hook_after_basket_array( @@ -561,7 +562,7 @@ def __init__(self, model, members, original=None): self._original = original super().__init__(_unravel_members(members)) - @ property + @property def model(self): """ The full Uproot deserialization model for the data @@ -569,7 +570,7 @@ def model(self): """ return self._model - @ property + @property def members(self): """ The name (str) and fixed-width @@ -578,7 +579,7 @@ def members(self): """ return self._members - @ property + @property def original(self): """ If not None, this was the original @@ -599,7 +600,7 @@ def __repr__(self): def __eq__(self, other): return isinstance(other, AsStridedObjects) and self._model == other._model - @ property + @property def numpy_dtype(self): return numpy.dtype(object) @@ -622,11 +623,11 @@ def awkward_form( form = awkward.forms.RegularForm(form, dim) return form - @ property + @property def cache_key(self): return f"{type(self).__name__}({self._model.__name__})" - @ property + @property def typename(self): return uproot.model.classname_decode(self._model.__name__)[0] @@ -702,7 +703,7 @@ def __repr__(self): self._cursor_offset, ) - @ property + @property def model(self): """ The full Uproot deserialization model for the data @@ -710,28 +711,28 @@ def model(self): """ return self._model - @ property + @property def branch(self): """ The ``TBranch`` from which the data are drawn. """ return self._branch - @ property + @property def context(self): """ Auxiliary data used in deserialization (dict). """ return self._context - @ property + @property def byte_offsets(self): """ Index where each entry of the ``byte_content`` starts and stops. """ return self._byte_offsets - @ property + @property def byte_content(self): """ Raw but uncompressed data, directly from @@ -739,7 +740,7 @@ def byte_content(self): """ return self._byte_content - @ property + @property def cursor_offset(self): """ Correction to the integer keys used in @@ -841,21 +842,21 @@ def __init__(self, interpretation, array): "non-numpy temporary arrays require changes in library interpretations" ) - @ property + @property def interpretation(self): """ The interpretation that produced this array. """ return self._interpretation - @ property + @property def array(self): """ Underlying array object, which may be NumPy or another temporary array. """ return self._array - @ property + @property def shape(self): assert self._array.shape == (len(self),) + self._interpretation.inner_shape return self._array.shape diff --git a/src/uproot/model.py b/src/uproot/model.py index 82ce7d1bc..7d29d2d24 100644 --- a/src/uproot/model.py +++ b/src/uproot/model.py @@ -824,6 +824,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): chunk=chunk, cursor=cursor, context=context, file=file ) if helper_obj.is_forth(): + #raise NotImplementedError forth_obj.add_node( "model828", helper_obj.get_pre(), @@ -1308,6 +1309,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): versioned_cls = cls.class_of_version(version) bytes_skipped = cursor._index - start_index if helper_obj.is_forth(): + #raise NotImplementedError helper_obj.add_to_pre(f"{bytes_skipped} stream skip \n") forth_obj.add_node( "Model1319", diff --git a/src/uproot/models/TObject.py b/src/uproot/models/TObject.py index d56523343..739d9dce4 100644 --- a/src/uproot/models/TObject.py +++ b/src/uproot/models/TObject.py @@ -53,7 +53,7 @@ def read_members(self, chunk, cursor, context, file): skip_length = cursor._index - start_index helper_obj.add_to_pre(f"{skip_length} stream skip \n") if forth_obj.should_add_form(): - temp_aform = '{"class": "RecordArray", "contents":{}, "parameters": {"__record__": "TObject"}}' + temp_aform = '{"class": "RecordArray", "contents":[], "parameters": {"__record__": "TObject"}}' forth_obj.add_form(json.loads(temp_aform)) forth_obj.add_node( "TObjext", diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index 2b18656d2..10819f14e 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -273,7 +273,7 @@ def class_code(self): class_flags, ) read_members.append( - " if helper_obj.is_forth():\n if forth_obj.should_add_form():\n forth_obj.add_form({'class': 'RecordArray', 'contents': content, 'parameters': {'record': 'TVector2'}})\n temp = forth_obj.add_node(f\"node{{forth_obj.get_keys()}}\",helper_obj.get_pre(),helper_obj.get_post(),helper_obj.get_init(),helper_obj.get_header(),\"i64\",0,None)" + " if helper_obj.is_forth():\n if forth_obj.should_add_form():\n forth_obj.add_form({'class': 'RecordArray', 'contents': content, 'parameters': {'record': 'TVector2'}})\n temp = forth_obj.add_node('dynamic',helper_obj.get_pre(),helper_obj.get_post(),helper_obj.get_init(),helper_obj.get_header(),\"i64\",0,None)" ) if len(read_members) == 1: # untested as of PR #629 @@ -684,15 +684,18 @@ def class_code( read_member_n.append(f" if member_index == {i}:") # @aryan26roy: test_0637's 01,02,08,09,11,12,13,15,16,29,38,45,46,49,50 + # raise NotImplementedError read_members.append( - " if helper_obj.is_forth():\n forth_obj.dummy_form = True" + " if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {'name': 'TOP', 'content': {}})\n" ) read_members.append( f" self._bases.append(c({self.name!r}, {self.base_version!r}).read(chunk, cursor, context, file, self._file, self._parent, concrete=self.concrete))" ) - read_members.append( - " if helper_obj.is_forth():\n temp_form = forth_obj.get_temp_form_top()\n content.update(temp_form['contents'])\n forth_obj.set_dummy_none()\n" - ) + read_members.append(" if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" + ) + # read_members.append( + # " if helper_obj.is_forth():\n temp_form = forth_obj.get_temp_form_top()\n content.update(temp_form['contents'])\n forth_obj.set_dummy_none(temp_top_dummy, temp_dummy, temp_top_flag)\n" + # ) read_member_n.append(" " + read_members[-1]) strided_interpretation.append( f" members.extend(file.class_named({self.name!r}, {self.base_version!r}).strided_interpretation(file, header, tobject_header, breadcrumbs).members)" @@ -912,13 +915,30 @@ def class_code( or not isinstance(elements[i + 1], Model_TStreamerBasicType) or elements[i + 1].array_length != 0 ): - if len(fields[-1]) == 1: # @aryan26roy: test_0637's 01,02,29,38,44,56 - + read_members.append(" if helper_obj.is_forth():") + read_members.append(" key = forth_obj.get_keys(1)") + read_members.append( + ' form_key = f"part0-node{key}-data"' + ) + read_members.append( + f' helper_obj.add_to_header(f"output part0-node{{key}}-data {uproot._awkward_forth.convert_dtype(formats[-1][0])}\\n")' + ) + read_members.append( + f' content["{fields[-1][0]}"] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[-1][0])}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}' + ) + read_members.append( + f' helper_obj.add_to_pre(f"stream !{formats[-1][0]}-> part0-node{{key}}-data\\n")' + ) + read_members.append( + " if forth_obj.should_add_form():" + ) read_members.append( - f" self._members[{fields[-1][0]!r}] = cursor.field(chunk, self._format{len(formats) - 1}, context)" + " forth_obj.add_form_key(form_key)" ) + read_members.append(f" self._members[{fields[-1][0]!r}] = cursor.field(chunk, self._format{len(formats) - 1}, context)" + ) else: read_members.append(" if helper_obj.is_forth():") @@ -959,7 +979,6 @@ def class_code( else: # @aryan26roy: test_0637's 44,56 - read_members.append( f" self._members[{self.name!r}] = cursor.array(chunk, {self.array_length}, self._dtype{len(dtypes)}, context)" ) @@ -1091,7 +1110,7 @@ class Model_TStreamerLoop(Model_TStreamerElement): instead of creating a behavior class to mix in functionality. """ - @property + @ property def count_name(self): """ The count name (``fCountName``) of this ``TStreamerLoop``. @@ -1180,14 +1199,14 @@ class Model_TStreamerSTL(Model_TStreamerElement): instead of creating a behavior class to mix in functionality. """ - @property + @ property def stl_type(self): """ The STL type code (``fSTLtype``) of this ``TStreamerSTL``. """ return self._members["fSTLtype"] - @property + @ property def fCtype(self): """ The type code (``fCtype``) of this ``TStreamerSTL``. @@ -1458,9 +1477,9 @@ def class_code( # @aryan26roy: test_0637's 01,02,29,45,46,49,50,56 read_members.append( - f" self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)" + f" if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {{'name': 'TOP', 'content': {{}}}})\n self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)\n if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n temp_var = forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content['{self.name}'] = temp_form1\n forth_obj.enable_adding()" ) - read_member_n.append(" " + read_members[-1]) + read_member_n.append(" " + f" self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)") strided_interpretation.append( f" members.append(({self.name!r}, file.class_named({self.typename.rstrip('*')!r}, 'max').strided_interpretation(file, header, tobject_header, breadcrumbs)))" diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index 83ed3cc74..0ab878fbf 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -15,7 +15,7 @@ def test_00(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, entry_stop=2) - assert py[0]['0'][0] == 'expskin_FluxUnisim' + assert py[0]["0"][0] == "expskin_FluxUnisim" # py[-1] == @@ -23,7 +23,10 @@ def test_01(): with uproot.open(skhep_testdata.data_path("uproot-delphes-pr442.root")) as file: branch = file["Delphes/GenJet/GenJet.SoftDroppedSubJet1"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0]['1'] == pytest.approx(84.56447925448748) + assert py[0][0]['0']['fZ'] == pytest.approx(-81.600465) # py[-1] == array([, ], dtype=object) From 3e4f7c8937e08f9d6b196bc6ba00e564f714da94 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Wed, 20 Jul 2022 09:13:54 -0500 Subject: [PATCH 13/41] generalized forth generation in streamers.py --- src/uproot/_awkward_forth.py | 11 +++++--- src/uproot/containers.py | 25 +++++++++++++------ src/uproot/interpretation/objects.py | 3 +-- src/uproot/model.py | 4 +-- src/uproot/streamers.py | 23 ++++++++++------- .../test_0637-setup-tests-for-AwkwardForth.py | 9 ++++--- 6 files changed, 49 insertions(+), 26 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index 4a898d47f..f0f2ba318 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -67,7 +67,7 @@ def set_dummy_none(self, temp_top, temp_form, temp_flag): self.dummy_aform = temp_form self.dummy_form = temp_flag - def add_form(self, aform, conlen=0): + def add_form(self, aform, conlen=0, traverse=1): if self.dummy_form: if self.dummy_aform is None: self.dummy_aform = aform @@ -91,15 +91,20 @@ def add_form(self, aform, conlen=0): if self.aform is None: self.aform = aform self.top_form = self.aform + if traverse == 2: + self.aform = self.aform["content"] else: if "content" in self.aform.keys(): if self.aform["content"] == "NULL": self.aform["content"] = aform - self.aform = self.aform["content"] + if traverse == 2: + self.aform = self.aform["content"]["content"] + else: + self.aform = self.aform["content"] else: raise ValueError elif "contents" in self.aform.keys(): - if len(self.aform["content"]) == conlen: + if len(self.aform["contents"]) == conlen: return else: raise ValueError diff --git a/src/uproot/containers.py b/src/uproot/containers.py index c831c60a2..e3ecb4174 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -791,17 +791,26 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): helper_obj.add_to_pre( "0 bytestops I-> stack \nbegin\ndup stream pos <>\nwhile\nswap 1 + swap\n" ) - helper_obj.add_to_post( - f"repeat\nswap part0-node{offsets_num}-offsets +<- stack drop\n" - ) + if len(self.inner_shape) > 0: + helper_obj.add_to_post( + f"repeat\nswap {self.inner_shape[0]} / part0-node{offsets_num}-offsets +<- stack drop\n" + ) + else: + helper_obj.add_to_post( + f"repeat\nswap part0-node{offsets_num}-offsets +<- stack drop\n" + ) if forth_obj.should_add_form(): forth_obj.add_form_key(form_key) if self._header: temp_bool = "true" else: temp_bool = "false" - temp_aform = f'{{ "class":"ListOffsetArray", "offsets":"i64", "content": "NULL", "has_identifier": false, "parameters": {{"uproot": {{"as": "vector", "header": {temp_bool}}}}}, "form_key": "node{offsets_num}"}}' - forth_obj.add_form(json.loads(temp_aform)) + if len(self.inner_shape) > 0: + temp_aform = f'{{ "class":"ListOffsetArray", "offsets":"i64", "content": {{"class": "RegularArray", "content": "NULL", "size": {self.inner_shape[0]}}}, "has_identifier": false, "parameters": {{"uproot": {{"as": "vector", "header": {temp_bool}}}}}, "form_key": "node{offsets_num}"}}' + forth_obj.add_form(json.loads(temp_aform), traverse=2) + else: + temp_aform = f'{{ "class":"ListOffsetArray", "offsets":"i64", "content": "NULL", "has_identifier": false, "parameters": {{"uproot": {{"as": "vector", "header": {temp_bool}}}}}, "form_key": "node{offsets_num}"}}' + forth_obj.add_form(json.loads(temp_aform)) temp = forth_obj.add_node( f"node{offsets_num}", helper_obj.get_pre(), @@ -816,8 +825,11 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if cursor.index >= chunk.stop and helper_obj.is_forth(): forth_obj.var_set = True out = [] - + if helper_obj.is_forth(): + temp_count = forth_obj.count_obj while cursor.index < chunk.stop: + if helper_obj.is_forth(): + forth_obj.count_obj = temp_count out.append( self._values.read( chunk, cursor, context, file, selffile, parent @@ -946,7 +958,6 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): ) out = ROOTRVec(values) - if self._header and header: uproot.deserialization.numbytes_check( chunk, diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index 002d87a6d..c36ee5a44 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -267,7 +267,6 @@ def basket_array_forth( container[elem] = context["forth"].vm.output_Index64(elem) else: container[elem] = context["forth"].vm.output_NumpyArray(elem) - print(self._form) output = awkward.from_buffers(self._form, len(byte_offsets) - 1, container) self.hook_after_basket_array( @@ -303,7 +302,7 @@ def _discover_forth(self, data, byte_offsets, branch, context, cursor_offset): branch.file.detached, branch, ) - + # print(output, "ERERERER") if not context["forth"].gen.var_set: context["forth"].prereaddone = True self._assemble_forth( diff --git a/src/uproot/model.py b/src/uproot/model.py index 7d29d2d24..5e6d7c8fd 100644 --- a/src/uproot/model.py +++ b/src/uproot/model.py @@ -824,7 +824,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): chunk=chunk, cursor=cursor, context=context, file=file ) if helper_obj.is_forth(): - #raise NotImplementedError + # raise NotImplementedError forth_obj.add_node( "model828", helper_obj.get_pre(), @@ -1309,7 +1309,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): versioned_cls = cls.class_of_version(version) bytes_skipped = cursor._index - start_index if helper_obj.is_forth(): - #raise NotImplementedError + # raise NotImplementedError helper_obj.add_to_pre(f"{bytes_skipped} stream skip \n") forth_obj.add_node( "Model1319", diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index 10819f14e..dea554ebb 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -273,7 +273,7 @@ def class_code(self): class_flags, ) read_members.append( - " if helper_obj.is_forth():\n if forth_obj.should_add_form():\n forth_obj.add_form({'class': 'RecordArray', 'contents': content, 'parameters': {'record': 'TVector2'}})\n temp = forth_obj.add_node('dynamic',helper_obj.get_pre(),helper_obj.get_post(),helper_obj.get_init(),helper_obj.get_header(),\"i64\",0,None)" + " if helper_obj.is_forth():\n if forth_obj.should_add_form():\n forth_obj.add_form({'class': 'RecordArray', 'contents': content, 'parameters': {'record': 'TVector2'}}, len(content))\n temp = forth_obj.add_node('dynamic',helper_obj.get_pre(),helper_obj.get_post(),helper_obj.get_init(),helper_obj.get_header(),\"i64\",0,None)" ) if len(read_members) == 1: # untested as of PR #629 @@ -691,8 +691,9 @@ def class_code( read_members.append( f" self._bases.append(c({self.name!r}, {self.base_version!r}).read(chunk, cursor, context, file, self._file, self._parent, concrete=self.concrete))" ) - read_members.append(" if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" - ) + read_members.append( + " if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" + ) # read_members.append( # " if helper_obj.is_forth():\n temp_form = forth_obj.get_temp_form_top()\n content.update(temp_form['contents'])\n forth_obj.set_dummy_none(temp_top_dummy, temp_dummy, temp_top_flag)\n" # ) @@ -937,8 +938,9 @@ def class_code( read_members.append( " forth_obj.add_form_key(form_key)" ) - read_members.append(f" self._members[{fields[-1][0]!r}] = cursor.field(chunk, self._format{len(formats) - 1}, context)" - ) + read_members.append( + f" self._members[{fields[-1][0]!r}] = cursor.field(chunk, self._format{len(formats) - 1}, context)" + ) else: read_members.append(" if helper_obj.is_forth():") @@ -1110,7 +1112,7 @@ class Model_TStreamerLoop(Model_TStreamerElement): instead of creating a behavior class to mix in functionality. """ - @ property + @property def count_name(self): """ The count name (``fCountName``) of this ``TStreamerLoop``. @@ -1199,14 +1201,14 @@ class Model_TStreamerSTL(Model_TStreamerElement): instead of creating a behavior class to mix in functionality. """ - @ property + @property def stl_type(self): """ The STL type code (``fSTLtype``) of this ``TStreamerSTL``. """ return self._members["fSTLtype"] - @ property + @property def fCtype(self): """ The type code (``fCtype``) of this ``TStreamerSTL``. @@ -1479,7 +1481,10 @@ def class_code( read_members.append( f" if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {{'name': 'TOP', 'content': {{}}}})\n self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)\n if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n temp_var = forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content['{self.name}'] = temp_form1\n forth_obj.enable_adding()" ) - read_member_n.append(" " + f" self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)") + read_member_n.append( + " " + + f" self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)" + ) strided_interpretation.append( f" members.append(({self.name!r}, file.class_named({self.typename.rstrip('*')!r}, 'max').strided_interpretation(file, header, tobject_header, breadcrumbs)))" diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index 0ab878fbf..a72dc2c81 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -25,8 +25,8 @@ def test_01(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0]['1'] == pytest.approx(84.56447925448748) - assert py[0][0]['0']['fZ'] == pytest.approx(-81.600465) + assert py[0][0]["1"] == pytest.approx(84.56447925448748) + assert py[0][0]["0"]["fZ"] == pytest.approx(-81.600465) # py[-1] == array([, ], dtype=object) @@ -34,7 +34,10 @@ def test_02(): with uproot.open(skhep_testdata.data_path("uproot-delphes-pr442.root")) as file: branch = file["Delphes/GenJet/GenJet.TrimmedP4[5]"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak") + assert py[0][0][0]["fE"] == 0 + assert py[-1][8][3]["fE"] == 0 # py[-1] == array([[, , , , ], [, , , , ]], dtype=object) From 180b0c38c32f7c0203025189104f2fb18089fc1f Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Thu, 21 Jul 2022 16:54:02 -0500 Subject: [PATCH 14/41] added more bug fixes --- src/uproot/_awkward_forth.py | 10 + src/uproot/containers.py | 114 +++++++++- src/uproot/interpretation/objects.py | 2 +- src/uproot/model.py | 4 +- src/uproot/streamers.py | 11 +- .../test_0637-setup-tests-for-AwkwardForth.py | 215 +++++++++++++----- 6 files changed, 290 insertions(+), 66 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index f0f2ba318..7c4c0b888 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -11,6 +11,9 @@ np.dtype(">f8"): "d", np.dtype(">i8"): "q", np.dtype(">i4"): "i", + np.dtype(">i2"): "h", + np.dtype(">u4"): "I", + np.dtype("bool"): "?", } @@ -149,6 +152,7 @@ def enable_adding(self): self.awkward_model["content"] = {} def add_node_whole(self, new_node, ref_latest): + print(self._prev_node, '=========================') if "content" in self.awkward_model.keys(): self.awkward_model["content"] = new_node else: @@ -302,3 +306,9 @@ def convert_dtype(format): return "int64" elif format == "i": return "int32" + elif format == "I": + return "uint32" + elif format == "?": + return "bool" + elif format == "h": + return "int16" diff --git a/src/uproot/containers.py b/src/uproot/containers.py index e3ecb4174..007e381f1 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -55,6 +55,7 @@ def _read_nested( if isinstance(model, numpy.dtype): symbol = uproot._awkward_forth.symbol_dict.get(model) + if symbol is None and helper_obj.is_forth(): raise TypeError("Cannot be awkward") if helper_obj.is_forth(): @@ -473,6 +474,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): helper_obj = uproot._awkward_forth.GenHelper(context) if helper_obj.is_forth(): + #raise NotImplementedError forth_obj = helper_obj.get_gen_obj() keys = forth_obj.get_keys(2) offsets_num = keys[0] @@ -500,7 +502,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): out = cursor.string_with_length(chunk, context, length) if helper_obj.is_forth(): helper_obj.add_to_pre( - f"stream I-> stack dup part0-node{offsets_num}-offsets <- stack stream #B-> part0-node{data_num}-data\n" + f"stream !I-> stack dup part0-node{offsets_num}-offsets +<- stack stream #B-> part0-node{data_num}-data\n" ) else: raise AssertionError(repr(self._length_bytes)) @@ -522,7 +524,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): temp_header = "true" else: temp_header = "false" - temp_aform = f'{{"class": "ListOffsetArray", "offsets": "i64", "content": {{"class": "NumpyArray", "primitive": "uint8", "inner_shape": [], "has_identifier": false, "parameters": {{"__array__": "char"}}, "form_key": "node{data_num}"}}, "has_identifier": false, "parameters": {{"uproot": {{"as": "vector", "header": {temp_header}}}}}, "form_key": "node{offsets_num}"}}' + temp_aform = f'{{"class": "ListOffsetArray", "offsets": "i64", "content": {{"class": "NumpyArray", "primitive": "uint8", "inner_shape": [], "has_identifier": false, "parameters": {{"__array__": "char"}}, "form_key": "node{data_num}"}}, "has_identifier": false, "parameters": {{"__array__": "string" ,"uproot": {{"as": "vector", "header": {temp_header}}}}}, "form_key": "node{offsets_num}"}}' forth_obj.add_form(json.loads(temp_aform)) form_keys = [ @@ -747,13 +749,63 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): return remainder.view(self._values).reshape(-1, *self.inner_shape) else: + if helper_obj.is_forth(): + helper_obj.add_to_header( + f"output part0-node{offsets_num}-offsets int64\n" + ) + form_key = f"part0-node{offsets_num}-offsets" + helper_obj.add_to_init( + f"0 part0-node{offsets_num}-offsets <- stack\n" + ) + helper_obj.add_to_pre( + "0 bytestops I-> stack \nbegin\ndup stream pos <>\nwhile\nswap 1 + swap\n" + ) + if len(self.inner_shape) > 0: + helper_obj.add_to_post( + f"repeat\nswap {self.inner_shape[0]} / part0-node{offsets_num}-offsets +<- stack drop\n" + ) + else: + helper_obj.add_to_post( + f"repeat\nswap part0-node{offsets_num}-offsets +<- stack drop\n" + ) + if forth_obj.should_add_form(): + forth_obj.add_form_key(form_key) + if self._header: + temp_bool = "true" + else: + temp_bool = "false" + if len(self.inner_shape) > 0: + temp_aform = f'{{ "class":"ListOffsetArray", "offsets":"i64", "content": {{"class": "RegularArray", "content": "NULL", "size": {self.inner_shape[0]}}}, "has_identifier": false, "parameters": {{"uproot": {{"as": "vector", "header": {temp_bool}}}}}, "form_key": "node{offsets_num}"}}' + forth_obj.add_form(json.loads(temp_aform), traverse=2) + else: + temp_aform = f'{{ "class":"ListOffsetArray", "offsets":"i64", "content": "NULL", "has_identifier": false, "parameters": {{"uproot": {{"as": "vector", "header": {temp_bool}}}}}, "form_key": "node{offsets_num}"}}' + forth_obj.add_form(json.loads(temp_aform)) + temp = forth_obj.add_node( + f"node{offsets_num}", + helper_obj.get_pre(), + helper_obj.get_post(), + helper_obj.get_init(), + helper_obj.get_header(), + "i64", + 1, + {}, + ) + + if cursor.index >= chunk.stop and helper_obj.is_forth(): + forth_obj.var_set = True out = [] + if helper_obj.is_forth(): + temp_count = forth_obj.count_obj while cursor.displacement(start_cursor) < num_bytes: + if helper_obj.is_forth(): + forth_obj.count_obj = temp_count out.append( self._values.read( chunk, cursor, context, file, selffile, parent ) ) + if helper_obj.is_forth(): + forth_obj.go_to(temp["content"]) if self._header and header: uproot.deserialization.numbytes_check( @@ -1125,7 +1177,6 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if helper_obj.is_forth(): key = forth_obj.get_keys(1) - form_key = f"part0-node{key}-offsets" helper_obj.add_to_header(f"output part0-node{key}-offsets int64\n") helper_obj.add_to_init(f"0 part0-node{key}-offsets <- stack\n") @@ -1258,6 +1309,11 @@ def awkward_form(self, file, context): def read(self, chunk, cursor, context, file, selffile, parent, header=True): # @aryan26roy: test_0637's 62,63,64,65,69,70,74,75,77 + helper_obj = uproot._awkward_forth.GenHelper(context) + + if helper_obj.is_forth(): + forth_obj = helper_obj.get_gen_obj() + if self._header and header: start_cursor = cursor.copy() ( @@ -1265,6 +1321,10 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): instance_version, is_memberwise, ) = uproot.deserialization.numbytes_version(chunk, cursor, context) + if helper_obj.is_forth(): + temp_jump = cursor._index - start_cursor._index + if temp_jump != 0: + helper_obj.add_to_pre(f"{temp_jump} stream skip\n") else: is_memberwise = False @@ -1277,10 +1337,44 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): ) length = cursor.field(chunk, _stl_container_size, context) + if helper_obj.is_forth(): + key = forth_obj.get_keys(1) + form_key = f"part0-node{key}-offsets" + helper_obj.add_to_header(f"output part0-node{key}-offsets int64\n") + helper_obj.add_to_init(f"0 part0-node{key}-offsets <- stack\n") + helper_obj.add_to_pre( + f"stream !I-> stack\ndup part0-node{key}-offsets +<- stack\n" + ) + # helper_obj.add_to_post("loop\n") + if forth_obj.should_add_form(): + forth_obj.add_form_key(form_key) + if self._header: + temp_bool = "true" + else: + temp_bool = "false" + temp_aform = f'{{ "class":"ListOffsetArray", "offsets":"i64", "content": "NULL", "has_identifier": false, "parameters": {{"__array__": "set","uproot": {{"as": "set", "header": {temp_bool}}}}}, "form_key": "node{key}"}}' + forth_obj.add_form(json.loads(temp_aform)) + if not isinstance(self._keys, numpy.dtype): + helper_obj.add_to_pre("0 do\n") + helper_obj.add_to_post("loop\n") + temp = forth_obj.add_node( + f"node{key}", + helper_obj.get_pre(), + helper_obj.get_post(), + helper_obj.get_init(), + helper_obj.get_header(), + "i64", + 1, + {}, + ) + if length == 0 and helper_obj.is_forth(): + forth_obj.var_set = True keys = _read_nested( self._keys, length, chunk, cursor, context, file, selffile, parent ) + if helper_obj.is_forth(): + forth_obj.go_to(temp) out = STLSet(keys) if self._header and header: @@ -1446,7 +1540,10 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): helper_obj.add_to_pre("6 stream skip\n") cursor.skip(6) if helper_obj.is_forth(): - helper_obj.add_to_pre("dup 0 do\n") + if not isinstance(self._keys, numpy.dtype): + helper_obj.add_to_pre("dup 0 do\n") + else: + helper_obj.add_to_pre("dup\n") if helper_obj.is_forth(): ( temp_node, @@ -1479,13 +1576,15 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): None, {"name": "TOP", "content": {}} ) if helper_obj.is_forth(): - keys_model["content"]["post_code"].append("loop\n") + if not isinstance(self._keys, numpy.dtype): + keys_model["content"]["post_code"].append("loop\n") if _has_nested_header(self._values) and header: cursor.skip(6) if helper_obj.is_forth(): keys_model["content"]["post_code"].append("6 stream skip\n") if helper_obj.is_forth(): - keys_model["content"]["post_code"].append("0 do\n") + if not isinstance(self._values, numpy.dtype): + keys_model["content"]["post_code"].append("0 do\n") values = _read_nested( self._values, length, @@ -1500,7 +1599,8 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if helper_obj.is_forth(): values_form = forth_obj.top_form values_model = forth_obj._prev_node - values_model["content"]["post_code"].append("loop \n") + if not isinstance(self._values, numpy.dtype): + values_model["content"]["post_code"].append("loop \n") forth_obj.awkward_model = temp_node forth_obj._prev_node = temp_node_top forth_obj.aform = temp_form diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index c36ee5a44..32ffae578 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -322,7 +322,7 @@ def _discover_forth(self, data, byte_offsets, branch, context, cursor_offset): """ self._forth_form_keys = tuple(context["forth"].gen.form_keys) self._form = context["forth"].gen.top_form - + print(self._complete_forth_code) return None # we should re-read all the data with Forth return output # Forth-generation was unsuccessful: this is Python output diff --git a/src/uproot/model.py b/src/uproot/model.py index 5e6d7c8fd..54b6d470e 100644 --- a/src/uproot/model.py +++ b/src/uproot/model.py @@ -773,7 +773,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): self.read_numbytes_version(chunk, cursor, context) length = cursor._index - temp_index if length != 0: - helper_obj.add_to_pre(f"{length} stream skip \n") + helper_obj.add_to_pre(f"{length} stream skip\n") if ( issubclass(cls, VersionedModel) and self._instance_version != classname_version(cls.__name__) @@ -824,7 +824,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): chunk=chunk, cursor=cursor, context=context, file=file ) if helper_obj.is_forth(): - # raise NotImplementedError + #print(forth_obj.awkward_model, "OPOPOPOPOPO") forth_obj.add_node( "model828", helper_obj.get_pre(), diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index dea554ebb..9ec1708f1 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -214,11 +214,14 @@ def class_code(self): " )", ] read_members.append( - """ + f""" helper_obj = uproot._awkward_forth.GenHelper(context) if helper_obj.is_forth(): forth_obj = helper_obj.get_gen_obj() - content = {}\n""" + content = {{}} + #if '{self.name}' == 'ElementLinkBase': + # raise NotImplementedError + #print(forth_obj.awkward_model, '======', forth_obj._prev_node, '{self.name}')\n""" ) read_member_n = [ " def read_member_n(self, chunk, cursor, context, file, member_index):" @@ -691,13 +694,13 @@ def class_code( read_members.append( f" self._bases.append(c({self.name!r}, {self.base_version!r}).read(chunk, cursor, context, file, self._file, self._parent, concrete=self.concrete))" ) + read_member_n.append(" " + read_members[-1]) read_members.append( - " if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" + f" if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" ) # read_members.append( # " if helper_obj.is_forth():\n temp_form = forth_obj.get_temp_form_top()\n content.update(temp_form['contents'])\n forth_obj.set_dummy_none(temp_top_dummy, temp_dummy, temp_top_flag)\n" # ) - read_member_n.append(" " + read_members[-1]) strided_interpretation.append( f" members.extend(file.class_named({self.name!r}, {self.base_version!r}).strided_interpretation(file, header, tobject_header, breadcrumbs).members)" ) diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index a72dc2c81..9d2996258 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -45,7 +45,10 @@ def test_03(): with uproot.open(skhep_testdata.data_path("uproot-FCCDelphesOutput.root")) as file: branch = file["metadata/gaudiConfigOptions"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0] == 'Converter.hepmcStatusList = "[]";\n' + assert py[0][14] == 'ToolSvc.photons.delphesArrayName = "PhotonEfficiency/photons";\nToolSvc.photons.isolationTags = "photonITags";\nToolSvc.photons.mcAssociations = "photonsToMC";\nToolSvc.photons.particles = "photons";\n' # py[-1] == @@ -53,15 +56,18 @@ def test_04(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["POOLContainerForm/DataHeaderForm/m_uints"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0][0] == 2 + assert py[0][3][2] == 589824 # py[-1] == -def test_05(): - with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - branch = file["CollectionTree/TrigConfKeys"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) +# def test_05(): +# with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: +# branch = file["CollectionTree/TrigConfKeys"] +# interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) +# py = branch.array(interp, library="np", entry_stop=2) # py[-1] == @@ -69,7 +75,9 @@ def test_06(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["CollectionTree/AnalysisJetsAuxDyn.NumTrkPt500"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][-1][-2] == 0 # py[-1] == @@ -77,7 +85,9 @@ def test_07(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["CollectionTree/AnalysisJetsAuxDyn.SumPtTrkPt500"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][2][0] == pytest.approx(53949.015625) # py[-1] == @@ -103,7 +113,9 @@ def test_10(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["CollectionTree/CaloCalTopoClustersAuxDyn.e_sampl"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=3) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=3) + assert py[0][0][0] == pytest.approx(168.03048706054688) # py[-1] == @@ -137,7 +149,9 @@ def test_14(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["CollectionTree/egammaClustersAuxDyn.eta_sampl"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=3) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=3) + assert py[0][0][-1] == pytest.approx(0.4663555920124054) # py[-1] == @@ -161,7 +175,9 @@ def test_17(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: branch = file["E/Evt/AAObject/usr_names"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=None) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=None) + assert len(py[0]) == 0 # py[-1] == @@ -169,7 +185,9 @@ def test_18(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: branch = file["E/Evt/hits/hits.t"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0] == pytest.approx(70104010.0) # py[-1] == array([80860738., 80861689., 80862014., 80861709., 80861737., 80861158., 80862362., 80860821., 80862271., 80862273., 80861294., 80861860., 80862548., 80861733., 80861605., 80860467., 80860408., 80861562., 80862012., 80862350., 80861491., 80860384., 80860930., 80861541., 80861461., 80861749., 80862352., 80861813., 80861822., 80861871., 80862000., 80862255., 80862253., 80862249., 80862266., 80862248., 80862246., 80862847., 80863032., 80861952., 80861954., 80861953., 80861957., 80861951., 80861961., 80861959., 80861955., 80861994., 80862060., 80861971., 80862004., 80862002., 80862059., 80861695., 80861813., 80861967., 80862919., 80862043., 80862054., 80862044., 80862044., 80862040., 80862043., 80862037., 80862040., 80862039., 80862070., 80862042., 80862322., 80861605., 80861865., 80863034., 80862987., 80861545., 80860392., 80861003., 80861564., 80862109., 80861821., 80862083., 80861121., 80862513., 80862513., 80862731., 80861604., 80862003., 80861910., 80861854., 80862297., 80860989., 80862948., 80862075., 80862141., 80862117., 80862039., 80862114., 80862075., 80862042., 80862072., 80862439., 80862481., 80861656., 80862096., 80862215., 80862215., 80862195., 80862458., 80862432., 80861915., 80861012., 80862208., 80861885., 80861888., 80861994., 80861883., 80862194., 80861812., 80862184., 80862309., 80862297., 80862840., 80862400., 80861565., 80862226., 80862149.]) @@ -177,7 +195,9 @@ def test_19(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: branch = file["E/Evt/hits/hits.a"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][-1] == 0.0 # py[-1] == array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]) @@ -185,7 +205,9 @@ def test_20(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: branch = file["E/Evt/hits/hits.trig"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = True py = branch.array(interp, library="np", entry_stop=2) + assert py[0][5] == 1 # py[-1] == array([0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0], dtype=int32) @@ -193,7 +215,9 @@ def test_21(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: branch = file["E/Evt/hits/hits.tot"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0] == 24 # py[-1] == array([29, 26, 22, 18, 22, 28, 28, 28, 21, 24, 28, 30, 25, 24, 30, 28, 29, 4, 21, 25, 26, 22, 23, 22, 23, 29, 23, 30, 24, 29, 31, 27, 32, 28, 30, 33, 33, 31, 29, 18, 23, 34, 21, 33, 33, 29, 37, 23, 21, 40, 25, 29, 22, 17, 31, 25, 28, 26, 21, 20, 25, 51, 38, 64, 42, 28, 29, 26, 21, 31, 22, 18, 41, 28, 29, 28, 29, 15, 25, 27, 24, 28, 28, 34, 28, 21, 19, 21, 20, 24, 26, 24, 13, 22, 30, 25, 17, 27, 24, 16, 31, 27, 29, 23, 26, 25, 26, 28, 12, 18, 30, 27, 48, 16, 25, 24, 27, 10, 21, 25, 30, 26, 26, 28, 24], dtype=uint32) @@ -201,7 +225,9 @@ def test_22(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: branch = file["E/Evt/hits/hits.pos.x"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][2] == 0.0 # py[-1] == array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]) @@ -209,7 +235,9 @@ def test_23(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: branch = file["E/Evt/trks/trks.usr_names"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert len(py[0][0]) == 0 # py[-1] == array([, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ], dtype=object) @@ -217,7 +245,9 @@ def test_24(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: branch = file["E/Evt/trks/trks.rec_stages"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = True py = branch.array(interp, library="np", entry_stop=2) + assert py[0][1][-1] == 5 # py[-1] == array([, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ], dtype=object) @@ -225,9 +255,13 @@ def test_25(): with uproot.open(skhep_testdata.data_path("uproot-issue-208.root")) as file: branch = file["config/VERSION/VERSION._name"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = True py = branch.array(interp, library="np", entry_stop=2) + assert py[0][-1] == 'numuCCAnalysis' # py[-1] == array(['psychePolicy', 'psycheEventModel', 'psycheCore', 'psycheUtils', 'psycheND280Utils', 'psycheIO', 'psycheSelections', 'psycheSystematics', 'highlandEventModel', 'highlandTools', 'highlandCore', 'highlandCorrections', 'highlandIO', 'baseAnalysis', 'baseTrackerAnalysis', 'numuCCAnalysis'], dtype=object) +# make it last + def test_26(): with uproot.open(skhep_testdata.data_path("uproot-issue-208.root")) as file: @@ -241,7 +275,9 @@ def test_27(): with uproot.open(skhep_testdata.data_path("uproot-issue-208.root")) as file: branch = file["config/SEL/SEL._branchAlias"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0][-1] == 'muFGD+Np' # py[-1] == array([], dtype=object) @@ -249,9 +285,13 @@ def test_28(): with uproot.open(skhep_testdata.data_path("uproot-issue-208.root")) as file: branch = file["config/SEL/SEL._nCutsInBranch"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0][-1] == 10 # py[-1] == array([], dtype=object) +# some fields missing + def test_29(): with uproot.open(skhep_testdata.data_path("uproot-issue213.root")) as file: @@ -273,7 +313,9 @@ def test_31(): with uproot.open(skhep_testdata.data_path("uproot-issue-308.root")) as file: branch = file["MetaData/BranchIDLists"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0][-1] == 2428801822 # py[-1] == @@ -281,7 +323,9 @@ def test_32(): with uproot.open(skhep_testdata.data_path("uproot-issue31.root")) as file: branch = file["T/data/name"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = True py = branch.array(interp, library="np", entry_stop=2) + assert py[-1] == 'two' # py[-1] == "two" @@ -289,7 +333,9 @@ def test_33(): with uproot.open(skhep_testdata.data_path("uproot-issue367b.root")) as file: branch = file["tree/weights"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][4]['1'][-1] == 1.0 # py[-1] == @@ -297,8 +343,9 @@ def test_34(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: branch = file["Header/Header./Header.geant4Version"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) - # py[-1] == '$Name: geant4-10-05-patch-01 $' + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1] == '$Name: geant4-10-05-patch-01 $' def test_35(): @@ -314,6 +361,7 @@ def test_36(): branch = file["Model/Model./Model.samplerNamesUnique"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) py = branch.array(interp, library="np", entry_stop=2) + assert py[0][-1] == 'MP_F_99.' # py[-1] == @@ -353,7 +401,9 @@ def test_41(): with uproot.open(skhep_testdata.data_path("uproot-issue468.root")) as file: branch = file["Event/Trajectory./Trajectory.XYZ"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = True py = branch.array(interp, library="np", entry_stop=2) + assert len(py[0]) == 0 # py[-1] == @@ -361,7 +411,9 @@ def test_42(): with uproot.open(skhep_testdata.data_path("uproot-issue468.root")) as file: branch = file["Event/Trajectory./Trajectory.energyDeposit"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = True py = branch.array(interp, library="np", entry_stop=2) + assert len(py[0]) == 0 # py[-1] == @@ -369,7 +421,9 @@ def test_43(): with uproot.open(skhep_testdata.data_path("uproot-issue468.root")) as file: branch = file["Event/Trajectory./Trajectory.ionA"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert len(py[0]) == 0 # py[-1] == @@ -433,15 +487,18 @@ def test_51(): with uproot.open(skhep_testdata.data_path("uproot-issue510b.root")) as file: branch = file["EDepSimEvents/Event/Primaries/Primaries.GeneratorName"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) - # py[-1] == array(['GENIE:fixed@density-fixed'], dtype=object) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1] == 'GENIE:fixed@density-fixed' def test_52(): with uproot.open(skhep_testdata.data_path("uproot-issue513.root")) as file: branch = file["Delphes/EFlowPhoton/EFlowPhoton.Particles"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][-1]['refs'][-1] == 2223 # py[-1] == array([, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ], dtype=object) @@ -449,7 +506,9 @@ def test_53(): with uproot.open(skhep_testdata.data_path("uproot-issue513.root")) as file: branch = file["Delphes/Jet/Jet.SoftDroppedSubJet2"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][-1]['fE'] == 0.0 # py[-1] == array([, , , , , ], dtype=object) @@ -457,7 +516,9 @@ def test_54(): with uproot.open(skhep_testdata.data_path("uproot-issue513.root")) as file: branch = file["Delphes/Jet/Jet.TrimmedP4[5]"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][-1][4]['fE'] == 0.0 # py[-1] == array([[, , , , ], [, , , , ], [, , , , ], [, , , , ], [, , , , ], [, , , , ]], dtype=object) @@ -465,7 +526,9 @@ def test_55(): with uproot.open(skhep_testdata.data_path("uproot-issue519.root")) as file: branch = file["testtree/testbranch"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1][5] == pytest.approx(0.346174418926239) # py[-1] == @@ -483,7 +546,9 @@ def test_57(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/vector_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1] == 'two' # py[-1] == @@ -491,7 +556,9 @@ def test_58(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/vector_tstring"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1] == 'two' # py[-1] == @@ -499,7 +566,9 @@ def test_59(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/vector_vector_int32"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1][1] == 2 # py[-1] == @@ -507,7 +576,9 @@ def test_60(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/vector_vector_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = True py = branch.array(interp, library="np", entry_stop=2) + assert py[-1][1][1] == 'two' # py[-1] == @@ -515,7 +586,9 @@ def test_61(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/vector_vector_tstring"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = True py = branch.array(interp, library="np", entry_stop=2) + assert py[-1][1][1] == 'two' # py[-1] == @@ -523,7 +596,9 @@ def test_62(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/vector_set_int32"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1][1] == 2 # py[-1] == @@ -531,7 +606,9 @@ def test_63(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/vector_set_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1][1] == 'two' # py[-1] == @@ -539,7 +616,9 @@ def test_64(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/set_int32"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1] == 2 # py[-1] == @@ -547,7 +626,9 @@ def test_65(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/set_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1] == 'two' # py[-1] == @@ -555,7 +636,9 @@ def test_66(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_int32_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1]['1'][1] == 2 # py[-1] == @@ -563,7 +646,9 @@ def test_67(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_int32_vector_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]['1'][1] == 2 # py[-1] == @@ -571,7 +656,9 @@ def test_68(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_int32_vector_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]['1'][1] == 'two' # py[-1] == @@ -579,7 +666,9 @@ def test_69(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_int32_set_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]['1'][1] == 2 # py[-1] == @@ -587,7 +676,9 @@ def test_70(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_int32_set_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]['1'][1] == 'two' # py[-1] == @@ -595,7 +686,9 @@ def test_71(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_string_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]['1'] == 2 # py[-1] == @@ -603,7 +696,9 @@ def test_72(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_string_vector_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]['1'][1] == 2 # py[-1] == @@ -611,7 +706,9 @@ def test_73(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_string_vector_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]['1'][1] == 'two' # py[-1] == @@ -619,7 +716,9 @@ def test_74(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_string_set_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]['1'][1] == 2 # py[-1] == @@ -627,7 +726,9 @@ def test_75(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_string_set_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]['1'][1] == 'two' # py[-1] == @@ -635,7 +736,9 @@ def test_76(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_int32_vector_vector_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]['1'][1][1] == 2 # py[-1] == @@ -643,7 +746,9 @@ def test_77(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_int32_vector_set_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]['1'][1][1] == 2 # py[-1] == @@ -651,7 +756,9 @@ def test_78(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_string_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]['1'] == 'TWO' # py[-1] == @@ -659,7 +766,9 @@ def test_79(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: branch = file["tree/map_string_tstring"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]['1'] == 'TWO' # py[-1] == @@ -677,5 +786,7 @@ def test_81(): with uproot.open(skhep_testdata.data_path("uproot-HZZ-objects.root")) as file: branch = file["events/MET"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0]['1'] == pytest.approx(2.5636332035064697) # py[-1] == From 4d113f3672e05553f2909d8381dda28d4c35b5cf Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Fri, 22 Jul 2022 16:34:42 -0500 Subject: [PATCH 15/41] added DaTime --- src/uproot/_awkward_forth.py | 1 - src/uproot/containers.py | 8 +++- src/uproot/models/TDatime.py | 27 +++++++++++ src/uproot/streamers.py | 4 +- .../test_0637-setup-tests-for-AwkwardForth.py | 45 ++++++++++++------- 5 files changed, 65 insertions(+), 20 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index 7c4c0b888..a360eeaff 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -152,7 +152,6 @@ def enable_adding(self): self.awkward_model["content"] = {} def add_node_whole(self, new_node, ref_latest): - print(self._prev_node, '=========================') if "content" in self.awkward_model.keys(): self.awkward_model["content"] = new_node else: diff --git a/src/uproot/containers.py b/src/uproot/containers.py index 007e381f1..78d29a087 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -103,8 +103,13 @@ def _read_nested( temp_count = context["forth"].gen.count_obj for i in range(length): if helper_obj.is_forth(): + if 'temp_ref' in context.keys(): + context["forth"].gen.go_to(context['temp_ref']) context["forth"].gen.count_obj = temp_count values[i] = model.read(chunk, cursor, context, file, selffile, parent) + if helper_obj.is_forth(): + if 'temp_ref' in context.keys(): + del context['temp_ref'] return values @@ -888,7 +893,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): ) ) if helper_obj.is_forth(): - forth_obj.go_to(temp["content"]) + forth_obj.go_to(temp) # if helper_obj.is_forth(): # forth_obj.go_to(temp) @@ -1205,6 +1210,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): 1, {}, ) + context['temp_ref'] = temp if length == 0 and helper_obj.is_forth(): forth_obj.var_set = True diff --git a/src/uproot/models/TDatime.py b/src/uproot/models/TDatime.py index 32ad47ef6..f55928ed5 100644 --- a/src/uproot/models/TDatime.py +++ b/src/uproot/models/TDatime.py @@ -11,6 +11,8 @@ import uproot import uproot.behaviors.TDatime +import uproot._awkward_forth +import json _tdatime_format1 = struct.Struct(">I") @@ -24,6 +26,31 @@ def read_numbytes_version(self, chunk, cursor, context): pass def read_members(self, chunk, cursor, context, file): + helper_obj = uproot._awkward_forth.GenHelper(context) + if helper_obj.is_forth(): + forth_obj = helper_obj.get_gen_obj() + key = forth_obj.get_keys(1) + form_key = f"part0-node{key}-data" + helper_obj.add_to_header(f"output part0-node{key}-data int32\n") + helper_obj.add_to_pre( + f"stream !I-> part0-node{key}-data\n" + ) + form_key = f"part0-node{key}-data" + if forth_obj.should_add_form(): + forth_obj.add_form_key(form_key) + temp_aform = {"class": "RecordArray", "contents": {'fDatime': {"class": "NumpyArray", "primitive": "uint32", "form_key": f"node{key}"}}, "parameters": {"__record__": "TDatime"}} + forth_obj.add_form(temp_aform) + temp_form = forth_obj.add_node( + f"node{key}", + helper_obj.get_pre(), + helper_obj.get_post(), + helper_obj.get_init(), + helper_obj.get_header(), + "i64", + 0, + None, + ) + forth_obj.go_to(temp_form) self._members["fDatime"] = cursor.field(chunk, _tdatime_format1, context) @classmethod diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index 9ec1708f1..3c542d493 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -219,7 +219,7 @@ def class_code(self): if helper_obj.is_forth(): forth_obj = helper_obj.get_gen_obj() content = {{}} - #if '{self.name}' == 'ElementLinkBase': + #if '{self.name}' == 'TVector3': # raise NotImplementedError #print(forth_obj.awkward_model, '======', forth_obj._prev_node, '{self.name}')\n""" ) @@ -696,7 +696,7 @@ def class_code( ) read_member_n.append(" " + read_members[-1]) read_members.append( - f" if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" + f" if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n #print(forth_obj._prev_node,'??????',forth_obj.awkward_model, '======',temp_model1, '/////////')\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" ) # read_members.append( # " if helper_obj.is_forth():\n temp_form = forth_obj.get_temp_form_top()\n content.update(temp_form['contents'])\n forth_obj.set_dummy_none(temp_top_dummy, temp_dummy, temp_top_flag)\n" diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index 9d2996258..317ed8d92 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -14,7 +14,7 @@ def test_00(): branch = file["tree/weights"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True - py = branch.array(interp, entry_stop=2) + py = branch.array(interp, library="ak", entry_stop=2) assert py[0]["0"][0] == "expskin_FluxUnisim" # py[-1] == @@ -105,7 +105,9 @@ def test_09(): "AntiKt10UFOCSSKJetsAuxDyn.GhostVR30Rmax4Rmin02TrackJet_BTagging201903" ] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0]['0'][0] == 352341021 # py[-1] == > (version 1) at 0x7febbf1b2fa0>], ...] at 0x7febbf1b2f40> @@ -125,7 +127,9 @@ def test_11(): "TruthBosonsWithDecayVerticesAuxDyn.incomingParticleLinks" ] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0]['0'][0] == 921521854 # py[-1] == > (version 1) at 0x7f636a9484c0>], ...] at 0x7f636a948eb0> @@ -133,7 +137,9 @@ def test_12(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["CollectionTree/TruthBottomAuxDyn.parentLinks"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0]['1'][0] == 2 # py[-1] == > (version 1) at 0x7fc259ae37c0>], ...] at 0x7fc259ae3f10> @@ -141,7 +147,9 @@ def test_13(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["CollectionTree/egammaClustersAuxDyn.constituentClusterLinks"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=3) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=3) + assert py[0][0]['1'][0] == 0 # py[-1] == > (version 1) at 0x7fa94e968c10>], ...] at 0x7fa94e968c70> @@ -206,7 +214,7 @@ def test_20(): branch = file["E/Evt/hits/hits.trig"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True - py = branch.array(interp, library="np", entry_stop=2) + py = branch.array(interp, library="ak", entry_stop=2) assert py[0][5] == 1 # py[-1] == array([0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0], dtype=int32) @@ -246,7 +254,7 @@ def test_24(): branch = file["E/Evt/trks/trks.rec_stages"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True - py = branch.array(interp, library="np", entry_stop=2) + py = branch.array(interp, library="ak", entry_stop=2) assert py[0][1][-1] == 5 # py[-1] == array([, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ], dtype=object) @@ -256,7 +264,7 @@ def test_25(): branch = file["config/VERSION/VERSION._name"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True - py = branch.array(interp, library="np", entry_stop=2) + py = branch.array(interp, library="ak", entry_stop=2) assert py[0][-1] == 'numuCCAnalysis' # py[-1] == array(['psychePolicy', 'psycheEventModel', 'psycheCore', 'psycheUtils', 'psycheND280Utils', 'psycheIO', 'psycheSelections', 'psycheSystematics', 'highlandEventModel', 'highlandTools', 'highlandCore', 'highlandCorrections', 'highlandIO', 'baseAnalysis', 'baseTrackerAnalysis', 'numuCCAnalysis'], dtype=object) @@ -324,7 +332,7 @@ def test_32(): branch = file["T/data/name"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True - py = branch.array(interp, library="np", entry_stop=2) + py = branch.array(interp, library="ak", entry_stop=2) assert py[-1] == 'two' # py[-1] == "two" @@ -360,7 +368,8 @@ def test_36(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: branch = file["Model/Model./Model.samplerNamesUnique"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) assert py[0][-1] == 'MP_F_99.' # py[-1] == @@ -369,7 +378,9 @@ def test_37(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: branch = file["Model/Model./Model.staPos"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][555]['2'] == 100.94856572890848 # py[-1] == array([, , , ..., , , ], dtype=object) @@ -393,7 +404,9 @@ def test_40(): with uproot.open(skhep_testdata.data_path("uproot-issue-407.root")) as file: branch = file["tree/branch"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0]['fDatime'] == 1749155840 # py[-1] == @@ -402,7 +415,7 @@ def test_41(): branch = file["Event/Trajectory./Trajectory.XYZ"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True - py = branch.array(interp, library="np", entry_stop=2) + py = branch.array(interp, library="ak", entry_stop=2) assert len(py[0]) == 0 # py[-1] == @@ -412,7 +425,7 @@ def test_42(): branch = file["Event/Trajectory./Trajectory.energyDeposit"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True - py = branch.array(interp, library="np", entry_stop=2) + py = branch.array(interp, library="ak", entry_stop=2) assert len(py[0]) == 0 # py[-1] == @@ -577,7 +590,7 @@ def test_60(): branch = file["tree/vector_vector_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True - py = branch.array(interp, library="np", entry_stop=2) + py = branch.array(interp, library="ak", entry_stop=2) assert py[-1][1][1] == 'two' # py[-1] == @@ -587,7 +600,7 @@ def test_61(): branch = file["tree/vector_vector_tstring"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True - py = branch.array(interp, library="np", entry_stop=2) + py = branch.array(interp, library="ak", entry_stop=2) assert py[-1][1][1] == 'two' # py[-1] == From 24add28cd4873714f09d11ea467cd093507165e4 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Mon, 8 Aug 2022 22:04:01 +0530 Subject: [PATCH 16/41] fixed bug in forth code generation --- src/uproot/containers.py | 3 +-- src/uproot/interpretation/objects.py | 3 +-- tests/test_0637-setup-tests-for-AwkwardForth.py | 2 ++ 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/uproot/containers.py b/src/uproot/containers.py index 78d29a087..c218878f3 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -1105,7 +1105,6 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): # @aryan26roy: test_0637's 00,03,04,06,07,08,09,10,11,12,13,14,15,16,17,23,24,26,27,28,31,33,36,38,41,42,43,44,45,46,49,50,55,56,57,58,59,60,61,62,63,67,68,72,73,76,77,80 helper_obj = uproot._awkward_forth.GenHelper(context) - if helper_obj.is_forth(): forth_obj = helper_obj.get_gen_obj() @@ -1179,7 +1178,6 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): ) else: length = cursor.field(chunk, _stl_container_size, context) - if helper_obj.is_forth(): key = forth_obj.get_keys(1) form_key = f"part0-node{key}-offsets" @@ -1214,6 +1212,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if length == 0 and helper_obj.is_forth(): forth_obj.var_set = True + values = _read_nested( self._values, length, chunk, cursor, context, file, selffile, parent ) diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index 32ffae578..b277305bb 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -302,7 +302,7 @@ def _discover_forth(self, data, byte_offsets, branch, context, cursor_offset): branch.file.detached, branch, ) - # print(output, "ERERERER") + context["forth"].gen.awkward_model = context["forth"].gen._prev_node if not context["forth"].gen.var_set: context["forth"].prereaddone = True self._assemble_forth( @@ -322,7 +322,6 @@ def _discover_forth(self, data, byte_offsets, branch, context, cursor_offset): """ self._forth_form_keys = tuple(context["forth"].gen.form_keys) self._form = context["forth"].gen.top_form - print(self._complete_forth_code) return None # we should re-read all the data with Forth return output # Forth-generation was unsuccessful: this is Python output diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index 317ed8d92..19e6a373b 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -791,7 +791,9 @@ def test_80(): ) as file: branch = file["t/x"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = True py = branch.array(interp, library="np", entry_stop=2) + assert len(py[0]) == 0 # py[-1] == From c82fba0ab8e41487d04bffabfa0971899ef7f42f Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Sun, 14 Aug 2022 15:23:05 +0530 Subject: [PATCH 17/41] solved more bugs --- src/uproot/_awkward_forth.py | 20 ++++++ src/uproot/containers.py | 8 ++- src/uproot/interpretation/objects.py | 5 +- src/uproot/model.py | 2 +- src/uproot/streamers.py | 68 +++++++++++++++---- .../test_0637-setup-tests-for-AwkwardForth.py | 21 ++++-- 6 files changed, 101 insertions(+), 23 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index a360eeaff..18cfbbd86 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -5,6 +5,7 @@ """ import numpy as np +import uproot.containers symbol_dict = { np.dtype(">f4"): "f", @@ -17,6 +18,11 @@ } +def check_depth(node): + if isinstance(node, uproot.containers.AsVector): + return 1 + + class ForthGenerator: """ This class is passed through the Forth code generation, collecting Forth snippets and concatenating them at the end. @@ -55,6 +61,20 @@ def replace_form_and_model(self, form, model): self.aform = form return temp_node, temp_node_top, temp_form, temp_form_top + def get_code_recursive(self, node): + if 'content' in node.keys(): + if node['content'] is None: + return ''.join(node['pre_code']), ''.join(node['post_code']) , node['init_code'], node['header_code'] + else: + pre, post, init, header = self.get_code_recursive(node["content"]) + pre2 = ''.join(node['pre_code']) + pre2 = pre2 + pre + post2 = ''.join(node['post_code']) + post2 = post2 + post + init = node['init_code'] + init + header = node['header_code'] + header + return pre2, post2, init, header + def should_add_form(self): if "content" in self.awkward_model.keys(): if self.awkward_model["content"] is None: diff --git a/src/uproot/containers.py b/src/uproot/containers.py index c218878f3..a38bac721 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -1550,14 +1550,16 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): else: helper_obj.add_to_pre("dup\n") if helper_obj.is_forth(): + temp = {"name": "TOP", "content": {}} ( temp_node, temp_node_top, temp_form, temp_form_top, ) = forth_obj.replace_form_and_model( - None, {"name": "TOP", "content": {}} + None, temp ) + context['temp_ref'] = temp keys = _read_nested( self._keys, length, @@ -1570,6 +1572,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): header=False, ) if helper_obj.is_forth(): + temp = {"name": "TOP", "content": {}} keys_form = forth_obj.top_form keys_model = forth_obj._prev_node ( @@ -1578,8 +1581,9 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): temp_form1, temp_form_top1, ) = forth_obj.replace_form_and_model( - None, {"name": "TOP", "content": {}} + None, temp ) + context['temp_ref'] = temp if helper_obj.is_forth(): if not isinstance(self._keys, numpy.dtype): keys_model["content"]["post_code"].append("loop\n") diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index b277305bb..33df0080f 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -260,6 +260,9 @@ def basket_array_forth( } ) context["forth"].vm.stack_push(len(byte_offsets) - 1) + print(self._complete_forth_code) + print(byte_offsets) + print(temp_data[:100]) context["forth"].vm.resume() container = {} for elem in self._forth_form_keys: @@ -308,7 +311,7 @@ def _discover_forth(self, data, byte_offsets, branch, context, cursor_offset): self._assemble_forth( context["forth"].gen, context["forth"].gen._prev_node["content"] ) - + print(context["forth"].gen._prev_node["content"]) self._complete_forth_code = f"""input stream input byteoffsets input bytestops diff --git a/src/uproot/model.py b/src/uproot/model.py index 54b6d470e..c6686f475 100644 --- a/src/uproot/model.py +++ b/src/uproot/model.py @@ -814,6 +814,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): if self._num_bytes is None and self._instance_version != self.class_version: self._instance_version = None cursor = self._cursor + helper_obj._pre_code.pop(-1) elif self._instance_version == 0: helper_obj.add_to_pre("4 stream skip\n") @@ -824,7 +825,6 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): chunk=chunk, cursor=cursor, context=context, file=file ) if helper_obj.is_forth(): - #print(forth_obj.awkward_model, "OPOPOPOPOPO") forth_obj.add_node( "model828", helper_obj.get_pre(), diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index 3c542d493..c45649c0c 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -208,6 +208,7 @@ def class_code(self): """ read_members = [ " def read_members(self, chunk, cursor, context, file):", + " import uproot._awkward_forth", " if self.is_memberwise:", " raise NotImplementedError(", ' f"memberwise serialization of {type(self).__name__}\\nin file {self.file.file_path}"', @@ -218,10 +219,7 @@ def class_code(self): helper_obj = uproot._awkward_forth.GenHelper(context) if helper_obj.is_forth(): forth_obj = helper_obj.get_gen_obj() - content = {{}} - #if '{self.name}' == 'TVector3': - # raise NotImplementedError - #print(forth_obj.awkward_model, '======', forth_obj._prev_node, '{self.name}')\n""" + content = {{}}\n""" ) read_member_n = [ " def read_member_n(self, chunk, cursor, context, file, member_index):" @@ -276,7 +274,7 @@ def class_code(self): class_flags, ) read_members.append( - " if helper_obj.is_forth():\n if forth_obj.should_add_form():\n forth_obj.add_form({'class': 'RecordArray', 'contents': content, 'parameters': {'record': 'TVector2'}}, len(content))\n temp = forth_obj.add_node('dynamic',helper_obj.get_pre(),helper_obj.get_post(),helper_obj.get_init(),helper_obj.get_header(),\"i64\",0,None)" + " if helper_obj.is_forth():\n if forth_obj.should_add_form():\n forth_obj.add_form({'class': 'RecordArray', 'contents': content, 'parameters': {'record': 'TVector2'}}, len(content))\n temp = forth_obj.add_node('dynamic',helper_obj.get_pre(),helper_obj.get_post(),helper_obj.get_init(),helper_obj.get_header(),\"i64\",0,None)\n" ) if len(read_members) == 1: # untested as of PR #629 @@ -307,7 +305,6 @@ def class_code(self): class_data.append( f" _format_memberwise{i} = struct.Struct('>{''.join(format)}')" ) - for i, dt in enumerate(dtypes): class_data.append(f" _dtype{i} = {dt}") @@ -696,7 +693,7 @@ def class_code( ) read_member_n.append(" " + read_members[-1]) read_members.append( - f" if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n #print(forth_obj._prev_node,'??????',forth_obj.awkward_model, '======',temp_model1, '/////////')\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" + f" if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n print(temp_node,'OPOPO')\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n #print(forth_obj._prev_node,'??????',forth_obj.awkward_model, '======',temp_model1, '/////////')\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" ) # read_members.append( # " if helper_obj.is_forth():\n temp_form = forth_obj.get_temp_form_top()\n content.update(temp_form['contents'])\n forth_obj.set_dummy_none(temp_top_dummy, temp_dummy, temp_top_flag)\n" @@ -808,6 +805,8 @@ def class_code( """ if context.get('speedbump', True): cursor.skip(1) + if helper_obj.is_forth(): + helper_obj.add_to_pre('1 stream skip \\n') """.strip( "\n" ) @@ -816,8 +815,27 @@ def class_code( # @aryan26roy: test_0637's 29,44,56 + read_members.append(' if helper_obj.is_forth():') + read_members.append(' key = forth_obj.get_keys(1)') + read_members.append(' key2 = forth_obj.get_keys(1)') + read_members.append(' form_key = f"part0-node{key}-data"') + read_members.append(' form_key2 = f"part0-node{key2}-data"') + read_members.append( + f' helper_obj.add_to_header(f"output part0-node{{key}}-data {{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}\\n")' + ) + read_members.append( + ' helper_obj.add_to_header(f"output part0-node{key2}-offsets int64\\n")' + ) + read_members.append( + f' content["{self.name!r}"] = {{"class": "ListOffsetArray", "primitive": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' + ) + read_members.append(f' #raise NotImplementedError\n helper_obj.add_to_pre(f"{{self.member({self.count_name!r})}} dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")') + read_members.append(' if forth_obj.should_add_form():') + read_members.append(' forth_obj.add_form_key(form_key)') + read_members.append(' forth_obj.add_form_key(form_key2)') + read_members.append( - f" self._members[{self.name!r}] = cursor.array(chunk, self.member({self.count_name!r}), tmp, context)" + f" self._members[{self.name!r}] = cursor.array(chunk, self.member({self.count_name!r}), tmp, context);" ) read_member_n.append(" " + read_members[-1]) @@ -933,7 +951,7 @@ def class_code( f' content["{fields[-1][0]}"] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[-1][0])}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}' ) read_members.append( - f' helper_obj.add_to_pre(f"stream !{formats[-1][0]}-> part0-node{{key}}-data\\n")' + f' helper_obj.add_to_pre(f"stream !{formats[-1][0]}-> part0-node{{key}}-data\\n stream pos .s cr drop\\n")' ) read_members.append( " if forth_obj.should_add_form():" @@ -942,7 +960,7 @@ def class_code( " forth_obj.add_form_key(form_key)" ) read_members.append( - f" self._members[{fields[-1][0]!r}] = cursor.field(chunk, self._format{len(formats) - 1}, context)" + f" self._members[{fields[-1][0]!r}] = cursor.field(chunk, self._format{len(formats) - 1}, context)\n print(cursor._index)" ) else: @@ -984,8 +1002,26 @@ def class_code( else: # @aryan26roy: test_0637's 44,56 + read_members.append(' if helper_obj.is_forth():') + read_members.append(' key = forth_obj.get_keys(1)') + read_members.append(' key2 = forth_obj.get_keys(1)') + read_members.append(' form_key = f"part0-node{key}-data"') + read_members.append(' form_key2 = f"part0-node{key2}-data"') read_members.append( - f" self._members[{self.name!r}] = cursor.array(chunk, {self.array_length}, self._dtype{len(dtypes)}, context)" + f' helper_obj.add_to_header(f"output part0-node{{key}}-data {{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}\\n")' + ) + read_members.append( + ' helper_obj.add_to_header(f"output part0-node{key2}-offsets int64\\n")' + ) + read_members.append( + f' content["{self.name!r}"] = {{"class": "ListOffsetArray", "primitive": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' + ) + read_members.append(f' helper_obj.add_to_pre(f"{self.array_length} dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")\n') + read_members.append(' if forth_obj.should_add_form():') + read_members.append(' forth_obj.add_form_key(form_key)') + read_members.append(' forth_obj.add_form_key(form_key2)') + read_members.append( + f" self._members[{self.name!r}] = cursor.array(chunk, {self.array_length}, self._dtype{len(dtypes)}, context)\n #print(cursor._index,'\\n' ,chunk._raw_data);" ) dtypes.append(_ftype_to_dtype(self.fType)) @@ -1247,12 +1283,16 @@ def class_code( ) # @aryan26roy: test_0637's 35,38,39,44,45,47,50,56 - + read_members.append( + " if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {'name': 'TOP', 'content': {}})\n" + ) read_members.append( f" self._members[{self.name!r}] = self._stl_container{len(containers)}.read(chunk, cursor, context, file, self._file, self.concrete)" ) read_member_n.append(" " + read_members[-1]) - + read_members.append( + f" if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n print(temp_form1,'OPOPOPO')\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n content[{self.name!r}] = temp_form1\n pre,post,init,header = forth_obj.get_code_recursive(temp_model1)\n helper_obj.add_to_header(header)\n helper_obj.add_to_pre(pre)\n helper_obj.add_to_post(post)\n helper_obj.add_to_init(init)" + ) strided_interpretation.append( f" members.append(({self.name!r}, cls._stl_container{len(containers)}.strided_interpretation(file, header, tobject_header, breadcrumbs)))" ) @@ -1482,7 +1522,7 @@ def class_code( # @aryan26roy: test_0637's 01,02,29,45,46,49,50,56 read_members.append( - f" if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {{'name': 'TOP', 'content': {{}}}})\n self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)\n if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n temp_var = forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content['{self.name}'] = temp_form1\n forth_obj.enable_adding()" + f" if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {{'name': 'TOP', 'content': {{}}}})\n self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)\n if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n #print(temp_node,'OPOPOOP')\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n temp_var = forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content['{self.name}'] = temp_form1\n forth_obj.enable_adding()" ) read_member_n.append( " " diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index 19e6a373b..557b6ac33 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -313,7 +313,9 @@ def test_30(): with uproot.open(skhep_testdata.data_path("uproot-issue243-new.root")) as file: branch = file["triggerList/triggerMap/triggerMap.first"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=16) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=16) + assert len(py[0]) == 0 # py[-1] == array(['HLT_2j35_bmv2c2060_split_2j35_L14J15.0ETA25', 'HLT_j100_2j55_bmv2c2060_split'], dtype=object) @@ -360,7 +362,9 @@ def test_35(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: branch = file["Geant4Data/Geant4Data./Geant4Data.particles"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0]['1'][0]['name'] == 'anti_alpha' # py[-1] == , ...} at 0x7fb557a012e0> @@ -388,7 +392,9 @@ def test_38(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: branch = file["Event/PRBHF_46."] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0]['zp'][0] == pytest.approx(0.9999998807907104) # py[-1] == (version 4) at 0x7f9be2f7b2e0> @@ -396,7 +402,9 @@ def test_39(): with uproot.open(skhep_testdata.data_path("uproot-issue399.root")) as file: branch = file["Geant4Data/Geant4Data./Geant4Data.particles"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1]['0'] == -1000020030 # py[-1] == , ...} at 0x7f6d057397f0> @@ -468,7 +476,9 @@ def test_47(): with uproot.open(skhep_testdata.data_path("uproot-issue494.root")) as file: branch = file["Geant4Data/Geant4Data./Geant4Data.particles"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][-1]['1']['mass'] == pytest.approx(3.727379) # py[-1] == , ...} at 0x7f53a44278b0> @@ -477,6 +487,7 @@ def test_48(): branch = file["Geant4Data/Geant4Data./Geant4Data.ions"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) py = branch.array(interp, library="np", entry_stop=2) + #assert py[-1]['0'][-1] == 1000020040 # py[-1] == From 5aae82d056f9033bb3937c2535858f1399e51444 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Tue, 16 Aug 2022 16:01:38 +0530 Subject: [PATCH 18/41] 2 tests left --- src/uproot/_awkward_forth.py | 12 +++ src/uproot/containers.py | 5 +- src/uproot/interpretation/objects.py | 4 - src/uproot/streamers.py | 8 +- .../test_0637-setup-tests-for-AwkwardForth.py | 85 +++++++++---------- 5 files changed, 63 insertions(+), 51 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index 18cfbbd86..7a513c3f4 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -13,7 +13,10 @@ np.dtype(">i8"): "q", np.dtype(">i4"): "i", np.dtype(">i2"): "h", + np.dtype(">u1"): "B", + np.dtype(">u2"): "H", np.dtype(">u4"): "I", + np.dtype(">u8"): "Q", np.dtype("bool"): "?", } @@ -62,6 +65,7 @@ def replace_form_and_model(self, form, model): return temp_node, temp_node_top, temp_form, temp_form_top def get_code_recursive(self, node): + print(node, 'aass') if 'content' in node.keys(): if node['content'] is None: return ''.join(node['pre_code']), ''.join(node['post_code']) , node['init_code'], node['header_code'] @@ -74,6 +78,8 @@ def get_code_recursive(self, node): init = node['init_code'] + init header = node['header_code'] + header return pre2, post2, init, header + elif self.var_set: + return '', '', '', '' def should_add_form(self): if "content" in self.awkward_model.keys(): @@ -331,3 +337,9 @@ def convert_dtype(format): return "bool" elif format == "h": return "int16" + elif format == "H": + return "uint16" + elif format == "Q": + return "uint64" + elif format == "B": + return "uint8" diff --git a/src/uproot/containers.py b/src/uproot/containers.py index a38bac721..8de665d93 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -91,6 +91,8 @@ def _read_nested( values = numpy.empty(length, dtype=_stl_object_type) if isinstance(model, AsContainer): if helper_obj.is_forth(): + if length == 0: + forth_obj.var_set = True temp_count = context["forth"].gen.count_obj for i in range(length): if helper_obj.is_forth(): @@ -479,6 +481,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): helper_obj = uproot._awkward_forth.GenHelper(context) if helper_obj.is_forth(): + print('afefe') #raise NotImplementedError forth_obj = helper_obj.get_gen_obj() keys = forth_obj.get_keys(2) @@ -1103,7 +1106,6 @@ def awkward_form(self, file, context): def read(self, chunk, cursor, context, file, selffile, parent, header=True): # @aryan26roy: test_0637's 00,03,04,06,07,08,09,10,11,12,13,14,15,16,17,23,24,26,27,28,31,33,36,38,41,42,43,44,45,46,49,50,55,56,57,58,59,60,61,62,63,67,68,72,73,76,77,80 - helper_obj = uproot._awkward_forth.GenHelper(context) if helper_obj.is_forth(): forth_obj = helper_obj.get_gen_obj() @@ -1198,6 +1200,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if not isinstance(self._values, numpy.dtype): helper_obj.add_to_pre("0 do\n") helper_obj.add_to_post("loop\n") + print('ADFAEF') temp = forth_obj.add_node( f"node{key}", helper_obj.get_pre(), diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index 33df0080f..4a118e71c 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -260,9 +260,6 @@ def basket_array_forth( } ) context["forth"].vm.stack_push(len(byte_offsets) - 1) - print(self._complete_forth_code) - print(byte_offsets) - print(temp_data[:100]) context["forth"].vm.resume() container = {} for elem in self._forth_form_keys: @@ -311,7 +308,6 @@ def _discover_forth(self, data, byte_offsets, branch, context, cursor_offset): self._assemble_forth( context["forth"].gen, context["forth"].gen._prev_node["content"] ) - print(context["forth"].gen._prev_node["content"]) self._complete_forth_code = f"""input stream input byteoffsets input bytestops diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index c45649c0c..013fc2a4d 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -819,15 +819,16 @@ def class_code( read_members.append(' key = forth_obj.get_keys(1)') read_members.append(' key2 = forth_obj.get_keys(1)') read_members.append(' form_key = f"part0-node{key}-data"') - read_members.append(' form_key2 = f"part0-node{key2}-data"') + read_members.append(' form_key2 = f"part0-node{key2}-offsets"') read_members.append( f' helper_obj.add_to_header(f"output part0-node{{key}}-data {{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}\\n")' ) read_members.append( ' helper_obj.add_to_header(f"output part0-node{key2}-offsets int64\\n")' ) + read_members.append(' helper_obj.add_to_init(f"0 part0-node{key2}-offsets <- stack\\n")') read_members.append( - f' content["{self.name!r}"] = {{"class": "ListOffsetArray", "primitive": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' + f' content["{self.name!r}"] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' ) read_members.append(f' #raise NotImplementedError\n helper_obj.add_to_pre(f"{{self.member({self.count_name!r})}} dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")') read_members.append(' if forth_obj.should_add_form():') @@ -1013,6 +1014,7 @@ def class_code( read_members.append( ' helper_obj.add_to_header(f"output part0-node{key2}-offsets int64\\n")' ) + read_members.append(' helper_obj.add_to_init(f"0 part0-node{key2}-offsets <- stack\\n")') read_members.append( f' content["{self.name!r}"] = {{"class": "ListOffsetArray", "primitive": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' ) @@ -1291,7 +1293,7 @@ def class_code( ) read_member_n.append(" " + read_members[-1]) read_members.append( - f" if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n print(temp_form1,'OPOPOPO')\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n content[{self.name!r}] = temp_form1\n pre,post,init,header = forth_obj.get_code_recursive(temp_model1)\n helper_obj.add_to_header(header)\n helper_obj.add_to_pre(pre)\n helper_obj.add_to_post(post)\n helper_obj.add_to_init(init)" + f" if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n print(temp_form1,'OPOPOPO')\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n content[{self.name!r}] = temp_form1\n print({self.name!r},'opopopopo')\n pre,post,init,header = forth_obj.get_code_recursive(temp_model1)\n helper_obj.add_to_header(header)\n helper_obj.add_to_pre(pre)\n helper_obj.add_to_post(post)\n helper_obj.add_to_init(init)" ) strided_interpretation.append( f" members.append(({self.name!r}, cls._stl_container{len(containers)}.strided_interpretation(file, header, tobject_header, breadcrumbs)))" diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index 557b6ac33..5642ee1a7 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -167,7 +167,9 @@ def test_15(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["CollectionTree/AnalysisHLT_mu24_ilooseAuxDyn.TrigMatchedObjects"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][-1][-1]['m_persKey'] == 980095599 # py[-1] == > (version 1) at 0x7fb9b9d24c10>], ...] at 0x7fb9b9d29250> @@ -175,7 +177,9 @@ def test_16(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: branch = file["CollectionTree/AnalysisHLT_mu40AuxDyn.TrigMatchedObjects"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][0]['m_persKey'][0] == 980095599 # py[-1] == > (version 1) at 0x7f6e29be5cd0>], ...] at 0x7f6e29bea250> @@ -268,14 +272,12 @@ def test_25(): assert py[0][-1] == 'numuCCAnalysis' # py[-1] == array(['psychePolicy', 'psycheEventModel', 'psycheCore', 'psycheUtils', 'psycheND280Utils', 'psycheIO', 'psycheSelections', 'psycheSystematics', 'highlandEventModel', 'highlandTools', 'highlandCore', 'highlandCorrections', 'highlandIO', 'baseAnalysis', 'baseTrackerAnalysis', 'numuCCAnalysis'], dtype=object) -# make it last - -def test_26(): - with uproot.open(skhep_testdata.data_path("uproot-issue-208.root")) as file: - branch = file["config/SEL/SEL._firstSteps"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) +# def test_26(): +# with uproot.open(skhep_testdata.data_path("uproot-issue-208.root")) as file: +# branch = file["config/SEL/SEL._firstSteps"] +# interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) +# py = branch.array(interp, library="np", entry_stop=2) # py[-1] == array([], dtype=object) @@ -298,15 +300,13 @@ def test_28(): assert py[0][0][-1] == 10 # py[-1] == array([], dtype=object) -# some fields missing - -def test_29(): - with uproot.open(skhep_testdata.data_path("uproot-issue213.root")) as file: - branch = file["T/eventPack/fGenInfo"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) - # py[-1] == +# def test_29(): +# with uproot.open(skhep_testdata.data_path("uproot-issue213.root")) as file: +# branch = file["T/eventPack/fGenInfo"] +# interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) +# py = branch.array(interp, library="np", entry_stop=2) +# py[-1] == def test_30(): @@ -456,19 +456,19 @@ def test_44(): # py[-1] == -def test_45(): - with uproot.open(skhep_testdata.data_path("uproot-issue485.root")) as file: - branch = file["MCTrack/global"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) +# def test_45(): +# with uproot.open(skhep_testdata.data_path("uproot-issue485.root")) as file: +# branch = file["MCTrack/global"] +# interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) +# py = branch.array(interp, library="np", entry_stop=2) # py[-1] == , ...] at 0x7f88b9a8ad00> -def test_46(): - with uproot.open(skhep_testdata.data_path("uproot-issue485.root")) as file: - branch = file["MCParticle/detector1"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) +# def test_46(): +# with uproot.open(skhep_testdata.data_path("uproot-issue485.root")) as file: +# branch = file["MCParticle/detector1"] +# interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) +# py = branch.array(interp, library="np", entry_stop=2) # py[-1] == , ...] at 0x7f94bc223550> @@ -482,31 +482,30 @@ def test_47(): # py[-1] == , ...} at 0x7f53a44278b0> -def test_48(): - with uproot.open(skhep_testdata.data_path("uproot-issue494.root")) as file: - branch = file["Geant4Data/Geant4Data./Geant4Data.ions"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) +# def test_48(): +# with uproot.open(skhep_testdata.data_path("uproot-issue494.root")) as file: +# branch = file["Geant4Data/Geant4Data./Geant4Data.ions"] +# interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) +# py = branch.array(interp, library="np", entry_stop=2) #assert py[-1]['0'][-1] == 1000020040 # py[-1] == -def test_49(): - with uproot.open(skhep_testdata.data_path("uproot-issue498.root")) as file: - branch = file["MCParticle/timepix"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) +# def test_49(): +# with uproot.open(skhep_testdata.data_path("uproot-issue498.root")) as file: +# branch = file["MCParticle/timepix"] +# interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) +# py = branch.array(interp, library="np", entry_stop=2) # py[-1] == ] at 0x7f0697bf1a00> -def test_50(): - with uproot.open(skhep_testdata.data_path("uproot-issue498.root")) as file: - branch = file["PixelHit/timepix"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) +# def test_50(): +# with uproot.open(skhep_testdata.data_path("uproot-issue498.root")) as file: +# branch = file["PixelHit/timepix"] +# interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) +# py = branch.array(interp, library="np", entry_stop=2) # py[-1] == - def test_51(): with uproot.open(skhep_testdata.data_path("uproot-issue510b.root")) as file: branch = file["EDepSimEvents/Event/Primaries/Primaries.GeneratorName"] From 15f8bff371f3c0c018a81d3f90b58e827ea68a7d Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Sun, 21 Aug 2022 23:04:36 +0530 Subject: [PATCH 19/41] solved one Events class bug --- src/uproot/containers.py | 4 ++- src/uproot/interpretation/objects.py | 14 ++++++--- src/uproot/streamers.py | 31 ++++++++++++++----- .../test_0637-setup-tests-for-AwkwardForth.py | 4 ++- 4 files changed, 38 insertions(+), 15 deletions(-) diff --git a/src/uproot/containers.py b/src/uproot/containers.py index 8de665d93..7f6dfd544 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -1179,14 +1179,16 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): chunk, cursor, context, file, member_index ) else: + print(cursor._index, 'errererer') length = cursor.field(chunk, _stl_container_size, context) + print(cursor._index, 'errererer') if helper_obj.is_forth(): key = forth_obj.get_keys(1) form_key = f"part0-node{key}-offsets" helper_obj.add_to_header(f"output part0-node{key}-offsets int64\n") helper_obj.add_to_init(f"0 part0-node{key}-offsets <- stack\n") helper_obj.add_to_pre( - f"stream !I-> stack\ndup part0-node{key}-offsets +<- stack\n" + f"stream !I-> stack\n dup part0-node{key}-offsets +<- stack\n" ) # helper_obj.add_to_post("loop\n") if forth_obj.should_add_form(): diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index 4a118e71c..b74b417af 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -259,14 +259,18 @@ def basket_array_forth( "bytestops": numpy.array(byte_offsets[1:]), } ) + print(self._complete_forth_code) + print(temp_data[:100]) context["forth"].vm.stack_push(len(byte_offsets) - 1) context["forth"].vm.resume() container = {} - for elem in self._forth_form_keys: - if "offsets" in elem: - container[elem] = context["forth"].vm.output_Index64(elem) - else: - container[elem] = context["forth"].vm.output_NumpyArray(elem) + print(context["forth"].vm.outputs) + # for elem in self._forth_form_keys: + # if "offsets" in elem: + # container[elem] = context["forth"].vm.output_Index64(elem) + # else: + # container[elem] = context["forth"].vm.output_NumpyArray(elem) + container = context["forth"].vm.outputs output = awkward.from_buffers(self._form, len(byte_offsets) - 1, container) self.hook_after_basket_array( diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index 013fc2a4d..6dfba8158 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -15,6 +15,8 @@ import uproot import uproot._awkward_forth +COUNT_NAMES = [] + _canonical_typename_patterns = [ (re.compile(r"\bChar_t\b"), "char"), (re.compile(r"\bUChar_t\b"), "unsigned char"), @@ -206,6 +208,9 @@ def class_code(self): Returns Python code as a string that, when evaluated, would be a suitable :doc:`uproot.model.VersionedModel` for this class and version. """ + for element in self.elements: # (self is a TStreamerInfo) + if element.has_member("fCountName"): + COUNT_NAMES.append(element.member("fCountName")) read_members = [ " def read_members(self, chunk, cursor, context, file):", " import uproot._awkward_forth", @@ -805,6 +810,7 @@ def class_code( """ if context.get('speedbump', True): cursor.skip(1) + print(cursor._index, 'ioioioio') if helper_obj.is_forth(): helper_obj.add_to_pre('1 stream skip \\n') """.strip( @@ -830,13 +836,16 @@ def class_code( read_members.append( f' content["{self.name!r}"] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' ) - read_members.append(f' #raise NotImplementedError\n helper_obj.add_to_pre(f"{{self.member({self.count_name!r})}} dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")') + read_members.append(f' #raise NotImplementedError\n helper_obj.add_to_pre(f" dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")') read_members.append(' if forth_obj.should_add_form():') read_members.append(' forth_obj.add_form_key(form_key)') read_members.append(' forth_obj.add_form_key(form_key2)') read_members.append( - f" self._members[{self.name!r}] = cursor.array(chunk, self.member({self.count_name!r}), tmp, context);" + f" print(type(self.member({self.count_name!r})), 'YUYUYUYUY');\n" + ) + read_members.append( + f" self._members[{self.name!r}] = cursor.array(chunk, self.member({self.count_name!r}), tmp, context);\n print(self.member({self.count_name!r}), 'nmnmnmnm')\n" ) read_member_n.append(" " + read_members[-1]) @@ -951,9 +960,15 @@ def class_code( read_members.append( f' content["{fields[-1][0]}"] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[-1][0])}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}' ) - read_members.append( - f' helper_obj.add_to_pre(f"stream !{formats[-1][0]}-> part0-node{{key}}-data\\n stream pos .s cr drop\\n")' - ) + if fields[-1][0] in COUNT_NAMES: + read_members.append( + f' helper_obj.add_to_pre(f"stream !{formats[-1][0]}-> stack dup part0-node{{key}}-data <- stack\\n")' + ) + else: + read_members.append( + f' helper_obj.add_to_pre(f"stream !{formats[-1][0]}-> part0-node{{key}}-data\\n")' + ) + read_members.append( " if forth_obj.should_add_form():" ) @@ -1007,7 +1022,7 @@ def class_code( read_members.append(' key = forth_obj.get_keys(1)') read_members.append(' key2 = forth_obj.get_keys(1)') read_members.append(' form_key = f"part0-node{key}-data"') - read_members.append(' form_key2 = f"part0-node{key2}-data"') + read_members.append(' form_key2 = f"part0-node{key2}-offsets"') read_members.append( f' helper_obj.add_to_header(f"output part0-node{{key}}-data {{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}\\n")' ) @@ -1016,7 +1031,7 @@ def class_code( ) read_members.append(' helper_obj.add_to_init(f"0 part0-node{key2}-offsets <- stack\\n")') read_members.append( - f' content["{self.name!r}"] = {{"class": "ListOffsetArray", "primitive": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' + f' content["{self.name!r}"] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' ) read_members.append(f' helper_obj.add_to_pre(f"{self.array_length} dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")\n') read_members.append(' if forth_obj.should_add_form():') @@ -1289,7 +1304,7 @@ def class_code( " if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {'name': 'TOP', 'content': {}})\n" ) read_members.append( - f" self._members[{self.name!r}] = self._stl_container{len(containers)}.read(chunk, cursor, context, file, self._file, self.concrete)" + f" self._members[{self.name!r}] = self._stl_container{len(containers)}.read(chunk, cursor, context, file, self._file, self.concrete)\n print(cursor._index)" ) read_member_n.append(" " + read_members[-1]) read_members.append( diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index 5642ee1a7..e6befba33 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -452,7 +452,9 @@ def test_44(): with uproot.open(skhep_testdata.data_path("uproot-issue46.root")) as file: branch = file["tree/evt"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0]['1'][-1] == True # py[-1] == From 1c86a88eb61b1bfbc200e7345f24028483097650 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Sun, 28 Aug 2022 13:29:44 +0530 Subject: [PATCH 20/41] fixed last bug --- src/uproot/_awkward_forth.py | 78 +++++++++++++++++-- src/uproot/containers.py | 12 ++- src/uproot/interpretation/objects.py | 9 ++- src/uproot/models/TString.py | 39 +++++++++- src/uproot/streamers.py | 29 ++++--- .../test_0637-setup-tests-for-AwkwardForth.py | 4 +- 6 files changed, 137 insertions(+), 34 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index 7a513c3f4..07fe6147b 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -37,6 +37,7 @@ def __init__(self, aform=None, count_obj=0, var_set=False): self.dummy_aform = None self.aform = aform self.top_form = None + self.prev_form = None self.awkward_model = {"name": "TOP", "content": {}} self._prev_node = self.awkward_model self.ref_list = [] @@ -55,6 +56,7 @@ def traverse_aform(self): def replace_form_and_model(self, form, model): temp_node = self.awkward_model + temp_prev_form = self.prev_form temp_node_top = self._prev_node self.awkward_model = model self._prev_node = self.awkward_model @@ -62,7 +64,7 @@ def replace_form_and_model(self, form, model): temp_form_top = self.top_form self.top_form = None self.aform = form - return temp_node, temp_node_top, temp_form, temp_form_top + return temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form def get_code_recursive(self, node): print(node, 'aass') @@ -86,7 +88,13 @@ def should_add_form(self): if self.awkward_model["content"] is None: return False else: - return not bool(self.awkward_model["content"]) + print(self.awkward_model['content'].keys) + if len(self.awkward_model['content'].keys()) == 0: + return True + elif self.awkward_model['content']['name'] == 'dynamic': + return True + else: + return False def get_temp_form_top(self): return self.top_dummy @@ -97,6 +105,7 @@ def set_dummy_none(self, temp_top, temp_form, temp_flag): self.dummy_form = temp_flag def add_form(self, aform, conlen=0, traverse=1): + print(self.var_set, 'ARARARARR') if self.dummy_form: if self.dummy_aform is None: self.dummy_aform = aform @@ -126,17 +135,51 @@ def add_form(self, aform, conlen=0, traverse=1): if "content" in self.aform.keys(): if self.aform["content"] == "NULL": self.aform["content"] = aform + self.prev_form = self.aform if traverse == 2: self.aform = self.aform["content"]["content"] + self.prev_form = self.prev_form["content"] else: self.aform = self.aform["content"] else: raise ValueError elif "contents" in self.aform.keys(): - if len(self.aform["contents"]) == conlen: - return + if self.aform['class'] == 'RecordArray': + if self.prev_form != None: + self.prev_form['content'] = aform + self.aform = aform + else: + self.top_form = aform + self.aform = aform + # for elem in aform['contents'].keys(): + # if self.depth(self.aform['contents'][elem])< self.depth(aform['contents'][elem]): + # aform['contents'][elem] = self.replace_keys(self.aform['contents'][elem],aform['contents'][elem]) + # self.aform['contents'][elem] = aform['contents'][elem] + elif len(self.aform["contents"]) == conlen: + pass else: raise ValueError + def depth(self,form): + count = 0 + temp = form + while 'content' in temp.keys(): + count += 1 + if isinstance(temp['content'],str): + break + temp = temp['content'] + return count + def replace_keys(self, old, new): + temp = new + print('++++++++++++++++++++++') + print(old, new) + print('++++++++++++++++++++++') + while old != 'NULL': + new['form_key'] = old['form_key'] + new = new['content'] + old = old['content'] + print(temp) + return temp + def get_keys(self, num_keys): if num_keys == 1: @@ -206,9 +249,30 @@ def add_node( self.awkward_model = self.awkward_model["content"] return temp_obj else: - temp_node = self.awkward_model - self.awkward_model = self.awkward_model["content"] - return temp_node + if self.awkward_model['content'] != None: + print(self.awkward_model['content']) + if self.awkward_model['content']['name'] == 'dynamic': + temp_obj = { + "name": name, + "type": dtype, + "pre_code": precode, + "post_code": postcode, + "init_code": initcode, + "header_code": headercode, + "num_child": num_child, + "content": content, + } + self.awkward_model['content'] = temp_obj + self.awkward_model = self.awkward_model["content"] + return temp_obj + else: + temp_node = self.awkward_model + self.awkward_model = self.awkward_model["content"] + return temp_node + else: + temp_node = self.awkward_model + self.awkward_model = self.awkward_model["content"] + return temp_node if isinstance(self.awkward_model, list): for elem in self.awkward_model: if name in elem.values(): diff --git a/src/uproot/containers.py b/src/uproot/containers.py index 7f6dfd544..08c1541e6 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -481,7 +481,6 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): helper_obj = uproot._awkward_forth.GenHelper(context) if helper_obj.is_forth(): - print('afefe') #raise NotImplementedError forth_obj = helper_obj.get_gen_obj() keys = forth_obj.get_keys(2) @@ -1179,9 +1178,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): chunk, cursor, context, file, member_index ) else: - print(cursor._index, 'errererer') length = cursor.field(chunk, _stl_container_size, context) - print(cursor._index, 'errererer') if helper_obj.is_forth(): key = forth_obj.get_keys(1) form_key = f"part0-node{key}-offsets" @@ -1202,7 +1199,6 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if not isinstance(self._values, numpy.dtype): helper_obj.add_to_pre("0 do\n") helper_obj.add_to_post("loop\n") - print('ADFAEF') temp = forth_obj.add_node( f"node{key}", helper_obj.get_pre(), @@ -1217,12 +1213,17 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if length == 0 and helper_obj.is_forth(): forth_obj.var_set = True + if helper_obj.is_forth(): + print(forth_obj.var_set,'=========================================') values = _read_nested( self._values, length, chunk, cursor, context, file, selffile, parent ) if helper_obj.is_forth(): + print('=========================================') + print(forth_obj.top_form) + print('=========================================') forth_obj.go_to(temp) out = STLVector(values) @@ -1561,6 +1562,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): temp_node_top, temp_form, temp_form_top, + temp_prev_form ) = forth_obj.replace_form_and_model( None, temp ) @@ -1585,6 +1587,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): temp_node_top1, temp_form1, temp_form_top1, + temp_prev_form1 ) = forth_obj.replace_form_and_model( None, temp ) @@ -1619,6 +1622,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): forth_obj._prev_node = temp_node_top forth_obj.aform = temp_form forth_obj.top_form = temp_form_top + forth_obj.prev_form = temp_prev_form aform = { "class": "ListOffsetArray", "offsets": "i64", diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index b74b417af..a6299b6d0 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -259,18 +259,19 @@ def basket_array_forth( "bytestops": numpy.array(byte_offsets[1:]), } ) - print(self._complete_forth_code) - print(temp_data[:100]) context["forth"].vm.stack_push(len(byte_offsets) - 1) context["forth"].vm.resume() container = {} - print(context["forth"].vm.outputs) # for elem in self._forth_form_keys: # if "offsets" in elem: # container[elem] = context["forth"].vm.output_Index64(elem) # else: # container[elem] = context["forth"].vm.output_NumpyArray(elem) container = context["forth"].vm.outputs + # print('==================================') + # print(self._form) + # print('==================================') + # print('==================================') output = awkward.from_buffers(self._form, len(byte_offsets) - 1, container) self.hook_after_basket_array( @@ -326,7 +327,7 @@ def _discover_forth(self, data, byte_offsets, branch, context, cursor_offset): self._forth_form_keys = tuple(context["forth"].gen.form_keys) self._form = context["forth"].gen.top_form return None # we should re-read all the data with Forth - + print() return output # Forth-generation was unsuccessful: this is Python output def _assemble_forth(self, forth_obj, awkward_model): diff --git a/src/uproot/models/TString.py b/src/uproot/models/TString.py index 6e07d4d2b..8fa278291 100644 --- a/src/uproot/models/TString.py +++ b/src/uproot/models/TString.py @@ -4,9 +4,9 @@ This module defines a versionless model of ``TString``. """ - +import json import uproot - +import uproot._awkward_forth class Model_TString(uproot.model.Model, str): """ @@ -19,6 +19,12 @@ def read_numbytes_version(self, chunk, cursor, context): pass def read_members(self, chunk, cursor, context, file): + helper_obj = uproot._awkward_forth.GenHelper(context) + if helper_obj.is_forth(): + forth_obj = helper_obj.get_gen_obj() + keys = forth_obj.get_keys(2) + offsets_num = keys[0] + data_num = keys[1] if self.is_memberwise: raise NotImplementedError( """memberwise serialization of {} @@ -26,6 +32,35 @@ def read_members(self, chunk, cursor, context, file): type(self).__name__, self.file.file_path ) ) + if helper_obj.is_forth(): + helper_obj.add_to_pre( + f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup part0-node{offsets_num}-offsets +<- stack stream #!B-> part0-node{data_num}-data\n" + ) + if forth_obj.should_add_form(): + temp_aform = {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "uint8", "inner_shape": [], "has_identifier": False, "parameters": {"__array__": "char"}, "form_key": f"node{data_num}"}, "has_identifier": False, "parameters": {"__array__": "string" ,"uproot": {"as": "TString", "header": False}}, "form_key": f"node{offsets_num}"} + forth_obj.add_form(temp_aform) + + form_keys = [ + f"part0-node{data_num}-data", + f"part0-node{offsets_num}-offsets", + ] + for elem in form_keys: + forth_obj.add_form_key(elem) + helper_obj.add_to_header( + f"output part0-node{offsets_num}-offsets int64\noutput part0-node{data_num}-data uint8\n" + ) + helper_obj.add_to_init(f"0 part0-node{offsets_num}-offsets <- stack\n") + temp_form = forth_obj.add_node( + f"node{offsets_num}", + helper_obj.get_pre(), + helper_obj.get_post(), + helper_obj.get_init(), + helper_obj.get_header(), + "i64", + 0, + None, + ) + forth_obj.go_to(temp_form) self._data = cursor.string(chunk, context) def postprocess(self, chunk, cursor, context, file): diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index 6dfba8158..5b1bb2c3d 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -691,14 +691,14 @@ def class_code( # @aryan26roy: test_0637's 01,02,08,09,11,12,13,15,16,29,38,45,46,49,50 # raise NotImplementedError read_members.append( - " if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {'name': 'TOP', 'content': {}})\n" + " if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form = forth_obj.replace_form_and_model(None, {'name': 'TOP', 'content': {}})\n" ) read_members.append( f" self._bases.append(c({self.name!r}, {self.base_version!r}).read(chunk, cursor, context, file, self._file, self._parent, concrete=self.concrete))" ) read_member_n.append(" " + read_members[-1]) read_members.append( - f" if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n print(temp_node,'OPOPO')\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n #print(forth_obj._prev_node,'??????',forth_obj.awkward_model, '======',temp_model1, '/////////')\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" + f" if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n print(temp_node,'OPOPO')\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.prev_form = temp_prev_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n #print(forth_obj._prev_node,'??????',forth_obj.awkward_model, '======',temp_model1, '/////////')\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" ) # read_members.append( # " if helper_obj.is_forth():\n temp_form = forth_obj.get_temp_form_top()\n content.update(temp_form['contents'])\n forth_obj.set_dummy_none(temp_top_dummy, temp_dummy, temp_top_flag)\n" @@ -810,7 +810,6 @@ def class_code( """ if context.get('speedbump', True): cursor.skip(1) - print(cursor._index, 'ioioioio') if helper_obj.is_forth(): helper_obj.add_to_pre('1 stream skip \\n') """.strip( @@ -834,16 +833,13 @@ def class_code( ) read_members.append(' helper_obj.add_to_init(f"0 part0-node{key2}-offsets <- stack\\n")') read_members.append( - f' content["{self.name!r}"] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' + f' content[{self.name!r}] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' ) - read_members.append(f' #raise NotImplementedError\n helper_obj.add_to_pre(f" dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")') + read_members.append(f' #raise NotImplementedError\n helper_obj.add_to_pre(f" var_{self.count_name} @ dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")') read_members.append(' if forth_obj.should_add_form():') read_members.append(' forth_obj.add_form_key(form_key)') read_members.append(' forth_obj.add_form_key(form_key2)') - read_members.append( - f" print(type(self.member({self.count_name!r})), 'YUYUYUYUY');\n" - ) read_members.append( f" self._members[{self.name!r}] = cursor.array(chunk, self.member({self.count_name!r}), tmp, context);\n print(self.member({self.count_name!r}), 'nmnmnmnm')\n" ) @@ -958,11 +954,12 @@ def class_code( f' helper_obj.add_to_header(f"output part0-node{{key}}-data {uproot._awkward_forth.convert_dtype(formats[-1][0])}\\n")' ) read_members.append( - f' content["{fields[-1][0]}"] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[-1][0])}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}' + f' content[{fields[-1][0]!r}] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[-1][0])}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}' ) if fields[-1][0] in COUNT_NAMES: + read_members.append(f' helper_obj.add_to_init(f"variable var_{fields[-1][0]}\\n")') read_members.append( - f' helper_obj.add_to_pre(f"stream !{formats[-1][0]}-> stack dup part0-node{{key}}-data <- stack\\n")' + f' helper_obj.add_to_pre(f"stream !{formats[-1][0]}-> stack dup var_{fields[-1][0]} ! part0-node{{key}}-data <- stack\\n")' ) else: read_members.append( @@ -976,7 +973,7 @@ def class_code( " forth_obj.add_form_key(form_key)" ) read_members.append( - f" self._members[{fields[-1][0]!r}] = cursor.field(chunk, self._format{len(formats) - 1}, context)\n print(cursor._index)" + f" self._members[{fields[-1][0]!r}] = cursor.field(chunk, self._format{len(formats) - 1}, context)\n #print(cursor._index)" ) else: @@ -991,7 +988,7 @@ def class_code( ) # read_members.append(' helper_obj.add_to_init(f"0 part0-node{key}-offsets <- stack\\n")') read_members.append( - f' content["{fields[0][i]}"] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[0][i])}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}' + f' content[{fields[0][i]!r}] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[0][i])}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}' ) read_members.append( f' helper_obj.add_to_pre(f"stream !{formats[0][i]}-> part0-node{{key}}-data\\n")' @@ -1031,7 +1028,7 @@ def class_code( ) read_members.append(' helper_obj.add_to_init(f"0 part0-node{key2}-offsets <- stack\\n")') read_members.append( - f' content["{self.name!r}"] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' + f' content[{self.name!r}] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' ) read_members.append(f' helper_obj.add_to_pre(f"{self.array_length} dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")\n') read_members.append(' if forth_obj.should_add_form():') @@ -1301,14 +1298,14 @@ def class_code( # @aryan26roy: test_0637's 35,38,39,44,45,47,50,56 read_members.append( - " if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {'name': 'TOP', 'content': {}})\n" + " if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form = forth_obj.replace_form_and_model(None, {'name': 'TOP', 'content': {}})\n" ) read_members.append( f" self._members[{self.name!r}] = self._stl_container{len(containers)}.read(chunk, cursor, context, file, self._file, self.concrete)\n print(cursor._index)" ) read_member_n.append(" " + read_members[-1]) read_members.append( - f" if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n print(temp_form1,'OPOPOPO')\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n content[{self.name!r}] = temp_form1\n print({self.name!r},'opopopopo')\n pre,post,init,header = forth_obj.get_code_recursive(temp_model1)\n helper_obj.add_to_header(header)\n helper_obj.add_to_pre(pre)\n helper_obj.add_to_post(post)\n helper_obj.add_to_init(init)" + f" if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj.prev_form = temp_prev_form\n print(temp_form1,'OPOPOPO', {len(containers)})\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n content[{self.name!r}] = temp_form1\n #print({self.name!r},'opopopopo')\n pre,post,init,header = forth_obj.get_code_recursive(temp_model1)\n helper_obj.add_to_header(header)\n helper_obj.add_to_pre(pre)\n helper_obj.add_to_post(post)\n helper_obj.add_to_init(init)" ) strided_interpretation.append( f" members.append(({self.name!r}, cls._stl_container{len(containers)}.strided_interpretation(file, header, tobject_header, breadcrumbs)))" @@ -1539,7 +1536,7 @@ def class_code( # @aryan26roy: test_0637's 01,02,29,45,46,49,50,56 read_members.append( - f" if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top = forth_obj.replace_form_and_model(None, {{'name': 'TOP', 'content': {{}}}})\n self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)\n if helper_obj.is_forth():\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n #print(temp_node,'OPOPOOP')\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n temp_var = forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content['{self.name}'] = temp_form1\n forth_obj.enable_adding()" + f" if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form = forth_obj.replace_form_and_model(None, {{'name': 'TOP', 'content': {{}}}})\n self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)\n if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj.prev_form = temp_prev_form\n #print(temp_form1,'OPOPOPO')\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n content[{self.name!r}] = temp_form1\n #print({self.name!r},'opopopopo')\n print(temp_form_top,'========')\n pre,post,init,header = forth_obj.get_code_recursive(temp_model1)\n helper_obj.add_to_header(header)\n helper_obj.add_to_pre(pre)\n helper_obj.add_to_post(post)\n helper_obj.add_to_init(init)" ) read_member_n.append( " " diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index e6befba33..86aa5a4af 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -563,7 +563,9 @@ def test_56(): ) as file: branch = file["tree/evt"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) + interp._forth = True + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1]['SliceU64'][0] == 1 # py[-1] == From 7f35e2c3c8202cfc6f8e9a35146f455be9c50517 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Sun, 28 Aug 2022 13:34:59 +0530 Subject: [PATCH 21/41] removed all debug statements --- src/uproot/_awkward_forth.py | 8 -------- src/uproot/containers.py | 6 +----- src/uproot/interpretation/objects.py | 9 --------- src/uproot/models/TRef.py | 3 --- src/uproot/streamers.py | 14 +++++++------- 5 files changed, 8 insertions(+), 32 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index 07fe6147b..ac0ab2297 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -67,7 +67,6 @@ def replace_form_and_model(self, form, model): return temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form def get_code_recursive(self, node): - print(node, 'aass') if 'content' in node.keys(): if node['content'] is None: return ''.join(node['pre_code']), ''.join(node['post_code']) , node['init_code'], node['header_code'] @@ -88,7 +87,6 @@ def should_add_form(self): if self.awkward_model["content"] is None: return False else: - print(self.awkward_model['content'].keys) if len(self.awkward_model['content'].keys()) == 0: return True elif self.awkward_model['content']['name'] == 'dynamic': @@ -105,7 +103,6 @@ def set_dummy_none(self, temp_top, temp_form, temp_flag): self.dummy_form = temp_flag def add_form(self, aform, conlen=0, traverse=1): - print(self.var_set, 'ARARARARR') if self.dummy_form: if self.dummy_aform is None: self.dummy_aform = aform @@ -170,14 +167,10 @@ def depth(self,form): return count def replace_keys(self, old, new): temp = new - print('++++++++++++++++++++++') - print(old, new) - print('++++++++++++++++++++++') while old != 'NULL': new['form_key'] = old['form_key'] new = new['content'] old = old['content'] - print(temp) return temp @@ -250,7 +243,6 @@ def add_node( return temp_obj else: if self.awkward_model['content'] != None: - print(self.awkward_model['content']) if self.awkward_model['content']['name'] == 'dynamic': temp_obj = { "name": name, diff --git a/src/uproot/containers.py b/src/uproot/containers.py index 08c1541e6..e69ef3f2f 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -1213,17 +1213,13 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if length == 0 and helper_obj.is_forth(): forth_obj.var_set = True - if helper_obj.is_forth(): - print(forth_obj.var_set,'=========================================') values = _read_nested( self._values, length, chunk, cursor, context, file, selffile, parent ) if helper_obj.is_forth(): - print('=========================================') - print(forth_obj.top_form) - print('=========================================') + forth_obj.go_to(temp) out = STLVector(values) diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index a6299b6d0..27659beab 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -262,16 +262,7 @@ def basket_array_forth( context["forth"].vm.stack_push(len(byte_offsets) - 1) context["forth"].vm.resume() container = {} - # for elem in self._forth_form_keys: - # if "offsets" in elem: - # container[elem] = context["forth"].vm.output_Index64(elem) - # else: - # container[elem] = context["forth"].vm.output_NumpyArray(elem) container = context["forth"].vm.outputs - # print('==================================') - # print(self._form) - # print('==================================') - # print('==================================') output = awkward.from_buffers(self._form, len(byte_offsets) - 1, container) self.hook_after_basket_array( diff --git a/src/uproot/models/TRef.py b/src/uproot/models/TRef.py index 6ae1e557c..16e65a54d 100644 --- a/src/uproot/models/TRef.py +++ b/src/uproot/models/TRef.py @@ -149,9 +149,6 @@ def read_members(self, chunk, cursor, context, file): f"part0-node{form_keys[2]}-data", f"part0-node{form_keys[5]}-data", ] - # print("===========================================") - # print(not bool(forth_obj.awkward_model["content"]), "YUYIYIIy") - # print("===========================================") if forth_obj.should_add_form(): for elem in keys: forth_obj.add_form_key(elem) diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index 5b1bb2c3d..6045051dd 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -698,7 +698,7 @@ def class_code( ) read_member_n.append(" " + read_members[-1]) read_members.append( - f" if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n print(temp_node,'OPOPO')\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.prev_form = temp_prev_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n #print(forth_obj._prev_node,'??????',forth_obj.awkward_model, '======',temp_model1, '/////////')\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" + f" if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.prev_form = temp_prev_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" ) # read_members.append( # " if helper_obj.is_forth():\n temp_form = forth_obj.get_temp_form_top()\n content.update(temp_form['contents'])\n forth_obj.set_dummy_none(temp_top_dummy, temp_dummy, temp_top_flag)\n" @@ -841,7 +841,7 @@ def class_code( read_members.append(' forth_obj.add_form_key(form_key2)') read_members.append( - f" self._members[{self.name!r}] = cursor.array(chunk, self.member({self.count_name!r}), tmp, context);\n print(self.member({self.count_name!r}), 'nmnmnmnm')\n" + f" self._members[{self.name!r}] = cursor.array(chunk, self.member({self.count_name!r}), tmp, context);\n" ) read_member_n.append(" " + read_members[-1]) @@ -973,7 +973,7 @@ def class_code( " forth_obj.add_form_key(form_key)" ) read_members.append( - f" self._members[{fields[-1][0]!r}] = cursor.field(chunk, self._format{len(formats) - 1}, context)\n #print(cursor._index)" + f" self._members[{fields[-1][0]!r}] = cursor.field(chunk, self._format{len(formats) - 1}, context)" ) else: @@ -1035,7 +1035,7 @@ def class_code( read_members.append(' forth_obj.add_form_key(form_key)') read_members.append(' forth_obj.add_form_key(form_key2)') read_members.append( - f" self._members[{self.name!r}] = cursor.array(chunk, {self.array_length}, self._dtype{len(dtypes)}, context)\n #print(cursor._index,'\\n' ,chunk._raw_data);" + f" self._members[{self.name!r}] = cursor.array(chunk, {self.array_length}, self._dtype{len(dtypes)}, context)" ) dtypes.append(_ftype_to_dtype(self.fType)) @@ -1301,11 +1301,11 @@ def class_code( " if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form = forth_obj.replace_form_and_model(None, {'name': 'TOP', 'content': {}})\n" ) read_members.append( - f" self._members[{self.name!r}] = self._stl_container{len(containers)}.read(chunk, cursor, context, file, self._file, self.concrete)\n print(cursor._index)" + f" self._members[{self.name!r}] = self._stl_container{len(containers)}.read(chunk, cursor, context, file, self._file, self.concrete)" ) read_member_n.append(" " + read_members[-1]) read_members.append( - f" if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj.prev_form = temp_prev_form\n print(temp_form1,'OPOPOPO', {len(containers)})\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n content[{self.name!r}] = temp_form1\n #print({self.name!r},'opopopopo')\n pre,post,init,header = forth_obj.get_code_recursive(temp_model1)\n helper_obj.add_to_header(header)\n helper_obj.add_to_pre(pre)\n helper_obj.add_to_post(post)\n helper_obj.add_to_init(init)" + f" if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj.prev_form = temp_prev_form\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n content[{self.name!r}] = temp_form1\n pre,post,init,header = forth_obj.get_code_recursive(temp_model1)\n helper_obj.add_to_header(header)\n helper_obj.add_to_pre(pre)\n helper_obj.add_to_post(post)\n helper_obj.add_to_init(init)" ) strided_interpretation.append( f" members.append(({self.name!r}, cls._stl_container{len(containers)}.strided_interpretation(file, header, tobject_header, breadcrumbs)))" @@ -1536,7 +1536,7 @@ def class_code( # @aryan26roy: test_0637's 01,02,29,45,46,49,50,56 read_members.append( - f" if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form = forth_obj.replace_form_and_model(None, {{'name': 'TOP', 'content': {{}}}})\n self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)\n if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj.prev_form = temp_prev_form\n #print(temp_form1,'OPOPOPO')\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n content[{self.name!r}] = temp_form1\n #print({self.name!r},'opopopopo')\n print(temp_form_top,'========')\n pre,post,init,header = forth_obj.get_code_recursive(temp_model1)\n helper_obj.add_to_header(header)\n helper_obj.add_to_pre(pre)\n helper_obj.add_to_post(post)\n helper_obj.add_to_init(init)" + f" if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form = forth_obj.replace_form_and_model(None, {{'name': 'TOP', 'content': {{}}}})\n self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)\n if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj.prev_form = temp_prev_form\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n content[{self.name!r}] = temp_form1\n pre,post,init,header = forth_obj.get_code_recursive(temp_model1)\n helper_obj.add_to_header(header)\n helper_obj.add_to_pre(pre)\n helper_obj.add_to_post(post)\n helper_obj.add_to_init(init)" ) read_member_n.append( " " From f7388cb9b0cd732e02529962304f96ff71fe1236 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 5 Sep 2022 14:36:58 +0000 Subject: [PATCH 22/41] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/uproot/_awkward_forth.py | 73 ++++++------ src/uproot/containers.py | 28 ++--- src/uproot/models/TDatime.py | 20 +++- src/uproot/models/TString.py | 75 +++++++----- src/uproot/streamers.py | 64 ++++++---- .../test_0637-setup-tests-for-AwkwardForth.py | 112 +++++++++--------- 6 files changed, 214 insertions(+), 158 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index ac0ab2297..2b46a9df4 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -5,6 +5,7 @@ """ import numpy as np + import uproot.containers symbol_dict = { @@ -67,29 +68,34 @@ def replace_form_and_model(self, form, model): return temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form def get_code_recursive(self, node): - if 'content' in node.keys(): - if node['content'] is None: - return ''.join(node['pre_code']), ''.join(node['post_code']) , node['init_code'], node['header_code'] + if "content" in node.keys(): + if node["content"] is None: + return ( + "".join(node["pre_code"]), + "".join(node["post_code"]), + node["init_code"], + node["header_code"], + ) else: pre, post, init, header = self.get_code_recursive(node["content"]) - pre2 = ''.join(node['pre_code']) + pre2 = "".join(node["pre_code"]) pre2 = pre2 + pre - post2 = ''.join(node['post_code']) + post2 = "".join(node["post_code"]) post2 = post2 + post - init = node['init_code'] + init - header = node['header_code'] + header + init = node["init_code"] + init + header = node["header_code"] + header return pre2, post2, init, header elif self.var_set: - return '', '', '', '' + return "", "", "", "" def should_add_form(self): if "content" in self.awkward_model.keys(): if self.awkward_model["content"] is None: return False else: - if len(self.awkward_model['content'].keys()) == 0: + if len(self.awkward_model["content"].keys()) == 0: return True - elif self.awkward_model['content']['name'] == 'dynamic': + elif self.awkward_model["content"]["name"] == "dynamic": return True else: return False @@ -141,9 +147,9 @@ def add_form(self, aform, conlen=0, traverse=1): else: raise ValueError elif "contents" in self.aform.keys(): - if self.aform['class'] == 'RecordArray': + if self.aform["class"] == "RecordArray": if self.prev_form != None: - self.prev_form['content'] = aform + self.prev_form["content"] = aform self.aform = aform else: self.top_form = aform @@ -156,24 +162,25 @@ def add_form(self, aform, conlen=0, traverse=1): pass else: raise ValueError - def depth(self,form): + + def depth(self, form): count = 0 temp = form - while 'content' in temp.keys(): + while "content" in temp.keys(): count += 1 - if isinstance(temp['content'],str): + if isinstance(temp["content"], str): break - temp = temp['content'] + temp = temp["content"] return count + def replace_keys(self, old, new): temp = new - while old != 'NULL': - new['form_key'] = old['form_key'] - new = new['content'] - old = old['content'] + while old != "NULL": + new["form_key"] = old["form_key"] + new = new["content"] + old = old["content"] return temp - def get_keys(self, num_keys): if num_keys == 1: key = self.count_obj @@ -242,19 +249,19 @@ def add_node( self.awkward_model = self.awkward_model["content"] return temp_obj else: - if self.awkward_model['content'] != None: - if self.awkward_model['content']['name'] == 'dynamic': + if self.awkward_model["content"] != None: + if self.awkward_model["content"]["name"] == "dynamic": temp_obj = { - "name": name, - "type": dtype, - "pre_code": precode, - "post_code": postcode, - "init_code": initcode, - "header_code": headercode, - "num_child": num_child, - "content": content, - } - self.awkward_model['content'] = temp_obj + "name": name, + "type": dtype, + "pre_code": precode, + "post_code": postcode, + "init_code": initcode, + "header_code": headercode, + "num_child": num_child, + "content": content, + } + self.awkward_model["content"] = temp_obj self.awkward_model = self.awkward_model["content"] return temp_obj else: diff --git a/src/uproot/containers.py b/src/uproot/containers.py index 731d055e6..09f1516ce 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -105,13 +105,13 @@ def _read_nested( temp_count = context["forth"].gen.count_obj for i in range(length): if helper_obj.is_forth(): - if 'temp_ref' in context.keys(): - context["forth"].gen.go_to(context['temp_ref']) + if "temp_ref" in context.keys(): + context["forth"].gen.go_to(context["temp_ref"]) context["forth"].gen.count_obj = temp_count values[i] = model.read(chunk, cursor, context, file, selffile, parent) if helper_obj.is_forth(): - if 'temp_ref' in context.keys(): - del context['temp_ref'] + if "temp_ref" in context.keys(): + del context["temp_ref"] return values @@ -481,7 +481,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): helper_obj = uproot._awkward_forth.GenHelper(context) if helper_obj.is_forth(): - #raise NotImplementedError + # raise NotImplementedError forth_obj = helper_obj.get_gen_obj() keys = forth_obj.get_keys(2) offsets_num = keys[0] @@ -1207,7 +1207,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): 1, {}, ) - context['temp_ref'] = temp + context["temp_ref"] = temp if length == 0 and helper_obj.is_forth(): forth_obj.var_set = True @@ -1557,11 +1557,9 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): temp_node_top, temp_form, temp_form_top, - temp_prev_form - ) = forth_obj.replace_form_and_model( - None, temp - ) - context['temp_ref'] = temp + temp_prev_form, + ) = forth_obj.replace_form_and_model(None, temp) + context["temp_ref"] = temp keys = _read_nested( self._keys, length, @@ -1582,11 +1580,9 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): temp_node_top1, temp_form1, temp_form_top1, - temp_prev_form1 - ) = forth_obj.replace_form_and_model( - None, temp - ) - context['temp_ref'] = temp + temp_prev_form1, + ) = forth_obj.replace_form_and_model(None, temp) + context["temp_ref"] = temp if helper_obj.is_forth(): if not isinstance(self._keys, numpy.dtype): keys_model["content"]["post_code"].append("loop\n") diff --git a/src/uproot/models/TDatime.py b/src/uproot/models/TDatime.py index f2adf145a..5ad2d48bd 100644 --- a/src/uproot/models/TDatime.py +++ b/src/uproot/models/TDatime.py @@ -5,14 +5,14 @@ """ +import json import struct import numpy import uproot -import uproot.behaviors.TDatime import uproot._awkward_forth -import json +import uproot.behaviors.TDatime _tdatime_format1 = struct.Struct(">I") @@ -32,13 +32,21 @@ def read_members(self, chunk, cursor, context, file): key = forth_obj.get_keys(1) form_key = f"part0-node{key}-data" helper_obj.add_to_header(f"output part0-node{key}-data int32\n") - helper_obj.add_to_pre( - f"stream !I-> part0-node{key}-data\n" - ) + helper_obj.add_to_pre(f"stream !I-> part0-node{key}-data\n") form_key = f"part0-node{key}-data" if forth_obj.should_add_form(): forth_obj.add_form_key(form_key) - temp_aform = {"class": "RecordArray", "contents": {'fDatime': {"class": "NumpyArray", "primitive": "uint32", "form_key": f"node{key}"}}, "parameters": {"__record__": "TDatime"}} + temp_aform = { + "class": "RecordArray", + "contents": { + "fDatime": { + "class": "NumpyArray", + "primitive": "uint32", + "form_key": f"node{key}", + } + }, + "parameters": {"__record__": "TDatime"}, + } forth_obj.add_form(temp_aform) temp_form = forth_obj.add_node( f"node{key}", diff --git a/src/uproot/models/TString.py b/src/uproot/models/TString.py index 8fa278291..01505fe2c 100644 --- a/src/uproot/models/TString.py +++ b/src/uproot/models/TString.py @@ -5,9 +5,11 @@ """ import json + import uproot import uproot._awkward_forth + class Model_TString(uproot.model.Model, str): """ A versionless :doc:`uproot.model.Model` for ``TString``. @@ -33,34 +35,51 @@ def read_members(self, chunk, cursor, context, file): ) ) if helper_obj.is_forth(): - helper_obj.add_to_pre( - f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup part0-node{offsets_num}-offsets +<- stack stream #!B-> part0-node{data_num}-data\n" - ) - if forth_obj.should_add_form(): - temp_aform = {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "uint8", "inner_shape": [], "has_identifier": False, "parameters": {"__array__": "char"}, "form_key": f"node{data_num}"}, "has_identifier": False, "parameters": {"__array__": "string" ,"uproot": {"as": "TString", "header": False}}, "form_key": f"node{offsets_num}"} - forth_obj.add_form(temp_aform) - - form_keys = [ - f"part0-node{data_num}-data", - f"part0-node{offsets_num}-offsets", - ] - for elem in form_keys: - forth_obj.add_form_key(elem) - helper_obj.add_to_header( - f"output part0-node{offsets_num}-offsets int64\noutput part0-node{data_num}-data uint8\n" - ) - helper_obj.add_to_init(f"0 part0-node{offsets_num}-offsets <- stack\n") - temp_form = forth_obj.add_node( - f"node{offsets_num}", - helper_obj.get_pre(), - helper_obj.get_post(), - helper_obj.get_init(), - helper_obj.get_header(), - "i64", - 0, - None, - ) - forth_obj.go_to(temp_form) + helper_obj.add_to_pre( + f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup part0-node{offsets_num}-offsets +<- stack stream #!B-> part0-node{data_num}-data\n" + ) + if forth_obj.should_add_form(): + temp_aform = { + "class": "ListOffsetArray", + "offsets": "i64", + "content": { + "class": "NumpyArray", + "primitive": "uint8", + "inner_shape": [], + "has_identifier": False, + "parameters": {"__array__": "char"}, + "form_key": f"node{data_num}", + }, + "has_identifier": False, + "parameters": { + "__array__": "string", + "uproot": {"as": "TString", "header": False}, + }, + "form_key": f"node{offsets_num}", + } + forth_obj.add_form(temp_aform) + + form_keys = [ + f"part0-node{data_num}-data", + f"part0-node{offsets_num}-offsets", + ] + for elem in form_keys: + forth_obj.add_form_key(elem) + helper_obj.add_to_header( + f"output part0-node{offsets_num}-offsets int64\noutput part0-node{data_num}-data uint8\n" + ) + helper_obj.add_to_init(f"0 part0-node{offsets_num}-offsets <- stack\n") + temp_form = forth_obj.add_node( + f"node{offsets_num}", + helper_obj.get_pre(), + helper_obj.get_post(), + helper_obj.get_init(), + helper_obj.get_header(), + "i64", + 0, + None, + ) + forth_obj.go_to(temp_form) self._data = cursor.string(chunk, context) def postprocess(self, chunk, cursor, context, file): diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index f65eb859d..426b9f2b4 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -208,7 +208,7 @@ def class_code(self): Returns Python code as a string that, when evaluated, would be a suitable :doc:`uproot.model.VersionedModel` for this class and version. """ - for element in self.elements: # (self is a TStreamerInfo) + for element in self.elements: # (self is a TStreamerInfo) if element.has_member("fCountName"): COUNT_NAMES.append(element.member("fCountName")) read_members = [ @@ -828,27 +828,37 @@ def class_code( ) read_member_n.append(" " + read_members[-1].replace("\n", "\n ")) - # @aryan26roy: test_0637's 29,44,56 + # @aryan26roy: test_0637's 29,44,56 - read_members.append(' if helper_obj.is_forth():') - read_members.append(' key = forth_obj.get_keys(1)') - read_members.append(' key2 = forth_obj.get_keys(1)') + read_members.append(" if helper_obj.is_forth():") + read_members.append(" key = forth_obj.get_keys(1)") + read_members.append(" key2 = forth_obj.get_keys(1)") read_members.append(' form_key = f"part0-node{key}-data"') - read_members.append(' form_key2 = f"part0-node{key2}-offsets"') + read_members.append( + ' form_key2 = f"part0-node{key2}-offsets"' + ) read_members.append( f' helper_obj.add_to_header(f"output part0-node{{key}}-data {{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}\\n")' ) read_members.append( ' helper_obj.add_to_header(f"output part0-node{key2}-offsets int64\\n")' ) - read_members.append(' helper_obj.add_to_init(f"0 part0-node{key2}-offsets <- stack\\n")') + read_members.append( + ' helper_obj.add_to_init(f"0 part0-node{key2}-offsets <- stack\\n")' + ) read_members.append( f' content[{self.name!r}] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' ) - read_members.append(f' #raise NotImplementedError\n helper_obj.add_to_pre(f" var_{self.count_name} @ dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")') - read_members.append(' if forth_obj.should_add_form():') - read_members.append(' forth_obj.add_form_key(form_key)') - read_members.append(' forth_obj.add_form_key(form_key2)') + read_members.append( + f' #raise NotImplementedError\n helper_obj.add_to_pre(f" var_{self.count_name} @ dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")' + ) + read_members.append(" if forth_obj.should_add_form():") + read_members.append( + " forth_obj.add_form_key(form_key)" + ) + read_members.append( + " forth_obj.add_form_key(form_key2)" + ) read_members.append( f" self._members[{self.name!r}] = cursor.array(chunk, self.member({self.count_name!r}), tmp, context);\n" @@ -967,7 +977,9 @@ def class_code( f' content[{fields[-1][0]!r}] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[-1][0])}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}' ) if fields[-1][0] in COUNT_NAMES: - read_members.append(f' helper_obj.add_to_init(f"variable var_{fields[-1][0]}\\n")') + read_members.append( + f' helper_obj.add_to_init(f"variable var_{fields[-1][0]}\\n")' + ) read_members.append( f' helper_obj.add_to_pre(f"stream !{formats[-1][0]}-> stack dup var_{fields[-1][0]} ! part0-node{{key}}-data <- stack\\n")' ) @@ -1025,25 +1037,35 @@ def class_code( else: # @aryan26roy: test_0637's 44,56 - read_members.append(' if helper_obj.is_forth():') - read_members.append(' key = forth_obj.get_keys(1)') - read_members.append(' key2 = forth_obj.get_keys(1)') + read_members.append(" if helper_obj.is_forth():") + read_members.append(" key = forth_obj.get_keys(1)") + read_members.append(" key2 = forth_obj.get_keys(1)") read_members.append(' form_key = f"part0-node{key}-data"') - read_members.append(' form_key2 = f"part0-node{key2}-offsets"') + read_members.append( + ' form_key2 = f"part0-node{key2}-offsets"' + ) read_members.append( f' helper_obj.add_to_header(f"output part0-node{{key}}-data {{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}\\n")' ) read_members.append( ' helper_obj.add_to_header(f"output part0-node{key2}-offsets int64\\n")' ) - read_members.append(' helper_obj.add_to_init(f"0 part0-node{key2}-offsets <- stack\\n")') + read_members.append( + ' helper_obj.add_to_init(f"0 part0-node{key2}-offsets <- stack\\n")' + ) read_members.append( f' content[{self.name!r}] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' ) - read_members.append(f' helper_obj.add_to_pre(f"{self.array_length} dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")\n') - read_members.append(' if forth_obj.should_add_form():') - read_members.append(' forth_obj.add_form_key(form_key)') - read_members.append(' forth_obj.add_form_key(form_key2)') + read_members.append( + f' helper_obj.add_to_pre(f"{self.array_length} dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")\n' + ) + read_members.append(" if forth_obj.should_add_form():") + read_members.append( + " forth_obj.add_form_key(form_key)" + ) + read_members.append( + " forth_obj.add_form_key(form_key2)" + ) read_members.append( f" self._members[{self.name!r}] = cursor.array(chunk, {self.array_length}, self._dtype{len(dtypes)}, context)" ) diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index 86aa5a4af..865169b19 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -48,7 +48,10 @@ def test_03(): interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) assert py[0][0] == 'Converter.hepmcStatusList = "[]";\n' - assert py[0][14] == 'ToolSvc.photons.delphesArrayName = "PhotonEfficiency/photons";\nToolSvc.photons.isolationTags = "photonITags";\nToolSvc.photons.mcAssociations = "photonsToMC";\nToolSvc.photons.particles = "photons";\n' + assert ( + py[0][14] + == 'ToolSvc.photons.delphesArrayName = "PhotonEfficiency/photons";\nToolSvc.photons.isolationTags = "photonITags";\nToolSvc.photons.mcAssociations = "photonsToMC";\nToolSvc.photons.particles = "photons";\n' + ) # py[-1] == @@ -68,7 +71,7 @@ def test_04(): # branch = file["CollectionTree/TrigConfKeys"] # interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) # py = branch.array(interp, library="np", entry_stop=2) - # py[-1] == +# py[-1] == def test_06(): @@ -107,7 +110,7 @@ def test_09(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0]['0'][0] == 352341021 + assert py[0][0]["0"][0] == 352341021 # py[-1] == > (version 1) at 0x7febbf1b2fa0>], ...] at 0x7febbf1b2f40> @@ -129,7 +132,7 @@ def test_11(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0]['0'][0] == 921521854 + assert py[0][0]["0"][0] == 921521854 # py[-1] == > (version 1) at 0x7f636a9484c0>], ...] at 0x7f636a948eb0> @@ -139,7 +142,7 @@ def test_12(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0]['1'][0] == 2 + assert py[0][0]["1"][0] == 2 # py[-1] == > (version 1) at 0x7fc259ae37c0>], ...] at 0x7fc259ae3f10> @@ -149,7 +152,7 @@ def test_13(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=3) - assert py[0][0]['1'][0] == 0 + assert py[0][0]["1"][0] == 0 # py[-1] == > (version 1) at 0x7fa94e968c10>], ...] at 0x7fa94e968c70> @@ -169,7 +172,7 @@ def test_15(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][-1][-1]['m_persKey'] == 980095599 + assert py[1][-1][-1]["m_persKey"] == 980095599 # py[-1] == > (version 1) at 0x7fb9b9d24c10>], ...] at 0x7fb9b9d29250> @@ -179,7 +182,7 @@ def test_16(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][0]['m_persKey'][0] == 980095599 + assert py[1][0]["m_persKey"][0] == 980095599 # py[-1] == > (version 1) at 0x7f6e29be5cd0>], ...] at 0x7f6e29bea250> @@ -269,7 +272,7 @@ def test_25(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][-1] == 'numuCCAnalysis' + assert py[0][-1] == "numuCCAnalysis" # py[-1] == array(['psychePolicy', 'psycheEventModel', 'psycheCore', 'psycheUtils', 'psycheND280Utils', 'psycheIO', 'psycheSelections', 'psycheSystematics', 'highlandEventModel', 'highlandTools', 'highlandCore', 'highlandCorrections', 'highlandIO', 'baseAnalysis', 'baseTrackerAnalysis', 'numuCCAnalysis'], dtype=object) @@ -278,7 +281,7 @@ def test_25(): # branch = file["config/SEL/SEL._firstSteps"] # interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) # py = branch.array(interp, library="np", entry_stop=2) - # py[-1] == array([], dtype=object) +# py[-1] == array([], dtype=object) def test_27(): @@ -287,7 +290,7 @@ def test_27(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0][-1] == 'muFGD+Np' + assert py[0][0][-1] == "muFGD+Np" # py[-1] == array([], dtype=object) @@ -335,7 +338,7 @@ def test_32(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1] == 'two' + assert py[-1] == "two" # py[-1] == "two" @@ -345,7 +348,7 @@ def test_33(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][4]['1'][-1] == 1.0 + assert py[0][4]["1"][-1] == 1.0 # py[-1] == @@ -355,7 +358,7 @@ def test_34(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1] == '$Name: geant4-10-05-patch-01 $' + assert py[-1] == "$Name: geant4-10-05-patch-01 $" def test_35(): @@ -364,7 +367,7 @@ def test_35(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]['1'][0]['name'] == 'anti_alpha' + assert py[0]["1"][0]["name"] == "anti_alpha" # py[-1] == , ...} at 0x7fb557a012e0> @@ -374,7 +377,7 @@ def test_36(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][-1] == 'MP_F_99.' + assert py[0][-1] == "MP_F_99." # py[-1] == @@ -384,7 +387,7 @@ def test_37(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][555]['2'] == 100.94856572890848 + assert py[0][555]["2"] == 100.94856572890848 # py[-1] == array([, , , ..., , , ], dtype=object) @@ -394,7 +397,7 @@ def test_38(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]['zp'][0] == pytest.approx(0.9999998807907104) + assert py[0]["zp"][0] == pytest.approx(0.9999998807907104) # py[-1] == (version 4) at 0x7f9be2f7b2e0> @@ -404,7 +407,7 @@ def test_39(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1]['0'] == -1000020030 + assert py[-1][1]["0"] == -1000020030 # py[-1] == , ...} at 0x7f6d057397f0> @@ -414,7 +417,7 @@ def test_40(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]['fDatime'] == 1749155840 + assert py[0]["fDatime"] == 1749155840 # py[-1] == @@ -454,7 +457,7 @@ def test_44(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]['1'][-1] == True + assert py[0]["1"][-1] == True # py[-1] == @@ -463,7 +466,7 @@ def test_44(): # branch = file["MCTrack/global"] # interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) # py = branch.array(interp, library="np", entry_stop=2) - # py[-1] == , ...] at 0x7f88b9a8ad00> +# py[-1] == , ...] at 0x7f88b9a8ad00> # def test_46(): @@ -471,7 +474,7 @@ def test_44(): # branch = file["MCParticle/detector1"] # interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) # py = branch.array(interp, library="np", entry_stop=2) - # py[-1] == , ...] at 0x7f94bc223550> +# py[-1] == , ...] at 0x7f94bc223550> def test_47(): @@ -480,7 +483,7 @@ def test_47(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][-1]['1']['mass'] == pytest.approx(3.727379) + assert py[-1][-1]["1"]["mass"] == pytest.approx(3.727379) # py[-1] == , ...} at 0x7f53a44278b0> @@ -489,8 +492,8 @@ def test_47(): # branch = file["Geant4Data/Geant4Data./Geant4Data.ions"] # interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) # py = branch.array(interp, library="np", entry_stop=2) - #assert py[-1]['0'][-1] == 1000020040 - # py[-1] == +# assert py[-1]['0'][-1] == 1000020040 +# py[-1] == # def test_49(): @@ -498,7 +501,7 @@ def test_47(): # branch = file["MCParticle/timepix"] # interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) # py = branch.array(interp, library="np", entry_stop=2) - # py[-1] == ] at 0x7f0697bf1a00> +# py[-1] == ] at 0x7f0697bf1a00> # def test_50(): @@ -506,7 +509,8 @@ def test_47(): # branch = file["PixelHit/timepix"] # interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) # py = branch.array(interp, library="np", entry_stop=2) - # py[-1] == +# py[-1] == + def test_51(): with uproot.open(skhep_testdata.data_path("uproot-issue510b.root")) as file: @@ -514,7 +518,7 @@ def test_51(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1] == 'GENIE:fixed@density-fixed' + assert py[-1] == "GENIE:fixed@density-fixed" def test_52(): @@ -523,7 +527,7 @@ def test_52(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][-1]['refs'][-1] == 2223 + assert py[1][-1]["refs"][-1] == 2223 # py[-1] == array([, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ], dtype=object) @@ -533,7 +537,7 @@ def test_53(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][-1]['fE'] == 0.0 + assert py[0][-1]["fE"] == 0.0 # py[-1] == array([, , , , , ], dtype=object) @@ -543,7 +547,7 @@ def test_54(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][-1][4]['fE'] == 0.0 + assert py[0][-1][4]["fE"] == 0.0 # py[-1] == array([[, , , , ], [, , , , ], [, , , , ], [, , , , ], [, , , , ], [, , , , ]], dtype=object) @@ -565,7 +569,7 @@ def test_56(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1]['SliceU64'][0] == 1 + assert py[1]["SliceU64"][0] == 1 # py[-1] == @@ -575,7 +579,7 @@ def test_57(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1] == 'two' + assert py[-1][1] == "two" # py[-1] == @@ -585,7 +589,7 @@ def test_58(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1] == 'two' + assert py[1][1] == "two" # py[-1] == @@ -605,7 +609,7 @@ def test_60(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1][1] == 'two' + assert py[-1][1][1] == "two" # py[-1] == @@ -615,7 +619,7 @@ def test_61(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1][1] == 'two' + assert py[-1][1][1] == "two" # py[-1] == @@ -635,7 +639,7 @@ def test_63(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1][1] == 'two' + assert py[-1][1][1] == "two" # py[-1] == @@ -655,7 +659,7 @@ def test_65(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1] == 'two' + assert py[-1][1] == "two" # py[-1] == @@ -665,7 +669,7 @@ def test_66(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1]['1'][1] == 2 + assert py[-1]["1"][1] == 2 # py[-1] == @@ -675,7 +679,7 @@ def test_67(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1] == 2 + assert py[1][1]["1"][1] == 2 # py[-1] == @@ -685,7 +689,7 @@ def test_68(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1] == 'two' + assert py[1][1]["1"][1] == "two" # py[-1] == @@ -695,7 +699,7 @@ def test_69(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1] == 2 + assert py[1][1]["1"][1] == 2 # py[-1] == @@ -705,7 +709,7 @@ def test_70(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1] == 'two' + assert py[1][1]["1"][1] == "two" # py[-1] == @@ -715,7 +719,7 @@ def test_71(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'] == 2 + assert py[1][1]["1"] == 2 # py[-1] == @@ -725,7 +729,7 @@ def test_72(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1] == 2 + assert py[1][1]["1"][1] == 2 # py[-1] == @@ -735,7 +739,7 @@ def test_73(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1] == 'two' + assert py[1][1]["1"][1] == "two" # py[-1] == @@ -745,7 +749,7 @@ def test_74(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1] == 2 + assert py[1][1]["1"][1] == 2 # py[-1] == @@ -755,7 +759,7 @@ def test_75(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1] == 'two' + assert py[1][1]["1"][1] == "two" # py[-1] == @@ -765,7 +769,7 @@ def test_76(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1][1] == 2 + assert py[1][1]["1"][1][1] == 2 # py[-1] == @@ -775,7 +779,7 @@ def test_77(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1][1] == 2 + assert py[1][1]["1"][1][1] == 2 # py[-1] == @@ -785,7 +789,7 @@ def test_78(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'] == 'TWO' + assert py[1][1]["1"] == "TWO" # py[-1] == @@ -795,7 +799,7 @@ def test_79(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'] == 'TWO' + assert py[1][1]["1"] == "TWO" # py[-1] == @@ -817,5 +821,5 @@ def test_81(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]['1'] == pytest.approx(2.5636332035064697) + assert py[0]["1"] == pytest.approx(2.5636332035064697) # py[-1] == From d0468cec89d0fe2e174f25055417deb3e936c6c2 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Mon, 5 Sep 2022 20:23:42 +0530 Subject: [PATCH 23/41] linting --- .github/workflows/build-test.yml | 1 - src/uproot/_awkward_forth.py | 80 +++++++------ src/uproot/containers.py | 30 ++--- src/uproot/interpretation/objects.py | 1 - src/uproot/models/TDatime.py | 19 ++- src/uproot/models/TString.py | 76 +++++++----- src/uproot/streamers.py | 70 +++++++---- .../test_0637-setup-tests-for-AwkwardForth.py | 112 +++++++++--------- 8 files changed, 222 insertions(+), 167 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 23a6ab103..69c4db814 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -24,7 +24,6 @@ jobs: python-version: "3.6" - platform: "windows-latest" python-version: "3.7" - fail-fast: false runs-on: "${{ matrix.platform }}" timeout-minutes: 30 diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index ac0ab2297..7399da599 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -5,6 +5,7 @@ """ import numpy as np + import uproot.containers symbol_dict = { @@ -22,6 +23,9 @@ def check_depth(node): + """ + This method checks the depth of the provided container + """ if isinstance(node, uproot.containers.AsVector): return 1 @@ -67,29 +71,34 @@ def replace_form_and_model(self, form, model): return temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form def get_code_recursive(self, node): - if 'content' in node.keys(): - if node['content'] is None: - return ''.join(node['pre_code']), ''.join(node['post_code']) , node['init_code'], node['header_code'] + if "content" in node.keys(): + if node["content"] is None: + return ( + "".join(node["pre_code"]), + "".join(node["post_code"]), + node["init_code"], + node["header_code"], + ) else: pre, post, init, header = self.get_code_recursive(node["content"]) - pre2 = ''.join(node['pre_code']) + pre2 = "".join(node["pre_code"]) pre2 = pre2 + pre - post2 = ''.join(node['post_code']) + post2 = "".join(node["post_code"]) post2 = post2 + post - init = node['init_code'] + init - header = node['header_code'] + header + init = node["init_code"] + init + header = node["header_code"] + header return pre2, post2, init, header elif self.var_set: - return '', '', '', '' + return "", "", "", "" def should_add_form(self): if "content" in self.awkward_model.keys(): if self.awkward_model["content"] is None: return False else: - if len(self.awkward_model['content'].keys()) == 0: + if len(self.awkward_model["content"].keys()) == 0: return True - elif self.awkward_model['content']['name'] == 'dynamic': + elif self.awkward_model["content"]["name"] == "dynamic": return True else: return False @@ -141,9 +150,9 @@ def add_form(self, aform, conlen=0, traverse=1): else: raise ValueError elif "contents" in self.aform.keys(): - if self.aform['class'] == 'RecordArray': - if self.prev_form != None: - self.prev_form['content'] = aform + if self.aform["class"] == "RecordArray": + if self.prev_form is not None: + self.prev_form["content"] = aform self.aform = aform else: self.top_form = aform @@ -156,24 +165,25 @@ def add_form(self, aform, conlen=0, traverse=1): pass else: raise ValueError - def depth(self,form): + + def depth(self, form): count = 0 temp = form - while 'content' in temp.keys(): + while "content" in temp.keys(): count += 1 - if isinstance(temp['content'],str): + if isinstance(temp["content"], str): break - temp = temp['content'] + temp = temp["content"] return count + def replace_keys(self, old, new): temp = new - while old != 'NULL': - new['form_key'] = old['form_key'] - new = new['content'] - old = old['content'] + while old != "NULL": + new["form_key"] = old["form_key"] + new = new["content"] + old = old["content"] return temp - def get_keys(self, num_keys): if num_keys == 1: key = self.count_obj @@ -210,7 +220,7 @@ def get_ref(self, index): def enable_adding(self): if "content" in self.awkward_model.keys(): - if self.awkward_model["content"] == None: + if self.awkward_model["content"] is None: self.awkward_model["content"] = {} def add_node_whole(self, new_node, ref_latest): @@ -242,19 +252,19 @@ def add_node( self.awkward_model = self.awkward_model["content"] return temp_obj else: - if self.awkward_model['content'] != None: - if self.awkward_model['content']['name'] == 'dynamic': + if self.awkward_model["content"] is not None: + if self.awkward_model["content"]["name"] == "dynamic": temp_obj = { - "name": name, - "type": dtype, - "pre_code": precode, - "post_code": postcode, - "init_code": initcode, - "header_code": headercode, - "num_child": num_child, - "content": content, - } - self.awkward_model['content'] = temp_obj + "name": name, + "type": dtype, + "pre_code": precode, + "post_code": postcode, + "init_code": initcode, + "header_code": headercode, + "num_child": num_child, + "content": content, + } + self.awkward_model["content"] = temp_obj self.awkward_model = self.awkward_model["content"] return temp_obj else: diff --git a/src/uproot/containers.py b/src/uproot/containers.py index 731d055e6..4fa5509cf 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -105,13 +105,13 @@ def _read_nested( temp_count = context["forth"].gen.count_obj for i in range(length): if helper_obj.is_forth(): - if 'temp_ref' in context.keys(): - context["forth"].gen.go_to(context['temp_ref']) + if "temp_ref" in context.keys(): + context["forth"].gen.go_to(context["temp_ref"]) context["forth"].gen.count_obj = temp_count values[i] = model.read(chunk, cursor, context, file, selffile, parent) if helper_obj.is_forth(): - if 'temp_ref' in context.keys(): - del context['temp_ref'] + if "temp_ref" in context.keys(): + del context["temp_ref"] return values @@ -481,7 +481,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): helper_obj = uproot._awkward_forth.GenHelper(context) if helper_obj.is_forth(): - #raise NotImplementedError + # raise NotImplementedError forth_obj = helper_obj.get_gen_obj() keys = forth_obj.get_keys(2) offsets_num = keys[0] @@ -1207,7 +1207,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): 1, {}, ) - context['temp_ref'] = temp + context["temp_ref"] = temp if length == 0 and helper_obj.is_forth(): forth_obj.var_set = True @@ -1557,11 +1557,9 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): temp_node_top, temp_form, temp_form_top, - temp_prev_form - ) = forth_obj.replace_form_and_model( - None, temp - ) - context['temp_ref'] = temp + temp_prev_form, + ) = forth_obj.replace_form_and_model(None, temp) + context["temp_ref"] = temp keys = _read_nested( self._keys, length, @@ -1582,11 +1580,9 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): temp_node_top1, temp_form1, temp_form_top1, - temp_prev_form1 - ) = forth_obj.replace_form_and_model( - None, temp - ) - context['temp_ref'] = temp + temp_prev_form1, + ) = forth_obj.replace_form_and_model(None, temp) + context["temp_ref"] = temp if helper_obj.is_forth(): if not isinstance(self._keys, numpy.dtype): keys_model["content"]["post_code"].append("loop\n") @@ -1632,7 +1628,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): forth_obj.add_form_key(form_key) forth_obj.add_form(aform) temp = forth_obj.add_node( - f"nodeMap", + "nodeMap", helper_obj.get_pre(), helper_obj.get_post(), helper_obj.get_init(), diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index ff32ad8f0..d2c99a94f 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -281,7 +281,6 @@ def _discover_forth(self, data, byte_offsets, branch, context, cursor_offset): self._forth_form_keys = tuple(context["forth"].gen.form_keys) self._form = context["forth"].gen.top_form return None # we should re-read all the data with Forth - print() return output # Forth-generation was unsuccessful: this is Python output def _assemble_forth(self, forth_obj, awkward_model): diff --git a/src/uproot/models/TDatime.py b/src/uproot/models/TDatime.py index f2adf145a..23bdec5f8 100644 --- a/src/uproot/models/TDatime.py +++ b/src/uproot/models/TDatime.py @@ -10,9 +10,8 @@ import numpy import uproot -import uproot.behaviors.TDatime import uproot._awkward_forth -import json +import uproot.behaviors.TDatime _tdatime_format1 = struct.Struct(">I") @@ -32,13 +31,21 @@ def read_members(self, chunk, cursor, context, file): key = forth_obj.get_keys(1) form_key = f"part0-node{key}-data" helper_obj.add_to_header(f"output part0-node{key}-data int32\n") - helper_obj.add_to_pre( - f"stream !I-> part0-node{key}-data\n" - ) + helper_obj.add_to_pre(f"stream !I-> part0-node{key}-data\n") form_key = f"part0-node{key}-data" if forth_obj.should_add_form(): forth_obj.add_form_key(form_key) - temp_aform = {"class": "RecordArray", "contents": {'fDatime': {"class": "NumpyArray", "primitive": "uint32", "form_key": f"node{key}"}}, "parameters": {"__record__": "TDatime"}} + temp_aform = { + "class": "RecordArray", + "contents": { + "fDatime": { + "class": "NumpyArray", + "primitive": "uint32", + "form_key": f"node{key}", + } + }, + "parameters": {"__record__": "TDatime"}, + } forth_obj.add_form(temp_aform) temp_form = forth_obj.add_node( f"node{key}", diff --git a/src/uproot/models/TString.py b/src/uproot/models/TString.py index 8fa278291..a2988d235 100644 --- a/src/uproot/models/TString.py +++ b/src/uproot/models/TString.py @@ -4,10 +4,11 @@ This module defines a versionless model of ``TString``. """ -import json + import uproot import uproot._awkward_forth + class Model_TString(uproot.model.Model, str): """ A versionless :doc:`uproot.model.Model` for ``TString``. @@ -33,34 +34,51 @@ def read_members(self, chunk, cursor, context, file): ) ) if helper_obj.is_forth(): - helper_obj.add_to_pre( - f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup part0-node{offsets_num}-offsets +<- stack stream #!B-> part0-node{data_num}-data\n" - ) - if forth_obj.should_add_form(): - temp_aform = {"class": "ListOffsetArray", "offsets": "i64", "content": {"class": "NumpyArray", "primitive": "uint8", "inner_shape": [], "has_identifier": False, "parameters": {"__array__": "char"}, "form_key": f"node{data_num}"}, "has_identifier": False, "parameters": {"__array__": "string" ,"uproot": {"as": "TString", "header": False}}, "form_key": f"node{offsets_num}"} - forth_obj.add_form(temp_aform) - - form_keys = [ - f"part0-node{data_num}-data", - f"part0-node{offsets_num}-offsets", - ] - for elem in form_keys: - forth_obj.add_form_key(elem) - helper_obj.add_to_header( - f"output part0-node{offsets_num}-offsets int64\noutput part0-node{data_num}-data uint8\n" - ) - helper_obj.add_to_init(f"0 part0-node{offsets_num}-offsets <- stack\n") - temp_form = forth_obj.add_node( - f"node{offsets_num}", - helper_obj.get_pre(), - helper_obj.get_post(), - helper_obj.get_init(), - helper_obj.get_header(), - "i64", - 0, - None, - ) - forth_obj.go_to(temp_form) + helper_obj.add_to_pre( + f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup part0-node{offsets_num}-offsets +<- stack stream #!B-> part0-node{data_num}-data\n" + ) + if forth_obj.should_add_form(): + temp_aform = { + "class": "ListOffsetArray", + "offsets": "i64", + "content": { + "class": "NumpyArray", + "primitive": "uint8", + "inner_shape": [], + "has_identifier": False, + "parameters": {"__array__": "char"}, + "form_key": f"node{data_num}", + }, + "has_identifier": False, + "parameters": { + "__array__": "string", + "uproot": {"as": "TString", "header": False}, + }, + "form_key": f"node{offsets_num}", + } + forth_obj.add_form(temp_aform) + + form_keys = [ + f"part0-node{data_num}-data", + f"part0-node{offsets_num}-offsets", + ] + for elem in form_keys: + forth_obj.add_form_key(elem) + helper_obj.add_to_header( + f"output part0-node{offsets_num}-offsets int64\noutput part0-node{data_num}-data uint8\n" + ) + helper_obj.add_to_init(f"0 part0-node{offsets_num}-offsets <- stack\n") + temp_form = forth_obj.add_node( + f"node{offsets_num}", + helper_obj.get_pre(), + helper_obj.get_post(), + helper_obj.get_init(), + helper_obj.get_header(), + "i64", + 0, + None, + ) + forth_obj.go_to(temp_form) self._data = cursor.string(chunk, context) def postprocess(self, chunk, cursor, context, file): diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index f65eb859d..84e73de0a 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -208,7 +208,7 @@ def class_code(self): Returns Python code as a string that, when evaluated, would be a suitable :doc:`uproot.model.VersionedModel` for this class and version. """ - for element in self.elements: # (self is a TStreamerInfo) + for element in self.elements: # (self is a TStreamerInfo) if element.has_member("fCountName"): COUNT_NAMES.append(element.member("fCountName")) read_members = [ @@ -220,11 +220,11 @@ def class_code(self): " )", ] read_members.append( - f""" + """ helper_obj = uproot._awkward_forth.GenHelper(context) if helper_obj.is_forth(): forth_obj = helper_obj.get_gen_obj() - content = {{}}\n""" + content = {}\n""" ) read_member_n = [ " def read_member_n(self, chunk, cursor, context, file, member_index):" @@ -705,7 +705,7 @@ def class_code( ) read_member_n.append(" " + read_members[-1]) read_members.append( - f" if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.prev_form = temp_prev_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" + " if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.prev_form = temp_prev_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" ) # read_members.append( # " if helper_obj.is_forth():\n temp_form = forth_obj.get_temp_form_top()\n content.update(temp_form['contents'])\n forth_obj.set_dummy_none(temp_top_dummy, temp_dummy, temp_top_flag)\n" @@ -828,27 +828,37 @@ def class_code( ) read_member_n.append(" " + read_members[-1].replace("\n", "\n ")) - # @aryan26roy: test_0637's 29,44,56 + # @aryan26roy: test_0637's 29,44,56 - read_members.append(' if helper_obj.is_forth():') - read_members.append(' key = forth_obj.get_keys(1)') - read_members.append(' key2 = forth_obj.get_keys(1)') + read_members.append(" if helper_obj.is_forth():") + read_members.append(" key = forth_obj.get_keys(1)") + read_members.append(" key2 = forth_obj.get_keys(1)") read_members.append(' form_key = f"part0-node{key}-data"') - read_members.append(' form_key2 = f"part0-node{key2}-offsets"') + read_members.append( + ' form_key2 = f"part0-node{key2}-offsets"' + ) read_members.append( f' helper_obj.add_to_header(f"output part0-node{{key}}-data {{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}\\n")' ) read_members.append( ' helper_obj.add_to_header(f"output part0-node{key2}-offsets int64\\n")' ) - read_members.append(' helper_obj.add_to_init(f"0 part0-node{key2}-offsets <- stack\\n")') + read_members.append( + ' helper_obj.add_to_init(f"0 part0-node{key2}-offsets <- stack\\n")' + ) read_members.append( f' content[{self.name!r}] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' ) - read_members.append(f' #raise NotImplementedError\n helper_obj.add_to_pre(f" var_{self.count_name} @ dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")') - read_members.append(' if forth_obj.should_add_form():') - read_members.append(' forth_obj.add_form_key(form_key)') - read_members.append(' forth_obj.add_form_key(form_key2)') + read_members.append( + f' #raise NotImplementedError\n helper_obj.add_to_pre(f" var_{self.count_name} @ dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")' + ) + read_members.append(" if forth_obj.should_add_form():") + read_members.append( + " forth_obj.add_form_key(form_key)" + ) + read_members.append( + " forth_obj.add_form_key(form_key2)" + ) read_members.append( f" self._members[{self.name!r}] = cursor.array(chunk, self.member({self.count_name!r}), tmp, context);\n" @@ -967,7 +977,9 @@ def class_code( f' content[{fields[-1][0]!r}] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[-1][0])}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}' ) if fields[-1][0] in COUNT_NAMES: - read_members.append(f' helper_obj.add_to_init(f"variable var_{fields[-1][0]}\\n")') + read_members.append( + f' helper_obj.add_to_init(f"variable var_{fields[-1][0]}\\n")' + ) read_members.append( f' helper_obj.add_to_pre(f"stream !{formats[-1][0]}-> stack dup var_{fields[-1][0]} ! part0-node{{key}}-data <- stack\\n")' ) @@ -1025,25 +1037,35 @@ def class_code( else: # @aryan26roy: test_0637's 44,56 - read_members.append(' if helper_obj.is_forth():') - read_members.append(' key = forth_obj.get_keys(1)') - read_members.append(' key2 = forth_obj.get_keys(1)') + read_members.append(" if helper_obj.is_forth():") + read_members.append(" key = forth_obj.get_keys(1)") + read_members.append(" key2 = forth_obj.get_keys(1)") read_members.append(' form_key = f"part0-node{key}-data"') - read_members.append(' form_key2 = f"part0-node{key2}-offsets"') + read_members.append( + ' form_key2 = f"part0-node{key2}-offsets"' + ) read_members.append( f' helper_obj.add_to_header(f"output part0-node{{key}}-data {{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}\\n")' ) read_members.append( ' helper_obj.add_to_header(f"output part0-node{key2}-offsets int64\\n")' ) - read_members.append(' helper_obj.add_to_init(f"0 part0-node{key2}-offsets <- stack\\n")') + read_members.append( + ' helper_obj.add_to_init(f"0 part0-node{key2}-offsets <- stack\\n")' + ) read_members.append( f' content[{self.name!r}] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' ) - read_members.append(f' helper_obj.add_to_pre(f"{self.array_length} dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")\n') - read_members.append(' if forth_obj.should_add_form():') - read_members.append(' forth_obj.add_form_key(form_key)') - read_members.append(' forth_obj.add_form_key(form_key2)') + read_members.append( + f' helper_obj.add_to_pre(f"{self.array_length} dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")\n' + ) + read_members.append(" if forth_obj.should_add_form():") + read_members.append( + " forth_obj.add_form_key(form_key)" + ) + read_members.append( + " forth_obj.add_form_key(form_key2)" + ) read_members.append( f" self._members[{self.name!r}] = cursor.array(chunk, {self.array_length}, self._dtype{len(dtypes)}, context)" ) diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index 86aa5a4af..865169b19 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -48,7 +48,10 @@ def test_03(): interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) assert py[0][0] == 'Converter.hepmcStatusList = "[]";\n' - assert py[0][14] == 'ToolSvc.photons.delphesArrayName = "PhotonEfficiency/photons";\nToolSvc.photons.isolationTags = "photonITags";\nToolSvc.photons.mcAssociations = "photonsToMC";\nToolSvc.photons.particles = "photons";\n' + assert ( + py[0][14] + == 'ToolSvc.photons.delphesArrayName = "PhotonEfficiency/photons";\nToolSvc.photons.isolationTags = "photonITags";\nToolSvc.photons.mcAssociations = "photonsToMC";\nToolSvc.photons.particles = "photons";\n' + ) # py[-1] == @@ -68,7 +71,7 @@ def test_04(): # branch = file["CollectionTree/TrigConfKeys"] # interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) # py = branch.array(interp, library="np", entry_stop=2) - # py[-1] == +# py[-1] == def test_06(): @@ -107,7 +110,7 @@ def test_09(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0]['0'][0] == 352341021 + assert py[0][0]["0"][0] == 352341021 # py[-1] == > (version 1) at 0x7febbf1b2fa0>], ...] at 0x7febbf1b2f40> @@ -129,7 +132,7 @@ def test_11(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0]['0'][0] == 921521854 + assert py[0][0]["0"][0] == 921521854 # py[-1] == > (version 1) at 0x7f636a9484c0>], ...] at 0x7f636a948eb0> @@ -139,7 +142,7 @@ def test_12(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0]['1'][0] == 2 + assert py[0][0]["1"][0] == 2 # py[-1] == > (version 1) at 0x7fc259ae37c0>], ...] at 0x7fc259ae3f10> @@ -149,7 +152,7 @@ def test_13(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=3) - assert py[0][0]['1'][0] == 0 + assert py[0][0]["1"][0] == 0 # py[-1] == > (version 1) at 0x7fa94e968c10>], ...] at 0x7fa94e968c70> @@ -169,7 +172,7 @@ def test_15(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][-1][-1]['m_persKey'] == 980095599 + assert py[1][-1][-1]["m_persKey"] == 980095599 # py[-1] == > (version 1) at 0x7fb9b9d24c10>], ...] at 0x7fb9b9d29250> @@ -179,7 +182,7 @@ def test_16(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][0]['m_persKey'][0] == 980095599 + assert py[1][0]["m_persKey"][0] == 980095599 # py[-1] == > (version 1) at 0x7f6e29be5cd0>], ...] at 0x7f6e29bea250> @@ -269,7 +272,7 @@ def test_25(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][-1] == 'numuCCAnalysis' + assert py[0][-1] == "numuCCAnalysis" # py[-1] == array(['psychePolicy', 'psycheEventModel', 'psycheCore', 'psycheUtils', 'psycheND280Utils', 'psycheIO', 'psycheSelections', 'psycheSystematics', 'highlandEventModel', 'highlandTools', 'highlandCore', 'highlandCorrections', 'highlandIO', 'baseAnalysis', 'baseTrackerAnalysis', 'numuCCAnalysis'], dtype=object) @@ -278,7 +281,7 @@ def test_25(): # branch = file["config/SEL/SEL._firstSteps"] # interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) # py = branch.array(interp, library="np", entry_stop=2) - # py[-1] == array([], dtype=object) +# py[-1] == array([], dtype=object) def test_27(): @@ -287,7 +290,7 @@ def test_27(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0][-1] == 'muFGD+Np' + assert py[0][0][-1] == "muFGD+Np" # py[-1] == array([], dtype=object) @@ -335,7 +338,7 @@ def test_32(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1] == 'two' + assert py[-1] == "two" # py[-1] == "two" @@ -345,7 +348,7 @@ def test_33(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][4]['1'][-1] == 1.0 + assert py[0][4]["1"][-1] == 1.0 # py[-1] == @@ -355,7 +358,7 @@ def test_34(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1] == '$Name: geant4-10-05-patch-01 $' + assert py[-1] == "$Name: geant4-10-05-patch-01 $" def test_35(): @@ -364,7 +367,7 @@ def test_35(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]['1'][0]['name'] == 'anti_alpha' + assert py[0]["1"][0]["name"] == "anti_alpha" # py[-1] == , ...} at 0x7fb557a012e0> @@ -374,7 +377,7 @@ def test_36(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][-1] == 'MP_F_99.' + assert py[0][-1] == "MP_F_99." # py[-1] == @@ -384,7 +387,7 @@ def test_37(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][555]['2'] == 100.94856572890848 + assert py[0][555]["2"] == 100.94856572890848 # py[-1] == array([, , , ..., , , ], dtype=object) @@ -394,7 +397,7 @@ def test_38(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]['zp'][0] == pytest.approx(0.9999998807907104) + assert py[0]["zp"][0] == pytest.approx(0.9999998807907104) # py[-1] == (version 4) at 0x7f9be2f7b2e0> @@ -404,7 +407,7 @@ def test_39(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1]['0'] == -1000020030 + assert py[-1][1]["0"] == -1000020030 # py[-1] == , ...} at 0x7f6d057397f0> @@ -414,7 +417,7 @@ def test_40(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]['fDatime'] == 1749155840 + assert py[0]["fDatime"] == 1749155840 # py[-1] == @@ -454,7 +457,7 @@ def test_44(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]['1'][-1] == True + assert py[0]["1"][-1] == True # py[-1] == @@ -463,7 +466,7 @@ def test_44(): # branch = file["MCTrack/global"] # interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) # py = branch.array(interp, library="np", entry_stop=2) - # py[-1] == , ...] at 0x7f88b9a8ad00> +# py[-1] == , ...] at 0x7f88b9a8ad00> # def test_46(): @@ -471,7 +474,7 @@ def test_44(): # branch = file["MCParticle/detector1"] # interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) # py = branch.array(interp, library="np", entry_stop=2) - # py[-1] == , ...] at 0x7f94bc223550> +# py[-1] == , ...] at 0x7f94bc223550> def test_47(): @@ -480,7 +483,7 @@ def test_47(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][-1]['1']['mass'] == pytest.approx(3.727379) + assert py[-1][-1]["1"]["mass"] == pytest.approx(3.727379) # py[-1] == , ...} at 0x7f53a44278b0> @@ -489,8 +492,8 @@ def test_47(): # branch = file["Geant4Data/Geant4Data./Geant4Data.ions"] # interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) # py = branch.array(interp, library="np", entry_stop=2) - #assert py[-1]['0'][-1] == 1000020040 - # py[-1] == +# assert py[-1]['0'][-1] == 1000020040 +# py[-1] == # def test_49(): @@ -498,7 +501,7 @@ def test_47(): # branch = file["MCParticle/timepix"] # interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) # py = branch.array(interp, library="np", entry_stop=2) - # py[-1] == ] at 0x7f0697bf1a00> +# py[-1] == ] at 0x7f0697bf1a00> # def test_50(): @@ -506,7 +509,8 @@ def test_47(): # branch = file["PixelHit/timepix"] # interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) # py = branch.array(interp, library="np", entry_stop=2) - # py[-1] == +# py[-1] == + def test_51(): with uproot.open(skhep_testdata.data_path("uproot-issue510b.root")) as file: @@ -514,7 +518,7 @@ def test_51(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1] == 'GENIE:fixed@density-fixed' + assert py[-1] == "GENIE:fixed@density-fixed" def test_52(): @@ -523,7 +527,7 @@ def test_52(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][-1]['refs'][-1] == 2223 + assert py[1][-1]["refs"][-1] == 2223 # py[-1] == array([, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ], dtype=object) @@ -533,7 +537,7 @@ def test_53(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][-1]['fE'] == 0.0 + assert py[0][-1]["fE"] == 0.0 # py[-1] == array([, , , , , ], dtype=object) @@ -543,7 +547,7 @@ def test_54(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][-1][4]['fE'] == 0.0 + assert py[0][-1][4]["fE"] == 0.0 # py[-1] == array([[, , , , ], [, , , , ], [, , , , ], [, , , , ], [, , , , ], [, , , , ]], dtype=object) @@ -565,7 +569,7 @@ def test_56(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1]['SliceU64'][0] == 1 + assert py[1]["SliceU64"][0] == 1 # py[-1] == @@ -575,7 +579,7 @@ def test_57(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1] == 'two' + assert py[-1][1] == "two" # py[-1] == @@ -585,7 +589,7 @@ def test_58(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1] == 'two' + assert py[1][1] == "two" # py[-1] == @@ -605,7 +609,7 @@ def test_60(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1][1] == 'two' + assert py[-1][1][1] == "two" # py[-1] == @@ -615,7 +619,7 @@ def test_61(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1][1] == 'two' + assert py[-1][1][1] == "two" # py[-1] == @@ -635,7 +639,7 @@ def test_63(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1][1] == 'two' + assert py[-1][1][1] == "two" # py[-1] == @@ -655,7 +659,7 @@ def test_65(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1] == 'two' + assert py[-1][1] == "two" # py[-1] == @@ -665,7 +669,7 @@ def test_66(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1]['1'][1] == 2 + assert py[-1]["1"][1] == 2 # py[-1] == @@ -675,7 +679,7 @@ def test_67(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1] == 2 + assert py[1][1]["1"][1] == 2 # py[-1] == @@ -685,7 +689,7 @@ def test_68(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1] == 'two' + assert py[1][1]["1"][1] == "two" # py[-1] == @@ -695,7 +699,7 @@ def test_69(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1] == 2 + assert py[1][1]["1"][1] == 2 # py[-1] == @@ -705,7 +709,7 @@ def test_70(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1] == 'two' + assert py[1][1]["1"][1] == "two" # py[-1] == @@ -715,7 +719,7 @@ def test_71(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'] == 2 + assert py[1][1]["1"] == 2 # py[-1] == @@ -725,7 +729,7 @@ def test_72(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1] == 2 + assert py[1][1]["1"][1] == 2 # py[-1] == @@ -735,7 +739,7 @@ def test_73(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1] == 'two' + assert py[1][1]["1"][1] == "two" # py[-1] == @@ -745,7 +749,7 @@ def test_74(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1] == 2 + assert py[1][1]["1"][1] == 2 # py[-1] == @@ -755,7 +759,7 @@ def test_75(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1] == 'two' + assert py[1][1]["1"][1] == "two" # py[-1] == @@ -765,7 +769,7 @@ def test_76(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1][1] == 2 + assert py[1][1]["1"][1][1] == 2 # py[-1] == @@ -775,7 +779,7 @@ def test_77(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'][1][1] == 2 + assert py[1][1]["1"][1][1] == 2 # py[-1] == @@ -785,7 +789,7 @@ def test_78(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'] == 'TWO' + assert py[1][1]["1"] == "TWO" # py[-1] == @@ -795,7 +799,7 @@ def test_79(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]['1'] == 'TWO' + assert py[1][1]["1"] == "TWO" # py[-1] == @@ -817,5 +821,5 @@ def test_81(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]['1'] == pytest.approx(2.5636332035064697) + assert py[0]["1"] == pytest.approx(2.5636332035064697) # py[-1] == From ea288d38083672f9d38fa85578e0c853d1d4e3c0 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Mon, 5 Sep 2022 20:27:51 +0530 Subject: [PATCH 24/41] removed unused import --- src/uproot/models/TDatime.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/uproot/models/TDatime.py b/src/uproot/models/TDatime.py index 5ad2d48bd..23bdec5f8 100644 --- a/src/uproot/models/TDatime.py +++ b/src/uproot/models/TDatime.py @@ -5,7 +5,6 @@ """ -import json import struct import numpy From 22c5a8e38a35417c7588762fc2a68f0f8d370609 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Tue, 6 Sep 2022 09:13:49 +0530 Subject: [PATCH 25/41] fixed bug --- src/uproot/containers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/uproot/containers.py b/src/uproot/containers.py index 4fa5509cf..889587359 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -541,9 +541,9 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): for elem in form_keys: forth_obj.add_form_key(elem) helper_obj.add_to_header( - f"output node{offsets_num}-offsets int64\noutput node{data_num}-data uint8\n" + f"output part0-node{offsets_num}-offsets int64\noutput part0-node{data_num}-data uint8\n" ) - helper_obj.add_to_init(f"0 node{offsets_num}-offsets <- stack\n") + helper_obj.add_to_init(f"0 part0-node{offsets_num}-offsets <- stack\n") temp_form = forth_obj.add_node( f"node{offsets_num}", helper_obj.get_pre(), @@ -841,10 +841,10 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if helper_obj.is_forth(): helper_obj.add_to_header( - f"output node{offsets_num}-offsets int64\n" + f"output part0-node{offsets_num}-offsets int64\n" ) - form_key = f"node{offsets_num}-offsets" - helper_obj.add_to_init(f"0 node{offsets_num}-offsets <- stack\n") + form_key = f"part0-node{offsets_num}-offsets" + helper_obj.add_to_init(f"0 part0-node{offsets_num}-offsets <- stack\n") helper_obj.add_to_pre( "0 bytestops I-> stack \nbegin\ndup stream pos <>\nwhile\nswap 1 + swap\n" ) From 88755f5984d1d0afe76ed94b5e48fa022a5179d9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 6 Sep 2022 03:44:45 +0000 Subject: [PATCH 26/41] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/uproot/containers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/uproot/containers.py b/src/uproot/containers.py index 889587359..fe75ca3db 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -844,7 +844,9 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): f"output part0-node{offsets_num}-offsets int64\n" ) form_key = f"part0-node{offsets_num}-offsets" - helper_obj.add_to_init(f"0 part0-node{offsets_num}-offsets <- stack\n") + helper_obj.add_to_init( + f"0 part0-node{offsets_num}-offsets <- stack\n" + ) helper_obj.add_to_pre( "0 bytestops I-> stack \nbegin\ndup stream pos <>\nwhile\nswap 1 + swap\n" ) From 31620612ffe94acd413e4805b61035dc07babe54 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Tue, 6 Sep 2022 09:47:39 +0530 Subject: [PATCH 27/41] fixed awkward-forth tests --- pyproject.toml | 2 +- src/uproot/models/TRef.py | 6 +++--- .../test_0637-setup-tests-for-AwkwardForth.py | 20 +++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 18e6c2eaf..ccff4cd4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,7 +87,7 @@ markers = [ "network", "xrootd", ] -timeout = 600 +#timeout = 600 [tool.isort] profile = "black" diff --git a/src/uproot/models/TRef.py b/src/uproot/models/TRef.py index fda7b2c73..66c097768 100644 --- a/src/uproot/models/TRef.py +++ b/src/uproot/models/TRef.py @@ -137,14 +137,14 @@ def read_members(self, chunk, cursor, context, file): helper_obj.add_to_pre("10 stream skip\n") helper_obj.add_to_pre( - f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup node{form_keys[1]}-offsets +<- stack stream #!B-> node{form_keys[2]}-data\n" + f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup part0-node{form_keys[1]}-offsets +<- stack stream #!B-> part0-node{form_keys[2]}-data\n" ) helper_obj.add_to_pre( - f"stream !I-> stack dup node{form_keys[3]}-data <- stack\n" + f"stream !I-> stack dup part0-node{form_keys[3]}-data <- stack\n" ) helper_obj.add_to_pre("6 stream skip\n") helper_obj.add_to_pre( - f"dup node{form_keys[4]}-offsets +<- stack stream #!I-> node{form_keys[5]}-data\n" + f"dup part0-node{form_keys[4]}-offsets +<- stack stream #!I-> part0-node{form_keys[5]}-data\n" ) keys = [ f"node{form_keys[1]}-offsets", diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index 865169b19..98c017d85 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -25,8 +25,8 @@ def test_01(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0]["1"] == pytest.approx(84.56447925448748) - assert py[0][0]["0"]["fZ"] == pytest.approx(-81.600465) + assert py[0][0]["fE"] == pytest.approx(84.56447925448748) + assert py[0][0]["fP"]["fZ"] == pytest.approx(-81.600465) # py[-1] == array([, ], dtype=object) @@ -110,7 +110,7 @@ def test_09(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0]["0"][0] == 352341021 + assert py[0][0][0]['m_persKey'] == 352341021 # py[-1] == > (version 1) at 0x7febbf1b2fa0>], ...] at 0x7febbf1b2f40> @@ -132,7 +132,7 @@ def test_11(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0]["0"][0] == 921521854 + assert py[0][0][0]['m_persKey'] == 921521854 # py[-1] == > (version 1) at 0x7f636a9484c0>], ...] at 0x7f636a948eb0> @@ -142,7 +142,7 @@ def test_12(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0]["1"][0] == 2 + assert py[0][0][0]['m_persIndex'] == 2 # py[-1] == > (version 1) at 0x7fc259ae37c0>], ...] at 0x7fc259ae3f10> @@ -152,7 +152,7 @@ def test_13(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=3) - assert py[0][0]["1"][0] == 0 + assert py[0][0][0]['m_persIndex'] == 0 # py[-1] == > (version 1) at 0x7fa94e968c10>], ...] at 0x7fa94e968c70> @@ -387,7 +387,7 @@ def test_37(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][555]["2"] == 100.94856572890848 + assert py[0][555]["fZ"] == 100.94856572890848 # py[-1] == array([, , , ..., , , ], dtype=object) @@ -457,7 +457,7 @@ def test_44(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]["1"][-1] == True + assert py[0]["ArrayBool"][-1] == True # py[-1] == @@ -518,7 +518,7 @@ def test_51(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1] == "GENIE:fixed@density-fixed" + assert py[-1][0] == "GENIE:fixed@density-fixed" def test_52(): @@ -821,5 +821,5 @@ def test_81(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]["1"] == pytest.approx(2.5636332035064697) + assert py[0]["fY"] == pytest.approx(2.5636332035064697) # py[-1] == From 7d15b438b88d85751517d28a8543d59b1be30c8b Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Tue, 6 Sep 2022 09:48:36 +0530 Subject: [PATCH 28/41] linting --- tests/test_0637-setup-tests-for-AwkwardForth.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index 98c017d85..65460381c 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -110,7 +110,7 @@ def test_09(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0][0]['m_persKey'] == 352341021 + assert py[0][0][0]["m_persKey"] == 352341021 # py[-1] == > (version 1) at 0x7febbf1b2fa0>], ...] at 0x7febbf1b2f40> @@ -132,7 +132,7 @@ def test_11(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0][0]['m_persKey'] == 921521854 + assert py[0][0][0]["m_persKey"] == 921521854 # py[-1] == > (version 1) at 0x7f636a9484c0>], ...] at 0x7f636a948eb0> @@ -142,7 +142,7 @@ def test_12(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0][0]['m_persIndex'] == 2 + assert py[0][0][0]["m_persIndex"] == 2 # py[-1] == > (version 1) at 0x7fc259ae37c0>], ...] at 0x7fc259ae3f10> @@ -152,7 +152,7 @@ def test_13(): interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = True py = branch.array(interp, library="ak", entry_stop=3) - assert py[0][0][0]['m_persIndex'] == 0 + assert py[0][0][0]["m_persIndex"] == 0 # py[-1] == > (version 1) at 0x7fa94e968c10>], ...] at 0x7fa94e968c70> From bd3ebb0dac2f1bf6e759abd925ac90e3ef93a6f8 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Tue, 6 Sep 2022 10:03:24 +0530 Subject: [PATCH 29/41] fixed forth generation bug --- src/uproot/model.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/uproot/model.py b/src/uproot/model.py index c6686f475..8e060eed4 100644 --- a/src/uproot/model.py +++ b/src/uproot/model.py @@ -814,7 +814,8 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): if self._num_bytes is None and self._instance_version != self.class_version: self._instance_version = None cursor = self._cursor - helper_obj._pre_code.pop(-1) + if helper_obj.is_forth(): + helper_obj._pre_code.pop(-1) elif self._instance_version == 0: helper_obj.add_to_pre("4 stream skip\n") From 5565d4b7a246c6b8e9c719921e16a8bb513bfc58 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Tue, 6 Sep 2022 10:17:55 +0530 Subject: [PATCH 30/41] switched to awkward._v2 --- src/uproot/containers.py | 62 +++++++++++++--------------- src/uproot/interpretation/objects.py | 2 +- src/uproot/models/TDatime.py | 8 ++-- src/uproot/models/TRef.py | 10 ++--- src/uproot/models/TString.py | 10 ++--- src/uproot/streamers.py | 48 +++++++++------------ 6 files changed, 64 insertions(+), 76 deletions(-) diff --git a/src/uproot/containers.py b/src/uproot/containers.py index fe75ca3db..88fe2fddc 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -60,11 +60,11 @@ def _read_nested( raise TypeError("Cannot be awkward") if helper_obj.is_forth(): key = forth_obj.get_keys(1) - form_key = f"part0-node{key}-data" + form_key = f"node{key}-data" helper_obj.add_to_header( - f"output part0-node{key}-data {uproot._awkward_forth.convert_dtype(symbol)}\n" + f"output node{key}-data {uproot._awkward_forth.convert_dtype(symbol)}\n" ) - helper_obj.add_to_pre(f"stream #!{symbol}-> part0-node{key}-data\n") + helper_obj.add_to_pre(f"stream #!{symbol}-> node{key}-data\n") if forth_obj.should_add_form(): forth_obj.add_form_key(form_key) forth_obj.add_form( @@ -502,14 +502,14 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): out = cursor.string(chunk, context) if helper_obj.is_forth(): helper_obj.add_to_pre( - f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup part0-node{offsets_num}-offsets +<- stack stream #!B-> part0-node{data_num}-data\n" + f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup node{offsets_num}-offsets +<- stack stream #!B-> node{data_num}-data\n" ) elif self._length_bytes == "4": length = cursor.field(chunk, _stl_container_size, context) out = cursor.string_with_length(chunk, context, length) if helper_obj.is_forth(): helper_obj.add_to_pre( - f"stream !I-> stack dup part0-node{offsets_num}-offsets +<- stack stream #B-> part0-node{data_num}-data\n" + f"stream !I-> stack dup node{offsets_num}-offsets +<- stack stream #B-> node{data_num}-data\n" ) else: raise AssertionError(repr(self._length_bytes)) @@ -541,9 +541,9 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): for elem in form_keys: forth_obj.add_form_key(elem) helper_obj.add_to_header( - f"output part0-node{offsets_num}-offsets int64\noutput part0-node{data_num}-data uint8\n" + f"output node{offsets_num}-offsets int64\noutput node{data_num}-data uint8\n" ) - helper_obj.add_to_init(f"0 part0-node{offsets_num}-offsets <- stack\n") + helper_obj.add_to_init(f"0 node{offsets_num}-offsets <- stack\n") temp_form = forth_obj.add_node( f"node{offsets_num}", helper_obj.get_pre(), @@ -758,22 +758,20 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): else: if helper_obj.is_forth(): helper_obj.add_to_header( - f"output part0-node{offsets_num}-offsets int64\n" - ) - form_key = f"part0-node{offsets_num}-offsets" - helper_obj.add_to_init( - f"0 part0-node{offsets_num}-offsets <- stack\n" + f"output node{offsets_num}-offsets int64\n" ) + form_key = f"node{offsets_num}-offsets" + helper_obj.add_to_init(f"0 node{offsets_num}-offsets <- stack\n") helper_obj.add_to_pre( "0 bytestops I-> stack \nbegin\ndup stream pos <>\nwhile\nswap 1 + swap\n" ) if len(self.inner_shape) > 0: helper_obj.add_to_post( - f"repeat\nswap {self.inner_shape[0]} / part0-node{offsets_num}-offsets +<- stack drop\n" + f"repeat\nswap {self.inner_shape[0]} / node{offsets_num}-offsets +<- stack drop\n" ) else: helper_obj.add_to_post( - f"repeat\nswap part0-node{offsets_num}-offsets +<- stack drop\n" + f"repeat\nswap node{offsets_num}-offsets +<- stack drop\n" ) if forth_obj.should_add_form(): forth_obj.add_form_key(form_key) @@ -841,22 +839,20 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if helper_obj.is_forth(): helper_obj.add_to_header( - f"output part0-node{offsets_num}-offsets int64\n" - ) - form_key = f"part0-node{offsets_num}-offsets" - helper_obj.add_to_init( - f"0 part0-node{offsets_num}-offsets <- stack\n" + f"output node{offsets_num}-offsets int64\n" ) + form_key = f"node{offsets_num}-offsets" + helper_obj.add_to_init(f"0 node{offsets_num}-offsets <- stack\n") helper_obj.add_to_pre( "0 bytestops I-> stack \nbegin\ndup stream pos <>\nwhile\nswap 1 + swap\n" ) if len(self.inner_shape) > 0: helper_obj.add_to_post( - f"repeat\nswap {self.inner_shape[0]} / part0-node{offsets_num}-offsets +<- stack drop\n" + f"repeat\nswap {self.inner_shape[0]} / node{offsets_num}-offsets +<- stack drop\n" ) else: helper_obj.add_to_post( - f"repeat\nswap part0-node{offsets_num}-offsets +<- stack drop\n" + f"repeat\nswap node{offsets_num}-offsets +<- stack drop\n" ) if forth_obj.should_add_form(): forth_obj.add_form_key(form_key) @@ -1181,11 +1177,11 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): length = cursor.field(chunk, _stl_container_size, context) if helper_obj.is_forth(): key = forth_obj.get_keys(1) - form_key = f"part0-node{key}-offsets" - helper_obj.add_to_header(f"output part0-node{key}-offsets int64\n") - helper_obj.add_to_init(f"0 part0-node{key}-offsets <- stack\n") + form_key = f"node{key}-offsets" + helper_obj.add_to_header(f"output node{key}-offsets int64\n") + helper_obj.add_to_init(f"0 node{key}-offsets <- stack\n") helper_obj.add_to_pre( - f"stream !I-> stack\n dup part0-node{key}-offsets +<- stack\n" + f"stream !I-> stack\n dup node{key}-offsets +<- stack\n" ) # helper_obj.add_to_post("loop\n") if forth_obj.should_add_form(): @@ -1347,11 +1343,11 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if helper_obj.is_forth(): key = forth_obj.get_keys(1) - form_key = f"part0-node{key}-offsets" - helper_obj.add_to_header(f"output part0-node{key}-offsets int64\n") - helper_obj.add_to_init(f"0 part0-node{key}-offsets <- stack\n") + form_key = f"node{key}-offsets" + helper_obj.add_to_header(f"output node{key}-offsets int64\n") + helper_obj.add_to_init(f"0 node{key}-offsets <- stack\n") helper_obj.add_to_pre( - f"stream !I-> stack\ndup part0-node{key}-offsets +<- stack\n" + f"stream !I-> stack\ndup node{key}-offsets +<- stack\n" ) # helper_obj.add_to_post("loop\n") if forth_obj.should_add_form(): @@ -1537,11 +1533,11 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): length = cursor.field(chunk, _stl_container_size, context) if helper_obj.is_forth(): key = forth_obj.get_keys(1) - form_key = f"part0-node{key}-offsets" - helper_obj.add_to_header(f"output part0-node{key}-offsets int64\n") - helper_obj.add_to_init(f"0 part0-node{key}-offsets <- stack\n") + form_key = f"node{key}-offsets" + helper_obj.add_to_header(f"output node{key}-offsets int64\n") + helper_obj.add_to_init(f"0 node{key}-offsets <- stack\n") helper_obj.add_to_pre( - f"stream !I-> stack\n dup part0-node{key}-offsets +<- stack\n" + f"stream !I-> stack\n dup node{key}-offsets +<- stack\n" ) if _has_nested_header(self._keys) and header: if helper_obj.is_forth(): diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index d2c99a94f..4350568b0 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -226,7 +226,7 @@ def basket_array_forth( context["forth"].vm.resume() container = {} container = context["forth"].vm.outputs - output = awkward.from_buffers(self._form, len(byte_offsets) - 1, container) + output = awkward._v2.from_buffers(self._form, len(byte_offsets) - 1, container) self.hook_after_basket_array( data=data, diff --git a/src/uproot/models/TDatime.py b/src/uproot/models/TDatime.py index 23bdec5f8..20eb9a9c2 100644 --- a/src/uproot/models/TDatime.py +++ b/src/uproot/models/TDatime.py @@ -29,10 +29,10 @@ def read_members(self, chunk, cursor, context, file): if helper_obj.is_forth(): forth_obj = helper_obj.get_gen_obj() key = forth_obj.get_keys(1) - form_key = f"part0-node{key}-data" - helper_obj.add_to_header(f"output part0-node{key}-data int32\n") - helper_obj.add_to_pre(f"stream !I-> part0-node{key}-data\n") - form_key = f"part0-node{key}-data" + form_key = f"node{key}-data" + helper_obj.add_to_header(f"output node{key}-data int32\n") + helper_obj.add_to_pre(f"stream !I-> node{key}-data\n") + form_key = f"node{key}-data" if forth_obj.should_add_form(): forth_obj.add_form_key(form_key) temp_aform = { diff --git a/src/uproot/models/TRef.py b/src/uproot/models/TRef.py index 66c097768..9f5c5807c 100644 --- a/src/uproot/models/TRef.py +++ b/src/uproot/models/TRef.py @@ -137,14 +137,14 @@ def read_members(self, chunk, cursor, context, file): helper_obj.add_to_pre("10 stream skip\n") helper_obj.add_to_pre( - f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup part0-node{form_keys[1]}-offsets +<- stack stream #!B-> part0-node{form_keys[2]}-data\n" + f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup node{form_keys[1]}-offsets +<- stack stream #!B-> node{form_keys[2]}-data\n" ) helper_obj.add_to_pre( - f"stream !I-> stack dup part0-node{form_keys[3]}-data <- stack\n" + f"stream !I-> stack dup node{form_keys[3]}-data <- stack\n" ) helper_obj.add_to_pre("6 stream skip\n") helper_obj.add_to_pre( - f"dup part0-node{form_keys[4]}-offsets +<- stack stream #!I-> part0-node{form_keys[5]}-data\n" + f"dup node{form_keys[4]}-offsets +<- stack stream #!I-> node{form_keys[5]}-data\n" ) keys = [ f"node{form_keys[1]}-offsets", @@ -160,10 +160,10 @@ def read_members(self, chunk, cursor, context, file): temp_aform = f'{{"class": "RecordArray", "contents": {{"fname": {{"class": "ListOffsetArray", "offsets": "i64", "content": {{"class": "NumpyArray", "primitive": "uint8", "inner_shape": [], "has_identifier": false, "parameters": {{"__array__": "char"}}, "form_key": "node{form_keys[2]}"}}, "has_identifier": false, "parameters": {{"uproot": {{"as": "vector", "header": {temp_bool}}}}}, "form_key": "node{form_keys[1]}"}}, "fSize": {{"class": "NumpyArray", "primitive": "int64", "inner_shape": [], "has_identifier": false, "parameters": {{}}, "form_key": "node{form_keys[3]}"}}, "refs": {{"class": "ListOffsetArray", "offsets": "i64", "content": {{"class": "NumpyArray", "primitive": "int64", "inner_shape": [], "has_identifier": false, "parameters": {{}}, "form_key": "node{form_keys[5]}"}}, "has_identifier": false, "parameters": {{}}, "form_key": "node{form_keys[4]}"}}}}, "has_identifier": false, "parameters": {{}}, "form_key": "node{form_keys[0]}"}}' forth_obj.add_form(json.loads(temp_aform)) helper_obj.add_to_header( - f"output part0-node{form_keys[1]}-offsets int64\noutput part0-node{form_keys[2]}-data uint8\noutput part0-node{form_keys[3]}-data int64\noutput part0-node{form_keys[4]}-offsets int64\noutput part0-node{form_keys[5]}-data int64\n" + f"output node{form_keys[1]}-offsets int64\noutput node{form_keys[2]}-data uint8\noutput node{form_keys[3]}-data int64\noutput node{form_keys[4]}-offsets int64\noutput node{form_keys[5]}-data int64\n" ) helper_obj.add_to_init( - f"0 part0-node{form_keys[1]}-offsets <- stack\n0 part0-node{form_keys[4]}-offsets <- stack\n" + f"0 node{form_keys[1]}-offsets <- stack\n0 node{form_keys[4]}-offsets <- stack\n" ) forth_obj.add_node( f"node{form_keys[0]}", diff --git a/src/uproot/models/TString.py b/src/uproot/models/TString.py index a2988d235..e78791ed3 100644 --- a/src/uproot/models/TString.py +++ b/src/uproot/models/TString.py @@ -35,7 +35,7 @@ def read_members(self, chunk, cursor, context, file): ) if helper_obj.is_forth(): helper_obj.add_to_pre( - f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup part0-node{offsets_num}-offsets +<- stack stream #!B-> part0-node{data_num}-data\n" + f"stream !B-> stack dup 255 = if drop stream !I-> stack then dup node{offsets_num}-offsets +<- stack stream #!B-> node{data_num}-data\n" ) if forth_obj.should_add_form(): temp_aform = { @@ -59,15 +59,15 @@ def read_members(self, chunk, cursor, context, file): forth_obj.add_form(temp_aform) form_keys = [ - f"part0-node{data_num}-data", - f"part0-node{offsets_num}-offsets", + f"node{data_num}-data", + f"node{offsets_num}-offsets", ] for elem in form_keys: forth_obj.add_form_key(elem) helper_obj.add_to_header( - f"output part0-node{offsets_num}-offsets int64\noutput part0-node{data_num}-data uint8\n" + f"output node{offsets_num}-offsets int64\noutput node{data_num}-data uint8\n" ) - helper_obj.add_to_init(f"0 part0-node{offsets_num}-offsets <- stack\n") + helper_obj.add_to_init(f"0 node{offsets_num}-offsets <- stack\n") temp_form = forth_obj.add_node( f"node{offsets_num}", helper_obj.get_pre(), diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index 84e73de0a..2bc297c10 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -833,24 +833,22 @@ def class_code( read_members.append(" if helper_obj.is_forth():") read_members.append(" key = forth_obj.get_keys(1)") read_members.append(" key2 = forth_obj.get_keys(1)") - read_members.append(' form_key = f"part0-node{key}-data"') + read_members.append(' form_key = f"node{key}-data"') + read_members.append(' form_key2 = f"node{key2}-offsets"') read_members.append( - ' form_key2 = f"part0-node{key2}-offsets"' + f' helper_obj.add_to_header(f"output node{{key}}-data {{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}\\n")' ) read_members.append( - f' helper_obj.add_to_header(f"output part0-node{{key}}-data {{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}\\n")' + ' helper_obj.add_to_header(f"output node{key2}-offsets int64\\n")' ) read_members.append( - ' helper_obj.add_to_header(f"output part0-node{key2}-offsets int64\\n")' - ) - read_members.append( - ' helper_obj.add_to_init(f"0 part0-node{key2}-offsets <- stack\\n")' + ' helper_obj.add_to_init(f"0 node{key2}-offsets <- stack\\n")' ) read_members.append( f' content[{self.name!r}] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' ) read_members.append( - f' #raise NotImplementedError\n helper_obj.add_to_pre(f" var_{self.count_name} @ dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")' + f' #raise NotImplementedError\n helper_obj.add_to_pre(f" var_{self.count_name} @ dup node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> node{{key}}-data\\n")' ) read_members.append(" if forth_obj.should_add_form():") read_members.append( @@ -967,11 +965,9 @@ def class_code( # @aryan26roy: test_0637's 01,02,29,38,44,56 read_members.append(" if helper_obj.is_forth():") read_members.append(" key = forth_obj.get_keys(1)") + read_members.append(' form_key = f"node{key}-data"') read_members.append( - ' form_key = f"part0-node{key}-data"' - ) - read_members.append( - f' helper_obj.add_to_header(f"output part0-node{{key}}-data {uproot._awkward_forth.convert_dtype(formats[-1][0])}\\n")' + f' helper_obj.add_to_header(f"output node{{key}}-data {uproot._awkward_forth.convert_dtype(formats[-1][0])}\\n")' ) read_members.append( f' content[{fields[-1][0]!r}] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[-1][0])}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}' @@ -981,11 +977,11 @@ def class_code( f' helper_obj.add_to_init(f"variable var_{fields[-1][0]}\\n")' ) read_members.append( - f' helper_obj.add_to_pre(f"stream !{formats[-1][0]}-> stack dup var_{fields[-1][0]} ! part0-node{{key}}-data <- stack\\n")' + f' helper_obj.add_to_pre(f"stream !{formats[-1][0]}-> stack dup var_{fields[-1][0]} ! node{{key}}-data <- stack\\n")' ) else: read_members.append( - f' helper_obj.add_to_pre(f"stream !{formats[-1][0]}-> part0-node{{key}}-data\\n")' + f' helper_obj.add_to_pre(f"stream !{formats[-1][0]}-> node{{key}}-data\\n")' ) read_members.append( @@ -1002,18 +998,16 @@ def class_code( read_members.append(" if helper_obj.is_forth():") for i in range(len(formats[0])): read_members.append(" key = forth_obj.get_keys(1)") + read_members.append(' form_key = f"node{key}-data"') read_members.append( - ' form_key = f"part0-node{key}-data"' + f' helper_obj.add_to_header(f"output node{{key}}-data {uproot._awkward_forth.convert_dtype(formats[0][i])}\\n")' ) - read_members.append( - f' helper_obj.add_to_header(f"output part0-node{{key}}-data {uproot._awkward_forth.convert_dtype(formats[0][i])}\\n")' - ) - # read_members.append(' helper_obj.add_to_init(f"0 part0-node{key}-offsets <- stack\\n")') + # read_members.append(' helper_obj.add_to_init(f"0 node{key}-offsets <- stack\\n")') read_members.append( f' content[{fields[0][i]!r}] = {{ "class": "NumpyArray", "primitive": "{uproot._awkward_forth.convert_dtype(formats[0][i])}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}' ) read_members.append( - f' helper_obj.add_to_pre(f"stream !{formats[0][i]}-> part0-node{{key}}-data\\n")' + f' helper_obj.add_to_pre(f"stream !{formats[0][i]}-> node{{key}}-data\\n")' ) read_members.append( " if forth_obj.should_add_form():" @@ -1040,24 +1034,22 @@ def class_code( read_members.append(" if helper_obj.is_forth():") read_members.append(" key = forth_obj.get_keys(1)") read_members.append(" key2 = forth_obj.get_keys(1)") - read_members.append(' form_key = f"part0-node{key}-data"') - read_members.append( - ' form_key2 = f"part0-node{key2}-offsets"' - ) + read_members.append(' form_key = f"node{key}-data"') + read_members.append(' form_key2 = f"node{key2}-offsets"') read_members.append( - f' helper_obj.add_to_header(f"output part0-node{{key}}-data {{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}\\n")' + f' helper_obj.add_to_header(f"output node{{key}}-data {{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}\\n")' ) read_members.append( - ' helper_obj.add_to_header(f"output part0-node{key2}-offsets int64\\n")' + ' helper_obj.add_to_header(f"output node{key2}-offsets int64\\n")' ) read_members.append( - ' helper_obj.add_to_init(f"0 part0-node{key2}-offsets <- stack\\n")' + ' helper_obj.add_to_init(f"0 node{key2}-offsets <- stack\\n")' ) read_members.append( f' content[{self.name!r}] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' ) read_members.append( - f' helper_obj.add_to_pre(f"{self.array_length} dup part0-node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> part0-node{{key}}-data\\n")\n' + f' helper_obj.add_to_pre(f"{self.array_length} dup node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> node{{key}}-data\\n")\n' ) read_members.append(" if forth_obj.should_add_form():") read_members.append( From ea182831052bef354ac9222f5887306093cf3eb6 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Tue, 6 Sep 2022 18:52:39 +0530 Subject: [PATCH 31/41] add timeout back in --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ccff4cd4f..18e6c2eaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,7 +87,7 @@ markers = [ "network", "xrootd", ] -#timeout = 600 +timeout = 600 [tool.isort] profile = "black" From 9193f0bb3103ab76327f854c1859e6bfad9b9ac0 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Thu, 8 Sep 2022 11:21:51 +0530 Subject: [PATCH 32/41] test old awkward array code as well --- src/uproot/streamers.py | 4 +- .../test_0637-setup-tests-for-AwkwardForth.py | 835 ++++++++++-------- 2 files changed, 456 insertions(+), 383 deletions(-) diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index 2bc297c10..a646c1c29 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -224,7 +224,7 @@ def class_code(self): helper_obj = uproot._awkward_forth.GenHelper(context) if helper_obj.is_forth(): forth_obj = helper_obj.get_gen_obj() - content = {}\n""" + content = {}""" ) read_member_n = [ " def read_member_n(self, chunk, cursor, context, file, member_index):" @@ -1022,7 +1022,7 @@ def class_code( # @aryan26roy: test_0637's 01,02,05,08,09,11,12,13,15,16,29,35,39,45,46,47,49,50,56 read_members.append( - f" {assign_members} = cursor.fields(chunk, self._format{len(formats) - 1}, context)" + f"\n {assign_members} = cursor.fields(chunk, self._format{len(formats) - 1}, context)" ) read_member_n.append( diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index 65460381c..7147c733b 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -11,58 +11,63 @@ def test_00(): with uproot.open(skhep_testdata.data_path("issue367b.root")) as file: - branch = file["tree/weights"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]["0"][0] == "expskin_FluxUnisim" + for elem in [True,False]: + branch = file["tree/weights"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0]["0"][0] == "expskin_FluxUnisim" # py[-1] == def test_01(): with uproot.open(skhep_testdata.data_path("uproot-delphes-pr442.root")) as file: - branch = file["Delphes/GenJet/GenJet.SoftDroppedSubJet1"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0]["fE"] == pytest.approx(84.56447925448748) - assert py[0][0]["fP"]["fZ"] == pytest.approx(-81.600465) + for elem in [True,False]: + branch = file["Delphes/GenJet/GenJet.SoftDroppedSubJet1"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0]["fE"] == pytest.approx(84.56447925448748) + assert py[0][0]["fP"]["fZ"] == pytest.approx(-81.600465) # py[-1] == array([, ], dtype=object) def test_02(): with uproot.open(skhep_testdata.data_path("uproot-delphes-pr442.root")) as file: - branch = file["Delphes/GenJet/GenJet.TrimmedP4[5]"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak") - assert py[0][0][0]["fE"] == 0 - assert py[-1][8][3]["fE"] == 0 + for elem in [True,False]: + branch = file["Delphes/GenJet/GenJet.TrimmedP4[5]"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak") + assert py[0][0][0]["fE"] == 0 + assert py[-1][8][3]["fE"] == 0 # py[-1] == array([[, , , , ], [, , , , ]], dtype=object) def test_03(): with uproot.open(skhep_testdata.data_path("uproot-FCCDelphesOutput.root")) as file: - branch = file["metadata/gaudiConfigOptions"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0] == 'Converter.hepmcStatusList = "[]";\n' - assert ( - py[0][14] - == 'ToolSvc.photons.delphesArrayName = "PhotonEfficiency/photons";\nToolSvc.photons.isolationTags = "photonITags";\nToolSvc.photons.mcAssociations = "photonsToMC";\nToolSvc.photons.particles = "photons";\n' - ) + for elem in [True,False]: + branch = file["metadata/gaudiConfigOptions"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0] == 'Converter.hepmcStatusList = "[]";\n' + assert ( + py[0][14] + == 'ToolSvc.photons.delphesArrayName = "PhotonEfficiency/photons";\nToolSvc.photons.isolationTags = "photonITags";\nToolSvc.photons.mcAssociations = "photonsToMC";\nToolSvc.photons.particles = "photons";\n' + ) # py[-1] == def test_04(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - branch = file["POOLContainerForm/DataHeaderForm/m_uints"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0][0] == 2 - assert py[0][3][2] == 589824 + for elem in [True,False]: + branch = file["POOLContainerForm/DataHeaderForm/m_uints"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0][0] == 2 + assert py[0][3][2] == 589824 # py[-1] == @@ -76,203 +81,222 @@ def test_04(): def test_06(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - branch = file["CollectionTree/AnalysisJetsAuxDyn.NumTrkPt500"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][-1][-2] == 0 + for elem in [True,False]: + branch = file["CollectionTree/AnalysisJetsAuxDyn.NumTrkPt500"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][-1][-2] == 0 # py[-1] == def test_07(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - branch = file["CollectionTree/AnalysisJetsAuxDyn.SumPtTrkPt500"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][2][0] == pytest.approx(53949.015625) + for elem in [True,False]: + branch = file["CollectionTree/AnalysisJetsAuxDyn.SumPtTrkPt500"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][2][0] == pytest.approx(53949.015625) # py[-1] == -def test_08(): - with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - branch = file["CollectionTree/AnalysisJetsAuxDyn.GhostTrack"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - py = branch.array(interp, library="np", entry_stop=2) +# def test_08(): +# with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: +# branch = file["CollectionTree/AnalysisJetsAuxDyn.GhostTrack"] +# interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) +# py = branch.array(interp, library="np", entry_stop=2) # py[-1] == > (version 1) at 0x7fc6a08f2f70>, ...], ...] at 0x7fc6a08f2f10> def test_09(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - branch = file["CollectionTree"][ - "AntiKt10UFOCSSKJetsAuxDyn.GhostVR30Rmax4Rmin02TrackJet_BTagging201903" - ] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0][0]["m_persKey"] == 352341021 + for elem in [True,False]: + branch = file["CollectionTree"][ + "AntiKt10UFOCSSKJetsAuxDyn.GhostVR30Rmax4Rmin02TrackJet_BTagging201903" + ] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0][0]["m_persKey"] == 352341021 # py[-1] == > (version 1) at 0x7febbf1b2fa0>], ...] at 0x7febbf1b2f40> def test_10(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - branch = file["CollectionTree/CaloCalTopoClustersAuxDyn.e_sampl"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=3) - assert py[0][0][0] == pytest.approx(168.03048706054688) + for elem in [True,False]: + branch = file["CollectionTree/CaloCalTopoClustersAuxDyn.e_sampl"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=3) + assert py[0][0][0] == pytest.approx(168.03048706054688) # py[-1] == def test_11(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - branch = file["CollectionTree"][ - "TruthBosonsWithDecayVerticesAuxDyn.incomingParticleLinks" - ] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0][0]["m_persKey"] == 921521854 + for elem in [True,False]: + branch = file["CollectionTree"][ + "TruthBosonsWithDecayVerticesAuxDyn.incomingParticleLinks" + ] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0][0]["m_persKey"] == 921521854 # py[-1] == > (version 1) at 0x7f636a9484c0>], ...] at 0x7f636a948eb0> def test_12(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - branch = file["CollectionTree/TruthBottomAuxDyn.parentLinks"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0][0]["m_persIndex"] == 2 + for elem in [True,False]: + branch = file["CollectionTree/TruthBottomAuxDyn.parentLinks"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0][0]["m_persIndex"] == 2 # py[-1] == > (version 1) at 0x7fc259ae37c0>], ...] at 0x7fc259ae3f10> def test_13(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - branch = file["CollectionTree/egammaClustersAuxDyn.constituentClusterLinks"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=3) - assert py[0][0][0]["m_persIndex"] == 0 + for elem in [True,False]: + branch = file["CollectionTree/egammaClustersAuxDyn.constituentClusterLinks"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=3) + assert py[0][0][0]["m_persIndex"] == 0 # py[-1] == > (version 1) at 0x7fa94e968c10>], ...] at 0x7fa94e968c70> def test_14(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - branch = file["CollectionTree/egammaClustersAuxDyn.eta_sampl"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=3) - assert py[0][0][-1] == pytest.approx(0.4663555920124054) + for elem in [True,False]: + branch = file["CollectionTree/egammaClustersAuxDyn.eta_sampl"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=3) + assert py[0][0][-1] == pytest.approx(0.4663555920124054) # py[-1] == def test_15(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - branch = file["CollectionTree/AnalysisHLT_mu24_ilooseAuxDyn.TrigMatchedObjects"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][-1][-1]["m_persKey"] == 980095599 + for elem in [True,False]: + branch = file["CollectionTree/AnalysisHLT_mu24_ilooseAuxDyn.TrigMatchedObjects"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][-1][-1]["m_persKey"] == 980095599 # py[-1] == > (version 1) at 0x7fb9b9d24c10>], ...] at 0x7fb9b9d29250> def test_16(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - branch = file["CollectionTree/AnalysisHLT_mu40AuxDyn.TrigMatchedObjects"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][0]["m_persKey"][0] == 980095599 + for elem in [True,False]: + branch = file["CollectionTree/AnalysisHLT_mu40AuxDyn.TrigMatchedObjects"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][0]["m_persKey"][0] == 980095599 # py[-1] == > (version 1) at 0x7f6e29be5cd0>], ...] at 0x7f6e29bea250> def test_17(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: - branch = file["E/Evt/AAObject/usr_names"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=None) - assert len(py[0]) == 0 + for elem in [True,False]: + branch = file["E/Evt/AAObject/usr_names"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=None) + assert len(py[0]) == 0 # py[-1] == def test_18(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: - branch = file["E/Evt/hits/hits.t"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0] == pytest.approx(70104010.0) + for elem in [True,False]: + branch = file["E/Evt/hits/hits.t"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0] == pytest.approx(70104010.0) # py[-1] == array([80860738., 80861689., 80862014., 80861709., 80861737., 80861158., 80862362., 80860821., 80862271., 80862273., 80861294., 80861860., 80862548., 80861733., 80861605., 80860467., 80860408., 80861562., 80862012., 80862350., 80861491., 80860384., 80860930., 80861541., 80861461., 80861749., 80862352., 80861813., 80861822., 80861871., 80862000., 80862255., 80862253., 80862249., 80862266., 80862248., 80862246., 80862847., 80863032., 80861952., 80861954., 80861953., 80861957., 80861951., 80861961., 80861959., 80861955., 80861994., 80862060., 80861971., 80862004., 80862002., 80862059., 80861695., 80861813., 80861967., 80862919., 80862043., 80862054., 80862044., 80862044., 80862040., 80862043., 80862037., 80862040., 80862039., 80862070., 80862042., 80862322., 80861605., 80861865., 80863034., 80862987., 80861545., 80860392., 80861003., 80861564., 80862109., 80861821., 80862083., 80861121., 80862513., 80862513., 80862731., 80861604., 80862003., 80861910., 80861854., 80862297., 80860989., 80862948., 80862075., 80862141., 80862117., 80862039., 80862114., 80862075., 80862042., 80862072., 80862439., 80862481., 80861656., 80862096., 80862215., 80862215., 80862195., 80862458., 80862432., 80861915., 80861012., 80862208., 80861885., 80861888., 80861994., 80861883., 80862194., 80861812., 80862184., 80862309., 80862297., 80862840., 80862400., 80861565., 80862226., 80862149.]) def test_19(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: - branch = file["E/Evt/hits/hits.a"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][-1] == 0.0 + for elem in [True,False]: + branch = file["E/Evt/hits/hits.a"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][-1] == 0.0 # py[-1] == array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]) def test_20(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: - branch = file["E/Evt/hits/hits.trig"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][5] == 1 + for elem in [True,False]: + branch = file["E/Evt/hits/hits.trig"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][5] == 1 # py[-1] == array([0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0], dtype=int32) def test_21(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: - branch = file["E/Evt/hits/hits.tot"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0] == 24 + for elem in [True,False]: + branch = file["E/Evt/hits/hits.tot"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0] == 24 # py[-1] == array([29, 26, 22, 18, 22, 28, 28, 28, 21, 24, 28, 30, 25, 24, 30, 28, 29, 4, 21, 25, 26, 22, 23, 22, 23, 29, 23, 30, 24, 29, 31, 27, 32, 28, 30, 33, 33, 31, 29, 18, 23, 34, 21, 33, 33, 29, 37, 23, 21, 40, 25, 29, 22, 17, 31, 25, 28, 26, 21, 20, 25, 51, 38, 64, 42, 28, 29, 26, 21, 31, 22, 18, 41, 28, 29, 28, 29, 15, 25, 27, 24, 28, 28, 34, 28, 21, 19, 21, 20, 24, 26, 24, 13, 22, 30, 25, 17, 27, 24, 16, 31, 27, 29, 23, 26, 25, 26, 28, 12, 18, 30, 27, 48, 16, 25, 24, 27, 10, 21, 25, 30, 26, 26, 28, 24], dtype=uint32) def test_22(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: - branch = file["E/Evt/hits/hits.pos.x"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][2] == 0.0 + for elem in [True,False]: + branch = file["E/Evt/hits/hits.pos.x"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][2] == 0.0 # py[-1] == array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]) def test_23(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: - branch = file["E/Evt/trks/trks.usr_names"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert len(py[0][0]) == 0 + for elem in [True,False]: + branch = file["E/Evt/trks/trks.usr_names"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert len(py[0][0]) == 0 # py[-1] == array([, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ], dtype=object) def test_24(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: - branch = file["E/Evt/trks/trks.rec_stages"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][1][-1] == 5 + for elem in [True,False]: + branch = file["E/Evt/trks/trks.rec_stages"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][1][-1] == 5 # py[-1] == array([, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ], dtype=object) def test_25(): with uproot.open(skhep_testdata.data_path("uproot-issue-208.root")) as file: - branch = file["config/VERSION/VERSION._name"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][-1] == "numuCCAnalysis" + for elem in [True,False]: + branch = file["config/VERSION/VERSION._name"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][-1] == "numuCCAnalysis" # py[-1] == array(['psychePolicy', 'psycheEventModel', 'psycheCore', 'psycheUtils', 'psycheND280Utils', 'psycheIO', 'psycheSelections', 'psycheSystematics', 'highlandEventModel', 'highlandTools', 'highlandCore', 'highlandCorrections', 'highlandIO', 'baseAnalysis', 'baseTrackerAnalysis', 'numuCCAnalysis'], dtype=object) @@ -286,21 +310,23 @@ def test_25(): def test_27(): with uproot.open(skhep_testdata.data_path("uproot-issue-208.root")) as file: - branch = file["config/SEL/SEL._branchAlias"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0][-1] == "muFGD+Np" + for elem in [True,False]: + branch = file["config/SEL/SEL._branchAlias"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0][-1] == "muFGD+Np" # py[-1] == array([], dtype=object) def test_28(): with uproot.open(skhep_testdata.data_path("uproot-issue-208.root")) as file: - branch = file["config/SEL/SEL._nCutsInBranch"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0][-1] == 10 + for elem in [True,False]: + branch = file["config/SEL/SEL._nCutsInBranch"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0][-1] == 10 # py[-1] == array([], dtype=object) @@ -314,150 +340,165 @@ def test_28(): def test_30(): with uproot.open(skhep_testdata.data_path("uproot-issue243-new.root")) as file: - branch = file["triggerList/triggerMap/triggerMap.first"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=16) - assert len(py[0]) == 0 + for elem in [True,False]: + branch = file["triggerList/triggerMap/triggerMap.first"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=16) + assert len(py[0]) == 0 # py[-1] == array(['HLT_2j35_bmv2c2060_split_2j35_L14J15.0ETA25', 'HLT_j100_2j55_bmv2c2060_split'], dtype=object) def test_31(): with uproot.open(skhep_testdata.data_path("uproot-issue-308.root")) as file: - branch = file["MetaData/BranchIDLists"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0][-1] == 2428801822 + for elem in [True,False]: + branch = file["MetaData/BranchIDLists"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0][-1] == 2428801822 # py[-1] == def test_32(): with uproot.open(skhep_testdata.data_path("uproot-issue31.root")) as file: - branch = file["T/data/name"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1] == "two" + for elem in [True,False]: + branch = file["T/data/name"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1] == "two" # py[-1] == "two" def test_33(): with uproot.open(skhep_testdata.data_path("uproot-issue367b.root")) as file: - branch = file["tree/weights"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][4]["1"][-1] == 1.0 + for elem in [True,False]: + branch = file["tree/weights"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][4]["1"][-1] == 1.0 # py[-1] == def test_34(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: - branch = file["Header/Header./Header.geant4Version"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1] == "$Name: geant4-10-05-patch-01 $" + for elem in [True,False]: + branch = file["Header/Header./Header.geant4Version"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1] == "$Name: geant4-10-05-patch-01 $" def test_35(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: - branch = file["Geant4Data/Geant4Data./Geant4Data.particles"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]["1"][0]["name"] == "anti_alpha" + for elem in [True,False]: + branch = file["Geant4Data/Geant4Data./Geant4Data.particles"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0]["1"][0]["name"] == "anti_alpha" # py[-1] == , ...} at 0x7fb557a012e0> def test_36(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: - branch = file["Model/Model./Model.samplerNamesUnique"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][-1] == "MP_F_99." + for elem in [True,False]: + branch = file["Model/Model./Model.samplerNamesUnique"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][-1] == "MP_F_99." # py[-1] == def test_37(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: - branch = file["Model/Model./Model.staPos"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][555]["fZ"] == 100.94856572890848 + for elem in [True,False]: + branch = file["Model/Model./Model.staPos"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][555]["fZ"] == 100.94856572890848 # py[-1] == array([, , , ..., , , ], dtype=object) def test_38(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: - branch = file["Event/PRBHF_46."] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]["zp"][0] == pytest.approx(0.9999998807907104) + for elem in [True,False]: + branch = file["Event/PRBHF_46."] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0]["zp"][0] == pytest.approx(0.9999998807907104) # py[-1] == (version 4) at 0x7f9be2f7b2e0> def test_39(): with uproot.open(skhep_testdata.data_path("uproot-issue399.root")) as file: - branch = file["Geant4Data/Geant4Data./Geant4Data.particles"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1]["0"] == -1000020030 + for elem in [True,False]: + branch = file["Geant4Data/Geant4Data./Geant4Data.particles"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1]["0"] == -1000020030 # py[-1] == , ...} at 0x7f6d057397f0> def test_40(): with uproot.open(skhep_testdata.data_path("uproot-issue-407.root")) as file: - branch = file["tree/branch"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]["fDatime"] == 1749155840 + for elem in [True,False]: + branch = file["tree/branch"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0]["fDatime"] == 1749155840 # py[-1] == def test_41(): with uproot.open(skhep_testdata.data_path("uproot-issue468.root")) as file: - branch = file["Event/Trajectory./Trajectory.XYZ"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert len(py[0]) == 0 + for elem in [True,False]: + branch = file["Event/Trajectory./Trajectory.XYZ"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert len(py[0]) == 0 # py[-1] == def test_42(): with uproot.open(skhep_testdata.data_path("uproot-issue468.root")) as file: - branch = file["Event/Trajectory./Trajectory.energyDeposit"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert len(py[0]) == 0 + for elem in [True,False]: + branch = file["Event/Trajectory./Trajectory.energyDeposit"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert len(py[0]) == 0 # py[-1] == def test_43(): with uproot.open(skhep_testdata.data_path("uproot-issue468.root")) as file: - branch = file["Event/Trajectory./Trajectory.ionA"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert len(py[0]) == 0 + for elem in [True,False]: + branch = file["Event/Trajectory./Trajectory.ionA"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert len(py[0]) == 0 # py[-1] == def test_44(): with uproot.open(skhep_testdata.data_path("uproot-issue46.root")) as file: - branch = file["tree/evt"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]["ArrayBool"][-1] == True + for elem in [True,False]: + branch = file["tree/evt"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0]["ArrayBool"][-1] == True # py[-1] == @@ -479,11 +520,12 @@ def test_44(): def test_47(): with uproot.open(skhep_testdata.data_path("uproot-issue494.root")) as file: - branch = file["Geant4Data/Geant4Data./Geant4Data.particles"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][-1]["1"]["mass"] == pytest.approx(3.727379) + for elem in [True,False]: + branch = file["Geant4Data/Geant4Data./Geant4Data.particles"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][-1]["1"]["mass"] == pytest.approx(3.727379) # py[-1] == , ...} at 0x7f53a44278b0> @@ -514,50 +556,55 @@ def test_47(): def test_51(): with uproot.open(skhep_testdata.data_path("uproot-issue510b.root")) as file: - branch = file["EDepSimEvents/Event/Primaries/Primaries.GeneratorName"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][0] == "GENIE:fixed@density-fixed" + for elem in [True,False]: + branch = file["EDepSimEvents/Event/Primaries/Primaries.GeneratorName"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][0] == "GENIE:fixed@density-fixed" def test_52(): with uproot.open(skhep_testdata.data_path("uproot-issue513.root")) as file: - branch = file["Delphes/EFlowPhoton/EFlowPhoton.Particles"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][-1]["refs"][-1] == 2223 + for elem in [True,False]: + branch = file["Delphes/EFlowPhoton/EFlowPhoton.Particles"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][-1]["refs"][-1] == 2223 # py[-1] == array([, , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , , ], dtype=object) def test_53(): with uproot.open(skhep_testdata.data_path("uproot-issue513.root")) as file: - branch = file["Delphes/Jet/Jet.SoftDroppedSubJet2"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][-1]["fE"] == 0.0 + for elem in [True,False]: + branch = file["Delphes/Jet/Jet.SoftDroppedSubJet2"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][-1]["fE"] == 0.0 # py[-1] == array([, , , , , ], dtype=object) def test_54(): with uproot.open(skhep_testdata.data_path("uproot-issue513.root")) as file: - branch = file["Delphes/Jet/Jet.TrimmedP4[5]"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][-1][4]["fE"] == 0.0 + for elem in [True,False]: + branch = file["Delphes/Jet/Jet.TrimmedP4[5]"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][-1][4]["fE"] == 0.0 # py[-1] == array([[, , , , ], [, , , , ], [, , , , ], [, , , , ], [, , , , ], [, , , , ]], dtype=object) def test_55(): with uproot.open(skhep_testdata.data_path("uproot-issue519.root")) as file: - branch = file["testtree/testbranch"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1][5] == pytest.approx(0.346174418926239) + for elem in [True,False]: + branch = file["testtree/testbranch"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1][5] == pytest.approx(0.346174418926239) # py[-1] == @@ -565,241 +612,265 @@ def test_56(): with uproot.open( skhep_testdata.data_path("uproot-small-evnt-tree-nosplit.root") ) as file: - branch = file["tree/evt"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1]["SliceU64"][0] == 1 + for elem in [True,False]: + branch = file["tree/evt"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1]["SliceU64"][0] == 1 # py[-1] == def test_57(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/vector_string"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1] == "two" + for elem in [True,False]: + branch = file["tree/vector_string"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1] == "two" # py[-1] == def test_58(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/vector_tstring"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1] == "two" + for elem in [True,False]: + branch = file["tree/vector_tstring"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1] == "two" # py[-1] == def test_59(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/vector_vector_int32"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1][1] == 2 + for elem in [True,False]: + branch = file["tree/vector_vector_int32"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1][1] == 2 # py[-1] == def test_60(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/vector_vector_string"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1][1] == "two" + for elem in [True,False]: + branch = file["tree/vector_vector_string"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1][1] == "two" # py[-1] == def test_61(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/vector_vector_tstring"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1][1] == "two" + for elem in [True,False]: + branch = file["tree/vector_vector_tstring"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1][1] == "two" # py[-1] == def test_62(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/vector_set_int32"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1][1] == 2 + for elem in [True,False]: + branch = file["tree/vector_set_int32"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1][1] == 2 # py[-1] == def test_63(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/vector_set_string"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1][1] == "two" + for elem in [True,False]: + branch = file["tree/vector_set_string"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1][1] == "two" # py[-1] == def test_64(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/set_int32"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1] == 2 + for elem in [True,False]: + branch = file["tree/set_int32"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1] == 2 # py[-1] == def test_65(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/set_string"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1][1] == "two" + for elem in [True,False]: + branch = file["tree/set_string"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1][1] == "two" # py[-1] == def test_66(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/map_int32_int16"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[-1]["1"][1] == 2 + for elem in [True,False]: + branch = file["tree/map_int32_int16"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[-1]["1"][1] == 2 # py[-1] == def test_67(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/map_int32_vector_int16"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]["1"][1] == 2 + for elem in [True,False]: + branch = file["tree/map_int32_vector_int16"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]["1"][1] == 2 # py[-1] == def test_68(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/map_int32_vector_string"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]["1"][1] == "two" + for elem in [True,False]: + branch = file["tree/map_int32_vector_string"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]["1"][1] == "two" # py[-1] == def test_69(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/map_int32_set_int16"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]["1"][1] == 2 + for elem in [True,False]: + branch = file["tree/map_int32_set_int16"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]["1"][1] == 2 # py[-1] == def test_70(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/map_int32_set_string"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]["1"][1] == "two" + for elem in [True,False]: + branch = file["tree/map_int32_set_string"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]["1"][1] == "two" # py[-1] == def test_71(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/map_string_int16"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]["1"] == 2 + for elem in [True,False]: + branch = file["tree/map_string_int16"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]["1"] == 2 # py[-1] == def test_72(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/map_string_vector_int16"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]["1"][1] == 2 + for elem in [True,False]: + branch = file["tree/map_string_vector_int16"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]["1"][1] == 2 # py[-1] == def test_73(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/map_string_vector_string"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]["1"][1] == "two" + for elem in [True,False]: + branch = file["tree/map_string_vector_string"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]["1"][1] == "two" # py[-1] == def test_74(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/map_string_set_int16"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]["1"][1] == 2 + for elem in [True,False]: + branch = file["tree/map_string_set_int16"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]["1"][1] == 2 # py[-1] == def test_75(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/map_string_set_string"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]["1"][1] == "two" + for elem in [True,False]: + branch = file["tree/map_string_set_string"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]["1"][1] == "two" # py[-1] == def test_76(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/map_int32_vector_vector_int16"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]["1"][1][1] == 2 + for elem in [True,False]: + branch = file["tree/map_int32_vector_vector_int16"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]["1"][1][1] == 2 # py[-1] == def test_77(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/map_int32_vector_set_int16"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]["1"][1][1] == 2 + for elem in [True,False]: + branch = file["tree/map_int32_vector_set_int16"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]["1"][1][1] == 2 # py[-1] == def test_78(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/map_string_string"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]["1"] == "TWO" + for elem in [True,False]: + branch = file["tree/map_string_string"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]["1"] == "TWO" # py[-1] == def test_79(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - branch = file["tree/map_string_tstring"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[1][1]["1"] == "TWO" + for elem in [True,False]: + branch = file["tree/map_string_tstring"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[1][1]["1"] == "TWO" # py[-1] == @@ -807,19 +878,21 @@ def test_80(): with uproot.open( skhep_testdata.data_path("uproot-vectorVectorDouble.root") ) as file: - branch = file["t/x"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="np", entry_stop=2) - assert len(py[0]) == 0 + for elem in [True,False]: + branch = file["t/x"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="np", entry_stop=2) + assert len(py[0]) == 0 # py[-1] == def test_81(): with uproot.open(skhep_testdata.data_path("uproot-HZZ-objects.root")) as file: - branch = file["events/MET"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = True - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0]["fY"] == pytest.approx(2.5636332035064697) + for elem in [True,False]: + branch = file["events/MET"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0]["fY"] == pytest.approx(2.5636332035064697) # py[-1] == From 5cf8e48525518157e71a3d86b6d1400081ff3e6c Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Mon, 12 Sep 2022 16:31:08 +0530 Subject: [PATCH 33/41] refactor forth generation --- src/uproot/_awkward_forth.py | 106 ++++------- src/uproot/containers.py | 6 +- src/uproot/interpretation/objects.py | 4 +- src/uproot/streamers.py | 6 +- .../test_0637-setup-tests-for-AwkwardForth.py | 164 +++++++++--------- 5 files changed, 121 insertions(+), 165 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index 7399da599..2f2376e39 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -36,14 +36,11 @@ class ForthGenerator: """ def __init__(self, aform=None, count_obj=0, var_set=False): - self.dummy_form = False - self.top_dummy = None - self.dummy_aform = None self.aform = aform self.top_form = None self.prev_form = None self.awkward_model = {"name": "TOP", "content": {}} - self._prev_node = self.awkward_model + self.top_node = self.awkward_model self.ref_list = [] self.forth_code = {} self.forth_keys = {} @@ -61,9 +58,9 @@ def traverse_aform(self): def replace_form_and_model(self, form, model): temp_node = self.awkward_model temp_prev_form = self.prev_form - temp_node_top = self._prev_node + temp_node_top = self.top_node self.awkward_model = model - self._prev_node = self.awkward_model + self.top_node = self.awkward_model temp_form = self.aform temp_form_top = self.top_form self.top_form = None @@ -112,77 +109,35 @@ def set_dummy_none(self, temp_top, temp_form, temp_flag): self.dummy_form = temp_flag def add_form(self, aform, conlen=0, traverse=1): - if self.dummy_form: - if self.dummy_aform is None: - self.dummy_aform = aform - self.top_dummy = aform - else: - if "content" in self.dummy_aform.keys(): - if self.dummy_aform["content"] == "NULL": - self.dummy_aform["content"] = aform - self.dummy_aform = self.dummy_aform["content"] - else: - raise ValueError - elif "contents" in self.dummy_aform.keys(): - if ( - len(self.dummy_aform["content"]) - < self.dummy_aform["parameters"]["lencon"] - ): - self.dummy_aform["contents"].append(aform) - else: - raise ValueError + if self.aform is None: + self.aform = aform + self.top_form = self.aform + if traverse == 2: + self.aform = self.aform["content"] else: - if self.aform is None: - self.aform = aform - self.top_form = self.aform - if traverse == 2: - self.aform = self.aform["content"] - else: - if "content" in self.aform.keys(): - if self.aform["content"] == "NULL": - self.aform["content"] = aform - self.prev_form = self.aform - if traverse == 2: - self.aform = self.aform["content"]["content"] - self.prev_form = self.prev_form["content"] - else: - self.aform = self.aform["content"] + if "content" in self.aform.keys(): + if self.aform["content"] == "NULL": + self.aform["content"] = aform + self.prev_form = self.aform + if traverse == 2: + self.aform = self.aform["content"]["content"] + self.prev_form = self.prev_form["content"] else: - raise ValueError - elif "contents" in self.aform.keys(): - if self.aform["class"] == "RecordArray": - if self.prev_form is not None: - self.prev_form["content"] = aform - self.aform = aform - else: - self.top_form = aform - self.aform = aform - # for elem in aform['contents'].keys(): - # if self.depth(self.aform['contents'][elem])< self.depth(aform['contents'][elem]): - # aform['contents'][elem] = self.replace_keys(self.aform['contents'][elem],aform['contents'][elem]) - # self.aform['contents'][elem] = aform['contents'][elem] - elif len(self.aform["contents"]) == conlen: - pass + self.aform = self.aform["content"] + else: + raise ValueError + elif "contents" in self.aform.keys(): + if self.aform["class"] == "RecordArray": + if self.prev_form is not None: + self.prev_form["content"] = aform + self.aform = aform else: - raise ValueError - - def depth(self, form): - count = 0 - temp = form - while "content" in temp.keys(): - count += 1 - if isinstance(temp["content"], str): - break - temp = temp["content"] - return count - - def replace_keys(self, old, new): - temp = new - while old != "NULL": - new["form_key"] = old["form_key"] - new = new["content"] - old = old["content"] - return temp + self.top_form = aform + self.aform = aform + elif len(self.aform["contents"]) == conlen: + pass + else: + raise ValueError def get_keys(self, num_keys): if num_keys == 1: @@ -202,7 +157,6 @@ def add_form_key(self, key): self.form_keys.append(key) def go_to(self, aform): - # aform["content"] = self.awkward_model self.awkward_model = aform def become(self, aform): @@ -212,7 +166,7 @@ def check_model(self): return bool(self.awkward_model) def get_current_node(self): - self.ref_list.append(self._prev_node) + self.ref_list.append(self.top_node) return len(self.ref_list) - 1 def get_ref(self, index): diff --git a/src/uproot/containers.py b/src/uproot/containers.py index 88fe2fddc..d3245f640 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -1572,7 +1572,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): if helper_obj.is_forth(): temp = {"name": "TOP", "content": {}} keys_form = forth_obj.top_form - keys_model = forth_obj._prev_node + keys_model = forth_obj.top_node ( temp_node1, temp_node_top1, @@ -1604,11 +1604,11 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): ) if helper_obj.is_forth(): values_form = forth_obj.top_form - values_model = forth_obj._prev_node + values_model = forth_obj.top_node if not isinstance(self._values, numpy.dtype): values_model["content"]["post_code"].append("loop \n") forth_obj.awkward_model = temp_node - forth_obj._prev_node = temp_node_top + forth_obj.top_node = temp_node_top forth_obj.aform = temp_form forth_obj.top_form = temp_form_top forth_obj.prev_form = temp_prev_form diff --git a/src/uproot/interpretation/objects.py b/src/uproot/interpretation/objects.py index 4350568b0..f70443aaa 100644 --- a/src/uproot/interpretation/objects.py +++ b/src/uproot/interpretation/objects.py @@ -261,11 +261,11 @@ def _discover_forth(self, data, byte_offsets, branch, context, cursor_offset): branch.file.detached, branch, ) - context["forth"].gen.awkward_model = context["forth"].gen._prev_node + context["forth"].gen.awkward_model = context["forth"].gen.top_node if not context["forth"].gen.var_set: context["forth"].prereaddone = True self._assemble_forth( - context["forth"].gen, context["forth"].gen._prev_node["content"] + context["forth"].gen, context["forth"].gen.top_node["content"] ) self._complete_forth_code = f"""input stream input byteoffsets diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index a646c1c29..a2093815d 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -705,7 +705,7 @@ def class_code( ) read_member_n.append(" " + read_members[-1]) read_members.append( - " if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.prev_form = temp_prev_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" + " if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj.top_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj.top_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.prev_form = temp_prev_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n forth_obj.add_node_whole(temp_model1, temp_model_ref)\n content.update(temp_form1['contents'])\n forth_obj.enable_adding()" ) # read_members.append( # " if helper_obj.is_forth():\n temp_form = forth_obj.get_temp_form_top()\n content.update(temp_form['contents'])\n forth_obj.set_dummy_none(temp_top_dummy, temp_dummy, temp_top_flag)\n" @@ -1329,7 +1329,7 @@ def class_code( ) read_member_n.append(" " + read_members[-1]) read_members.append( - f" if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj.prev_form = temp_prev_form\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n content[{self.name!r}] = temp_form1\n pre,post,init,header = forth_obj.get_code_recursive(temp_model1)\n helper_obj.add_to_header(header)\n helper_obj.add_to_pre(pre)\n helper_obj.add_to_post(post)\n helper_obj.add_to_init(init)" + f" if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj.top_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj.prev_form = temp_prev_form\n forth_obj.top_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n content[{self.name!r}] = temp_form1\n pre,post,init,header = forth_obj.get_code_recursive(temp_model1)\n helper_obj.add_to_header(header)\n helper_obj.add_to_pre(pre)\n helper_obj.add_to_post(post)\n helper_obj.add_to_init(init)" ) strided_interpretation.append( f" members.append(({self.name!r}, cls._stl_container{len(containers)}.strided_interpretation(file, header, tobject_header, breadcrumbs)))" @@ -1560,7 +1560,7 @@ def class_code( # @aryan26roy: test_0637's 01,02,29,45,46,49,50,56 read_members.append( - f" if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form = forth_obj.replace_form_and_model(None, {{'name': 'TOP', 'content': {{}}}})\n self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)\n if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj._prev_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj.prev_form = temp_prev_form\n forth_obj._prev_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n content[{self.name!r}] = temp_form1\n pre,post,init,header = forth_obj.get_code_recursive(temp_model1)\n helper_obj.add_to_header(header)\n helper_obj.add_to_pre(pre)\n helper_obj.add_to_post(post)\n helper_obj.add_to_init(init)" + f" if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form = forth_obj.replace_form_and_model(None, {{'name': 'TOP', 'content': {{}}}})\n self._members[{self.name!r}] = c({self.typename.rstrip('*')!r}).read(chunk, cursor, context, file, self._file, self.concrete)\n if helper_obj.is_forth():\n temp_prev_form1 = forth_obj.prev_form\n temp_form1 = forth_obj.top_form\n temp_model1 = forth_obj.top_node\n temp_model_ref = forth_obj.awkward_model\n forth_obj.awkward_model = temp_node\n forth_obj.prev_form = temp_prev_form\n forth_obj.top_node = temp_node_top\n forth_obj.aform = temp_form\n forth_obj.top_form = temp_form_top\n temp_model1 = temp_model1['content']\n content[{self.name!r}] = temp_form1\n pre,post,init,header = forth_obj.get_code_recursive(temp_model1)\n helper_obj.add_to_header(header)\n helper_obj.add_to_pre(pre)\n helper_obj.add_to_post(post)\n helper_obj.add_to_init(init)" ) read_member_n.append( " " diff --git a/tests/test_0637-setup-tests-for-AwkwardForth.py b/tests/test_0637-setup-tests-for-AwkwardForth.py index 7147c733b..1167d7722 100644 --- a/tests/test_0637-setup-tests-for-AwkwardForth.py +++ b/tests/test_0637-setup-tests-for-AwkwardForth.py @@ -11,7 +11,7 @@ def test_00(): with uproot.open(skhep_testdata.data_path("issue367b.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/weights"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -22,19 +22,19 @@ def test_00(): def test_01(): with uproot.open(skhep_testdata.data_path("uproot-delphes-pr442.root")) as file: - for elem in [True,False]: - branch = file["Delphes/GenJet/GenJet.SoftDroppedSubJet1"] - interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) - interp._forth = elem - py = branch.array(interp, library="ak", entry_stop=2) - assert py[0][0]["fE"] == pytest.approx(84.56447925448748) - assert py[0][0]["fP"]["fZ"] == pytest.approx(-81.600465) + for elem in [True, False]: + branch = file["Delphes/GenJet/GenJet.SoftDroppedSubJet1"] + interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) + interp._forth = elem + py = branch.array(interp, library="ak", entry_stop=2) + assert py[0][0]["fE"] == pytest.approx(84.56447925448748) + assert py[0][0]["fP"]["fZ"] == pytest.approx(-81.600465) # py[-1] == array([, ], dtype=object) def test_02(): with uproot.open(skhep_testdata.data_path("uproot-delphes-pr442.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["Delphes/GenJet/GenJet.TrimmedP4[5]"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -46,7 +46,7 @@ def test_02(): def test_03(): with uproot.open(skhep_testdata.data_path("uproot-FCCDelphesOutput.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["metadata/gaudiConfigOptions"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -61,7 +61,7 @@ def test_03(): def test_04(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["POOLContainerForm/DataHeaderForm/m_uints"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -81,7 +81,7 @@ def test_04(): def test_06(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["CollectionTree/AnalysisJetsAuxDyn.NumTrkPt500"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -92,7 +92,7 @@ def test_06(): def test_07(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["CollectionTree/AnalysisJetsAuxDyn.SumPtTrkPt500"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -106,12 +106,12 @@ def test_07(): # branch = file["CollectionTree/AnalysisJetsAuxDyn.GhostTrack"] # interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) # py = branch.array(interp, library="np", entry_stop=2) - # py[-1] == > (version 1) at 0x7fc6a08f2f70>, ...], ...] at 0x7fc6a08f2f10> +# py[-1] == > (version 1) at 0x7fc6a08f2f70>, ...], ...] at 0x7fc6a08f2f10> def test_09(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["CollectionTree"][ "AntiKt10UFOCSSKJetsAuxDyn.GhostVR30Rmax4Rmin02TrackJet_BTagging201903" ] @@ -124,7 +124,7 @@ def test_09(): def test_10(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["CollectionTree/CaloCalTopoClustersAuxDyn.e_sampl"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -135,7 +135,7 @@ def test_10(): def test_11(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["CollectionTree"][ "TruthBosonsWithDecayVerticesAuxDyn.incomingParticleLinks" ] @@ -148,7 +148,7 @@ def test_11(): def test_12(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["CollectionTree/TruthBottomAuxDyn.parentLinks"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -159,7 +159,7 @@ def test_12(): def test_13(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["CollectionTree/egammaClustersAuxDyn.constituentClusterLinks"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -170,7 +170,7 @@ def test_13(): def test_14(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["CollectionTree/egammaClustersAuxDyn.eta_sampl"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -181,8 +181,10 @@ def test_14(): def test_15(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - for elem in [True,False]: - branch = file["CollectionTree/AnalysisHLT_mu24_ilooseAuxDyn.TrigMatchedObjects"] + for elem in [True, False]: + branch = file[ + "CollectionTree/AnalysisHLT_mu24_ilooseAuxDyn.TrigMatchedObjects" + ] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem py = branch.array(interp, library="ak", entry_stop=2) @@ -192,7 +194,7 @@ def test_15(): def test_16(): with uproot.open(skhep_testdata.data_path("uproot-issue-123a.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["CollectionTree/AnalysisHLT_mu40AuxDyn.TrigMatchedObjects"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -203,7 +205,7 @@ def test_16(): def test_17(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["E/Evt/AAObject/usr_names"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -214,7 +216,7 @@ def test_17(): def test_18(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["E/Evt/hits/hits.t"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -225,7 +227,7 @@ def test_18(): def test_19(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["E/Evt/hits/hits.a"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -236,7 +238,7 @@ def test_19(): def test_20(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["E/Evt/hits/hits.trig"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -247,7 +249,7 @@ def test_20(): def test_21(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["E/Evt/hits/hits.tot"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -258,7 +260,7 @@ def test_21(): def test_22(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["E/Evt/hits/hits.pos.x"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -269,7 +271,7 @@ def test_22(): def test_23(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["E/Evt/trks/trks.usr_names"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -280,7 +282,7 @@ def test_23(): def test_24(): with uproot.open(skhep_testdata.data_path("uproot-issue390.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["E/Evt/trks/trks.rec_stages"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -291,7 +293,7 @@ def test_24(): def test_25(): with uproot.open(skhep_testdata.data_path("uproot-issue-208.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["config/VERSION/VERSION._name"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -310,7 +312,7 @@ def test_25(): def test_27(): with uproot.open(skhep_testdata.data_path("uproot-issue-208.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["config/SEL/SEL._branchAlias"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -321,7 +323,7 @@ def test_27(): def test_28(): with uproot.open(skhep_testdata.data_path("uproot-issue-208.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["config/SEL/SEL._nCutsInBranch"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -340,7 +342,7 @@ def test_28(): def test_30(): with uproot.open(skhep_testdata.data_path("uproot-issue243-new.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["triggerList/triggerMap/triggerMap.first"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -351,7 +353,7 @@ def test_30(): def test_31(): with uproot.open(skhep_testdata.data_path("uproot-issue-308.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["MetaData/BranchIDLists"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -362,7 +364,7 @@ def test_31(): def test_32(): with uproot.open(skhep_testdata.data_path("uproot-issue31.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["T/data/name"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -373,7 +375,7 @@ def test_32(): def test_33(): with uproot.open(skhep_testdata.data_path("uproot-issue367b.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/weights"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -384,7 +386,7 @@ def test_33(): def test_34(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["Header/Header./Header.geant4Version"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -394,7 +396,7 @@ def test_34(): def test_35(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["Geant4Data/Geant4Data./Geant4Data.particles"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -405,7 +407,7 @@ def test_35(): def test_36(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["Model/Model./Model.samplerNamesUnique"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -416,7 +418,7 @@ def test_36(): def test_37(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["Model/Model./Model.staPos"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -427,7 +429,7 @@ def test_37(): def test_38(): with uproot.open(skhep_testdata.data_path("uproot-issue371.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["Event/PRBHF_46."] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -438,7 +440,7 @@ def test_38(): def test_39(): with uproot.open(skhep_testdata.data_path("uproot-issue399.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["Geant4Data/Geant4Data./Geant4Data.particles"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -449,7 +451,7 @@ def test_39(): def test_40(): with uproot.open(skhep_testdata.data_path("uproot-issue-407.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/branch"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -460,7 +462,7 @@ def test_40(): def test_41(): with uproot.open(skhep_testdata.data_path("uproot-issue468.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["Event/Trajectory./Trajectory.XYZ"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -471,7 +473,7 @@ def test_41(): def test_42(): with uproot.open(skhep_testdata.data_path("uproot-issue468.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["Event/Trajectory./Trajectory.energyDeposit"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -482,7 +484,7 @@ def test_42(): def test_43(): with uproot.open(skhep_testdata.data_path("uproot-issue468.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["Event/Trajectory./Trajectory.ionA"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -493,7 +495,7 @@ def test_43(): def test_44(): with uproot.open(skhep_testdata.data_path("uproot-issue46.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/evt"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -520,7 +522,7 @@ def test_44(): def test_47(): with uproot.open(skhep_testdata.data_path("uproot-issue494.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["Geant4Data/Geant4Data./Geant4Data.particles"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -556,7 +558,7 @@ def test_47(): def test_51(): with uproot.open(skhep_testdata.data_path("uproot-issue510b.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["EDepSimEvents/Event/Primaries/Primaries.GeneratorName"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -566,7 +568,7 @@ def test_51(): def test_52(): with uproot.open(skhep_testdata.data_path("uproot-issue513.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["Delphes/EFlowPhoton/EFlowPhoton.Particles"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -577,7 +579,7 @@ def test_52(): def test_53(): with uproot.open(skhep_testdata.data_path("uproot-issue513.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["Delphes/Jet/Jet.SoftDroppedSubJet2"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -588,7 +590,7 @@ def test_53(): def test_54(): with uproot.open(skhep_testdata.data_path("uproot-issue513.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["Delphes/Jet/Jet.TrimmedP4[5]"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -599,7 +601,7 @@ def test_54(): def test_55(): with uproot.open(skhep_testdata.data_path("uproot-issue519.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["testtree/testbranch"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -612,7 +614,7 @@ def test_56(): with uproot.open( skhep_testdata.data_path("uproot-small-evnt-tree-nosplit.root") ) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/evt"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -623,7 +625,7 @@ def test_56(): def test_57(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/vector_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -634,7 +636,7 @@ def test_57(): def test_58(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/vector_tstring"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -645,7 +647,7 @@ def test_58(): def test_59(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/vector_vector_int32"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -656,7 +658,7 @@ def test_59(): def test_60(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/vector_vector_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -667,7 +669,7 @@ def test_60(): def test_61(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/vector_vector_tstring"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -678,7 +680,7 @@ def test_61(): def test_62(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/vector_set_int32"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -689,7 +691,7 @@ def test_62(): def test_63(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/vector_set_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -700,7 +702,7 @@ def test_63(): def test_64(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/set_int32"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -711,7 +713,7 @@ def test_64(): def test_65(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/set_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -722,7 +724,7 @@ def test_65(): def test_66(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/map_int32_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -733,7 +735,7 @@ def test_66(): def test_67(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/map_int32_vector_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -744,7 +746,7 @@ def test_67(): def test_68(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/map_int32_vector_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -755,7 +757,7 @@ def test_68(): def test_69(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/map_int32_set_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -766,7 +768,7 @@ def test_69(): def test_70(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/map_int32_set_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -777,7 +779,7 @@ def test_70(): def test_71(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/map_string_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -788,7 +790,7 @@ def test_71(): def test_72(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/map_string_vector_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -799,7 +801,7 @@ def test_72(): def test_73(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/map_string_vector_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -810,7 +812,7 @@ def test_73(): def test_74(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/map_string_set_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -821,7 +823,7 @@ def test_74(): def test_75(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/map_string_set_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -832,7 +834,7 @@ def test_75(): def test_76(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/map_int32_vector_vector_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -843,7 +845,7 @@ def test_76(): def test_77(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/map_int32_vector_set_int16"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -854,7 +856,7 @@ def test_77(): def test_78(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/map_string_string"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -865,7 +867,7 @@ def test_78(): def test_79(): with uproot.open(skhep_testdata.data_path("uproot-stl_containers.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["tree/map_string_tstring"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -878,7 +880,7 @@ def test_80(): with uproot.open( skhep_testdata.data_path("uproot-vectorVectorDouble.root") ) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["t/x"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem @@ -889,7 +891,7 @@ def test_80(): def test_81(): with uproot.open(skhep_testdata.data_path("uproot-HZZ-objects.root")) as file: - for elem in [True,False]: + for elem in [True, False]: branch = file["events/MET"] interp = uproot.interpretation.identify.interpretation_of(branch, {}, False) interp._forth = elem From 8b61178cb20132007e4dd547fc2ead75c1ff10fd Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Tue, 13 Sep 2022 13:13:15 +0530 Subject: [PATCH 34/41] remove unused function --- src/uproot/_awkward_forth.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index 2f2376e39..ad2db4859 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -6,8 +6,6 @@ import numpy as np -import uproot.containers - symbol_dict = { np.dtype(">f4"): "f", np.dtype(">f8"): "d", @@ -22,14 +20,6 @@ } -def check_depth(node): - """ - This method checks the depth of the provided container - """ - if isinstance(node, uproot.containers.AsVector): - return 1 - - class ForthGenerator: """ This class is passed through the Forth code generation, collecting Forth snippets and concatenating them at the end. @@ -103,11 +93,6 @@ def should_add_form(self): def get_temp_form_top(self): return self.top_dummy - def set_dummy_none(self, temp_top, temp_form, temp_flag): - self.top_dummy = temp_top - self.dummy_aform = temp_form - self.dummy_form = temp_flag - def add_form(self, aform, conlen=0, traverse=1): if self.aform is None: self.aform = aform From 5745f27cc6fd623451f7ded88d92b6ae05b5e409 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Tue, 13 Sep 2022 13:31:05 +0530 Subject: [PATCH 35/41] refactor code retrieval from helper class --- src/uproot/_awkward_forth.py | 46 ++++++++++++++++-------------------- src/uproot/containers.py | 35 ++++++--------------------- src/uproot/model.py | 15 +++--------- src/uproot/models/TDatime.py | 5 +--- src/uproot/models/TObject.py | 5 +--- src/uproot/models/TRef.py | 5 +--- src/uproot/models/TString.py | 5 +--- src/uproot/streamers.py | 2 +- 8 files changed, 35 insertions(+), 83 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index ad2db4859..4b4966eb2 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -169,9 +169,7 @@ def add_node_whole(self, new_node, ref_latest): self.awkward_model["contents"].append(new_node) self.awkward_model = ref_latest - def add_node( - self, name, precode, postcode, initcode, headercode, dtype, num_child, content - ): + def add_node(self, name, code_attrs, dtype, num_child, content): if isinstance(self.awkward_model, dict): if ( not bool(self.awkward_model["content"]) @@ -180,10 +178,10 @@ def add_node( temp_obj = { "name": name, "type": dtype, - "pre_code": precode, - "post_code": postcode, - "init_code": initcode, - "header_code": headercode, + "pre_code": code_attrs["precode"], + "post_code": code_attrs["postcode"], + "init_code": code_attrs["initcode"], + "header_code": code_attrs["headercode"], "num_child": num_child, "content": content, } @@ -196,10 +194,10 @@ def add_node( temp_obj = { "name": name, "type": dtype, - "pre_code": precode, - "post_code": postcode, - "init_code": initcode, - "header_code": headercode, + "pre_code": code_attrs["precode"], + "post_code": code_attrs["postcode"], + "init_code": code_attrs["initcode"], + "header_code": code_attrs["headercode"], "num_child": num_child, "content": content, } @@ -222,10 +220,10 @@ def add_node( { "name": name, "type": dtype, - "pre_code": precode, - "post_code": postcode, - "init_code": initcode, - "header_code": headercode, + "pre_code": code_attrs["precode"], + "post_code": code_attrs["postcode"], + "init_code": code_attrs["initcode"], + "header_code": code_attrs["headercode"], "num_child": num_child, "content": content, } @@ -297,27 +295,23 @@ def get_gen_obj(self): def add_to_pre(self, code): self._pre_code.append(code) - def get_pre(self): - return self._pre_code - def add_to_post(self, code): self._post_code.append(code) - def get_post(self): - return self._post_code + def get_attrs(self): + return { + "precode": self._pre_code, + "postcode": self._post_code, + "initcode": self._init, + "headercode": self._header, + } def add_to_header(self, code): self._header += code - def get_header(self): - return self._header - def add_to_init(self, code): self._init += code - def get_init(self): - return self._init - def convert_dtype(format): """Takes datatype codes from classses and returns the full datatype name. diff --git a/src/uproot/containers.py b/src/uproot/containers.py index d3245f640..dfd807d8a 100644 --- a/src/uproot/containers.py +++ b/src/uproot/containers.py @@ -76,10 +76,7 @@ def _read_nested( ) temp = forth_obj.add_node( f"node{key}", - helper_obj.get_pre(), - helper_obj.get_post(), - helper_obj.get_init(), - helper_obj.get_header(), + helper_obj.get_attrs(), "i64", 1, None, @@ -546,10 +543,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): helper_obj.add_to_init(f"0 node{offsets_num}-offsets <- stack\n") temp_form = forth_obj.add_node( f"node{offsets_num}", - helper_obj.get_pre(), - helper_obj.get_post(), - helper_obj.get_init(), - helper_obj.get_header(), + helper_obj.get_attrs(), "i64", 0, None, @@ -787,10 +781,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): forth_obj.add_form(json.loads(temp_aform)) temp = forth_obj.add_node( f"node{offsets_num}", - helper_obj.get_pre(), - helper_obj.get_post(), - helper_obj.get_init(), - helper_obj.get_header(), + helper_obj.get_attrs(), "i64", 1, {}, @@ -868,10 +859,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): forth_obj.add_form(json.loads(temp_aform)) temp = forth_obj.add_node( f"node{offsets_num}", - helper_obj.get_pre(), - helper_obj.get_post(), - helper_obj.get_init(), - helper_obj.get_header(), + helper_obj.get_attrs(), "i64", 1, {}, @@ -1197,10 +1185,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): helper_obj.add_to_post("loop\n") temp = forth_obj.add_node( f"node{key}", - helper_obj.get_pre(), - helper_obj.get_post(), - helper_obj.get_init(), - helper_obj.get_header(), + helper_obj.get_attrs(), "i64", 1, {}, @@ -1363,10 +1348,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): helper_obj.add_to_post("loop\n") temp = forth_obj.add_node( f"node{key}", - helper_obj.get_pre(), - helper_obj.get_post(), - helper_obj.get_init(), - helper_obj.get_header(), + helper_obj.get_attrs(), "i64", 1, {}, @@ -1627,10 +1609,7 @@ def read(self, chunk, cursor, context, file, selffile, parent, header=True): forth_obj.add_form(aform) temp = forth_obj.add_node( "nodeMap", - helper_obj.get_pre(), - helper_obj.get_post(), - helper_obj.get_init(), - helper_obj.get_header(), + helper_obj.get_attrs(), "i64", 1, [keys_model["content"], values_model["content"]], diff --git a/src/uproot/model.py b/src/uproot/model.py index 8e060eed4..14453401c 100644 --- a/src/uproot/model.py +++ b/src/uproot/model.py @@ -786,10 +786,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): if helper_obj.is_forth(): forth_obj.add_node( "pass", - helper_obj.get_pre(), - helper_obj.get_post(), - helper_obj.get_init(), - helper_obj.get_header(), + helper_obj.get_attrs(), "i64", 1, {}, @@ -828,10 +825,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): if helper_obj.is_forth(): forth_obj.add_node( "model828", - helper_obj.get_pre(), - helper_obj.get_post(), - helper_obj.get_init(), - helper_obj.get_header(), + helper_obj.get_attrs(), "i64", 1, {}, @@ -1314,10 +1308,7 @@ def read(cls, chunk, cursor, context, file, selffile, parent, concrete=None): helper_obj.add_to_pre(f"{bytes_skipped} stream skip \n") forth_obj.add_node( "Model1319", - helper_obj.get_pre(), - helper_obj.get_post(), - helper_obj.get_init(), - helper_obj.get_header(), + helper_obj.get_attrs(), "i64", 1, {}, diff --git a/src/uproot/models/TDatime.py b/src/uproot/models/TDatime.py index 20eb9a9c2..bca77fa5f 100644 --- a/src/uproot/models/TDatime.py +++ b/src/uproot/models/TDatime.py @@ -49,10 +49,7 @@ def read_members(self, chunk, cursor, context, file): forth_obj.add_form(temp_aform) temp_form = forth_obj.add_node( f"node{key}", - helper_obj.get_pre(), - helper_obj.get_post(), - helper_obj.get_init(), - helper_obj.get_header(), + helper_obj.get_attrs(), "i64", 0, None, diff --git a/src/uproot/models/TObject.py b/src/uproot/models/TObject.py index 5e5617b41..3e4dacd27 100644 --- a/src/uproot/models/TObject.py +++ b/src/uproot/models/TObject.py @@ -57,10 +57,7 @@ def read_members(self, chunk, cursor, context, file): forth_obj.add_form(json.loads(temp_aform)) forth_obj.add_node( "TObjext", - helper_obj.get_pre(), - helper_obj.get_post(), - helper_obj.get_init(), - helper_obj.get_header(), + helper_obj.get_attrs(), "i64", 0, {}, diff --git a/src/uproot/models/TRef.py b/src/uproot/models/TRef.py index 9f5c5807c..7a5afdb0a 100644 --- a/src/uproot/models/TRef.py +++ b/src/uproot/models/TRef.py @@ -167,10 +167,7 @@ def read_members(self, chunk, cursor, context, file): ) forth_obj.add_node( f"node{form_keys[0]}", - helper_obj.get_pre(), - helper_obj.get_post(), - helper_obj.get_init(), - helper_obj.get_header(), + helper_obj.get_attrs(), "i64", 1, None, diff --git a/src/uproot/models/TString.py b/src/uproot/models/TString.py index e78791ed3..f0dcaa89f 100644 --- a/src/uproot/models/TString.py +++ b/src/uproot/models/TString.py @@ -70,10 +70,7 @@ def read_members(self, chunk, cursor, context, file): helper_obj.add_to_init(f"0 node{offsets_num}-offsets <- stack\n") temp_form = forth_obj.add_node( f"node{offsets_num}", - helper_obj.get_pre(), - helper_obj.get_post(), - helper_obj.get_init(), - helper_obj.get_header(), + helper_obj.get_attrs(), "i64", 0, None, diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index a2093815d..bb1b871af 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -279,7 +279,7 @@ def class_code(self): class_flags, ) read_members.append( - " if helper_obj.is_forth():\n if forth_obj.should_add_form():\n forth_obj.add_form({'class': 'RecordArray', 'contents': content, 'parameters': {'record': 'TVector2'}}, len(content))\n temp = forth_obj.add_node('dynamic',helper_obj.get_pre(),helper_obj.get_post(),helper_obj.get_init(),helper_obj.get_header(),\"i64\",0,None)\n" + " if helper_obj.is_forth():\n if forth_obj.should_add_form():\n forth_obj.add_form({'class': 'RecordArray', 'contents': content, 'parameters': {'record': 'TVector2'}}, len(content))\n temp = forth_obj.add_node('dynamic',helper_obj.get_attrs(),\"i64\",0,None)\n" ) if len(read_members) == 1: # untested as of PR #629 From 40b646f6756012c6ceb23e34d360a9fac4b7b3ab Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Tue, 13 Sep 2022 13:40:21 +0530 Subject: [PATCH 36/41] remove redundant functions --- src/uproot/_awkward_forth.py | 57 ------------------------------------ 1 file changed, 57 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index 8ef618cfa..405f9dd98 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -127,68 +127,11 @@ def add_form(self, aform, conlen=0, traverse=1): else: return False - def get_temp_form_top(self): - return self.top_dummy - def set_dummy_none(self, temp_top, temp_form, temp_flag): self.top_dummy = temp_top self.dummy_aform = temp_form self.dummy_form = temp_flag - def add_form(self, aform, conlen=0, traverse=1): - if self.dummy_form: - if self.dummy_aform is None: - self.dummy_aform = aform - self.top_dummy = aform - else: - if "content" in self.dummy_aform.keys(): - if self.dummy_aform["content"] == "NULL": - self.dummy_aform["content"] = aform - self.dummy_aform = self.dummy_aform["content"] - else: - raise ValueError - elif "contents" in self.dummy_aform.keys(): - if ( - len(self.dummy_aform["content"]) - < self.dummy_aform["parameters"]["lencon"] - ): - self.dummy_aform["contents"].append(aform) - else: - raise ValueError - else: - if self.aform is None: - self.aform = aform - self.top_form = self.aform - if traverse == 2: - self.aform = self.aform["content"] - else: - if "content" in self.aform.keys(): - if self.aform["content"] == "NULL": - self.aform["content"] = aform - self.prev_form = self.aform - if traverse == 2: - self.aform = self.aform["content"]["content"] - self.prev_form = self.prev_form["content"] - else: - self.aform = self.aform["content"] - else: - raise ValueError - elif "contents" in self.aform.keys(): - if self.aform["class"] == "RecordArray": - if self.prev_form is not None: - self.prev_form["content"] = aform - self.aform = aform - else: - self.top_form = aform - self.aform = aform - # for elem in aform['contents'].keys(): - # if self.depth(self.aform['contents'][elem])< self.depth(aform['contents'][elem]): - # aform['contents'][elem] = self.replace_keys(self.aform['contents'][elem],aform['contents'][elem]) - # self.aform['contents'][elem] = aform['contents'][elem] - elif len(self.aform["contents"]) == conlen: - pass - else: - raise ValueError def depth(self, form): count = 0 From 45f1f58af9736953502b3cc991810d2cadba80e8 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Tue, 13 Sep 2022 13:40:55 +0530 Subject: [PATCH 37/41] linting --- src/uproot/_awkward_forth.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index 405f9dd98..1a39f4bda 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -132,7 +132,6 @@ def set_dummy_none(self, temp_top, temp_form, temp_flag): self.dummy_aform = temp_form self.dummy_form = temp_flag - def depth(self, form): count = 0 temp = form From 10e2c2e306037588c99687c1cb53c82d4786d529 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Thu, 15 Sep 2022 19:22:45 +0530 Subject: [PATCH 38/41] remove unused functions and improve error messages --- src/uproot/_awkward_forth.py | 22 ++-------------------- src/uproot/streamers.py | 2 +- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index 1a39f4bda..9df1be268 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -113,7 +113,7 @@ def add_form(self, aform, conlen=0, traverse=1): else: self.aform = self.aform["content"] else: - raise ValueError + raise ValueError("Form representation is corrupted.") elif "contents" in self.aform.keys(): if self.aform["class"] == "RecordArray": if self.prev_form is not None: @@ -132,24 +132,6 @@ def set_dummy_none(self, temp_top, temp_form, temp_flag): self.dummy_aform = temp_form self.dummy_form = temp_flag - def depth(self, form): - count = 0 - temp = form - while "content" in temp.keys(): - count += 1 - if isinstance(temp["content"], str): - break - temp = temp["content"] - return count - - def replace_keys(self, old, new): - temp = new - while old != "NULL": - new["form_key"] = old["form_key"] - new = new["content"] - old = old["content"] - return temp - def get_keys(self, num_keys): if num_keys == 1: key = self.count_obj @@ -283,7 +265,7 @@ def add_forth_code(self, ref, forth_exec_pre, forth_exec_post): def add_to_final(self, code): if not isinstance(code, list): - raise TypeError + raise TypeError("Something went wrong with Forth code generation.") self.final_code.extend(code) return diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index bb1b871af..140494973 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -848,7 +848,7 @@ def class_code( f' content[{self.name!r}] = {{"class": "ListOffsetArray", "offsets": "i64", "content": {{ "class": "NumpyArray", "primitive": f"{{uproot._awkward_forth.convert_dtype(uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}])}}", "inner_shape": [], "has_identifier": False, "parameters": {{}}, "form_key": f"node{{key}}"}}, "form_key": f"node{{key2}}"}}' ) read_members.append( - f' #raise NotImplementedError\n helper_obj.add_to_pre(f" var_{self.count_name} @ dup node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> node{{key}}-data\\n")' + f' helper_obj.add_to_pre(f" var_{self.count_name} @ dup node{{key2}}-offsets +<- stack \\n stream #!{{uproot._awkward_forth.symbol_dict[self._dtype{len(dtypes)}]}}-> node{{key}}-data\\n")' ) read_members.append(" if forth_obj.should_add_form():") read_members.append( From 23fb863229674979f6926aa09d0ad720ee14f4e7 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Sun, 18 Sep 2022 19:52:49 +0530 Subject: [PATCH 39/41] fix error types --- src/uproot/_awkward_forth.py | 6 +++--- src/uproot/streamers.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index 9df1be268..ae57e8f50 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -113,7 +113,7 @@ def add_form(self, aform, conlen=0, traverse=1): else: self.aform = self.aform["content"] else: - raise ValueError("Form representation is corrupted.") + raise AssertionError("Form representation is corrupted.") elif "contents" in self.aform.keys(): if self.aform["class"] == "RecordArray": if self.prev_form is not None: @@ -144,7 +144,7 @@ def get_keys(self, num_keys): self.count_obj += 1 return out else: - raise ValueError("Number of keys cannot be less than 1") + raise AssertionError("Number of keys cannot be less than 1") def add_form_key(self, key): self.form_keys.append(key) @@ -265,7 +265,7 @@ def add_forth_code(self, ref, forth_exec_pre, forth_exec_post): def add_to_final(self, code): if not isinstance(code, list): - raise TypeError("Something went wrong with Forth code generation.") + raise AssertionError("Something went wrong with Forth code generation.") self.final_code.extend(code) return diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index 140494973..d5be91d80 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -696,7 +696,6 @@ def class_code( read_member_n.append(f" if member_index == {i}:") # @aryan26roy: test_0637's 01,02,08,09,11,12,13,15,16,29,38,45,46,49,50 - # raise NotImplementedError read_members.append( " if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form = forth_obj.replace_form_and_model(None, {'name': 'TOP', 'content': {}})\n" ) From 9bb7307e15a113961074fe37d039440a47fbf109 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Sun, 18 Sep 2022 20:13:33 +0530 Subject: [PATCH 40/41] rolling back --- src/uproot/_awkward_forth.py | 6 +++--- src/uproot/streamers.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index ae57e8f50..9df1be268 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -113,7 +113,7 @@ def add_form(self, aform, conlen=0, traverse=1): else: self.aform = self.aform["content"] else: - raise AssertionError("Form representation is corrupted.") + raise ValueError("Form representation is corrupted.") elif "contents" in self.aform.keys(): if self.aform["class"] == "RecordArray": if self.prev_form is not None: @@ -144,7 +144,7 @@ def get_keys(self, num_keys): self.count_obj += 1 return out else: - raise AssertionError("Number of keys cannot be less than 1") + raise ValueError("Number of keys cannot be less than 1") def add_form_key(self, key): self.form_keys.append(key) @@ -265,7 +265,7 @@ def add_forth_code(self, ref, forth_exec_pre, forth_exec_post): def add_to_final(self, code): if not isinstance(code, list): - raise AssertionError("Something went wrong with Forth code generation.") + raise TypeError("Something went wrong with Forth code generation.") self.final_code.extend(code) return diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index d5be91d80..140494973 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -696,6 +696,7 @@ def class_code( read_member_n.append(f" if member_index == {i}:") # @aryan26roy: test_0637's 01,02,08,09,11,12,13,15,16,29,38,45,46,49,50 + # raise NotImplementedError read_members.append( " if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form = forth_obj.replace_form_and_model(None, {'name': 'TOP', 'content': {}})\n" ) From 20ebe9b81ae61b39157e5d725da0d67940a3e212 Mon Sep 17 00:00:00 2001 From: Aryan Roy Date: Sun, 18 Sep 2022 21:59:19 +0530 Subject: [PATCH 41/41] fix error types --- src/uproot/_awkward_forth.py | 6 +++--- src/uproot/streamers.py | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/uproot/_awkward_forth.py b/src/uproot/_awkward_forth.py index 9df1be268..ae57e8f50 100644 --- a/src/uproot/_awkward_forth.py +++ b/src/uproot/_awkward_forth.py @@ -113,7 +113,7 @@ def add_form(self, aform, conlen=0, traverse=1): else: self.aform = self.aform["content"] else: - raise ValueError("Form representation is corrupted.") + raise AssertionError("Form representation is corrupted.") elif "contents" in self.aform.keys(): if self.aform["class"] == "RecordArray": if self.prev_form is not None: @@ -144,7 +144,7 @@ def get_keys(self, num_keys): self.count_obj += 1 return out else: - raise ValueError("Number of keys cannot be less than 1") + raise AssertionError("Number of keys cannot be less than 1") def add_form_key(self, key): self.form_keys.append(key) @@ -265,7 +265,7 @@ def add_forth_code(self, ref, forth_exec_pre, forth_exec_post): def add_to_final(self, code): if not isinstance(code, list): - raise TypeError("Something went wrong with Forth code generation.") + raise AssertionError("Something went wrong with Forth code generation.") self.final_code.extend(code) return diff --git a/src/uproot/streamers.py b/src/uproot/streamers.py index 140494973..d5be91d80 100644 --- a/src/uproot/streamers.py +++ b/src/uproot/streamers.py @@ -696,7 +696,6 @@ def class_code( read_member_n.append(f" if member_index == {i}:") # @aryan26roy: test_0637's 01,02,08,09,11,12,13,15,16,29,38,45,46,49,50 - # raise NotImplementedError read_members.append( " if helper_obj.is_forth():\n temp_node, temp_node_top, temp_form, temp_form_top, temp_prev_form = forth_obj.replace_form_and_model(None, {'name': 'TOP', 'content': {}})\n" )