From a54ca7dd1cefec22d75a39583ad0fbf0b6074ecf Mon Sep 17 00:00:00 2001 From: Andrew Hilger Date: Thu, 19 Dec 2024 16:44:27 -0800 Subject: [PATCH] pickle backward compat test Summary: This test verifies that thrift-py3 auto-migrated Structs, Unions, Enums, and containers are backwards compatible with pickle use cases, even where a thrift-py3 type has been serialized and stored, it can still be unpickled into an equivalent thrift-python type. This makes sense because both py3 and python follow the same pickling protocols (just delegate to serialize-deserialize). The containers work only because of recent D66985311. Reviewed By: Filip-F Differential Revision: D67479759 fbshipit-source-id: d35e5f0eff3ce6c20385950e9091a03dda00b521 --- .../lib/py3/test/auto_migrate/serializer.py | 82 ++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/third-party/thrift/src/thrift/lib/py3/test/auto_migrate/serializer.py b/third-party/thrift/src/thrift/lib/py3/test/auto_migrate/serializer.py index 932b412b8f94db..cf5f961c0c79b4 100644 --- a/third-party/thrift/src/thrift/lib/py3/test/auto_migrate/serializer.py +++ b/third-party/thrift/src/thrift/lib/py3/test/auto_migrate/serializer.py @@ -167,13 +167,31 @@ def test_bad_deserialize(self) -> None: def pickle_round_robin( self, - control: Union[Struct, Hashable], + control: Struct | Hashable, ) -> None: encoded = pickle.dumps(control, protocol=pickle.HIGHEST_PROTOCOL) decoded = pickle.loads(encoded) self.assertIsInstance(decoded, type(control)) self.assertEqual(control, decoded) + # tests py3 auto-migrate backwards compatibility + # bytes produced by pickle.dumps(control, protocol=pickle.HIGHEST_PROTOCOL) + # when run in normal mode (auto-migrate off) + def assert_unpickle_compat(self, stored: bytes, control: Struct | Hashable) -> None: + decoded = None + try: + decoded = pickle.loads(stored) + except pickle.UnpicklingError: + self.fail( + f"failed to unpickle {stored=}" + f"encoded control={pickle.dumps(control, protocol=pickle.HIGHEST_PROTOCOL)}" + ) + self.assertIsInstance(decoded, type(control)) + self.assertEqual( + control, + decoded, + ) + def test_serialize_easy_struct(self) -> None: control = easy(val=5, val_list=[1, 2, 3, 4]) fixtures: Mapping[Protocol, bytes] = { @@ -191,6 +209,18 @@ def test_pickle_easy_struct(self) -> None: control = easy(val=0, val_list=[5, 6, 7]) self.pickle_round_robin(control) + def test_unpickle_stored_easy_struct(self) -> None: + control = easy(val=0, val_list=[5, 6, 7]) + # string produced with pickle.dumps(control, protocol=pickle.HIGHEST_PROTOCOL) + # when run in normal mode (auto-migrate off) + stored = ( + b"\x80\x05\x95U\x00\x00\x00\x00\x00\x00\x00\x8c\x15thrift.py3." + b"serializer\x94\x8c\x0bdeserialize\x94\x93\x94\x8c\rtesting.types" + b"\x94\x8c\x04easy\x94\x93\x94C\x0c\x15\x00\x195\n\x0c\x0e,\x00\x16" + b"\x00\x00\x94\x86\x94R\x94." + ) + self.assert_unpickle_compat(stored, control) + def test_serialize_hard_struct(self) -> None: control = hard( val=0, val_list=[1, 2, 3, 4], name="foo", an_int=Integers(tiny=1) @@ -231,6 +261,29 @@ def test_pickle_Integers_union(self) -> None: control = Integers(large=2**32) self.pickle_round_robin(control) + def test_unpickle_stored_Integers_union(self) -> None: + control = Integers(large=2**32) + stored = ( + b"\x80\x05\x95T\x00\x00\x00\x00\x00\x00\x00\x8c\x15thrift.py3." + b"serializer\x94\x8c\x0bdeserialize\x94\x93\x94\x8c\rtesting.types" + b"\x94\x8c\x08Integers\x94\x93\x94C\x07F\x80\x80\x80\x80 \x00\x94" + b"\x86\x94R\x94." + ) + self.assert_unpickle_compat(stored, control) + + def test_pickle_enum(self) -> None: + control = MyEnum.ME1 + self.pickle_round_robin(control) + + def test_unpickle_stored_enum(self) -> None: + control = MyEnum.ME1 + stored = ( + b"\x80\x05\x95E\x00\x00\x00\x00\x00\x00\x00\x8c0apache.thrift.test." + b"terse_write.terse_write.types\x94\x8c\x06MyEnum\x94\x93\x94K\x01" + b"\x85\x94R\x94." + ) + self.assert_unpickle_compat(stored, control) + def test_pickle_sequence(self) -> None: control = I32List([1, 2, 3, 4]) self.pickle_round_robin(control) @@ -240,14 +293,41 @@ def test_pickle_sequence(self) -> None: assert data self.pickle_round_robin(data) + def test_unpickle_stored_sequence(self) -> None: + control = I32List([1, 2, 3, 4]) + stored = ( + b"\x80\x05\x95/\x00\x00\x00\x00\x00\x00\x00\x8c\rtesting.types" + b"\x94\x8c\tList__i32\x94\x93\x94]\x94(K\x01K\x02K\x03K\x04e" + b"\x85\x94R\x94." + ) + self.assert_unpickle_compat(stored, control) + def test_pickle_set(self) -> None: control = SetI32({1, 2, 3, 4}) self.pickle_round_robin(control) + def test_unpickle_stored_set(self) -> None: + control = SetI32({1, 2, 3, 4}) + stored = ( + b"\x80\x05\x95.\x00\x00\x00\x00\x00\x00\x00\x8c\rtesting.types" + b"\x94\x8c\x08Set__i32\x94\x93\x94\x8f\x94(K\x01K\x02K\x03K\x04" + b"\x90\x85\x94R\x94." + ) + self.assert_unpickle_compat(stored, control) + def test_pickle_mapping(self) -> None: control = StrStrMap({"test": "test", "foo": "bar"}) self.pickle_round_robin(control) + def test_unpickle_stored_mapping(self) -> None: + control = StrStrMap({"test": "test", "foo": "bar"}) + stored = ( + b"\x80\x05\x95E\x00\x00\x00\x00\x00\x00\x00\x8c\rtesting.types" + b"\x94\x8c\x12Map__string_string\x94\x93\x94}\x94(\x8c\x04test" + b"\x94h\x04\x8c\x03foo\x94\x8c\x03bar\x94u\x85\x94R\x94." + ) + self.assert_unpickle_compat(stored, control) + def test_deserialize_with_length(self) -> None: control = easy(val=5, val_list=[1, 2, 3, 4, 5]) self.with_length_round_robin(control)