From 4dd5ada16fd99ebc0f88e17afa12661313577314 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Thu, 14 Jan 2021 22:52:10 +0100 Subject: [PATCH 01/16] add in pypy support + change test timeout to 60 seconds --- synapse/python_dependencies.py | 8 ++++++-- synapse/storage/engines/sqlite.py | 6 ++++++ tests/__init__.py | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index bfd46a3730db..968a1bc85bde 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -86,8 +86,12 @@ CONDITIONAL_REQUIREMENTS = { "matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"], - # we use execute_values with the fetch param, which arrived in psycopg 2.8. - "postgres": ["psycopg2>=2.8"], + "postgres": [ + # we use execute_values with the fetch param, which arrived in psycopg 2.8. + "psycopg2>=2.8 ; platform_python_implementation != 'PyPy'", + "psycopg2cffi>=2.7 ; platform_python_implementation == 'PyPy'", + "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'" + ], # ACME support is required to provision TLS certificates from authorities # that use the protocol, such as Let's Encrypt. "acme": [ diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py index 5db0f0b520db..2ea7bd94a314 100644 --- a/synapse/storage/engines/sqlite.py +++ b/synapse/storage/engines/sqlite.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import platform import struct import threading import typing @@ -30,6 +31,11 @@ def __init__(self, database_module, database_config): database = database_config.get("args", {}).get("database") self._is_in_memory = database in (None, ":memory:",) + if platform.python_implementation() == "PyPy": + # related to #6186, pypy's sqlite3 module doesn't like bytearrays, + # so here we're converting them back + database_module.register_adapter(bytearray, lambda array: bytes(array)) + # The current max state_group, or None if we haven't looked # in the DB yet. self._current_state_group_id = None diff --git a/tests/__init__.py b/tests/__init__.py index ed805db1c2d0..0f0ea28ae3c9 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -21,4 +21,4 @@ # attempt to do the patch before we load any synapse code do_patch() -util.DEFAULT_TIMEOUT_DURATION = 20 +util.DEFAULT_TIMEOUT_DURATION = 60.0 From 6dd497438bbf205fe8ec64e062604833b1051227 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Thu, 14 Jan 2021 22:56:15 +0100 Subject: [PATCH 02/16] add news --- changelog.d/9123.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/9123.misc diff --git a/changelog.d/9123.misc b/changelog.d/9123.misc new file mode 100644 index 000000000000..85563b8be71d --- /dev/null +++ b/changelog.d/9123.misc @@ -0,0 +1 @@ +Add in preliminary PyPy support. \ No newline at end of file From 6f7bd6aa3bb619b90eebaf93822e42228c8ebafe Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Thu, 14 Jan 2021 23:01:18 +0100 Subject: [PATCH 03/16] make style checker happy --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 968a1bc85bde..c062491e87df 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -90,7 +90,7 @@ # we use execute_values with the fetch param, which arrived in psycopg 2.8. "psycopg2>=2.8 ; platform_python_implementation != 'PyPy'", "psycopg2cffi>=2.7 ; platform_python_implementation == 'PyPy'", - "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'" + "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'", ], # ACME support is required to provision TLS certificates from authorities # that use the protocol, such as Let's Encrypt. From 74a126fcecd114b4ad9359b20ac9537641af4c06 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Fri, 29 Jan 2021 21:53:34 +0100 Subject: [PATCH 04/16] pet the linter --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index c062491e87df..760edb83eeda 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -87,7 +87,7 @@ CONDITIONAL_REQUIREMENTS = { "matrix-synapse-ldap3": ["matrix-synapse-ldap3>=0.1"], "postgres": [ - # we use execute_values with the fetch param, which arrived in psycopg 2.8. + # we use execute_values with the fetch param, which arrived in psycopg 2.8. "psycopg2>=2.8 ; platform_python_implementation != 'PyPy'", "psycopg2cffi>=2.7 ; platform_python_implementation == 'PyPy'", "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'", From e20aec88e89692bdaf997ed869543d4593483e66 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Sat, 30 Jan 2021 14:08:03 +0100 Subject: [PATCH 05/16] update psycopg2cffi --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 760edb83eeda..8a2b73b75e3c 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -89,7 +89,7 @@ "postgres": [ # we use execute_values with the fetch param, which arrived in psycopg 2.8. "psycopg2>=2.8 ; platform_python_implementation != 'PyPy'", - "psycopg2cffi>=2.7 ; platform_python_implementation == 'PyPy'", + "psycopg2cffi>=2.8 ; platform_python_implementation == 'PyPy'", "psycopg2cffi-compat==1.1 ; platform_python_implementation == 'PyPy'", ], # ACME support is required to provision TLS certificates from authorities From f45af9eef097fab2a6439900824abd2c24fb77b8 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Sat, 30 Jan 2021 21:17:50 +0100 Subject: [PATCH 06/16] simplify psycopg2 imports Co-authored-by: intelfx --- synapse/storage/engines/__init__.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index 035f9ea6e98b..a406deb2d490 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -28,11 +28,8 @@ def create_engine(database_config) -> BaseDatabaseEngine: return Sqlite3Engine(sqlite3, database_config) if name == "psycopg2": - # pypy requires psycopg2cffi rather than psycopg2 - if platform.python_implementation() == "PyPy": - import psycopg2cffi as psycopg2 # type: ignore - else: - import psycopg2 # type: ignore + # note: pypy is still supported by using psycopg2cffi-compat to monkey-patch imports to psycopg2 + import psycopg2 # type: ignore return PostgresEngine(psycopg2, database_config) From aa2a3a4696d8c6e0ae5e0225f73804b75889740e Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Sat, 30 Jan 2021 23:15:49 +0100 Subject: [PATCH 07/16] pet the linter --- synapse/storage/engines/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index a406deb2d490..8225a6746c99 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import platform from ._base import BaseDatabaseEngine, IncorrectDatabaseSetup from .postgres import PostgresEngine From 3985c2452039cea341853c4151d37fcfe7ab7eee Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Tue, 2 Feb 2021 09:53:46 +0100 Subject: [PATCH 08/16] make tests slightly faster on PyPy --- tests/unittest.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unittest.py b/tests/unittest.py index 767d5d607738..2485f3b26920 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -19,6 +19,7 @@ import hmac import inspect import logging +import platform import time from typing import Callable, Dict, Iterable, Optional, Tuple, Type, TypeVar, Union @@ -126,7 +127,8 @@ def tearDown(orig): ret = orig() # force a GC to workaround problems with deferreds leaking logcontexts when # they are GCed (see the logcontext docs) - gc.collect() + if platform.python_implementation() != "PyPy": + gc.collect() set_current_context(SENTINEL_CONTEXT) return ret From aba36dfaa374fa6bcd0409d519a4aa3ba37fb946 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Wed, 3 Feb 2021 17:28:39 +0100 Subject: [PATCH 09/16] Update synapse/storage/engines/sqlite.py Co-authored-by: Patrick Cloke --- synapse/storage/engines/sqlite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py index 2ea7bd94a314..b3d1834efbec 100644 --- a/synapse/storage/engines/sqlite.py +++ b/synapse/storage/engines/sqlite.py @@ -32,8 +32,8 @@ def __init__(self, database_module, database_config): self._is_in_memory = database in (None, ":memory:",) if platform.python_implementation() == "PyPy": - # related to #6186, pypy's sqlite3 module doesn't like bytearrays, - # so here we're converting them back + # pypy's sqlite3 module doesn't handle bytearrays, convert them + # back to bytes. database_module.register_adapter(bytearray, lambda array: bytes(array)) # The current max state_group, or None if we haven't looked From e81576f027a160bb50201ad0a9fef96e6ac07469 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Wed, 3 Feb 2021 17:29:15 +0100 Subject: [PATCH 10/16] Update psycopg2cffi-compat comment note Co-authored-by: Patrick Cloke --- synapse/storage/engines/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index 8225a6746c99..8bc6a6046cea 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -27,7 +27,7 @@ def create_engine(database_config) -> BaseDatabaseEngine: return Sqlite3Engine(sqlite3, database_config) if name == "psycopg2": - # note: pypy is still supported by using psycopg2cffi-compat to monkey-patch imports to psycopg2 + # Note that psycopg2cffi-compat provides the psycopg2 module on mypy. import psycopg2 # type: ignore return PostgresEngine(psycopg2, database_config) From 0d00f25ddc64b4fdc4a00eed2971690a9eaa0cf0 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Wed, 3 Feb 2021 17:30:52 +0100 Subject: [PATCH 11/16] revert default timeout duration --- tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index 0f0ea28ae3c9..785f60520da5 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -21,4 +21,4 @@ # attempt to do the patch before we load any synapse code do_patch() -util.DEFAULT_TIMEOUT_DURATION = 60.0 +util.DEFAULT_TIMEOUT_DURATION = 20.0 From a028df1758c122107f5cbfe2000cd0b0d7d9c4bf Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Wed, 3 Feb 2021 18:04:16 +0100 Subject: [PATCH 12/16] Update changelog.d/9123.misc Co-authored-by: Patrick Cloke --- changelog.d/9123.misc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/9123.misc b/changelog.d/9123.misc index 85563b8be71d..329600c40c2b 100644 --- a/changelog.d/9123.misc +++ b/changelog.d/9123.misc @@ -1 +1 @@ -Add in preliminary PyPy support. \ No newline at end of file +Add experimental support for running Synapse with PyPy. From 52025ddbbb949a7f98afac812842539cc2423165 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Wed, 3 Feb 2021 18:08:49 +0100 Subject: [PATCH 13/16] revert DEFAULT_TIMEOUT_DURATION Co-authored-by: Patrick Cloke --- tests/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__init__.py b/tests/__init__.py index 785f60520da5..ed805db1c2d0 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -21,4 +21,4 @@ # attempt to do the patch before we load any synapse code do_patch() -util.DEFAULT_TIMEOUT_DURATION = 20.0 +util.DEFAULT_TIMEOUT_DURATION = 20 From bbab091a878a47a6bf604644f3aa11a90af19827 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Wed, 3 Feb 2021 21:16:00 +0100 Subject: [PATCH 14/16] revert gc.collect removal --- tests/unittest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unittest.py b/tests/unittest.py index 2485f3b26920..4254df8d2071 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -127,8 +127,7 @@ def tearDown(orig): ret = orig() # force a GC to workaround problems with deferreds leaking logcontexts when # they are GCed (see the logcontext docs) - if platform.python_implementation() != "PyPy": - gc.collect() + gc.collect() set_current_context(SENTINEL_CONTEXT) return ret From c443cf3bb2ece005b0a14537f7fd06823c6eea03 Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Wed, 3 Feb 2021 21:19:18 +0100 Subject: [PATCH 15/16] rename mypy -> pypy --- synapse/storage/engines/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index 8bc6a6046cea..d15ccfacdeb7 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -27,7 +27,7 @@ def create_engine(database_config) -> BaseDatabaseEngine: return Sqlite3Engine(sqlite3, database_config) if name == "psycopg2": - # Note that psycopg2cffi-compat provides the psycopg2 module on mypy. + # Note that psycopg2cffi-compat provides the psycopg2 module on pypy. import psycopg2 # type: ignore return PostgresEngine(psycopg2, database_config) From 39657ba6339ad4bc6ca5b657a520bbecadfa6a9d Mon Sep 17 00:00:00 2001 From: Jonathan de Jong Date: Wed, 3 Feb 2021 21:21:23 +0100 Subject: [PATCH 16/16] remove platform import --- tests/unittest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unittest.py b/tests/unittest.py index 4254df8d2071..767d5d607738 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -19,7 +19,6 @@ import hmac import inspect import logging -import platform import time from typing import Callable, Dict, Iterable, Optional, Tuple, Type, TypeVar, Union