From 68788fa3fea47992e97f220f04cc8873b28a551b Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 5 May 2022 23:04:01 +0100 Subject: [PATCH 01/16] Bump mypy and mypy-zope --- poetry.lock | 58 +++++++++++++++++++++++++++----------------------- pyproject.toml | 4 ++-- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/poetry.lock b/poetry.lock index 89e78576b7b6..64b82d27c4a9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -572,7 +572,7 @@ python-versions = "*" [[package]] name = "mypy" -version = "0.931" +version = "0.950" description = "Optional static typing for Python" category = "dev" optional = false @@ -580,13 +580,14 @@ python-versions = ">=3.6" [package.dependencies] mypy-extensions = ">=0.4.3" -tomli = ">=1.1.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} typed-ast = {version = ">=1.4.0,<2", markers = "python_version < \"3.8\""} typing-extensions = ">=3.10" [package.extras] dmypy = ["psutil (>=4.0)"] python2 = ["typed-ast (>=1.4.0,<2)"] +reports = ["lxml"] [[package]] name = "mypy-extensions" @@ -598,14 +599,14 @@ python-versions = "*" [[package]] name = "mypy-zope" -version = "0.3.5" +version = "0.3.7" description = "Plugin for mypy to support zope interfaces" category = "dev" optional = false python-versions = "*" [package.dependencies] -mypy = "0.931" +mypy = "0.950" "zope.interface" = "*" "zope.schema" = "*" @@ -1562,7 +1563,7 @@ url_preview = ["lxml"] [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "2bda1a7cfc8cc02832b4a7d16bf7e1615cb05e0639bdb30688aadf692d851942" +content-hash = "f24699464828ac1a63f1034b4a18c841ef585737b9a802fd8311836444f1d702" [metadata.files] attrs = [ @@ -2089,34 +2090,37 @@ msgpack = [ {file = "msgpack-1.0.3.tar.gz", hash = "sha256:51fdc7fb93615286428ee7758cecc2f374d5ff363bdd884c7ea622a7a327a81e"}, ] mypy = [ - {file = "mypy-0.931-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c5b42d0815e15518b1f0990cff7a705805961613e701db60387e6fb663fe78a"}, - {file = "mypy-0.931-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c89702cac5b302f0c5d33b172d2b55b5df2bede3344a2fbed99ff96bddb2cf00"}, - {file = "mypy-0.931-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:300717a07ad09525401a508ef5d105e6b56646f7942eb92715a1c8d610149714"}, - {file = "mypy-0.931-cp310-cp310-win_amd64.whl", hash = "sha256:7b3f6f557ba4afc7f2ce6d3215d5db279bcf120b3cfd0add20a5d4f4abdae5bc"}, - {file = "mypy-0.931-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:1bf752559797c897cdd2c65f7b60c2b6969ffe458417b8d947b8340cc9cec08d"}, - {file = "mypy-0.931-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4365c60266b95a3f216a3047f1d8e3f895da6c7402e9e1ddfab96393122cc58d"}, - {file = "mypy-0.931-cp36-cp36m-win_amd64.whl", hash = "sha256:1b65714dc296a7991000b6ee59a35b3f550e0073411ac9d3202f6516621ba66c"}, - {file = "mypy-0.931-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e839191b8da5b4e5d805f940537efcaa13ea5dd98418f06dc585d2891d228cf0"}, - {file = "mypy-0.931-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:50c7346a46dc76a4ed88f3277d4959de8a2bd0a0fa47fa87a4cde36fe247ac05"}, - {file = "mypy-0.931-cp37-cp37m-win_amd64.whl", hash = "sha256:d8f1ff62f7a879c9fe5917b3f9eb93a79b78aad47b533911b853a757223f72e7"}, - {file = "mypy-0.931-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f9fe20d0872b26c4bba1c1be02c5340de1019530302cf2dcc85c7f9fc3252ae0"}, - {file = "mypy-0.931-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1b06268df7eb53a8feea99cbfff77a6e2b205e70bf31743e786678ef87ee8069"}, - {file = "mypy-0.931-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8c11003aaeaf7cc2d0f1bc101c1cc9454ec4cc9cb825aef3cafff8a5fdf4c799"}, - {file = "mypy-0.931-cp38-cp38-win_amd64.whl", hash = "sha256:d9d2b84b2007cea426e327d2483238f040c49405a6bf4074f605f0156c91a47a"}, - {file = "mypy-0.931-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ff3bf387c14c805ab1388185dd22d6b210824e164d4bb324b195ff34e322d166"}, - {file = "mypy-0.931-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5b56154f8c09427bae082b32275a21f500b24d93c88d69a5e82f3978018a0266"}, - {file = "mypy-0.931-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:8ca7f8c4b1584d63c9a0f827c37ba7a47226c19a23a753d52e5b5eddb201afcd"}, - {file = "mypy-0.931-cp39-cp39-win_amd64.whl", hash = "sha256:74f7eccbfd436abe9c352ad9fb65872cc0f1f0a868e9d9c44db0893440f0c697"}, - {file = "mypy-0.931-py3-none-any.whl", hash = "sha256:1171f2e0859cfff2d366da2c7092b06130f232c636a3f7301e3feb8b41f6377d"}, - {file = "mypy-0.931.tar.gz", hash = "sha256:0038b21890867793581e4cb0d810829f5fd4441aa75796b53033af3aa30430ce"}, + {file = "mypy-0.950-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cf9c261958a769a3bd38c3e133801ebcd284ffb734ea12d01457cb09eacf7d7b"}, + {file = "mypy-0.950-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5b5bd0ffb11b4aba2bb6d31b8643902c48f990cc92fda4e21afac658044f0c0"}, + {file = "mypy-0.950-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5e7647df0f8fc947388e6251d728189cfadb3b1e558407f93254e35abc026e22"}, + {file = "mypy-0.950-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:eaff8156016487c1af5ffa5304c3e3fd183edcb412f3e9c72db349faf3f6e0eb"}, + {file = "mypy-0.950-cp310-cp310-win_amd64.whl", hash = "sha256:563514c7dc504698fb66bb1cf897657a173a496406f1866afae73ab5b3cdb334"}, + {file = "mypy-0.950-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:dd4d670eee9610bf61c25c940e9ade2d0ed05eb44227275cce88701fee014b1f"}, + {file = "mypy-0.950-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ca75ecf2783395ca3016a5e455cb322ba26b6d33b4b413fcdedfc632e67941dc"}, + {file = "mypy-0.950-cp36-cp36m-win_amd64.whl", hash = "sha256:6003de687c13196e8a1243a5e4bcce617d79b88f83ee6625437e335d89dfebe2"}, + {file = "mypy-0.950-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4c653e4846f287051599ed8f4b3c044b80e540e88feec76b11044ddc5612ffed"}, + {file = "mypy-0.950-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e19736af56947addedce4674c0971e5dceef1b5ec7d667fe86bcd2b07f8f9075"}, + {file = "mypy-0.950-cp37-cp37m-win_amd64.whl", hash = "sha256:ef7beb2a3582eb7a9f37beaf38a28acfd801988cde688760aea9e6cc4832b10b"}, + {file = "mypy-0.950-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0112752a6ff07230f9ec2f71b0d3d4e088a910fdce454fdb6553e83ed0eced7d"}, + {file = "mypy-0.950-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ee0a36edd332ed2c5208565ae6e3a7afc0eabb53f5327e281f2ef03a6bc7687a"}, + {file = "mypy-0.950-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:77423570c04aca807508a492037abbd72b12a1fb25a385847d191cd50b2c9605"}, + {file = "mypy-0.950-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5ce6a09042b6da16d773d2110e44f169683d8cc8687e79ec6d1181a72cb028d2"}, + {file = "mypy-0.950-cp38-cp38-win_amd64.whl", hash = "sha256:5b231afd6a6e951381b9ef09a1223b1feabe13625388db48a8690f8daa9b71ff"}, + {file = "mypy-0.950-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0384d9f3af49837baa92f559d3fa673e6d2652a16550a9ee07fc08c736f5e6f8"}, + {file = "mypy-0.950-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1fdeb0a0f64f2a874a4c1f5271f06e40e1e9779bf55f9567f149466fc7a55038"}, + {file = "mypy-0.950-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:61504b9a5ae166ba5ecfed9e93357fd51aa693d3d434b582a925338a2ff57fd2"}, + {file = "mypy-0.950-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a952b8bc0ae278fc6316e6384f67bb9a396eb30aced6ad034d3a76120ebcc519"}, + {file = "mypy-0.950-cp39-cp39-win_amd64.whl", hash = "sha256:eaea21d150fb26d7b4856766e7addcf929119dd19fc832b22e71d942835201ef"}, + {file = "mypy-0.950-py3-none-any.whl", hash = "sha256:a4d9898f46446bfb6405383b57b96737dcfd0a7f25b748e78ef3e8c576bba3cb"}, + {file = "mypy-0.950.tar.gz", hash = "sha256:1b333cfbca1762ff15808a0ef4f71b5d3eed8528b23ea1c3fb50543c867d68de"}, ] mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] mypy-zope = [ - {file = "mypy-zope-0.3.5.tar.gz", hash = "sha256:489e7da1c2af887f2cfe3496995fc247f296512b495b57817edddda9d22308f3"}, - {file = "mypy_zope-0.3.5-py3-none-any.whl", hash = "sha256:3bd0cc9a3e5933b02931af4b214ba32a4f4ff98adb30c979ce733857db91a18b"}, + {file = "mypy-zope-0.3.7.tar.gz", hash = "sha256:9da171e78e8ef7ac8922c86af1a62f1b7f3244f121020bd94a2246bc3f33c605"}, + {file = "mypy_zope-0.3.7-py3-none-any.whl", hash = "sha256:9c7637d066e4d1bafa0651abc091c752009769098043b236446e6725be2bc9c2"}, ] netaddr = [ {file = "netaddr-0.8.0-py2.py3-none-any.whl", hash = "sha256:9666d0232c32d2656e5e5f8d735f58fd6c7457ce52fc21c98d45f2af78f990ac"}, diff --git a/pyproject.toml b/pyproject.toml index 446895711b01..877f19708d37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -248,8 +248,8 @@ flake8-bugbear = "==21.3.2" flake8 = "*" # Typechecking -mypy = "==0.931" -mypy-zope = "==0.3.5" +mypy = "*" +mypy-zope = "*" types-bleach = ">=4.1.0" types-commonmark = ">=0.9.2" types-jsonschema = ">=3.2.0" From bfd491310a0576d403c595425ca7585d80a540ab Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 5 May 2022 23:04:12 +0100 Subject: [PATCH 02/16] Ignore false positive in synapse.handler.message Not really fair to call this a false positive since our annotation is incorrect, but screw it. --- synapse/handlers/message.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 95a89ac01f4b..c073b4e41ce3 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1455,7 +1455,10 @@ async def persist_and_notify_client_event( # If the old version of alt_aliases is of an unknown form, # completely replace it. if not isinstance(original_alt_aliases, (list, tuple)): - original_alt_aliases = [] + # type-ignore: although original_alt_aliases is defined as List[str], + # it may be set to `alt_aliases` from `original_event`, i.e. to any + # JSON value. Therefore this is not unreachable. + original_alt_aliases = [] # type: ignore[unreachable] # Check that each alias is currently valid. new_alt_aliases = set(alt_aliases) - set(original_alt_aliases) From fb11e63e8cd1e936e44c2d77b028a1f42bcebfe5 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 5 May 2022 23:06:27 +0100 Subject: [PATCH 03/16] Fix false positive in synapse.appservice.api This function is passed an arbitrary JSON value, not a JSONDict. --- synapse/appservice/api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index adc6b074dae6..d19f8dd996b2 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple from prometheus_client import Counter +from typing_extensions import TypeGuard from synapse.api.constants import EventTypes, Membership, ThirdPartyEntityKind from synapse.api.errors import CodeMessageException @@ -66,7 +67,7 @@ def _is_valid_3pe_metadata(info: JsonDict) -> bool: return True -def _is_valid_3pe_result(r: JsonDict, field: str) -> bool: +def _is_valid_3pe_result(r: object, field: str) -> TypeGuard[JsonDict]: if not isinstance(r, dict): return False From fd5b09beee7f69fbafa2e79ec626d28837f41a90 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 5 May 2022 23:16:27 +0100 Subject: [PATCH 04/16] Ignore false positive in synapse.config.appservice --- synapse/config/appservice.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index 720b90a28386..99b7f584a869 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -56,7 +56,8 @@ def load_appservices( ) -> List[ApplicationService]: """Returns a list of Application Services from the config files.""" if not isinstance(config_files, list): - logger.warning("Expected %s to be a list of AS config files.", config_files) + # type-ignore: this function gets arbitrary json value; we do use this path. + logger.warning("Expected %s to be a list of AS config files.", config_files) # type: ignore[unreachable] return [] # Dicts of value -> filename From b5228cab0a839110d43025039cfbadc771d39722 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 6 May 2022 00:20:07 +0100 Subject: [PATCH 05/16] Ignore false positive in synapse.events.presence_router --- synapse/events/presence_router.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/synapse/events/presence_router.py b/synapse/events/presence_router.py index a58f313e8b1c..b6ea5178cea9 100644 --- a/synapse/events/presence_router.py +++ b/synapse/events/presence_router.py @@ -153,7 +153,9 @@ async def get_users_for_states( continue if not isinstance(result, Dict): - logger.warning( + # Type ignore: we don't trust modules to return the types that + # they're supposed to. + logger.warning( # type: ignore[unreachable] "Wrong type returned by module API callback %s: %s, expected Dict", callback, result, @@ -162,7 +164,9 @@ async def get_users_for_states( for key, new_entries in result.items(): if not isinstance(new_entries, Set): - logger.warning( + # Type ignore: we don't trust modules to return the types that + # they're supposed to. + logger.warning( # type: ignore[unreachable] "Wrong type returned by module API callback %s: %s, expected Set", callback, new_entries, From 8459efa6e8d4b2b5623265fb87fe2c23ae9cc3a4 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 6 May 2022 00:49:33 +0100 Subject: [PATCH 06/16] Try to fix complaints about unawaited coroutines --- synapse/metrics/background_process_metrics.py | 47 +++++++++++++++---- 1 file changed, 37 insertions(+), 10 deletions(-) diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index f61396bb79a9..e9c24938695f 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -28,11 +28,11 @@ Type, TypeVar, Union, - cast, ) from prometheus_client import Metric from prometheus_client.core import REGISTRY, Counter, Gauge +from typing_extensions import ParamSpec from twisted.internet import defer @@ -256,24 +256,51 @@ async def run() -> Optional[R]: return defer.ensureDeferred(run()) -F = TypeVar("F", bound=Callable[..., Awaitable[Optional[Any]]]) +P = ParamSpec("P") -def wrap_as_background_process(desc: str) -> Callable[[F], F]: - """Decorator that wraps a function that gets called as a background - process. +def wrap_as_background_process( + desc: str, +) -> Callable[ + [Callable[P, Awaitable[Optional[R]]]], + Callable[P, "defer.Deferred[Optional[R]]"], +]: + """Decorator that wraps an asynchronous function `func`, returning a synchronous + decorated function. Calling the decorated version runs `func` as a background + process, forwarding all arguments verbatim. - Equivalent to calling the function with `run_as_background_process` - """ + That is, + + @wrap_as_background_process + def func(*args): ... + func(1, 2, third=3) - def wrap_as_background_process_inner(func: F) -> F: + is equivalent to: + + def func(*args): ... + run_as_background_process(func, 1, 2, third=3) + + The former can be convenient if `func` needs to be run as a background process in + multiple places. + """ + # Note: the decorated version of `func` (`wrap_as_background_process_inner`) could be + # considered an asynchronous function. However, recent mypy versions warn us that + # we have forgotten to `await` a coroutine if we mark the decorated function as + # returning an Awaitable. + # + # error: Value of type "Coroutine[Any, Any, None]" must be used [unused-coroutine] + # note: Are you missing an await? + # It seems happier when the decorated version of `func` returns a Deferred. + def wrap_as_background_process_inner( + func: Callable[P, Awaitable[Optional[R]]] + ) -> Callable[P, "defer.Deferred[Optional[R]]"]: @wraps(func) def wrap_as_background_process_inner_2( - *args: Any, **kwargs: Any + *args: P.args, **kwargs: P.kwargs ) -> "defer.Deferred[Optional[R]]": return run_as_background_process(desc, func, *args, **kwargs) - return cast(F, wrap_as_background_process_inner_2) + return wrap_as_background_process_inner_2 return wrap_as_background_process_inner From 227e4cb0c4f6f8038c80892dd47a3d055daf63cf Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 6 May 2022 01:02:41 +0100 Subject: [PATCH 07/16] Fix tests.storage.test_monthly_active_users Before the recent changes, both expressions assigned to D were coroutines. Now the second expression is a synchronous function returning a deferred. ``` tests/storage/test_monthly_active_users.py:237: error: Incompatible types in assignment (expression has type "Deferred[None]", variable has type "Coroutine[Any, Any, None]") [assignment] ``` --- tests/storage/test_monthly_active_users.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py index 0fbf46567091..18449fd84e88 100644 --- a/tests/storage/test_monthly_active_users.py +++ b/tests/storage/test_monthly_active_users.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, List +from typing import Any, Awaitable, Dict, List from unittest.mock import Mock from twisted.test.proto_helpers import MemoryReactor @@ -222,7 +222,7 @@ def test_reap_monthly_active_users_reserved_users(self): self.store.user_add_threepid(user, "email", email, now, now) ) - d = self.store.db_pool.runInteraction( + d: Awaitable[None] = self.store.db_pool.runInteraction( "initialise", self.store._initialise_reserved_users, threepids ) self.get_success(d) From e1993b4974200e8d89de097c7750a306202de7bf Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 6 May 2022 01:23:00 +0100 Subject: [PATCH 08/16] Fix sorteddict stubs Mypy now includes https://github.com/python/typeshed/pull/6653 which triggered ``` stubs/sortedcontainers/sorteddict.pyi:88: error: Signature of "update" incompatible with supertype "MutableMapping" [override] stubs/sortedcontainers/sorteddict.pyi:88: note: Superclass: stubs/sortedcontainers/sorteddict.pyi:88: note: @overload stubs/sortedcontainers/sorteddict.pyi:88: note: def update(self, SupportsKeysAndGetItem[_KT, _VT], **kwargs: _VT) -> None stubs/sortedcontainers/sorteddict.pyi:88: note: @overload stubs/sortedcontainers/sorteddict.pyi:88: note: def update(self, Iterable[Tuple[_KT, _VT]], **kwargs: _VT) -> None stubs/sortedcontainers/sorteddict.pyi:88: note: @overload stubs/sortedcontainers/sorteddict.pyi:88: note: def update(self, **kwargs: _VT) -> None stubs/sortedcontainers/sorteddict.pyi:88: note: Subclass: stubs/sortedcontainers/sorteddict.pyi:88: note: @overload stubs/sortedcontainers/sorteddict.pyi:88: note: def update(self, Mapping[_KT, _VT], **kwargs: _VT) -> None stubs/sortedcontainers/sorteddict.pyi:88: note: @overload stubs/sortedcontainers/sorteddict.pyi:88: note: def update(self, Iterable[Tuple[_KT, _VT]], **kwargs: _VT) -> None stubs/sortedcontainers/sorteddict.pyi:88: note: @overload stubs/sortedcontainers/sorteddict.pyi:88: note: def update(self, **kwargs: _VT) -> None ``` --- stubs/sortedcontainers/sorteddict.pyi | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/stubs/sortedcontainers/sorteddict.pyi b/stubs/sortedcontainers/sorteddict.pyi index 3a4f9c307685..7c399ab38d5e 100644 --- a/stubs/sortedcontainers/sorteddict.pyi +++ b/stubs/sortedcontainers/sorteddict.pyi @@ -85,12 +85,19 @@ class SortedDict(Dict[_KT, _VT]): def popitem(self, index: int = ...) -> Tuple[_KT, _VT]: ... def peekitem(self, index: int = ...) -> Tuple[_KT, _VT]: ... def setdefault(self, key: _KT, default: Optional[_VT] = ...) -> _VT: ... - @overload - def update(self, __map: Mapping[_KT, _VT], **kwargs: _VT) -> None: ... - @overload - def update(self, __iterable: Iterable[Tuple[_KT, _VT]], **kwargs: _VT) -> None: ... - @overload - def update(self, **kwargs: _VT) -> None: ... + # Mypy now reports the first overload as an error, because typeshed widened the type + # of `__map` to its internal `_typeshed.SupportsKeysAndGetItem` type in + # https://github.com/python/typeshed/pull/6653 + # Since sorteddicts don't change the signature of `update` from that of `dict`, we + # let the stubs for `update` inherit from the stubs for `dict`. (I suspect we could + # do the same for many othe methods.) We leave the stubs commented to better track + # how this file has evolved from the original stubs. + # @overload + # def update(self, __map: Mapping[_KT, _VT], **kwargs: _VT) -> None: ... + # @overload + # def update(self, __iterable: Iterable[Tuple[_KT, _VT]], **kwargs: _VT) -> None: ... + # @overload + # def update(self, **kwargs: _VT) -> None: ... def __reduce__( self, ) -> Tuple[ From 7dade80a2de398769eaee4f0e73425703b34f327 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 6 May 2022 01:34:34 +0100 Subject: [PATCH 09/16] Suppress false positive when calling run_as_background_process --- synapse/metrics/background_process_metrics.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index e9c24938695f..368e3eef3bad 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -298,7 +298,11 @@ def wrap_as_background_process_inner( def wrap_as_background_process_inner_2( *args: P.args, **kwargs: P.kwargs ) -> "defer.Deferred[Optional[R]]": - return run_as_background_process(desc, func, *args, **kwargs) + # type-ignore: mypy is confusing kwargs with the bg_start_span kwarg. + # Argument 4 to "run_as_background_process" has incompatible type + # "**P.kwargs"; expected "bool" + # See https://github.com/python/mypy/issues/8862 + return run_as_background_process(desc, func, *args, **kwargs) # type: ignore[arg-type] return wrap_as_background_process_inner_2 From 36d74ebbd6cdae7e5af180761fcb6dd90dd384c0 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 6 May 2022 01:37:04 +0100 Subject: [PATCH 10/16] Update sentry-sdk This suppresses the complaint ``` synapse/app/_base.py:491: error: Cannot instantiate abstract class "init" with abstract attribute "__exit__" [abstract] ``` by including the change in https://github.com/getsentry/sentry-python/p I can't see anything obviously terrible in the changelogs; see https://github.com/getsentry/sentry-python/releases Additionally, we don't upper bound sentry-sdk, so fresh installations might be using this version anyway. --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index 64b82d27c4a9..564ba7ec02c4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1021,7 +1021,7 @@ jeepney = ">=0.6" [[package]] name = "sentry-sdk" -version = "1.5.7" +version = "1.5.11" description = "Python client for Sentry (https://sentry.io)" category = "main" optional = true @@ -2390,8 +2390,8 @@ secretstorage = [ {file = "SecretStorage-3.3.1.tar.gz", hash = "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195"}, ] sentry-sdk = [ - {file = "sentry-sdk-1.5.7.tar.gz", hash = "sha256:aa52da941c56b5a76fd838f8e9e92a850bf893a9eb1e33ffce6c21431d07ee30"}, - {file = "sentry_sdk-1.5.7-py2.py3-none-any.whl", hash = "sha256:411a8495bd18cf13038e5749e4710beb4efa53da6351f67b4c2f307c2d9b6d49"}, + {file = "sentry-sdk-1.5.11.tar.gz", hash = "sha256:6c01d9d0b65935fd275adc120194737d1df317dce811e642cbf0394d0d37a007"}, + {file = "sentry_sdk-1.5.11-py2.py3-none-any.whl", hash = "sha256:c17179183cac614e900cbd048dab03f49a48e2820182ec686c25e7ce46f8548f"}, ] service-identity = [ {file = "service-identity-21.1.0.tar.gz", hash = "sha256:6e6c6086ca271dc11b033d17c3a8bea9f24ebff920c587da090afc9519419d34"}, From b99d5ab2635688f813d7ff76712a18e9fa75b22a Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 6 May 2022 01:42:18 +0100 Subject: [PATCH 11/16] Changelog innit --- changelog.d/12650.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/12650.misc diff --git a/changelog.d/12650.misc b/changelog.d/12650.misc new file mode 100644 index 000000000000..07bb4ce5a91a --- /dev/null +++ b/changelog.d/12650.misc @@ -0,0 +1 @@ +Update to mypy 0.950. \ No newline at end of file From a58d9738f4831d5b01dffd9a5a587b35987d8acf Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 6 May 2022 12:16:47 +0100 Subject: [PATCH 12/16] Use `result: object` to make ignores unnecessary --- synapse/events/presence_router.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/synapse/events/presence_router.py b/synapse/events/presence_router.py index b6ea5178cea9..98555c8c0c65 100644 --- a/synapse/events/presence_router.py +++ b/synapse/events/presence_router.py @@ -147,15 +147,15 @@ async def get_users_for_states( # run all the callbacks for get_users_for_states and combine the results for callback in self._get_users_for_states_callbacks: try: - result = await callback(state_updates) + # Note: result is an object here, because we don't trust modules to + # return the types they're supposed to. + result: object = await callback(state_updates) except Exception as e: logger.warning("Failed to run module API callback %s: %s", callback, e) continue if not isinstance(result, Dict): - # Type ignore: we don't trust modules to return the types that - # they're supposed to. - logger.warning( # type: ignore[unreachable] + logger.warning( "Wrong type returned by module API callback %s: %s, expected Dict", callback, result, @@ -164,9 +164,7 @@ async def get_users_for_states( for key, new_entries in result.items(): if not isinstance(new_entries, Set): - # Type ignore: we don't trust modules to return the types that - # they're supposed to. - logger.warning( # type: ignore[unreachable] + logger.warning( "Wrong type returned by module API callback %s: %s, expected Set", callback, new_entries, From 7c4f6dc7b62a0026eb9943f7ac9ad2e5d14f6d7e Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 6 May 2022 12:31:10 +0100 Subject: [PATCH 13/16] Inline calls to `get_success` --- tests/storage/test_monthly_active_users.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py index 18449fd84e88..c951b481f37e 100644 --- a/tests/storage/test_monthly_active_users.py +++ b/tests/storage/test_monthly_active_users.py @@ -222,10 +222,11 @@ def test_reap_monthly_active_users_reserved_users(self): self.store.user_add_threepid(user, "email", email, now, now) ) - d: Awaitable[None] = self.store.db_pool.runInteraction( - "initialise", self.store._initialise_reserved_users, threepids + self.get_success( + self.store.db_pool.runInteraction( + "initialise", self.store._initialise_reserved_users, threepids + ) ) - self.get_success(d) count = self.get_success(self.store.get_monthly_active_count()) self.assertEqual(count, initial_users) @@ -233,8 +234,7 @@ def test_reap_monthly_active_users_reserved_users(self): users = self.get_success(self.store.get_registered_reserved_users()) self.assertEqual(len(users), reserved_user_number) - d = self.store.reap_monthly_active_users() - self.get_success(d) + self.get_success(self.store.reap_monthly_active_users()) count = self.get_success(self.store.get_monthly_active_count()) self.assertEqual(count, self.hs.config.server.max_mau_value) From 88fe871a28658acf31369f1233c30fbdc28a5011 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 6 May 2022 13:00:06 +0100 Subject: [PATCH 14/16] Remove comment that is confused about life --- synapse/metrics/background_process_metrics.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index 368e3eef3bad..7d65ff3a4804 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -283,14 +283,6 @@ def func(*args): ... The former can be convenient if `func` needs to be run as a background process in multiple places. """ - # Note: the decorated version of `func` (`wrap_as_background_process_inner`) could be - # considered an asynchronous function. However, recent mypy versions warn us that - # we have forgotten to `await` a coroutine if we mark the decorated function as - # returning an Awaitable. - # - # error: Value of type "Coroutine[Any, Any, None]" must be used [unused-coroutine] - # note: Are you missing an await? - # It seems happier when the decorated version of `func` returns a Deferred. def wrap_as_background_process_inner( func: Callable[P, Awaitable[Optional[R]]] ) -> Callable[P, "defer.Deferred[Optional[R]]"]: From 25571991380cb0422a1584f033e096111ece8b82 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 6 May 2022 13:02:30 +0100 Subject: [PATCH 15/16] Workaround typeignore in synapse.handlers.message grrr at not validating. Y'all need more static typing in your lives. --- synapse/handlers/message.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index c073b4e41ce3..c28b792e6fe2 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1427,7 +1427,7 @@ async def persist_and_notify_client_event( # Validate a newly added alias or newly added alt_aliases. original_alias = None - original_alt_aliases: List[str] = [] + original_alt_aliases: object = [] original_event_id = event.unsigned.get("replaces_state") if original_event_id: @@ -1455,10 +1455,8 @@ async def persist_and_notify_client_event( # If the old version of alt_aliases is of an unknown form, # completely replace it. if not isinstance(original_alt_aliases, (list, tuple)): - # type-ignore: although original_alt_aliases is defined as List[str], - # it may be set to `alt_aliases` from `original_event`, i.e. to any - # JSON value. Therefore this is not unreachable. - original_alt_aliases = [] # type: ignore[unreachable] + # TODO: check that the original_alt_aliases' entries are all strings + original_alt_aliases = [] # Check that each alias is currently valid. new_alt_aliases = set(alt_aliases) - set(original_alt_aliases) From bf59faea05e1bd4b041c5b0bdc6d21853dc5e257 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 6 May 2022 13:08:15 +0100 Subject: [PATCH 16/16] Appease linter --- synapse/metrics/background_process_metrics.py | 1 + tests/storage/test_monthly_active_users.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/metrics/background_process_metrics.py b/synapse/metrics/background_process_metrics.py index 7d65ff3a4804..298809742a94 100644 --- a/synapse/metrics/background_process_metrics.py +++ b/synapse/metrics/background_process_metrics.py @@ -283,6 +283,7 @@ def func(*args): ... The former can be convenient if `func` needs to be run as a background process in multiple places. """ + def wrap_as_background_process_inner( func: Callable[P, Awaitable[Optional[R]]] ) -> Callable[P, "defer.Deferred[Optional[R]]"]: diff --git a/tests/storage/test_monthly_active_users.py b/tests/storage/test_monthly_active_users.py index c951b481f37e..4c29ad79b643 100644 --- a/tests/storage/test_monthly_active_users.py +++ b/tests/storage/test_monthly_active_users.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Awaitable, Dict, List +from typing import Any, Dict, List from unittest.mock import Mock from twisted.test.proto_helpers import MemoryReactor