Relax ignore-missing-imports for modules that have stubs now and update mypy

mypy.ini

@@ -192,98 +192,97 @@ disallow_untyped_defs = True
 [mypy-tests.storage.test_user_directory]
 disallow_untyped_defs = True
 
-[mypy-pymacaroons.*]
-ignore_missing_imports = True
+;; Dependencies without annotations
+;; Before ignoring a module, check to see if type stubs are available.
+;; The `typeshed` project maintains stubs here:
+;;     https://github.com/python/typeshed/tree/master/stubs
+;; and for each pacakge `foo` there's a corresponding `types-foo` package on PyPI,
+;; which we can pull in as a dev dependency by adding to `setup.py`'s
+;; `CONDITIONAL_REQUIREMENTS["mypy"]` list.
 
-[mypy-zope]
+[mypy-authlib.*]
 ignore_missing_imports = True
 
 [mypy-bcrypt]
 ignore_missing_imports = True
 
-[mypy-constantly]
-ignore_missing_imports = True
-
-[mypy-twisted.*]
+[mypy-canonicaljson]
 ignore_missing_imports = True
 
-[mypy-treq.*]
+[mypy-constantly]
 ignore_missing_imports = True
 
-[mypy-hyperlink]
+[mypy-daemonize]
 ignore_missing_imports = True
 
 [mypy-h11]
 ignore_missing_imports = True
 
-[mypy-msgpack]
-ignore_missing_imports = True
-
-[mypy-opentracing]
+[mypy-hiredis]
 ignore_missing_imports = True
 
-[mypy-OpenSSL.*]
+[mypy-hyperlink]
 ignore_missing_imports = True
 
-[mypy-netaddr]
+[mypy-ijson.*]
 ignore_missing_imports = True
 
-[mypy-saml2.*]
+[mypy-jaeger_client.*]
 ignore_missing_imports = True
 
-[mypy-canonicaljson]
+[mypy-josepy.*]
 ignore_missing_imports = True
 
-[mypy-jaeger_client.*]
+[mypy-jwt.*]
 ignore_missing_imports = True
 
-[mypy-jsonschema]
+[mypy-lxml]
 ignore_missing_imports = True
 
-[mypy-signedjson.*]
+[mypy-msgpack]
 ignore_missing_imports = True
 
-[mypy-prometheus_client.*]
+[mypy-nacl.*]
 ignore_missing_imports = True
 
-[mypy-service_identity.*]
+[mypy-netaddr]
 ignore_missing_imports = True
 
-[mypy-daemonize]
+[mypy-opentracing]
 ignore_missing_imports = True
 
-[mypy-sentry_sdk]
+[mypy-phonenumbers.*]
 ignore_missing_imports = True
 
-[mypy-PIL.*]
+[mypy-prometheus_client.*]
 ignore_missing_imports = True
 
-[mypy-lxml]
+[mypy-pymacaroons.*]
 ignore_missing_imports = True
 
-[mypy-jwt.*]
+[mypy-pympler.*]
 ignore_missing_imports = True
 
-[mypy-authlib.*]
+[mypy-rust_python_jaeger_reporter.*]
 ignore_missing_imports = True
 
-[mypy-rust_python_jaeger_reporter.*]
+[mypy-saml2.*]
 ignore_missing_imports = True
 
-[mypy-nacl.*]
+[mypy-sentry_sdk]
 ignore_missing_imports = True
 
-[mypy-hiredis]
+[mypy-service_identity.*]
 ignore_missing_imports = True
 
-[mypy-josepy.*]
+[mypy-signedjson.*]
 ignore_missing_imports = True
 
-[mypy-pympler.*]
+[mypy-treq.*]
 ignore_missing_imports = True
 
-[mypy-phonenumbers.*]
+[mypy-twisted.*]
 ignore_missing_imports = True
 
-[mypy-ijson.*]
+[mypy-zope]
 ignore_missing_imports = True

setup.py

@@ -112,7 +112,13 @@ def exec_file(path_segments):
     "pygithub==1.55",
 ]
 
-CONDITIONAL_REQUIREMENTS["mypy"] = ["mypy==0.812", "mypy-zope==0.2.13"]
+CONDITIONAL_REQUIREMENTS["mypy"] = [
+    "mypy==0.812",
+    "mypy-zope==0.2.13",
+    "types-jsonschema>=3.2.0",
+    "types-pyOpenSSL>=20.0.7",
+    "types-Pillow>=8.3.4",
+]
 
 # Dependencies which are exclusively required by unit test code. This is
 # NOT a list of all modules that are necessary to run the unit tests.

synapse/config/tls.py

@@ -172,9 +172,12 @@ def is_disk_cert_valid(self, allow_self_signed=True):
                 )
 
         # YYYYMMDDhhmmssZ -- in UTC
-        expires_on = datetime.strptime(
-            tls_certificate.get_notAfter().decode("ascii"), "%Y%m%d%H%M%SZ"
-        )
+        expiry_data = tls_certificate.get_notAfter()
+        if expiry_data is None:
+            raise ValueError(
+                "TLS Certificate has no expiry date, and this is not permitted"
+            )
+        expires_on = datetime.strptime(expiry_data.decode("ascii"), "%Y%m%d%H%M%SZ")
         now = datetime.utcnow()
         days_remaining = (expires_on - now).days
         return days_remaining

synapse/http/client.py

@@ -912,7 +912,7 @@ class InsecureInterceptableContextFactory(ssl.ContextFactory):
 
     def __init__(self):
         self._context = SSL.Context(SSL.SSLv23_METHOD)
-        self._context.set_verify(VERIFY_NONE, lambda *_: None)
+        self._context.set_verify(VERIFY_NONE, lambda *_: False)
 
     def getContext(self, hostname=None, port=None):
         return self._context

synapse/rest/media/v1/__init__.py

@@ -12,33 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import PIL.Image
+from PIL.features import check_codec
 
 # check for JPEG support.
-try:
-    PIL.Image._getdecoder("rgb", "jpeg", None)
-except OSError as e:
-    if str(e).startswith("decoder jpeg not available"):
-        raise Exception(
-            "FATAL: jpeg codec not supported. Install pillow correctly! "
-            " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&"
-            " pip install pillow --user'"
-        )
-except Exception:
-    # any other exception is fine
-    pass
+if not check_codec("jpg"):
+    raise Exception(
+        "FATAL: jpeg codec not supported. Install pillow correctly! "
+        " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&"
+        " pip install pillow --user'"
+    )
 
 
 # check for PNG support.
-try:
-    PIL.Image._getdecoder("rgb", "zip", None)
-except OSError as e:
-    if str(e).startswith("decoder zip not available"):
-        raise Exception(
-            "FATAL: zip codec not supported. Install pillow correctly! "
-            " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&"
-            " pip install pillow --user'"
-        )
-except Exception:
-    # any other exception is fine
-    pass
+if not check_codec("zlib"):
+    raise Exception(
+        "FATAL: zip codec not supported. Install pillow correctly! "
+        " 'sudo apt-get install libjpeg-dev' then 'pip uninstall pillow &&"
+        " pip install pillow --user'"
+    )

synapse/rest/media/v1/thumbnailer.py

@@ -61,9 +61,14 @@ def __init__(self, input_path: str):
         self.transpose_method = None
         try:
             # We don't use ImageOps.exif_transpose since it crashes with big EXIF
-            image_exif = self.image._getexif()
+            # Safety: Pillow seems to acknowledge that this method is
+            # "private, experimental, but generally widely used". Pillow 6
+            # includes a public getexif() method (no underscore) that we might
+            # consider using?
+            image_exif = self.image._getexif()  # type: ignore
             if image_exif is not None:
                 image_orientation = image_exif.get(EXIF_ORIENTATION_TAG)
+                assert isinstance(image_orientation, int)
                 self.transpose_method = EXIF_TRANSPOSE_MAPPINGS.get(image_orientation)
         except Exception as e:
             # A lot of parsing errors can happen when parsing EXIF
@@ -76,7 +81,10 @@ def transpose(self) -> Tuple[int, int]:
             A tuple containing the new image size in pixels as (width, height).
         """
         if self.transpose_method is not None:
-            self.image = self.image.transpose(self.transpose_method)
+            # Safety: `transpose` takes an int rather than e.g. an IntEnum.
+            # self.transpose_method is set above to be a value in
+            # EXIF_TRANSPOSE_MAPPINGS, and that only contains correct values.
+            self.image = self.image.transpose(self.transpose_method)  # type: ignore[arg-type]
             self.width, self.height = self.image.size
             self.transpose_method = None
             # We don't need EXIF any more
@@ -101,7 +109,7 @@ def aspect(self, max_width: int, max_height: int) -> Tuple[int, int]:
         else:
             return (max_height * self.width) // self.height, max_height
 
-    def _resize(self, width: int, height: int) -> Image:
+    def _resize(self, width: int, height: int) -> Image.Image:
         # 1-bit or 8-bit color palette images need converting to RGB
         # otherwise they will be scaled using nearest neighbour which
         # looks awful.
@@ -151,7 +159,7 @@ def crop(self, width: int, height: int, output_type: str) -> BytesIO:
             cropped = scaled_image.crop((crop_left, 0, crop_right, height))
         return self._encode_image(cropped, output_type)
 
-    def _encode_image(self, output_image: Image, output_type: str) -> BytesIO:
+    def _encode_image(self, output_image: Image.Image, output_type: str) -> BytesIO:
         output_bytes_io = BytesIO()
         fmt = self.FORMATS[output_type]
         if fmt == "JPEG":