From e71dc6a84f6f334aaea44f1738346014d646f43c Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Sun, 2 Oct 2022 08:56:09 +0200 Subject: [PATCH] Enable strict mode for `mypy` + other error codes (#149) * chore(mypy): enable strict mode and more error codes * chore: comply with `mypy` new rules * chore: ignore some type errors related to `importlib_metadata` --- deptry/cli.py | 8 +++---- deptry/compat.py | 2 ++ deptry/dependency.py | 6 ++--- deptry/dependency_getter/__init__.py | 0 deptry/dependency_getter/pyproject_toml.py | 6 ++--- deptry/dependency_getter/requirements_txt.py | 4 ++-- deptry/import_parser.py | 8 +++---- deptry/issue_finders/__init__.py | 0 deptry/issue_finders/transitive.py | 8 ++++--- deptry/json_writer.py | 4 ++-- deptry/module.py | 3 ++- deptry/notebook_import_extractor.py | 12 +++++----- deptry/stdlibs/__init__.py | 0 deptry/utils.py | 4 ++-- pyproject.toml | 24 +++++++++++++++----- 15 files changed, 53 insertions(+), 36 deletions(-) create mode 100644 deptry/dependency_getter/__init__.py create mode 100644 deptry/issue_finders/__init__.py create mode 100644 deptry/stdlibs/__init__.py diff --git a/deptry/cli.py b/deptry/cli.py index 227df3a3..f12959ad 100644 --- a/deptry/cli.py +++ b/deptry/cli.py @@ -1,6 +1,6 @@ import logging -import pathlib import sys +from pathlib import Path from typing import List, Optional, Tuple, Union import click @@ -37,7 +37,7 @@ def configure_logger(ctx: click.Context, _param: click.Parameter, value: bool) - @click.command() -@click.argument("root", type=click.Path(exists=True), required=False) +@click.argument("root", type=click.Path(exists=True, path_type=Path), required=False) @click.option( "--verbose", "-v", @@ -178,7 +178,7 @@ def configure_logger(ctx: click.Context, _param: click.Parameter, value: bool) - hidden=True, ) def deptry( - root: pathlib.Path, + root: Optional[Path], verbose: bool, ignore_obsolete: Tuple[str, ...], ignore_missing: Tuple[str, ...], @@ -232,4 +232,4 @@ def deptry( def display_deptry_version() -> None: - logging.info(f'deptry {metadata.version("deptry")}') + logging.info(f'deptry {metadata.version("deptry")}') # type: ignore[no-untyped-call] diff --git a/deptry/compat.py b/deptry/compat.py index 6ec17f04..9deac8b2 100644 --- a/deptry/compat.py +++ b/deptry/compat.py @@ -6,3 +6,5 @@ else: import importlib_metadata as metadata # noqa: F401 from importlib_metadata import PackageNotFoundError # noqa: F401 + +__all__ = ("metadata", "PackageNotFoundError") diff --git a/deptry/dependency.py b/deptry/dependency.py index ef9cfa34..a1ec76b6 100644 --- a/deptry/dependency.py +++ b/deptry/dependency.py @@ -44,7 +44,7 @@ def __str__(self) -> str: def find_metadata(self, name: str) -> bool: try: - metadata.distribution(name) + metadata.distribution(name) # type: ignore[no-untyped-call] return True except PackageNotFoundError: logging.warning( @@ -80,7 +80,7 @@ def _get_top_level_module_names_from_top_level_txt(self) -> List[str]: This function extracts these names, if a top-level.txt file exists. """ - metadata_top_levels = metadata.distribution(self.name).read_text("top_level.txt") + metadata_top_levels = metadata.distribution(self.name).read_text("top_level.txt") # type: ignore[no-untyped-call] if metadata_top_levels: return [x for x in metadata_top_levels.split("\n") if len(x) > 0] else: @@ -101,7 +101,7 @@ def _get_top_level_module_names_from_record_file(self) -> List[str]: """ top_levels = [] try: - metadata_records = metadata.distribution(self.name).read_text("RECORD") + metadata_records = metadata.distribution(self.name).read_text("RECORD") # type: ignore[no-untyped-call] if not metadata_records: return [] diff --git a/deptry/dependency_getter/__init__.py b/deptry/dependency_getter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/deptry/dependency_getter/pyproject_toml.py b/deptry/dependency_getter/pyproject_toml.py index 6d74b7f8..efa936f3 100644 --- a/deptry/dependency_getter/pyproject_toml.py +++ b/deptry/dependency_getter/pyproject_toml.py @@ -36,7 +36,7 @@ def get(self) -> List[Dependency]: def _get_pyproject_toml_dependencies(self) -> Dict[str, Any]: pyproject_data = load_pyproject_toml() - dependencies = pyproject_data["tool"]["poetry"]["dependencies"] + dependencies: Dict[str, Any] = pyproject_data["tool"]["poetry"]["dependencies"] return dependencies def _get_pyproject_toml_dev_dependencies(self) -> Dict[str, Any]: @@ -66,14 +66,14 @@ def _log_dependencies(self, dependencies: List[Dependency]) -> None: logging.debug("") @staticmethod - def _is_optional(dep: str, spec: Union[str, dict]) -> bool: + def _is_optional(dep: str, spec: Union[str, Dict[str, Any]]) -> bool: # if of the shape `isodate = {version = "*", optional = true}` mark as optional` if isinstance(spec, dict) and "optional" in spec and spec["optional"]: return True return False @staticmethod - def _is_conditional(dep: str, spec: Union[str, dict]) -> bool: + def _is_conditional(dep: str, spec: Union[str, Dict[str, Any]]) -> bool: # if of the shape `tomli = { version = "^2.0.1", python = "<3.11" }`, mark as conditional. if isinstance(spec, dict) and "python" in spec and "version" in spec: return True diff --git a/deptry/dependency_getter/requirements_txt.py b/deptry/dependency_getter/requirements_txt.py index d1a54ff3..6b045249 100644 --- a/deptry/dependency_getter/requirements_txt.py +++ b/deptry/dependency_getter/requirements_txt.py @@ -2,7 +2,7 @@ import logging import os import re -from typing import List, Optional, Tuple +from typing import List, Match, Optional, Tuple from deptry.dependency import Dependency @@ -117,7 +117,7 @@ def _log_dependencies(self, dependencies: List[Dependency]) -> None: logging.debug("") @staticmethod - def _line_is_url(line: str) -> Optional[re.Match]: + def _line_is_url(line: str) -> Optional[Match[str]]: return re.search("^(http|https|git\+https)", line) @staticmethod diff --git a/deptry/import_parser.py b/deptry/import_parser.py index 58b58ab3..1d218294 100644 --- a/deptry/import_parser.py +++ b/deptry/import_parser.py @@ -49,7 +49,7 @@ def get_imported_modules_from_str(self, file_str: str) -> List[str]: def _get_imported_modules_from_py(self, path_to_py_file: Path) -> List[str]: try: with open(path_to_py_file) as f: - root = ast.parse(f.read(), path_to_py_file) # type: ignore + root = ast.parse(f.read(), path_to_py_file) # type: ignore[call-overload] import_nodes = self._get_import_nodes_from(root) return self._get_import_modules_from(import_nodes) except UnicodeDecodeError: @@ -58,7 +58,7 @@ def _get_imported_modules_from_py(self, path_to_py_file: Path) -> List[str]: def _get_imported_modules_from_py_and_guess_encoding(self, path_to_py_file: Path) -> List[str]: try: with open(path_to_py_file, encoding=self._get_file_encoding(path_to_py_file)) as f: - root = ast.parse(f.read(), path_to_py_file) # type: ignore + root = ast.parse(f.read(), path_to_py_file) # type: ignore[call-overload] import_nodes = self._get_import_nodes_from(root) return self._get_import_modules_from(import_nodes) except UnicodeDecodeError: @@ -95,12 +95,12 @@ def _get_import_modules_from(nodes: List[Union[ast.Import, ast.ImportFrom]]) -> if isinstance(node, ast.Import): modules += [x.name.split(".")[0] for x in node.names] # nodes for imports like `from . import foo` do not have a module attribute. - elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0: + elif node.module and node.level == 0: modules.append(node.module.split(".")[0]) return modules @staticmethod - def _flatten_list(modules_per_file: List[List]) -> List: + def _flatten_list(modules_per_file: List[List[str]]) -> List[str]: all_modules = [] for modules in modules_per_file: if modules: diff --git a/deptry/issue_finders/__init__.py b/deptry/issue_finders/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/deptry/issue_finders/transitive.py b/deptry/issue_finders/transitive.py index a53c3af1..f645597b 100644 --- a/deptry/issue_finders/transitive.py +++ b/deptry/issue_finders/transitive.py @@ -1,5 +1,5 @@ import logging -from typing import List, Optional, Tuple +from typing import List, Tuple, cast from deptry.dependency import Dependency from deptry.module import Module @@ -24,13 +24,15 @@ def __init__( self.dependencies = dependencies self.ignore_transitive = ignore_transitive - def find(self) -> List[Optional[str]]: + def find(self) -> List[str]: logging.debug("\nScanning for transitive dependencies...") transitive_dependencies = [] for module in self.imported_modules: logging.debug(f"Scanning module {module.name}...") if self._is_transitive(module): - transitive_dependencies.append(module.package) + # `self._is_transitive` only returns `True` if the package is not None. + module_package = cast(str, module.package) + transitive_dependencies.append(module_package) return transitive_dependencies def _is_transitive(self, module: Module) -> bool: diff --git a/deptry/json_writer.py b/deptry/json_writer.py index 021898ba..143413c8 100644 --- a/deptry/json_writer.py +++ b/deptry/json_writer.py @@ -1,5 +1,5 @@ import json -from typing import Dict +from typing import Dict, List class JsonWriter: @@ -13,6 +13,6 @@ class JsonWriter: def __init__(self, json_output: str) -> None: self.json_output = json_output - def write(self, issues: Dict) -> None: + def write(self, issues: Dict[str, List[str]]) -> None: with open(self.json_output, "w", encoding="utf-8") as f: json.dump(issues, f, ensure_ascii=False, indent=4) diff --git a/deptry/module.py b/deptry/module.py index 99606702..29cb6e16 100644 --- a/deptry/module.py +++ b/deptry/module.py @@ -95,7 +95,8 @@ def _get_package_name_from_metadata(self) -> Optional[str]: Most packages simply have a field called "Name" in their metadata. This method extracts that field. """ try: - return metadata.metadata(self.name)["Name"] + name: str = metadata.metadata(self.name)["Name"] # type: ignore[no-untyped-call] + return name except PackageNotFoundError: return None diff --git a/deptry/notebook_import_extractor.py b/deptry/notebook_import_extractor.py index 3fa712ec..de605e3f 100644 --- a/deptry/notebook_import_extractor.py +++ b/deptry/notebook_import_extractor.py @@ -1,7 +1,7 @@ import json import re from pathlib import Path -from typing import List +from typing import Any, Dict, List class NotebookImportExtractor: @@ -26,22 +26,22 @@ def extract(self, path_to_ipynb: Path) -> List[str]: return self._flatten(import_statements) @staticmethod - def _read_ipynb_file(path_to_ipynb: Path) -> dict: + def _read_ipynb_file(path_to_ipynb: Path) -> Dict[str, Any]: with open(path_to_ipynb, "r") as f: - notebook = json.load(f) + notebook: Dict[str, Any] = json.load(f) return notebook @staticmethod - def _keep_code_cells(notebook: dict) -> List[dict]: + def _keep_code_cells(notebook: Dict[str, Any]) -> List[Dict[str, Any]]: return [cell for cell in notebook["cells"] if cell["cell_type"] == "code"] @staticmethod def _contains_import_statements(line: str) -> bool: return re.search(r"^(?:from\s+(\w+)(?:\.\w+)?\s+)?import\s+([^\s,.]+)(?:\.\w+)?", line) is not None - def _extract_import_statements_from_cell(self, cell: dict) -> List[str]: + def _extract_import_statements_from_cell(self, cell: Dict[str, Any]) -> List[str]: return [line for line in cell["source"] if self._contains_import_statements(line)] @staticmethod - def _flatten(list_of_lists: List[List]) -> List: + def _flatten(list_of_lists: List[List[str]]) -> List[str]: return [item for sublist in list_of_lists for item in sublist] diff --git a/deptry/stdlibs/__init__.py b/deptry/stdlibs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/deptry/utils.py b/deptry/utils.py index 7a7eb883..6000d300 100644 --- a/deptry/utils.py +++ b/deptry/utils.py @@ -2,7 +2,7 @@ import sys from contextlib import contextmanager from pathlib import Path -from typing import Dict, Generator +from typing import Any, Dict, Generator if sys.version_info >= (3, 11): import tomllib @@ -33,7 +33,7 @@ def run_within_dir(path: Path) -> Generator[None, None, None]: os.chdir(oldpwd) -def load_pyproject_toml(pyproject_toml_path: str = PYPROJECT_TOML_PATH) -> Dict: +def load_pyproject_toml(pyproject_toml_path: str = PYPROJECT_TOML_PATH) -> Dict[str, Any]: try: with Path(pyproject_toml_path).open("rb") as pyproject_file: return tomllib.load(pyproject_file) diff --git a/pyproject.toml b/pyproject.toml index 47a1441e..003586ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,14 +48,26 @@ profile = "black" [tool.mypy] files = ["deptry", "scripts"] -disallow_untyped_defs = true disallow_any_unimported = true -no_implicit_optional = true -check_untyped_defs = true -warn_return_any = false -warn_unused_ignores = true +enable_error_code = [ + "ignore-without-code", + "redundant-expr", + "truthy-bool", +] +strict = true +warn_unreachable = true +pretty = true show_error_codes = true -ignore_missing_imports = true + +# Ignore warnings for unused ignores because of https://github.com/python/mypy/issues/8823. +# In some Python versions, we can end up with backport modules being untyped, while they are typed on others. +[[tool.mypy.overrides]] +module = [ + "deptry.cli", + "deptry.dependency", + "deptry.module", +] +warn_unused_ignores = false [tool.deptry] exclude = [