Add DEP005 to detect dependencies that are in the standard library (#…

…761)


Co-authored-by: Mathieu Kniewallner <mathieu.kniewallner@gmail.com>

docs/rules-violations.md

@@ -8,6 +8,7 @@ _deptry_ checks your project against the following rules related to dependencies
 | DEP002 | Project should not contain unused dependencies               | [link](#unused-dependencies-dep002)                 |
 | DEP003 | Project should not use transitive dependencies            | [link](#transitive-dependencies-dep003)             |
 | DEP004 | Project should not use development dependencies in non-development code | [link](#misplaced-development-dependencies-dep004)  |
+| DEP005 | Project should not contain dependencies that are in the standard library    | [link](#standard-library-dependencies-dep005)       |
 
 Any of the checks can be disabled with the [`ignore`](usage.md#ignore) flag. Specific dependencies or modules can be
 ignored with the [`per-rule-ignores`](usage.md#per-rule-ignores) flag.
@@ -170,3 +171,36 @@ dependencies = [
 [tool.pdm.dev-dependencies]
 test = ["pytest==7.2.0"]
 ```
+
+## Standard library dependencies (DEP005)
+
+Dependencies that are part of the Python standard library should not be defined as dependencies in your project.
+
+### Example
+
+On a project with the following dependencies:
+
+```toml
+[project]
+dependencies = [
+    "asyncio",
+]
+```
+
+and the following `main.py` in the project:
+
+```python
+import asyncio
+
+def async_example():
+    return asyncio.run(some_coroutine())
+```
+
+_deptry_ will report `asyncio` as a standard library dependency because it is part of the standard library, yet it is defined as a dependency in the project.
+
+To fix the issue, `asyncio` should be removed from `[project.dependencies]`:
+
+```toml
+[project]
+dependencies = []
+```

python/deptry/core.py

@@ -59,29 +59,28 @@ def run(self) -> None:
 
         python_files = self._find_python_files()
         local_modules = self._get_local_modules()
-        stdlib_modules = self._get_stdlib_modules()
+        standard_library_modules = self._get_standard_library_modules()
 
         imported_modules_with_locations = [
             ModuleLocations(
                 ModuleBuilder(
                     module,
                     local_modules,
-                    stdlib_modules,
+                    standard_library_modules,
                     dependencies_extract.dependencies,
                     dependencies_extract.dev_dependencies,
                 ).build(),
                 locations,
             )
             for module, locations in get_imported_modules_from_list_of_files(python_files).items()
         ]
-        imported_modules_with_locations = [
-            module_with_locations
-            for module_with_locations in imported_modules_with_locations
-            if not module_with_locations.module.standard_library
-        ]
 
         violations = find_violations(
-            imported_modules_with_locations, dependencies_extract.dependencies, self.ignore, self.per_rule_ignores
+            imported_modules_with_locations,
+            dependencies_extract.dependencies,
+            self.ignore,
+            self.per_rule_ignores,
+            standard_library_modules,
         )
         TextReporter(violations, use_ansi=not self.no_ansi).report()
 
@@ -126,7 +125,7 @@ def _is_local_module(path: Path) -> bool:
         )
 
     @staticmethod
-    def _get_stdlib_modules() -> frozenset[str]:
+    def _get_standard_library_modules() -> frozenset[str]:
         if sys.version_info[:2] >= (3, 10):
             return sys.stdlib_module_names
 

python/deptry/module.py

@@ -64,7 +64,7 @@ def __init__(
         self,
         name: str,
         local_modules: set[str],
-        stdlib_modules: frozenset[str],
+        standard_library_modules: frozenset[str],
         dependencies: list[Dependency] | None = None,
         dev_dependencies: list[Dependency] | None = None,
     ) -> None:
@@ -74,13 +74,13 @@ def __init__(
         Args:
             name: The name of the imported module
             local_modules: The list of local modules
-            stdlib_modules: The list of Python stdlib modules
+            standard_library_modules: The list of Python stdlib modules
             dependencies: A list of the project's dependencies
             dev_dependencies: A list of the project's development dependencies
         """
         self.name = name
         self.local_modules = local_modules
-        self.stdlib_modules = stdlib_modules
+        self.standard_library_modules = standard_library_modules
         self.dependencies = dependencies or []
         self.dev_dependencies = dev_dependencies or []
 
@@ -137,7 +137,7 @@ def _get_corresponding_top_levels_from(self, dependencies: list[Dependency]) ->
         ]
 
     def _in_standard_library(self) -> bool:
-        return self.name in self.stdlib_modules
+        return self.name in self.standard_library_modules
 
     def _is_local_module(self) -> bool:
         """

python/deptry/violations/__init__.py

@@ -9,16 +9,20 @@
 from deptry.violations.dep003_transitive.violation import DEP003TransitiveDependencyViolation
 from deptry.violations.dep004_misplaced_dev.finder import DEP004MisplacedDevDependenciesFinder
 from deptry.violations.dep004_misplaced_dev.violation import DEP004MisplacedDevDependencyViolation
+from deptry.violations.dep005_standard_library.finder import DEP005StandardLibraryDependenciesFinder
+from deptry.violations.dep005_standard_library.violation import DEP005StandardLibraryDependencyViolation
 
 __all__ = (
     "DEP001MissingDependencyViolation",
     "DEP002UnusedDependencyViolation",
     "DEP003TransitiveDependencyViolation",
     "DEP004MisplacedDevDependencyViolation",
+    "DEP005StandardLibraryDependencyViolation",
     "DEP001MissingDependenciesFinder",
     "DEP002UnusedDependenciesFinder",
     "DEP003TransitiveDependenciesFinder",
     "DEP004MisplacedDevDependenciesFinder",
+    "DEP005StandardLibraryDependenciesFinder",
     "Violation",
     "ViolationsFinder",
 )

python/deptry/violations/base.py

@@ -23,12 +23,13 @@ class ViolationsFinder(ABC):
         dependencies: A list of Dependency objects representing the project's dependencies.
         ignored_modules: A tuple of module names to ignore when scanning for issues. Defaults to an
             empty tuple.
-
+        standard_library_modules: A set of modules that are part of the standard library
     """
 
     violation: ClassVar[type[Violation]]
     imported_modules_with_locations: list[ModuleLocations]
     dependencies: list[Dependency]
+    standard_library_modules: frozenset[str]
     ignored_modules: tuple[str, ...] = ()
 
     @abstractmethod

python/deptry/violations/dep001_missing/finder.py

@@ -27,6 +27,9 @@ def find(self) -> list[Violation]:
         for module_with_locations in self.imported_modules_with_locations:
             module = module_with_locations.module
 
+            if module.standard_library:
+                continue
+
             logging.debug("Scanning module %s...", module.name)
 
             if self._is_missing(module):

python/deptry/violations/dep003_transitive/finder.py

@@ -34,6 +34,9 @@ def find(self) -> list[Violation]:
         for module_with_locations in self.imported_modules_with_locations:
             module = module_with_locations.module
 
+            if module.standard_library:
+                continue
+
             logging.debug("Scanning module %s...", module.name)
 
             if self._is_transitive(module):

python/deptry/violations/dep004_misplaced_dev/finder.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import logging
-from dataclasses import dataclass
 from typing import TYPE_CHECKING
 
 from deptry.violations.base import ViolationsFinder
@@ -11,6 +10,8 @@
     from deptry.module import Module
     from deptry.violations import Violation
 
+from dataclasses import dataclass
+
 
 @dataclass
 class DEP004MisplacedDevDependenciesFinder(ViolationsFinder):
@@ -35,6 +36,9 @@ def find(self) -> list[Violation]:
         for module_with_locations in self.imported_modules_with_locations:
             module = module_with_locations.module
 
+            if module.standard_library:
+                continue
+
             logging.debug("Scanning module %s...", module.name)
             corresponding_package_name = self._get_package_name(module)
 

python/deptry/violations/dep005_standard_library/finder.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from deptry.imports.location import Location
+from deptry.violations.base import ViolationsFinder
+from deptry.violations.dep005_standard_library.violation import DEP005StandardLibraryDependencyViolation
+
+if TYPE_CHECKING:
+    from deptry.violations import Violation
+
+
+@dataclass
+class DEP005StandardLibraryDependenciesFinder(ViolationsFinder):
+    """
+    Finds dependencies that are part of the standard library but are defined as dependencies.
+    """
+
+    violation = DEP005StandardLibraryDependencyViolation
+
+    def find(self) -> list[Violation]:
+        logging.debug("\nScanning for dependencies that are part of the standard library...")
+        stdlib_violations: list[Violation] = []
+
+        for dependency in self.dependencies:
+            logging.debug("Scanning module %s...", dependency.name)
+
+            if dependency.name in self.standard_library_modules:
+                if dependency.name in self.ignored_modules:
+                    logging.debug(
+                        "Dependency '%s' found to be a dependency that is part of the standard library, but ignoring.",
+                        dependency.name,
+                    )
+                    continue
+
+                logging.debug(
+                    "Dependency '%s' marked as a dependency that is part of the standard library.", dependency.name
+                )
+                stdlib_violations.append(self.violation(dependency, Location(dependency.definition_file)))
+
+        return stdlib_violations

python/deptry/violations/dep005_standard_library/violation.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, ClassVar
+
+from deptry.violations.base import Violation
+
+if TYPE_CHECKING:
+    from deptry.dependency import Dependency
+
+
+@dataclass
+class DEP005StandardLibraryDependencyViolation(Violation):
+    error_code: ClassVar[str] = "DEP005"
+    error_template: ClassVar[str] = (
+        "'{name}' is defined as a dependency but it is included in the Python standard library."
+    )
+    issue: Dependency
+
+    def get_error_message(self) -> str:
+        return self.error_template.format(name=self.issue.name)

python/deptry/violations/finder.py

@@ -8,6 +8,7 @@
     DEP002UnusedDependenciesFinder,
     DEP003TransitiveDependenciesFinder,
     DEP004MisplacedDevDependenciesFinder,
+    DEP005StandardLibraryDependenciesFinder,
 )
 
 if TYPE_CHECKING:
@@ -23,6 +24,7 @@
     DEP002UnusedDependenciesFinder,
     DEP003TransitiveDependenciesFinder,
     DEP004MisplacedDevDependenciesFinder,
+    DEP005StandardLibraryDependenciesFinder,
 )
 
 
@@ -31,19 +33,20 @@ def find_violations(
     dependencies: list[Dependency],
     ignore: tuple[str, ...],
     per_rule_ignores: Mapping[str, tuple[str, ...]],
+    standard_library_modules: frozenset[str],
 ) -> list[Violation]:
     violations = []
 
     for violation_finder in _VIOLATIONS_FINDERS:
         if violation_finder.violation.error_code not in ignore:
             violations.extend(
                 violation_finder(
-                    imported_modules_with_locations,
-                    dependencies,
-                    per_rule_ignores.get(violation_finder.violation.error_code, ()),
+                    imported_modules_with_locations=imported_modules_with_locations,
+                    dependencies=dependencies,
+                    ignored_modules=per_rule_ignores.get(violation_finder.violation.error_code, ()),
+                    standard_library_modules=standard_library_modules,
                 ).find()
             )
-
     return _get_sorted_violations(violations)
 
 

scripts/generate_stdlibs.py

@@ -45,7 +45,7 @@ def handle_data(self, data: str) -> None:
             self.modules.append(data)
 
 
-def get_stdlib_modules_for_python_version(python_version: tuple[int, int]) -> list[str]:
+def get_standard_library_modules_for_python_version(python_version: tuple[int, int]) -> list[str]:
     with urllib.request.urlopen(  # noqa: S310
         STDLIB_MODULES_URL.format(python_version[0], python_version[1])
     ) as response:
@@ -60,9 +60,9 @@ def get_stdlib_modules_for_python_version(python_version: tuple[int, int]) -> li
     return sorted(modules)
 
 
-def get_stdlib_modules() -> dict[str, list[str]]:
+def get_standard_library_modules() -> dict[str, list[str]]:
     return {
-        f"{python_version[0]}{python_version[1]}": get_stdlib_modules_for_python_version(python_version)
+        f"{python_version[0]}{python_version[1]}": get_standard_library_modules_for_python_version(python_version)
         for python_version in PYTHON_VERSIONS
     }
 
@@ -78,10 +78,10 @@ def write_stdlibs_file(stdlib_python: dict[str, list[str]]) -> None:
                     values=[
                         ast.Call(
                             func=ast.Name(id="frozenset"),
-                            args=[ast.Set(elts=[ast.Constant(module) for module in python_stdlib_modules])],
+                            args=[ast.Set(elts=[ast.Constant(module) for module in python_standard_library_modules])],
                             keywords=[],
                         )
-                        for python_stdlib_modules in stdlib_python.values()
+                        for python_standard_library_modules in stdlib_python.values()
                     ],
                 ),
                 lineno=0,
@@ -95,4 +95,4 @@ def write_stdlibs_file(stdlib_python: dict[str, list[str]]) -> None:
 
 
 if __name__ == "__main__":
-    write_stdlibs_file(get_stdlib_modules())
+    write_stdlibs_file(get_standard_library_modules())

tests/data/pep_621_project/pyproject.toml

@@ -11,6 +11,7 @@ dependencies = [
     "click>=8.1.3",
     "requests>=2.28.1",
     "pkginfo>=1.8.3",
+    "asyncio",
 ]
 
 [project.optional-dependencies]

tests/data/pep_621_project/src/main.py

@@ -5,3 +5,4 @@
 import click
 import white as w
 from urllib3 import contrib
+import asyncio

tests/functional/cli/test_cli_pep_621.py

@@ -47,6 +47,14 @@ def test_cli_with_pep_621(pip_venv_factory: PipVenvFactory) -> None:
                 "module": "matplotlib",
                 "location": {"file": str(Path("pyproject.toml")), "line": None, "column": None},
             },
+            {
+                "error": {
+                    "code": "DEP005",
+                    "message": "'asyncio' is defined as a dependency but it is included in the Python standard library.",
+                },
+                "module": "asyncio",
+                "location": {"file": "pyproject.toml", "line": None, "column": None},
+            },
             {
                 "error": {"code": "DEP004", "message": "'black' imported but declared as a dev dependency"},
                 "module": "black",