feat: add back PEP 420 support behind feature flag (#808)

* feat(cli): add `--experimental-namespace-package` option

* test(core): more robust local modules tests

* feat(core): handle subdirectories in PEP 420 mode

* test(functional): add tests for namespaced package

* docs(usage): document `--experimental-namespace-package`

docs/usage.md

@@ -489,3 +489,28 @@ pep621_dev_dependency_groups = ["test", "docs"]
 ```shell
 deptry . --pep621-dev-dependency-groups "test,docs"
 ```
+
+#### Experimental namespace package
+
+!!! warning
+    This option is experimental and disabled by default for now, as it could degrade performance in large codebases.
+
+Enable experimental namespace package ([PEP 420](https://peps.python.org/pep-0420/)) support.
+
+When enabled, deptry will not only rely on the presence of `__init__.py` file in a directory to determine if it is a
+local Python module or not, but will consider any Python file in the directory or its subdirectories, recursively. If a
+Python file is found, then the directory will be considered as a local Python module.
+
+- Type: `bool`
+- Default: `False`
+- `pyproject.toml` option name: `experimental_namespace_package`
+- CLI option name: `--experimental-namespace-package`
+- `pyproject.toml` example:
+```toml
+[tool.deptry]
+experimental_namespace_package = true
+```
+- CLI example:
+```shell
+deptry . --experimental-namespace-package
+```

python/deptry/cli.py

@@ -245,6 +245,11 @@ def display_deptry_version(ctx: click.Context, _param: click.Parameter, value: b
     default=(),
     show_default=False,
 )
+@click.option(
+    "--experimental-namespace-package",
+    is_flag=True,
+    help="Enable experimental support for namespace package (PEP 420) when detecting local modules (https://peps.python.org/pep-0420/).",
+)
 def deptry(
     root: tuple[Path, ...],
     config: Path,
@@ -262,6 +267,7 @@ def deptry(
     json_output: str,
     package_module_name_map: MutableMapping[str, tuple[str, ...]],
     pep621_dev_dependency_groups: tuple[str, ...],
+    experimental_namespace_package: bool,
 ) -> None:
     """Find dependency issues in your Python project.
 
@@ -282,6 +288,7 @@ def deptry(
 
     if requirements_txt_dev:
         logging.warning(REQUIREMENTS_TXT_DEV_DEPRECATION_MESSAGE)
+
     Core(
         root=root,
         config=config,
@@ -299,4 +306,5 @@ def deptry(
         json_output=json_output,
         package_module_name_map=package_module_name_map,
         pep621_dev_dependency_groups=pep621_dev_dependency_groups,
+        experimental_namespace_package=experimental_namespace_package,
     ).run()

python/deptry/core.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+import os
 import sys
 from dataclasses import dataclass
 from typing import TYPE_CHECKING
@@ -40,6 +41,7 @@ class Core:
     json_output: str
     package_module_name_map: Mapping[str, tuple[str, ...]]
     pep621_dev_dependency_groups: tuple[str, ...]
+    experimental_namespace_package: bool
 
     def run(self) -> None:
         self._log_config()
@@ -116,14 +118,25 @@ def _get_local_modules(self) -> set[str]:
 
         return guessed_local_modules | set(self.known_first_party)
 
-    @staticmethod
-    def _is_local_module(path: Path) -> bool:
+    def _is_local_module(self, path: Path) -> bool:
         """Guess if a module is a local Python module."""
         return bool(
             (path.is_file() and path.name != "__init__.py" and path.suffix == ".py")
-            or (path.is_dir() and list(path.glob("*.py")))
+            or (path.is_dir() and self._directory_has_python_files(path))
         )
 
+    def _directory_has_python_files(self, path: Path) -> bool:
+        """Check if there is any Python file in the current directory. If experimental support for namespace packages
+        (PEP 420) is enabled, also search for Python files in subdirectories."""
+        if self.experimental_namespace_package:
+            for _root, _dirs, files in os.walk(path):
+                for file in files:
+                    if file.endswith(".py"):
+                        return True
+            return False
+
+        return bool(list(path.glob("*.py")))
+
     @staticmethod
     def _get_standard_library_modules() -> frozenset[str]:
         if sys.version_info[:2] >= (3, 10):

tests/data/project_using_namespace/.gitignore

@@ -0,0 +1 @@
+/build

tests/data/project_using_namespace/foo/database/bar.py

@@ -0,0 +1,7 @@
+from os import chdir, walk
+from pathlib import Path
+
+import flake8
+import white as w
+
+from foo import api

tests/data/project_using_namespace/pyproject.toml

@@ -0,0 +1,17 @@
+[project]
+# PEP 621 project metadata
+# See https://www.python.org/dev/peps/pep-0621/
+name = "foo"
+version = "1.2.3"
+requires-python = ">=3.7"
+dependencies = ["tomli==2.0.1"]
+
+[project.optional-dependencies]
+dev = ["flake8==7.1.1"]
+
+[build-system]
+requires = ["setuptools>=61.0.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.deptry]
+pep621_dev_dependency_groups = ["dev"]

tests/functional/cli/test_cli_namespace.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import uuid
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+
+from tests.functional.utils import Project
+from tests.utils import get_issues_report
+
+if TYPE_CHECKING:
+    from tests.utils import PipVenvFactory
+
+
+@pytest.mark.xdist_group(name=Project.NAMESPACE)
+def test_cli_with_namespace(pip_venv_factory: PipVenvFactory) -> None:
+    with pip_venv_factory(Project.NAMESPACE) as virtual_env:
+        issue_report = f"{uuid.uuid4()}.json"
+        result = virtual_env.run(f"deptry . --experimental-namespace-package -o {issue_report}")
+
+        assert result.returncode == 1
+        assert get_issues_report(Path(issue_report)) == [
+            {
+                "error": {"code": "DEP004", "message": "'flake8' imported but declared as a dev dependency"},
+                "module": "flake8",
+                "location": {"file": str(Path("foo/database/bar.py")), "line": 4, "column": 8},
+            },
+            {
+                "error": {"code": "DEP001", "message": "'white' imported but missing from the dependency definitions"},
+                "module": "white",
+                "location": {"file": str(Path("foo/database/bar.py")), "line": 5, "column": 8},
+            },
+            {
+                "error": {"code": "DEP002", "message": "'tomli' defined as a dependency but not used in the codebase"},
+                "module": "tomli",
+                "location": {"file": str(Path("pyproject.toml")), "line": None, "column": None},
+            },
+        ]
+
+
+@pytest.mark.xdist_group(name=Project.NAMESPACE)
+def test_cli_with_namespace_without_experimental_flag(pip_venv_factory: PipVenvFactory) -> None:
+    with pip_venv_factory(Project.NAMESPACE) as virtual_env:
+        issue_report = f"{uuid.uuid4()}.json"
+        result = virtual_env.run(f"deptry . -o {issue_report}")
+
+        assert result.returncode == 1
+        assert get_issues_report(Path(issue_report)) == [
+            {
+                "error": {"code": "DEP004", "message": "'flake8' imported but declared as a dev dependency"},
+                "module": "flake8",
+                "location": {"file": str(Path("foo/database/bar.py")), "line": 4, "column": 8},
+            },
+            {
+                "error": {"code": "DEP001", "message": "'white' imported but missing from the dependency definitions"},
+                "module": "white",
+                "location": {"file": str(Path("foo/database/bar.py")), "line": 5, "column": 8},
+            },
+            {
+                "error": {"code": "DEP003", "message": "'foo' imported but it is a transitive dependency"},
+                "module": "foo",
+                "location": {"file": str(Path("foo/database/bar.py")), "line": 7, "column": 1},
+            },
+            {
+                "error": {"code": "DEP002", "message": "'tomli' defined as a dependency but not used in the codebase"},
+                "module": "tomli",
+                "location": {"file": str(Path("pyproject.toml")), "line": None, "column": None},
+            },
+        ]

tests/functional/utils.py

@@ -11,6 +11,7 @@ class Project(str, Enum):
     FUTURE_DEPRECATED_OBSOLETE_ARGUMENT = "project_with_future_deprecated_obsolete_argument"
     GITIGNORE = "project_with_gitignore"
     MULTIPLE_SOURCE_DIRECTORIES = "project_with_multiple_source_directories"
+    NAMESPACE = "project_using_namespace"
     PDM = "project_with_pdm"
     POETRY = "project_with_poetry"
     PYPROJECT_DIFFERENT_DIRECTORY = "project_with_pyproject_different_directory"

tests/unit/deprecate/test_requirements_txt.py

@@ -25,6 +25,7 @@
     "package_module_name_map": ANY,
     "pep621_dev_dependency_groups": ANY,
     "using_default_requirements_files": ANY,
+    "experimental_namespace_package": ANY,
 }
 
 

tests/unit/test_core.py

@@ -25,30 +25,83 @@
 
 
 @pytest.mark.parametrize(
-    ("known_first_party", "root_suffix", "expected"),
+    ("known_first_party", "root_suffix", "experimental_namespace_package", "expected"),
     [
         (
             (),
             "",
-            {"module_with_init", "module_without_init", "local_file"},
+            False,
+            {
+                "local_file",
+                "module_with_init",
+                "module_without_init",
+            },
         ),
         (
             ("module_with_init", "module_without_init"),
             "",
-            {"module_with_init", "module_without_init", "local_file"},
+            False,
+            {
+                "local_file",
+                "module_with_init",
+                "module_without_init",
+            },
         ),
         (
             ("module_without_init",),
             "module_with_init",
-            {"foo", "module_without_init", "subdirectory"},
+            False,
+            {
+                "foo",
+                "module_without_init",
+                "subdirectory",
+            },
+        ),
+        (
+            (),
+            "",
+            True,
+            {
+                "local_file",
+                "module_using_namespace",
+                "module_with_init",
+                "module_without_init",
+            },
+        ),
+        (
+            ("module_with_init", "module_without_init"),
+            "",
+            True,
+            {
+                "local_file",
+                "module_using_namespace",
+                "module_with_init",
+                "module_without_init",
+            },
+        ),
+        (
+            ("module_without_init",),
+            "module_with_init",
+            True,
+            {
+                "foo",
+                "module_without_init",
+                "subdirectory",
+            },
         ),
     ],
 )
 def test__get_local_modules(
-    tmp_path: Path, known_first_party: tuple[str, ...], root_suffix: str, expected: set[str]
+    tmp_path: Path,
+    known_first_party: tuple[str, ...],
+    root_suffix: str,
+    experimental_namespace_package: bool,
+    expected: set[str],
 ) -> None:
     with run_within_dir(tmp_path):
         create_files([
+            Path("directory_without_python_files/foo.txt"),
+            Path("module_using_namespace/subdirectory_namespace/foo.py"),
             Path("module_with_init/__init__.py"),
             Path("module_with_init/foo.py"),
             Path("module_with_init/subdirectory/__init__.py"),
@@ -75,6 +128,7 @@ def test__get_local_modules(
                 package_module_name_map={},
                 pep621_dev_dependency_groups=(),
                 using_default_requirements_files=True,
+                experimental_namespace_package=experimental_namespace_package,
             )._get_local_modules()
             == expected
         )