From 2550b1c805fe7472b93998790c6b5e2bc5b2e68d Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Thu, 8 Sep 2022 10:25:00 +0200 Subject: [PATCH 1/5] Added a check for development dependencies, added unit tests, added fixture for cli unit tests --- README.md | 2 +- deptry/cli.py | 45 +++++-- deptry/config.py | 54 ++++++--- deptry/core.py | 33 +++-- deptry/dependency_getter.py | 36 ++++-- deptry/issue_finders/dev.py | 36 ++++++ deptry/issue_finders/issue_finder.py | 50 -------- deptry/issue_finders/missing.py | 19 ++- deptry/issue_finders/obsolete.py | 27 ++++- deptry/issue_finders/transitive.py | 19 +-- deptry/module.py | 113 ++++++++++++------ deptry/python_file_finder.py | 10 +- docs/docs/pyproject-toml.md | 4 +- docs/docs/usage.md | 4 +- pyproject.toml | 4 +- .../README.md | 28 +++-- .../poetry.toml | 0 .../pyproject.toml | 3 + .../src/main.py | 3 +- .../src/notebook.ipynb | 1 - .../project_without_obsolete/README.md | 4 - .../project_without_obsolete/poetry.toml | 2 - .../project_without_obsolete/pyproject.toml | 10 -- .../project_without_obsolete/src/main.py | 6 - tests/test_cli.py | 74 ++++++------ tests/test_missing_dependencies_finder.py | 8 +- tests/test_module.py | 8 +- tests/test_obsolete_dependencies_finder.py | 12 +- tests/test_transitive_dependencies_finder.py | 8 +- 29 files changed, 367 insertions(+), 256 deletions(-) create mode 100644 deptry/issue_finders/dev.py delete mode 100644 deptry/issue_finders/issue_finder.py rename tests/data/{projects/project_with_obsolete => example_project}/README.md (52%) rename tests/data/{projects/project_with_obsolete => example_project}/poetry.toml (100%) rename tests/data/{projects/project_with_obsolete => example_project}/pyproject.toml (86%) rename tests/data/{projects/project_with_obsolete => example_project}/src/main.py (75%) rename tests/data/{projects/project_with_obsolete => example_project}/src/notebook.ipynb (95%) delete mode 100644 tests/data/projects/project_without_obsolete/README.md delete mode 100644 tests/data/projects/project_without_obsolete/poetry.toml delete mode 100644 tests/data/projects/project_without_obsolete/pyproject.toml delete mode 100644 tests/data/projects/project_without_obsolete/src/main.py diff --git a/README.md b/README.md index de13c937..3d84fd6a 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ --- -_deptry_ is a command line tool to check for issues with dependencies in a poetry managed Python project. It checks for three types of issues: +_deptry_ is a command line tool to check for issues with dependencies in a poetry managed Python project. It checks for fourtypes of issues: - Obsolete dependencies: Dependencies which are added to your project's dependencies, but which are not used within the codebase. - Transitive dependencies: Packages from which code is imported, but the package (A) itself is not in your projects dependencies. Instead, another package (B) is in your list of dependencies, which depends on (A). Package (A) should be added to your project's list of dependencies. diff --git a/deptry/cli.py b/deptry/cli.py index 72f751a1..5e53a751 100644 --- a/deptry/cli.py +++ b/deptry/cli.py @@ -33,6 +33,11 @@ is_flag=True, help="Boolean flag to specify if deptry should skip scanning the project for transitive dependencies.", ) +@click.option( + "--skip-develop", + is_flag=True, + help="Boolean flag to specify if deptry should skip scanning the project for development dependencies that should be regular dependencies.", +) @click.option( "--ignore-obsolete", "-io", @@ -57,10 +62,16 @@ Can be used multiple times. For example; `deptry . -it foo -io bar`.""", ) @click.option( - "--ignore-directories", + "--ignore-develop", "-id", multiple=True, - help="""Directories in which .py files should not be scanned for imports to determine if dependencies are obsolete, missing or transitive. + help="""Modules that should never be marked as 'missing due to transitive' even though deptry determines them to be transitive. + Can be used multiple times. For example; `deptry . -it foo -io bar`.""", +) +@click.option( + "--exclude", + multiple=True, + help="""Directories or files in which .py files should not be scanned for imports to determine if there are dependency issues. Defaults to ['venv','tests']. Specify multiple directories by using this flag twice, e.g. `-id .venv -id tests -id other_dir`.""", ) @click.option( @@ -80,10 +91,12 @@ def deptry( ignore_obsolete: List[str], ignore_missing: List[str], ignore_transitive: List[str], + ignore_develop: List[str], skip_obsolete: bool, skip_missing: bool, skip_transitive: bool, - ignore_directories: List[str], + skip_develop: bool, + exclude: List[str], ignore_notebooks: bool, version: bool, ) -> None: @@ -108,22 +121,26 @@ def deptry( ignore_obsolete=ignore_obsolete if ignore_obsolete else None, ignore_missing=ignore_missing if ignore_missing else None, ignore_transitive=ignore_transitive if ignore_transitive else None, - ignore_directories=ignore_directories if ignore_directories else None, + ignore_develop=ignore_develop if ignore_develop else None, + exclude=exclude if exclude else None, ignore_notebooks=ignore_notebooks if ignore_notebooks else None, skip_obsolete=skip_obsolete if skip_obsolete else None, skip_missing=skip_missing if skip_missing else None, skip_transitive=skip_transitive if skip_transitive else None, + skip_develop=skip_develop if skip_develop else None, ) result = Core( ignore_obsolete=config.ignore_obsolete, ignore_missing=config.ignore_missing, ignore_transitive=config.ignore_transitive, - ignore_directories=config.ignore_directories, + ignore_develop=config.ignore_develop, + exclude=config.exclude, ignore_notebooks=config.ignore_notebooks, skip_obsolete=config.skip_obsolete, skip_missing=config.skip_missing, skip_transitive=config.skip_transitive, + skip_develop=config.skip_develop, ).run() issue_found = False if not skip_obsolete and "obsolete" in result and result["obsolete"]: @@ -135,6 +152,9 @@ def deptry( if not skip_transitive and "transitive" in result and result["transitive"]: log_transitive_dependencies(result["transitive"]) issue_found = True + if not skip_develop and "develop" in result and result["develop"]: + log_develop_dependencies(result["develop"]) + issue_found = True if issue_found: log_additional_info() @@ -165,9 +185,16 @@ def log_transitive_dependencies(dependencies: List[str], sep="\n\t") -> None: logging.info( f"There are transitive dependencies that should be explicitly defined as dependencies in pyproject.toml:\n{sep}{sep.join(dependencies)}\n" ) + logging.info("""They are currently imported but not specified directly as your project's dependencies.""") + + +def log_develop_dependencies(dependencies: List[str], sep="\n\t") -> None: + logging.info("\n-----------------------------------------------------\n") + logging.info(f"There are imported modules from development dependencies detected:\n{sep}{sep.join(dependencies)}\n") logging.info( - """They are currently imported but not specified directly as your project's dependencies. This issue also be caused -by a development dependency that is found to be used within the scanned Python files.""" + """Consider moving them to `[tool.poetry.dependencies]` in pyproject.toml. If this is not correct and the +dependencies listed above are indeed development dependencies, it's likely that files were scanned that are used +for development. Run `deptry -v .` to see a list of scanned files.""" ) @@ -175,7 +202,7 @@ def log_additional_info(): logging.info("\n-----------------------------------------------------\n") logging.info( """Dependencies and directories can be ignored by passing additional command-line arguments. See `deptry --help` for more details. -Alternatively, deptry can be configured through `pyproject.toml`: +Alternatively, deptry can be configured through `pyproject.toml`. An example: ``` [tool.deptry] @@ -188,7 +215,7 @@ def log_additional_info(): ignore_transitive = [ 'your-dependency' ] -ignore_directories = [ +exclude = [ '.venv', 'tests', 'docs' ] ``` diff --git a/deptry/config.py b/deptry/config.py index 7d7f7001..3ee8a1bc 100644 --- a/deptry/config.py +++ b/deptry/config.py @@ -8,11 +8,13 @@ "ignore_obsolete": [], "ignore_missing": [], "ignore_transitive": [], - "ignore_directories": [".venv", "tests"], + "ignore_develop": [], + "exclude": [".venv", "tests"], "ignore_notebooks": False, "skip_obsolete": False, "skip_missing": False, "skip_transitive": False, + "skip_develop": False, } @@ -28,34 +30,40 @@ def __init__( ignore_obsolete: Optional[List[str]], ignore_missing: Optional[List[str]], ignore_transitive: Optional[List[str]], + ignore_develop: Optional[List[str]], skip_obsolete: Optional[bool], skip_missing: Optional[bool], skip_transitive: Optional[bool], - ignore_directories: Optional[List[str]], + skip_develop: Optional[bool], + exclude: Optional[List[str]], ignore_notebooks: Optional[bool], ) -> None: self._set_defaults() self._override_config_with_pyproject_toml() self._override_config_with_cli_arguments( - ignore_obsolete, - ignore_missing, - ignore_transitive, - ignore_directories, - ignore_notebooks, - skip_obsolete, - skip_missing, - skip_transitive, + ignore_obsolete=ignore_obsolete, + ignore_missing=ignore_missing, + ignore_transitive=ignore_transitive, + ignore_develop=ignore_develop, + exclude=exclude, + ignore_notebooks=ignore_notebooks, + skip_obsolete=skip_obsolete, + skip_missing=skip_missing, + skip_transitive=skip_transitive, + skip_develop=skip_develop, ) def _set_defaults(self) -> None: self.ignore_obsolete = DEFAULTS["ignore_obsolete"] self.ignore_missing = DEFAULTS["ignore_missing"] self.ignore_transitive = DEFAULTS["ignore_transitive"] - self.ignore_directories = DEFAULTS["ignore_directories"] + self.ignore_develop = DEFAULTS["ignore_develop"] + self.exclude = DEFAULTS["exclude"] self.ignore_notebooks = DEFAULTS["ignore_notebooks"] self.skip_obsolete = DEFAULTS["skip_obsolete"] self.skip_missing = DEFAULTS["skip_missing"] self.skip_transitive = DEFAULTS["skip_transitive"] + self.skip_develop = DEFAULTS["skip_develop"] def _override_config_with_pyproject_toml(self) -> None: pyproject_toml_config = self._read_configuration_from_pyproject_toml() @@ -63,10 +71,12 @@ def _override_config_with_pyproject_toml(self) -> None: self._override_with_toml_argument("ignore_obsolete", List[str], pyproject_toml_config) self._override_with_toml_argument("ignore_missing", List[str], pyproject_toml_config) self._override_with_toml_argument("ignore_transitive", List[str], pyproject_toml_config) + self._override_with_toml_argument("ignore_develop", List[str], pyproject_toml_config) self._override_with_toml_argument("skip_missing", List[str], pyproject_toml_config) self._override_with_toml_argument("skip_obsolete", List[str], pyproject_toml_config) self._override_with_toml_argument("skip_transitive", List[str], pyproject_toml_config) - self._override_with_toml_argument("ignore_directories", List[str], pyproject_toml_config) + self._override_with_toml_argument("skip_develop", List[str], pyproject_toml_config) + self._override_with_toml_argument("exclude", List[str], pyproject_toml_config) self._override_with_toml_argument("ignore_notebooks", List[str], pyproject_toml_config) def _read_configuration_from_pyproject_toml(self) -> Optional[Dict]: @@ -87,16 +97,18 @@ def _override_with_toml_argument(self, argument: str, expected_type: Any, pyproj setattr(self, argument, value) self._log_changed_by_pyproject_toml(argument, value) - def _override_config_with_cli_arguments( + def _override_config_with_cli_arguments( # noqa self, ignore_obsolete: Optional[List[str]], ignore_missing: Optional[List[str]], ignore_transitive: Optional[List[str]], - ignore_directories: Optional[List[str]], + ignore_develop: Optional[List[str]], + exclude: Optional[List[str]], ignore_notebooks: Optional[bool], skip_obsolete: Optional[bool], skip_missing: Optional[bool], skip_transitive: Optional[bool], + skip_develop: Optional[bool], ) -> None: if ignore_obsolete: @@ -111,6 +123,10 @@ def _override_config_with_cli_arguments( self.ignore_transitive = ignore_transitive self._log_changed_by_command_line_argument("ignore_transitive", ignore_transitive) + if ignore_develop: + self.ignore_develop = ignore_develop + self._log_changed_by_command_line_argument("ignore_develop", ignore_develop) + if skip_obsolete: self.skip_obsolete = skip_obsolete self._log_changed_by_command_line_argument("skip_obsolete", skip_obsolete) @@ -123,9 +139,13 @@ def _override_config_with_cli_arguments( self.skip_transitive = skip_transitive self._log_changed_by_command_line_argument("skip_transitive", skip_transitive) - if ignore_directories: - self.ignore_directories = ignore_directories - self._log_changed_by_command_line_argument("ignore_directories", ignore_directories) + if skip_develop: + self.skip_develop = skip_develop + self._log_changed_by_command_line_argument("skip_develop", skip_develop) + + if exclude: + self.exclude = exclude + self._log_changed_by_command_line_argument("exclude", exclude) if ignore_notebooks: self.ignore_notebooks = ignore_notebooks diff --git a/deptry/core.py b/deptry/core.py index c152f379..a924afd1 100644 --- a/deptry/core.py +++ b/deptry/core.py @@ -4,10 +4,11 @@ from deptry.dependency_getter import DependencyGetter from deptry.import_parser import ImportParser +from deptry.issue_finders.dev import DevDependenciesFinder from deptry.issue_finders.missing import MissingDependenciesFinder from deptry.issue_finders.obsolete import ObsoleteDependenciesFinder from deptry.issue_finders.transitive import TransitiveDependenciesFinder -from deptry.module import Module +from deptry.module import ModuleBuilder from deptry.python_file_finder import PythonFileFinder @@ -17,53 +18,65 @@ def __init__( ignore_obsolete: List[str], ignore_missing: List[str], ignore_transitive: List[str], + ignore_develop: List[str], skip_obsolete: bool, skip_missing: bool, skip_transitive: bool, - ignore_directories: List[str], + skip_develop: bool, + exclude: List[str], ignore_notebooks: bool, ) -> None: self.ignore_obsolete = ignore_obsolete self.ignore_missing = ignore_missing self.ignore_transitive = ignore_transitive - self.ignore_directories = ignore_directories + self.ignore_develop = ignore_develop + self.exclude = exclude self.ignore_notebooks = ignore_notebooks self.skip_obsolete = skip_obsolete self.skip_missing = skip_missing self.skip_transitive = skip_transitive + self.skip_develop = skip_develop logging.debug("Running with the following configuration:") logging.debug(f"ignore_obsolete: {ignore_obsolete}") logging.debug(f"ignore_missing: {ignore_missing}") logging.debug(f"ignore_transitive: {ignore_transitive}") + logging.debug(f"ignore_develop: {ignore_develop}") logging.debug(f"skip_obsolete: {skip_obsolete}") logging.debug(f"skip_missing: {skip_missing}") logging.debug(f"skip_transitive: {skip_transitive}") - logging.debug(f"ignore_directories: {ignore_directories}") + logging.debug(f"skip_develop {skip_develop}") + logging.debug(f"exclude: {exclude}") logging.debug(f"ignore_notebooks: {ignore_notebooks}\n") def run(self) -> Dict: dependencies = DependencyGetter().get() + dev_dependencies = DependencyGetter(dev=True).get() + all_python_files = PythonFileFinder( - ignore_directories=self.ignore_directories, ignore_notebooks=self.ignore_notebooks + exclude=self.exclude, ignore_notebooks=self.ignore_notebooks ).get_all_python_files_in(Path(".")) imported_modules = ImportParser().get_imported_modules_for_list_of_files(all_python_files) - imported_modules = [Module(mod, dependencies) for mod in imported_modules] - imported_modules = [mod for mod in imported_modules if not mod.is_standard_library()] + imported_modules = [ModuleBuilder(mod, dependencies, dev_dependencies).build() for mod in imported_modules] + imported_modules = [mod for mod in imported_modules if not mod.standard_library] result = {} if not self.skip_obsolete: result["obsolete"] = ObsoleteDependenciesFinder( - imported_modules=imported_modules, dependencies=dependencies, list_to_ignore=self.ignore_obsolete + imported_modules=imported_modules, dependencies=dependencies, ignore_obsolete=self.ignore_obsolete ).find() if not self.skip_missing: result["missing"] = MissingDependenciesFinder( - imported_modules=imported_modules, dependencies=dependencies, list_to_ignore=self.ignore_missing + imported_modules=imported_modules, dependencies=dependencies, ignore_missing=self.ignore_missing ).find() if not self.skip_transitive: result["transitive"] = TransitiveDependenciesFinder( - imported_modules=imported_modules, dependencies=dependencies, list_to_ignore=self.ignore_transitive + imported_modules=imported_modules, dependencies=dependencies, ignore_transitive=self.ignore_transitive + ).find() + if not self.skip_develop: + result["develop"] = DevDependenciesFinder( + imported_modules=imported_modules, dependencies=dependencies, ignore_develop=self.ignore_develop ).find() return result diff --git a/deptry/dependency_getter.py b/deptry/dependency_getter.py index 9de31337..895706c9 100644 --- a/deptry/dependency_getter.py +++ b/deptry/dependency_getter.py @@ -1,6 +1,6 @@ import logging from pathlib import Path -from typing import List +from typing import Dict, List import toml @@ -12,14 +12,18 @@ class DependencyGetter: Class to get a project's list of dependencies from pyproject.toml. Args: - ignore_dependencies: A list of dependencies which should be omitted from the resulting list. + dev (bool): Read either the regular, or the dev dependencies, based on this argument. """ - def __init__(self) -> None: - pass + def __init__(self, dev: bool = False) -> None: + self.dev = dev def get(self): - pyproject_toml_dependencies = self._get_pyproject_toml_dependencies() + if self.dev: + pyproject_toml_dependencies = self._get_pyproject_toml_dev_dependencies() + else: + pyproject_toml_dependencies = self._get_pyproject_toml_dependencies() + dependencies = [] for dep in pyproject_toml_dependencies: if not dep == "python": @@ -27,14 +31,30 @@ def get(self): self._log_dependencies(dependencies) return dependencies - def _get_pyproject_toml_dependencies(self) -> List[str]: + def _load_pyproject_toml(self) -> Dict: pyproject_text = Path("./pyproject.toml").read_text() - pyproject_data = toml.loads(pyproject_text) + return toml.loads(pyproject_text) + + def _get_pyproject_toml_dependencies(self) -> List[str]: + pyproject_data = self._load_pyproject_toml() dependencies = list(pyproject_data["tool"]["poetry"]["dependencies"].keys()) return sorted(dependencies) + def _get_pyproject_toml_dev_dependencies(self) -> List[str]: + dev_dependencies = {} + pyproject_data = self._load_pyproject_toml() + try: + dev_dependencies = {**pyproject_data["tool"]["poetry"]["dev-dependencies"], **dev_dependencies} + except KeyError: + pass + try: + dev_dependencies = {**pyproject_data["tool"]["poetry"]["group"]["dev"]["dependencies"], **dev_dependencies} + except KeyError: + pass + return sorted(dev_dependencies) + def _log_dependencies(self, dependencies: List[Dependency]) -> None: - logging.debug("The project contains the following dependencies:") + logging.debug(f"The project contains the following {'dev-' if self.dev else ''}dependencies:") for dependency in dependencies: logging.debug(str(dependency)) logging.debug("") diff --git a/deptry/issue_finders/dev.py b/deptry/issue_finders/dev.py new file mode 100644 index 00000000..97944544 --- /dev/null +++ b/deptry/issue_finders/dev.py @@ -0,0 +1,36 @@ +import logging +from typing import List + +from deptry.dependency import Dependency +from deptry.module import Module + + +class DevDependenciesFinder: + """ + Given a list of imported modules and a list of project dependencies, determine which ones are transitive. + """ + + def __init__( + self, imported_modules: List[Module], dependencies: List[Dependency], ignore_develop: List[str] = [] + ) -> None: + self.imported_modules = imported_modules + self.dependencies = dependencies + self.ignore_develop = ignore_develop + + def find(self) -> List[str]: + logging.debug("\nScanning for incorrect development dependencies...") + dev_dependencies = [] + for module in self.imported_modules: + logging.debug(f"Scanning module {module.name}...") + if self._is_development_dependency(module): + dev_dependencies.append(module.package) + return dev_dependencies + + def _is_development_dependency(self, module: Module) -> bool: + if module.dev_dependency: + if module.name in self.ignore_develop: + logging.debug(f"Module '{module.package}' found to be a development dependency, but ignoring.") + else: + logging.debug(f"Dependency '{module.package}' marked as a transitive dependency.") + return True + return False diff --git a/deptry/issue_finders/issue_finder.py b/deptry/issue_finders/issue_finder.py deleted file mode 100644 index f8c9f962..00000000 --- a/deptry/issue_finders/issue_finder.py +++ /dev/null @@ -1,50 +0,0 @@ -import logging -from typing import List - -from deptry.dependency import Dependency -from deptry.module import Module - - -class IssueFinder: - """ - Helper class to find issues within a project's dependencies - """ - - def __init__( - self, imported_modules: List[Module], dependencies: List[Dependency], list_to_ignore: List[str] = [] - ) -> None: - self.imported_modules = imported_modules - self.dependencies = dependencies - self.list_to_ignore = list_to_ignore - - def _module_in_any_top_level(self, module: Module) -> bool: - for dependency in self.dependencies: - if dependency.top_levels and module.name in dependency.top_levels: - return True - return False - - def _module_in_dependencies(self, module: Module) -> bool: - for dependency in self.dependencies: - if module.package == dependency.name: - return True - return False - - def _dependency_found_in_imported_modules(self, dependency: Dependency) -> bool: - for module in self.imported_modules: - if module.package == dependency.name: - logging.debug(f"Dependency '{dependency.name}' is used as module '{module.name}'.") - return True - else: - return False - - def _any_of_the_top_levels_imported(self, dependency: Dependency) -> bool: - if not dependency.top_levels: - return False - else: - for top_level in dependency.top_levels: - if any(module.name == top_level for module in self.imported_modules): - logging.debug( - f"Dependency '{dependency.name}' is not obsolete, since imported module '{top_level}' is in its top-level module names" - ) - return True - return False diff --git a/deptry/issue_finders/missing.py b/deptry/issue_finders/missing.py index 05a5929f..d255476e 100644 --- a/deptry/issue_finders/missing.py +++ b/deptry/issue_finders/missing.py @@ -2,20 +2,20 @@ from typing import List from deptry.dependency import Dependency -from deptry.issue_finders.issue_finder import IssueFinder from deptry.module import Module -class MissingDependenciesFinder(IssueFinder): +class MissingDependenciesFinder: """ Given a list of imported modules and a list of project dependencies, determine which ones are missing. - TODO make one class dependencyfinder that the other three inherit from """ def __init__( - self, imported_modules: List[Module], dependencies: List[Dependency], list_to_ignore: List[str] = [] + self, imported_modules: List[Module], dependencies: List[Dependency], ignore_missing: List[str] = [] ) -> None: - super().__init__(imported_modules, dependencies, list_to_ignore) + self.imported_modules = imported_modules + self.dependencies = dependencies + self.ignore_missing = ignore_missing def find(self) -> List[str]: logging.debug("\nScanning for missing dependencies...") @@ -28,13 +28,8 @@ def find(self) -> List[str]: def _is_missing(self, module: Module) -> bool: - if ( - module.package is None - and not self._module_in_any_top_level(module) - and not self._module_in_dependencies(module) - and not module.is_local_module() - ): - if module.name in self.list_to_ignore: + if module.package is None and not module.dependency and not module.dev_dependency and not module.local_module: + if module.name in self.ignore_missing: logging.debug(f"Identified module '{module.name}' as a missing dependency, but ignoring.") else: logging.debug( diff --git a/deptry/issue_finders/obsolete.py b/deptry/issue_finders/obsolete.py index 388f817d..58b6cede 100644 --- a/deptry/issue_finders/obsolete.py +++ b/deptry/issue_finders/obsolete.py @@ -2,11 +2,10 @@ from typing import List from deptry.dependency import Dependency -from deptry.issue_finders.issue_finder import IssueFinder from deptry.module import Module -class ObsoleteDependenciesFinder(IssueFinder): +class ObsoleteDependenciesFinder: """ Given a list of imported modules and a list of project dependencies, determine which ones are obsolete. @@ -18,9 +17,11 @@ class ObsoleteDependenciesFinder(IssueFinder): """ def __init__( - self, imported_modules: List[Module], dependencies: List[Dependency], list_to_ignore: List[str] = [] + self, imported_modules: List[Module], dependencies: List[Dependency], ignore_obsolete: List[str] = [] ) -> None: - super().__init__(imported_modules, dependencies, list_to_ignore) + self.imported_modules = imported_modules + self.dependencies = dependencies + self.ignore_obsolete = ignore_obsolete def find(self) -> List[str]: logging.debug("\nScanning for obsolete dependencies...") @@ -36,9 +37,25 @@ def _is_obsolete(self, dependency: Dependency) -> bool: if not self._dependency_found_in_imported_modules(dependency) and not self._any_of_the_top_levels_imported( dependency ): - if dependency.name in self.list_to_ignore: + if dependency.name in self.ignore_obsolete: logging.debug(f"Dependency '{dependency.name}' found to be obsolete, but ignoring.") else: logging.debug(f"Dependency '{dependency.name}' does not seem to be used.") return True return False + + def _dependency_found_in_imported_modules(self, dependency: Dependency) -> bool: + for module in self.imported_modules: + if module.package == dependency.name: + return True + else: + return False + + def _any_of_the_top_levels_imported(self, dependency: Dependency) -> bool: + if not dependency.top_levels: + return False + else: + for top_level in dependency.top_levels: + if any(module.name == top_level for module in self.imported_modules): + return True + return False diff --git a/deptry/issue_finders/transitive.py b/deptry/issue_finders/transitive.py index 27838a85..a7f2c006 100644 --- a/deptry/issue_finders/transitive.py +++ b/deptry/issue_finders/transitive.py @@ -2,19 +2,20 @@ from typing import List from deptry.dependency import Dependency -from deptry.issue_finders.issue_finder import IssueFinder from deptry.module import Module -class TransitiveDependenciesFinder(IssueFinder): +class TransitiveDependenciesFinder: """ Given a list of imported modules and a list of project dependencies, determine which ones are transitive. """ def __init__( - self, imported_modules: List[Module], dependencies: List[Dependency], list_to_ignore: List[str] = [] + self, imported_modules: List[Module], dependencies: List[Dependency], ignore_transitive: List[str] = [] ) -> None: - super().__init__(imported_modules, dependencies, list_to_ignore) + self.imported_modules = imported_modules + self.dependencies = dependencies + self.ignore_transitive = ignore_transitive def find(self) -> List[str]: logging.debug("\nScanning for transitive dependencies...") @@ -28,12 +29,12 @@ def find(self) -> List[str]: def _is_transitive(self, module: Module) -> bool: if ( module.package is not None - and not self._module_in_any_top_level(module) - and not self._module_in_dependencies(module) - and not module.is_local_module() + and not module.dependency + and not module.dev_dependency + and not module.local_module ): - if module.name in self.list_to_ignore: - logging.debug(f"Module '{module.package}' found to be a transitive dependency, but ignoring.") + if module.name in self.ignore_transitive: + logging.debug(f"Dependency '{module.package}' found to be a transitive dependency, but ignoring.") else: logging.debug(f"Dependency '{module.package}' marked as a transitive dependency.") return True diff --git a/deptry/module.py b/deptry/module.py index b195a7e4..532f1091 100644 --- a/deptry/module.py +++ b/deptry/module.py @@ -16,7 +16,41 @@ class Module: - def __init__(self, name: str, dependencies: List[Dependency] = None) -> None: + def __init__( + self, + name: str, + standard_library: bool = False, + local_module: bool = False, + package: str = None, + top_levels: List[str] = None, + dev_top_levels: List[str] = None, + dependency: bool = None, + dev_dependency: bool = None, + ): + self.name = name + self.standard_library = standard_library + self.local_module = local_module + self.package = package + self.top_levels = top_levels + self.dev_top_levels = dev_top_levels + self.dependency = dependency + self.dev_dependency = dev_dependency + self._log() + + def _log(self): + logging.debug("--- MODULE ---") + logging.debug(self.__str__()) + logging.debug("") + + def __repr__(self) -> str: + return f"Module '{self.name}'" + + def __str__(self) -> str: + return "\n".join("%s: %s" % item for item in vars(self).items()) + + +class ModuleBuilder: + def __init__(self, name: str, dependencies: List[Dependency] = [], dev_dependencies: List[Dependency] = []) -> None: """ A Module object that represents an imported module. If the metadata field 'Name' is found for a module, it's added as the associated package name. Otherwise, check if it is part of the standard library. @@ -27,52 +61,57 @@ def __init__(self, name: str, dependencies: List[Dependency] = None) -> None: we do know that this information can be used in detecting obsolete dependencies, so there is no need to raise a warning. If nothing is found, a warning is logged to inform the user that there is no information available about this package. + TODO: Move name arg to build func """ self.name = name - self.standard_library = False - self.package = self._get_package_name(dependencies) - self.local_module = self._module_is_local_directory() + self.dependencies = dependencies + self.dev_dependencies = dev_dependencies + + def build(self) -> Module: - def _get_package_name(self, dependencies: List[Dependency]) -> str: + standard_library = self._in_standard_library() + if standard_library: + return Module(self.name, standard_library=True) + else: + local_module = self._is_local_directory() + if local_module: + return Module(self.name, local_module=True) + else: + package = self._get_package_name() + top_levels = self._get_corresponding_top_levels(self.dependencies) + dev_top_levels = self._get_corresponding_top_levels(self.dev_dependencies) + dependency = self._has_matching_dependency(package, top_levels) + dev_dependency = self._has_matching_dev_dependency(package, dev_top_levels) + return Module( + self.name, + package=package, + top_levels=top_levels, + dev_top_levels=dev_top_levels, + dependency=dependency, + dev_dependency=dev_dependency, + ) + + def _get_package_name(self) -> str: try: return self._extract_package_name_from_metadata() except PackageNotFoundError: - if self._module_in_standard_library(): - self.standard_library = True - elif dependencies and self._module_found_in_top_levels(dependencies): - logging.debug( - f"Failed to find metadata for import `{self.name}` in current environment, but it was encountered in a dependency's top-level module names." - ) - else: - logging.debug( - f"Failed to find corresponding package name for import `{self.name}` in current environment." - ) + pass - def _module_found_in_top_levels(self, dependencies: List[Dependency]) -> bool: - return any([self.name in dependency.top_levels for dependency in dependencies if dependency.top_levels]) + def _get_corresponding_top_levels(self, dependencies: List[Dependency]) -> bool: + return [ + dependency.name + for dependency in dependencies + if dependency.top_levels and self.name in dependency.top_levels + ] def _extract_package_name_from_metadata(self) -> str: package = metadata.metadata(self.name)["Name"] - logging.debug(f"Corresponding package name for imported module `{self.name}` is `{package}`.") return package - def _module_in_standard_library(self): + def _in_standard_library(self): if self.name in self._get_stdlib_packages(): - logging.debug(f"module `{self.name}` is in the Python standard library.") return True - def __repr__(self) -> str: - return f"Module '{self.name}'" - - def __str__(self) -> str: - if self.standard_library: - return f"Module '{self.name}' from standard library" - else: - if self.package: - return f"Module '{self.name}' from package '{self.package}'" - else: - return f"Module '{self.name}' from unknown source" - def _get_stdlib_packages(self) -> Set[str]: incorrect_version_error = ValueError( f"Incorrect Python version {'.'.join([str(x) for x in sys.version_info[0:3]])}. Only 3.7, 3.8, 3.9 and 3.10 are currently supported." @@ -93,13 +132,13 @@ def _get_stdlib_packages(self) -> Set[str]: else: raise incorrect_version_error - def _module_is_local_directory(self): + def _is_local_directory(self): directories = [f for f in os.listdir() if Path(f).is_dir()] local_modules = [subdir for subdir in directories if "__init__.py" in os.listdir(subdir)] return self.name in local_modules - def is_local_module(self): - return self.local_module + def _has_matching_dependency(self, package, top_levels): + return (package in [dep.name for dep in self.dependencies]) or len(top_levels) > 0 - def is_standard_library(self): - return self.standard_library + def _has_matching_dev_dependency(self, package, dev_top_levels): + return (package in [dep.name for dep in self.dev_dependencies]) or len(dev_top_levels) > 0 diff --git a/deptry/python_file_finder.py b/deptry/python_file_finder.py index a0d94ff4..f2731407 100644 --- a/deptry/python_file_finder.py +++ b/deptry/python_file_finder.py @@ -9,8 +9,8 @@ class PythonFileFinder: If ignore_notebooks is set to True, .ipynb files are ignored and only .py files are returned. """ - def __init__(self, ignore_directories: List[str] = [".venv"], ignore_notebooks: bool = False) -> None: - self.ignore_directories = ignore_directories + def __init__(self, exclude: List[str] = [".venv"], ignore_notebooks: bool = False) -> None: + self.exclude = exclude self.ignore_notebooks = ignore_notebooks def get_all_python_files_in(self, directory: Path) -> List[Path]: @@ -29,8 +29,4 @@ def _get_all_ipynb_files_in(self, directory: Path) -> List[Path]: return [path for path in directory.rglob("*.ipynb")] def _remove_directories_to_ignore(self, all_py_files: List[Path]) -> List[Path]: - return [ - path - for path in all_py_files - if not any([str(path).startswith(pattern) for pattern in self.ignore_directories]) - ] + return [path for path in all_py_files if not any([str(path).startswith(pattern) for pattern in self.exclude])] diff --git a/docs/docs/pyproject-toml.md b/docs/docs/pyproject-toml.md index 42ab5eb5..476e9d4c 100644 --- a/docs/docs/pyproject-toml.md +++ b/docs/docs/pyproject-toml.md @@ -4,7 +4,7 @@ _deptry_ can be configured by adding a `[tool.deptry]` section to _pyproject.toml_. The possible arguments are: -- `ignore_directories`: `List` +- `exclude`: `List` - `ignore_notebooks`: `bool` - `ignore_obsolete`: `List` - `ignore_missing`: `List` @@ -17,7 +17,7 @@ An example of such a section is given below. ``` [tool.deptry] -ignore_directories = [ +exclude = [ '.venv','tests','docs' ] ignore_obsolete = [ diff --git a/docs/docs/usage.md b/docs/docs/usage.md index f70b593d..0141fd40 100644 --- a/docs/docs/usage.md +++ b/docs/docs/usage.md @@ -30,11 +30,11 @@ deptry . -v ## Ignore directories _deptry_ scans the working directory and it's subdirectories recursively for `.py` and `.ipynb` files to scan for import statements. By default, -the `.venv` directory is ignored. To ignore other directories, use the `-id` flag. Note that this overwrites the default, so to ignore +the `.venv` directory is ignored. To ignore other directories, use the `--exclude` flag. Note that this overwrites the default, so to ignore both the `.venv` directory and another directory, use the flag twice: ```sh -deptry . -id .venv -id other_directory +deptry . --exclude .venv --exclude tests --exclude other_directory ``` ## Skip checks for obsolete, transitive or missing dependencies. diff --git a/pyproject.toml b/pyproject.toml index dc5dde05..39a4b898 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,8 +68,8 @@ exclude = [ ] [tool.deptry] -ignore_directories = [ - '.venv','docs', 'tests' +exclude = [ + '.venv','docs','tests' ] [tool.poetry.scripts] diff --git a/tests/data/projects/project_with_obsolete/README.md b/tests/data/example_project/README.md similarity index 52% rename from tests/data/projects/project_with_obsolete/README.md rename to tests/data/example_project/README.md index 7eb73e5f..b7aaba0e 100644 --- a/tests/data/projects/project_with_obsolete/README.md +++ b/tests/data/example_project/README.md @@ -1,3 +1,5 @@ +## Dependencies + Dependencies are: ``` @@ -9,12 +11,21 @@ requests pkginfo ``` +dev-dependencies: + +``` +black +``` + +## Imports + Imported in .py files are ``` click -requests urllib3 +black +white ``` Additional imports in .ipynb file: @@ -23,20 +34,21 @@ Additional imports in .ipynb file: toml ``` +## Config + pyproject.toml specifies to ignore the dependency: ``` pkginfo ``` -So expected output for obsolete packages when ignoring ipynb: +## Output -``` -isort -toml -``` -Expected output for obsolete packages when including ipynb files: +So expected output without any additional configuration: ``` -isort +obsolete: requests (because pkginfo is ignored) +missing: white +transitive: None +dev: black ``` diff --git a/tests/data/projects/project_with_obsolete/poetry.toml b/tests/data/example_project/poetry.toml similarity index 100% rename from tests/data/projects/project_with_obsolete/poetry.toml rename to tests/data/example_project/poetry.toml diff --git a/tests/data/projects/project_with_obsolete/pyproject.toml b/tests/data/example_project/pyproject.toml similarity index 86% rename from tests/data/projects/project_with_obsolete/pyproject.toml rename to tests/data/example_project/pyproject.toml index a677fb25..69110bb9 100644 --- a/tests/data/projects/project_with_obsolete/pyproject.toml +++ b/tests/data/example_project/pyproject.toml @@ -13,6 +13,9 @@ click = "^8.1.3" requests = "^2.28.1" pkginfo = "^1.8.3" +[tool.poetry.dev-dependencies] +black = "^22.6.0" + [tool.deptry] ignore_obsolete = [ 'pkginfo' diff --git a/tests/data/projects/project_with_obsolete/src/main.py b/tests/data/example_project/src/main.py similarity index 75% rename from tests/data/projects/project_with_obsolete/src/main.py rename to tests/data/example_project/src/main.py index 693bbaca..2ee666ac 100644 --- a/tests/data/projects/project_with_obsolete/src/main.py +++ b/tests/data/example_project/src/main.py @@ -1,6 +1,7 @@ from os import chdir, walk from pathlib import Path +import black import click -import requests as req +import white as w from urllib3 import contrib diff --git a/tests/data/projects/project_with_obsolete/src/notebook.ipynb b/tests/data/example_project/src/notebook.ipynb similarity index 95% rename from tests/data/projects/project_with_obsolete/src/notebook.ipynb rename to tests/data/example_project/src/notebook.ipynb index d840e399..a51bdb9d 100644 --- a/tests/data/projects/project_with_obsolete/src/notebook.ipynb +++ b/tests/data/example_project/src/notebook.ipynb @@ -8,7 +8,6 @@ "outputs": [], "source": [ "import click\n", - "import requests as req\n", "from urllib3 import contrib\n", "import toml" ] diff --git a/tests/data/projects/project_without_obsolete/README.md b/tests/data/projects/project_without_obsolete/README.md deleted file mode 100644 index 8399f7a5..00000000 --- a/tests/data/projects/project_without_obsolete/README.md +++ /dev/null @@ -1,4 +0,0 @@ -Imported are toml, urllib3 -Dependencies are toml, urllib3 - -So expected output; no obsolete packages. diff --git a/tests/data/projects/project_without_obsolete/poetry.toml b/tests/data/projects/project_without_obsolete/poetry.toml deleted file mode 100644 index ab1033bd..00000000 --- a/tests/data/projects/project_without_obsolete/poetry.toml +++ /dev/null @@ -1,2 +0,0 @@ -[virtualenvs] -in-project = true diff --git a/tests/data/projects/project_without_obsolete/pyproject.toml b/tests/data/projects/project_without_obsolete/pyproject.toml deleted file mode 100644 index f75bdde5..00000000 --- a/tests/data/projects/project_without_obsolete/pyproject.toml +++ /dev/null @@ -1,10 +0,0 @@ -[tool.poetry] -name = "test" -version = "0.0.1" -description = "A test project" -authors = ["test "] - -[tool.poetry.dependencies] -python = ">=3.7,<3.11" -toml = "^0.10.2" -urllib3 = "^1.26.12" diff --git a/tests/data/projects/project_without_obsolete/src/main.py b/tests/data/projects/project_without_obsolete/src/main.py deleted file mode 100644 index f5cd77e5..00000000 --- a/tests/data/projects/project_without_obsolete/src/main.py +++ /dev/null @@ -1,6 +0,0 @@ -from os import chdir, walk -from pathlib import Path -from typing import List - -import toml -from urllib3 import _collections diff --git a/tests/test_cli.py b/tests/test_cli.py index 53290969..df4150c9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,59 +2,63 @@ import shutil import subprocess -from deptry.utils import run_within_dir - +import pytest -def test_cli_returns_error(tmp_path): - """ - data/projects/project_with_obsolete has obsolete dependencies. - Verify that `deptry` returns status code 1 and verify that it finds the right obsolete dependencies. - """ +from deptry.utils import run_within_dir - tmp_path_proj = tmp_path / "project_with_obsolete" - shutil.copytree("tests/data/projects/project_with_obsolete", tmp_path_proj) +@pytest.fixture(scope="session") +def dir_with_venv_installed(tmp_path_factory): + tmp_path_proj = tmp_path_factory.getbasetemp() + shutil.copytree("tests/data/example_project", str(tmp_path_proj), dirs_exist_ok=True) with run_within_dir(str(tmp_path_proj)): subprocess.check_call(shlex.split("poetry install --no-interaction --no-root")) == 0 + return tmp_path_proj + + +def test_cli_returns_error(dir_with_venv_installed): + with run_within_dir(str(dir_with_venv_installed)): result = subprocess.run(shlex.split("poetry run deptry ."), capture_output=True, text=True) assert result.returncode == 1 - assert "pyproject.toml contains obsolete dependencies:\n\n\tisort\n\n" in result.stderr + assert "pyproject.toml contains obsolete dependencies:\n\n\tisort\n\trequests\n\n" in result.stderr + assert "There are dependencies missing from pyproject.toml:\n\n\twhite\n\n" in result.stderr + assert "There are imported modules from development dependencies detected:\n\n\tblack\n\n" in result.stderr + +def test_cli_ignore_notebooks(dir_with_venv_installed): + with run_within_dir(str(dir_with_venv_installed)): result = subprocess.run(shlex.split("poetry run deptry . --ignore-notebooks"), capture_output=True, text=True) assert result.returncode == 1 - assert "pyproject.toml contains obsolete dependencies:\n\n\tisort\n\ttoml\n\n" in result.stderr - + assert "pyproject.toml contains obsolete dependencies:\n\n\tisort\n\trequests\n\ttoml\n\n" in result.stderr -def test_cli_returns_no_error(tmp_path): - """ - data/projects/project_without_obsolete has no obsolete dependencies. - Verify that `deptry` completes with status code 0. - """ - tmp_path_proj = tmp_path / "project_without_obsolete" - shutil.copytree("tests/data/projects/project_without_obsolete", tmp_path_proj) - - with run_within_dir(str(tmp_path_proj)): - subprocess.check_call(shlex.split("poetry install --no-interaction --no-root")) == 0 - result = subprocess.run(shlex.split("poetry run deptry ."), capture_output=True, text=True) +def test_cli_ignore_flags(dir_with_venv_installed): + with run_within_dir(str(dir_with_venv_installed)): + result = subprocess.run( + shlex.split("poetry run deptry . -io isort -io pkginfo -io requests -im white -id black"), + capture_output=True, + text=True, + ) assert result.returncode == 0 -def test_cli_argument_overwrites_pyproject_toml_argument(tmp_path): - """ - The cli argument should overwrite the pyproject.toml argument. In project_with_obsolete, pyproject.toml specifies - to ignore 'pkginfo' and the obsolete dependencies are ['isort','toml']. - Verify that this is changed to ['isort','pkginfo'] if we run the command with `-io toml` (so cli argument overwrites the toml argument) - """ +def test_cli_skip_flags(dir_with_venv_installed): + with run_within_dir(str(dir_with_venv_installed)): + result = subprocess.run( + shlex.split("poetry run deptry . --skip-obsolete --skip-missing --skip-develop --skip-transitive"), + capture_output=True, + text=True, + ) + assert result.returncode == 0 - tmp_path_proj = tmp_path / "project_with_obsolete" - shutil.copytree("tests/data/projects/project_with_obsolete", tmp_path_proj) - with run_within_dir(str(tmp_path_proj)): - subprocess.check_call(shlex.split("poetry install --no-interaction --no-root")) == 0 - result = subprocess.run(shlex.split("poetry run deptry . -io toml"), capture_output=True, text=True) +def test_cli_exclude(dir_with_venv_installed): + with run_within_dir(str(dir_with_venv_installed)): + result = subprocess.run( + shlex.split("poetry run deptry . --exclude src/notebook.ipynb "), capture_output=True, text=True + ) assert result.returncode == 1 - assert "pyproject.toml contains obsolete dependencies:\n\n\tisort\n\tpkginfo\n\n" in result.stderr + assert "pyproject.toml contains obsolete dependencies:\n\n\tisort\n\trequests\n\ttoml\n\n" in result.stderr def test_cli_help(): diff --git a/tests/test_missing_dependencies_finder.py b/tests/test_missing_dependencies_finder.py index 982cdc48..470a2ba9 100644 --- a/tests/test_missing_dependencies_finder.py +++ b/tests/test_missing_dependencies_finder.py @@ -1,11 +1,11 @@ from deptry.dependency import Dependency from deptry.issue_finders.missing import MissingDependenciesFinder -from deptry.module import Module +from deptry.module import ModuleBuilder def test_simple(): dependencies = [] - modules = [Module("foobar", dependencies)] + modules = [ModuleBuilder("foobar", dependencies).build()] deps = MissingDependenciesFinder(imported_modules=modules, dependencies=dependencies).find() assert len(deps) == 1 assert deps[0] == "foobar" @@ -13,8 +13,8 @@ def test_simple(): def test_simple_with_ignore(): dependencies = [] - modules = [Module("foobar", dependencies)] + modules = [ModuleBuilder("foobar", dependencies).build()] deps = MissingDependenciesFinder( - imported_modules=modules, dependencies=dependencies, list_to_ignore=["foobar"] + imported_modules=modules, dependencies=dependencies, ignore_missing=["foobar"] ).find() assert len(deps) == 0 diff --git a/tests/test_module.py b/tests/test_module.py index bce58a8b..147a5728 100644 --- a/tests/test_module.py +++ b/tests/test_module.py @@ -1,9 +1,9 @@ from deptry.dependency import Dependency -from deptry.module import Module +from deptry.module import ModuleBuilder def test_simple_import(): - module = Module("click") + module = ModuleBuilder("click").build() assert module.package == "click" @@ -11,11 +11,11 @@ def test_top_level(): # Test if no error is raised, argument is accepted. dependency = Dependency("beautifulsoup4") dependency.top_levels = ["bs4"] - module = Module("bs4", [dependency]) + module = ModuleBuilder("bs4", [dependency]).build() assert module.package == None def test_stdlib(): - module = Module("sys") + module = ModuleBuilder("sys").build() assert module.package is None assert module.standard_library diff --git a/tests/test_obsolete_dependencies_finder.py b/tests/test_obsolete_dependencies_finder.py index f7904b29..8744f5b5 100644 --- a/tests/test_obsolete_dependencies_finder.py +++ b/tests/test_obsolete_dependencies_finder.py @@ -1,11 +1,11 @@ from deptry.dependency import Dependency from deptry.issue_finders.obsolete import ObsoleteDependenciesFinder -from deptry.module import Module +from deptry.module import ModuleBuilder def test_simple(): dependencies = [Dependency("click"), Dependency("toml")] - modules = [Module("click", dependencies)] + modules = [ModuleBuilder("click", dependencies).build()] deps = ObsoleteDependenciesFinder(imported_modules=modules, dependencies=dependencies).find() assert len(deps) == 1 assert deps[0] == "toml" @@ -13,9 +13,9 @@ def test_simple(): def test_simple_with_ignore(): dependencies = [Dependency("click"), Dependency("toml")] - modules = [Module("toml", dependencies)] + modules = [ModuleBuilder("toml", dependencies).build()] deps = ObsoleteDependenciesFinder( - imported_modules=modules, dependencies=dependencies, list_to_ignore=["click"] + imported_modules=modules, dependencies=dependencies, ignore_obsolete=["click"] ).find() assert len(deps) == 0 @@ -26,7 +26,7 @@ def test_top_level(): mpl_toolkits is in the top-level of matplotlib, so matplotlib should not be marked as an obsolete dependency. """ dependencies = [Dependency("matplotlib")] - modules = [Module("mpl_toolkits", dependencies)] + modules = [ModuleBuilder("mpl_toolkits", dependencies).build()] deps = ObsoleteDependenciesFinder(imported_modules=modules, dependencies=dependencies).find() assert len(deps) == 0 @@ -36,6 +36,6 @@ def test_without_top_level(): Test if packages without top-level information are correctly maked as obsolete """ dependencies = [Dependency("isort")] - modules = [Module("isort", dependencies)] + modules = [ModuleBuilder("isort", dependencies).build()] deps = ObsoleteDependenciesFinder(imported_modules=modules, dependencies=dependencies).find() assert len(deps) == 0 diff --git a/tests/test_transitive_dependencies_finder.py b/tests/test_transitive_dependencies_finder.py index 42e1397e..17e03b10 100644 --- a/tests/test_transitive_dependencies_finder.py +++ b/tests/test_transitive_dependencies_finder.py @@ -1,6 +1,6 @@ from deptry.dependency import Dependency from deptry.issue_finders.transitive import TransitiveDependenciesFinder -from deptry.module import Module +from deptry.module import ModuleBuilder def test_simple(): @@ -8,7 +8,7 @@ def test_simple(): matplotlib is in testing environment which requires pillow, so pillow should be found as transitive. """ dependencies = [] - modules = [Module("pillow", dependencies)] + modules = [ModuleBuilder("pillow", dependencies).build()] deps = TransitiveDependenciesFinder(imported_modules=modules, dependencies=dependencies).find() assert len(deps) == 1 assert deps[0] == "Pillow" @@ -16,8 +16,8 @@ def test_simple(): def test_simple_with_ignore(): dependencies = [] - modules = [Module("foobar", dependencies)] + modules = [ModuleBuilder("foobar", dependencies).build()] deps = TransitiveDependenciesFinder( - imported_modules=modules, dependencies=dependencies, list_to_ignore=["foobar"] + imported_modules=modules, dependencies=dependencies, ignore_transitive=["foobar"] ).find() assert len(deps) == 0 From 3ba75febb1e7230e95b4b09a421fad2ca898b5fd Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Thu, 8 Sep 2022 10:57:45 +0200 Subject: [PATCH 2/5] fixed unit tests, renamed misplaced to misplaced-dev, added docs --- README.md | 5 +-- deptry/cli.py | 26 +++++++------- deptry/config.py | 36 +++++++++---------- deptry/core.py | 22 ++++++------ .../{dev.py => misplaced_dev.py} | 13 +++---- docs/docs/index.md | 5 +-- docs/docs/pyproject-toml.md | 2 ++ docs/docs/usage.md | 18 +++++----- tests/test_cli.py | 2 +- tests/test_import_parser.py | 4 +-- tests/test_notebook_import_extractor.py | 6 ++-- tests/test_python_file_finder.py | 8 ++--- 12 files changed, 75 insertions(+), 72 deletions(-) rename deptry/issue_finders/{dev.py => misplaced_dev.py} (77%) diff --git a/README.md b/README.md index 3d84fd6a..96609abe 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,12 @@ --- -_deptry_ is a command line tool to check for issues with dependencies in a poetry managed Python project. It checks for fourtypes of issues: +_deptry_ is a command line tool to check for issues with dependencies in a poetry managed Python project. It checks for four types of issues: - Obsolete dependencies: Dependencies which are added to your project's dependencies, but which are not used within the codebase. -- Transitive dependencies: Packages from which code is imported, but the package (A) itself is not in your projects dependencies. Instead, another package (B) is in your list of dependencies, which depends on (A). Package (A) should be added to your project's list of dependencies. - Missing dependencies: Modules that are imported within your project, but no corresponding package is found in the environment. +- Transitive dependencies: Packages from which code is imported, but the package (A) itself is not in your projects dependencies. Instead, another package (B) is in your list of dependencies, which depends on (A). Package (A) should be added to your project's list of dependencies. +- Misplaced dependencies: Development dependencies that should be included as regular dependencies. _deptry_ detects these issue by scanning the imported modules within all Python files in a directory and it's subdirectories, and comparing those to the dependencies listed in _pyproject.toml_. diff --git a/deptry/cli.py b/deptry/cli.py index 5e53a751..c553df80 100644 --- a/deptry/cli.py +++ b/deptry/cli.py @@ -34,7 +34,7 @@ help="Boolean flag to specify if deptry should skip scanning the project for transitive dependencies.", ) @click.option( - "--skip-develop", + "--skip-misplaced-dev", is_flag=True, help="Boolean flag to specify if deptry should skip scanning the project for development dependencies that should be regular dependencies.", ) @@ -62,7 +62,7 @@ Can be used multiple times. For example; `deptry . -it foo -io bar`.""", ) @click.option( - "--ignore-develop", + "--ignore-misplaced-dev", "-id", multiple=True, help="""Modules that should never be marked as 'missing due to transitive' even though deptry determines them to be transitive. @@ -91,11 +91,11 @@ def deptry( ignore_obsolete: List[str], ignore_missing: List[str], ignore_transitive: List[str], - ignore_develop: List[str], + ignore_misplaced_dev: List[str], skip_obsolete: bool, skip_missing: bool, skip_transitive: bool, - skip_develop: bool, + skip_misplaced_dev: bool, exclude: List[str], ignore_notebooks: bool, version: bool, @@ -121,26 +121,26 @@ def deptry( ignore_obsolete=ignore_obsolete if ignore_obsolete else None, ignore_missing=ignore_missing if ignore_missing else None, ignore_transitive=ignore_transitive if ignore_transitive else None, - ignore_develop=ignore_develop if ignore_develop else None, + ignore_misplaced_dev=ignore_misplaced_dev if ignore_misplaced_dev else None, exclude=exclude if exclude else None, ignore_notebooks=ignore_notebooks if ignore_notebooks else None, skip_obsolete=skip_obsolete if skip_obsolete else None, skip_missing=skip_missing if skip_missing else None, skip_transitive=skip_transitive if skip_transitive else None, - skip_develop=skip_develop if skip_develop else None, + skip_misplaced_dev=skip_misplaced_dev if skip_misplaced_dev else None, ) result = Core( ignore_obsolete=config.ignore_obsolete, ignore_missing=config.ignore_missing, ignore_transitive=config.ignore_transitive, - ignore_develop=config.ignore_develop, + ignore_misplaced_dev=config.ignore_misplaced_dev, exclude=config.exclude, ignore_notebooks=config.ignore_notebooks, skip_obsolete=config.skip_obsolete, skip_missing=config.skip_missing, skip_transitive=config.skip_transitive, - skip_develop=config.skip_develop, + skip_misplaced_dev=config.skip_misplaced_dev, ).run() issue_found = False if not skip_obsolete and "obsolete" in result and result["obsolete"]: @@ -152,8 +152,8 @@ def deptry( if not skip_transitive and "transitive" in result and result["transitive"]: log_transitive_dependencies(result["transitive"]) issue_found = True - if not skip_develop and "develop" in result and result["develop"]: - log_develop_dependencies(result["develop"]) + if not skip_misplaced_dev and "misplaced_dev" in result and result["misplaced_dev"]: + log_misplaced_develop_dependencies(result["misplaced_dev"]) issue_found = True if issue_found: @@ -188,13 +188,13 @@ def log_transitive_dependencies(dependencies: List[str], sep="\n\t") -> None: logging.info("""They are currently imported but not specified directly as your project's dependencies.""") -def log_develop_dependencies(dependencies: List[str], sep="\n\t") -> None: +def log_misplaced_develop_dependencies(dependencies: List[str], sep="\n\t") -> None: logging.info("\n-----------------------------------------------------\n") logging.info(f"There are imported modules from development dependencies detected:\n{sep}{sep.join(dependencies)}\n") logging.info( """Consider moving them to `[tool.poetry.dependencies]` in pyproject.toml. If this is not correct and the -dependencies listed above are indeed development dependencies, it's likely that files were scanned that are used -for development. Run `deptry -v .` to see a list of scanned files.""" +dependencies listed above are indeed development dependencies, it's likely that files were scanned that are only used +for development purposes. Run `deptry -v .` to see a list of scanned files.""" ) diff --git a/deptry/config.py b/deptry/config.py index 3ee8a1bc..ad90e492 100644 --- a/deptry/config.py +++ b/deptry/config.py @@ -8,13 +8,13 @@ "ignore_obsolete": [], "ignore_missing": [], "ignore_transitive": [], - "ignore_develop": [], + "ignore_misplaced_dev": [], "exclude": [".venv", "tests"], "ignore_notebooks": False, "skip_obsolete": False, "skip_missing": False, "skip_transitive": False, - "skip_develop": False, + "skip_misplaced_dev": False, } @@ -30,11 +30,11 @@ def __init__( ignore_obsolete: Optional[List[str]], ignore_missing: Optional[List[str]], ignore_transitive: Optional[List[str]], - ignore_develop: Optional[List[str]], + ignore_misplaced_dev: Optional[List[str]], skip_obsolete: Optional[bool], skip_missing: Optional[bool], skip_transitive: Optional[bool], - skip_develop: Optional[bool], + skip_misplaced_dev: Optional[bool], exclude: Optional[List[str]], ignore_notebooks: Optional[bool], ) -> None: @@ -44,26 +44,26 @@ def __init__( ignore_obsolete=ignore_obsolete, ignore_missing=ignore_missing, ignore_transitive=ignore_transitive, - ignore_develop=ignore_develop, + ignore_misplaced_dev=ignore_misplaced_dev, exclude=exclude, ignore_notebooks=ignore_notebooks, skip_obsolete=skip_obsolete, skip_missing=skip_missing, skip_transitive=skip_transitive, - skip_develop=skip_develop, + skip_misplaced_dev=skip_misplaced_dev, ) def _set_defaults(self) -> None: self.ignore_obsolete = DEFAULTS["ignore_obsolete"] self.ignore_missing = DEFAULTS["ignore_missing"] self.ignore_transitive = DEFAULTS["ignore_transitive"] - self.ignore_develop = DEFAULTS["ignore_develop"] + self.ignore_misplaced_dev = DEFAULTS["ignore_misplaced_dev"] self.exclude = DEFAULTS["exclude"] self.ignore_notebooks = DEFAULTS["ignore_notebooks"] self.skip_obsolete = DEFAULTS["skip_obsolete"] self.skip_missing = DEFAULTS["skip_missing"] self.skip_transitive = DEFAULTS["skip_transitive"] - self.skip_develop = DEFAULTS["skip_develop"] + self.skip_misplaced_dev = DEFAULTS["skip_misplaced_dev"] def _override_config_with_pyproject_toml(self) -> None: pyproject_toml_config = self._read_configuration_from_pyproject_toml() @@ -71,11 +71,11 @@ def _override_config_with_pyproject_toml(self) -> None: self._override_with_toml_argument("ignore_obsolete", List[str], pyproject_toml_config) self._override_with_toml_argument("ignore_missing", List[str], pyproject_toml_config) self._override_with_toml_argument("ignore_transitive", List[str], pyproject_toml_config) - self._override_with_toml_argument("ignore_develop", List[str], pyproject_toml_config) + self._override_with_toml_argument("ignore_misplaced_dev", List[str], pyproject_toml_config) self._override_with_toml_argument("skip_missing", List[str], pyproject_toml_config) self._override_with_toml_argument("skip_obsolete", List[str], pyproject_toml_config) self._override_with_toml_argument("skip_transitive", List[str], pyproject_toml_config) - self._override_with_toml_argument("skip_develop", List[str], pyproject_toml_config) + self._override_with_toml_argument("skip_misplaced_dev", List[str], pyproject_toml_config) self._override_with_toml_argument("exclude", List[str], pyproject_toml_config) self._override_with_toml_argument("ignore_notebooks", List[str], pyproject_toml_config) @@ -102,13 +102,13 @@ def _override_config_with_cli_arguments( # noqa ignore_obsolete: Optional[List[str]], ignore_missing: Optional[List[str]], ignore_transitive: Optional[List[str]], - ignore_develop: Optional[List[str]], + ignore_misplaced_dev: Optional[List[str]], exclude: Optional[List[str]], ignore_notebooks: Optional[bool], skip_obsolete: Optional[bool], skip_missing: Optional[bool], skip_transitive: Optional[bool], - skip_develop: Optional[bool], + skip_misplaced_dev: Optional[bool], ) -> None: if ignore_obsolete: @@ -123,9 +123,9 @@ def _override_config_with_cli_arguments( # noqa self.ignore_transitive = ignore_transitive self._log_changed_by_command_line_argument("ignore_transitive", ignore_transitive) - if ignore_develop: - self.ignore_develop = ignore_develop - self._log_changed_by_command_line_argument("ignore_develop", ignore_develop) + if ignore_misplaced_dev: + self.ignore_misplaced_dev = ignore_misplaced_dev + self._log_changed_by_command_line_argument("ignore_misplaced_dev", ignore_misplaced_dev) if skip_obsolete: self.skip_obsolete = skip_obsolete @@ -139,9 +139,9 @@ def _override_config_with_cli_arguments( # noqa self.skip_transitive = skip_transitive self._log_changed_by_command_line_argument("skip_transitive", skip_transitive) - if skip_develop: - self.skip_develop = skip_develop - self._log_changed_by_command_line_argument("skip_develop", skip_develop) + if skip_misplaced_dev: + self.skip_misplaced_dev = skip_misplaced_dev + self._log_changed_by_command_line_argument("skip_misplaced_dev", skip_misplaced_dev) if exclude: self.exclude = exclude diff --git a/deptry/core.py b/deptry/core.py index a924afd1..39e8c0b1 100644 --- a/deptry/core.py +++ b/deptry/core.py @@ -4,7 +4,7 @@ from deptry.dependency_getter import DependencyGetter from deptry.import_parser import ImportParser -from deptry.issue_finders.dev import DevDependenciesFinder +from deptry.issue_finders.misplaced_dev import MisplacedDevDependenciesFinder from deptry.issue_finders.missing import MissingDependenciesFinder from deptry.issue_finders.obsolete import ObsoleteDependenciesFinder from deptry.issue_finders.transitive import TransitiveDependenciesFinder @@ -18,33 +18,33 @@ def __init__( ignore_obsolete: List[str], ignore_missing: List[str], ignore_transitive: List[str], - ignore_develop: List[str], + ignore_misplaced_dev: List[str], skip_obsolete: bool, skip_missing: bool, skip_transitive: bool, - skip_develop: bool, + skip_misplaced_dev: bool, exclude: List[str], ignore_notebooks: bool, ) -> None: self.ignore_obsolete = ignore_obsolete self.ignore_missing = ignore_missing self.ignore_transitive = ignore_transitive - self.ignore_develop = ignore_develop + self.ignore_misplaced_dev = ignore_misplaced_dev self.exclude = exclude self.ignore_notebooks = ignore_notebooks self.skip_obsolete = skip_obsolete self.skip_missing = skip_missing self.skip_transitive = skip_transitive - self.skip_develop = skip_develop + self.skip_misplaced_dev = skip_misplaced_dev logging.debug("Running with the following configuration:") logging.debug(f"ignore_obsolete: {ignore_obsolete}") logging.debug(f"ignore_missing: {ignore_missing}") logging.debug(f"ignore_transitive: {ignore_transitive}") - logging.debug(f"ignore_develop: {ignore_develop}") + logging.debug(f"ignore_misplaced_dev: {ignore_misplaced_dev}") logging.debug(f"skip_obsolete: {skip_obsolete}") logging.debug(f"skip_missing: {skip_missing}") logging.debug(f"skip_transitive: {skip_transitive}") - logging.debug(f"skip_develop {skip_develop}") + logging.debug(f"skip_misplaced_dev {skip_misplaced_dev}") logging.debug(f"exclude: {exclude}") logging.debug(f"ignore_notebooks: {ignore_notebooks}\n") @@ -74,9 +74,11 @@ def run(self) -> Dict: result["transitive"] = TransitiveDependenciesFinder( imported_modules=imported_modules, dependencies=dependencies, ignore_transitive=self.ignore_transitive ).find() - if not self.skip_develop: - result["develop"] = DevDependenciesFinder( - imported_modules=imported_modules, dependencies=dependencies, ignore_develop=self.ignore_develop + if not self.skip_misplaced_dev: + result["misplaced_dev"] = MisplacedDevDependenciesFinder( + imported_modules=imported_modules, + dependencies=dependencies, + ignore_misplaced_dev=self.ignore_misplaced_dev, ).find() return result diff --git a/deptry/issue_finders/dev.py b/deptry/issue_finders/misplaced_dev.py similarity index 77% rename from deptry/issue_finders/dev.py rename to deptry/issue_finders/misplaced_dev.py index 97944544..2d1e94df 100644 --- a/deptry/issue_finders/dev.py +++ b/deptry/issue_finders/misplaced_dev.py @@ -5,17 +5,18 @@ from deptry.module import Module -class DevDependenciesFinder: +class MisplacedDevDependenciesFinder: """ - Given a list of imported modules and a list of project dependencies, determine which ones are transitive. + Given a list of imported modules and a list of project dependencies, determine which development dependencies + should be regular dependencies. """ def __init__( - self, imported_modules: List[Module], dependencies: List[Dependency], ignore_develop: List[str] = [] + self, imported_modules: List[Module], dependencies: List[Dependency], ignore_misplaced_dev: List[str] = [] ) -> None: self.imported_modules = imported_modules self.dependencies = dependencies - self.ignore_develop = ignore_develop + self.ignore_misplaced_dev = ignore_misplaced_dev def find(self) -> List[str]: logging.debug("\nScanning for incorrect development dependencies...") @@ -28,9 +29,9 @@ def find(self) -> List[str]: def _is_development_dependency(self, module: Module) -> bool: if module.dev_dependency: - if module.name in self.ignore_develop: + if module.name in self.ignore_misplaced_dev: logging.debug(f"Module '{module.package}' found to be a development dependency, but ignoring.") else: - logging.debug(f"Dependency '{module.package}' marked as a transitive dependency.") + logging.debug(f"Dependency '{module.package}' marked as a misplaced development dependency.") return True return False diff --git a/docs/docs/index.md b/docs/docs/index.md index d0c18816..a6e268fd 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -9,11 +9,12 @@ [![License](https://img.shields.io/github/license/fpgmaas/deptry)](https://img.shields.io/github/license/fpgmaas/deptry) --- -_deptry_ is a command line tool to check for issues with dependencies in a poetry managed Python project. It checks for three types of issues: +_deptry_ is a command line tool to check for issues with dependencies in a poetry managed Python project. It checks for four types of issues: - Obsolete dependencies: Dependencies which are added to your project's dependencies, but which are not used within the codebase. -- Transitive dependencies: Packages from which code is imported, but the package (A) itself is not in your projects dependencies. Instead, another package (B) is in your list of dependencies, which depends on (A). Package (A) should be added to your project's list of dependencies. - Missing dependencies: Modules that are imported within your project, but no corresponding package is found in the environment. +- Transitive dependencies: Packages from which code is imported, but the package (A) itself is not in your projects dependencies. Instead, another package (B) is in your list of dependencies, which depends on (A). Package (A) should be added to your project's list of dependencies. +- Misplaced dependencies: Development dependencies that should be included as regular dependencies. _deptry_ detects these issue by scanning the imported modules within all Python files in a directory and it's subdirectories, and comparing those to the dependencies listed in `pyproject.toml`. diff --git a/docs/docs/pyproject-toml.md b/docs/docs/pyproject-toml.md index 476e9d4c..b037bd75 100644 --- a/docs/docs/pyproject-toml.md +++ b/docs/docs/pyproject-toml.md @@ -8,10 +8,12 @@ _deptry_ can be configured by adding a `[tool.deptry]` section to _pyproject.tom - `ignore_notebooks`: `bool` - `ignore_obsolete`: `List` - `ignore_missing`: `List` +- `ignore_misplaced_dev`: `List` - `ignore_transitive`: `List` - `skip_obsolete`: `bool` - `skip_missing`: `bool` - `skip_transitive`: `bool` +- `skip_misplaced_dev`: `bool` An example of such a section is given below. diff --git a/docs/docs/usage.md b/docs/docs/usage.md index 0141fd40..af006102 100644 --- a/docs/docs/usage.md +++ b/docs/docs/usage.md @@ -4,7 +4,6 @@ In order to check for obsolete imports, _deptry_ requires a _pyproject.toml_ file to be present in the directory passed as the first argument, and it requires the corresponding environment to be activated. - ## Configuration _deptry_ can be configured with command line arguments or by adding a `[tool.deptry]` section to _pyproject.toml_. Explanation for the command line arguments can @@ -27,31 +26,32 @@ To show more details about the scanned python files, the imported modules found, deptry . -v ``` -## Ignore directories +## Exclude files and directories -_deptry_ scans the working directory and it's subdirectories recursively for `.py` and `.ipynb` files to scan for import statements. By default, -the `.venv` directory is ignored. To ignore other directories, use the `--exclude` flag. Note that this overwrites the default, so to ignore -both the `.venv` directory and another directory, use the flag twice: +_deptry_ scans the working directory and it's subdirectories recursively for `.py` and `.ipynb` files to scan for import statements. Any files solely used for development purposes should not be scanned. +By default, the `.venv` and the `tests` directory are excluded. To ignore other directories, use the `--exclude` flag. Note that this overwrites the default, so to ignore +both the `.venv` and `tests` directories and another directory, use the flag thrice: ```sh deptry . --exclude .venv --exclude tests --exclude other_directory ``` -## Skip checks for obsolete, transitive or missing dependencies. +## Skip checks for obsolete, transitive or misplaced development dependencies. -Checks for obsolete, transitive or missing dependencies can be skipped by using the corresponding flags: +Checks for obsolete, transitive, missing, or misplaced development dependencies can be skipped by using the corresponding flags: ```sh deptry . --skip-obsolete deptry . --skip-transitive deptry . --skip-missing +deptry . --skip-misplaced-dev ``` ## Ignore dependencies Sometimes, you might want _deptry_ to ignore certain dependencies in certain checks, for example when you have an module that is used but not imported. -Dependencies can be ignored for each check separately with the `--ignore-obsolete`, `--ignore-transitive`, or `--ignore-missing` flag, or with their -respective abbreviations `-io`, `-it`, `-im`. Each argument can be used multiple times to ignore multiple dependencies. Some examples: +Dependencies can be ignored for each check separately with the `--ignore-obsolete`, `--ignore-transitive`, `--ignore-missing` or `--ignore-misplaced-dev` flag, or with their +respective abbreviations `-io`, `-it`, `-im` and `-id`. Each argument can be used multiple times to ignore multiple dependencies. Some examples: The following will ignore dependency foo while checking for obsolete dependencies and will ignore module bar while checking for missing dependencies diff --git a/tests/test_cli.py b/tests/test_cli.py index df4150c9..9868995f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -45,7 +45,7 @@ def test_cli_ignore_flags(dir_with_venv_installed): def test_cli_skip_flags(dir_with_venv_installed): with run_within_dir(str(dir_with_venv_installed)): result = subprocess.run( - shlex.split("poetry run deptry . --skip-obsolete --skip-missing --skip-develop --skip-transitive"), + shlex.split("poetry run deptry . --skip-obsolete --skip-missing --skip-misplaced-dev --skip-transitive"), capture_output=True, text=True, ) diff --git a/tests/test_import_parser.py b/tests/test_import_parser.py index 015dbae2..f201fd93 100644 --- a/tests/test_import_parser.py +++ b/tests/test_import_parser.py @@ -11,9 +11,9 @@ def test_import_parser_py(): def test_import_parser_ipynb(): imported_modules = ImportParser().get_imported_modules_from_file( - Path("tests/data/projects/project_with_obsolete/src/notebook.ipynb") + Path("tests/data/example_project/src/notebook.ipynb") ) - assert set(imported_modules) == set(["click", "requests", "urllib3", "toml"]) + assert set(imported_modules) == set(["click", "urllib3", "toml"]) def test_import_parser_ifelse(): diff --git a/tests/test_notebook_import_extractor.py b/tests/test_notebook_import_extractor.py index dd5de8d7..984b2420 100644 --- a/tests/test_notebook_import_extractor.py +++ b/tests/test_notebook_import_extractor.py @@ -8,7 +8,7 @@ def test_convert_notebook(): - imports = NotebookImportExtractor().extract("tests/data/projects/project_with_obsolete/src/notebook.ipynb") + imports = NotebookImportExtractor().extract("tests/data/example_project/src/notebook.ipynb") assert "import click" in imports[0] - assert "import requests as req" in imports[1] - assert len(imports) == 4 + assert "from urllib3 import contrib" in imports[1] + assert len(imports) == 3 diff --git a/tests/test_python_file_finder.py b/tests/test_python_file_finder.py index 3c2784fa..5b4e05b6 100644 --- a/tests/test_python_file_finder.py +++ b/tests/test_python_file_finder.py @@ -14,9 +14,7 @@ def test_find_only_py_files(): """ Should only find src/main.py """ - files = PythonFileFinder(ignore_notebooks=True).get_all_python_files_in( - Path("tests/data/projects/project_with_obsolete") - ) + files = PythonFileFinder(ignore_notebooks=True).get_all_python_files_in(Path("tests/data/example_project")) assert len(files) == 1 assert "main.py" in (str(files[0])) @@ -25,9 +23,7 @@ def test_find_py_and_ipynb_files(): """ Should find src/main.py and src/notebook.ipynb """ - files = PythonFileFinder(ignore_notebooks=False).get_all_python_files_in( - Path("tests/data/projects/project_with_obsolete") - ) + files = PythonFileFinder(ignore_notebooks=False).get_all_python_files_in(Path("tests/data/example_project")) assert len(files) == 2 assert any(["main.py" in str(x) for x in files]) assert any(["notebook.ipynb" in str(x) for x in files]) From a7c4cb32ee9c03314df74ce6cc7a7d240d897330 Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Thu, 8 Sep 2022 11:47:26 +0200 Subject: [PATCH 3/5] fixed python3.7 unit test --- tests/test_cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 9868995f..56c6fcfe 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,8 +9,8 @@ @pytest.fixture(scope="session") def dir_with_venv_installed(tmp_path_factory): - tmp_path_proj = tmp_path_factory.getbasetemp() - shutil.copytree("tests/data/example_project", str(tmp_path_proj), dirs_exist_ok=True) + tmp_path_proj = tmp_path_factory.getbasetemp() / 'example_project' + shutil.copytree("tests/data/example_project", str(tmp_path_proj)) with run_within_dir(str(tmp_path_proj)): subprocess.check_call(shlex.split("poetry install --no-interaction --no-root")) == 0 return tmp_path_proj From 64fbd89ec4f5916c8a2f4d52b0f41ec76b6e39f2 Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Thu, 8 Sep 2022 11:51:19 +0200 Subject: [PATCH 4/5] formatting --- tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 56c6fcfe..76c87e59 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -9,7 +9,7 @@ @pytest.fixture(scope="session") def dir_with_venv_installed(tmp_path_factory): - tmp_path_proj = tmp_path_factory.getbasetemp() / 'example_project' + tmp_path_proj = tmp_path_factory.getbasetemp() / "example_project" shutil.copytree("tests/data/example_project", str(tmp_path_proj)) with run_within_dir(str(tmp_path_proj)): subprocess.check_call(shlex.split("poetry install --no-interaction --no-root")) == 0 From 943049ee852dc40c354dd666f65243bbae57ee8e Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Thu, 8 Sep 2022 12:16:54 +0200 Subject: [PATCH 5/5] documentation --- deptry/cli.py | 11 ++++++----- docs/docs/index.md | 6 +++--- docs/docs/usage.md | 19 +++++++++++-------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/deptry/cli.py b/deptry/cli.py index c553df80..c64835f4 100644 --- a/deptry/cli.py +++ b/deptry/cli.py @@ -51,28 +51,29 @@ "--ignore-missing", "-im", multiple=True, - help="""Modules that should never be marked as having missing dependencies, even if the matching package for the import statement cannot be found. + help="""Modules that should never be marked as missing dependencies, even if the matching package for the import statement cannot be found. Can be used multiple times. For example; `deptry . -io foo -io bar`.""", ) @click.option( "--ignore-transitive", "-it", multiple=True, - help="""Modules that should never be marked as 'missing due to transitive' even though deptry determines them to be transitive. + help="""Dependencies that should never be marked as an issue due to it being a transitive dependency, even though deptry determines them to be transitive. Can be used multiple times. For example; `deptry . -it foo -io bar`.""", ) @click.option( "--ignore-misplaced-dev", "-id", multiple=True, - help="""Modules that should never be marked as 'missing due to transitive' even though deptry determines them to be transitive. - Can be used multiple times. For example; `deptry . -it foo -io bar`.""", + help="""Modules that should never be marked as a misplaced development dependency, even though it seems to not be used solely for development purposes. + Can be used multiple times. For example; `deptry . -id foo -id bar`.""", ) @click.option( "--exclude", multiple=True, help="""Directories or files in which .py files should not be scanned for imports to determine if there are dependency issues. - Defaults to ['venv','tests']. Specify multiple directories by using this flag twice, e.g. `-id .venv -id tests -id other_dir`.""", + Defaults to ['venv','tests']. Specify multiple directories by using this flag multiple times, e.g. `--exclude .venv --exclude tests + --exclude other_dir`.""", ) @click.option( "--ignore-notebooks", diff --git a/docs/docs/index.md b/docs/docs/index.md index a6e268fd..26388cea 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -17,7 +17,7 @@ _deptry_ is a command line tool to check for issues with dependencies in a poetr - Misplaced dependencies: Development dependencies that should be included as regular dependencies. _deptry_ detects these issue by scanning the imported modules within all Python files in -a directory and it's subdirectories, and comparing those to the dependencies listed in `pyproject.toml`. +a directory and it's subdirectories, and comparing those to the dependencies listed in _pyproject.toml_. ## Quickstart @@ -37,11 +37,11 @@ poetry add --dev deptry ### Prerequisites -In order to check for obsolete imports, _deptry_ requires a _pyproject.toml_ file to be present in the directory passed as the first argument, and it requires the corresponding environment to be activated. +In order to check for dependency issues, _deptry_ requires a _pyproject.toml_ file to be present in the directory passed as the first argument, and it requires the corresponding environment to be activated. ### Usage -To scan your project for obsolete imports, run +To scan your project for dependency issues, run ```sh deptry . diff --git a/docs/docs/usage.md b/docs/docs/usage.md index af006102..6360c989 100644 --- a/docs/docs/usage.md +++ b/docs/docs/usage.md @@ -18,22 +18,25 @@ _deptry_ can be run with deptry . ``` -## Increased verbosity +To determine issues with imported modules and dependencies, _deptry_ will scan the working directory and it's subdirectories recursively for `.py` and `.ipynb` files, so it can +extract the imported modules from those files. Any files solely used for development purposes, such as files used for unit testing, should not be scanned. By default, the directories +`.venv` and `tests` are excluded. -To show more details about the scanned python files, the imported modules found, and how deptry determined which dependencies are obsolete, add the `-v` flag: +## Excluding files and directories + +To ignore other directories than the default `.venv` and `tests`, use the `--exclude` flag. Note that this overwrites the defaults, so to ignore +both the `.venv` and `tests` directories and another directory, use the flag thrice: ```sh -deptry . -v +deptry . --exclude .venv --exclude tests --exclude other_directory ``` -## Exclude files and directories +## Increased verbosity -_deptry_ scans the working directory and it's subdirectories recursively for `.py` and `.ipynb` files to scan for import statements. Any files solely used for development purposes should not be scanned. -By default, the `.venv` and the `tests` directory are excluded. To ignore other directories, use the `--exclude` flag. Note that this overwrites the default, so to ignore -both the `.venv` and `tests` directories and another directory, use the flag thrice: +To show more details about the scanned python files, the imported modules found, and how deptry determined which dependencies are obsolete, add the `-v` flag: ```sh -deptry . --exclude .venv --exclude tests --exclude other_directory +deptry . -v ``` ## Skip checks for obsolete, transitive or misplaced development dependencies.