Added support for requirement.txt (#87)

* Added support for requirements.txt (#84)
* Solved bug with relative imports being marked as missing (#86)

README.md

@@ -9,9 +9,12 @@
 
 ---
 
-_deptry_ is a command line tool to check for issues with dependencies in a poetry managed Python project, such as obsolete or missing dependencies.
+_deptry_ is a command line tool to check for issues with dependencies in a Python project, such as obsolete or missing dependencies. It supports the following types of projects:
 
-Dependency issues are detected by scanning for imported modules within all Python files in a directory and its subdirectories, and comparing those to the dependencies listed in pyproject.toml.
+- Projects that use [Poetry](https://python-poetry.org/) and a corresponding _pyproject.toml_ file
+- Projects that use a _requirements.txt_ file according to the [pip](https://pip.pypa.io/en/stable/user_guide/) standards
+
+Dependency issues are detected by scanning for imported modules within all Python files in a directory and its subdirectories, and comparing those to the dependencies listed in the project's requirements.
 
 ---
 
@@ -25,22 +28,22 @@ Dependency issues are detected by scanning for imported modules within all Pytho
 
 _deptry_ can be added to your project with 
 
-```
+```shell
 poetry add --group dev deptry
 ```
 
-or for older versions of poetry:
+or with 
 
 ```
-poetry add --dev deptry
+pip install deptry
 ```
 
 > **Warning**
 > _deptry_ is still in the early phases of development. For one-off testing of your project's dependencies, this is no issue. However, if you plan to use _deptry_ in a CI/CD pipeline, it is a good idea to pin the version.
 
 ### Prerequisites
 
-In order to check for obsolete imports, _deptry_ requires a _pyproject.toml_ file to be present in the directory passed as the first argument, and it requires the corresponding environment to be activated.
+_deptry_ should be run withing the root directory of the project to be scanned, and the proejct should be running in its own dedicated virtual environment.
 
 ### Usage
 
@@ -50,8 +53,8 @@ To scan your project for obsolete imports, run
 deptry .
 ```
 
-__deptry__ can be configured by using additional command line arguments, or 
-by adding a `[tool.deptry]` section in __pyproject.toml__.
+_deptry_ can be configured by using additional command line arguments, or 
+by adding a `[tool.deptry]` section in _pyproject.toml_.
 
 For more information, see the [documentation](https://fpgmaas.github.io/deptry/).
 

deptry/cli.py

@@ -44,7 +44,7 @@
     "-io",
     type=click.STRING,
     help="""
-    Comma-separated list of dependencies listed in pyproject.toml that should never be marked as obsolete, even if they are not imported in any of the files scanned.
+    Comma-separated list of dependencies that should never be marked as obsolete, even if they are not imported in any of the files scanned.
     For example; `deptry . --ignore-obsolete foo,bar`.
     """,
     default=DEFAULTS["ignore_obsolete"],
@@ -81,7 +81,7 @@
     "-e",
     type=click.STRING,
     help="""Comma-separated list of directories or files in which .py files should not be scanned for imports to determine if there are dependency issues.
-    For example: `deptry . --exclude .venv,tests,foo,bar.py`
+    For example: `deptry . --exclude venv,.venv,tests,foo,bar.py`
     """,
     default=DEFAULTS["exclude"],
     show_default=True,
@@ -106,6 +106,24 @@
     is_flag=True,
     help="Display the current version and exit.",
 )
+@click.option(
+    "--requirements-txt",
+    "-rt",
+    type=click.STRING,
+    help="""A .txt files with the project's dependencies. If a file called pyproject.toml with a [tool.poetry.dependencies] section is found, this argument is ignored
+    and the dependencies are extracted from the pyproject.toml file instead. Example use: `deptry . --requirements-txt req/prod.txt`""",
+    default=DEFAULTS["requirements_txt"],
+    show_default=True,
+)
+@click.option(
+    "--requirements-txt-dev",
+    "-rtd",
+    type=click.STRING,
+    help=""".txt files to scan for additional development dependencies. If a file called pyproject.toml with a [tool.poetry.dependencies] section is found, this argument is ignored
+    and the dependencies are extracted from the pyproject.toml file instead. Can be multiple e.g. `deptry . --requirements-txt-dev req/dev.txt,req/test.txt`""",
+    default=DEFAULTS["requirements_txt_dev"],
+    show_default=True,
+)
 def deptry(
     directory: pathlib.Path,
     verbose: bool,
@@ -120,6 +138,8 @@ def deptry(
     exclude: List[str],
     extend_exclude: List[str],
     ignore_notebooks: bool,
+    requirements_txt: str,
+    requirements_txt_dev: str,
     version: bool,
 ) -> None:
 
@@ -151,6 +171,8 @@ def deptry(
             skip_missing=skip_missing,
             skip_transitive=skip_transitive,
             skip_misplaced_dev=skip_misplaced_dev,
+            requirements_txt=requirements_txt,
+            requirements_txt_dev=requirements_txt_dev,
         )
 
         result = Core(
@@ -165,6 +187,8 @@ def deptry(
             skip_missing=config.skip_missing,
             skip_transitive=config.skip_transitive,
             skip_misplaced_dev=config.skip_misplaced_dev,
+            requirements_txt=config.requirements_txt,
+            requirements_txt_dev=config.requirements_txt_dev,
         ).run()
         issue_found = False
         if not skip_obsolete and "obsolete" in result and result["obsolete"]:
@@ -191,7 +215,7 @@ def deptry(
 
 def log_obsolete_dependencies(dependencies: List[str], sep="\n\t") -> None:
     logging.info("\n-----------------------------------------------------\n")
-    logging.info(f"pyproject.toml contains obsolete dependencies:\n{sep}{sep.join(sorted(dependencies))}\n")
+    logging.info(f"The project contains obsolete dependencies:\n{sep}{sep.join(sorted(dependencies))}\n")
     logging.info(
         """Consider removing them from your projects dependencies. If a package is used for development purposes, you should add
 it to your development dependencies instead."""
@@ -200,7 +224,9 @@ def log_obsolete_dependencies(dependencies: List[str], sep="\n\t") -> None:
 
 def log_missing_dependencies(dependencies: List[str], sep="\n\t") -> None:
     logging.info("\n-----------------------------------------------------\n")
-    logging.info(f"There are dependencies missing from pyproject.toml:\n{sep}{sep.join(sorted(dependencies))}\n")
+    logging.info(
+        f"There are dependencies missing from the project's list of dependencies:\n{sep}{sep.join(sorted(dependencies))}\n"
+    )
     logging.info("""Consider adding them to your project's dependencies. """)
 
 
@@ -218,7 +244,7 @@ def log_misplaced_develop_dependencies(dependencies: List[str], sep="\n\t") -> N
         f"There are imported modules from development dependencies detected:\n{sep}{sep.join(sorted(dependencies))}\n"
     )
     logging.info(
-        """Consider moving them to `[tool.poetry.dependencies]` in pyproject.toml. If this is not correct and the
+        """Consider moving them to your project's 'regular' dependencies. If this is not correct and the
 dependencies listed above are indeed development dependencies, it's likely that files were scanned that are only used
 for development purposes. Run `deptry -v .` to see a list of scanned files."""
     )
@@ -242,7 +268,7 @@ def log_additional_info():
   'your-dependency'
 ]
 exclude = [
-  '.venv', 'tests', 'docs'
+  'venv','.venv', 'tests', 'docs'
 ]
 ```
 

deptry/cli_defaults.py

@@ -3,11 +3,13 @@
     "ignore_missing": "",
     "ignore_transitive": "",
     "ignore_misplaced_dev": "",
-    "exclude": ".venv,tests",
+    "exclude": "venv,.venv,tests",
     "extend_exclude": "",
     "ignore_notebooks": False,
     "skip_obsolete": False,
     "skip_missing": False,
     "skip_transitive": False,
     "skip_misplaced_dev": False,
+    "requirements_txt": "requirements.txt",
+    "requirements_txt_dev": "dev-requirements.txt,requirements-dev.txt",
 }

deptry/config.py

@@ -1,4 +1,5 @@
 import logging
+import os
 from typing import Any, Dict
 
 from deptry.cli_defaults import DEFAULTS
@@ -25,6 +26,8 @@ def __init__(
         exclude: str = None,
         extend_exclude: str = None,
         ignore_notebooks: bool = None,
+        requirements_txt: str = None,
+        requirements_txt_dev: str = None,
     ) -> None:
 
         self.pyproject_data = self._read_configuration_from_pyproject_toml()
@@ -39,6 +42,8 @@ def __init__(
         self._set_bool_config("skip_transitive", skip_transitive)
         self._set_bool_config("skip_misplaced_dev", skip_misplaced_dev)
         self._set_bool_config("ignore_notebooks", ignore_notebooks)
+        self._set_string_config("requirements_txt", requirements_txt)
+        self._set_string_to_list_config("requirements_txt_dev", requirements_txt_dev)
 
     def _set_string_to_list_config(self, attribute: str, cli_value: str):
         """
@@ -56,18 +61,34 @@ def _set_bool_config(self, attribute: str, cli_value: str):
         self._override_with_toml_argument(attribute)
         self._override_with_cli_argument_boolean(attribute, cli_value)
 
+    def _set_string_config(self, attribute: str, cli_value: str):
+        """
+        Set configuration for arguments that are supplied as strings in the CLI, but should be converted to a list.
+        """
+        self._set_default_string(attribute)
+        self._override_with_toml_argument(attribute)
+        self._override_with_cli_argument_string(attribute, cli_value)
+
     def _set_default_string_to_list(self, attribute: str):
         setattr(self, attribute, self._comma_separated_string_to_list(DEFAULTS[attribute]))
 
     def _set_default_boolean(self, attribute: str):
         setattr(self, attribute, DEFAULTS[attribute])
 
+    def _set_default_string(self, attribute: str):
+        setattr(self, attribute, DEFAULTS[attribute])
+
     def _override_with_cli_argument_string_to_list(self, attribute, value):
         if value and not value == DEFAULTS[attribute]:
             value_as_list = self._comma_separated_string_to_list(value)
             self._log_changed_by_command_line_argument(attribute, value_as_list)
             setattr(self, attribute, value_as_list)
 
+    def _override_with_cli_argument_string(self, attribute, value):
+        if value and not value == DEFAULTS[attribute]:
+            self._log_changed_by_command_line_argument(attribute, value)
+            setattr(self, attribute, value)
+
     def _override_with_cli_argument_boolean(self, attribute, value):
         if value and not value == DEFAULTS[attribute]:
             self._log_changed_by_command_line_argument(attribute, value)
@@ -83,7 +104,11 @@ def _override_with_toml_argument(self, argument: str) -> None:
             self._log_changed_by_pyproject_toml(argument, value)
 
     def _read_configuration_from_pyproject_toml(self) -> Dict:
-        pyproject_data = load_pyproject_toml()
+        if self._pyproject_toml_exists():
+            pyproject_data = load_pyproject_toml()
+        else:
+            logging.debug("No pyproject.toml file to read configuration from.")
+            return None
         try:
             return pyproject_data["tool"]["deptry"]
         except KeyError:  # noqa
@@ -104,3 +129,9 @@ def _comma_separated_string_to_list(string: str):
             return string.split(",")
         else:
             return []
+
+    @staticmethod
+    def _pyproject_toml_exists() -> bool:
+        if "pyproject.toml" in os.listdir():
+            return True
+        return False

deptry/core.py

@@ -2,13 +2,16 @@
 from pathlib import Path
 from typing import Dict, List
 
-from deptry.dependency_getter import DependencyGetter
+from deptry.dependency import Dependency
+from deptry.dependency_getter.pyproject_toml import PyprojectTomlDependencyGetter
+from deptry.dependency_getter.requirements_txt import RequirementsTxtDependencyGetter
+from deptry.dependency_specification_detector import DependencySpecificationDetector
 from deptry.import_parser import ImportParser
 from deptry.issue_finders.misplaced_dev import MisplacedDevDependenciesFinder
 from deptry.issue_finders.missing import MissingDependenciesFinder
 from deptry.issue_finders.obsolete import ObsoleteDependenciesFinder
 from deptry.issue_finders.transitive import TransitiveDependenciesFinder
-from deptry.module import ModuleBuilder
+from deptry.module import Module, ModuleBuilder
 from deptry.python_file_finder import PythonFileFinder
 
 
@@ -26,6 +29,8 @@ def __init__(
         exclude: List[str],
         extend_exclude: List[str],
         ignore_notebooks: bool,
+        requirements_txt: str,
+        requirements_txt_dev: List[str],
     ) -> None:
         self.ignore_obsolete = ignore_obsolete
         self.ignore_missing = ignore_missing
@@ -38,13 +43,15 @@ def __init__(
         self.skip_missing = skip_missing
         self.skip_transitive = skip_transitive
         self.skip_misplaced_dev = skip_misplaced_dev
+        self.requirements_txt = requirements_txt
+        self.requirements_txt_dev = requirements_txt_dev
 
     def run(self) -> Dict:
 
         self._log_config()
 
-        dependencies = DependencyGetter().get()
-        dev_dependencies = DependencyGetter(dev=True).get()
+        dependency_management_format = DependencySpecificationDetector(requirements_txt=self.requirements_txt).detect()
+        dependencies, dev_dependencies = self._get_dependencies(dependency_management_format)
 
         all_python_files = PythonFileFinder(
             exclude=self.exclude + self.extend_exclude, ignore_notebooks=self.ignore_notebooks
@@ -54,6 +61,11 @@ def run(self) -> Dict:
         imported_modules = [ModuleBuilder(mod, dependencies, dev_dependencies).build() for mod in imported_modules]
         imported_modules = [mod for mod in imported_modules if not mod.standard_library]
 
+        issues = self._find_issues(imported_modules, dependencies)
+
+        return issues
+
+    def _find_issues(self, imported_modules: List[Module], dependencies: List[Dependency]):
         result = {}
         if not self.skip_obsolete:
             result["obsolete"] = ObsoleteDependenciesFinder(
@@ -73,11 +85,25 @@ def run(self) -> Dict:
                 dependencies=dependencies,
                 ignore_misplaced_dev=self.ignore_misplaced_dev,
             ).find()
-
         return result
 
+    def _get_dependencies(self, dependency_management_format: str):
+        if dependency_management_format == "pyproject_toml":
+            dependencies = PyprojectTomlDependencyGetter().get()
+            dev_dependencies = PyprojectTomlDependencyGetter(dev=True).get()
+        elif dependency_management_format == "requirements_txt":
+            dependencies = RequirementsTxtDependencyGetter(requirements_txt=self.requirements_txt).get()
+            dev_dependencies = RequirementsTxtDependencyGetter(
+                dev=True, requirements_txt_dev=self.requirements_txt_dev
+            ).get()
+        else:
+            raise ValueError(
+                "Incorrect dependency manage format. Only pyproject.toml and requirements.txt are supported."
+            )
+        return dependencies, dev_dependencies
+
     def _log_config(self):
         logging.debug("Running with the following configuration:")
         for key, value in vars(self).items():
             logging.debug(f"{key}: {value}")
-        logging.debug("\n")
+        logging.debug("")

deptry/dependency.py

@@ -17,7 +17,7 @@ class Dependency:
     def __init__(self, name: str, conditional: bool = False, optional: bool = False) -> None:
         """
         Args:
-            name: Name of the dependency, as shown in pyproject.toml
+            name: Name of the dependency, as shown in pyproject.toml or requirements.txt
             conditional: boolean to indicate if the dependency is conditional, e.g. 'importlib-metadata': {'version': '*', 'python': '<=3.7'}
         """
         self.name = name

deptry/dependency_getter.py → deptry/dependency_getter/pyproject_toml.py

@@ -5,7 +5,7 @@
 from deptry.utils import load_pyproject_toml
 
 
-class DependencyGetter:
+class PyprojectTomlDependencyGetter:
     """
     Class to get a project's list of dependencies from pyproject.toml.