diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 24959e454d..129747af13 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -339,6 +339,7 @@ contributors: - kdestin <101366538+kdestin@users.noreply.github.com> - jaydesl <35102795+jaydesl@users.noreply.github.com> - jab +- gremat <50012463+gremat@users.noreply.github.com> - gracejiang16 <70730457+gracejiang16@users.noreply.github.com> - glmdgrielson <32415403+glmdgrielson@users.noreply.github.com> - glegoux diff --git a/doc/user_guide/configuration/all-options.rst b/doc/user_guide/configuration/all-options.rst index eb7d72f2a3..d2d74308ff 100644 --- a/doc/user_guide/configuration/all-options.rst +++ b/doc/user_guide/configuration/all-options.rst @@ -181,6 +181,13 @@ Standard Checkers **Default:** ``sys.version_info[:2]`` +--pythonpath +"""""""""""" +*Add paths to sys.path. Supports globbing patterns. Paths are absolute or relative to the current working directory.* + +**Default:** ``()`` + + --recursive """"""""""" *Discover python modules and packages in the file system subtree.* @@ -282,6 +289,8 @@ Standard Checkers py-version = "sys.version_info[:2]" + pythonpath = [] + recursive = false reports = false diff --git a/doc/user_guide/usage/run.rst b/doc/user_guide/usage/run.rst index e7462a8f5d..a47ab8ae0d 100644 --- a/doc/user_guide/usage/run.rst +++ b/doc/user_guide/usage/run.rst @@ -39,7 +39,7 @@ you can give it a file name if it's possible to guess a module name from the fil path using the python path. Some examples: ``pylint mymodule.py`` should always work since the current working -directory is automatically added on top of the python path +directory is automatically added on top of the python path. ``pylint directory/mymodule.py`` will work if: ``directory`` is a python package (i.e. has an ``__init__.py`` file), an implicit namespace package @@ -52,6 +52,19 @@ If the analyzed sources use implicit namespace packages (PEP 420), the source ro be specified using the ``--source-roots`` option. Otherwise, the package names are detected incorrectly, since implicit namespace packages don't contain an ``__init__.py``. +In out-of-source directories +---------------------------- + +If you are analyzing a file that is not located under the main source directory of your +project but needs to import modules from there, for instance and most prominantly a test +file in ``tests/``, you can use ``--pythonpath`` to add the main source directory to the +python path. +For example, if your project features a directory layout with a dedicated source +directory ``src/`` and a test directory ``tests/`` at the top level, you can use +``--pythonpath=src`` (or the appropriate configuration setting) to successfully lint +your tests. + + Globbing support ---------------- diff --git a/doc/whatsnew/fragments/9507.feature b/doc/whatsnew/fragments/9507.feature new file mode 100644 index 0000000000..8ea70e00d5 --- /dev/null +++ b/doc/whatsnew/fragments/9507.feature @@ -0,0 +1,5 @@ +Support linting in out-of-source directories with new main.pythonpath argument that adds relative or absolute paths to sys.path. + +Refs #9507 +Refs #7357 +Refs #5644 diff --git a/pylint/lint/__init__.py b/pylint/lint/__init__.py index 1c0c6d9f58..1caf53a30b 100644 --- a/pylint/lint/__init__.py +++ b/pylint/lint/__init__.py @@ -27,7 +27,11 @@ report_total_messages_stats, ) from pylint.lint.run import Run -from pylint.lint.utils import _augment_sys_path, augmented_sys_path +from pylint.lint.utils import ( + _augment_sys_path, + augmented_sys_path, + realpath_transformer, +) __all__ = [ "check_parallel", @@ -39,6 +43,7 @@ "ArgumentPreprocessingError", "_augment_sys_path", "augmented_sys_path", + "realpath_transformer", "discover_package_path", "save_results", "load_results", diff --git a/pylint/lint/base_options.py b/pylint/lint/base_options.py index 59a811d9c6..1339427813 100644 --- a/pylint/lint/base_options.py +++ b/pylint/lint/base_options.py @@ -377,6 +377,16 @@ def _make_linter_options(linter: PyLinter) -> Options: ), }, ), + ( + "pythonpath", + { + "type": "glob_paths_csv", + "metavar": "[,...]", + "default": (), + "help": "Add paths to sys.path. Supports globbing patterns. Paths are absolute " + "or relative to the current working directory.", + }, + ), ( "ignored-modules", { diff --git a/pylint/lint/parallel.py b/pylint/lint/parallel.py index af381494c3..92af977dc3 100644 --- a/pylint/lint/parallel.py +++ b/pylint/lint/parallel.py @@ -36,12 +36,12 @@ def _worker_initialize( - linter: bytes, extra_packages_paths: Sequence[str] | None = None + linter: bytes, extra_sys_paths: Sequence[str] | None = None ) -> None: """Function called to initialize a worker for a Process within a concurrent Pool. :param linter: A linter-class (PyLinter) instance pickled with dill - :param extra_packages_paths: Extra entries to be added to `sys.path` + :param extra_sys_paths: Extra entries to be added to `sys.path` """ global _worker_linter # pylint: disable=global-statement _worker_linter = dill.loads(linter) @@ -57,8 +57,8 @@ def _worker_initialize( _worker_linter.load_plugin_modules(_worker_linter._dynamic_plugins, force=True) _worker_linter.load_plugin_configuration() - if extra_packages_paths: - _augment_sys_path(extra_packages_paths) + if extra_sys_paths: + _augment_sys_path(extra_sys_paths) def _worker_check_single_file( @@ -125,7 +125,7 @@ def check_parallel( linter: PyLinter, jobs: int, files: Iterable[FileItem], - extra_packages_paths: Sequence[str] | None = None, + extra_sys_paths: Sequence[str] | None = None, ) -> None: """Use the given linter to lint the files with given amount of workers (jobs). @@ -135,9 +135,7 @@ def check_parallel( # The linter is inherited by all the pool's workers, i.e. the linter # is identical to the linter object here. This is required so that # a custom PyLinter object can be used. - initializer = functools.partial( - _worker_initialize, extra_packages_paths=extra_packages_paths - ) + initializer = functools.partial(_worker_initialize, extra_sys_paths=extra_sys_paths) with ProcessPoolExecutor( max_workers=jobs, initializer=initializer, initargs=(dill.dumps(linter),) ) as executor: diff --git a/pylint/lint/pylinter.py b/pylint/lint/pylinter.py index eff15cc444..1195af0b93 100644 --- a/pylint/lint/pylinter.py +++ b/pylint/lint/pylinter.py @@ -52,6 +52,7 @@ augmented_sys_path, get_fatal_error_message, prepare_crash_report, + realpath_transformer, ) from pylint.message import Message, MessageDefinition, MessageDefinitionStore from pylint.reporters.base_reporter import BaseReporter @@ -671,6 +672,10 @@ def check(self, files_or_modules: Sequence[str]) -> None: for file_or_module in files_or_modules } ) + # Prefer package paths detected per module over user-defined PYTHONPATH additions + extra_sys_paths = extra_packages_paths + realpath_transformer( + self.config.pythonpath + ) # TODO: Move the parallel invocation into step 3 of the checking process if not self.config.from_stdin and self.config.jobs > 1: @@ -679,13 +684,13 @@ def check(self, files_or_modules: Sequence[str]) -> None: self, self.config.jobs, self._iterate_file_descrs(files_or_modules), - extra_packages_paths, + extra_sys_paths, ) sys.path = original_sys_path return # 1) Get all FileItems - with augmented_sys_path(extra_packages_paths): + with augmented_sys_path(extra_sys_paths): if self.config.from_stdin: fileitems = self._get_file_descr_from_stdin(files_or_modules[0]) data: str | None = _read_stdin() @@ -694,7 +699,7 @@ def check(self, files_or_modules: Sequence[str]) -> None: data = None # The contextmanager also opens all checkers and sets up the PyLinter class - with augmented_sys_path(extra_packages_paths): + with augmented_sys_path(extra_sys_paths): with self._astroid_module_checker() as check_astroid_module: # 2) Get the AST for each FileItem ast_per_fileitem = self._get_asts(fileitems, data) diff --git a/pylint/lint/utils.py b/pylint/lint/utils.py index a7fbfd0bc3..194fb9d9fd 100644 --- a/pylint/lint/utils.py +++ b/pylint/lint/utils.py @@ -112,6 +112,11 @@ def get_fatal_error_message(filepath: str, issue_template_path: Path) -> str: ) +def realpath_transformer(paths: Sequence[str]) -> list[str]: + """Transforms paths to real paths while expanding user vars.""" + return [str(Path(path).resolve().expanduser()) for path in paths] + + def _augment_sys_path(additional_paths: Sequence[str]) -> list[str]: original = list(sys.path) changes = [] diff --git a/tests/lint/unittest_lint.py b/tests/lint/unittest_lint.py index 086c192110..2a8def887c 100644 --- a/tests/lint/unittest_lint.py +++ b/tests/lint/unittest_lint.py @@ -1250,6 +1250,33 @@ def test_import_sibling_module_from_namespace(initialized_linter: PyLinter) -> N assert not linter.stats.by_msg +def test_import_external_module_with_relative_pythonpath_config( + initialized_linter: PyLinter, +) -> None: + """Given a module that imports an external module, ensure that the external module + is found when the path to the external module is configured in `main.pythonpath`. + + Note: The setup is similar to `test_import_sibling_module_from_namespace` but the + manual sys.path setup is replaced with a `main.pythonpath` configuration. + """ + linter = initialized_linter + with tempdir() as tmpdir: + create_files(["namespace_main/module.py", "namespace_ext/ext_module.py"]) + main_path = Path("namespace_main/module.py") + with open(main_path, "w", encoding="utf-8") as f: + f.write( + """\"\"\"This module imports ext_module.\"\"\" +import ext_module +print(ext_module) +""" + ) + + os.chdir(tmpdir) + linter.config.pythonpath = ["namespace_ext"] + linter.check(["namespace_main/module.py"]) + assert not linter.stats.by_msg + + def test_lint_namespace_package_under_dir(initialized_linter: PyLinter) -> None: """Regression test for https://github.com/pylint-dev/pylint/issues/1667.""" linter = initialized_linter diff --git a/tests/test_check_parallel.py b/tests/test_check_parallel.py index d7c936b97d..77f23bab95 100644 --- a/tests/test_check_parallel.py +++ b/tests/test_check_parallel.py @@ -186,9 +186,7 @@ def test_worker_initialize(self) -> None: def test_worker_initialize_with_package_paths(self) -> None: linter = PyLinter(reporter=Reporter()) with augmented_sys_path([]): - worker_initialize( - linter=dill.dumps(linter), extra_packages_paths=["fake-path"] - ) + worker_initialize(linter=dill.dumps(linter), extra_sys_paths=["fake-path"]) assert "fake-path" in sys.path def test_worker_initialize_reregisters_custom_plugins(self) -> None: @@ -629,7 +627,7 @@ def test_no_deadlock_due_to_initializer_error(self) -> None: files=iter(single_file_container), # This will trigger an exception in the initializer for the parallel jobs # because arguments has to be an Iterable. - extra_packages_paths=1, # type: ignore[arg-type] + extra_sys_paths=1, # type: ignore[arg-type] ) @pytest.mark.needs_two_cores