diff --git a/binder/environment.yml b/binder/environment.yml index a13690d78cc..0f033c13e4c 100644 --- a/binder/environment.yml +++ b/binder/environment.yml @@ -39,7 +39,7 @@ dependencies: - pyqt >=5.15,<5.16 - pyqtwebengine >=5.15,<5.16 - python-lsp-black >=2.0.0,<3.0.0 -- python-lsp-server >=1.12.0,<1.13.0 +- python-lsp-server >=1.12.2,<1.13.0 - pyuca >=1.2 - pyxdg >=0.26 - pyzmq >=24.0.0 diff --git a/changelogs/Spyder-6.md b/changelogs/Spyder-6.md index 01024eeadf8..9a096bde41a 100644 --- a/changelogs/Spyder-6.md +++ b/changelogs/Spyder-6.md @@ -4,6 +4,7 @@ ### API changes +* **Breaking** - The `sig_pythonpath_changed` signal of the Python path manager plugin now emits a list of strings and a bool, instead of two dictionaries. * Add `early_return` and `return_awaitable` kwargs to `AsyncDispatcher` constructor. * Add `register_api` and `get_api` methods to `RemoteClient` plugin in order to get and register new rest API modules for the remote client. diff --git a/external-deps/spyder-kernels/.gitrepo b/external-deps/spyder-kernels/.gitrepo index 57d90f4bf1b..b1737842092 100644 --- a/external-deps/spyder-kernels/.gitrepo +++ b/external-deps/spyder-kernels/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/spyder-ide/spyder-kernels.git branch = master - commit = 67259709972e39c6d13b463076670a62dc9e832e - parent = 00f4b6630c6846ebd51514b8bc63f49c4821b38b + commit = 633c29712f7260c1dceca25bcecbb5def91e5818 + parent = 1aab01be14f79b17da61348d564b20cad69dabc4 method = merge cmdver = 0.4.9 diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index 477fcd95ff2..43e47d7118d 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -93,6 +93,11 @@ def __init__(self, *args, **kwargs): # To save the python env info self.pythonenv_info: PythonEnvInfo = {} + # Store original sys.path. Kernels are started with PYTHONPATH + # removed from environment variables, so this will never have + # user paths and should be clean. + self._sys_path = sys.path.copy() + @property def kernel_info(self): # Used for checking correct version by spyder @@ -770,27 +775,42 @@ def set_special_kernel(self, special): raise NotImplementedError(f"{special}") @comm_handler - def update_syspath(self, path_dict, new_path_dict): + def update_syspath(self, new_path, prioritize): """ Update the PYTHONPATH of the kernel. - `path_dict` and `new_path_dict` have the paths as keys and the state - as values. The state is `True` for active and `False` for inactive. - - `path_dict` corresponds to the previous state of the PYTHONPATH. - `new_path_dict` corresponds to the new state of the PYTHONPATH. + Parameters + ---------- + new_path: list of str + List of PYTHONPATH paths. + prioritize: bool + Whether to place PYTHONPATH paths at the front (True) or + back (False) of sys.path. + + Notes + ----- + A copy of sys.path is made at instantiation, which should be clean, + so we can just prepend/append to the copy without having to explicitly + remove old user paths. PYTHONPATH can just be overwritten. """ - # Remove old paths - for path in path_dict: - while path in sys.path: - sys.path.remove(path) - - # Add new paths - pypath = [path for path, active in new_path_dict.items() if active] - if pypath: - sys.path.extend(pypath) - os.environ.update({'PYTHONPATH': os.pathsep.join(pypath)}) + if new_path is not None: + # Overwrite PYTHONPATH + os.environ.update({'PYTHONPATH': os.pathsep.join(new_path)}) + + # Add new paths to original sys.path + if prioritize: + sys.path[:] = new_path + self._sys_path + + # Ensure current directory is always first to imitate Python + # standard behavior + if '' in sys.path: + sys.path.remove('') + sys.path.insert(0, '') + else: + sys.path[:] = self._sys_path + new_path else: + # Restore original sys.path and remove PYTHONPATH + sys.path[:] = self._sys_path os.environ.pop('PYTHONPATH', None) @comm_handler diff --git a/external-deps/spyder-kernels/spyder_kernels/console/start.py b/external-deps/spyder-kernels/spyder_kernels/console/start.py index b8a423af2a9..3f7607eb05b 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/start.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/start.py @@ -16,6 +16,14 @@ import sys import site +# Remove current directory from sys.path to prevent kernel crashes when people +# name Python files or modules with the same name as standard library modules. +# See spyder-ide/spyder#8007 +# Inject it back into sys.path after all imports in this module but +# before the kernel is initialized +while '' in sys.path: + sys.path.remove('') + # Third-party imports from traitlets import DottedObjectName @@ -29,13 +37,6 @@ def import_spydercustomize(): parent = osp.dirname(here) customize_dir = osp.join(parent, 'customize') - # Remove current directory from sys.path to prevent kernel - # crashes when people name Python files or modules with - # the same name as standard library modules. - # See spyder-ide/spyder#8007 - while '' in sys.path: - sys.path.remove('') - # Import our customizations site.addsitedir(customize_dir) import spydercustomize # noqa @@ -46,6 +47,7 @@ def import_spydercustomize(): except ValueError: pass + def kernel_config(): """Create a config object with IPython kernel options.""" from IPython.core.application import get_ipython_dir @@ -150,13 +152,6 @@ def main(): # Import our customizations into the kernel import_spydercustomize() - # Remove current directory from sys.path to prevent kernel - # crashes when people name Python files or modules with - # the same name as standard library modules. - # See spyder-ide/spyder#8007 - while '' in sys.path: - sys.path.remove('') - # Main imports from ipykernel.kernelapp import IPKernelApp from spyder_kernels.console.kernel import SpyderKernel @@ -189,6 +184,12 @@ def close(self): kernel.config = kernel_config() except: pass + + # Re-add current working directory path into sys.path after all of the + # import statements, but before initializing the kernel. + if '' not in sys.path: + sys.path.insert(0, '') + kernel.initialize() # Set our own magics diff --git a/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py index 7e623b30107..dfcd10086b0 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py @@ -77,13 +77,15 @@ def setup_kernel(cmd): ) # wait for connection file to exist, timeout after 5s tic = time.time() - while not os.path.exists(connection_file) \ - and kernel.poll() is None \ - and time.time() < tic + SETUP_TIMEOUT: + while ( + not os.path.exists(connection_file) + and kernel.poll() is None + and time.time() < tic + SETUP_TIMEOUT + ): time.sleep(0.1) if kernel.poll() is not None: - o,e = kernel.communicate() + o, e = kernel.communicate() raise IOError("Kernel failed to start:\n%s" % e) if not os.path.exists(connection_file): @@ -229,7 +231,7 @@ def kernel(request): 'True_' ], 'minmax': False, - 'filter_on':True + 'filter_on': True } # Teardown @@ -468,8 +470,11 @@ def test_is_defined(kernel): def test_get_doc(kernel): """Test to get object documentation dictionary.""" objtxt = 'help' - assert ("Define the builtin 'help'" in kernel.get_doc(objtxt)['docstring'] or - "Define the built-in 'help'" in kernel.get_doc(objtxt)['docstring']) + assert ( + "Define the builtin 'help'" in kernel.get_doc(objtxt)['docstring'] + or "Define the built-in 'help'" in kernel.get_doc(objtxt)['docstring'] + ) + def test_get_source(kernel): """Test to get object source.""" @@ -507,7 +512,7 @@ def test_cwd_in_sys_path(): with setup_kernel(cmd) as client: reply = client.execute_interactive( "import sys; sys_path = sys.path", - user_expressions={'output':'sys_path'}, timeout=TIMEOUT) + user_expressions={'output': 'sys_path'}, timeout=TIMEOUT) # Transform value obtained through user_expressions user_expressions = reply['content']['user_expressions'] @@ -518,6 +523,21 @@ def test_cwd_in_sys_path(): assert '' in value +def test_prioritize(kernel): + """Test that user path priority is honored in sys.path.""" + syspath = kernel.get_syspath() + append_path = ['/test/append/path'] + prepend_path = ['/test/prepend/path'] + + kernel.update_syspath(append_path, prioritize=False) + new_syspath = kernel.get_syspath() + assert new_syspath == syspath + append_path + + kernel.update_syspath(prepend_path, prioritize=True) + new_syspath = kernel.get_syspath() + assert new_syspath == prepend_path + syspath + + @flaky(max_runs=3) def test_multiprocessing(tmpdir): """ @@ -701,8 +721,10 @@ def test_runfile(tmpdir): assert content['found'] # Run code file `u` with current namespace - msg = client.execute_interactive("%runfile {} --current-namespace" - .format(repr(str(u))), timeout=TIMEOUT) + msg = client.execute_interactive( + "%runfile {} --current-namespace".format(repr(str(u))), + timeout=TIMEOUT + ) content = msg['content'] # Verify that the variable `result3` is defined @@ -727,7 +749,9 @@ def test_runfile(tmpdir): sys.platform == 'darwin' and sys.version_info[:2] == (3, 8), reason="Fails on Mac with Python 3.8") def test_np_threshold(kernel): - """Test that setting Numpy threshold doesn't make the Variable Explorer slow.""" + """ + Test that setting Numpy threshold doesn't make the Variable Explorer slow. + """ cmd = "from spyder_kernels.console import start; start.main()" @@ -786,7 +810,9 @@ def test_np_threshold(kernel): while "data" not in msg['content']: msg = client.get_shell_msg(timeout=TIMEOUT) content = msg['content']['data']['text/plain'] - assert "{'float_kind': =5.15,<5.16 - pyqtwebengine >=5.15,<5.16 - python-lsp-black >=2.0.0,<3.0.0 - - python-lsp-server >=1.12.0,<1.13.0 + - python-lsp-server >=1.12.2,<1.13.0 - pyuca >=1.2 - pyzmq >=24.0.0 - qdarkstyle >=3.2.0,<3.3.0 diff --git a/setup.py b/setup.py index bc986ce5441..788be4c1d0c 100644 --- a/setup.py +++ b/setup.py @@ -299,7 +299,7 @@ def run(self): 'pylint-venv>=3.0.2', 'pyls-spyder>=0.4.0', 'python-lsp-black>=2.0.0,<3.0.0', - 'python-lsp-server[all]>=1.12.0,<1.13.0', + 'python-lsp-server[all]>=1.12.2,<1.13.0', 'pyuca>=1.2', 'pyxdg>=0.26;platform_system=="Linux"', 'pyzmq>=24.0.0', @@ -329,7 +329,7 @@ def run(self): install_requires = [req for req in install_requires if req.split(">")[0] not in reqs_to_loosen] - install_requires.append('python-lsp-server[all]>=1.12.0,<1.14.0') + install_requires.append('python-lsp-server[all]>=1.12.2,<1.14.0') install_requires.append('qtconsole>=5.5.1,<5.7.0') extras_require = { diff --git a/spyder/app/tests/conftest.py b/spyder/app/tests/conftest.py index 79da753cf70..c639ee02580 100755 --- a/spyder/app/tests/conftest.py +++ b/spyder/app/tests/conftest.py @@ -27,15 +27,13 @@ from spyder.api.plugin_registration.registry import PLUGIN_REGISTRY from spyder.api.plugins import Plugins from spyder.app import start -from spyder.config.base import get_home_dir, running_in_ci +from spyder.config.base import get_home_dir from spyder.config.manager import CONF from spyder.plugins.ipythonconsole.utils.kernelspec import SpyderKernelSpec from spyder.plugins.projects.api import EmptyProject from spyder.plugins.run.api import RunActions, StoredRunConfigurationExecutor from spyder.plugins.toolbar.api import ApplicationToolbars from spyder.utils import encoding -from spyder.utils.environ import (get_user_env, set_user_env, - amend_user_shell_init) # ============================================================================= # ---- Constants @@ -624,20 +622,3 @@ def threads_condition(): CONF.reset_manager() PLUGIN_REGISTRY.reset() raise - - -@pytest.fixture -def restore_user_env(): - """Set user environment variables and restore upon test exit""" - if not running_in_ci(): - pytest.skip("Skipped because not in CI.") - - if os.name == "nt": - orig_env = get_user_env() - - yield - - if os.name == "nt": - set_user_env(orig_env) - else: - amend_user_shell_init(restore=True) diff --git a/spyder/app/tests/test_mainwindow.py b/spyder/app/tests/test_mainwindow.py index c0fd72a14b0..3a29ed5cfdd 100644 --- a/spyder/app/tests/test_mainwindow.py +++ b/spyder/app/tests/test_mainwindow.py @@ -11,6 +11,7 @@ """ # Standard library imports +from collections import OrderedDict import gc import os import os.path as osp @@ -96,7 +97,6 @@ ) from spyder.plugins.shortcuts.widgets.table import SEQUENCE from spyder.py3compat import qbytearray_to_str, to_text_string -from spyder.utils.environ import set_user_env from spyder.utils.conda import get_list_conda_envs from spyder.utils.misc import remove_backslashes, rename_file from spyder.utils.clipboard_helper import CLIPBOARD_HELPER @@ -6518,8 +6518,7 @@ def test_switch_to_plugin(main_window, qtbot): @flaky(max_runs=5) -def test_PYTHONPATH_in_consoles(main_window, qtbot, tmp_path, - restore_user_env): +def test_PYTHONPATH_in_consoles(main_window, qtbot, tmp_path): """ Test that PYTHONPATH is passed to IPython consoles under different scenarios. @@ -6527,42 +6526,36 @@ def test_PYTHONPATH_in_consoles(main_window, qtbot, tmp_path, # Wait until the window is fully up ipyconsole = main_window.ipyconsole shell = ipyconsole.get_current_shellwidget() - qtbot.waitUntil(lambda: shell._prompt_html is not None, - timeout=SHELL_TIMEOUT) + qtbot.waitUntil( + lambda: shell._prompt_html is not None, timeout=SHELL_TIMEOUT + ) # Main variables ppm = main_window.get_plugin(Plugins.PythonpathManager) - # Add a directory to PYTHONPATH - sys_dir = tmp_path / 'sys_dir' - sys_dir.mkdir() - set_user_env({"PYTHONPATH": str(sys_dir)}) - - # Add a directory to the current list of paths to simulate a path added by - # users + # Create a directory to use as a user path user_dir = tmp_path / 'user_dir' user_dir.mkdir() - if os.name != "nt": - assert ppm.get_container().path == () - ppm.get_container().path = (str(user_dir),) + ppm.get_container().path - # Open Pythonpath dialog to detect sys_dir + # Check that the inital configured spyder_pythonpath is empty + assert ppm.get_container()._spyder_pythonpath == [] + + # Add a directory to the current list of paths to simulate a path added by + # the user ppm.show_path_manager() qtbot.wait(500) - # Check we're showing two headers - assert len(ppm.path_manager_dialog.headers) == 2 + ppm.path_manager_dialog.add_path(directory=user_dir) - # Check the PPM emits the right signal after closing the dialog with qtbot.waitSignal(ppm.sig_pythonpath_changed, timeout=1000): - ppm.path_manager_dialog.close() + ppm.path_manager_dialog.accept() - # Check directories were added to sys.path in the right order + # Check that user_dir was added to sys.path in the right order with qtbot.waitSignal(shell.executed, timeout=2000): shell.execute("import sys; sys_path = sys.path") sys_path = shell.get_value("sys_path") - assert sys_path[-2:] == [str(user_dir), str(sys_dir)] + assert sys_path[-1] == str(user_dir) # Path should be at the end # Create new console ipyconsole.create_new_client() @@ -6570,12 +6563,33 @@ def test_PYTHONPATH_in_consoles(main_window, qtbot, tmp_path, qtbot.waitUntil(lambda: shell1._prompt_html is not None, timeout=SHELL_TIMEOUT) - # Check directories are part of the new console's sys.path + # Check user_dir is part of the new console's sys.path with qtbot.waitSignal(shell1.executed, timeout=2000): shell1.execute("import sys; sys_path = sys.path") sys_path = shell1.get_value("sys_path") - assert sys_path[-2:] == [str(user_dir), str(sys_dir)] + assert sys_path[-1] == str(user_dir) # Path should be at the end + + # Check that user path can be prepended to sys.path + ppm.show_path_manager() + qtbot.wait(500) + + # ??? Why does this work... + ppm.path_manager_dialog.prioritize_button.setChecked(True) + qtbot.wait(500) + # ...but this does not? + # with qtbot.waitUntil(ppm.path_manager_dialog.prioritize_button.isChecked): + # ppm.path_manager_dialog.prioritize_button.animateClick() + + with qtbot.waitSignal(ppm.sig_pythonpath_changed, timeout=1000): + ppm.path_manager_dialog.accept() + + for s in [shell, shell1]: + with qtbot.waitSignal(s.executed, timeout=2000): + s.execute("sys_path = sys.path") + + sys_path = shell.get_value("sys_path") + assert sys_path[1] == str(user_dir) # Path should be ['', user_dir, ...] # Check that disabling a path from the PPM removes it from sys.path in all # consoles diff --git a/spyder/config/lsp.py b/spyder/config/lsp.py index b391d006bd8..4410e90b99b 100644 --- a/spyder/config/lsp.py +++ b/spyder/config/lsp.py @@ -79,6 +79,7 @@ 'environment': None, 'extra_paths': [], 'env_vars': None, + 'prioritize_extra_paths': False, # Until we have a graphical way for users to add modules to # this option 'auto_import_modules': [ diff --git a/spyder/config/main.py b/spyder/config/main.py index 258fde12685..8277c1243fd 100644 --- a/spyder/config/main.py +++ b/spyder/config/main.py @@ -116,6 +116,9 @@ ('pythonpath_manager', { 'spyder_pythonpath': [], + 'prioritize': False, + 'system_paths': {}, + 'user_paths': {}, }), ('quick_layouts', { diff --git a/spyder/dependencies.py b/spyder/dependencies.py index 837d1311bdf..8a64654399b 100644 --- a/spyder/dependencies.py +++ b/spyder/dependencies.py @@ -60,7 +60,7 @@ PYGMENTS_REQVER = '>=2.0' PYLINT_REQVER = '>=3.1,<4' PYLINT_VENV_REQVER = '>=3.0.2' -PYLSP_REQVER = '>=1.12.0,<1.13.0' +PYLSP_REQVER = '>=1.12.2,<1.13.0' PYLSP_BLACK_REQVER = '>=2.0.0,<3.0.0' PYLS_SPYDER_REQVER = '>=0.4.0' PYUCA_REQVER = '>=1.2' diff --git a/spyder/plugins/completion/api.py b/spyder/plugins/completion/api.py index f9de9a37526..fb55fc218f5 100644 --- a/spyder/plugins/completion/api.py +++ b/spyder/plugins/completion/api.py @@ -1059,17 +1059,17 @@ def project_path_update(self, project_path: str, update_kind: str, """ pass - @Slot(object, object) - def python_path_update(self, previous_path, new_path): + @Slot(object, bool) + def python_path_update(self, new_path, prioritize): """ Handle Python path updates on Spyder. Parameters ---------- - previous_path: Dict - Dictionary containing the previous Python path values. - new_path: Dict + new_path: list of str Dictionary containing the current Python path values. + prioritize: bool + Whether to prioritize Python path values in sys.path """ pass diff --git a/spyder/plugins/completion/plugin.py b/spyder/plugins/completion/plugin.py index e519b58de24..55c76b6bef1 100644 --- a/spyder/plugins/completion/plugin.py +++ b/spyder/plugins/completion/plugin.py @@ -124,16 +124,17 @@ class CompletionPlugin(SpyderPluginV2): Name of the completion client. """ - sig_pythonpath_changed = Signal(object, object) + sig_pythonpath_changed = Signal(object, bool) """ - This signal is used to receive changes on the PythonPath. + This signal is used to receive changes on the Python path values handled + by Spyder. Parameters ---------- - prev_path: dict - Previous PythonPath settings. - new_path: dict + new_path: list of str New PythonPath settings. + prioritize + Whether to prioritize PYTHONPATH in sys.path """ _sig_interpreter_changed = Signal(str) diff --git a/spyder/plugins/completion/providers/languageserver/provider.py b/spyder/plugins/completion/providers/languageserver/provider.py index e5d9f90e987..fa6f96756d4 100644 --- a/spyder/plugins/completion/providers/languageserver/provider.py +++ b/spyder/plugins/completion/providers/languageserver/provider.py @@ -377,7 +377,6 @@ def project_path_update(self, project_path, update_kind, projects): self.stop_completion_services_for_language(language) self.start_completion_services_for_language(language) - def report_server_error(self, error): """Report server errors in our error report dialog.""" error_data = dict( @@ -532,26 +531,25 @@ def shutdown(self): for language in self.clients: self.stop_completion_services_for_language(language) - @Slot(object, object) - def python_path_update(self, path_dict, new_path_dict): + @Slot(object, bool) + def python_path_update(self, new_path, prioritize): """ Update server configuration after a change in Spyder's Python path. - `path_dict` corresponds to the previous state of the Python path. - `new_path_dict` corresponds to the new state of the Python path. + Parameters + ---------- + new_path: list of str + New state of the Python path handled by Spyder. + prioritize: bool + Whether to prioritize Python path in sys.path. """ - # If path_dict and new_path_dict are the same, it means the change - # was generated by opening or closing a project. In that case, we - # don't need to request an update because that's done through the - # addition/deletion of workspaces. - update = True - if path_dict == new_path_dict: - update = False - - if update: - logger.debug("Update server's sys.path") - self.update_lsp_configuration(python_only=True) + # Opening/closing a project will create a diff between old_path + # and new_path, but we don't know if prioritize changed. + # sig_pythonpath_changed is only emitted if there is a change so we + # should always update the confguration when this method is called. + logger.debug("Update server's sys.path") + self.update_lsp_configuration(python_only=True) @qdebounced(timeout=600) def interpreter_changed(self, interpreter: str): @@ -589,8 +587,11 @@ def on_pyls_spyder_configuration_change(self, option, value): def on_code_snippets_enabled_disabled(self, value): self.update_lsp_configuration() - @on_conf_change(section='pythonpath_manager', option='spyder_pythonpath') - def on_pythonpath_option_update(self, value): + @on_conf_change( + section='pythonpath_manager', + option=['spyder_pythonpath', 'prioritize'] + ) + def on_pythonpath_option_update(self, option, value): # This is only useful to run some self-contained tests if running_under_pytest(): self.update_lsp_configuration(python_only=True) @@ -806,13 +807,15 @@ def generate_python_config(self): # Jedi configuration env_vars = os.environ.copy() # Ensure env is indepependent of PyLSP's - env_vars.pop('PYTHONPATH', None) jedi = { 'environment': self._interpreter, 'extra_paths': self.get_conf('spyder_pythonpath', section='pythonpath_manager', default=[]), + 'prioritize_extra_paths': self.get_conf( + 'prioritize', section='pythonpath_manager', default=False + ), 'env_vars': env_vars, } jedi_completion = { diff --git a/spyder/plugins/ipythonconsole/plugin.py b/spyder/plugins/ipythonconsole/plugin.py index 45d9a3dcac3..40d7849e49a 100644 --- a/spyder/plugins/ipythonconsole/plugin.py +++ b/spyder/plugins/ipythonconsole/plugin.py @@ -500,7 +500,7 @@ def on_main_menu_teardown(self): mainmenu.remove_item_from_application_menu( IPythonConsoleWidgetMenus.Documentation, menu_id=ApplicationMenus.Help - ) + ) @on_plugin_teardown(plugin=Plugins.Editor) def on_editor_teardown(self): @@ -1024,7 +1024,7 @@ def save_working_directory(self, dirname): """ self.get_widget().save_working_directory(dirname) - def update_path(self, path_dict, new_path_dict): + def update_path(self, new_path, prioritize): """ Update path on consoles. @@ -1033,16 +1033,16 @@ def update_path(self, path_dict, new_path_dict): Parameters ---------- - path_dict : dict - Corresponds to the previous state of the PYTHONPATH. - new_path_dict : dict - Corresponds to the new state of the PYTHONPATH. + new_path : list of str + New state of the Python path handled by Spyder. + prioritize : bool + Whether to prioritize Python path in sys.path Returns ------- None. """ - self.get_widget().update_path(path_dict, new_path_dict) + self.get_widget().update_path(new_path, prioritize) def restart(self): """ diff --git a/spyder/plugins/ipythonconsole/utils/kernelspec.py b/spyder/plugins/ipythonconsole/utils/kernelspec.py index 3d225b9ed7e..43d4e36ad87 100644 --- a/spyder/plugins/ipythonconsole/utils/kernelspec.py +++ b/spyder/plugins/ipythonconsole/utils/kernelspec.py @@ -174,12 +174,6 @@ def env(self): # Do not pass PYTHONPATH to kernels directly, spyder-ide/spyder#13519 env_vars.pop('PYTHONPATH', None) - # List of paths declared by the user, plus project's path, to - # add to PYTHONPATH - pathlist = self.get_conf( - 'spyder_pythonpath', default=[], section='pythonpath_manager') - pypath = os.pathsep.join(pathlist) - # List of modules to exclude from our UMR umr_namelist = self.get_conf( 'umr/namelist', section='main_interpreter') @@ -201,7 +195,6 @@ def env(self): 'SPY_JEDI_O': self.get_conf('jedi_completer'), 'SPY_TESTING': running_under_pytest() or get_safe_mode(), 'SPY_HIDE_CMD': self.get_conf('hide_cmd_windows'), - 'SPY_PYTHONPATH': pypath, # This env var avoids polluting the OS default temp directory with # files generated by `conda run`. It's restored/removed in the # kernel after initialization. diff --git a/spyder/plugins/ipythonconsole/utils/tests/test_spyder_kernel.py b/spyder/plugins/ipythonconsole/utils/tests/test_spyder_kernel.py index 0503af6a6a0..7fe7baa5fb4 100644 --- a/spyder/plugins/ipythonconsole/utils/tests/test_spyder_kernel.py +++ b/spyder/plugins/ipythonconsole/utils/tests/test_spyder_kernel.py @@ -8,42 +8,10 @@ Tests for the Spyder kernel """ -import os import pytest from spyder.config.manager import CONF from spyder.plugins.ipythonconsole.utils.kernelspec import SpyderKernelSpec -from spyder.py3compat import to_text_string - - -@pytest.mark.parametrize('default_interpreter', [True, False]) -def test_kernel_pypath(tmpdir, default_interpreter): - """ - Test that PYTHONPATH and spyder_pythonpath option are properly handled - when an external interpreter is used or not. - - Regression test for spyder-ide/spyder#8681. - Regression test for spyder-ide/spyder#17511. - """ - # Set default interpreter value - CONF.set('main_interpreter', 'default', default_interpreter) - - # Add a path to PYTHONPATH and spyder_pythonpath config option - pypath = to_text_string(tmpdir.mkdir('test-pypath')) - os.environ['PYTHONPATH'] = pypath - CONF.set('pythonpath_manager', 'spyder_pythonpath', [pypath]) - - kernel_spec = SpyderKernelSpec() - - # Check that PYTHONPATH is not in our kernelspec - # and pypath is in SPY_PYTHONPATH - assert 'PYTHONPATH' not in kernel_spec.env - assert pypath in kernel_spec.env['SPY_PYTHONPATH'] - - # Restore default values - CONF.set('main_interpreter', 'default', True) - CONF.set('pythonpath_manager', 'spyder_pythonpath', []) - del os.environ['PYTHONPATH'] def test_python_interpreter(tmpdir): diff --git a/spyder/plugins/ipythonconsole/widgets/main_widget.py b/spyder/plugins/ipythonconsole/widgets/main_widget.py index 71d151b27e7..e1d9bbf74a8 100644 --- a/spyder/plugins/ipythonconsole/widgets/main_widget.py +++ b/spyder/plugins/ipythonconsole/widgets/main_widget.py @@ -2472,13 +2472,13 @@ def on_working_directory_changed(self, dirname): if dirname and osp.isdir(dirname): self.sig_current_directory_changed.emit(dirname) - def update_path(self, path_dict, new_path_dict): + def update_path(self, new_path, prioritize): """Update path on consoles.""" logger.debug("Update sys.path in all console clients") for client in self.clients: shell = client.shellwidget if shell is not None: - shell.update_syspath(path_dict, new_path_dict) + shell.update_syspath(new_path, prioritize) def get_active_project_path(self): """Get the active project path.""" diff --git a/spyder/plugins/ipythonconsole/widgets/shell.py b/spyder/plugins/ipythonconsole/widgets/shell.py index e2b1d5b9d12..a72d0c93f1e 100644 --- a/spyder/plugins/ipythonconsole/widgets/shell.py +++ b/spyder/plugins/ipythonconsole/widgets/shell.py @@ -413,6 +413,15 @@ def setup_spyder_kernel(self): self.send_spyder_kernel_configuration() + # Update sys.path + paths = self.get_conf( + "spyder_pythonpath", section="pythonpath_manager" + ) + prioritize = self.get_conf( + "prioritize", section="pythonpath_manager" + ) + self.update_syspath(paths, prioritize) + run_lines = self.get_conf('startup/run_lines') if run_lines: self.execute(run_lines, hidden=True) @@ -712,14 +721,14 @@ def set_color_scheme(self, color_scheme, reset=True): "color scheme", "dark" if not dark_color else "light" ) - def update_syspath(self, path_dict, new_path_dict): + def update_syspath(self, new_paths, prioritize): """Update sys.path contents in the kernel.""" # Prevent error when the kernel is not available and users open/close # projects or use the Python path manager. # Fixes spyder-ide/spyder#21563 if self.kernel_handler is not None: self.call_kernel(interrupt=True, blocking=False).update_syspath( - path_dict, new_path_dict + new_paths, prioritize ) def request_syspath(self): diff --git a/spyder/plugins/pythonpath/container.py b/spyder/plugins/pythonpath/container.py index 84baa967280..7325287d74e 100644 --- a/spyder/plugins/pythonpath/container.py +++ b/spyder/plugins/pythonpath/container.py @@ -9,6 +9,7 @@ from collections import OrderedDict import logging +import os import os.path as osp from qtpy.QtCore import Signal @@ -34,32 +35,34 @@ class PythonpathActions: # ----------------------------------------------------------------------------- class PythonpathContainer(PluginMainContainer): - sig_pythonpath_changed = Signal(object, object) + sig_pythonpath_changed = Signal(object, bool) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.path = () - self.not_active_path = () - self.project_path = () # ---- PluginMainContainer API # ------------------------------------------------------------------------- def setup(self): - - # Migrate from old conf files to config options - if self.get_conf('paths_in_conf_files', default=True): + # Migrate to new config options if necessary + if not self.get_conf("config_options_migrated", False): self._migrate_to_config_options() - # Load Python path - self._load_pythonpath() + # This attribute is only used to detect changes and after initializing + # here should only be set in update_active_project_path. + self._project_path = OrderedDict() - # Save current Pythonpath at startup so plugins can use it afterwards - self.set_conf('spyder_pythonpath', self.get_spyder_pythonpath()) + # These attributes are only used to detect changes and after + # initializing here should only be set in _save_paths. + self._user_paths = OrderedDict(self.get_conf('user_paths')) + self._system_paths = self.get_conf('system_paths') + self._prioritize = self.get_conf('prioritize') + self._spyder_pythonpath = self.get_conf('spyder_pythonpath') # Path manager dialog self.path_manager_dialog = PathManager(parent=self, sync=True) self.path_manager_dialog.sig_path_changed.connect( - self._update_python_path) + self._save_paths + ) self.path_manager_dialog.redirect_stdio.connect( self.sig_redirect_stdio_requested) @@ -74,48 +77,38 @@ def setup(self): def update_actions(self): pass - def on_close(self): - # Save current system path to detect changes next time Spyder starts - self.set_conf('system_path', get_system_pythonpath()) - # ---- Public API # ------------------------------------------------------------------------- def update_active_project_path(self, path): - """Update active project path.""" + """ + Update active project path. + + _project_path is set in this method and nowhere else. + """ + # _project_path should be reset whenever it is updated. + self._project_path = OrderedDict() if path is None: - logger.debug("Update Pythonpath because project was closed") - path = () + logger.debug("Update Spyder PYTHONPATH because project was closed") else: - logger.debug(f"Add to Pythonpath project's path -> {path}") - path = (path,) - - # Old path - old_path_dict_p = self._get_spyder_pythonpath_dict() - - # Change project path - self.project_path = path - self.path_manager_dialog.project_path = path + logger.debug(f"Add project paths to Spyder PYTHONPATH: {path}") + self._project_path.update({path: True}) - # New path - new_path_dict_p = self._get_spyder_pythonpath_dict() - - # Update path - self.set_conf('spyder_pythonpath', self.get_spyder_pythonpath()) - self.sig_pythonpath_changed.emit(old_path_dict_p, new_path_dict_p) + self._save_paths() def show_path_manager(self): - """Show path manager dialog.""" - # Do not update paths or run setup if widget is already open, - # see spyder-ide/spyder#20808 + """ + Show path manager dialog. + """ + # Do not update paths if widget is already open, + # see spyder-ide/spyder#20808. if not self.path_manager_dialog.isVisible(): - # Set main attributes saved here self.path_manager_dialog.update_paths( - self.path, self.not_active_path, get_system_pythonpath() + project_path=self._project_path, + user_paths=self._user_paths, + system_paths=self._system_paths, + prioritize=self._prioritize ) - # Setup its contents again - self.path_manager_dialog.setup() - # Show and give it focus self.path_manager_dialog.show() self.path_manager_dialog.activateWindow() @@ -123,146 +116,151 @@ def show_path_manager(self): self.path_manager_dialog.setFocus() def get_spyder_pythonpath(self): - """ - Return active Spyder PYTHONPATH plus project path as a list of paths. - """ - path_dict = self._get_spyder_pythonpath_dict() - path = [k for k, v in path_dict.items() if v] - return path + """Return active Spyder PYTHONPATH as a list of paths.""" + # Desired behavior is project_path | user_paths | system_paths, but + # Python 3.8 does not support | operator for OrderedDict. + all_paths = OrderedDict(reversed(self._system_paths.items())) + all_paths.update(reversed(self._user_paths.items())) + all_paths.update(reversed(self._project_path.items())) + all_paths = OrderedDict(reversed(all_paths.items())) + + return [p for p, v in all_paths.items() if v] # ---- Private API # ------------------------------------------------------------------------- - def _load_pythonpath(self): - """Load Python paths.""" - # Get current system PYTHONPATH - system_path = get_system_pythonpath() - - # Get previous system PYTHONPATH - previous_system_path = self.get_conf('system_path', default=()) - - # Load all paths - paths = [] - previous_paths = self.get_conf('path') - for path in previous_paths: - # Path was removed since last time or it's not a directory - # anymore - if not osp.isdir(path): - continue - - # Path was removed from system path - if path in previous_system_path and path not in system_path: - continue - - paths.append(path) - - self.path = tuple(paths) - - # Update path option. This avoids loading paths that were removed in - # this session in later ones. - self.set_conf('path', self.path) - - # Update system path so that path_manager_dialog can work with its - # latest contents. - self.set_conf('system_path', system_path) - - # Add system path - if system_path: - self.path = self.path + system_path - - # Load not active paths - not_active_paths = self.get_conf('not_active_path') - self.not_active_path = tuple( - name for name in not_active_paths if osp.isdir(name) + def _get_system_paths(self): + system_paths = get_system_pythonpath() + conf_system_paths = self.get_conf('system_paths', {}) + + # If a system path already exists in the configuration, use the + # configuration active state. If it does not exist in the + # configuration, then set the active state to True. + system_paths = OrderedDict( + {p: conf_system_paths.get(p, True) for p in system_paths} ) - def _save_paths(self, new_path_dict): - """ - Save tuples for all paths and not active ones to config system and - update their associated attributes. + return system_paths - `new_path_dict` is an OrderedDict that has the new paths as keys and - the state as values. The state is `True` for active and `False` for - inactive. + def _save_paths(self, user_paths=None, system_paths=None, prioritize=None): """ - path = tuple(p for p in new_path_dict) - not_active_path = tuple( - p for p in new_path_dict if not new_path_dict[p] - ) - - # Don't set options unless necessary - if path != self.path: - self.set_conf('path', path) - self.path = path + Save user and system path dictionaries and prioritize to config. + + Parameters + ---------- + user_paths: OrderedDict + Paths set by the user. + system_paths: OrderedDict + Paths set in the PYTHONPATH environment variable. + prioritize: bool + Whether paths should be prepended (True) or appended (False) to + sys.path. + + Notes + ----- + - Each dictionary key is a path and the value is the active state. + - sig_pythonpath_changed is emitted from this method, and nowhere else, + on condition that _spyder_pythonpath changed. - if not_active_path != self.not_active_path: - self.set_conf('not_active_path', not_active_path) - self.not_active_path = not_active_path - - def _get_spyder_pythonpath_dict(self): - """ - Return Spyder PYTHONPATH plus project path as dictionary of paths. - - The returned ordered dictionary has the paths as keys and the state - as values. The state is `True` for active and `False` for inactive. - - Example: - OrderedDict([('/some/path, True), ('/some/other/path, False)]) - """ - path_dict = OrderedDict() - - # Make project path to be the first one so that modules developed in a - # project are not shadowed by those present in other paths. - for path in self.project_path: - path_dict[path] = True - - for path in self.path: - path_dict[path] = path not in self.not_active_path - - return path_dict - - def _update_python_path(self, new_path_dict=None): """ - Update Python path on language server and kernels. + assert isinstance(user_paths, (type(None), OrderedDict)) + assert isinstance(system_paths, (type(None), OrderedDict)) + assert isinstance(prioritize, (type(None), bool)) - The new_path_dict should not include the project path. - """ - # Load existing path plus project path - old_path_dict_p = self._get_spyder_pythonpath_dict() - - # Save new path - if new_path_dict is not None: - self._save_paths(new_path_dict) - - # Load new path plus project path - new_path_dict_p = self._get_spyder_pythonpath_dict() + emit = False - # Do not notify observers unless necessary - if new_path_dict_p != old_path_dict_p: - pypath = self.get_spyder_pythonpath() - logger.debug(f"Update Pythonpath to {pypath}") - self.set_conf('spyder_pythonpath', pypath) - self.sig_pythonpath_changed.emit(old_path_dict_p, new_path_dict_p) + # Don't set options unless necessary + if user_paths is not None and user_paths != self._user_paths: + logger.debug(f"Saving user paths: {user_paths}") + self.set_conf('user_paths', dict(user_paths)) + self._user_paths = user_paths + + if system_paths is not None and system_paths != self._system_paths: + logger.debug(f"Saving system paths: {system_paths}") + self.set_conf('system_paths', dict(system_paths)) + self._system_paths = system_paths + + if prioritize is not None and prioritize != self._prioritize: + logger.debug(f"Saving prioritize: {prioritize}") + self.set_conf('prioritize', prioritize) + self._prioritize = prioritize + emit = True + + spyder_pythonpath = self.get_spyder_pythonpath() + if spyder_pythonpath != self._spyder_pythonpath: + logger.debug(f"Saving Spyder pythonpath: {spyder_pythonpath}") + self.set_conf('spyder_pythonpath', spyder_pythonpath) + self._spyder_pythonpath = spyder_pythonpath + emit = True + + # Only emit signal if spyder_pythonpath or prioritize changed + if emit: + self.sig_pythonpath_changed.emit( + self._spyder_pythonpath, self._prioritize + ) def _migrate_to_config_options(self): """ Migrate paths saved in the `path` and `not_active_path` files located in our config directory to our config system. - This was the way we save those paths in Spyder 5 and before. + # TODO: Remove for Spyder 7 """ path_file = get_conf_path('path') not_active_path_file = get_conf_path('not_active_path') + config_path = self.get_conf('path', None) + config_not_active_path = self.get_conf('not_active_path', None) + paths_in_conf_files = self.get_conf('paths_in_conf_files', None) + system_path = self.get_conf('system_path', None) path = [] + not_active_path = [] + + # Get path from file if osp.isfile(path_file): with open(path_file, 'r', encoding='utf-8') as f: path = f.read().splitlines() + try: + os.remove(path_file) + except OSError: + pass - not_active_path = [] + # Get inactive paths from file if osp.isfile(not_active_path_file): with open(not_active_path_file, 'r', encoding='utf-8') as f: not_active_path = f.read().splitlines() - - self.set_conf('path', tuple(path)) - self.set_conf('not_active_path', tuple(not_active_path)) - self.set_conf('paths_in_conf_files', False) + try: + os.remove(not_active_path_file) + except OSError: + pass + + # Get path from config; supersedes paths from file + if config_path is not None: + path = config_path + self.remove_conf('path') + + # Get inactive path from config; supersedes paths from file + if config_not_active_path is not None: + not_active_path = config_not_active_path + self.remove_conf('not_active_path') + + if paths_in_conf_files is not None: + self.remove_conf('paths_in_conf_files') + + # Get system path + system_paths = {} + if system_path is not None: + system_paths = {p: p not in not_active_path for p in system_path} + self.remove_conf('system_path') + + # path config has all user and system paths; only want user paths + user_paths = { + p: p not in not_active_path for p in path if p not in system_path + } + + + # Update the configuration + self.set_conf('user_paths', user_paths) + self.set_conf('system_paths', system_paths) + + # Do not migrate again + self.set_conf("config_options_migrated", True) diff --git a/spyder/plugins/pythonpath/plugin.py b/spyder/plugins/pythonpath/plugin.py index 879bedd1eb9..d7e5c15ee4b 100644 --- a/spyder/plugins/pythonpath/plugin.py +++ b/spyder/plugins/pythonpath/plugin.py @@ -34,25 +34,18 @@ class PythonpathManager(SpyderPluginV2): CONF_SECTION = NAME CONF_FILE = False - sig_pythonpath_changed = Signal(object, object) + sig_pythonpath_changed = Signal(object, bool) """ This signal is emitted when there is a change in the Pythonpath handled by Spyder. Parameters ---------- - old_path_dict: OrderedDict - Previous Pythonpath ordered dictionary. Its keys correspond to the - project, user and system paths declared by users or detected by Spyder, - and its values are their state (i.e. True for enabled and False for - disabled). - - new_path_dict: OrderedDict - New Pythonpath dictionary. - - See Also - -------- - :py:meth:`.PythonpathContainer._get_spyder_pythonpath_dict` + new_path_list: list of str + New list of PYTHONPATH paths. + + prioritize + Whether to prioritize PYTHONPATH in sys.path """ # ---- SpyderPluginV2 API diff --git a/spyder/plugins/pythonpath/widgets/pathmanager.py b/spyder/plugins/pythonpath/widgets/pathmanager.py index c04994d7657..0b9ed6ddb67 100644 --- a/spyder/plugins/pythonpath/widgets/pathmanager.py +++ b/spyder/plugins/pythonpath/widgets/pathmanager.py @@ -43,20 +43,21 @@ class PathManagerToolbuttons: MoveToBottom = 'move_to_bottom' AddPath = 'add_path' RemovePath = 'remove_path' + ImportPaths = 'import_paths' ExportPaths = 'export_paths' + Prioritize = 'prioritize' class PathManager(QDialog, SpyderWidgetMixin): """Path manager dialog.""" redirect_stdio = Signal(bool) - sig_path_changed = Signal(object) + sig_path_changed = Signal(object, object, bool) # This is required for our tests CONF_SECTION = 'pythonpath_manager' - def __init__(self, parent, path=None, project_path=None, - not_active_path=None, sync=True): + def __init__(self, parent, sync=True): """Path manager dialog.""" if PYQT5 or PYQT6: super().__init__(parent, class_parent=parent) @@ -64,24 +65,11 @@ def __init__(self, parent, path=None, project_path=None, QDialog.__init__(self, parent) SpyderWidgetMixin.__init__(self, class_parent=parent) - assert isinstance(path, (tuple, type(None))) - # Style # NOTE: This needs to be here so all buttons are styled correctly self.setStyleSheet(self._stylesheet) - self.path = path or () - self.project_path = project_path or () - self.not_active_path = not_active_path or () - self.last_path = getcwd_or_home() - self.original_path_dict = None - self.system_path = () - self.user_path = [] - - # This is necessary to run our tests - if self.path: - self.update_paths(system_path=get_system_pythonpath()) # Widgets self.add_button = None @@ -91,6 +79,7 @@ def __init__(self, parent, path=None, project_path=None, self.movedown_button = None self.movebottom_button = None self.export_button = None + self.prioritize_button = None self.user_header = None self.project_header = None self.system_header = None @@ -143,8 +132,11 @@ def __init__(self, parent, path=None, project_path=None, self.bbox.accepted.connect(self.accept) self.bbox.rejected.connect(self.reject) - # Setup - self.setup() + # Attributes + self.project_path = None + self.user_paths = None + self.system_paths = None + self.prioritize = None # ---- Private methods # ------------------------------------------------------------------------- @@ -185,32 +177,41 @@ def _setup_right_toolbar(self): tip=_('Remove path'), icon=self.create_icon('editclear'), triggered=lambda x: self.remove_path()) + self.import_button = self.create_toolbutton( + PathManagerToolbuttons.ImportPaths, + tip=_('Import from PYTHONPATH environment variable'), + icon=self.create_icon('fileimport'), + triggered=lambda x: self.import_pythonpath()) self.export_button = self.create_toolbutton( PathManagerToolbuttons.ExportPaths, icon=self.create_icon('fileexport'), triggered=self.export_pythonpath, tip=_("Export to PYTHONPATH environment variable")) + self.prioritize_button = self.create_toolbutton( + PathManagerToolbuttons.Prioritize, + option='prioritize', + triggered=self.refresh, + ) + self.prioritize_button.setCheckable(True) self.selection_widgets = [self.movetop_button, self.moveup_button, self.movedown_button, self.movebottom_button] return ( [self.add_button, self.remove_button] + - self.selection_widgets + [self.export_button] + self.selection_widgets + [self.import_button, self.export_button] + + [self.prioritize_button] ) - def _create_item(self, path): + def _create_item(self, path, active): """Helper to create a new list item.""" item = QListWidgetItem(path) if path in self.project_path: item.setFlags(Qt.NoItemFlags | Qt.ItemIsUserCheckable) item.setCheckState(Qt.Checked) - elif path in self.not_active_path: - item.setFlags(item.flags() | Qt.ItemIsUserCheckable) - item.setCheckState(Qt.Unchecked) else: item.setFlags(item.flags() | Qt.ItemIsUserCheckable) - item.setCheckState(Qt.Checked) + item.setCheckState(Qt.Checked if active else Qt.Unchecked) return item @@ -259,6 +260,23 @@ def _stylesheet(self): return css.toString() + def _setup_system_paths(self, paths): + """Add system paths, creating system header if necessary""" + if not paths: + return + + if not self.system_header: + self.system_header, system_widget = ( + self._create_header(_("System PYTHONPATH")) + ) + self.headers.append(self.system_header) + self.listwidget.addItem(self.system_header) + self.listwidget.setItemWidget(self.system_header, system_widget) + + for path, active in paths.items(): + item = self._create_item(path, active) + self.listwidget.addItem(item) + # ---- Public methods # ------------------------------------------------------------------------- @property @@ -269,7 +287,7 @@ def editable_bottom_row(self): if self.project_header: bottom_row += len(self.project_path) + 1 if self.user_header: - bottom_row += len(self.user_path) + bottom_row += len(self.get_user_paths()) return bottom_row @@ -302,12 +320,12 @@ def setup(self): self.listwidget.addItem(self.project_header) self.listwidget.setItemWidget(self.project_header, project_widget) - for path in self.project_path: - item = self._create_item(path) + for path, active in self.project_path.items(): + item = self._create_item(path, active) self.listwidget.addItem(item) # Paths added by the user - if self.user_path: + if self.user_paths: self.user_header, user_widget = ( self._create_header(_("User paths")) ) @@ -315,25 +333,17 @@ def setup(self): self.listwidget.addItem(self.user_header) self.listwidget.setItemWidget(self.user_header, user_widget) - for path in self.user_path: - item = self._create_item(path) + for path, active in self.user_paths.items(): + item = self._create_item(path, active) self.listwidget.addItem(item) - # System path - if self.system_path: - self.system_header, system_widget = ( - self._create_header(_("System PYTHONPATH")) - ) - self.headers.append(self.system_header) - self.listwidget.addItem(self.system_header) - self.listwidget.setItemWidget(self.system_header, system_widget) + # System paths + self._setup_system_paths(self.system_paths) - for path in self.system_path: - item = self._create_item(path) - self.listwidget.addItem(item) + # Prioritize + self.prioritize_button.setChecked(self.prioritize) self.listwidget.setCurrentRow(0) - self.original_path_dict = self.get_path_dict() self.refresh() @Slot() @@ -341,6 +351,15 @@ def export_pythonpath(self): """ Export to PYTHONPATH environment variable Only apply to: current user. + + If the user chooses to clear the contents of the system PYTHONPATH, + then the active user paths are prepended to active system paths and + the resulting list is saved to the system PYTHONPATH. Inactive system + paths are discarded. If the user chooses not to clear the contents of + the system PYTHONPATH, then the new system PYTHONPATH comprises the + inactive system paths + active user paths + active system paths, and + inactive system paths remain inactive. With either choice, inactive + user paths are retained in the user paths and remain inactive. """ answer = QMessageBox.question( self, @@ -358,77 +377,96 @@ def export_pythonpath(self): if answer == QMessageBox.Cancel: return + user_paths = self.get_user_paths() + active_user_paths = OrderedDict( + {p: v for p, v in user_paths.items() if v} + ) + new_user_paths = OrderedDict( + {p: v for p, v in user_paths.items() if not v} + ) + + system_paths = self.get_system_paths() + active_system_paths = OrderedDict( + {p: v for p, v in system_paths.items() if v} + ) + inactive_system_paths = OrderedDict( + {p: v for p, v in system_paths.items() if not v} + ) + + # Desired behavior is active_user | active_system, but Python 3.8 does + # not support | operator for OrderedDict. + new_system_paths = OrderedDict(reversed(active_system_paths.items())) + new_system_paths.update(reversed(active_user_paths.items())) + if answer == QMessageBox.No: + # Desired behavior is inactive_system | active_user | active_system + new_system_paths.update(reversed(inactive_system_paths.items())) + new_system_paths = OrderedDict(reversed(new_system_paths.items())) + env = get_user_env() + env['PYTHONPATH'] = list(new_system_paths.keys()) + set_user_env(env, parent=self) + + self.update_paths( + user_paths=new_user_paths, system_paths=new_system_paths + ) - # This doesn't include the project path because it's a transient - # directory, i.e. only used in Spyder and during specific - # circumstances. - active_path = [k for k, v in self.get_path_dict().items() if v] + def get_user_paths(self): + """Get current user paths as displayed on listwidget.""" + paths = OrderedDict() - if answer == QMessageBox.Yes: - ppath = active_path - else: - ppath = env.get('PYTHONPATH', []) - if not isinstance(ppath, list): - ppath = [ppath] + if self.user_header is None: + return paths - ppath = [p for p in ppath if p not in active_path] - ppath = ppath + active_path + start = self.listwidget.row(self.user_header) + 1 + stop = self.listwidget.count() + if self.system_header is not None: + stop = self.listwidget.row(self.system_header) - os.environ['PYTHONPATH'] = os.pathsep.join(ppath) + for row in range(start, stop): + item = self.listwidget.item(row) + paths.update({item.text(): item.checkState() == Qt.Checked}) - # Update widget so changes are reflected on it immediately - self.update_paths(system_path=tuple(ppath)) - self.set_conf('system_path', tuple(ppath)) - self.setup() + return paths - env['PYTHONPATH'] = list(ppath) - set_user_env(env, parent=self) + def get_system_paths(self): + """Get current system paths as displayed on listwidget.""" + paths = OrderedDict() + + if self.system_header is None: + return paths + + start = self.listwidget.row(self.system_header) + 1 + for row in range(start, self.listwidget.count()): + item = self.listwidget.item(row) + paths.update({item.text(): item.checkState() == Qt.Checked}) + + return paths - def get_path_dict(self, project_path=False): + def update_paths( + self, + project_path=None, + user_paths=None, + system_paths=None, + prioritize=None + ): """ - Return an ordered dict with the path entries as keys and the active - state as the value. + Update path attributes. - If `project_path` is True, its entries are also included. + These attributes should only be set in this method and upon activating + the dialog. They should remain fixed while the dialog is active and are + used to compare with what is shown in the listwidget in order to detect + changes. """ - odict = OrderedDict() - for row in range(self.listwidget.count()): - item = self.listwidget.item(row) - path = item.text() - if item not in self.headers: - if path in self.project_path and not project_path: - continue - odict[path] = item.checkState() == Qt.Checked - - return odict - - def get_user_path(self): - """Get current user path as displayed on listwidget.""" - user_path = [] - for row in range(self.listwidget.count()): - item = self.listwidget.item(row) - path = item.text() - if item not in self.headers: - if path not in (self.project_path + self.system_path): - user_path.append(path) - - return user_path - - def update_paths(self, path=None, not_active_path=None, system_path=None): - """Update path attributes.""" - if path is not None: - self.path = path - if not_active_path is not None: - self.not_active_path = not_active_path - if system_path is not None: - self.system_path = system_path - - previous_system_path = self.get_conf('system_path', ()) - self.user_path = [ - path for path in self.path - if path not in (self.system_path + previous_system_path) - ] + if project_path is not None: + self.project_path = project_path + if user_paths is not None: + self.user_paths = user_paths + if system_paths is not None: + self.system_paths = system_paths + if prioritize is not None: + self.prioritize = prioritize + + self.setup() def refresh(self): """Refresh toolbar widgets.""" @@ -462,15 +500,24 @@ def refresh(self): # Enable remove button only for user paths self.remove_button.setEnabled( - not current_item in self.headers + current_item not in self.headers and (self.editable_top_row <= row <= self.editable_bottom_row) ) + if self.prioritize_button.isChecked(): + self.prioritize_button.setIcon(self.create_icon('prepend')) + self.prioritize_button.setToolTip(_("Paths are prepended to sys.path")) + else: + self.prioritize_button.setIcon(self.create_icon('append')) + self.prioritize_button.setToolTip(_("Paths are appended to sys.path")) + self.export_button.setEnabled(self.listwidget.count() > 0) # Ok button only enabled if actual changes occur self.button_ok.setEnabled( - self.original_path_dict != self.get_path_dict() + self.user_paths != self.get_user_paths() + or self.system_paths != self.get_system_paths() + or self.prioritize != self.prioritize_button.isChecked() ) @Slot() @@ -491,7 +538,7 @@ def add_path(self, directory=None): directory = osp.abspath(directory) self.last_path = directory - if directory in self.get_path_dict(): + if directory in self.get_user_paths(): item = self.listwidget.findItems(directory, Qt.MatchExactly)[0] item.setCheckState(Qt.Checked) answer = QMessageBox.question( @@ -499,7 +546,7 @@ def add_path(self, directory=None): _("Add path"), _("This directory is already included in the list." "
" - "Do you want to move it to the top of it?"), + "Do you want to move it to the top of the list?"), QMessageBox.Yes | QMessageBox.No) if answer == QMessageBox.Yes: @@ -524,11 +571,9 @@ def add_path(self, directory=None): ) # Add new path - item = self._create_item(directory) + item = self._create_item(directory, True) self.listwidget.insertItem(self.editable_top_row, item) self.listwidget.setCurrentRow(self.editable_top_row) - - self.user_path.insert(0, directory) else: answer = QMessageBox.warning( self, @@ -565,15 +610,11 @@ def remove_path(self, force=False): QMessageBox.Yes | QMessageBox.No) if force or answer == QMessageBox.Yes: - # Remove current item from user_path - item = self.listwidget.currentItem() - self.user_path.remove(item.text()) - # Remove selected item from view self.listwidget.takeItem(self.listwidget.currentRow()) # Remove user header if there are no more user paths - if len(self.user_path) == 0: + if len(self.get_user_paths()) == 0: self.listwidget.takeItem( self.listwidget.row(self.user_header) ) @@ -583,6 +624,33 @@ def remove_path(self, force=False): # Refresh widget self.refresh() + @Slot() + def import_pythonpath(self): + """Import PYTHONPATH from environment.""" + current_system_paths = self.get_system_paths() + system_paths = get_system_pythonpath() + + # Inherit active state from current system paths + system_paths = OrderedDict( + {p: current_system_paths.get(p, True) for p in system_paths} + ) + + # Remove system paths + if self.system_header: + header_row = self.listwidget.row(self.system_header) + for row in range(self.listwidget.count(), header_row, -1): + self.listwidget.takeItem(row) + + # Also remove system header + if not system_paths: + self.listwidget.takeItem(header_row) + self.headers.remove(self.system_header) + self.system_header = None + + self._setup_system_paths(system_paths) + + self.refresh() + def move_to(self, absolute=None, relative=None): """Move items of list widget.""" index = self.listwidget.currentRow() @@ -599,7 +667,6 @@ def move_to(self, absolute=None, relative=None): self.listwidget.insertItem(new_index, item) self.listwidget.setCurrentRow(new_index) - self.user_path = self.get_user_path() self.refresh() def current_row(self): @@ -626,30 +693,15 @@ def count(self): # ---- Qt methods # ------------------------------------------------------------------------- - def _update_system_path(self): - """ - Request to update path values on main window if current and previous - system paths are different. - """ - if self.system_path != self.get_conf('system_path', default=()): - self.sig_path_changed.emit(self.get_path_dict()) - self.set_conf('system_path', self.system_path) - def accept(self): """Override Qt method.""" - path_dict = self.get_path_dict() - if self.original_path_dict != path_dict: - self.sig_path_changed.emit(path_dict) + self.sig_path_changed.emit( + self.get_user_paths(), + self.get_system_paths(), + self.prioritize_button.isChecked() + ) super().accept() - def reject(self): - self._update_system_path() - super().reject() - - def closeEvent(self, event): - self._update_system_path() - super().closeEvent(event) - def test(): """Run path manager test.""" @@ -658,12 +710,25 @@ def test(): _ = qapplication() dlg = PathManager( None, - path=tuple(sys.path[:1]), - project_path=tuple(sys.path[-2:]), + ) + dlg.update_paths( + user_paths={p: True for p in sys.path[1:-2]}, + project_path={p: True for p in sys.path[:1]}, + system_paths={p: True for p in sys.path[-2:]}, + prioritize=False ) - def callback(path_dict): - sys.stdout.write(str(path_dict)) + def callback(user_paths, system_paths, prioritize): + sys.stdout.write(f"Prioritize: {prioritize}") + sys.stdout.write("\n---- User paths ----\n") + sys.stdout.write( + '\n'.join([f'{k}: {v}' for k, v in user_paths.items()]) + ) + sys.stdout.write("\n---- System paths ----\n") + sys.stdout.write( + '\n'.join([f'{k}: {v}' for k, v in system_paths.items()]) + ) + sys.stdout.write('\n') dlg.sig_path_changed.connect(callback) sys.exit(dlg.exec_()) diff --git a/spyder/plugins/pythonpath/widgets/tests/test_pathmanager.py b/spyder/plugins/pythonpath/widgets/tests/test_pathmanager.py index f6e73a0e384..375385cd4cc 100644 --- a/spyder/plugins/pythonpath/widgets/tests/test_pathmanager.py +++ b/spyder/plugins/pythonpath/widgets/tests/test_pathmanager.py @@ -8,6 +8,7 @@ Tests for pathmanager.py """ # Standard library imports +from collections import OrderedDict import sys import os @@ -17,7 +18,9 @@ from qtpy.QtWidgets import QMessageBox, QPushButton # Local imports +from spyder.utils.environ import get_user_env, set_user_env from spyder.utils.programs import is_module_installed +from spyder.utils.tests.conftest import restore_user_env from spyder.plugins.pythonpath.utils import check_path from spyder.plugins.pythonpath.widgets import pathmanager as pathmanager_mod @@ -25,29 +28,57 @@ @pytest.fixture def pathmanager(qtbot, request): """Set up PathManager.""" - path, project_path, not_active_path = request.param - widget = pathmanager_mod.PathManager( - None, - path=tuple(path), - project_path=tuple(project_path), - not_active_path=tuple(not_active_path)) + user_paths, project_path, system_paths = request.param + + widget = pathmanager_mod.PathManager(None) + widget.update_paths( + user_paths=OrderedDict({p: True for p in user_paths}), + project_path=OrderedDict({p: True for p in project_path}), + system_paths=OrderedDict({p: True for p in system_paths}), + prioritize=False + ) widget.show() qtbot.addWidget(widget) return widget -@pytest.mark.parametrize('pathmanager', - [(sys.path[:-10], sys.path[-10:], ())], - indirect=True) -def test_pathmanager(pathmanager, qtbot): +@pytest.mark.parametrize( + 'pathmanager', [(sys.path[:-10], sys.path[-10:], ())], indirect=True +) +def test_pathmanager(qtbot, pathmanager): """Run PathManager test""" pathmanager.show() assert pathmanager -@pytest.mark.parametrize('pathmanager', - [(sys.path[:-10], sys.path[-10:], ())], - indirect=True) +@pytest.mark.parametrize('pathmanager', [((), (), ())], indirect=True) +def test_import_PYTHONPATH(qtbot, pathmanager, tmp_path, restore_user_env): + """ + Test that PYTHONPATH is imported. + """ + + # Add a directory to PYTHONPATH environment variable + sys_dir = tmp_path / 'sys_dir' + sys_dir.mkdir() + set_user_env({"PYTHONPATH": str(sys_dir)}) + + # Open Pythonpath dialog + pathmanager.show() + qtbot.wait(500) + + assert len(pathmanager.headers) == 0 + assert pathmanager.get_system_paths() == OrderedDict() + + # Import PYTHONPATH from environment + pathmanager.import_pythonpath() + assert len(pathmanager.headers) == 1 + + assert pathmanager.get_system_paths() == OrderedDict({str(sys_dir): True}) + + +@pytest.mark.parametrize( + 'pathmanager', [(sys.path[:-10], sys.path[-10:], ())], indirect=True +) def test_check_uncheck_path(pathmanager): """ Test that checking and unchecking a path in the PathManager correctly @@ -60,20 +91,16 @@ def test_check_uncheck_path(pathmanager): assert item.checkState() == Qt.Checked -@pytest.mark.skipif(os.name != 'nt' or not is_module_installed('win32con'), - reason=("This feature is not applicable for Unix " - "systems and pywin32 is needed")) -@pytest.mark.parametrize('pathmanager', - [(['p1', 'p2', 'p3'], ['p4', 'p5', 'p6'], [])], - indirect=True) -def test_export_to_PYTHONPATH(pathmanager, mocker): - # Import here to prevent an ImportError when testing on unix systems - from spyder.utils.environ import (get_user_env, set_user_env, - listdict2envdict) - - # Store PYTHONPATH original state - env = get_user_env() - original_pathlist = env.get('PYTHONPATH', []) +@pytest.mark.skipif( + os.name != 'nt' or not is_module_installed('win32con'), + reason=("This feature is not applicable for Unix " + "systems and pywin32 is needed") +) +@pytest.mark.parametrize( + 'pathmanager', [(['p1', 'p2', 'p3'], ['p4', 'p5', 'p6'], [])], + indirect=True +) +def test_export_to_PYTHONPATH(pathmanager, mocker, restore_user_env): # Mock the dialog window and answer "Yes" to clear contents of PYTHONPATH # before adding Spyder's path list @@ -107,14 +134,10 @@ def test_export_to_PYTHONPATH(pathmanager, mocker): env = get_user_env() assert env['PYTHONPATH'] == expected_pathlist - # Restore PYTHONPATH to its original state - env['PYTHONPATH'] = original_pathlist - set_user_env(listdict2envdict(env)) - -@pytest.mark.parametrize('pathmanager', - [(sys.path[:-10], sys.path[-10:], ())], - indirect=True) +@pytest.mark.parametrize( + 'pathmanager', [(sys.path[:-10], sys.path[-10:], ())], indirect=True +) def test_invalid_directories(qtbot, pathmanager): """Check [site/dist]-packages are invalid paths.""" if os.name == 'nt': @@ -137,9 +160,9 @@ def interact_message_box(): pathmanager.add_path(path) -@pytest.mark.parametrize('pathmanager', - [(('/spam', '/bar'), ('/foo', ), ())], - indirect=True) +@pytest.mark.parametrize( + 'pathmanager', [(('/spam', '/bar'), ('/foo', ), ())], indirect=True +) def test_remove_item_and_reply_no(qtbot, pathmanager): """Check that the item is not removed after answering 'No'.""" pathmanager.show() @@ -163,9 +186,9 @@ def interact_message_box(): assert pathmanager.count() == count -@pytest.mark.parametrize('pathmanager', - [(('/spam', '/bar'), ('/foo', ), ())], - indirect=True) +@pytest.mark.parametrize( + 'pathmanager', [(('/spam', '/bar'), ('/foo', ), ())], indirect=True +) def test_remove_item_and_reply_yes(qtbot, pathmanager): """Check that the item is indeed removed after answering 'Yes'.""" pathmanager.show() @@ -190,9 +213,7 @@ def interact_message_box(): assert pathmanager.count() == (count - 1) -@pytest.mark.parametrize('pathmanager', - [((), (), ())], - indirect=True) +@pytest.mark.parametrize('pathmanager', [((), (), ())], indirect=True) def test_add_repeated_item(qtbot, pathmanager, tmpdir): """ Check behavior when an unchecked item that is already on the list is added. @@ -207,7 +228,7 @@ def test_add_repeated_item(qtbot, pathmanager, tmpdir): pathmanager.add_path(dir2) pathmanager.add_path(dir3) pathmanager.set_row_check_state(2, Qt.Unchecked) - assert not all(pathmanager.get_path_dict().values()) + assert not all(pathmanager.get_user_paths().values()) def interact_message_box(): messagebox = pathmanager.findChild(QMessageBox) @@ -222,17 +243,17 @@ def interact_message_box(): timer.timeout.connect(interact_message_box) timer.start(500) pathmanager.add_path(dir2) - print(pathmanager.get_path_dict()) + print(pathmanager.get_user_paths()) # Back to main thread assert pathmanager.count() == 4 - assert list(pathmanager.get_path_dict().keys())[0] == dir2 - assert all(pathmanager.get_path_dict().values()) + assert list(pathmanager.get_user_paths().keys())[0] == dir2 + assert all(pathmanager.get_user_paths().values()) -@pytest.mark.parametrize('pathmanager', - [(('/spam', '/bar'), ('/foo', ), ())], - indirect=True) +@pytest.mark.parametrize( + 'pathmanager', [(('/spam', '/bar'), ('/foo', ), ())], indirect=True +) def test_buttons_state(qtbot, pathmanager, tmpdir): """Check buttons are enabled/disabled based on items and position.""" pathmanager.show() @@ -277,6 +298,14 @@ def test_buttons_state(qtbot, pathmanager, tmpdir): pathmanager.remove_path(True) assert not pathmanager.button_ok.isEnabled() + # Check prioritize button + assert pathmanager.prioritize_button.isEnabled() + assert not pathmanager.prioritize_button.isChecked() + pathmanager.prioritize_button.animateClick() + qtbot.waitUntil(pathmanager.prioritize_button.isChecked) + assert pathmanager.prioritize_button.isChecked() + assert pathmanager.button_ok.isEnabled() + if __name__ == "__main__": pytest.main([os.path.basename(__file__)]) diff --git a/spyder/utils/environ.py b/spyder/utils/environ.py index 5b1d36606ed..74c9ea3b388 100644 --- a/spyder/utils/environ.py +++ b/spyder/utils/environ.py @@ -27,7 +27,9 @@ from qtpy.QtWidgets import QMessageBox # Local imports -from spyder.config.base import _, running_in_ci, get_conf_path +from spyder.config.base import ( + _, running_in_ci, get_conf_path, running_under_pytest +) from spyder.widgets.collectionseditor import CollectionsEditor from spyder.utils.icon_manager import ima from spyder.utils.programs import run_shell_command @@ -111,7 +113,7 @@ def get_user_environment_variables(): # We only need to do this if Spyder was **not** launched from a # terminal. Otherwise, it'll inherit the env vars present in it. # Fixes spyder-ide/spyder#22415 - if not launched_from_terminal: + if not launched_from_terminal or running_under_pytest(): try: user_env_script = _get_user_env_script() proc = run_shell_command(user_env_script, env={}, text=True) diff --git a/spyder/utils/icon_manager.py b/spyder/utils/icon_manager.py index 1ce13d1216c..6c709d55262 100644 --- a/spyder/utils/icon_manager.py +++ b/spyder/utils/icon_manager.py @@ -290,6 +290,8 @@ def __init__(self): '1uparrow': [('mdi.arrow-up',), {'color': self.MAIN_FG_COLOR}], '2downarrow': [('mdi.arrow-collapse-down',), {'color': self.MAIN_FG_COLOR}], '1downarrow': [('mdi.arrow-down',), {'color': self.MAIN_FG_COLOR}], + 'prepend': [('mdi.arrow-collapse-left',), {'color': self.MAIN_FG_COLOR}], + 'append': [('mdi.arrow-collapse-right',), {'color': self.MAIN_FG_COLOR}], 'undock': [('mdi.open-in-new',), {'color': self.MAIN_FG_COLOR}], 'close_pane': [('mdi.window-close',), {'color': self.MAIN_FG_COLOR}], 'toolbar_ext_button': [('mdi.dots-horizontal',), {'color': self.MAIN_FG_COLOR}], diff --git a/spyder/utils/tests/conftest.py b/spyder/utils/tests/conftest.py new file mode 100644 index 00000000000..361d6442845 --- /dev/null +++ b/spyder/utils/tests/conftest.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Standard library imports +import os + +# Third-party imports +import pytest + +# Local imports +from spyder.config.base import running_in_ci +from spyder.utils.environ import ( + get_user_env, set_user_env, amend_user_shell_init +) + + +@pytest.fixture +def restore_user_env(): + """Set user environment variables and restore upon test exit""" + if not running_in_ci(): + pytest.skip("Skipped because not in CI.") + + if os.name == "nt": + orig_env = get_user_env() + + yield + + if os.name == "nt": + set_user_env(orig_env) + else: + amend_user_shell_init(restore=True) diff --git a/spyder/utils/tests/test_environ.py b/spyder/utils/tests/test_environ.py index 277f2418be6..d4f771d9848 100644 --- a/spyder/utils/tests/test_environ.py +++ b/spyder/utils/tests/test_environ.py @@ -18,15 +18,15 @@ from qtpy.QtCore import QTimer # Local imports -from spyder.utils.environ import (get_user_environment_variables, - UserEnvDialog, amend_user_shell_init) +from spyder.utils.environ import ( + get_user_environment_variables, UserEnvDialog, amend_user_shell_init +) from spyder.utils.test import close_message_box -from spyder.app.tests.conftest import restore_user_env @pytest.fixture def environ_dialog(qtbot): - "Setup the Environment variables Dialog." + """Setup the Environment variables Dialog.""" QTimer.singleShot(1000, lambda: close_message_box(qtbot)) dialog = UserEnvDialog() qtbot.addWidget(dialog) @@ -44,8 +44,10 @@ def test_get_user_environment_variables(): @pytest.mark.skipif(os.name == "nt", reason="Does not apply to Windows") def test_get_user_env_newline(restore_user_env): - # Test variable value with newline characters. - # Regression test for spyder-ide#20097 + """ + Test variable value with newline characters. + Regression test for spyder-ide#20097. + """ text = "myfunc() { echo hello;\n echo world\n}\nexport -f myfunc" amend_user_shell_init(text) user_env = get_user_environment_variables()