diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index cbae97f12f5377..affd366011af8e 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -205,6 +205,9 @@ typedef struct PyConfig { wchar_t *run_module; wchar_t *run_filename; + /* --- Set by Py_Main() -------------------------- */ + wchar_t *sys_path_0; + /* --- Private fields ---------------------------- */ // Install importlib? If equals to 0, importlib is not initialized at all. diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 24617ab24c6958..1d8d9f5b604835 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -500,6 +500,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'run_command': None, 'run_module': None, 'run_filename': None, + 'sys_path_0': None, '_install_importlib': 1, 'check_hash_pycs_mode': 'default', @@ -1119,6 +1120,7 @@ def test_init_run_main(self): 'program_name': './python3', 'run_command': code + '\n', 'parse_argv': 2, + 'sys_path_0': '', } self.check_all_configs("test_init_run_main", config, api=API_PYTHON) @@ -1134,6 +1136,7 @@ def test_init_main(self): 'run_command': code + '\n', 'parse_argv': 2, '_init_main': 0, + 'sys_path_0': '', } self.check_all_configs("test_init_main", config, api=API_PYTHON, diff --git a/Lib/test/test_interpreters.py b/Lib/test/test_interpreters.py index 27a143c7f5f38d..1f7b946227b9a8 100644 --- a/Lib/test/test_interpreters.py +++ b/Lib/test/test_interpreters.py @@ -1,5 +1,8 @@ import contextlib +import json import os +import os.path +import sys import threading from textwrap import dedent import unittest @@ -8,6 +11,7 @@ from test import support from test.support import import_helper from test.support import threading_helper +from test.support import os_helper _interpreters = import_helper.import_module('_xxsubinterpreters') _channels = import_helper.import_module('_xxinterpchannels') from test.support import interpreters @@ -488,6 +492,154 @@ def task(): pass +class StartupTests(TestBase): + + # We want to ensure the initial state of subinterpreters + # matches expectations. + + _subtest_count = 0 + + @contextlib.contextmanager + def subTest(self, *args): + with super().subTest(*args) as ctx: + self._subtest_count += 1 + try: + yield ctx + finally: + if self._debugged_in_subtest: + if self._subtest_count == 1: + # The first subtest adds a leading newline, so we + # compensate here by not printing a trailing newline. + print('### end subtest debug ###', end='') + else: + print('### end subtest debug ###') + self._debugged_in_subtest = False + + def debug(self, msg, *, header=None): + if header: + self._debug(f'--- {header} ---') + if msg: + if msg.endswith(os.linesep): + self._debug(msg[:-len(os.linesep)]) + else: + self._debug(msg) + self._debug('') + self._debug('------') + else: + self._debug(msg) + + _debugged = False + _debugged_in_subtest = False + def _debug(self, msg): + if not self._debugged: + print() + self._debugged = True + if self._subtest is not None: + if True: + if not self._debugged_in_subtest: + self._debugged_in_subtest = True + print('### start subtest debug ###') + print(msg) + else: + print(msg) + + def create_temp_dir(self): + import tempfile + tmp = tempfile.mkdtemp(prefix='test_interpreters_') + tmp = os.path.realpath(tmp) + self.addCleanup(os_helper.rmtree, tmp) + return tmp + + def write_script(self, *path, text): + filename = os.path.join(*path) + dirname = os.path.dirname(filename) + if dirname: + os.makedirs(dirname, exist_ok=True) + with open(filename, 'w', encoding='utf-8') as outfile: + outfile.write(dedent(text)) + return filename + + @support.requires_subprocess() + def run_python(self, argv, *, cwd=None): + # This method is inspired by + # EmbeddingTestsMixin.run_embedded_interpreter() in test_embed.py. + import shlex + import subprocess + if isinstance(argv, str): + argv = shlex.split(argv) + argv = [sys.executable, *argv] + try: + proc = subprocess.run( + argv, + cwd=cwd, + capture_output=True, + text=True, + ) + except Exception as exc: + self.debug(f'# cmd: {shlex.join(argv)}') + if isinstance(exc, FileNotFoundError) and not exc.filename: + if os.path.exists(argv[0]): + exists = 'exists' + else: + exists = 'does not exist' + self.debug(f'{argv[0]} {exists}') + raise # re-raise + assert proc.stderr == '' or proc.returncode != 0, proc.stderr + if proc.returncode != 0 and support.verbose: + self.debug(f'# python3 {shlex.join(argv[1:])} failed:') + self.debug(proc.stdout, header='stdout') + self.debug(proc.stderr, header='stderr') + self.assertEqual(proc.returncode, 0) + self.assertEqual(proc.stderr, '') + return proc.stdout + + def test_sys_path_0(self): + # The main interpreter's sys.path[0] should be used by subinterpreters. + script = ''' + import sys + from test.support import interpreters + + orig = sys.path[0] + + interp = interpreters.create() + interp.run(f"""if True: + import json + import sys + print(json.dumps({{ + 'main': {orig!r}, + 'sub': sys.path[0], + }}, indent=4), flush=True) + """) + ''' + # / + # pkg/ + # __init__.py + # __main__.py + # script.py + # script.py + cwd = self.create_temp_dir() + self.write_script(cwd, 'pkg', '__init__.py', text='') + self.write_script(cwd, 'pkg', '__main__.py', text=script) + self.write_script(cwd, 'pkg', 'script.py', text=script) + self.write_script(cwd, 'script.py', text=script) + + cases = [ + ('script.py', cwd), + ('-m script', cwd), + ('-m pkg', cwd), + ('-m pkg.script', cwd), + ('-c "import script"', ''), + ] + for argv, expected in cases: + with self.subTest(f'python3 {argv}'): + out = self.run_python(argv, cwd=cwd) + data = json.loads(out) + sp0_main, sp0_sub = data['main'], data['sub'] + self.assertEqual(sp0_sub, sp0_main) + self.assertEqual(sp0_sub, expected) + # XXX Also check them all with the -P cmdline flag? + + class TestIsShareable(TestBase): def test_default_shareables(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-09-27-18-01-06.gh-issue-109853.coQQiL.rst b/Misc/NEWS.d/next/Core and Builtins/2023-09-27-18-01-06.gh-issue-109853.coQQiL.rst new file mode 100644 index 00000000000000..45de3ba8877b01 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-09-27-18-01-06.gh-issue-109853.coQQiL.rst @@ -0,0 +1 @@ +``sys.path[0]`` is now set correctly for subinterpreters. diff --git a/Modules/main.c b/Modules/main.c index 7edfeb3365b4c6..489f1cf6c461a2 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -559,6 +559,11 @@ pymain_run_python(int *exitcode) goto error; } + // XXX Calculate config->sys_path_0 in getpath.py. + // The tricky part is that we can't check the path importers yet + // at that point. + assert(config->sys_path_0 == NULL); + if (config->run_filename != NULL) { /* If filename is a package (ex: directory or ZIP file) which contains __main__.py, main_importer_path is set to filename and will be @@ -574,24 +579,37 @@ pymain_run_python(int *exitcode) // import readline and rlcompleter before script dir is added to sys.path pymain_import_readline(config); + PyObject *path0 = NULL; if (main_importer_path != NULL) { - if (pymain_sys_path_add_path0(interp, main_importer_path) < 0) { - goto error; - } + path0 = Py_NewRef(main_importer_path); } else if (!config->safe_path) { - PyObject *path0 = NULL; int res = _PyPathConfig_ComputeSysPath0(&config->argv, &path0); if (res < 0) { goto error; } - - if (res > 0) { - if (pymain_sys_path_add_path0(interp, path0) < 0) { - Py_DECREF(path0); - goto error; - } + else if (res == 0) { + Py_CLEAR(path0); + } + } + // XXX Apply config->sys_path_0 in init_interp_main(). We have + // to be sure to get readline/rlcompleter imported at the correct time. + if (path0 != NULL) { + wchar_t *wstr = PyUnicode_AsWideCharString(path0, NULL); + if (wstr == NULL) { Py_DECREF(path0); + goto error; + } + config->sys_path_0 = _PyMem_RawWcsdup(wstr); + PyMem_Free(wstr); + if (config->sys_path_0 == NULL) { + Py_DECREF(path0); + goto error; + } + int res = pymain_sys_path_add_path0(interp, path0); + Py_DECREF(path0); + if (res < 0) { + goto error; } } diff --git a/Python/initconfig.c b/Python/initconfig.c index 4e5d4bb9876e3b..7e5cc4e20d8734 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -739,6 +739,7 @@ PyConfig_Clear(PyConfig *config) CLEAR(config->exec_prefix); CLEAR(config->base_exec_prefix); CLEAR(config->platlibdir); + CLEAR(config->sys_path_0); CLEAR(config->filesystem_encoding); CLEAR(config->filesystem_errors); @@ -993,6 +994,7 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) COPY_WSTR_ATTR(exec_prefix); COPY_WSTR_ATTR(base_exec_prefix); COPY_WSTR_ATTR(platlibdir); + COPY_WSTR_ATTR(sys_path_0); COPY_ATTR(site_import); COPY_ATTR(bytes_warning); @@ -1102,6 +1104,7 @@ _PyConfig_AsDict(const PyConfig *config) SET_ITEM_WSTR(exec_prefix); SET_ITEM_WSTR(base_exec_prefix); SET_ITEM_WSTR(platlibdir); + SET_ITEM_WSTR(sys_path_0); SET_ITEM_INT(site_import); SET_ITEM_INT(bytes_warning); SET_ITEM_INT(warn_default_encoding); @@ -1403,6 +1406,7 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) GET_WSTR_OPT(pythonpath_env); GET_WSTR_OPT(home); GET_WSTR(platlibdir); + GET_WSTR(sys_path_0); // Path configuration output GET_UINT(module_search_paths_set); @@ -3165,6 +3169,7 @@ _Py_DumpPathConfig(PyThreadState *tstate) PySys_WriteStderr(" import site = %i\n", config->site_import); PySys_WriteStderr(" is in build tree = %i\n", config->_is_python_build); DUMP_CONFIG("stdlib dir", stdlib_dir); + DUMP_CONFIG("sys.path[0]", sys_path_0); #undef DUMP_CONFIG #define DUMP_SYS(NAME) \ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 29771e07ae6a2c..ba4fd5c1c1c1e5 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1200,6 +1200,31 @@ init_interp_main(PyThreadState *tstate) #endif } + if (!is_main_interp) { + // The main interpreter is handled in Py_Main(), for now. + if (config->sys_path_0 != NULL) { + PyObject *path0 = PyUnicode_FromWideChar(config->sys_path_0, -1); + if (path0 == NULL) { + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + PyObject *sysdict = interp->sysdict; + if (sysdict == NULL) { + Py_DECREF(path0); + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + PyObject *sys_path = PyDict_GetItemWithError(sysdict, &_Py_ID(path)); + if (sys_path == NULL) { + Py_DECREF(path0); + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + int res = PyList_Insert(sys_path, 0, path0); + Py_DECREF(path0); + if (res) { + return _PyStatus_ERR("can't initialize sys.path[0]"); + } + } + } + assert(!_PyErr_Occurred(tstate)); return _PyStatus_OK();