Avoid symlinking the contents of /usr into PyPy3 virtualenvs (#2310)

Co-authored-by: Bernát Gábor <bgabor8@bloomberg.net>

docs/changelog/2310.bugfix.rst

@@ -0,0 +1 @@
+Avoid symlinking the contents of ``/usr`` into PyPy3.8+ virtualenvs - by :user:`stefanor`.

src/virtualenv/create/via_global_ref/builtin/pypy/pypy3.py

@@ -42,6 +42,11 @@ def to_lib(self, src):
     def sources(cls, interpreter):
         for src in super(PyPy3Posix, cls).sources(interpreter):
             yield src
+        # PyPy >= 3.8 supports a standard prefix installation, where older
+        # versions always used a portable/developent style installation.
+        # If this is a standard prefix installation, skip the below:
+        if interpreter.system_prefix == "/usr":
+            return
         # Also copy/symlink anything under prefix/lib, which, for "portable"
         # PyPy builds, includes the tk,tcl runtime and a number of shared
         # objects. In distro-specific builds or on conda this should be empty

src/virtualenv/util/path/_pathlib/via_os_path.py

@@ -49,6 +49,9 @@ def __ne__(self, other):
     def __hash__(self):
         return hash(self._path)
 
+    def as_posix(self):
+        return str(self).replace(os.sep, "/")
+
     def exists(self):
         return os.path.exists(self._path)
 

tests/unit/create/via_global_ref/builtin/pypy/deb_pypy37.json

@@ -0,0 +1,64 @@
+{
+  "platform": "linux",
+  "implementation": "PyPy",
+  "pypy_version_info": [7, 3, 7, "final", 0],
+  "version_info": {
+    "major": 3,
+    "minor": 7,
+    "micro": 12,
+    "releaselevel": "final",
+    "serial": 0
+  },
+  "architecture": 64,
+  "version": "3.7.12 (7.3.7+dfsg-5, Jan 27 2022, 12:27:44)\\n[PyPy 7.3.7 with GCC 11.2.0]",
+  "os": "posix",
+  "prefix": "/usr/lib/pypy3",
+  "base_prefix": "/usr/lib/pypy3",
+  "real_prefix": null,
+  "base_exec_prefix": "/usr/lib/pypy3",
+  "exec_prefix": "/usr/lib/pypy3",
+  "executable": "/usr/bin/pypy3",
+  "original_executable": "/usr/bin/pypy3",
+  "system_executable": "/usr/bin/pypy3",
+  "has_venv": true,
+  "path": [
+    "/usr/lib/pypy3/lib_pypy/__extensions__",
+    "/usr/lib/pypy3/lib_pypy",
+    "/usr/lib/pypy3/lib-python/3",
+    "/usr/lib/pypy3/lib-python/3/lib-tk",
+    "/usr/lib/pypy3/lib-python/3/plat-linux2",
+    "/usr/local/lib/pypy3.7/dist-packages",
+    "/usr/lib/python3/dist-packages"
+  ],
+  "file_system_encoding": "utf-8",
+  "stdout_encoding": "UTF-8",
+  "sysconfig_scheme": null,
+  "sysconfig_paths": {
+    "stdlib": "{base}/lib-python/{py_version_short}",
+    "platstdlib": "{base}/lib-python/{py_version_short}",
+    "purelib": "{base}/../../local/lib/pypy{py_version_short}/lib-python",
+    "platlib": "{base}/../../local/lib/pypy{py_version_short}/lib-python",
+    "include": "{base}/include",
+    "scripts": "{base}/../../local/bin",
+    "data": "{base}/../../local"
+  },
+  "distutils_install": {
+    "purelib": "site-packages",
+    "platlib": "site-packages",
+    "headers": "include/UNKNOWN",
+    "scripts": "bin",
+    "data": ""
+  },
+  "sysconfig": {
+    "makefile_filename": "/usr/lib/pypy3/lib-python/3.7/config-3.7-x86_64-linux-gnu/Makefile"
+  },
+  "sysconfig_vars": {
+    "base": "/usr/lib/pypy3",
+    "py_version_short": "3.7",
+    "PYTHONFRAMEWORK": ""
+  },
+  "system_stdlib": "/usr/lib/pypy3/lib-python/3.7",
+  "system_stdlib_platform": "/usr/lib/pypy3/lib-python/3.7",
+  "max_size": 9223372036854775807,
+  "_creators": null
+}

tests/unit/create/via_global_ref/builtin/pypy/deb_pypy38.json

@@ -0,0 +1,60 @@
+{
+  "platform": "linux",
+  "implementation": "PyPy",
+  "pypy_version_info": [7, 3, 8, "final", 0],
+  "version_info": {
+    "major": 3,
+    "minor": 8,
+    "micro": 12,
+    "releaselevel": "final",
+    "serial": 0
+  },
+  "architecture": 64,
+  "version": "3.8.12 (7.3.8+dfsg-2, Mar 05 2022, 02:04:42)\\n[PyPy 7.3.8 with GCC 11.2.0]",
+  "os": "posix",
+  "prefix": "/usr",
+  "base_prefix": "/usr",
+  "real_prefix": null,
+  "base_exec_prefix": "/usr",
+  "exec_prefix": "/usr",
+  "executable": "/usr/bin/pypy3",
+  "original_executable": "/usr/bin/pypy3",
+  "system_executable": "/usr/bin/pypy3",
+  "has_venv": true,
+  "path": ["/usr/lib/pypy3.8", "/usr/local/lib/pypy3.8/dist-packages", "/usr/lib/python3/dist-packages"],
+  "file_system_encoding": "utf-8",
+  "stdout_encoding": "UTF-8",
+  "sysconfig_scheme": null,
+  "sysconfig_paths": {
+    "stdlib": "{installed_base}/lib/{implementation_lower}{py_version_short}",
+    "platstdlib": "{platbase}/lib/{implementation_lower}{py_version_short}",
+    "purelib": "{base}/local/lib/{implementation_lower}{py_version_short}/dist-packages",
+    "platlib": "{platbase}/local/lib/{implementation_lower}{py_version_short}/dist-packages",
+    "include": "{installed_base}/local/include/{implementation_lower}{py_version_short}{abiflags}",
+    "scripts": "{base}/local/bin",
+    "data": "{base}"
+  },
+  "distutils_install": {
+    "purelib": "lib/pypy3.8/site-packages",
+    "platlib": "lib/pypy3.8/site-packages",
+    "headers": "include/pypy3.8/UNKNOWN",
+    "scripts": "bin",
+    "data": ""
+  },
+  "sysconfig": {
+    "makefile_filename": "/usr/lib/pypy3.8/config-3.8-x86_64-linux-gnu/Makefile"
+  },
+  "sysconfig_vars": {
+    "installed_base": "/usr",
+    "implementation_lower": "pypy",
+    "py_version_short": "3.8",
+    "platbase": "/usr",
+    "base": "/usr",
+    "abiflags": "",
+    "PYTHONFRAMEWORK": ""
+  },
+  "system_stdlib": "/usr/lib/pypy3.8",
+  "system_stdlib_platform": "/usr/lib/pypy3.8",
+  "max_size": 9223372036854775807,
+  "_creators": null
+}

tests/unit/create/via_global_ref/builtin/pypy/portable_pypy38.json

@@ -0,0 +1,60 @@
+{
+  "platform": "linux",
+  "implementation": "PyPy",
+  "pypy_version_info": [7, 3, 8, "final", 0],
+  "version_info": {
+    "major": 3,
+    "minor": 8,
+    "micro": 12,
+    "releaselevel": "final",
+    "serial": 0
+  },
+  "architecture": 64,
+  "version": "3.8.12 (d00b0afd2a5dd3c13fcda75d738262c864c62fa7, Feb 18 2022, 09:52:33)\\n[PyPy 7.3.8 with GCC 10.2.1 20210130 (Red Hat 10.2.1-11)]",
+  "os": "posix",
+  "prefix": "/tmp/pypy3.8-v7.3.8-linux64",
+  "base_prefix": "/tmp/pypy3.8-v7.3.8-linux64",
+  "real_prefix": null,
+  "base_exec_prefix": "/tmp/pypy3.8-v7.3.8-linux64",
+  "exec_prefix": "/tmp/pypy3.8-v7.3.8-linux64",
+  "executable": "/tmp/pypy3.8-v7.3.8-linux64/bin/pypy",
+  "original_executable": "/tmp/pypy3.8-v7.3.8-linux64/bin/pypy",
+  "system_executable": "/tmp/pypy3.8-v7.3.8-linux64/bin/pypy",
+  "has_venv": true,
+  "path": ["/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8", "/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8/site-packages"],
+  "file_system_encoding": "utf-8",
+  "stdout_encoding": "UTF-8",
+  "sysconfig_scheme": null,
+  "sysconfig_paths": {
+    "stdlib": "{installed_base}/lib/{implementation_lower}{py_version_short}",
+    "platstdlib": "{platbase}/lib/{implementation_lower}{py_version_short}",
+    "purelib": "{base}/lib/{implementation_lower}{py_version_short}/site-packages",
+    "platlib": "{platbase}/lib/{implementation_lower}{py_version_short}/site-packages",
+    "include": "{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}",
+    "scripts": "{base}/bin",
+    "data": "{base}"
+  },
+  "distutils_install": {
+    "purelib": "lib/pypy3.8/site-packages",
+    "platlib": "lib/pypy3.8/site-packages",
+    "headers": "include/pypy3.8/UNKNOWN",
+    "scripts": "bin",
+    "data": ""
+  },
+  "sysconfig": {
+    "makefile_filename": "/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8/config-3.8-x86_64-linux-gnu/Makefile"
+  },
+  "sysconfig_vars": {
+    "installed_base": "/tmp/pypy3.8-v7.3.8-linux64",
+    "implementation_lower": "pypy",
+    "py_version_short": "3.8",
+    "platbase": "/tmp/pypy3.8-v7.3.8-linux64",
+    "base": "/tmp/pypy3.8-v7.3.8-linux64",
+    "abiflags": "",
+    "PYTHONFRAMEWORK": ""
+  },
+  "system_stdlib": "/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8",
+  "system_stdlib_platform": "/tmp/pypy3.8-v7.3.8-linux64/lib/pypy3.8",
+  "max_size": 9223372036854775807,
+  "_creators": null
+}

tests/unit/create/via_global_ref/builtin/pypy/test_pypy3.py

@@ -0,0 +1,104 @@
+from __future__ import absolute_import, unicode_literals
+
+import fnmatch
+
+from virtualenv.create.via_global_ref.builtin.pypy.pypy3 import PyPy3Posix
+from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, PathRefToDest
+from virtualenv.discovery.py_info import PythonInfo
+from virtualenv.util.path import Path
+
+
+class FakePath(Path):
+    """
+    A Path() fake that only knows about files in existing_paths and the
+    directories that contain them.
+    """
+
+    existing_paths = []
+
+    if hasattr(Path(""), "_flavour"):
+        _flavour = Path("")._flavour
+
+    def exists(self):
+        return self.as_posix() in self.existing_paths or self.is_dir()
+
+    def glob(self, glob):
+        pattern = self.as_posix() + "/" + glob
+        for path in fnmatch.filter(self.existing_paths, pattern):
+            yield FakePath(path)
+
+    def is_dir(self):
+        prefix = self.as_posix() + "/"
+        return any(True for path in self.existing_paths if path.startswith(prefix))
+
+    def iterdir(self):
+        prefix = self.as_posix() + "/"
+        for path in self.existing_paths:
+            if path.startswith(prefix) and "/" not in path[len(prefix) :]:
+                yield FakePath(path)
+
+    def resolve(self):
+        return self
+
+    def __div__(self, key):
+        return FakePath(super(FakePath, self).__div__(key))
+
+    def __truediv__(self, key):
+        return FakePath(super(FakePath, self).__truediv__(key))
+
+
+def assert_contains_exe(sources, src):
+    """Assert that the one and only executeable in sources is src"""
+    exes = [source for source in sources if isinstance(source, ExePathRefToDest)]
+    assert len(exes) == 1
+    exe = exes[0]
+    assert exe.src.as_posix() == src
+
+
+def assert_contains_ref(sources, src):
+    """Assert that src appears in sources"""
+    assert any(source for source in sources if isinstance(source, PathRefToDest) and source.src.as_posix() == src)
+
+
+def inject_fake_path(mocker, existing_paths):
+    """Inject FakePath in all the correct places, and set existing_paths"""
+    FakePath.existing_paths = existing_paths
+    mocker.patch("virtualenv.create.via_global_ref.builtin.pypy.common.Path", FakePath)
+    mocker.patch("virtualenv.create.via_global_ref.builtin.pypy.pypy3.Path", FakePath)
+
+
+def _load_pypi_info(name):
+    return PythonInfo._from_json((Path(__file__).parent / "{}.json".format(name)).read_text())
+
+
+def test_portable_pypy3_virtualenvs_get_their_libs(mocker):
+    paths = ["/tmp/pypy3.8-v7.3.8-linux64/bin/pypy", "/tmp/pypy3.8-v7.3.8-linux64/lib/libgdbm.so.4"]
+    inject_fake_path(mocker, paths)
+    path = Path("/tmp/pypy3.8-v7.3.8-linux64/bin/libpypy3-c.so")
+    mocker.patch.object(PyPy3Posix, "_shared_libs", return_value=[path])
+
+    sources = list(PyPy3Posix.sources(interpreter=_load_pypi_info("portable_pypy38")))
+    assert_contains_exe(sources, "/tmp/pypy3.8-v7.3.8-linux64/bin/pypy")
+    assert len(sources) > 2
+    assert_contains_ref(sources, "/tmp/pypy3.8-v7.3.8-linux64/bin/libpypy3-c.so")
+    assert_contains_ref(sources, "/tmp/pypy3.8-v7.3.8-linux64/lib/libgdbm.so.4")
+
+
+def test_debian_pypy37_virtualenvs(mocker):
+    # Debian's pypy3 layout, installed to /usr, before 3.8 allowed a /usr prefix
+    inject_fake_path(mocker, ["/usr/bin/pypy3"])
+    mocker.patch.object(PyPy3Posix, "_shared_libs", return_value=[Path("/usr/lib/pypy3/bin/libpypy3-c.so")])
+    sources = list(PyPy3Posix.sources(interpreter=_load_pypi_info("deb_pypy37")))
+    assert_contains_exe(sources, "/usr/bin/pypy3")
+    assert_contains_ref(sources, "/usr/lib/pypy3/bin/libpypy3-c.so")
+    assert len(sources) == 2
+
+
+def test_debian_pypy38_virtualenvs_exclude_usr(mocker):
+    inject_fake_path(mocker, ["/usr/bin/pypy3", "/usr/lib/foo"])
+    # libpypy3-c.so lives on the ld search path
+    mocker.patch.object(PyPy3Posix, "_shared_libs", return_value=[])
+
+    sources = list(PyPy3Posix.sources(interpreter=_load_pypi_info("deb_pypy38")))
+    assert_contains_exe(sources, "/usr/bin/pypy3")
+    assert len(sources) == 1