From 2daeb24c64f5e0eb808c4e81be32c3e8c602e45a Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Wed, 31 Jul 2024 11:38:54 -0300 Subject: [PATCH 1/2] fix: fix Python venv in core24 classic snaps Changes to virtual environment handling made our default approach - creating a venv in the build system and then just moving it into the snap - not work for Python 3.12. My understanding of the issue is that the "home" key in the "pyvenv.cfg" file points to a path in the build system, which obviously does not exist at snap-run-time. This apparently works fine *somehow* in core22 - likely there's some fallback logic that makes the interpreter correctly find the bundled Python libraries, but not in core24. So to address this we update this "home" key to point to its final destination in the snap. Fixes #4942 --- snapcraft/parts/plugins/python_plugin.py | 33 ++++++++++++++- snapcraft/services/lifecycle.py | 12 ++++++ .../environment/test-variables/task.yaml | 2 +- .../python-hello/classic/snap/snapcraft.yaml | 3 ++ .../core24/python-hello/src/hello/__init__.py | 5 ++- .../python-hello/strict/snap/snapcraft.yaml | 2 + tests/spread/core24/python-hello/task.yaml | 7 +++- .../unit/parts/plugins/test_python_plugin.py | 40 ++++++++++++++++++- tests/unit/services/test_lifecycle.py | 10 ++++- 9 files changed, 107 insertions(+), 7 deletions(-) diff --git a/snapcraft/parts/plugins/python_plugin.py b/snapcraft/parts/plugins/python_plugin.py index fb2159a9ca..ab890cb63e 100644 --- a/snapcraft/parts/plugins/python_plugin.py +++ b/snapcraft/parts/plugins/python_plugin.py @@ -17,9 +17,10 @@ """The Snapcraft Python plugin.""" import logging +from pathlib import Path from typing import Optional -from craft_parts import errors +from craft_parts import StepInfo, errors from craft_parts.plugins import python_plugin from overrides import override @@ -64,3 +65,33 @@ def _get_system_python_interpreter(self) -> Optional[str]: confinement, ) return interpreter + + @classmethod + def post_prime(cls, step_info: StepInfo) -> None: + """Perform Python-specific actions right before packing.""" + base = step_info.project_base + + if base != "core24": + # Only fix pyvenv.cfg on core24 snaps + return + + root_path: Path = step_info.prime_dir + + pyvenv = root_path / "pyvenv.cfg" + if not pyvenv.is_file(): + return + + snap_path = Path(f"/snap/{step_info.project_name}/current") + new_home = f"home = {snap_path}" + + candidates = ( + step_info.part_install_dir, + step_info.stage_dir, + ) + + contents = pyvenv.read_text() + for candidate in candidates: + old_home = f"home = {candidate}" + contents = contents.replace(old_home, new_home) + + pyvenv.write_text(contents) diff --git a/snapcraft/services/lifecycle.py b/snapcraft/services/lifecycle.py index 6bfae8d160..61f8f95472 100644 --- a/snapcraft/services/lifecycle.py +++ b/snapcraft/services/lifecycle.py @@ -77,6 +77,7 @@ def setup(self) -> None: extra_build_snaps=project.get_extra_build_snaps(), confinement=project.confinement, project_base=project.base or "", + project_name=project.name, ) callbacks.register_prologue(parts.set_global_environment) callbacks.register_pre_step(parts.set_step_environment) @@ -85,8 +86,19 @@ def setup(self) -> None: @overrides def post_prime(self, step_info: StepInfo) -> bool: """Run post-prime parts steps for Snapcraft.""" + from snapcraft.parts import plugins + project = cast(models.Project, self._project) + part_name = step_info.part_name + plugin_name = project.parts[part_name]["plugin"] + + # Handle plugin-specific prime fixes + if plugin_name == "python": + plugins.PythonPlugin.post_prime(step_info) + + # Handle patch-elf + # do not use system libraries in classic confinement use_system_libs = not bool(project.confinement == "classic") diff --git a/tests/spread/core24-suites/environment/test-variables/task.yaml b/tests/spread/core24-suites/environment/test-variables/task.yaml index 15c6fbae11..f9fb09d0a2 100644 --- a/tests/spread/core24-suites/environment/test-variables/task.yaml +++ b/tests/spread/core24-suites/environment/test-variables/task.yaml @@ -33,7 +33,7 @@ execute: | cat "$file" for exp in \ "^SNAPCRAFT_PROJECT_GRADE=devel$" \ - "^SNAPCRAFT_PROJECT_NAME=None$" \ + "^SNAPCRAFT_PROJECT_NAME=variables$" \ "^SNAPCRAFT_PROJECT_VERSION=1$" \ "^SNAPCRAFT_PARALLEL_BUILD_COUNT=[0-9]\+$" \ "^SNAPCRAFT_PROJECT_DIR=${root}$" \ diff --git a/tests/spread/core24/python-hello/classic/snap/snapcraft.yaml b/tests/spread/core24/python-hello/classic/snap/snapcraft.yaml index bfac77d1a7..d9d88dff4d 100644 --- a/tests/spread/core24/python-hello/classic/snap/snapcraft.yaml +++ b/tests/spread/core24/python-hello/classic/snap/snapcraft.yaml @@ -16,6 +16,8 @@ parts: hello: plugin: python source: src + python-packages: + - black build-attributes: - enable-patchelf stage-packages: @@ -23,3 +25,4 @@ parts: - libpython3.12-stdlib - python3.12-minimal - python3.12-venv + - python3-minimal # (for the "python3" symlink) diff --git a/tests/spread/core24/python-hello/src/hello/__init__.py b/tests/spread/core24/python-hello/src/hello/__init__.py index e3095b2229..de8c28b98f 100644 --- a/tests/spread/core24/python-hello/src/hello/__init__.py +++ b/tests/spread/core24/python-hello/src/hello/__init__.py @@ -1,2 +1,5 @@ +import black + + def main(): - print("hello world") + print(f"hello world! black version: {black.__version__}") diff --git a/tests/spread/core24/python-hello/strict/snap/snapcraft.yaml b/tests/spread/core24/python-hello/strict/snap/snapcraft.yaml index 2192e0e234..f52a1ca966 100644 --- a/tests/spread/core24/python-hello/strict/snap/snapcraft.yaml +++ b/tests/spread/core24/python-hello/strict/snap/snapcraft.yaml @@ -12,3 +12,5 @@ parts: hello: plugin: python source: src + python-packages: + - black diff --git a/tests/spread/core24/python-hello/task.yaml b/tests/spread/core24/python-hello/task.yaml index 11c3f18fd1..2db4043b9e 100644 --- a/tests/spread/core24/python-hello/task.yaml +++ b/tests/spread/core24/python-hello/task.yaml @@ -1,5 +1,10 @@ summary: Build and run Python-based snaps in core24 +systems: + # Must *not* run this on 24.04, which can give false-positives due to the + # presence of the system Python 3.12. + - ubuntu-22.04* + environment: PARAM/strict: "" PARAM/classic: "--classic" @@ -17,4 +22,4 @@ execute: | # shellcheck disable=SC2086 snap install python-hello-"${SPREAD_VARIANT}"_1.0_*.snap --dangerous ${PARAM} - python-hello-"${SPREAD_VARIANT}" | MATCH "hello world" + python-hello-"${SPREAD_VARIANT}" | MATCH "hello world! black version" diff --git a/tests/unit/parts/plugins/test_python_plugin.py b/tests/unit/parts/plugins/test_python_plugin.py index 36487cfcaa..c4cc654dca 100644 --- a/tests/unit/parts/plugins/test_python_plugin.py +++ b/tests/unit/parts/plugins/test_python_plugin.py @@ -17,7 +17,7 @@ from textwrap import dedent import pytest -from craft_parts import Part, PartInfo, ProjectInfo, errors +from craft_parts import Part, PartInfo, ProjectInfo, Step, StepInfo, errors from snapcraft.parts.plugins import PythonPlugin @@ -190,3 +190,41 @@ def test_get_system_python_interpreter_unknown_base(confinement, new_dir): expected_error = "Don't know which interpreter to use for base core10" with pytest.raises(errors.PartsError, match=expected_error): plugin._get_system_python_interpreter() + + +@pytest.mark.parametrize("home_attr", ["part_install_dir", "stage_dir"]) +def test_fix_pyvenv(new_dir, home_attr): + part_info = PartInfo( + project_info=ProjectInfo( + application_name="test", + project_name="test-snap", + base="core24", + confinement="classic", + project_base="core24", + cache_dir=new_dir, + ), + part=Part("my-part", {"plugin": "python"}), + ) + + prime_dir = part_info.prime_dir + prime_dir.mkdir() + + pyvenv = prime_dir / "pyvenv.cfg" + pyvenv.write_text( + dedent( + f"""\ + home = {getattr(part_info, home_attr)}/usr/bin + include-system-site-packages = false + version = 3.12.3 + executable = /root/parts/my-part/install/usr/bin/python3.12 + command = /root/parts/my-part/install/usr/bin/python3 -m venv /root/parts/my-part/install + """ + ) + ) + + step_info = StepInfo(part_info, Step.PRIME) + + PythonPlugin.post_prime(step_info) + + new_contents = pyvenv.read_text() + assert "home = /snap/test-snap/current/usr/bin" in new_contents diff --git a/tests/unit/services/test_lifecycle.py b/tests/unit/services/test_lifecycle.py index 6eb47794af..3c7d63de10 100644 --- a/tests/unit/services/test_lifecycle.py +++ b/tests/unit/services/test_lifecycle.py @@ -43,7 +43,10 @@ def test_lifecycle_installs_base(lifecycle_service, mocker): ) -def test_post_prime_no_patchelf(fp, tmp_path, lifecycle_service): +def test_post_prime_no_patchelf(fp, tmp_path, lifecycle_service, default_project): + new_attrs = {"parts": {"my-part": {"plugin": "nil"}}} + default_project.__dict__.update(**new_attrs) + mock_step_info = mock.Mock() mock_step_info.configure_mock( **{ @@ -51,6 +54,7 @@ def test_post_prime_no_patchelf(fp, tmp_path, lifecycle_service): "build_attributes": [], "state.files": ["usr/bin/ls"], "prime_dir": tmp_path / "prime", + "part_name": "my-part", } ) @@ -82,7 +86,7 @@ def test_post_prime_patchelf( use_system_libs, ): patchelf_spy = mocker.spy(snapcraft.parts, "patch_elf") - new_attrs = {"confinement": confinement} + new_attrs = {"confinement": confinement, "parts": {"my-part": {"plugin": "nil"}}} default_project.__dict__.update(**new_attrs) mock_step_info = mock.Mock() @@ -92,6 +96,7 @@ def test_post_prime_patchelf( "build_attributes": ["enable-patchelf"], "state.files": ["usr/bin/ls"], "prime_dir": tmp_path / "prime", + "part_name": "my-part", } ) @@ -207,6 +212,7 @@ def test_lifecycle_custom_arguments( assert info.project_base == expected_base assert info.confinement == expected_confinement + assert info.project_name == default_project.name == "default" @pytest.mark.usefixtures("default_project") From 698300d1399713edf72893f9dc54ef451dd59300 Mon Sep 17 00:00:00 2001 From: Tiago Nobrega Date: Thu, 1 Aug 2024 08:56:58 -0300 Subject: [PATCH 2/2] fixup! fix: fix Python venv in core24 classic snaps --- snapcraft/parts/plugins/python_plugin.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/snapcraft/parts/plugins/python_plugin.py b/snapcraft/parts/plugins/python_plugin.py index ab890cb63e..df20f19871 100644 --- a/snapcraft/parts/plugins/python_plugin.py +++ b/snapcraft/parts/plugins/python_plugin.py @@ -71,8 +71,8 @@ def post_prime(cls, step_info: StepInfo) -> None: """Perform Python-specific actions right before packing.""" base = step_info.project_base - if base != "core24": - # Only fix pyvenv.cfg on core24 snaps + if base in ("core20", "core22"): + # Only fix pyvenv.cfg on core24+ snaps return root_path: Path = step_info.prime_dir @@ -89,9 +89,11 @@ def post_prime(cls, step_info: StepInfo) -> None: step_info.stage_dir, ) - contents = pyvenv.read_text() + old_contents = contents = pyvenv.read_text() for candidate in candidates: old_home = f"home = {candidate}" contents = contents.replace(old_home, new_home) - pyvenv.write_text(contents) + if old_contents != contents: + logger.debug("Updating pyvenv.cfg to:\n%s", contents) + pyvenv.write_text(contents)