From 5dbc70645e20dd508700b8404ccdacf86d1a9ba8 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 23 Jul 2024 20:18:03 -0700 Subject: [PATCH] Support statically linked CPython. (#2472) This pulls in a new vendored version of Pip with the recent fix in https://github.com/pypa/pip/pull/12716 applied in https://github.com/pex-tool/pip/pull/13. As a result, Pex can run using vendored Pip under statically linked musl libc CPython interpreters. This opens the door to bootstrapping newer unpatched Pip's that also have this same fix (versions 24.2 and later; see: #2471). Fixes #2017 --- pex/vendor/__init__.py | 11 ++- pex/vendor/_vendored/pip/.layout.json | 2 +- .../pip/pip/_internal/utils/glibc.py | 15 +++- tests/integration/test_issue_2017.py | 76 +++++++++++++++++++ 4 files changed, 99 insertions(+), 5 deletions(-) create mode 100644 tests/integration/test_issue_2017.py diff --git a/pex/vendor/__init__.py b/pex/vendor/__init__.py index dd6e7e428..0f81e0057 100644 --- a/pex/vendor/__init__.py +++ b/pex/vendor/__init__.py @@ -207,19 +207,24 @@ def iter_vendor_specs(filter_requires_python=None): # We shell out to pip at buildtime to resolve and install dependencies. # N.B.: We're currently using a patched version of Pip 20.3.4 housed at # https://github.com/pex-tool/pip/tree/pex/patches/generation-2. - # It has 2 patches: + # It has 3 patches: # 1.) https://github.com/pex-tool/pip/commit/06f462537c981116c763c1ba40cf40e9dd461bcf # The patch works around a bug in `pip download --constraint...` tracked at # https://github.com/pypa/pip/issues/9283 and fixed by https://github.com/pypa/pip/pull/9301 # there and https://github.com/pex-tool/pip/pull/8 in our fork. # 2.) https://github.com/pex-tool/pip/commit/386a54f097ece66775d0c7f34fd29bb596c6b0be # This is a cherry-pick of - # https://github.com/pex-tool/pip/commit/00fb5a0b224cde08e3e5ca034247baadfb646468 + # https://github.com/pypa/pip/commit/00fb5a0b224cde08e3e5ca034247baadfb646468 # (https://github.com/pypa/pip/pull/9533) from upstream that upgrades Pip's vendored # packaging to 20.9 to pick up support for mac universal2 wheels. + # 3.) https://github.com/pex-tool/pip/commit/00827ec9f4275a7786425cf006466c56f4cbd862 + # This is a cherry-pick of + # https://github.com/pypa/pip/commit/601bcf82eccfbc15c1ff6cc735aafb2c9dab81a5 + # (https://github.com/pypa/pip/pull/12716) from upstream that fixes glibc version probing on + # musl libc systems. yield VendorSpec.git( repo="https://github.com/pex-tool/pip", - commit="386a54f097ece66775d0c7f34fd29bb596c6b0be", + commit="00827ec9f4275a7786425cf006466c56f4cbd862", project_name="pip", rewrite=False, ) diff --git a/pex/vendor/_vendored/pip/.layout.json b/pex/vendor/_vendored/pip/.layout.json index e30534fe3..48d545b25 100644 --- a/pex/vendor/_vendored/pip/.layout.json +++ b/pex/vendor/_vendored/pip/.layout.json @@ -1 +1 @@ -{"fingerprint": "120267325b80f5c4b4adac019eb6617ab3319395c043d2871eedf70dd6ae2954", "record_relpath": "pip-20.3.4.dist-info/RECORD", "root_is_purelib": true, "stash_dir": ".prefix"} \ No newline at end of file +{"fingerprint": "23b678435eebb1c7423541b5663bdc518833a4b75a3a3a2192466a37cd7b1861", "record_relpath": "pip-20.3.4.dist-info/RECORD", "root_is_purelib": true, "stash_dir": ".prefix"} \ No newline at end of file diff --git a/pex/vendor/_vendored/pip/pip/_internal/utils/glibc.py b/pex/vendor/_vendored/pip/pip/_internal/utils/glibc.py index 361042441..fb65d32c5 100644 --- a/pex/vendor/_vendored/pip/pip/_internal/utils/glibc.py +++ b/pex/vendor/_vendored/pip/pip/_internal/utils/glibc.py @@ -49,7 +49,20 @@ def glibc_version_string_ctypes(): # manpage says, "If filename is NULL, then the returned handle is for the # main program". This way we can let the linker do the work to figure out # which libc our process is actually using. - process_namespace = ctypes.CDLL(None) + # + # We must also handle the special case where the executable is not a + # dynamically linked executable. This can occur when using musl libc, + # for example. In this situation, dlopen() will error, leading to an + # OSError. Interestingly, at least in the case of musl, there is no + # errno set on the OSError. The single string argument used to construct + # OSError comes from libc itself and is therefore not portable to + # hard code here. In any case, failure to call dlopen() means we + # can't proceed, so we bail on our attempt. + try: + process_namespace = ctypes.CDLL(None) + except OSError: + return None + try: gnu_get_libc_version = process_namespace.gnu_get_libc_version except AttributeError: diff --git a/tests/integration/test_issue_2017.py b/tests/integration/test_issue_2017.py new file mode 100644 index 000000000..468d4f61c --- /dev/null +++ b/tests/integration/test_issue_2017.py @@ -0,0 +1,76 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import os.path +import shutil +import subprocess +import tarfile +from textwrap import dedent + +import pytest + +from pex.common import is_exe, safe_open +from pex.compatibility import urlparse +from pex.fetcher import URLFetcher +from pex.pep_440 import Version +from pex.pip.version import PipVersion +from pex.typing import TYPE_CHECKING +from testing import IS_LINUX, run_pex_command + +if TYPE_CHECKING: + from typing import Any + + +# TODO(John Sirois): Include a test of >= Pip 24.2 when Pex adds support for it. +# See: https://github.com/pex-tool/pex/issues/2471 +@pytest.mark.skipif( + PipVersion.DEFAULT > PipVersion.VENDORED and PipVersion.DEFAULT.version < Version("24.2"), + reason=( + "Although Pex's vendored Pip is patched to handle statically linked musl libc CPython, no " + "version of Pip Pex supports handles these Pythons until Pip 24.2" + ), +) +@pytest.mark.skipif( + not IS_LINUX, + reason="This test tests statically linked musl libc CPython which is only available for Linux.", +) +def test_statically_linked_musl_libc_cpython_support(tmpdir): + # type: (Any) -> None + + pbs_distribution_url = ( + "https://github.com/indygreg/python-build-standalone/releases/download/20221220/" + "cpython-3.10.9+20221220-x86_64_v3-unknown-linux-musl-install_only.tar.gz" + ) + pbs_distribution = os.path.join( + str(tmpdir), + os.path.basename(urlparse.urlparse(pbs_distribution_url).path), + ) + with URLFetcher().get_body_stream(pbs_distribution_url) as read_fp, open( + pbs_distribution, "wb" + ) as write_fp: + shutil.copyfileobj(read_fp, write_fp) + with tarfile.open(pbs_distribution) as tf: + tf.extractall(str(tmpdir)) + statically_linked_musl_libc_cpython = os.path.join(str(tmpdir), "python", "bin", "python3") + assert is_exe(statically_linked_musl_libc_cpython) + + pex = os.path.join(str(tmpdir), "pex") + run_pex_command( + args=["fortune==1.1.1", "-c", "fortune", "-o", pex], + python=statically_linked_musl_libc_cpython, + ).assert_success() + + fortune_db = os.path.join(str(tmpdir), "fortunes") + with safe_open(fortune_db, "w") as fp: + fp.write( + dedent( + """\ + A day for firm decisions!!!!! Or is it? + % + """ + ) + ) + output = subprocess.check_output(args=[pex, fortune_db]) + assert b"A day for firm decisions!!!!! Or is it?\n" == output, output.decode("utf-8")