Support --pip-version 24.1 and Python 3.13. (#2435)

Pex has been testing against Python 3.13 alpha and then beta releases 
for a while now using a patched Pip with Python 3.13 support. Those
patches have made it into the Pip 24.1 release; so now Pex can
officially support Python 3.13.

The Pip 24.1 release notes: https://pip.pypa.io/en/stable/news/#v24-1

Closes #2406

.github/workflows/ci.yml

@@ -73,8 +73,8 @@ jobs:
           - py311-pip20
           - py311-pip22_3_1
           - py311-pip23_1_2
-          - py312-pip24_0
-          - py313-pip24_0_patched
+          - py312-pip24_1
+          - py313-pip24_1
           - pypy310-pip20
           - pypy310-pip22_3_1
           - pypy310-pip23_1_2
@@ -84,8 +84,8 @@ jobs:
           - py311-pip20-integration
           - py311-pip22_3_1-integration
           - py311-pip23_1_2-integration
-          - py312-pip24_0-integration
-          - py313-pip24_0_patched-integration
+          - py312-pip24_1-integration
+          - py313-pip24_1-integration
           - pypy310-pip20-integration
           - pypy310-pip22_3_1-integration
           - pypy310-pip23_1_2-integration
@@ -129,10 +129,10 @@ jobs:
       matrix:
         include:
           - python-version: [ 3, 12 ]
-            tox-env: py312-pip24_0
+            tox-env: py312-pip24_1
             tox-env-python: python3.11
           - python-version: [ 3, 12 ]
-            tox-env: py312-pip24_0-integration
+            tox-env: py312-pip24_1-integration
             tox-env-python: python3.11
     steps:
       - name: Calculate Pythons to Expose

CHANGES.md

@@ -1,5 +1,12 @@
 # Release Notes
 
+## 2.5.0
+
+This release brings support for Python 3.13 and `--pip-version 24.1`,
+which is the first Pip version to support it.
+
+* Support `--pip-version 24.1` and Python 3.13. (#2434)
+
 ## 2.4.1
 
 This release fixes `pex --only-binary X --lock ...` to work with lock

pex/pip/foreign_platform/__init__.py

@@ -5,26 +5,18 @@
 
 import json
 import os
-import re
 
 from pex.common import safe_mkdtemp
 from pex.interpreter_constraints import iter_compatible_versions
 from pex.pep_425 import CompatibilityTags
 from pex.pip.download_observer import DownloadObserver, Patch, PatchSet
-from pex.pip.log_analyzer import ErrorAnalyzer, ErrorMessage
 from pex.platforms import Platform
 from pex.targets import AbbreviatedPlatform, CompletePlatform, Target
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from typing import Iterable, Iterator, Optional
-
-    import attr  # vendor:skip
-
-    from pex.pip.log_analyzer import ErrorAnalysis
-else:
-    from pex.third_party import attr
+    from typing import Any, Iterable, Iterator, Mapping, Optional
 
 
 def iter_platform_args(
@@ -66,46 +58,91 @@ def iter_platform_args(
     yield platform.abi
 
 
-@attr.s(frozen=True)
-class _Issue10050Analyzer(ErrorAnalyzer):
-    # Part of the workaround for: https://github.com/pypa/pip/issues/10050
+class EvaluationEnvironment(dict):
+    class _Missing(str):
+        pass
 
-    _platform = attr.ib()  # type: Platform
+    class UndefinedName(Exception):
+        pass
 
-    def analyze(self, line):
-        # type: (str) -> ErrorAnalysis
-        # N.B.: Pip --log output looks like:
-        # 2021-06-20T19:06:00,981 pip._vendor.packaging.markers.UndefinedEnvironmentName: 'python_full_version' does not exist in evaluation environment.
-        match = re.match(
-            r"^[^ ]+ pip._vendor.packaging.markers.UndefinedEnvironmentName: "
-            r"(?P<missing_marker>.*)\.$",
-            line,
+    def __init__(
+        self,
+        target_description,  # type: str
+        *args,  # type: Any
+        **kwargs  # type: Any
+    ):
+        # type: (...) -> None
+        self._target_description = target_description
+        super(EvaluationEnvironment, self).__init__(*args, **kwargs)
+
+    def __missing__(self, key):
+        # type: (Any) -> Any
+        return self._Missing(
+            "Failed to resolve for {target_description}. Resolve requires evaluation of unknown "
+            "environment marker: {marker!r} does not exist in evaluation environment.".format(
+                target_description=self._target_description, marker=key
+            )
         )
-        if match:
-            return self.Complete(
-                ErrorMessage(
-                    "Failed to resolve for platform {}. Resolve requires evaluation of unknown "
-                    "environment marker: {}.".format(self._platform, match.group("missing_marker"))
-                )
+
+    def raise_if_missing(self, value):
+        # type: (Any) -> None
+        if isinstance(value, self._Missing):
+            raise self.UndefinedName(value)
+
+    def default(self):
+        # type: () -> EvaluationEnvironment
+        return EvaluationEnvironment(self._target_description, self.copy())
+
+
+class PatchContext(object):
+    _PEX_PATCHED_MARKERS_FILE_ENV_VAR_NAME = "_PEX_PATCHED_MARKERS_FILE"
+
+    @classmethod
+    def load_evaluation_environment(cls):
+        # type: () -> EvaluationEnvironment
+
+        patched_markers_file = os.environ.pop(cls._PEX_PATCHED_MARKERS_FILE_ENV_VAR_NAME)
+        with open(patched_markers_file) as fp:
+            data = json.load(fp)
+        return EvaluationEnvironment(data["target_description"], data["patched_environment"])
+
+    @classmethod
+    def dump_marker_environment(cls, target):
+        # type: (Target) -> Mapping[str, str]
+
+        target_description = target.render_description()
+        patched_environment = target.marker_environment.as_dict()
+        patches_file = os.path.join(safe_mkdtemp(), "markers.json")
+        with open(patches_file, "w") as markers_fp:
+            json.dump(
+                {
+                    "target_description": target_description,
+                    "patched_environment": patched_environment,
+                },
+                markers_fp,
             )
-        return self.Continue()
+        TRACER.log(
+            "Patching environment markers for {target_description} with "
+            "{patched_environment}".format(
+                target_description=target_description, patched_environment=patched_environment
+            ),
+            V=3,
+        )
+        return {cls._PEX_PATCHED_MARKERS_FILE_ENV_VAR_NAME: patches_file}
 
 
 def patch(target):
     # type: (Target) -> Optional[DownloadObserver]
     if not isinstance(target, (AbbreviatedPlatform, CompletePlatform)):
         return None
 
-    analyzer = _Issue10050Analyzer(target.platform)
-
     patches = []
     patches_dir = safe_mkdtemp()
 
-    patched_environment = target.marker_environment.as_dict()
-    with open(os.path.join(patches_dir, "markers.json"), "w") as markers_fp:
-        json.dump(patched_environment, markers_fp)
     patches.append(
-        Patch.from_code_resource(__name__, "markers.py", _PEX_PATCHED_MARKERS_FILE=markers_fp.name)
+        Patch.from_code_resource(
+            __name__, "markers.py", **PatchContext.dump_marker_environment(target)
+        )
     )
 
     compatible_tags = target.supported_tags
@@ -128,11 +165,7 @@ def patch(target):
         patch_requires_python(requires_python=[requires_python], patches_dir=patches_dir)
     )
 
-    TRACER.log(
-        "Patching environment markers for {} with {}".format(target, patched_environment),
-        V=3,
-    )
-    return DownloadObserver(analyzer=analyzer, patch_set=PatchSet(patches=tuple(patches)))
+    return DownloadObserver(analyzer=None, patch_set=PatchSet(patches=tuple(patches)))
 
 
 def patch_tags(

pex/pip/foreign_platform/markers.py

@@ -3,17 +3,53 @@
 
 from __future__ import absolute_import
 
-import json
-import os
+from pex.typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Any, Dict
 
 
 def patch():
+    # type: () -> None
+
     from pip._vendor.packaging import markers  # type: ignore[import]
 
-    # N.B.: The following environment variable is used by the Pex runtime to control Pip and must be
-    # kept in-sync with `__init__.py`.
-    patched_markers_file = os.environ.pop("_PEX_PATCHED_MARKERS_FILE")
-    with open(patched_markers_file) as fp:
-        patched_markers = json.load(fp)
+    from pex.exceptions import production_assert
+    from pex.pip.foreign_platform import EvaluationEnvironment, PatchContext
+
+    evaluation_environment = PatchContext.load_evaluation_environment()
+
+    def _get_env(
+        environment,  # type: Dict[Any, Any]
+        name,  # type: Any
+    ):
+        # type: (...) -> Any
+        production_assert(
+            isinstance(environment, EvaluationEnvironment),
+            "Expected environment to come from the {function} function, "
+            "which we patch to return {expected_type}, but was {actual_type}".format(
+                function=markers.default_environment,
+                expected_type=EvaluationEnvironment,
+                actual_type=type(environment),
+            ),
+        )
+        return environment[name]
+
+    # Works with all Pip vendored packaging distributions.
+    markers.default_environment = evaluation_environment.default
+    # Covers Pip<24.1 vendored packaging.
+    markers._get_env = _get_env
+
+    original_eval_op = markers._eval_op
+
+    def _eval_op(
+        lhs,  # type: Any
+        op,  # type: Any
+        rhs,  # type: Any
+    ):
+        # type: (...) -> Any
+        evaluation_environment.raise_if_missing(lhs)
+        evaluation_environment.raise_if_missing(rhs)
+        return original_eval_op(lhs, op, rhs)
 
-    markers.default_environment = patched_markers.copy
+    markers._eval_op = _eval_op

pex/pip/version.py

@@ -257,21 +257,13 @@ def values(cls):
         requires_python=">=3.7,<3.13",
     )
 
-    # This is https://github.com/pypa/pip/pull/12462 which is approved but not yet merged or
-    # released. It allows testing Python 3.13 pre-releases but should not be used by the public; so
-    # we keep it hidden.
-    v24_0_dev0_patched = PipVersionValue(
-        name="24.0.dev0-patched",
-        version="24.0.dev0+patched",
-        requirement=(
-            "pip @ git+https://github.com/jsirois/pip@0257c9422f7bb99a6f319b54f808a5c50339be6c"
-        ),
-        setuptools_version="69.0.3",
-        wheel_version="0.42.0",
-        requires_python=">=3.7",
-        hidden=True,
+    v24_1 = PipVersionValue(
+        version="24.1",
+        setuptools_version="70.1.0",
+        wheel_version="0.43.0",
+        requires_python=">=3.8,<3.14",
     )
 
     VENDORED = v20_3_4_patched
     LATEST = LatestPipVersion()
-    DEFAULT = DefaultPipVersion(preferred=(VENDORED, v23_2))
+    DEFAULT = DefaultPipVersion(preferred=(VENDORED, v23_2, v24_1))