diff --git a/AUTHORS.txt b/AUTHORS.txt index 10317a284b2..dda2ac30f85 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -105,6 +105,7 @@ Bogdan Opanchuk BorisZZZ Brad Erickson Bradley Ayers +Branch Vincent Brandon L. Reiss Brandt Bucher Brannon Dorsey @@ -131,11 +132,13 @@ Carol Willing Carter Thayer Cass Chandrasekhar Atina +Charlie Marsh Chih-Hsuan Yen Chris Brinker Chris Hunt Chris Jerdonek Chris Kuehl +Chris Markiewicz Chris McDonough Chris Pawley Chris Pryer @@ -234,6 +237,7 @@ Dos Moonen Douglas Thor DrFeathers Dustin Ingram +Dustin Rodrigues Dwayne Bailey Ed Morley Edgar Ramírez @@ -365,12 +369,14 @@ Jeff Dairiki Jeff Widman Jelmer Vernooij jenix21 +Jeremy Fleischman Jeremy Stanley Jeremy Zafran Jesse Rittner Jiashuo Li Jim Fisher Jim Garrison +Jinzhe Zeng Jiun Bae Jivan Amara Joe Bylund @@ -391,6 +397,7 @@ Jorge Niedbalski Joseph Bylund Joseph Long Josh Bronson +Josh Cannon Josh Hansen Josh Schneier Joshua @@ -425,6 +432,7 @@ konstin kpinc Krishna Oza Kumar McMillan +Kuntal Majumder Kurt McKee Kyle Persohn lakshmanaram @@ -513,6 +521,7 @@ Miro Hrončok Monica Baluna montefra Monty Taylor +morotti mrKazzila Muha Ajjan Nadav Wexler @@ -625,6 +634,7 @@ Richard Jones Richard Si Ricky Ng-Adam Rishi +rmorotti RobberPhex Robert Collins Robert McGibbon @@ -700,6 +710,7 @@ Stéphane Klein Sumana Harihareswara Surbhi Sharma Sviatoslav Sydorenko +Sviatoslav Sydorenko (Святослав Сидоренко) Swat009 Sylvain Takayuki SHIMIZUKAWA diff --git a/NEWS.rst b/NEWS.rst index 3e7940c5f41..397277444bc 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -9,6 +9,79 @@ .. towncrier release notes start +24.2 (2024-07-28) +================= + +Deprecations and Removals +------------------------- + +- Deprecate ``pip install --editable`` falling back to ``setup.py develop`` + when using a setuptools version that does not support :pep:`660` + (setuptools v63 and older). (`#11457 `_) + +Features +-------- + +- Check unsupported packages for the current platform. (`#11054 `_) +- Use system certificates *and* certifi certificates to verify HTTPS connections on Python 3.10+. + Python 3.9 and earlier only use certifi. + + To revert to previous behaviour, pass the flag ``--use-deprecated=legacy-certs``. (`#11647 `_) +- Improve discovery performance of installed packages when the ``importlib.metadata`` + backend is used to load distribution metadata (used by default under Python 3.11+). (`#12656 `_) +- Improve performance when the same requirement string appears many times during + resolution, by consistently caching the parsed requirement string. (`#12663 `_) +- Minor performance improvement of finding applicable package candidates by not + repeatedly calculating their versions (`#12664 `_) +- Disable pip's self version check when invoking a pip subprocess to install + PEP 517 build requirements. (`#12683 `_) +- Improve dependency resolution performance by caching platform compatibility + tags during wheel cache lookup. (`#12712 `_) +- ``wheel`` is no longer explicitly listed as a build dependency of ``pip``. + ``setuptools`` injects this dependency in the ``get_requires_for_build_wheel()`` + hook and no longer needs it on newer versions. (`#12728 `_) +- Ignore ``--require-virtualenv`` for ``pip check`` and ``pip freeze`` (`#12842 `_) +- Improve package download and install performance. + + Increase chunk sizes when downloading (256 kB, up from 10 kB) and reading files (1 MB, up from 8 kB). + This reduces the frequency of updates to pip's progress bar. (`#12810 `_) +- Improve pip install performance. + + Files are now extracted in 1MB blocks, or in one block matching the file size for + smaller files. A decompressor is no longer instantiated when extracting 0 bytes files, + it is not necessary because there is no data to decompress. (`#12803 `_) + +Bug Fixes +--------- + +- Set ``no_color`` to global ``rich.Console`` instance. (`#11045 `_) +- Fix resolution to respect ``--python-version`` when checking ``Requires-Python``. (`#12216 `_) +- Perform hash comparisons in a case-insensitive manner. (`#12680 `_) +- Avoid ``dlopen`` failure for glibc detection in musl builds (`#12716 `_) +- Avoid keyring logging crashes when pip is run in verbose mode. (`#12751 `_) +- Fix finding hardlink targets in tar files with an ignored top-level directory. (`#12781 `_) +- Improve pip install performance by only creating required parent + directories once, instead of before extracting every file in the wheel. (`#12782 `_) +- Improve pip install performance by calculating installed packages printout + in linear time instead of quadratic time. (`#12791 `_) + +Vendored Libraries +------------------ + +- Remove vendored tenacity. +- Update the preload list for the ``DEBUNDLED`` case, to replace ``pep517`` that has been renamed to ``pyproject_hooks``. +- Use tomllib from the stdlib if available, rather than tomli +- Upgrade certifi to 2024.7.4 +- Upgrade platformdirs to 4.2.2 +- Upgrade pygments to 2.18.0 +- Upgrade setuptools to 70.3.0 +- Upgrade typing_extensions to 4.12.2 + +Improved Documentation +---------------------- + +- Correct ``—-ignore-conflicts`` (including an em dash) to ``--ignore-conflicts``. (`#12851 `_) + 24.1.2 (2024-07-07) =================== diff --git a/news/10822.vendor.rst b/news/10822.vendor.rst deleted file mode 100644 index 27a3c234091..00000000000 --- a/news/10822.vendor.rst +++ /dev/null @@ -1 +0,0 @@ -Remove vendored tenacity. diff --git a/news/11045.bugfix.rst b/news/11045.bugfix.rst deleted file mode 100644 index cf08c9b8bcd..00000000000 --- a/news/11045.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Set ``no_color`` to global ``rich.Console`` instance. diff --git a/news/11054.feature.rst b/news/11054.feature.rst deleted file mode 100644 index 335468c12f9..00000000000 --- a/news/11054.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Check unsupported packages for the current platform. diff --git a/news/11457.removal.rst b/news/11457.removal.rst deleted file mode 100644 index 7af83fde3ce..00000000000 --- a/news/11457.removal.rst +++ /dev/null @@ -1,3 +0,0 @@ -Deprecate ``pip install --editable`` falling back to ``setup.py develop`` -when using a setuptools version that does not support :pep:`660` -(setuptools v63 and older). diff --git a/news/11647.feature.rst b/news/11647.feature.rst deleted file mode 100644 index 26d04d49165..00000000000 --- a/news/11647.feature.rst +++ /dev/null @@ -1,4 +0,0 @@ -Changed pip to use system certificates and certifi to verify HTTPS connections. -This change only affects Python 3.10 or later, Python 3.9 and earlier only use certifi. - -To revert to previous behavior pass the flag ``--use-deprecated=legacy-certs``. diff --git a/news/11865.feature.rst b/news/11865.feature.rst new file mode 100644 index 00000000000..5f4cb7b563d --- /dev/null +++ b/news/11865.feature.rst @@ -0,0 +1 @@ +Implement PEP-710 for storing provenance_url.json file. diff --git a/news/12216.bugfix.rst b/news/12216.bugfix.rst deleted file mode 100644 index 804bf1ee9bb..00000000000 --- a/news/12216.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix resolution to respect ``--python-version`` when checking ``Requires-Python``. diff --git a/news/12572.trivial.rst b/news/12572.trivial.rst deleted file mode 100644 index f50b78a217c..00000000000 --- a/news/12572.trivial.rst +++ /dev/null @@ -1 +0,0 @@ -Add ``groups`` in dependabot.yml to bump group updates diff --git a/news/12656.feature.rst b/news/12656.feature.rst deleted file mode 100644 index fdbba5484ba..00000000000 --- a/news/12656.feature.rst +++ /dev/null @@ -1,3 +0,0 @@ -Improve discovery performance of installed packages when the -``importlib.metadata`` backend is used to load distribution metadata -(used by default under Python 3.11+). diff --git a/news/12660.trivial.rst b/news/12660.trivial.rst deleted file mode 100644 index f02256eccbb..00000000000 --- a/news/12660.trivial.rst +++ /dev/null @@ -1,2 +0,0 @@ -Remove (suppressed) deprecation warning from vendored ``pkg_resources`` -to ensure builds succeed with ``PYTHONWARNINGS=error``. diff --git a/news/12663.feature.rst b/news/12663.feature.rst deleted file mode 100644 index 11300ae47d3..00000000000 --- a/news/12663.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Improve performance when the same requirement string appears many times during resolution, by consistently caching the parsed requirement string. diff --git a/news/12664.feature.rst b/news/12664.feature.rst deleted file mode 100644 index a5f85097c62..00000000000 --- a/news/12664.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Minor performance improvement of finding applicable package candidates by not repeatedly calculating their versions diff --git a/news/12680.bugfix.rst b/news/12680.bugfix.rst deleted file mode 100644 index fa821f9f339..00000000000 --- a/news/12680.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Perform hash comparisons in a case-insensitive manner. diff --git a/news/12683.feature.rst b/news/12683.feature.rst deleted file mode 100644 index 2e949cd0762..00000000000 --- a/news/12683.feature.rst +++ /dev/null @@ -1,2 +0,0 @@ -Disable pip's self version check when invoking a pip subprocess to install -PEP 517 build requirements. diff --git a/news/12712.feature.rst b/news/12712.feature.rst deleted file mode 100644 index 3a08cdb10e2..00000000000 --- a/news/12712.feature.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve dependency resolution performance by caching platform compatibility -tags during wheel cache lookup. diff --git a/news/12716.bugfix.rst b/news/12716.bugfix.rst deleted file mode 100644 index 7af794f7bb3..00000000000 --- a/news/12716.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid dlopen failure for glibc detection in musl builds diff --git a/news/12728.feature.rst b/news/12728.feature.rst deleted file mode 100644 index 923d18707e0..00000000000 --- a/news/12728.feature.rst +++ /dev/null @@ -1,5 +0,0 @@ -``wheel`` is no longer explicitly listed as a build depepndency of ``pip``. -``setuptools`` already injects this dependency in the ``get_requires_for_build_wheel()`` hook. -This makes no difference for users of ``pip``. -This makes no difference when building wheels of ``pip``. -This avoids an unnecessary dependency on ``wheel`` when building the source distribution of ``pip``. diff --git a/news/12751.bugfix.rst b/news/12751.bugfix.rst deleted file mode 100644 index 70c6680a8a9..00000000000 --- a/news/12751.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid keyring logging crashes when pip is run in verbose mode. diff --git a/news/12776.trivial.rst b/news/12776.trivial.rst deleted file mode 100644 index 87d2f8e7975..00000000000 --- a/news/12776.trivial.rst +++ /dev/null @@ -1 +0,0 @@ -Use prerelease version of CFFI for Python 3.13 testing diff --git a/news/12781.bugfix.rst b/news/12781.bugfix.rst deleted file mode 100644 index 6bd43d347db..00000000000 --- a/news/12781.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix finding hardlink targets in tar files with an ignored top-level directory. diff --git a/news/12782.bugfix.rst b/news/12782.bugfix.rst deleted file mode 100644 index 459a2838c32..00000000000 --- a/news/12782.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve pip install performance by only creating required parent -directories once, instead of before extracting every file in the wheel. diff --git a/news/12791.bugfix.rst b/news/12791.bugfix.rst deleted file mode 100644 index c19345d439c..00000000000 --- a/news/12791.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve pip install performance. The installed packages printout is -now calculated in linear time instead of quadratic time. diff --git a/news/12796.vendor.rst b/news/12796.vendor.rst deleted file mode 100644 index 6384a5b1476..00000000000 --- a/news/12796.vendor.rst +++ /dev/null @@ -1 +0,0 @@ -Update the preload list for the ``DEBUNDLED`` case, to replace ``pep517`` that has been renamed to ``pyproject_hooks``. diff --git a/news/12797.vendor.rst b/news/12797.vendor.rst deleted file mode 100644 index 7842883ddba..00000000000 --- a/news/12797.vendor.rst +++ /dev/null @@ -1 +0,0 @@ -Use tomllib from the stdlib if available, rather than tomli diff --git a/news/12803.bugfix.rst b/news/12803.bugfix.rst deleted file mode 100644 index 4193d33e05c..00000000000 --- a/news/12803.bugfix.rst +++ /dev/null @@ -1,4 +0,0 @@ -Improve pip install performance. Files are now extracted in 1MB blocks, -or in one block matching the file size for smaller files. -A decompressor is no longer instantiated when extracting 0 bytes files, -it is not necessary because there is no data to decompress. diff --git a/news/12805.trivial.rst b/news/12805.trivial.rst deleted file mode 100644 index 56f61f77238..00000000000 --- a/news/12805.trivial.rst +++ /dev/null @@ -1 +0,0 @@ -Update ruff to 0.5.0 diff --git a/news/12810.feature.rst b/news/12810.feature.rst deleted file mode 100644 index fd236947e4d..00000000000 --- a/news/12810.feature.rst +++ /dev/null @@ -1,5 +0,0 @@ -Improve download performance. Download packages and update the -progress bar in larger chunks of 256 kB, up from 10 kB. -Limit the progress bar to 5 refresh per second. -Improve hash performance. Read package files in larger chunks of 1 MB, -up from 8192 bytes. diff --git a/news/12842.feature.rst b/news/12842.feature.rst deleted file mode 100644 index 60ebc3245f2..00000000000 --- a/news/12842.feature.rst +++ /dev/null @@ -1 +0,0 @@ -Ignore ``--require-virtualenv`` for ``pip check`` and ``pip freeze`` diff --git a/news/12851.doc.rst b/news/12851.doc.rst deleted file mode 100644 index 844545c73ac..00000000000 --- a/news/12851.doc.rst +++ /dev/null @@ -1 +0,0 @@ -Correct ``—-ignore-conflicts`` (including an em dash) to ``--ignore-conflicts``. diff --git a/news/5dac0fc9-10c7-4d77-b8dc-8ff111c9557d.trivial.rst b/news/5dac0fc9-10c7-4d77-b8dc-8ff111c9557d.trivial.rst deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/news/72167f18-bd68-41e1-8404-4d23e6b2652f.trivial.rst b/news/72167f18-bd68-41e1-8404-4d23e6b2652f.trivial.rst deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/news/aa82171b-1578-4128-8db3-9aa72b3a6a84.trivial.rst b/news/aa82171b-1578-4128-8db3-9aa72b3a6a84.trivial.rst deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/news/bcfde343-3f44-464e-9229-7af962defac6.trivial.rst b/news/bcfde343-3f44-464e-9229-7af962defac6.trivial.rst deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/news/certifi.vendor.rst b/news/certifi.vendor.rst deleted file mode 100644 index bc4ad30e7f9..00000000000 --- a/news/certifi.vendor.rst +++ /dev/null @@ -1 +0,0 @@ -Upgrade certifi to 2024.7.4 diff --git a/news/d0281e66-f6f9-4fb6-ac44-b5a9d468d42b.trivial.rst b/news/d0281e66-f6f9-4fb6-ac44-b5a9d468d42b.trivial.rst deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/news/platformdirs.vendor.rst b/news/platformdirs.vendor.rst deleted file mode 100644 index b2007b95b09..00000000000 --- a/news/platformdirs.vendor.rst +++ /dev/null @@ -1 +0,0 @@ -Upgrade platformdirs to 4.2.2 diff --git a/news/pygments.vendor.rst b/news/pygments.vendor.rst deleted file mode 100644 index 5232695fa02..00000000000 --- a/news/pygments.vendor.rst +++ /dev/null @@ -1 +0,0 @@ -Upgrade pygments to 2.18.0 diff --git a/news/setuptools.vendor.rst b/news/setuptools.vendor.rst deleted file mode 100644 index afdd14c09dc..00000000000 --- a/news/setuptools.vendor.rst +++ /dev/null @@ -1 +0,0 @@ -Upgrade setuptools to 70.3.0 diff --git a/news/typing_extensions.vendor.rst b/news/typing_extensions.vendor.rst deleted file mode 100644 index 580ac5dfb2b..00000000000 --- a/news/typing_extensions.vendor.rst +++ /dev/null @@ -1 +0,0 @@ -Upgrade typing_extensions to 4.12.2 diff --git a/src/pip/__init__.py b/src/pip/__init__.py index 531eec2d928..bfc5eb111a0 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -1,6 +1,6 @@ from typing import List, Optional -__version__ = "24.2.dev0" +__version__ = "24.3.dev0" def main(args: Optional[List[str]] = None) -> int: diff --git a/src/pip/_internal/models/direct_url.py b/src/pip/_internal/models/direct_url.py index fc5ec8d4aa9..efaf017b2b8 100644 --- a/src/pip/_internal/models/direct_url.py +++ b/src/pip/_internal/models/direct_url.py @@ -17,6 +17,7 @@ T = TypeVar("T") DIRECT_URL_METADATA_NAME = "direct_url.json" +PROVENANCE_URL_METADATA_NAME = "provenance_url.json" ENV_VAR_RE = re.compile(r"^\$\{[A-Za-z0-9-_]+\}(:\$\{[A-Za-z0-9-_]+\})?$") @@ -205,20 +206,27 @@ def from_dict(cls, d: Dict[str, Any]) -> "DirectUrl": ), ) - def to_dict(self) -> Dict[str, Any]: + def to_dict(self, *, keep_legacy_hash_key: bool = True) -> Dict[str, Any]: res = _filter_none( url=self.redacted_url, subdirectory=self.subdirectory, ) - res[self.info.name] = self.info._to_dict() + + info_dict = self.info._to_dict() + if not keep_legacy_hash_key: + info_dict.pop("hash", None) + + res[self.info.name] = info_dict return res @classmethod def from_json(cls, s: str) -> "DirectUrl": return cls.from_dict(json.loads(s)) - def to_json(self) -> str: - return json.dumps(self.to_dict(), sort_keys=True) + def to_json(self, *, keep_legacy_hash_key: bool = True) -> str: + return json.dumps( + self.to_dict(keep_legacy_hash_key=keep_legacy_hash_key), sort_keys=True + ) def is_local_editable(self) -> bool: return isinstance(self.info, DirInfo) and self.info.editable diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py index aef42aa9eef..673e7b670de 100644 --- a/src/pip/_internal/operations/install/wheel.py +++ b/src/pip/_internal/operations/install/wheel.py @@ -48,7 +48,12 @@ FilesystemWheel, get_wheel_distribution, ) -from pip._internal.models.direct_url import DIRECT_URL_METADATA_NAME, DirectUrl +from pip._internal.models.direct_url import ( + DIRECT_URL_METADATA_NAME, + PROVENANCE_URL_METADATA_NAME, + ArchiveInfo, + DirectUrl, +) from pip._internal.models.scheme import SCHEME_KEYS, Scheme from pip._internal.utils.filesystem import adjacent_tmp_file, replace from pip._internal.utils.misc import StreamWrapper, ensure_dir, hash_file, partition @@ -424,9 +429,10 @@ def _install_wheel( # noqa: C901, PLR0915 function is too long wheel_zip: ZipFile, wheel_path: str, scheme: Scheme, + download_info: DirectUrl, + is_direct: bool, pycompile: bool = True, warn_script_location: bool = True, - direct_url: Optional[DirectUrl] = None, requested: bool = False, ) -> None: """Install a wheel. @@ -673,12 +679,25 @@ def _generate_file(path: str, **kwargs: Any) -> Generator[BinaryIO, None, None]: installer_file.write(b"pip\n") generated.append(installer_path) - # Record the PEP 610 direct URL reference - if direct_url is not None: + if is_direct: + # Record the PEP 610 direct URL reference direct_url_path = os.path.join(dest_info_dir, DIRECT_URL_METADATA_NAME) with _generate_file(direct_url_path) as direct_url_file: - direct_url_file.write(direct_url.to_json().encode("utf-8")) + direct_url_file.write(download_info.to_json().encode("utf-8")) generated.append(direct_url_path) + else: + # Record the PEP 710 provenance URL reference only if we have hashes for + # the given wheel. They can be missing when wheels are built using an old pip. + assert isinstance(download_info.info, ArchiveInfo) + if download_info.info.hashes: + provenance_url_path = os.path.join( + dest_info_dir, PROVENANCE_URL_METADATA_NAME + ) + with _generate_file(provenance_url_path) as provenance_url_file: + provenance_url_file.write( + download_info.to_json(keep_legacy_hash_key=False).encode("utf-8") + ) + generated.append(provenance_url_path) # Record the REQUESTED file if requested: @@ -721,10 +740,11 @@ def install_wheel( name: str, wheel_path: str, scheme: Scheme, + download_info: DirectUrl, + is_direct: bool, req_description: str, pycompile: bool = True, warn_script_location: bool = True, - direct_url: Optional[DirectUrl] = None, requested: bool = False, ) -> None: with ZipFile(wheel_path, allowZip64=True) as z: @@ -734,8 +754,9 @@ def install_wheel( wheel_zip=z, wheel_path=wheel_path, scheme=scheme, + download_info=download_info, + is_direct=is_direct, pycompile=pycompile, warn_script_location=warn_script_location, - direct_url=direct_url, requested=requested, ) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 834bc513356..0f162c43152 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -861,6 +861,7 @@ def install( self.install_succeeded = True return + assert self.download_info assert self.is_wheel assert self.local_file_path @@ -868,10 +869,11 @@ def install( self.req.name, self.local_file_path, scheme=scheme, + download_info=self.download_info, + is_direct=self.is_direct, req_description=str(self.req), pycompile=pycompile, warn_script_location=warn_script_location, - direct_url=self.download_info if self.is_direct else None, requested=self.user_supplied, ) self.install_succeeded = True diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index c6b5635f8fe..dc56cea76ee 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -1,5 +1,6 @@ import hashlib import io +import json import os import re import ssl @@ -9,7 +10,7 @@ import textwrap from os.path import curdir, join, pardir from pathlib import Path -from typing import Dict, Iterable, List, Optional, Tuple +from typing import Any, Dict, Iterable, List, Optional, Tuple import pytest @@ -2743,3 +2744,154 @@ def add_link(tar: tarfile.TarFile, name: str, linktype: str, target: str) -> Non # Run the internal test result = script.run("python", "-m", "linktest") assert result.stdout.strip() == "8 files checked" + + +def _check_provenance_url(provenance_url: Dict[str, Any]) -> None: + assert "archive_info" in provenance_url + assert "url" in provenance_url + assert ( + len(provenance_url) == 2 + ), "provenance_url.json should hold only archive_info and url keys" + + assert "hashes" in provenance_url["archive_info"] + assert len(provenance_url["archive_info"]["hashes"]) > 0 + + +@pytest.mark.parametrize( + "pkg_name, pkg_version, distribution", + [ + pytest.param( + "simplewheel", + "1.0", + "simplewheel-1.0-py2.py3-none-any.whl", + id="wheel", + ), + pytest.param( + "simple", + "1.0", + "simple-1.0.tar.gz", + id="sdist", + ), + ], +) +def test_install_provenance_url( + script: PipTestEnvironment, + data: TestData, + pkg_name: str, + pkg_version: str, + distribution: str, +) -> None: + """Test installing a distribution from a simple API produces provenance_url.json.""" + server = make_mock_server() + + distribution_path = f"/files/{distribution}" + server.mock.side_effect = [ + package_page( + { + distribution: distribution_path, + } + ), + file_response(data.packages.joinpath(distribution)), + ] + + index_url = f"http://{server.host}:{server.port}" + + pip_args = [ + "install", + "-i", + index_url, + f"{pkg_name}=={pkg_version}", + ] + with server_running(server): + result = script.pip(*pip_args) + + result.assert_installed( + pkg_name=pkg_name, without_egg_link=True, editable=False + ) + + provenance_url_path = ( + script.site_packages + / f"{pkg_name}-{pkg_version}.dist-info" + / "provenance_url.json" + ) + + assert result.files_created[ + provenance_url_path + ], "provenance_url.json was not created" + + provenance_url_full_path = result.files_created[provenance_url_path].full + + with open(provenance_url_full_path) as f: + provenance_url_content = json.load(f) + + _check_provenance_url(provenance_url_content) + assert provenance_url_content["url"] == f"{index_url}{distribution_path}" + + +def test_install_provenance_url_cached( + script: PipTestEnvironment, data: TestData +) -> None: + """Test installing a cached distribution produced provenance_url.json.""" + pkg_name = "simple" + pkg_version = "1.0" + distribution = "simple-1.0.tar.gz" + + server = make_mock_server() + + distribution_path = f"/files/{distribution}" + server.mock.side_effect = [ + package_page( + { + distribution: distribution_path, + } + ), + file_response(data.packages.joinpath(distribution)), + ] * 2 + + index_url = f"http://{server.host}:{server.port}" + + pip_args = [ + "install", + "-i", + index_url, + f"{pkg_name}=={pkg_version}", + ] + + with server_running(server): + result = script.pip(*pip_args) + + result.assert_installed( + pkg_name=pkg_name, without_egg_link=True, editable=False + ) + + provenance_url_path = ( + script.site_packages + / f"{pkg_name}-{pkg_version}.dist-info" + / "provenance_url.json" + ) + + assert result.files_created[ + provenance_url_path + ], "provenance_url.json was not created" + + provenance_url_full_path = result.files_created[provenance_url_path].full + + with open(provenance_url_full_path) as f: + provenance_url_content = json.load(f) + + _check_provenance_url(provenance_url_content) + assert provenance_url_content["url"] == f"{index_url}{distribution_path}" + + os.unlink(provenance_url_full_path) + + pip_args.append("--ignore-installed") + result = script.pip(*pip_args) + + assert f"Using cached {pkg_name}" in result.stdout + + assert os.path.exists(provenance_url_full_path) + with open(provenance_url_full_path) as f: + new_provenance_url_content = json.load(f) + + _check_provenance_url(new_provenance_url_content) + assert new_provenance_url_content == provenance_url_content diff --git a/tests/unit/test_direct_url.py b/tests/unit/test_direct_url.py index 151e0a30f5b..4851db34199 100644 --- a/tests/unit/test_direct_url.py +++ b/tests/unit/test_direct_url.py @@ -28,6 +28,26 @@ def test_to_json() -> None: ) +def test_to_json_no_keep_legacy_hash_key() -> None: + direct_url = DirectUrl( + url="https://pypi.org/simple/sample/sample-1.2.0-py3-none-any.whl", + info=ArchiveInfo( + hash="sha256=257ded4ea1fafa475f099e544b2d7560f674d42" + "917e096d462e8a46a64f51245", + hashes={ + "sha256": "257ded4ea1fafa475f099e544b2d7560f674d" + "42917e096d462e8a46a64f51245", + }, + ), + ) + direct_url.validate() + assert direct_url.to_json(keep_legacy_hash_key=False) == ( + '{"archive_info": {"hashes": {' + '"sha256": "257ded4ea1fafa475f099e544b2d7560f674d42917e096d462e8a46a64f51245"}' + '}, "url": "https://pypi.org/simple/sample/sample-1.2.0-py3-none-any.whl"}' + ) + + def test_archive_info() -> None: direct_url_dict = { "url": "file:///home/user/archive.tgz", diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index ed6f5821133..f7ad4d21fbc 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -18,6 +18,7 @@ from pip._internal.locations import get_scheme from pip._internal.models.direct_url import ( DIRECT_URL_METADATA_NAME, + PROVENANCE_URL_METADATA_NAME, ArchiveInfo, DirectUrl, ) @@ -330,6 +331,17 @@ def main(): "gui_scripts": ["sample2 = sample:main"], }, ).save_to_dir(tmpdir) + self.download_info = DirectUrl( + url="https://localhost:8080/sample/sample-1.2.0-py3-none-any.whl", + info=ArchiveInfo( + hash="sha256=257ded4ea1fafa475f099e544b2d7560f674d42917e096d462e" + "8a46a64f51245", + hashes={ + "sha256": "257ded4ea1fafa475f099e544b2d7560f674d42917e096d46" + "2e8a46a64f51245", + }, + ), + ) self.req = Requirement("sample") self.src = os.path.join(tmpdir, "src") self.dest = os.path.join(tmpdir, "dest") @@ -370,6 +382,8 @@ def test_std_install(self, data: TestData, tmpdir: Path) -> None: self.name, self.wheelpath, scheme=self.scheme, + download_info=self.download_info, + is_direct=False, req_description=str(self.req), ) self.assert_installed(0o644) @@ -388,6 +402,8 @@ def test_std_install_with_custom_umask( self.name, self.wheelpath, scheme=self.scheme, + download_info=self.download_info, + is_direct=False, req_description=str(self.req), ) self.assert_installed(expected_permission) @@ -400,6 +416,8 @@ def test_std_install_requested(self, data: TestData, tmpdir: Path) -> None: self.name, self.wheelpath, scheme=self.scheme, + download_info=self.download_info, + is_direct=False, req_description=str(self.req), requested=True, ) @@ -415,7 +433,7 @@ def test_std_install_with_direct_url(self, data: TestData, tmpdir: Path) -> None because wheelpath is typically the result of a local build. """ self.prep(data, tmpdir) - direct_url = DirectUrl( + download_info = DirectUrl( url="file:///home/user/archive.tgz", info=ArchiveInfo(), ) @@ -423,19 +441,94 @@ def test_std_install_with_direct_url(self, data: TestData, tmpdir: Path) -> None self.name, self.wheelpath, scheme=self.scheme, + download_info=download_info, + is_direct=True, req_description=str(self.req), - direct_url=direct_url, ) direct_url_path = os.path.join(self.dest_dist_info, DIRECT_URL_METADATA_NAME) self.assert_permission(direct_url_path, 0o644) with open(direct_url_path, "rb") as f1: - expected_direct_url_json = direct_url.to_json() + expected_direct_url_json = download_info.to_json() direct_url_json = f1.read().decode("utf-8") assert direct_url_json == expected_direct_url_json - # check that the direc_url file is part of RECORDS + # check that the direct_url file is part of RECORDS with open(os.path.join(self.dest_dist_info, "RECORD")) as f2: assert DIRECT_URL_METADATA_NAME in f2.read() + def test_std_install_with_provenance_url( + self, data: TestData, tmpdir: Path + ) -> None: + """Test that install_wheel creates provenance_url.json metadata.""" + self.prep(data, tmpdir) + download_info = DirectUrl( + url="https://pypi.org/simple/sample/sample-1.2.0-py3-none-any.whl", + info=ArchiveInfo( + hash="sha256=257ded4ea1fafa475f099e544b2d7560f674d42" + "917e096d462e8a46a64f51245", + hashes={ + "sha256": "257ded4ea1fafa475f099e544b2d7560f674d" + "42917e096d462e8a46a64f51245", + }, + ), + ) + wheel.install_wheel( + self.name, + self.wheelpath, + scheme=self.scheme, + download_info=download_info, + is_direct=False, + req_description=str(self.req), + ) + provenance_url_path = os.path.join( + self.dest_dist_info, PROVENANCE_URL_METADATA_NAME + ) + self.assert_permission(provenance_url_path, 0o644) + with open(provenance_url_path, "rb") as f1: + expected_provenance_url_json = download_info.to_json( + keep_legacy_hash_key=False + ) + provenance_url_json = f1.read().decode("utf-8") + assert provenance_url_json == expected_provenance_url_json + # check that the provenance_url.json file is part of RECORDS + with open(os.path.join(self.dest_dist_info, "RECORD")) as f2: + assert PROVENANCE_URL_METADATA_NAME in f2.read() + + @pytest.mark.parametrize( + "hashes", + [ + pytest.param(None, id="None"), + pytest.param({}, id="empty"), + ], + ) + def test_std_install_with_provenance_url_no_hashes( + self, data: TestData, tmpdir: Path, hashes: Optional[Dict[str, str]] + ) -> None: + """Test that install_wheel does not create provenance_url.json + when hashes are missing. + """ + self.prep(data, tmpdir) + download_info = DirectUrl( + url="https://pypi.org/simple/sample/sample-1.2.0-py3-none-any.whl", + info=ArchiveInfo( + hash=None, + hashes=hashes, + ), + ) + wheel.install_wheel( + self.name, + self.wheelpath, + scheme=self.scheme, + download_info=download_info, + is_direct=False, + req_description=str(self.req), + ) + provenance_url_path = os.path.join( + self.dest_dist_info, PROVENANCE_URL_METADATA_NAME + ) + assert not os.path.exists(provenance_url_path) + with open(os.path.join(self.dest_dist_info, "RECORD")) as f2: + assert PROVENANCE_URL_METADATA_NAME not in f2.read() + def test_install_prefix(self, data: TestData, tmpdir: Path) -> None: prefix = os.path.join(os.path.sep, "some", "path") self.prep(data, tmpdir) @@ -451,6 +544,8 @@ def test_install_prefix(self, data: TestData, tmpdir: Path) -> None: self.name, self.wheelpath, scheme=scheme, + download_info=self.download_info, + is_direct=False, req_description=str(self.req), ) @@ -468,6 +563,8 @@ def test_dist_info_contains_empty_dir(self, data: TestData, tmpdir: Path) -> Non self.name, self.wheelpath, scheme=self.scheme, + download_info=self.download_info, + is_direct=False, req_description=str(self.req), ) self.assert_installed(0o644) @@ -486,6 +583,8 @@ def test_wheel_install_rejects_bad_paths( "simple", str(wheel_path), scheme=self.scheme, + download_info=self.download_info, + is_direct=False, req_description="simple", ) @@ -508,6 +607,8 @@ def test_invalid_entrypoints_fail( "simple", str(wheel_path), scheme=self.scheme, + download_info=self.download_info, + is_direct=False, req_description="simple", )