diff --git a/docs/configuration.md b/docs/configuration.md index 7c64129d1ab..f2f1e4265ee 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -285,6 +285,21 @@ Use parallel execution when using the new (`>=1.1.0`) installer. Set the maximum number of retries in an unstable network. This setting has no effect if the server does not support HTTP range requests. +### `installer.re-resolve` + +**Type**: `boolean` + +**Default**: `true` + +**Environment Variable**: `POETRY_INSTALLER_RE_RESOLVE` + +*Introduced in 2.0.0* + +If the config option is _not_ set and the lock file is at least version 2.1 +(created by Poetry 2.0 or above), the installer will not re-resolve dependencies +but evaluate the locked markers to decide which of the locked dependencies have to +be installed into the target environment. + ### `solver.lazy-wheel` **Type**: `boolean` diff --git a/src/poetry/config/config.py b/src/poetry/config/config.py index bf2f0948d03..4f0612938eb 100644 --- a/src/poetry/config/config.py +++ b/src/poetry/config/config.py @@ -126,6 +126,7 @@ class Config: "max-retries": 0, }, "installer": { + "re-resolve": True, "parallel": True, "max-workers": None, "no-binary": None, @@ -303,6 +304,7 @@ def _get_normalizer(name: str) -> Callable[[str], Any]: "virtualenvs.options.system-site-packages", "virtualenvs.options.prefer-active-python", "experimental.system-git-client", + "installer.re-resolve", "installer.parallel", "solver.lazy-wheel", "warnings.export", diff --git a/src/poetry/console/commands/config.py b/src/poetry/console/commands/config.py index 1a8a8e1307c..59bc36d0b5b 100644 --- a/src/poetry/console/commands/config.py +++ b/src/poetry/console/commands/config.py @@ -71,6 +71,7 @@ def unique_config_values(self) -> dict[str, tuple[Any, Any]]: "virtualenvs.prompt": (str, str), "experimental.system-git-client": (boolean_validator, boolean_normalizer), "requests.max-retries": (lambda val: int(val) >= 0, int_normalizer), + "installer.re-resolve": (boolean_validator, boolean_normalizer), "installer.parallel": (boolean_validator, boolean_normalizer), "installer.max-workers": (lambda val: int(val) > 0, int_normalizer), "installer.no-binary": ( diff --git a/src/poetry/installation/installer.py b/src/poetry/installation/installer.py index b918a153fac..370b4c5ec04 100644 --- a/src/poetry/installation/installer.py +++ b/src/poetry/installation/installer.py @@ -7,6 +7,7 @@ from packaging.utils import canonicalize_name from poetry.installation.executor import Executor +from poetry.puzzle.transaction import Transaction from poetry.repositories import Repository from poetry.repositories import RepositoryPool from poetry.repositories.installed_repository import InstalledRepository @@ -206,6 +207,10 @@ def _do_install(self) -> int: from poetry.puzzle.solver import Solver locked_repository = Repository("poetry-locked") + reresolve = self._config.get("installer.re-resolve", True) + solved_packages: dict[Package, TransitivePackageInfo] = {} + lockfile_repo = LockfileRepository() + if self._update: if not self._lock and self._locker.is_locked(): locked_repository = self._locker.locked_repository() @@ -241,7 +246,6 @@ def _do_install(self) -> int: self._write_lock_file(solved_packages) return 0 - lockfile_repo = LockfileRepository() for package in solved_packages: if not lockfile_repo.has_package(package): lockfile_repo.add_package(package) @@ -254,6 +258,13 @@ def _do_install(self) -> int: "pyproject.toml changed significantly since poetry.lock was last" " generated. Run `poetry lock [--no-update]` to fix the lock file." ) + if not reresolve and not self._locker.is_locked_groups_and_markers(): + if self._io.is_verbose(): + self._io.write_line( + "Cannot install without re-resolving" + " because the lock file is not at least version 2.1" + ) + reresolve = True locker_extras = { canonicalize_name(extra) @@ -264,7 +275,10 @@ def _do_install(self) -> int: raise ValueError(f"Extra [{extra}] is not specified.") locked_repository = self._locker.locked_repository() - lockfile_repo = locked_repository + if reresolve: + lockfile_repo = locked_repository + else: + solved_packages = self._locker.locked_packages() if self._io.is_verbose(): self._io.write_line("") @@ -272,33 +286,52 @@ def _do_install(self) -> int: "Finding the necessary packages for the current system" ) - if self._groups is not None: - root = self._package.with_dependency_groups(list(self._groups), only=True) - else: - root = self._package.without_optional_dependency_groups() + if reresolve: + if self._groups is not None: + root = self._package.with_dependency_groups( + list(self._groups), only=True + ) + else: + root = self._package.without_optional_dependency_groups() - # We resolve again by only using the lock file - packages = lockfile_repo.packages + locked_repository.packages - pool = RepositoryPool.from_packages(packages, self._config) + # We resolve again by only using the lock file + packages = lockfile_repo.packages + locked_repository.packages + pool = RepositoryPool.from_packages(packages, self._config) - solver = Solver( - root, - pool, - self._installed_repository.packages, - locked_repository.packages, - NullIO(), - ) - # Everything is resolved at this point, so we no longer need - # to load deferred dependencies (i.e. VCS, URL and path dependencies) - solver.provider.load_deferred(False) - - with solver.use_environment(self._env): - ops = solver.solve(use_latest=self._whitelist).calculate_operations( - with_uninstalls=self._requires_synchronization or self._update, - synchronize=self._requires_synchronization, - skip_directory=self._skip_directory, - extras=set(self._extras), + solver = Solver( + root, + pool, + self._installed_repository.packages, + locked_repository.packages, + NullIO(), ) + # Everything is resolved at this point, so we no longer need + # to load deferred dependencies (i.e. VCS, URL and path dependencies) + solver.provider.load_deferred(False) + + with solver.use_environment(self._env): + transaction = solver.solve(use_latest=self._whitelist) + + else: + if self._groups is None: + groups = self._package.dependency_group_names() + else: + groups = set(self._groups) + transaction = Transaction( + locked_repository.packages, + solved_packages, + self._installed_repository.packages, + self._package, + self._env.marker_env, + groups, + ) + + ops = transaction.calculate_operations( + with_uninstalls=self._requires_synchronization or self._update, + synchronize=self._requires_synchronization, + skip_directory=self._skip_directory, + extras=set(self._extras), + ) # Validate the dependencies for op in ops: diff --git a/src/poetry/packages/locker.py b/src/poetry/packages/locker.py index fc80c1811d2..f87c490d70f 100644 --- a/src/poetry/packages/locker.py +++ b/src/poetry/packages/locker.py @@ -18,6 +18,7 @@ from poetry.core.constraints.version import parse_constraint from poetry.core.packages.dependency import Dependency from poetry.core.packages.package import Package +from poetry.core.version.markers import parse_marker from poetry.core.version.requirements import InvalidRequirementError from tomlkit import array from tomlkit import comment @@ -26,6 +27,7 @@ from tomlkit import table from poetry.__version__ import __version__ +from poetry.packages.transitive_package_info import TransitivePackageInfo from poetry.toml.file import TOMLFile from poetry.utils._compat import tomllib @@ -38,7 +40,6 @@ from poetry.core.packages.vcs_dependency import VCSDependency from tomlkit.toml_document import TOMLDocument - from poetry.packages.transitive_package_info import TransitivePackageInfo from poetry.repositories.lockfile_repository import LockfileRepository logger = logging.getLogger(__name__) @@ -103,6 +104,13 @@ def is_fresh(self) -> bool: return False + def is_locked_groups_and_markers(self) -> bool: + if not self.is_locked(): + return False + + version = Version.parse(self.lock_data["metadata"]["lock-version"]) + return version >= Version.parse("2.1") + def set_pyproject_data(self, pyproject_data: dict[str, Any]) -> None: self._pyproject_data = pyproject_data self._content_hash = self._get_content_hash() @@ -121,7 +129,6 @@ def locked_repository(self) -> LockfileRepository: """ Searches and returns a repository of locked packages. """ - from poetry.factory import Factory from poetry.repositories.lockfile_repository import LockfileRepository repository = LockfileRepository() @@ -129,113 +136,41 @@ def locked_repository(self) -> LockfileRepository: if not self.is_locked(): return repository - lock_data = self.lock_data - locked_packages = cast("list[dict[str, Any]]", lock_data["package"]) + locked_packages = cast("list[dict[str, Any]]", self.lock_data["package"]) if not locked_packages: return repository for info in locked_packages: - source = info.get("source", {}) - source_type = source.get("type") - url = source.get("url") - if source_type in ["directory", "file"]: - url = self.lock.parent.joinpath(url).resolve().as_posix() - - name = info["name"] - package = Package( - name, - info["version"], - source_type=source_type, - source_url=url, - source_reference=source.get("reference"), - source_resolved_reference=source.get("resolved_reference"), - source_subdirectory=source.get("subdirectory"), - ) - package.description = info.get("description", "") - package.optional = info["optional"] - metadata = cast("dict[str, Any]", lock_data["metadata"]) - - # Storing of package files and hashes has been through a few generations in - # the lockfile, we can read them all: - # - # - latest and preferred is that this is read per package, from - # package.files - # - oldest is that hashes were stored in metadata.hashes without filenames - # - in between those two, hashes were stored alongside filenames in - # metadata.files - package_files = info.get("files") - if package_files is not None: - package.files = package_files - elif "hashes" in metadata: - hashes = cast("dict[str, Any]", metadata["hashes"]) - package.files = [{"name": h, "hash": h} for h in hashes[name]] - elif source_type in {"git", "directory", "url"}: - package.files = [] - else: - files = metadata["files"][name] - if source_type == "file": - filename = Path(url).name - package.files = [item for item in files if item["file"] == filename] - else: - # Strictly speaking, this is not correct, but we have no chance - # to always determine which are the correct files because the - # lockfile doesn't keep track which files belong to which package. - package.files = files - - package.python_versions = info["python-versions"] - - package_extras: dict[NormalizedName, list[Dependency]] = {} - extras = info.get("extras", {}) - if extras: - for name, deps in extras.items(): - name = canonicalize_name(name) - package_extras[name] = [] - - for dep in deps: - try: - dependency = Dependency.create_from_pep_508(dep) - except InvalidRequirementError: - # handle lock files with invalid PEP 508 - m = re.match(r"^(.+?)(?:\[(.+?)])?(?:\s+\((.+)\))?$", dep) - if not m: - raise - dep_name = m.group(1) - extras = m.group(2) or "" - constraint = m.group(3) or "*" - dependency = Dependency( - dep_name, constraint, extras=extras.split(",") - ) - package_extras[name].append(dependency) - - package.extras = package_extras - - for dep_name, constraint in info.get("dependencies", {}).items(): - root_dir = self.lock.parent - if package.source_type == "directory": - # root dir should be the source of the package relative to the lock - # path - assert package.source_url is not None - root_dir = Path(package.source_url) + repository.add_package(self._get_locked_package(info)) - if isinstance(constraint, list): - for c in constraint: - package.add_dependency( - Factory.create_dependency(dep_name, c, root_dir=root_dir) - ) + return repository - continue + def locked_packages(self) -> dict[Package, TransitivePackageInfo]: + if not self.is_locked_groups_and_markers(): + raise RuntimeError( + "This method should not be called if the lock file" + " is not at least version 2.1." + ) - package.add_dependency( - Factory.create_dependency(dep_name, constraint, root_dir=root_dir) - ) + locked_packages: dict[Package, TransitivePackageInfo] = {} - if "develop" in info: - package.develop = info["develop"] + locked_package_info = cast("list[dict[str, Any]]", self.lock_data["package"]) - repository.add_package(package) + for info in locked_package_info: + package = self._get_locked_package(info, with_dependencies=False) + groups = set(info["groups"]) + locked_marker = info.get("markers", "*") + if isinstance(locked_marker, str): + markers = {group: parse_marker(locked_marker) for group in groups} + else: + markers = { + group: parse_marker(locked_marker.get(group, "*")) + for group in groups + } + locked_packages[package] = TransitivePackageInfo(0, groups, markers) - return repository + return locked_packages def set_lock_data( self, root: Package, packages: dict[Package, TransitivePackageInfo] @@ -410,6 +345,111 @@ def _get_lock_data(self) -> dict[str, Any]: return lock_data + def _get_locked_package( + self, info: dict[str, Any], with_dependencies: bool = True + ) -> Package: + source = info.get("source", {}) + source_type = source.get("type") + url = source.get("url") + if source_type in ["directory", "file"]: + url = self.lock.parent.joinpath(url).resolve().as_posix() + + name = info["name"] + package = Package( + name, + info["version"], + source_type=source_type, + source_url=url, + source_reference=source.get("reference"), + source_resolved_reference=source.get("resolved_reference"), + source_subdirectory=source.get("subdirectory"), + ) + package.description = info.get("description", "") + package.optional = info["optional"] + metadata = cast("dict[str, Any]", self.lock_data["metadata"]) + + # Storing of package files and hashes has been through a few generations in + # the lockfile, we can read them all: + # + # - latest and preferred is that this is read per package, from + # package.files + # - oldest is that hashes were stored in metadata.hashes without filenames + # - in between those two, hashes were stored alongside filenames in + # metadata.files + package_files = info.get("files") + if package_files is not None: + package.files = package_files + elif "hashes" in metadata: + hashes = cast("dict[str, Any]", metadata["hashes"]) + package.files = [{"name": h, "hash": h} for h in hashes[name]] + elif source_type in {"git", "directory", "url"}: + package.files = [] + else: + files = metadata["files"][name] + if source_type == "file": + filename = Path(url).name + package.files = [item for item in files if item["file"] == filename] + else: + # Strictly speaking, this is not correct, but we have no chance + # to always determine which are the correct files because the + # lockfile doesn't keep track which files belong to which package. + package.files = files + + package.python_versions = info["python-versions"] + + if "develop" in info: + package.develop = info["develop"] + + if with_dependencies: + from poetry.factory import Factory + + package_extras: dict[NormalizedName, list[Dependency]] = {} + extras = info.get("extras", {}) + if extras: + for name, deps in extras.items(): + name = canonicalize_name(name) + package_extras[name] = [] + + for dep in deps: + try: + dependency = Dependency.create_from_pep_508(dep) + except InvalidRequirementError: + # handle lock files with invalid PEP 508 + m = re.match(r"^(.+?)(?:\[(.+?)])?(?:\s+\((.+)\))?$", dep) + if not m: + raise + dep_name = m.group(1) + extras = m.group(2) or "" + constraint = m.group(3) or "*" + dependency = Dependency( + dep_name, constraint, extras=extras.split(",") + ) + package_extras[name].append(dependency) + + package.extras = package_extras + + for dep_name, constraint in info.get("dependencies", {}).items(): + root_dir = self.lock.parent + if package.source_type == "directory": + # root dir should be the source of the package relative to the lock + # path + assert package.source_url is not None + root_dir = Path(package.source_url) + + if isinstance(constraint, list): + for c in constraint: + package.add_dependency( + Factory.create_dependency(dep_name, c, root_dir=root_dir) + ) + + continue + + package.add_dependency( + Factory.create_dependency(dep_name, constraint, root_dir=root_dir) + ) + + return package + def _lock_packages( self, packages: dict[Package, TransitivePackageInfo] ) -> list[dict[str, Any]]: diff --git a/src/poetry/packages/transitive_package_info.py b/src/poetry/packages/transitive_package_info.py index 84fb6e12d5d..82960ec08d3 100644 --- a/src/poetry/packages/transitive_package_info.py +++ b/src/poetry/packages/transitive_package_info.py @@ -3,9 +3,12 @@ from dataclasses import dataclass from typing import TYPE_CHECKING +from poetry.core.version.markers import BaseMarker +from poetry.core.version.markers import EmptyMarker + if TYPE_CHECKING: - from poetry.core.version.markers import BaseMarker + from collections.abc import Iterable @dataclass @@ -13,3 +16,10 @@ class TransitivePackageInfo: depth: int # max depth in the dependency tree groups: set[str] markers: dict[str, BaseMarker] # group -> marker + + def get_marker(self, groups: Iterable[str]) -> BaseMarker: + marker: BaseMarker = EmptyMarker() + for group in groups: + if group_marker := self.markers.get(group): + marker = marker.union(group_marker) + return marker diff --git a/src/poetry/puzzle/transaction.py b/src/poetry/puzzle/transaction.py index 6fedc491315..3f18e7f5095 100644 --- a/src/poetry/puzzle/transaction.py +++ b/src/poetry/puzzle/transaction.py @@ -2,6 +2,7 @@ from collections import defaultdict from typing import TYPE_CHECKING +from typing import Any from poetry.utils.extras import get_extra_package_names @@ -21,6 +22,8 @@ def __init__( result_packages: list[Package] | dict[Package, TransitivePackageInfo], installed_packages: list[Package] | None = None, root_package: Package | None = None, + marker_env: dict[str, Any] | None = None, + groups: set[str] | None = None, ) -> None: self._current_packages = current_packages self._result_packages = result_packages @@ -30,6 +33,8 @@ def __init__( self._installed_packages = installed_packages self._root_package = root_package + self._marker_env = marker_env + self._groups = groups def get_solved_packages(self) -> dict[Package, TransitivePackageInfo]: assert isinstance(self._result_packages, dict) @@ -50,7 +55,11 @@ def calculate_operations( operations: list[Operation] = [] extra_packages: set[NormalizedName] = set() - if extras is not None: + if self._marker_env: + marker_env_with_extras = self._marker_env.copy() + if extras is not None: + marker_env_with_extras["extra"] = extras + elif extras is not None: assert self._root_package is not None extra_packages = get_extra_package_names( self._result_packages, @@ -64,13 +73,31 @@ def calculate_operations( } else: priorities = defaultdict(int) + relevant_result_packages: set[NormalizedName] = set() uninstalls: set[NormalizedName] = set() for result_package in self._result_packages: - installed = False - is_unsolicited_extra = extras is not None and ( - result_package.optional and result_package.name not in extra_packages - ) + is_unsolicited_extra = False + if self._marker_env: + assert self._groups is not None + assert isinstance(self._result_packages, dict) + info = self._result_packages[result_package] + + if info.groups & self._groups and info.get_marker( + self._groups + ).validate(marker_env_with_extras): + relevant_result_packages.add(result_package.name) + elif result_package.optional: + is_unsolicited_extra = True + else: + continue + else: + relevant_result_packages.add(result_package.name) + is_unsolicited_extra = extras is not None and ( + result_package.optional + and result_package.name not in extra_packages + ) + installed = False for installed_package in self._installed_packages: if result_package.name == installed_package.name: installed = True @@ -123,10 +150,7 @@ def calculate_operations( if with_uninstalls: for current_package in self._current_packages: - found = any( - current_package.name == result_package.name - for result_package in self._result_packages - ) + found = current_package.name in (relevant_result_packages | uninstalls) if not found: for installed_package in self._installed_packages: @@ -135,12 +159,9 @@ def calculate_operations( operations.append(Uninstall(installed_package)) if synchronize: - result_package_names = { - result_package.name for result_package in self._result_packages - } # We preserve pip when not managed by poetry, this is done to avoid # externally managed virtual environments causing unnecessary removals. - preserved_package_names = {"pip"} - result_package_names + preserved_package_names = {"pip"} - relevant_result_packages for installed_package in self._installed_packages: if installed_package.name in uninstalls: @@ -155,7 +176,7 @@ def calculate_operations( if installed_package.name in preserved_package_names: continue - if installed_package.name not in result_package_names: + if installed_package.name not in relevant_result_packages: uninstalls.add(installed_package.name) operations.append(Uninstall(installed_package)) diff --git a/tests/console/commands/test_config.py b/tests/console/commands/test_config.py index 0572c2c12a7..d34e2c5879a 100644 --- a/tests/console/commands/test_config.py +++ b/tests/console/commands/test_config.py @@ -59,6 +59,7 @@ def test_list_displays_default_value_if_not_set( installer.no-binary = null installer.only-binary = null installer.parallel = true +installer.re-resolve = true keyring.enabled = true requests.max-retries = 0 solver.lazy-wheel = true @@ -91,6 +92,7 @@ def test_list_displays_set_get_setting( installer.no-binary = null installer.only-binary = null installer.parallel = true +installer.re-resolve = true keyring.enabled = true requests.max-retries = 0 solver.lazy-wheel = true @@ -144,6 +146,7 @@ def test_unset_setting( installer.no-binary = null installer.only-binary = null installer.parallel = true +installer.re-resolve = true keyring.enabled = true requests.max-retries = 0 solver.lazy-wheel = true @@ -175,6 +178,7 @@ def test_unset_repo_setting( installer.no-binary = null installer.only-binary = null installer.parallel = true +installer.re-resolve = true keyring.enabled = true requests.max-retries = 0 solver.lazy-wheel = true @@ -304,6 +308,7 @@ def test_list_displays_set_get_local_setting( installer.no-binary = null installer.only-binary = null installer.parallel = true +installer.re-resolve = true keyring.enabled = true requests.max-retries = 0 solver.lazy-wheel = true @@ -343,6 +348,7 @@ def test_list_must_not_display_sources_from_pyproject_toml( installer.no-binary = null installer.only-binary = null installer.parallel = true +installer.re-resolve = true keyring.enabled = true repositories.foo.url = "https://foo.bar/simple/" requests.max-retries = 0 diff --git a/tests/installation/test_installer.py b/tests/installation/test_installer.py index e567269b549..f4b81757c82 100644 --- a/tests/installation/test_installer.py +++ b/tests/installation/test_installer.py @@ -15,6 +15,7 @@ from cleo.io.null_io import NullIO from cleo.io.outputs.output import Verbosity from packaging.utils import canonicalize_name +from poetry.core.constraints.version import Version from poetry.core.packages.dependency_group import MAIN_GROUP from poetry.core.packages.dependency_group import DependencyGroup from poetry.core.packages.package import Package @@ -36,6 +37,9 @@ if TYPE_CHECKING: + from collections.abc import Iterator + + from _pytest.fixtures import FixtureRequest from pytest_mock import MockerFixture from poetry.installation.operations.operation import Operation @@ -142,6 +146,14 @@ def _write_lock_data(self, data: dict[str, Any]) -> None: self._lock_data = data +@pytest.fixture(autouse=True, params=[False, True]) +def config_installer_reresolve( + config: Config, request: FixtureRequest +) -> Iterator[bool]: + config.config["installer"]["re-resolve"] = request.param + yield request.param + + @pytest.fixture() def package() -> ProjectPackage: p = ProjectPackage("root", "1.0") @@ -206,6 +218,14 @@ def fixture(name: str) -> dict[str, Any]: return content +def fix_lock_data(lock_data: dict[str, Any]) -> None: + if Version.parse(lock_data["metadata"]["lock-version"]) >= Version.parse("2.1"): + for locked_package in lock_data["package"]: + locked_package["groups"] = ["main"] + locked_package["files"] = [] + del lock_data["metadata"]["files"] + + def test_run_no_dependencies(installer: Installer, locker: Locker) -> None: result = installer.run() assert result == 0 @@ -244,50 +264,52 @@ def test_run_with_dependencies( assert locker.written_data == expected +@pytest.mark.parametrize("lock_version", ("1.1", "2.1")) def test_run_update_after_removing_dependencies( installer: Installer, locker: Locker, repo: Repository, package: ProjectPackage, installed: CustomInstalledRepository, + lock_version: str, ) -> None: - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": "A", - "version": "1.0", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - { - "name": "B", - "version": "1.1", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - { - "name": "C", - "version": "1.2", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - ], - "metadata": { + lock_data = { + "package": [ + { + "name": "A", + "version": "1.0", + "optional": False, + "platform": "*", "python-versions": "*", + "checksum": [], + }, + { + "name": "B", + "version": "1.1", + "optional": False, "platform": "*", - "content-hash": "123456789", - "files": {"A": [], "B": [], "C": []}, + "python-versions": "*", + "checksum": [], }, - } - ) + { + "name": "C", + "version": "1.2", + "optional": False, + "platform": "*", + "python-versions": "*", + "checksum": [], + }, + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + "files": {"A": [], "B": [], "C": []}, + }, + } + fix_lock_data(lock_data) + locker.locked(True) + locker.mock_lock_data(lock_data) package_a = get_package("A", "1.0") package_b = get_package("B", "1.1") package_c = get_package("C", "1.2") @@ -315,6 +337,7 @@ def test_run_update_after_removing_dependencies( def _configure_run_install_dev( + lock_version: str, locker: Locker, repo: Repository, package: ProjectPackage, @@ -325,43 +348,49 @@ def _configure_run_install_dev( """ Perform common test setup for `test_run_install_*dev*()` methods. """ - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": "A", - "version": "1.0", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - { - "name": "B", - "version": "1.1", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - { - "name": "C", - "version": "1.2", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - ], - "metadata": { + lock_data: dict[str, Any] = { + "package": [ + { + "name": "A", + "version": "1.0", + "optional": False, + "platform": "*", "python-versions": "*", + "checksum": [], + }, + { + "name": "B", + "version": "1.1", + "optional": False, "platform": "*", - "content-hash": "123456789", - "files": {"A": [], "B": [], "C": []}, + "python-versions": "*", + "checksum": [], }, - } - ) + { + "name": "C", + "version": "1.2", + "optional": False, + "platform": "*", + "python-versions": "*", + "checksum": [], + }, + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + "files": {"A": [], "B": [], "C": []}, + }, + } + if lock_version == "2.1": + for locked_package in lock_data["package"]: + locked_package["groups"] = [ + "dev" if locked_package["name"] == "C" else "main" + ] + locked_package["files"] = [] + del lock_data["metadata"]["files"] + locker.locked(True) + locker.mock_lock_data(lock_data) package_a = get_package("A", "1.0") package_b = get_package("B", "1.1") package_c = get_package("C", "1.2") @@ -382,6 +411,7 @@ def _configure_run_install_dev( package.add_dependency_group(group) +@pytest.mark.parametrize("lock_version", ("1.1", "2.1")) @pytest.mark.parametrize( ("groups", "installs", "updates", "removals", "with_packages_installed"), [ @@ -408,8 +438,10 @@ def test_run_install_with_dependency_groups( repo: Repository, package: ProjectPackage, installed: CustomInstalledRepository, + lock_version: str, ) -> None: _configure_run_install_dev( + lock_version, locker, repo, package, @@ -430,12 +462,14 @@ def test_run_install_with_dependency_groups( assert installer.executor.removals_count == removals +@pytest.mark.parametrize("lock_version", ("1.1", "2.1")) def test_run_install_does_not_remove_locked_packages_if_installed_but_not_required( installer: Installer, locker: Locker, repo: Repository, package: ProjectPackage, installed: CustomInstalledRepository, + lock_version: str, ) -> None: package_a = get_package("a", "1.0") package_b = get_package("b", "1.1") @@ -454,43 +488,43 @@ def test_run_install_does_not_remove_locked_packages_if_installed_but_not_requir Factory.create_dependency(package_a.name, str(package_a.version)) ) - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": package_a.name, - "version": package_a.version.text, - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - { - "name": package_b.name, - "version": package_b.version.text, - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - { - "name": package_c.name, - "version": package_c.version.text, - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - ], - "metadata": { + lock_data = { + "package": [ + { + "name": package_a.name, + "version": package_a.version.text, + "optional": False, + "platform": "*", "python-versions": "*", + "checksum": [], + }, + { + "name": package_b.name, + "version": package_b.version.text, + "optional": False, "platform": "*", - "content-hash": "123456789", - "files": {package_a.name: [], package_b.name: [], package_c.name: []}, + "python-versions": "*", + "checksum": [], }, - } - ) + { + "name": package_c.name, + "version": package_c.version.text, + "optional": False, + "platform": "*", + "python-versions": "*", + "checksum": [], + }, + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + "files": {package_a.name: [], package_b.name: [], package_c.name: []}, + }, + } + fix_lock_data(lock_data) + locker.locked(True) + locker.mock_lock_data(lock_data) result = installer.run() assert result == 0 @@ -500,12 +534,15 @@ def test_run_install_does_not_remove_locked_packages_if_installed_but_not_requir assert installer.executor.removals_count == 0 +@pytest.mark.parametrize("lock_version", ("1.1", "2.1")) def test_run_install_removes_locked_packages_if_installed_and_synchronization_is_required( installer: Installer, locker: Locker, repo: Repository, package: ProjectPackage, installed: CustomInstalledRepository, + lock_version: str, + config_installer_reresolve: bool, ) -> None: package_a = get_package("a", "1.0") package_b = get_package("b", "1.1") @@ -524,44 +561,45 @@ def test_run_install_removes_locked_packages_if_installed_and_synchronization_is Factory.create_dependency(package_a.name, str(package_a.version)) ) - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": package_a.name, - "version": package_a.version.text, - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - { - "name": package_b.name, - "version": package_b.version.text, - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - { - "name": package_c.name, - "version": package_c.version.text, - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - ], - "metadata": { + lock_data = { + "package": [ + { + "name": package_a.name, + "version": package_a.version.text, + "optional": False, + "platform": "*", + "python-versions": "*", + "checksum": [], + }, + { + "name": package_b.name, + "version": package_b.version.text, + "optional": False, + "platform": "*", "python-versions": "*", + "checksum": [], + }, + { + "name": package_c.name, + "version": package_c.version.text, + "optional": False, "platform": "*", - "content-hash": "123456789", - "files": {package_a.name: [], package_b.name: [], package_c.name: []}, + "python-versions": "*", + "checksum": [], }, - } - ) + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + "files": {package_a.name: [], package_b.name: [], package_c.name: []}, + }, + } + fix_lock_data(lock_data) + locker.locked(True) + locker.mock_lock_data(lock_data) + installer.update(True) installer.requires_synchronization(True) installer.run() @@ -570,12 +608,14 @@ def test_run_install_removes_locked_packages_if_installed_and_synchronization_is assert installer.executor.removals_count == 2 +@pytest.mark.parametrize("lock_version", ("1.1", "2.1")) def test_run_install_removes_no_longer_locked_packages_if_installed( installer: Installer, locker: Locker, repo: Repository, package: ProjectPackage, installed: CustomInstalledRepository, + lock_version: str, ) -> None: package_a = get_package("a", "1.0") package_b = get_package("b", "1.1") @@ -594,43 +634,43 @@ def test_run_install_removes_no_longer_locked_packages_if_installed( Factory.create_dependency(package_a.name, str(package_a.version)) ) - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": package_a.name, - "version": package_a.version.text, - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - { - "name": package_b.name, - "version": package_b.version.text, - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - { - "name": package_c.name, - "version": package_c.version.text, - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - ], - "metadata": { + lock_data = { + "package": [ + { + "name": package_a.name, + "version": package_a.version.text, + "optional": False, + "platform": "*", "python-versions": "*", + "checksum": [], + }, + { + "name": package_b.name, + "version": package_b.version.text, + "optional": False, "platform": "*", - "content-hash": "123456789", - "files": {package_a.name: [], package_b.name: [], package_c.name: []}, + "python-versions": "*", + "checksum": [], }, - } - ) + { + "name": package_c.name, + "version": package_c.version.text, + "optional": False, + "platform": "*", + "python-versions": "*", + "checksum": [], + }, + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + "files": {package_a.name: [], package_b.name: [], package_c.name: []}, + }, + } + fix_lock_data(lock_data) + locker.locked(True) + locker.mock_lock_data(lock_data) installer.update(True) result = installer.run() @@ -641,6 +681,7 @@ def test_run_install_removes_no_longer_locked_packages_if_installed( assert installer.executor.removals_count == 2 +@pytest.mark.parametrize("lock_version", ("1.1", "2.1")) @pytest.mark.parametrize( "managed_reserved_package_names", [(), ("pip",)], @@ -652,6 +693,7 @@ def test_run_install_with_synchronization( repo: Repository, package: ProjectPackage, installed: CustomInstalledRepository, + lock_version: str, ) -> None: package_a = get_package("a", "1.0") package_b = get_package("b", "1.1") @@ -680,36 +722,37 @@ def test_run_install_with_synchronization( Factory.create_dependency(package_a.name, str(package_a.version)) ) - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": pkg.name, - "version": pkg.version, - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - } - for pkg in locked_packages - ], - "metadata": { - "python-versions": "*", + lock_data = { + "package": [ + { + "name": pkg.name, + "version": pkg.version, + "optional": False, "platform": "*", - "content-hash": "123456789", - "files": {pkg.name: [] for pkg in locked_packages}, - }, - } - ) + "python-versions": "*", + "checksum": [], + } + for pkg in locked_packages + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + "files": {pkg.name: [] for pkg in locked_packages}, + }, + } + fix_lock_data(lock_data) + locker.locked(True) + locker.mock_lock_data(lock_data) + installer.update(True) installer.requires_synchronization(True) result = installer.run() assert result == 0 assert installer.executor.installations_count == 0 assert installer.executor.updates_count == 0 - assert 2 + len(managed_reserved_packages) == installer.executor.removals_count + assert installer.executor.removals_count == 2 + len(managed_reserved_packages) expected_removals = { package_b.name, @@ -721,30 +764,35 @@ def test_run_install_with_synchronization( assert {r.name for r in installer.executor.removals} == expected_removals +@pytest.mark.parametrize("lock_version", ("1.1", "2.1")) def test_run_whitelist_add( - installer: Installer, locker: Locker, repo: Repository, package: ProjectPackage + installer: Installer, + locker: Locker, + repo: Repository, + package: ProjectPackage, + lock_version: str, ) -> None: - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": "A", - "version": "1.0", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - } - ], - "metadata": { - "python-versions": "*", + lock_data = { + "package": [ + { + "name": "A", + "version": "1.0", + "optional": False, "platform": "*", - "content-hash": "123456789", - "files": {"A": []}, - }, - } - ) + "python-versions": "*", + "checksum": [], + } + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + "files": {"A": []}, + }, + } + fix_lock_data(lock_data) + locker.locked(True) + locker.mock_lock_data(lock_data) package_a = get_package("A", "1.0") package_a_new = get_package("A", "1.1") package_b = get_package("B", "1.1") @@ -765,42 +813,44 @@ def test_run_whitelist_add( assert locker.written_data == expected +@pytest.mark.parametrize("lock_version", ("1.1", "2.1")) def test_run_whitelist_remove( installer: Installer, locker: Locker, repo: Repository, package: ProjectPackage, installed: CustomInstalledRepository, + lock_version: str, ) -> None: - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": "A", - "version": "1.0", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - { - "name": "B", - "version": "1.1", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - ], - "metadata": { + lock_data = { + "package": [ + { + "name": "A", + "version": "1.0", + "optional": False, + "platform": "*", "python-versions": "*", + "checksum": [], + }, + { + "name": "B", + "version": "1.1", + "optional": False, "platform": "*", - "content-hash": "123456789", - "files": {"A": [], "B": []}, + "python-versions": "*", + "checksum": [], }, - } - ) + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + "files": {"A": [], "B": []}, + }, + } + fix_lock_data(lock_data) + locker.locked(True) + locker.mock_lock_data(lock_data) package_a = get_package("A", "1.0") package_b = get_package("B", "1.1") repo.add_package(package_a) @@ -1323,30 +1373,35 @@ def test_run_installs_with_local_setuptools_directory( assert installer.executor.installations_count == 3 +@pytest.mark.parametrize("lock_version", ("1.1", "2.1")) def test_run_with_prereleases( - installer: Installer, locker: Locker, repo: Repository, package: ProjectPackage + installer: Installer, + locker: Locker, + repo: Repository, + package: ProjectPackage, + lock_version: str, ) -> None: - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": "A", - "version": "1.0a2", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - } - ], - "metadata": { - "python-versions": "*", + lock_data = { + "package": [ + { + "name": "A", + "version": "1.0a2", + "optional": False, "platform": "*", - "content-hash": "123456789", - "files": {"A": []}, - }, - } - ) + "python-versions": "*", + "checksum": [], + } + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + "files": {"A": []}, + }, + } + fix_lock_data(lock_data) + locker.locked(True) + locker.mock_lock_data(lock_data) package_a = get_package("A", "1.0a2") package_b = get_package("B", "1.1") repo.add_package(package_a) @@ -1367,30 +1422,35 @@ def test_run_with_prereleases( assert locker.written_data == expected +@pytest.mark.parametrize("lock_version", ("1.1", "2.1")) def test_run_update_all_with_lock( - installer: Installer, locker: Locker, repo: Repository, package: ProjectPackage + installer: Installer, + locker: Locker, + repo: Repository, + package: ProjectPackage, + lock_version: str, ) -> None: - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": "A", - "version": "1.0", - "optional": True, - "platform": "*", - "python-versions": "*", - "checksum": [], - } - ], - "metadata": { - "python-versions": "*", + lock_data = { + "package": [ + { + "name": "A", + "version": "1.0", + "optional": True, "platform": "*", - "content-hash": "123456789", - "files": {"A": []}, - }, - } - ) + "python-versions": "*", + "checksum": [], + } + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + "files": {"A": []}, + }, + } + fix_lock_data(lock_data) + locker.locked(True) + locker.mock_lock_data(lock_data) package_a = get_package("A", "1.1") repo.add_package(get_package("A", "1.0")) repo.add_package(package_a) @@ -1406,48 +1466,53 @@ def test_run_update_all_with_lock( assert locker.written_data == expected +@pytest.mark.parametrize("lock_version", ("1.1", "2.1")) def test_run_update_with_locked_extras( - installer: Installer, locker: Locker, repo: Repository, package: ProjectPackage + installer: Installer, + locker: Locker, + repo: Repository, + package: ProjectPackage, + lock_version: str, ) -> None: - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": "A", - "version": "1.0", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - "dependencies": {"B": "^1.0", "C": "^1.0"}, - }, - { - "name": "B", - "version": "1.0", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - { - "name": "C", - "version": "1.1", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - "requirements": {"python": "~2.7"}, - }, - ], - "metadata": { + lock_data = { + "package": [ + { + "name": "A", + "version": "1.0", + "optional": False, + "platform": "*", "python-versions": "*", + "checksum": [], + "dependencies": {"B": "^1.0", "C": "^1.0"}, + }, + { + "name": "B", + "version": "1.0", + "optional": False, "platform": "*", - "content-hash": "123456789", - "files": {"A": [], "B": [], "C": []}, + "python-versions": "*", + "checksum": [], }, - } - ) + { + "name": "C", + "version": "1.1", + "optional": False, + "platform": "*", + "python-versions": "*", + "checksum": [], + "requirements": {"python": "~2.7"}, + }, + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + "files": {"A": [], "B": [], "C": []}, + }, + } + fix_lock_data(lock_data) + locker.locked(True) + locker.mock_lock_data(lock_data) package_a = get_package("A", "1.0") package_a.extras = {canonicalize_name("foo"): [get_dependency("B")]} b_dependency = get_dependency("B", "^1.0", optional=True) @@ -1522,72 +1587,78 @@ def test_run_install_duplicate_dependencies_different_constraints( assert installer.executor.removals_count == 0 +@pytest.mark.parametrize("lock_version", ("1.1", "2.1")) def test_run_install_duplicate_dependencies_different_constraints_with_lock( - installer: Installer, locker: Locker, repo: Repository, package: ProjectPackage + installer: Installer, + locker: Locker, + repo: Repository, + package: ProjectPackage, + lock_version: str, ) -> None: - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": "A", - "version": "1.0", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - "dependencies": { - "B": [ - {"version": "^1.0", "python": "<4.0"}, - {"version": "^2.0", "python": ">=4.0"}, - ] - }, - }, - { - "name": "B", - "version": "1.0", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - "dependencies": {"C": "1.2"}, - "requirements": {"python": "<4.0"}, - }, - { - "name": "B", - "version": "2.0", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - "dependencies": {"C": "1.5"}, - "requirements": {"python": ">=4.0"}, - }, - { - "name": "C", - "version": "1.2", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - { - "name": "C", - "version": "1.5", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], + lock_data = { + "package": [ + { + "name": "A", + "version": "1.0", + "optional": False, + "platform": "*", + "python-versions": "*", + "checksum": [], + "dependencies": { + "B": [ + {"version": "^1.0", "python": "<4.0"}, + {"version": "^2.0", "python": ">=4.0"}, + ] }, - ], - "metadata": { + }, + { + "name": "B", + "version": "1.0", + "optional": False, + "platform": "*", "python-versions": "*", + "checksum": [], + "dependencies": {"C": "1.2"}, + "requirements": {"python": "<4.0"}, + }, + { + "name": "B", + "version": "2.0", + "optional": False, "platform": "*", - "content-hash": "123456789", - "files": {"A": [], "B": [], "C": []}, + "python-versions": "*", + "checksum": [], + "dependencies": {"C": "1.5"}, + "requirements": {"python": ">=4.0"}, }, - } - ) + { + "name": "C", + "version": "1.2", + "optional": False, + "platform": "*", + "python-versions": "*", + "checksum": [], + }, + { + "name": "C", + "version": "1.5", + "optional": False, + "platform": "*", + "python-versions": "*", + "checksum": [], + }, + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + "files": {"A": [], "B": [], "C": []}, + }, + } + fix_lock_data(lock_data) + locker.locked(True) + locker.mock_lock_data(lock_data) + package.add_dependency(Factory.create_dependency("A", "*")) package_a = get_package("A", "1.0") @@ -1625,43 +1696,45 @@ def test_run_install_duplicate_dependencies_different_constraints_with_lock( assert installer.executor.removals_count == 0 +@pytest.mark.parametrize("lock_version", ("1.1", "2.1")) def test_run_update_uninstalls_after_removal_transitive_dependency( installer: Installer, locker: Locker, repo: Repository, package: ProjectPackage, installed: CustomInstalledRepository, + lock_version: str, ) -> None: - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": "A", - "version": "1.0", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - "dependencies": {"B": {"version": "^1.0", "python": "<2.0"}}, - }, - { - "name": "B", - "version": "1.0", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - ], - "metadata": { + lock_data = { + "package": [ + { + "name": "A", + "version": "1.0", + "optional": False, + "platform": "*", "python-versions": "*", + "checksum": [], + "dependencies": {"B": {"version": "^1.0", "python": "<2.0"}}, + }, + { + "name": "B", + "version": "1.0", + "optional": False, "platform": "*", - "content-hash": "123456789", - "files": {"A": [], "B": []}, + "python-versions": "*", + "checksum": [], }, - } - ) + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + "files": {"A": [], "B": []}, + }, + } + fix_lock_data(lock_data) + locker.locked(True) + locker.mock_lock_data(lock_data) package.add_dependency(Factory.create_dependency("A", "*")) package_a = get_package("A", "1.0") @@ -1686,76 +1759,78 @@ def test_run_update_uninstalls_after_removal_transitive_dependency( assert installer.executor.removals_count == 1 +@pytest.mark.parametrize("lock_version", ("1.1", "2.1")) def test_run_install_duplicate_dependencies_different_constraints_with_lock_update( installer: Installer, locker: Locker, repo: Repository, package: ProjectPackage, installed: CustomInstalledRepository, + lock_version: str, ) -> None: - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": "A", - "version": "1.0", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - "dependencies": { - "B": [ - {"version": "^1.0", "python": "<2.7"}, - {"version": "^2.0", "python": ">=2.7"}, - ] - }, - }, - { - "name": "B", - "version": "1.0", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - "dependencies": {"C": "1.2"}, - "requirements": {"python": "<2.7"}, - }, - { - "name": "B", - "version": "2.0", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - "dependencies": {"C": "1.5"}, - "requirements": {"python": ">=2.7"}, - }, - { - "name": "C", - "version": "1.2", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - }, - { - "name": "C", - "version": "1.5", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], + lock_data = { + "package": [ + { + "name": "A", + "version": "1.0", + "optional": False, + "platform": "*", + "python-versions": "*", + "checksum": [], + "dependencies": { + "B": [ + {"version": "^1.0", "python": "<2.7"}, + {"version": "^2.0", "python": ">=2.7"}, + ] }, - ], - "metadata": { + }, + { + "name": "B", + "version": "1.0", + "optional": False, + "platform": "*", "python-versions": "*", + "checksum": [], + "dependencies": {"C": "1.2"}, + "requirements": {"python": "<2.7"}, + }, + { + "name": "B", + "version": "2.0", + "optional": False, "platform": "*", - "content-hash": "123456789", - "files": {"A": [], "B": [], "C": []}, + "python-versions": "*", + "checksum": [], + "dependencies": {"C": "1.5"}, + "requirements": {"python": ">=2.7"}, }, - } - ) + { + "name": "C", + "version": "1.2", + "optional": False, + "platform": "*", + "python-versions": "*", + "checksum": [], + }, + { + "name": "C", + "version": "1.5", + "optional": False, + "platform": "*", + "python-versions": "*", + "checksum": [], + }, + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + "files": {"A": [], "B": [], "C": []}, + }, + } + fix_lock_data(lock_data) + locker.locked(True) + locker.mock_lock_data(lock_data) package.add_dependency(Factory.create_dependency("A", "*")) package_a = get_package("A", "1.1") @@ -2014,40 +2089,45 @@ def test_installer_required_extras_should_be_installed( assert installer.executor.removals_count == 0 +@pytest.mark.parametrize("lock_version", ("1.1", "2.1")) def test_update_multiple_times_with_split_dependencies_is_idempotent( - installer: Installer, locker: Locker, repo: Repository, package: ProjectPackage + installer: Installer, + locker: Locker, + repo: Repository, + package: ProjectPackage, + lock_version: str, ) -> None: - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": "A", - "version": "1.0", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - "dependencies": {"B": ">=1.0"}, - }, - { - "name": "B", - "version": "1.0.1", - "optional": False, - "platform": "*", - "python-versions": ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*", - "checksum": [], - "dependencies": {}, - }, - ], - "metadata": { + lock_data = { + "package": [ + { + "name": "A", + "version": "1.0", + "optional": False, + "platform": "*", "python-versions": "*", + "checksum": [], + "dependencies": {"B": ">=1.0"}, + }, + { + "name": "B", + "version": "1.0.1", + "optional": False, "platform": "*", - "content-hash": "123456789", - "files": {"A": [], "B": []}, + "python-versions": ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*", + "checksum": [], + "dependencies": {}, }, - } - ) + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + "files": {"A": [], "B": []}, + }, + } + fix_lock_data(lock_data) + locker.locked(True) + locker.mock_lock_data(lock_data) package.python_versions = "~2.7 || ^3.4" package.add_dependency(Factory.create_dependency("A", "^1.0")) @@ -2286,46 +2366,52 @@ def test_run_with_dependencies_quiet( assert output != "" +@pytest.mark.parametrize("lock_version", ("1.1", "2.1")) def test_installer_should_use_the_locked_version_of_git_dependencies( - installer: Installer, locker: Locker, package: ProjectPackage, repo: Repository + installer: Installer, + locker: Locker, + package: ProjectPackage, + repo: Repository, + lock_version: str, ) -> None: - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": "demo", - "version": "0.1.1", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - "dependencies": {"pendulum": ">=1.4.4"}, - "source": { - "type": "git", - "url": "https://github.com/demo/demo.git", - "reference": "master", - "resolved_reference": "123456", - }, - }, - { - "name": "pendulum", - "version": "1.4.4", - "optional": False, - "platform": "*", - "python-versions": "*", - "checksum": [], - "dependencies": {}, - }, - ], - "metadata": { + lock_data = { + "package": [ + { + "name": "demo", + "version": "0.1.1", + "optional": False, + "platform": "*", "python-versions": "*", + "checksum": [], + "dependencies": {"pendulum": ">=1.4.4"}, + "source": { + "type": "git", + "url": "https://github.com/demo/demo.git", + "reference": "master", + "resolved_reference": "123456", + }, + }, + { + "name": "pendulum", + "version": "1.4.4", + "optional": False, "platform": "*", - "content-hash": "123456789", - "files": {"demo": [], "pendulum": []}, + "python-versions": "*", + "checksum": [], + "dependencies": {}, }, - } - ) + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "platform": "*", + "content-hash": "123456789", + "files": {"demo": [], "pendulum": []}, + }, + } + fix_lock_data(lock_data) + locker.locked(True) + locker.mock_lock_data(lock_data) package.add_dependency( Factory.create_dependency( @@ -2339,7 +2425,12 @@ def test_installer_should_use_the_locked_version_of_git_dependencies( assert result == 0 assert isinstance(installer.executor, Executor) - assert installer.executor.installations[-1] == Package( + demo_installation = next( + package + for package in installer.executor.installations + if package.name == "demo" + ) + assert demo_installation == Package( "demo", "0.1.1", source_type="git", @@ -2383,7 +2474,12 @@ def test_installer_should_use_the_locked_version_of_git_dependencies_with_extras assert isinstance(installer.executor, Executor) assert len(installer.executor.installations) == 3 - assert installer.executor.installations[-1] == Package( + demo_installation = next( + package + for package in installer.executor.installations + if package.name == "demo" + ) + assert demo_installation == Package( "demo", "0.1.2", source_type="git", @@ -2423,7 +2519,12 @@ def test_installer_should_use_the_locked_version_of_git_dependencies_without_ref assert isinstance(installer.executor, Executor) assert len(installer.executor.installations) == 2 - assert installer.executor.installations[-1] == Package( + demo_installation = next( + package + for package in installer.executor.installations + if package.name == "demo" + ) + assert demo_installation == Package( "demo", "0.1.2", source_type="git", @@ -2433,6 +2534,7 @@ def test_installer_should_use_the_locked_version_of_git_dependencies_without_ref ) +@pytest.mark.parametrize("lock_version", ("2.0", "2.1")) @pytest.mark.parametrize("env_platform", ["darwin", "linux"]) def test_installer_distinguishes_locked_packages_with_local_version_by_source( pool: RepositoryPool, @@ -2442,6 +2544,7 @@ def test_installer_distinguishes_locked_packages_with_local_version_by_source( repo: Repository, package: ProjectPackage, env_platform: str, + lock_version: str, ) -> None: """https://github.com/python-poetry/poetry/issues/6710""" # Require 1.11.0+cpu from pytorch for most platforms, but specify 1.11.0 and pypi on @@ -2468,37 +2571,41 @@ def test_installer_distinguishes_locked_packages_with_local_version_by_source( ) # Locking finds both the pypi and the pytorch packages. - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": "torch", - "version": "1.11.0", - "optional": False, - "files": [], - "python-versions": "*", - }, - { - "name": "torch", - "version": "1.11.0+cpu", - "optional": False, - "files": [], - "python-versions": "*", - "source": { - "type": "legacy", - "url": "https://download.pytorch.org/whl", - "reference": "pytorch", - }, - }, - ], - "metadata": { + lock_data: dict[str, Any] = { + "package": [ + { + "name": "torch", + "version": "1.11.0", + "optional": False, + "files": [], "python-versions": "*", - "platform": "*", - "content-hash": "123456789", }, - } - ) + { + "name": "torch", + "version": "1.11.0+cpu", + "optional": False, + "files": [], + "python-versions": "*", + "source": { + "type": "legacy", + "url": "https://download.pytorch.org/whl", + "reference": "pytorch", + }, + }, + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + }, + } + if lock_version == "2.1": + lock_data["package"][0]["groups"] = ["main"] + lock_data["package"][0]["markers"] = "sys_platform == 'darwin'" + lock_data["package"][1]["groups"] = ["main"] + lock_data["package"][1]["markers"] = "sys_platform != 'darwin'" + locker.locked(True) + locker.mock_lock_data(lock_data) installer = Installer( NullIO(), MockEnv(platform=env_platform), @@ -2536,6 +2643,7 @@ def test_installer_distinguishes_locked_packages_with_local_version_by_source( ) +@pytest.mark.parametrize("lock_version", ("2.0", "2.1")) @pytest.mark.parametrize("env_platform_machine", ["aarch64", "amd64"]) def test_installer_distinguishes_locked_packages_with_same_version_by_source( pool: RepositoryPool, @@ -2545,6 +2653,7 @@ def test_installer_distinguishes_locked_packages_with_same_version_by_source( repo: Repository, package: ProjectPackage, env_platform_machine: str, + lock_version: str, ) -> None: """https://github.com/python-poetry/poetry/issues/8303""" package.add_dependency( @@ -2569,37 +2678,41 @@ def test_installer_distinguishes_locked_packages_with_same_version_by_source( ) # Locking finds both the pypi and the pyhweels packages. - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": "kivy", - "version": "2.2.1", - "optional": False, - "files": [], - "python-versions": "*", - }, - { - "name": "kivy", - "version": "2.2.1", - "optional": False, - "files": [], - "python-versions": "*", - "source": { - "type": "legacy", - "url": "https://www.piwheels.org/simple", - "reference": "pywheels", - }, - }, - ], - "metadata": { + lock_data: dict[str, Any] = { + "package": [ + { + "name": "kivy", + "version": "2.2.1", + "optional": False, + "files": [], "python-versions": "*", - "platform": "*", - "content-hash": "123456789", }, - } - ) + { + "name": "kivy", + "version": "2.2.1", + "optional": False, + "files": [], + "python-versions": "*", + "source": { + "type": "legacy", + "url": "https://www.piwheels.org/simple", + "reference": "pywheels", + }, + }, + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + }, + } + if lock_version == "2.1": + lock_data["package"][0]["groups"] = ["main"] + lock_data["package"][0]["markers"] = "platform_machine != 'aarch64'" + lock_data["package"][1]["groups"] = ["main"] + lock_data["package"][1]["markers"] = "platform_machine == 'aarch64'" + locker.locked(True) + locker.mock_lock_data(lock_data) installer = Installer( NullIO(), MockEnv(platform_machine=env_platform_machine), @@ -2640,6 +2753,7 @@ def test_installer_distinguishes_locked_packages_with_same_version_by_source( ) +@pytest.mark.parametrize("lock_version", ("2.0", "2.1")) @pytest.mark.parametrize("env_platform", ["darwin", "linux"]) def test_explicit_source_dependency_with_direct_origin_dependency( pool: RepositoryPool, @@ -2649,6 +2763,7 @@ def test_explicit_source_dependency_with_direct_origin_dependency( repo: Repository, package: ProjectPackage, env_platform: str, + lock_version: str, ) -> None: """ A dependency with explicit source should not be satisfied by @@ -2681,49 +2796,54 @@ def test_explicit_source_dependency_with_direct_origin_dependency( repo.add_package(get_package("demo", "0.1.0")) # Locking finds both the direct origin and the explicit source packages. - locker.locked(True) - locker.mock_lock_data( - { - "package": [ - { - "name": "demo", - "version": "0.1.0", - "optional": False, - "files": [], - "python-versions": "*", - "dependencies": {"pendulum": ">=1.4.4"}, - "source": { - "type": "url", - "url": demo_url, - }, - }, - { - "name": "demo", - "version": "0.1.0", - "optional": False, - "files": [], - "python-versions": "*", - "source": { - "type": "legacy", - "url": "https://www.demo.org/simple", - "reference": "repo", - }, + lock_data: dict[str, Any] = { + "package": [ + { + "name": "demo", + "version": "0.1.0", + "optional": False, + "files": [], + "python-versions": "*", + "dependencies": {"pendulum": ">=1.4.4"}, + "source": { + "type": "url", + "url": demo_url, }, - { - "name": "pendulum", - "version": "1.4.4", - "optional": False, - "files": [], - "python-versions": "*", + }, + { + "name": "demo", + "version": "0.1.0", + "optional": False, + "files": [], + "python-versions": "*", + "source": { + "type": "legacy", + "url": "https://www.demo.org/simple", + "reference": "repo", }, - ], - "metadata": { + }, + { + "name": "pendulum", + "version": "1.4.4", + "optional": False, + "files": [], "python-versions": "*", - "platform": "*", - "content-hash": "123456789", }, - } - ) + ], + "metadata": { + "lock-version": lock_version, + "python-versions": "*", + "content-hash": "123456789", + }, + } + if lock_version == "2.1": + for locked_package in lock_data["package"]: + locked_package["groups"] = ["main"] + lock_data["package"][0]["markers"] = "sys_platform != 'darwin'" + lock_data["package"][1]["markers"] = "sys_platform == 'darwin'" + lock_data["package"][2]["markers"] = "sys_platform != 'darwin'" + locker.locked(True) + locker.mock_lock_data(lock_data) installer = Installer( NullIO(), MockEnv(platform=env_platform), @@ -2745,7 +2865,7 @@ def test_explicit_source_dependency_with_direct_origin_dependency( assert result == 0 assert isinstance(installer.executor, Executor) if env_platform == "linux": - assert installer.executor.installations == [ + assert set(installer.executor.installations) == { Package("pendulum", "1.4.4"), Package( "demo", @@ -2753,7 +2873,7 @@ def test_explicit_source_dependency_with_direct_origin_dependency( source_type="url", source_url=demo_url, ), - ] + } else: assert installer.executor.installations == [ Package( diff --git a/tests/packages/test_locker.py b/tests/packages/test_locker.py index 9f64ccbdf80..f3d3876cc08 100644 --- a/tests/packages/test_locker.py +++ b/tests/packages/test_locker.py @@ -54,6 +54,42 @@ def transitive_info() -> TransitivePackageInfo: return TransitivePackageInfo(0, {MAIN_GROUP}, {}) +@pytest.mark.parametrize("is_locked", [True, False]) +def test_is_locked(locker: Locker, root: ProjectPackage, is_locked: bool) -> None: + if is_locked: + locker.set_lock_data(root, {}) + assert locker.is_locked() is is_locked + + +@pytest.mark.parametrize("is_fresh", [True, False]) +def test_is_fresh( + locker: Locker, + root: ProjectPackage, + transitive_info: TransitivePackageInfo, + is_fresh: bool, +) -> None: + locker.set_lock_data(root, {}) + if not is_fresh: + locker.set_pyproject_data( + {"tool": {"poetry": {"dependencies": {"tomli": "*"}}}} + ) + assert locker.is_fresh() is is_fresh + + +@pytest.mark.parametrize("lock_version", [None, "2.0", "2.1"]) +def test_is_locked_group_and_markers( + locker: Locker, root: ProjectPackage, lock_version: str | None +) -> None: + if lock_version: + locker.set_lock_data(root, {}) + with locker.lock.open("r", encoding="utf-8") as f: + content = f.read() + content = content.replace(locker._VERSION, lock_version) + with locker.lock.open("w", encoding="utf-8") as f: + f.write(content) + assert locker.is_locked_groups_and_markers() is (lock_version == "2.1") + + def test_lock_file_data_is_ordered( locker: Locker, root: ProjectPackage, transitive_info: TransitivePackageInfo ) -> None: @@ -427,6 +463,64 @@ def test_locker_properly_loads_subdir(locker: Locker) -> None: assert package.source_subdirectory == "subdir" +@pytest.mark.parametrize( + ("groups", "marker", "expected"), + [ + (["main"], None, {"main": "*"}), + ( + ["main"], + repr('python_version == "3.9"'), + {"main": 'python_version == "3.9"'}, + ), + ( + ["main", "dev"], + repr('python_version == "3.9"'), + {"main": 'python_version == "3.9"', "dev": 'python_version == "3.9"'}, + ), + ( + ["main", "dev"], + ( + '{"main" = \'python_version == "3.9"\',' + ' "dev" = \'sys_platform == "linux"\'}' + ), + {"main": 'python_version == "3.9"', "dev": 'sys_platform == "linux"'}, + ), + ( + ["main", "dev"], + '{"main" = \'python_version == "3.9"\'}', + {"main": 'python_version == "3.9"', "dev": "*"}, + ), + ], +) +def test_locker_properly_loads_groups_and_markers( + locker: Locker, groups: list[str], marker: str, expected: dict[str, str] +) -> None: + content = rf""" +[[package]] +name = "a" +version = "1.0" +optional = false +python-versions = "*" +groups = {groups} +{"markers = " + marker if marker else ""} +files = [] + +[metadata] +lock-version = "2.1" +python-versions = "*" +content-hash = "115cf985d932e9bf5f540555bbdd75decbb62cac81e399375fc19f6277f8c1d8" +""" + with open(locker.lock, "w", encoding="utf-8") as f: + f.write(content) + + packages = locker.locked_packages() + + a = get_package("a", "1.0") + assert len(packages) == 1 + assert packages[a].groups == set(groups) + assert packages[a].markers == {g: parse_marker(m) for g, m in expected.items()} + + def test_locker_properly_assigns_metadata_files(locker: Locker) -> None: """ For multiple constraints dependencies, there is only one common entry in diff --git a/tests/packages/test_transitive_package_info.py b/tests/packages/test_transitive_package_info.py new file mode 100644 index 00000000000..8532555ba98 --- /dev/null +++ b/tests/packages/test_transitive_package_info.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import pytest + +from poetry.core.version.markers import parse_marker + +from poetry.packages.transitive_package_info import TransitivePackageInfo + + +@pytest.mark.parametrize( + "groups, expected", + [ + ([], ""), + (["main"], 'sys_platform == "linux"'), + (["dev"], 'python_version < "3.9"'), + (["main", "dev"], 'sys_platform == "linux" or python_version < "3.9"'), + (["foo"], ""), + (["main", "foo", "dev"], 'sys_platform == "linux" or python_version < "3.9"'), + ], +) +def test_get_marker(groups: list[str], expected: str) -> None: + info = TransitivePackageInfo( + depth=0, + groups={"main", "dev"}, + markers={ + "main": parse_marker('sys_platform =="linux"'), + "dev": parse_marker('python_version < "3.9"'), + }, + ) + assert str(info.get_marker(groups)) == expected diff --git a/tests/puzzle/test_transaction.py b/tests/puzzle/test_transaction.py index 78640a54813..a8874ebe4eb 100644 --- a/tests/puzzle/test_transaction.py +++ b/tests/puzzle/test_transaction.py @@ -3,7 +3,14 @@ from typing import TYPE_CHECKING from typing import Any +import pytest + +from packaging.utils import canonicalize_name +from poetry.core.packages.dependency import Dependency from poetry.core.packages.package import Package +from poetry.core.packages.project_package import ProjectPackage +from poetry.core.version.markers import AnyMarker +from poetry.core.version.markers import parse_marker from poetry.installation.operations.update import Update from poetry.packages.transitive_package_info import TransitivePackageInfo @@ -186,3 +193,182 @@ def test_it_should_update_installed_packages_if_sources_are_different() -> None: } ], ) + + +@pytest.mark.parametrize( + ("groups", "expected"), + [ + (set(), []), + ({"main"}, ["a", "c"]), + ({"dev"}, ["b", "c"]), + ({"main", "dev"}, ["a", "b", "c"]), + ], +) +@pytest.mark.parametrize("installed", [False, True]) +@pytest.mark.parametrize("sync", [False, True]) +def test_calculate_operations_with_groups( + installed: bool, sync: bool, groups: set[str], expected: list[str] +) -> None: + transaction = Transaction( + [Package("a", "1"), Package("b", "1"), Package("c", "1")], + { + Package("a", "1"): TransitivePackageInfo( + 0, {"main"}, {"main": AnyMarker()} + ), + Package("b", "1"): TransitivePackageInfo(0, {"dev"}, {"dev": AnyMarker()}), + Package("c", "1"): TransitivePackageInfo( + 0, {"main", "dev"}, {"main": AnyMarker(), "dev": AnyMarker()} + ), + }, + [Package("a", "1"), Package("b", "1"), Package("c", "1")] if installed else [], + None, + {"python_version": "3.8"}, + groups, + ) + + expected_ops = [ + {"job": "install", "package": Package(name, "1")} for name in expected + ] + if installed: + for op in expected_ops: + op["skipped"] = True + if sync: + for name in sorted({"a", "b", "c"}.difference(expected), reverse=True): + expected_ops.insert(0, {"job": "remove", "package": Package(name, "1")}) + + check_operations(transaction.calculate_operations(sync), expected_ops) + + +@pytest.mark.parametrize( + ("python_version", "expected"), [("3.8", ["a"]), ("3.9", ["b"])] +) +@pytest.mark.parametrize("installed", [False, True]) +@pytest.mark.parametrize("sync", [False, True]) +def test_calculate_operations_with_markers( + installed: bool, sync: bool, python_version: str, expected: list[str] +) -> None: + transaction = Transaction( + [Package("a", "1"), Package("b", "1")], + { + Package("a", "1"): TransitivePackageInfo( + 0, {"main"}, {"main": parse_marker("python_version < '3.9'")} + ), + Package("b", "1"): TransitivePackageInfo( + 0, {"main"}, {"main": parse_marker("python_version >= '3.9'")} + ), + }, + [Package("a", "1"), Package("b", "1")] if installed else [], + None, + {"python_version": python_version}, + {"main"}, + ) + + expected_ops = [ + {"job": "install", "package": Package(name, "1")} for name in expected + ] + if installed: + for op in expected_ops: + op["skipped"] = True + if sync: + for name in sorted({"a", "b"}.difference(expected), reverse=True): + expected_ops.insert(0, {"job": "remove", "package": Package(name, "1")}) + + check_operations(transaction.calculate_operations(sync), expected_ops) + + +@pytest.mark.parametrize( + ("python_version", "sys_platform", "groups", "expected"), + [ + ("3.8", "win32", {"main"}, True), + ("3.9", "linux", {"main"}, False), + ("3.9", "linux", {"dev"}, True), + ("3.8", "win32", {"dev"}, False), + ("3.9", "linux", {"main", "dev"}, True), + ("3.8", "win32", {"main", "dev"}, True), + ("3.8", "linux", {"main", "dev"}, True), + ("3.9", "win32", {"main", "dev"}, False), + ], +) +def test_calculate_operations_with_groups_and_markers( + python_version: str, + sys_platform: str, + groups: set[str], + expected: bool, +) -> None: + transaction = Transaction( + [Package("a", "1")], + { + Package("a", "1"): TransitivePackageInfo( + 0, + {"main", "dev"}, + { + "main": parse_marker("python_version < '3.9'"), + "dev": parse_marker("sys_platform == 'linux'"), + }, + ), + }, + [], + None, + {"python_version": python_version, "sys_platform": sys_platform}, + groups, + ) + + expected_ops = ( + [{"job": "install", "package": Package("a", "1")}] if expected else [] + ) + + check_operations(transaction.calculate_operations(), expected_ops) + + +@pytest.mark.parametrize("extras", [False, True]) +@pytest.mark.parametrize("marker_env", [False, True]) +@pytest.mark.parametrize("installed", [False, True]) +@pytest.mark.parametrize("with_uninstalls", [False, True]) +@pytest.mark.parametrize("sync", [False, True]) +def test_calculate_operations_extras( + extras: bool, + marker_env: bool, + installed: bool, + with_uninstalls: bool, + sync: bool, +) -> None: + extra_name = canonicalize_name("foo") + package = ProjectPackage("root", "1.0") + dep_a = Dependency("a", "1", optional=True) + dep_a._in_extras = [extra_name] + package.add_dependency(dep_a) + package.extras = {extra_name: [dep_a]} + opt_a = Package("a", "1") + opt_a.optional = True + + transaction = Transaction( + [Package("a", "1")], + { + opt_a: TransitivePackageInfo( + 0, + {"main"} if marker_env else set(), + {"main": parse_marker("extra == 'foo'")} if marker_env else {}, + ) + }, + [Package("a", "1")] if installed else [], + package, + {"python_version": "3.8"} if marker_env else None, + {"main"} if marker_env else None, + ) + + if extras: + ops = [{"job": "install", "package": Package("a", "1"), "skipped": installed}] + elif installed: + # extras are always removed, even if with_uninstalls is False + ops = [{"job": "remove", "package": Package("a", "1")}] + else: + ops = [{"job": "install", "package": Package("a", "1"), "skipped": True}] + + check_operations( + transaction.calculate_operations( + with_uninstalls, + sync, + extras={extra_name} if extras else set(), + ), + ops, + )