Skip to content

Commit

Permalink
locker: propagate cumulative markers to nested deps
Browse files Browse the repository at this point in the history 
This change ensures that markers are propagated from top level
dependencies to the deepest level by walking top to bottom instead of
iterating over all available packages.

In addition, we also compress any dependencies with the same name and
constraint to provide a more concise representation.

Resolves: #3112 #3160
  • Loading branch information
@abn
abn committed Oct 12, 2020
1 parent 2ca46d0 commit a63e461
Show file tree
Hide file tree
Showing 2 changed files with 199 additions and 27 deletions.
86 changes: 59 additions & 27 deletions poetry/packages/locker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,30 +215,62 @@ def __get_locked_package(

for dependency in project_requires:
dependency = deepcopy(dependency)
if pinned_versions:
locked_package = __get_locked_package(dependency)
if locked_package:
dependency.set_constraint(locked_package.to_dependency().constraint)
locked_package = __get_locked_package(dependency)
if locked_package:
locked_dependency = locked_package.to_dependency()
locked_dependency.marker = dependency.marker.intersect(
locked_package.marker
)

if not pinned_versions:
locked_dependency.set_constraint(dependency.constraint)

dependency = locked_dependency

project_level_dependencies.add(dependency.name)
dependencies.append(dependency)

if not with_nested:
# return only with project level dependencies
return dependencies

nested_dependencies = list()
nested_dependencies = dict()

for pkg in packages: # type: Package
for requirement in pkg.requires: # type: Dependency
if requirement.name in project_level_dependencies:
def __walk_level(
__dependencies, __level
): # type: (List[Dependency], int) -> None
if not __dependencies:
return

__next_level = []

for requirement in __dependencies:
__locked_package = __get_locked_package(requirement)

if __locked_package:
for require in __locked_package.requires:
if require.marker.is_empty():
require.marker = requirement.marker
else:
require.marker = require.marker.intersect(
requirement.marker
)

require.marker = require.marker.intersect(
__locked_package.marker
)
__next_level.append(require)

if requirement.name in project_level_dependencies and __level == 0:
# project level dependencies take precedence
continue

locked_package = __get_locked_package(requirement)
if locked_package:
if __locked_package:
# create dependency from locked package to retain dependency metadata
# if this is not done, we can end-up with incorrect nested dependencies
requirement = locked_package.to_dependency()
marker = requirement.marker
requirement = __locked_package.to_dependency()
requirement.marker = requirement.marker.intersect(marker)
else:
# we make a copy to avoid any side-effects
requirement = deepcopy(requirement)
Expand All @@ -251,26 +283,26 @@ def __get_locked_package(
)

# dependencies use extra to indicate that it was activated via parent
# package's extras
marker = requirement.marker.without_extras()
for project_requirement in project_requires:
if (
pkg.name == project_requirement.name
and project_requirement.constraint.allows(pkg.version)
):
requirement.marker = marker.intersect(
project_requirement.marker
)
break
# package's extras, this is not required for nested exports as we assume
# the resolver already selected this dependency
requirement.marker = requirement.marker.without_extras().intersect(
pkg.marker
)

key = (requirement.name, requirement.pretty_constraint)
if key not in nested_dependencies:
nested_dependencies[key] = requirement
else:
# this dependency was not from a project requirement
requirement.marker = marker.intersect(pkg.marker)
nested_dependencies[key].marker = nested_dependencies[
key
].marker.intersect(requirement.marker)

return __walk_level(__next_level, __level + 1)

if requirement not in nested_dependencies:
nested_dependencies.append(requirement)
__walk_level(dependencies, 0)

return sorted(
itertools.chain(dependencies, nested_dependencies),
itertools.chain(dependencies, nested_dependencies.values()),
key=lambda x: x.name.lower(),
)

Expand Down
140 changes: 140 additions & 0 deletions tests/utils/test_exporter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import pytest

from poetry.core.packages import dependency_from_pep_508
from poetry.core.toml.file import TOMLFile
from poetry.factory import Factory
from poetry.packages import Locker as BaseLocker
Expand Down Expand Up @@ -175,6 +176,145 @@ def test_exporter_can_export_requirements_txt_with_standard_packages_and_markers
assert expected == content


def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers(
tmp_dir, poetry
):
poetry.locker.mock_lock_data(
{
"package": [
{
"name": "a",
"version": "1.2.3",
"category": "main",
"optional": False,
"python-versions": "*",
"marker": "python_version < '3.7'",
"dependencies": {"b": ">=0.0.0", "c": ">=0.0.0"},
},
{
"name": "b",
"version": "4.5.6",
"category": "main",
"optional": False,
"python-versions": "*",
"marker": "platform_system == 'Windows'",
"dependencies": {"d": ">=0.0.0"},
},
{
"name": "c",
"version": "7.8.9",
"category": "main",
"optional": False,
"python-versions": "*",
"marker": "sys_platform == 'win32'",
"dependencies": {"d": ">=0.0.0"},
},
{
"name": "d",
"version": "0.0.1",
"category": "main",
"optional": False,
"python-versions": "*",
},
],
"metadata": {
"python-versions": "*",
"content-hash": "123456789",
"hashes": {"a": [], "b": [], "c": [], "d": []},
},
}
)
set_package_requires(poetry, skip={"b", "c", "d"})

exporter = Exporter(poetry)

exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt")

with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f:
content = f.read()

expected = {
"a": dependency_from_pep_508("a==1.2.3; python_version < '3.7'"),
"b": dependency_from_pep_508(
"b==4.5.6; platform_system == 'Windows' and python_version < '3.7'"
),
"c": dependency_from_pep_508(
"c==7.8.9; sys_platform == 'win32' and python_version < '3.7'"
),
"d": dependency_from_pep_508(
"d==0.0.1; python_version < '3.7' and platform_system == 'Windows' and sys_platform == 'win32'"
),
}

for line in content.strip().split("\n"):
dependency = dependency_from_pep_508(line)
assert dependency.name in expected
expected_dependency = expected.pop(dependency.name)
assert dependency == expected_dependency
assert dependency.marker == expected_dependency.marker

assert expected == {}


def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers_any(
tmp_dir, poetry
):
poetry.locker.mock_lock_data(
{
"package": [
{
"name": "a",
"version": "1.2.3",
"category": "main",
"optional": False,
"python-versions": "*",
},
{
"name": "b",
"version": "4.5.6",
"category": "dev",
"optional": False,
"python-versions": "*",
"dependencies": {"a": ">=1.2.3"},
},
],
"metadata": {
"python-versions": "*",
"content-hash": "123456789",
"hashes": {"a": [], "b": []},
},
}
)

poetry.package.requires = [
Factory.create_dependency(
name="a", constraint=dict(version="^1.2.3", python="<3.8")
),
]
poetry.package.dev_requires = [
Factory.create_dependency(
name="b", constraint=dict(version="^4.5.6"), category="dev"
),
Factory.create_dependency(name="a", constraint=dict(version="^1.2.3")),
]

exporter = Exporter(poetry)

exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt", dev=True)

with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f:
content = f.read()

assert (
content
== """\
a==1.2.3
a==1.2.3; python_version < "3.8"
b==4.5.6
"""
)


def test_exporter_can_export_requirements_txt_with_standard_packages_and_hashes(
tmp_dir, poetry
):
Expand Down

0 comments on commit a63e461

Please sign in to comment.