Skip to content

Commit 1188b31

Browse files
abnfinswimmer
abn
authored and
finswimmer
committed
locker: propagate cumulative markers to nested deps
This change ensures that markers are propagated from top level dependencies to the deepest level by walking top to bottom instead of iterating over all available packages. In addition, we also compress any dependencies with the same name and constraint to provide a more concise representation. Resolves: #3112 #3160 (cherry picked from commit e78a67b)
1 parent 02307ba commit 1188b31

2 files changed

Lines changed: 199 additions & 27 deletions

File tree

‎poetry/packages/locker.py‎

Lines changed: 59 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -215,30 +215,62 @@ def __get_locked_package(
215215

216216
for dependency in project_requires:
217217
dependency = deepcopy(dependency)
218-
if pinned_versions:
219-
locked_package = __get_locked_package(dependency)
220-
if locked_package:
221-
dependency.set_constraint(locked_package.to_dependency().constraint)
218+
locked_package = __get_locked_package(dependency)
219+
if locked_package:
220+
locked_dependency = locked_package.to_dependency()
221+
locked_dependency.marker = dependency.marker.intersect(
222+
locked_package.marker
223+
)
224+
225+
if not pinned_versions:
226+
locked_dependency.set_constraint(dependency.constraint)
227+
228+
dependency = locked_dependency
229+
222230
project_level_dependencies.add(dependency.name)
223231
dependencies.append(dependency)
224232

225233
if not with_nested:
226234
# return only with project level dependencies
227235
return dependencies
228236

229-
nested_dependencies = list()
237+
nested_dependencies = dict()
230238

231-
for pkg in packages: # type: Package
232-
for requirement in pkg.requires: # type: Dependency
233-
if requirement.name in project_level_dependencies:
239+
def __walk_level(
240+
__dependencies, __level
241+
): # type: (List[Dependency], int) -> None
242+
if not __dependencies:
243+
return
244+
245+
__next_level = []
246+
247+
for requirement in __dependencies:
248+
__locked_package = __get_locked_package(requirement)
249+
250+
if __locked_package:
251+
for require in __locked_package.requires:
252+
if require.marker.is_empty():
253+
require.marker = requirement.marker
254+
else:
255+
require.marker = require.marker.intersect(
256+
requirement.marker
257+
)
258+
259+
require.marker = require.marker.intersect(
260+
__locked_package.marker
261+
)
262+
__next_level.append(require)
263+
264+
if requirement.name in project_level_dependencies and __level == 0:
234265
# project level dependencies take precedence
235266
continue
236267

237-
locked_package = __get_locked_package(requirement)
238-
if locked_package:
268+
if __locked_package:
239269
# create dependency from locked package to retain dependency metadata
240270
# if this is not done, we can end-up with incorrect nested dependencies
241-
requirement = locked_package.to_dependency()
271+
marker = requirement.marker
272+
requirement = __locked_package.to_dependency()
273+
requirement.marker = requirement.marker.intersect(marker)
242274
else:
243275
# we make a copy to avoid any side-effects
244276
requirement = deepcopy(requirement)
@@ -251,26 +283,26 @@ def __get_locked_package(
251283
)
252284

253285
# dependencies use extra to indicate that it was activated via parent
254-
# package's extras
255-
marker = requirement.marker.without_extras()
256-
for project_requirement in project_requires:
257-
if (
258-
pkg.name == project_requirement.name
259-
and project_requirement.constraint.allows(pkg.version)
260-
):
261-
requirement.marker = marker.intersect(
262-
project_requirement.marker
263-
)
264-
break
286+
# package's extras, this is not required for nested exports as we assume
287+
# the resolver already selected this dependency
288+
requirement.marker = requirement.marker.without_extras().intersect(
289+
pkg.marker
290+
)
291+
292+
key = (requirement.name, requirement.pretty_constraint)
293+
if key not in nested_dependencies:
294+
nested_dependencies[key] = requirement
265295
else:
266-
# this dependency was not from a project requirement
267-
requirement.marker = marker.intersect(pkg.marker)
296+
nested_dependencies[key].marker = nested_dependencies[
297+
key
298+
].marker.intersect(requirement.marker)
299+
300+
return __walk_level(__next_level, __level + 1)
268301

269-
if requirement not in nested_dependencies:
270-
nested_dependencies.append(requirement)
302+
__walk_level(dependencies, 0)
271303

272304
return sorted(
273-
itertools.chain(dependencies, nested_dependencies),
305+
itertools.chain(dependencies, nested_dependencies.values()),
274306
key=lambda x: x.name.lower(),
275307
)
276308

‎tests/utils/test_exporter.py‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import pytest
44

5+
from poetry.core.packages import dependency_from_pep_508
56
from poetry.core.toml.file import TOMLFile
67
from poetry.factory import Factory
78
from poetry.packages import Locker as BaseLocker
@@ -175,6 +176,145 @@ def test_exporter_can_export_requirements_txt_with_standard_packages_and_markers
175176
assert expected == content
176177

177178

179+
def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers(
180+
tmp_dir, poetry
181+
):
182+
poetry.locker.mock_lock_data(
183+
{
184+
"package": [
185+
{
186+
"name": "a",
187+
"version": "1.2.3",
188+
"category": "main",
189+
"optional": False,
190+
"python-versions": "*",
191+
"marker": "python_version < '3.7'",
192+
"dependencies": {"b": ">=0.0.0", "c": ">=0.0.0"},
193+
},
194+
{
195+
"name": "b",
196+
"version": "4.5.6",
197+
"category": "main",
198+
"optional": False,
199+
"python-versions": "*",
200+
"marker": "platform_system == 'Windows'",
201+
"dependencies": {"d": ">=0.0.0"},
202+
},
203+
{
204+
"name": "c",
205+
"version": "7.8.9",
206+
"category": "main",
207+
"optional": False,
208+
"python-versions": "*",
209+
"marker": "sys_platform == 'win32'",
210+
"dependencies": {"d": ">=0.0.0"},
211+
},
212+
{
213+
"name": "d",
214+
"version": "0.0.1",
215+
"category": "main",
216+
"optional": False,
217+
"python-versions": "*",
218+
},
219+
],
220+
"metadata": {
221+
"python-versions": "*",
222+
"content-hash": "123456789",
223+
"hashes": {"a": [], "b": [], "c": [], "d": []},
224+
},
225+
}
226+
)
227+
set_package_requires(poetry, skip={"b", "c", "d"})
228+
229+
exporter = Exporter(poetry)
230+
231+
exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt")
232+
233+
with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f:
234+
content = f.read()
235+
236+
expected = {
237+
"a": dependency_from_pep_508("a==1.2.3; python_version < '3.7'"),
238+
"b": dependency_from_pep_508(
239+
"b==4.5.6; platform_system == 'Windows' and python_version < '3.7'"
240+
),
241+
"c": dependency_from_pep_508(
242+
"c==7.8.9; sys_platform == 'win32' and python_version < '3.7'"
243+
),
244+
"d": dependency_from_pep_508(
245+
"d==0.0.1; python_version < '3.7' and platform_system == 'Windows' and sys_platform == 'win32'"
246+
),
247+
}
248+
249+
for line in content.strip().split("\n"):
250+
dependency = dependency_from_pep_508(line)
251+
assert dependency.name in expected
252+
expected_dependency = expected.pop(dependency.name)
253+
assert dependency == expected_dependency
254+
assert dependency.marker == expected_dependency.marker
255+
256+
assert expected == {}
257+
258+
259+
def test_exporter_can_export_requirements_txt_with_nested_packages_and_markers_any(
260+
tmp_dir, poetry
261+
):
262+
poetry.locker.mock_lock_data(
263+
{
264+
"package": [
265+
{
266+
"name": "a",
267+
"version": "1.2.3",
268+
"category": "main",
269+
"optional": False,
270+
"python-versions": "*",
271+
},
272+
{
273+
"name": "b",
274+
"version": "4.5.6",
275+
"category": "dev",
276+
"optional": False,
277+
"python-versions": "*",
278+
"dependencies": {"a": ">=1.2.3"},
279+
},
280+
],
281+
"metadata": {
282+
"python-versions": "*",
283+
"content-hash": "123456789",
284+
"hashes": {"a": [], "b": []},
285+
},
286+
}
287+
)
288+
289+
poetry.package.requires = [
290+
Factory.create_dependency(
291+
name="a", constraint=dict(version="^1.2.3", python="<3.8")
292+
),
293+
]
294+
poetry.package.dev_requires = [
295+
Factory.create_dependency(
296+
name="b", constraint=dict(version="^4.5.6"), category="dev"
297+
),
298+
Factory.create_dependency(name="a", constraint=dict(version="^1.2.3")),
299+
]
300+
301+
exporter = Exporter(poetry)
302+
303+
exporter.export("requirements.txt", Path(tmp_dir), "requirements.txt", dev=True)
304+
305+
with (Path(tmp_dir) / "requirements.txt").open(encoding="utf-8") as f:
306+
content = f.read()
307+
308+
assert (
309+
content
310+
== """\
311+
a==1.2.3
312+
a==1.2.3; python_version < "3.8"
313+
b==4.5.6
314+
"""
315+
)
316+
317+
178318
def test_exporter_can_export_requirements_txt_with_standard_packages_and_hashes(
179319
tmp_dir, poetry
180320
):

0 commit comments

Comments
 (0)