Skip to content

Commit 805dcff

Browse files
reksargaborbernatfrenzymadness
authored
Windows embedable support (#2353)
* Bump pip and setuptools (#2348) Signed-off-by: Bernát Gábor <bgabor8@bloomberg.net> * Use shlex.quote instead of deprecated pipes.quote (#2351) * Embeds the "python<VERSION>.zip" for Windows. For example, for Python 3.10 the embeddable file name would be "python310.zip". If this file would be found in `sys.path`, the virtualenv should copy it into the "<venv>\Scripts\python310.zip". * For Windows CPython3: *.dll/*.pyd -> to_bin * Fixture for a Python interpreter info. Helps to test virtualenv creator classes. * Creators tests: path_mock as separate module. * Clarifies tests, separates testing tools. * Tests for CPython3Windows sources. * Tests for the embedded Python std lib for Windows. * Add news entry. * Replaces `yield from` for backward compability. * FIX: Path mocking in pypy tests. * Wrap `sys` `Path` with `str` for importlib. The importlib accepts a Path-like objects from Python 3.6 * Makes PathMock ABC compatible with Python 2 * Does not collect tests for Python3 under Python 2 It is possible to make pass CPython3 tests under Python 2, but it's better to disable it instead of decreasing the readability and performance of Python 3 style. * Allows empty `Path()` in Windows with Python 2 * Allows to load fixture files with PY2 Windows Path * Skips one PY3 POSIX test in PY2 Windows Co-authored-by: Bernát Gábor <gaborjbernat@gmail.com> Co-authored-by: Lumír 'Frenzy' Balhar <lbalhar@redhat.com>
1 parent b01515b commit 805dcff

14 files changed

Lines changed: 438 additions & 102 deletions

File tree

‎docs/changelog/1774.feature.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Support for Windows embeddable Python package: includes ``python<VERSION>.zip``
2+
in the creator sources.

‎src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py‎

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
from __future__ import absolute_import, unicode_literals
22

33
import abc
4+
import fnmatch
5+
from itertools import chain
6+
from operator import methodcaller as method
47
from textwrap import dedent
58

69
from six import add_metaclass
@@ -53,11 +56,20 @@ def setup_meta(cls, interpreter):
5356

5457
@classmethod
5558
def sources(cls, interpreter):
56-
for src in super(CPython3Windows, cls).sources(interpreter):
57-
yield src
58-
if not cls.has_shim(interpreter):
59-
for src in cls.include_dll_and_pyd(interpreter):
60-
yield src
59+
if cls.has_shim(interpreter):
60+
refs = cls.executables(interpreter)
61+
else:
62+
refs = chain(
63+
cls.executables(interpreter),
64+
cls.dll_and_pyd(interpreter),
65+
cls.python_zip(interpreter),
66+
)
67+
for ref in refs:
68+
yield ref
69+
70+
@classmethod
71+
def executables(cls, interpreter):
72+
return super(CPython3Windows, cls).sources(interpreter)
6173

6274
@classmethod
6375
def has_shim(cls, interpreter):
@@ -79,13 +91,32 @@ def host_python(cls, interpreter):
7991
return super(CPython3Windows, cls).host_python(interpreter)
8092

8193
@classmethod
82-
def include_dll_and_pyd(cls, interpreter):
94+
def dll_and_pyd(cls, interpreter):
8395
dll_folder = Path(interpreter.system_prefix) / "DLLs"
8496
host_exe_folder = Path(interpreter.system_executable).parent
8597
for folder in [host_exe_folder, dll_folder]:
8698
for file in folder.iterdir():
8799
if file.suffix in (".pyd", ".dll"):
88-
yield PathRefToDest(file, dest=cls.to_dll_and_pyd)
100+
yield PathRefToDest(file, cls.to_bin)
89101

90-
def to_dll_and_pyd(self, src):
91-
return self.bin_dir / src.name
102+
@classmethod
103+
def python_zip(cls, interpreter):
104+
"""
105+
"python{VERSION}.zip" contains compiled *.pyc std lib packages, where
106+
"VERSION" is `py_version_nodot` var from the `sysconfig` module.
107+
:see: https://docs.python.org/3/using/windows.html#the-embeddable-package
108+
:see: `discovery.py_info.PythonInfo` class (interpreter).
109+
:see: `python -m sysconfig` output.
110+
111+
:note: The embeddable Python distribution for Windows includes
112+
"python{VERSION}.zip" and "python{VERSION}._pth" files. User can
113+
move/rename *zip* file and edit `sys.path` by editing *_pth* file.
114+
Here the `pattern` is used only for the default *zip* file name!
115+
"""
116+
pattern = "*python{}.zip".format(interpreter.version_nodot)
117+
matches = fnmatch.filter(interpreter.path, pattern)
118+
matched_paths = map(Path, matches)
119+
existing_paths = filter(method("exists"), matched_paths)
120+
path = next(existing_paths, None)
121+
if path is not None:
122+
yield PathRefToDest(path, cls.to_bin)

‎src/virtualenv/discovery/py_info.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,10 @@ def abs_path(v):
4747
self.version_info = VersionInfo(*list(u(i) for i in sys.version_info))
4848
self.architecture = 64 if sys.maxsize > 2**32 else 32
4949

50+
# Used to determine some file names.
51+
# See `CPython3Windows.python_zip()`.
52+
self.version_nodot = sysconfig.get_config_var("py_version_nodot")
53+
5054
self.version = u(sys.version)
5155
self.os = u(os.name)
5256

‎src/virtualenv/seed/wheels/embed/setuptools-62.3.2-py3-none-any.whl‎

1.17 MB
Binary file not shown.

‎src/virtualenv/util/path/_pathlib/via_os_path.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
from __future__ import absolute_import, unicode_literals
22

3+
import fnmatch
34
import os
45
import platform
56
from contextlib import contextmanager
@@ -10,7 +11,7 @@
1011

1112

1213
class Path(object):
13-
def __init__(self, path):
14+
def __init__(self, path=""):
1415
if isinstance(path, Path):
1516
_path = path._path
1617
else:
@@ -147,5 +148,13 @@ def chmod(self, mode):
147148
def absolute(self):
148149
return Path(os.path.abspath(self._path))
149150

151+
def rglob(self, pattern):
152+
"""
153+
Rough emulation of the origin method. Just for searching fixture files.
154+
"""
155+
for root, _dirs, files in os.walk(self._path):
156+
for filename in fnmatch.filter(files, pattern):
157+
yield Path(os.path.join(root, filename))
158+
150159

151160
__all__ = ("Path",)

‎tests/conftest.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from virtualenv.app_data import AppDataDiskFolder
1515
from virtualenv.discovery.builtin import get_interpreter
1616
from virtualenv.discovery.py_info import PythonInfo
17-
from virtualenv.info import IS_PYPY, IS_WIN, fs_supports_symlink
17+
from virtualenv.info import IS_PYPY, IS_WIN, PY2, fs_supports_symlink
1818
from virtualenv.report import LOGGER
1919
from virtualenv.util.path import Path
2020
from virtualenv.util.six import ensure_str, ensure_text
@@ -388,3 +388,11 @@ def skip_if_test_in_system(session_app_data):
388388
current = PythonInfo.current(session_app_data)
389389
if current.system_executable is not None:
390390
pytest.skip("test not valid if run under system")
391+
392+
393+
def pytest_ignore_collect(path):
394+
"""
395+
We can't just skip these tests due to syntax errors that occurs during
396+
collecting tests under a Python 2 host.
397+
"""
398+
return PY2 and str(path).endswith("test_cpython3_win.py")

‎tests/unit/create/via_global_ref/builtin/conftest.py‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import sys
2+
3+
import pytest
4+
from testing import path
5+
from testing.py_info import read_fixture
6+
7+
from virtualenv.util.path import Path
8+
9+
# Allows to import from `testing` into test submodules.
10+
sys.path.append(str(Path(__file__).parent))
11+
12+
13+
@pytest.fixture
14+
def py_info(py_info_name):
15+
return read_fixture(py_info_name)
16+
17+
18+
@pytest.fixture
19+
def mock_files(mocker):
20+
return lambda paths, files: path.mock_files(mocker, paths, files)
21+
22+
23+
@pytest.fixture
24+
def mock_pypy_libs(mocker):
25+
return lambda pypy, libs: path.mock_pypy_libs(mocker, pypy, libs)

‎tests/unit/create/via_global_ref/builtin/cpython/cpython3_win_embed.json‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
{
2+
"platform": "win32",
3+
"implementation": "CPython",
4+
"version_info": {
5+
"major": 3,
6+
"minor": 10,
7+
"micro": 4,
8+
"releaselevel": "final",
9+
"serial": 0
10+
},
11+
"architecture": 64,
12+
"version_nodot": "310",
13+
"version": "3.10.4 (tags/v3.10.4:9d38120, Mar 23 2022, 23:13:41) [MSC v.1929 64 bit (AMD64)]",
14+
"os": "nt",
15+
"prefix": "c:\\path\\to\\python",
16+
"base_prefix": "c:\\path\\to\\python",
17+
"real_prefix": null,
18+
"base_exec_prefix": "c:\\path\\to\\python",
19+
"exec_prefix": "c:\\path\\to\\python",
20+
"executable": "c:\\path\\to\\python\\python.exe",
21+
"original_executable": "c:\\path\\to\\python\\python.exe",
22+
"system_executable": "c:\\path\\to\\python\\python.exe",
23+
"has_venv": false,
24+
"path": [
25+
"c:\\path\\to\\python\\Scripts\\virtualenv.exe",
26+
"c:\\path\\to\\python\\python310.zip",
27+
"c:\\path\\to\\python",
28+
"c:\\path\\to\\python\\Lib\\site-packages"
29+
],
30+
"file_system_encoding": "utf-8",
31+
"stdout_encoding": "utf-8",
32+
"sysconfig_scheme": null,
33+
"sysconfig_paths": {
34+
"stdlib": "{installed_base}/Lib",
35+
"platstdlib": "{base}/Lib",
36+
"purelib": "{base}/Lib/site-packages",
37+
"platlib": "{base}/Lib/site-packages",
38+
"include": "{installed_base}/Include",
39+
"scripts": "{base}/Scripts",
40+
"data": "{base}"
41+
},
42+
"distutils_install": {
43+
"purelib": "Lib\\site-packages",
44+
"platlib": "Lib\\site-packages",
45+
"headers": "Include\\UNKNOWN",
46+
"scripts": "Scripts",
47+
"data": ""
48+
},
49+
"sysconfig": {
50+
"makefile_filename": "c:\\path\\to\\python\\Lib\\config\\Makefile"
51+
},
52+
"sysconfig_vars": {
53+
"PYTHONFRAMEWORK": "",
54+
"installed_base": "c:\\path\\to\\python",
55+
"base": "c:\\path\\to\\python"
56+
},
57+
"system_stdlib": "c:\\path\\to\\python\\Lib",
58+
"system_stdlib_platform": "c:\\path\\to\\python\\Lib",
59+
"max_size": 9223372036854775807,
60+
"_creators": null
61+
}

‎tests/unit/create/via_global_ref/builtin/cpython/test_cpython3_win.py‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
import pytest
2+
from testing.helpers import contains_exe, contains_ref
3+
from testing.path import join as path
4+
5+
from virtualenv.create.via_global_ref.builtin.cpython.cpython3 import CPython3Windows
6+
7+
CPYTHON3_PATH = (
8+
"virtualenv.create.via_global_ref.builtin.cpython.common.Path",
9+
"virtualenv.create.via_global_ref.builtin.cpython.cpython3.Path",
10+
)
11+
12+
13+
@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
14+
def test_2_exe_on_default_py_host(py_info, mock_files):
15+
mock_files(CPYTHON3_PATH, [py_info.system_executable])
16+
sources = tuple(CPython3Windows.sources(interpreter=py_info))
17+
# Default Python exe.
18+
assert contains_exe(sources, py_info.system_executable)
19+
# Should always exist.
20+
assert contains_exe(sources, path(py_info.prefix, "pythonw.exe"))
21+
22+
23+
@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
24+
def test_3_exe_on_not_default_py_host(py_info, mock_files):
25+
# Not default python host.
26+
py_info.system_executable = path(py_info.prefix, "python666.exe")
27+
mock_files(CPYTHON3_PATH, [py_info.system_executable])
28+
sources = tuple(CPython3Windows.sources(interpreter=py_info))
29+
# Not default Python exe linked to both the default name and origin.
30+
assert contains_exe(sources, py_info.system_executable, "python.exe")
31+
assert contains_exe(sources, py_info.system_executable, "python666.exe")
32+
# Should always exist.
33+
assert contains_exe(sources, path(py_info.prefix, "pythonw.exe"))
34+
35+
36+
@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
37+
def test_only_shim(py_info, mock_files):
38+
shim = path(py_info.system_stdlib, "venv\\scripts\\nt\\python.exe")
39+
py_files = (
40+
path(py_info.prefix, "libcrypto-1_1.dll"),
41+
path(py_info.prefix, "libffi-7.dll"),
42+
path(py_info.prefix, "_asyncio.pyd"),
43+
path(py_info.prefix, "_bz2.pyd"),
44+
)
45+
mock_files(CPYTHON3_PATH, [shim, *py_files])
46+
sources = tuple(CPython3Windows.sources(interpreter=py_info))
47+
assert CPython3Windows.has_shim(interpreter=py_info)
48+
assert contains_exe(sources, shim)
49+
assert not contains_exe(sources, py_info.system_executable)
50+
for file in py_files:
51+
assert not contains_ref(sources, file)
52+
53+
54+
@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
55+
def test_exe_dll_pyd_without_shim(py_info, mock_files):
56+
py_files = (
57+
path(py_info.prefix, "libcrypto-1_1.dll"),
58+
path(py_info.prefix, "libffi-7.dll"),
59+
path(py_info.prefix, "_asyncio.pyd"),
60+
path(py_info.prefix, "_bz2.pyd"),
61+
)
62+
mock_files(CPYTHON3_PATH, py_files)
63+
sources = tuple(CPython3Windows.sources(interpreter=py_info))
64+
assert not CPython3Windows.has_shim(interpreter=py_info)
65+
assert contains_exe(sources, py_info.system_executable)
66+
for file in py_files:
67+
assert contains_ref(sources, file)
68+
69+
70+
@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
71+
def test_python_zip_if_exists_and_set_in_path(py_info, mock_files):
72+
python_zip_name = "python{}.zip".format(py_info.version_nodot)
73+
python_zip = path(py_info.prefix, python_zip_name)
74+
mock_files(CPYTHON3_PATH, [python_zip])
75+
sources = tuple(CPython3Windows.sources(interpreter=py_info))
76+
assert python_zip in py_info.path
77+
assert contains_ref(sources, python_zip)
78+
79+
80+
@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
81+
def test_no_python_zip_if_exists_and_not_set_in_path(py_info, mock_files):
82+
python_zip_name = "python{}.zip".format(py_info.version_nodot)
83+
python_zip = path(py_info.prefix, python_zip_name)
84+
py_info.path.remove(python_zip)
85+
mock_files(CPYTHON3_PATH, [python_zip])
86+
sources = tuple(CPython3Windows.sources(interpreter=py_info))
87+
assert python_zip not in py_info.path
88+
assert not contains_ref(sources, python_zip)
89+
90+
91+
@pytest.mark.parametrize("py_info_name", ["cpython3_win_embed"])
92+
def test_no_python_zip_if_not_exists(py_info, mock_files):
93+
python_zip_name = "python{}.zip".format(py_info.version_nodot)
94+
python_zip = path(py_info.prefix, python_zip_name)
95+
# No `python_zip`, just python.exe file.
96+
mock_files(CPYTHON3_PATH, [py_info.system_executable])
97+
sources = tuple(CPython3Windows.sources(interpreter=py_info))
98+
assert python_zip in py_info.path
99+
assert not contains_ref(sources, python_zip)

0 commit comments

Comments
 (0)