Skip to content

Commit 9048c49

Browse files
zoobazooba
authored
bpo-37369: Fix initialization of sys members when launched via an app container (GH-14428)
sys._base_executable is now always defined on all platforms, and can be overridden through configuration. Also adds test.support.PythonSymlink to encapsulate platform-specific logic for symlinking sys.executable
1 parent 80097e0 commit 9048c49

File tree

17 files changed

+401
-268
lines changed

17 files changed

+401
-268
lines changed

‎Include/cpython/initconfig.h‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -373,10 +373,11 @@ typedef struct {
373373
module_search_paths_set is equal
374374
to zero. */
375375

376-
wchar_t *executable; /* sys.executable */
377-
wchar_t *prefix; /* sys.prefix */
378-
wchar_t *base_prefix; /* sys.base_prefix */
379-
wchar_t *exec_prefix; /* sys.exec_prefix */
376+
wchar_t *executable; /* sys.executable */
377+
wchar_t *base_executable; /* sys._base_executable */
378+
wchar_t *prefix; /* sys.prefix */
379+
wchar_t *base_prefix; /* sys.base_prefix */
380+
wchar_t *exec_prefix; /* sys.exec_prefix */
380381
wchar_t *base_exec_prefix; /* sys.base_exec_prefix */
381382

382383
/* --- Parameter only used by Py_Main() ---------- */

‎Include/internal/pycore_pathconfig.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ typedef struct _PyPathConfig {
2727
are ignored when their value are equal to -1 (unset). */
2828
int isolated;
2929
int site_import;
30+
/* Set when a venv is detected */
31+
wchar_t *base_executable;
3032
} _PyPathConfig;
3133

3234
#define _PyPathConfig_INIT \

‎Lib/multiprocessing/popen_spawn_win32.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,7 @@
2222
def _path_eq(p1, p2):
2323
return p1 == p2 or os.path.normcase(p1) == os.path.normcase(p2)
2424

25-
WINENV = (hasattr(sys, '_base_executable') and
26-
not _path_eq(sys.executable, sys._base_executable))
25+
WINENV = not _path_eq(sys.executable, sys._base_executable)
2726

2827

2928
def _close_handles(*handles):

‎Lib/site.py‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -459,13 +459,6 @@ def venv(known_paths):
459459
env = os.environ
460460
if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in env:
461461
executable = sys._base_executable = os.environ['__PYVENV_LAUNCHER__']
462-
elif sys.platform == 'win32' and '__PYVENV_LAUNCHER__' in env:
463-
executable = sys.executable
464-
import _winapi
465-
sys._base_executable = _winapi.GetModuleFileName(0)
466-
# bpo-35873: Clear the environment variable to avoid it being
467-
# inherited by child processes.
468-
del os.environ['__PYVENV_LAUNCHER__']
469462
else:
470463
executable = sys.executable
471464
exe_dir, _ = os.path.split(os.path.abspath(executable))

‎Lib/test/support/__init__.py‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import fnmatch
1313
import functools
1414
import gc
15+
import glob
1516
import importlib
1617
import importlib.util
1718
import io
@@ -2500,6 +2501,84 @@ def skip_unless_symlink(test):
25002501
msg = "Requires functional symlink implementation"
25012502
return test if ok else unittest.skip(msg)(test)
25022503

2504+
class PythonSymlink:
2505+
"""Creates a symlink for the current Python executable"""
2506+
def __init__(self, link=None):
2507+
self.link = link or os.path.abspath(TESTFN)
2508+
self._linked = []
2509+
self.real = os.path.realpath(sys.executable)
2510+
self._also_link = []
2511+
2512+
self._env = None
2513+
2514+
self._platform_specific()
2515+
2516+
def _platform_specific(self):
2517+
pass
2518+
2519+
if sys.platform == "win32":
2520+
def _platform_specific(self):
2521+
import _winapi
2522+
2523+
if os.path.lexists(self.real) and not os.path.exists(self.real):
2524+
# App symlink appears to not exist, but we want the
2525+
# real executable here anyway
2526+
self.real = _winapi.GetModuleFileName(0)
2527+
2528+
dll = _winapi.GetModuleFileName(sys.dllhandle)
2529+
src_dir = os.path.dirname(dll)
2530+
dest_dir = os.path.dirname(self.link)
2531+
self._also_link.append((
2532+
dll,
2533+
os.path.join(dest_dir, os.path.basename(dll))
2534+
))
2535+
for runtime in glob.glob(os.path.join(src_dir, "vcruntime*.dll")):
2536+
self._also_link.append((
2537+
runtime,
2538+
os.path.join(dest_dir, os.path.basename(runtime))
2539+
))
2540+
2541+
self._env = {k.upper(): os.getenv(k) for k in os.environ}
2542+
self._env["PYTHONHOME"] = os.path.dirname(self.real)
2543+
if sysconfig.is_python_build(True):
2544+
self._env["PYTHONPATH"] = os.path.dirname(os.__file__)
2545+
2546+
def __enter__(self):
2547+
os.symlink(self.real, self.link)
2548+
self._linked.append(self.link)
2549+
for real, link in self._also_link:
2550+
os.symlink(real, link)
2551+
self._linked.append(link)
2552+
return self
2553+
2554+
def __exit__(self, exc_type, exc_value, exc_tb):
2555+
for link in self._linked:
2556+
try:
2557+
os.remove(link)
2558+
except IOError as ex:
2559+
if verbose:
2560+
print("failed to clean up {}: {}".format(link, ex))
2561+
2562+
def _call(self, python, args, env, returncode):
2563+
cmd = [python, *args]
2564+
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
2565+
stderr=subprocess.PIPE, env=env)
2566+
r = p.communicate()
2567+
if p.returncode != returncode:
2568+
if verbose:
2569+
print(repr(r[0]))
2570+
print(repr(r[1]), file=sys.stderr)
2571+
raise RuntimeError(
2572+
'unexpected return code: {0} (0x{0:08X})'.format(p.returncode))
2573+
return r
2574+
2575+
def call_real(self, *args, returncode=0):
2576+
return self._call(self.real, args, None, returncode)
2577+
2578+
def call_link(self, *args, returncode=0):
2579+
return self._call(self.link, args, self._env, returncode)
2580+
2581+
25032582
_can_xattr = None
25042583
def can_xattr():
25052584
global _can_xattr

‎Lib/test/test_embed.py‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
362362
'pythonpath_env': None,
363363
'home': None,
364364
'executable': GET_DEFAULT_CONFIG,
365+
'base_executable': GET_DEFAULT_CONFIG,
365366

366367
'prefix': GET_DEFAULT_CONFIG,
367368
'base_prefix': GET_DEFAULT_CONFIG,
@@ -534,14 +535,16 @@ def get_expected_config(self, expected_preconfig, expected, env, api,
534535
if expected['stdio_errors'] is self.GET_DEFAULT_CONFIG:
535536
expected['stdio_errors'] = 'surrogateescape'
536537

538+
if sys.platform == 'win32':
539+
default_executable = self.test_exe
540+
elif expected['program_name'] is not self.GET_DEFAULT_CONFIG:
541+
default_executable = os.path.abspath(expected['program_name'])
542+
else:
543+
default_executable = os.path.join(os.getcwd(), '_testembed')
537544
if expected['executable'] is self.GET_DEFAULT_CONFIG:
538-
if sys.platform == 'win32':
539-
expected['executable'] = self.test_exe
540-
else:
541-
if expected['program_name'] is not self.GET_DEFAULT_CONFIG:
542-
expected['executable'] = os.path.abspath(expected['program_name'])
543-
else:
544-
expected['executable'] = os.path.join(os.getcwd(), '_testembed')
545+
expected['executable'] = default_executable
546+
if expected['base_executable'] is self.GET_DEFAULT_CONFIG:
547+
expected['base_executable'] = default_executable
545548
if expected['program_name'] is self.GET_DEFAULT_CONFIG:
546549
expected['program_name'] = './_testembed'
547550

‎Lib/test/test_httpservers.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -610,9 +610,10 @@ def setUp(self):
610610

611611
# The shebang line should be pure ASCII: use symlink if possible.
612612
# See issue #7668.
613+
self._pythonexe_symlink = None
613614
if support.can_symlink():
614615
self.pythonexe = os.path.join(self.parent_dir, 'python')
615-
os.symlink(sys.executable, self.pythonexe)
616+
self._pythonexe_symlink = support.PythonSymlink(self.pythonexe).__enter__()
616617
else:
617618
self.pythonexe = sys.executable
618619

@@ -655,8 +656,8 @@ def setUp(self):
655656
def tearDown(self):
656657
try:
657658
os.chdir(self.cwd)
658-
if self.pythonexe != sys.executable:
659-
os.remove(self.pythonexe)
659+
if self._pythonexe_symlink:
660+
self._pythonexe_symlink.__exit__(None, None, None)
660661
if self.nocgi_path:
661662
os.remove(self.nocgi_path)
662663
if self.file1_path:

‎Lib/test/test_platform.py‎

Lines changed: 8 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -20,37 +20,9 @@ def test_architecture(self):
2020

2121
@support.skip_unless_symlink
2222
def test_architecture_via_symlink(self): # issue3762
23-
# On Windows, the EXE needs to know where pythonXY.dll and *.pyd is at
24-
# so we add the directory to the path, PYTHONHOME and PYTHONPATH.
25-
env = None
26-
if sys.platform == "win32":
27-
env = {k.upper(): os.environ[k] for k in os.environ}
28-
env["PATH"] = "{};{}".format(
29-
os.path.dirname(sys.executable), env.get("PATH", ""))
30-
env["PYTHONHOME"] = os.path.dirname(sys.executable)
31-
if sysconfig.is_python_build(True):
32-
env["PYTHONPATH"] = os.path.dirname(os.__file__)
33-
34-
def get(python, env=None):
35-
cmd = [python, '-c',
36-
'import platform; print(platform.architecture())']
37-
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
38-
stderr=subprocess.PIPE, env=env)
39-
r = p.communicate()
40-
if p.returncode:
41-
print(repr(r[0]))
42-
print(repr(r[1]), file=sys.stderr)
43-
self.fail('unexpected return code: {0} (0x{0:08X})'
44-
.format(p.returncode))
45-
return r
46-
47-
real = os.path.realpath(sys.executable)
48-
link = os.path.abspath(support.TESTFN)
49-
os.symlink(real, link)
50-
try:
51-
self.assertEqual(get(real), get(link, env=env))
52-
finally:
53-
os.remove(link)
23+
with support.PythonSymlink() as py:
24+
cmd = "-c", "import platform; print(platform.architecture())"
25+
self.assertEqual(py.call_real(*cmd), py.call_link(*cmd))
5426

5527
def test_platform(self):
5628
for aliased in (False, True):
@@ -275,6 +247,11 @@ def test_libc_ver(self):
275247
os.path.exists(sys.executable+'.exe'):
276248
# Cygwin horror
277249
executable = sys.executable + '.exe'
250+
elif sys.platform == "win32" and not os.path.exists(sys.executable):
251+
# App symlink appears to not exist, but we want the
252+
# real executable here anyway
253+
import _winapi
254+
executable = _winapi.GetModuleFileName(0)
278255
else:
279256
executable = sys.executable
280257
platform.libc_ver(executable)

‎Lib/test/test_sysconfig.py‎

Lines changed: 6 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
from copy import copy
77

88
from test.support import (import_module, TESTFN, unlink, check_warnings,
9-
captured_stdout, skip_unless_symlink, change_cwd)
9+
captured_stdout, skip_unless_symlink, change_cwd,
10+
PythonSymlink)
1011

1112
import sysconfig
1213
from sysconfig import (get_paths, get_platform, get_config_vars,
@@ -232,39 +233,10 @@ def test_get_scheme_names(self):
232233
self.assertEqual(get_scheme_names(), wanted)
233234

234235
@skip_unless_symlink
235-
def test_symlink(self):
236-
# On Windows, the EXE needs to know where pythonXY.dll is at so we have
237-
# to add the directory to the path.
238-
env = None
239-
if sys.platform == "win32":
240-
env = {k.upper(): os.environ[k] for k in os.environ}
241-
env["PATH"] = "{};{}".format(
242-
os.path.dirname(sys.executable), env.get("PATH", ""))
243-
# Requires PYTHONHOME as well since we locate stdlib from the
244-
# EXE path and not the DLL path (which should be fixed)
245-
env["PYTHONHOME"] = os.path.dirname(sys.executable)
246-
if sysconfig.is_python_build(True):
247-
env["PYTHONPATH"] = os.path.dirname(os.__file__)
248-
249-
# Issue 7880
250-
def get(python, env=None):
251-
cmd = [python, '-c',
252-
'import sysconfig; print(sysconfig.get_platform())']
253-
p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
254-
stderr=subprocess.PIPE, env=env)
255-
out, err = p.communicate()
256-
if p.returncode:
257-
print((out, err))
258-
self.fail('Non-zero return code {0} (0x{0:08X})'
259-
.format(p.returncode))
260-
return out, err
261-
real = os.path.realpath(sys.executable)
262-
link = os.path.abspath(TESTFN)
263-
os.symlink(real, link)
264-
try:
265-
self.assertEqual(get(real), get(link, env))
266-
finally:
267-
unlink(link)
236+
def test_symlink(self): # Issue 7880
237+
with PythonSymlink() as py:
238+
cmd = "-c", "import sysconfig; print(sysconfig.get_platform())"
239+
self.assertEqual(py.call_real(*cmd), py.call_link(*cmd))
268240

269241
def test_user_similar(self):
270242
# Issue #8759: make sure the posix scheme for the users

‎Lib/test/test_venv.py‎

Lines changed: 19 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@
2828
# Platforms that set sys._base_executable can create venvs from within
2929
# another venv, so no need to skip tests that require venv.create().
3030
requireVenvCreate = unittest.skipUnless(
31-
hasattr(sys, '_base_executable')
32-
or sys.prefix == sys.base_prefix,
31+
sys.prefix == sys.base_prefix
32+
or sys._base_executable != sys.executable,
3333
'cannot run venv.create from within a venv on this platform')
3434

3535
def check_output(cmd, encoding=None):
@@ -57,8 +57,14 @@ def setUp(self):
5757
self.bindir = 'bin'
5858
self.lib = ('lib', 'python%d.%d' % sys.version_info[:2])
5959
self.include = 'include'
60-
executable = getattr(sys, '_base_executable', sys.executable)
60+
executable = sys._base_executable
6161
self.exe = os.path.split(executable)[-1]
62+
if (sys.platform == 'win32'
63+
and os.path.lexists(executable)
64+
and not os.path.exists(executable)):
65+
self.cannot_link_exe = True
66+
else:
67+
self.cannot_link_exe = False
6268

6369
def tearDown(self):
6470
rmtree(self.env_dir)
@@ -102,7 +108,7 @@ def test_defaults(self):
102108
else:
103109
self.assertFalse(os.path.exists(p))
104110
data = self.get_text_file_contents('pyvenv.cfg')
105-
executable = getattr(sys, '_base_executable', sys.executable)
111+
executable = sys._base_executable
106112
path = os.path.dirname(executable)
107113
self.assertIn('home = %s' % path, data)
108114
fn = self.get_env_file(self.bindir, self.exe)
@@ -158,20 +164,16 @@ def test_prefixes(self):
158164
"""
159165
Test that the prefix values are as expected.
160166
"""
161-
#check our prefixes
162-
self.assertEqual(sys.base_prefix, sys.prefix)
163-
self.assertEqual(sys.base_exec_prefix, sys.exec_prefix)
164-
165167
# check a venv's prefixes
166168
rmtree(self.env_dir)
167169
self.run_with_capture(venv.create, self.env_dir)
168170
envpy = os.path.join(self.env_dir, self.bindir, self.exe)
169171
cmd = [envpy, '-c', None]
170172
for prefix, expected in (
171173
('prefix', self.env_dir),
172-
('prefix', self.env_dir),
173-
('base_prefix', sys.prefix),
174-
('base_exec_prefix', sys.exec_prefix)):
174+
('exec_prefix', self.env_dir),
175+
('base_prefix', sys.base_prefix),
176+
('base_exec_prefix', sys.base_exec_prefix)):
175177
cmd[2] = 'import sys; print(sys.%s)' % prefix
176178
out, err = check_output(cmd)
177179
self.assertEqual(out.strip(), expected.encode())
@@ -283,7 +285,12 @@ def test_symlinking(self):
283285
# symlinked to 'python3.3' in the env, even when symlinking in
284286
# general isn't wanted.
285287
if usl:
286-
self.assertTrue(os.path.islink(fn))
288+
if self.cannot_link_exe:
289+
# Symlinking is skipped when our executable is already a
290+
# special app symlink
291+
self.assertFalse(os.path.islink(fn))
292+
else:
293+
self.assertTrue(os.path.islink(fn))
287294

288295
# If a venv is created from a source build and that venv is used to
289296
# run the test, the pyvenv.cfg in the venv created in the test will

0 commit comments

Comments
 (0)