Skip to content

Commit 0c90d6f

Browse files
vstinnervstinner
authored
[3.7] bpo-34247: Fix Python 3.7 initialization (#8659)
* -X dev: it is now possible to override the memory allocator using PYTHONMALLOC even if the developer mode is enabled. * Add _Py_InitializeFromConfig() * Add _Py_Initialize_ReadEnvVars() to set global configuration variables from environment variables * Fix the code to initialize Python: Py_Initialize() now also reads environment variables * _Py_InitializeCore() can now be called twice: the second call only replaces the configuration. * Write unit tests on Py_Initialize() and the different ways to configure Python * The isolated mode now always sets Py_IgnoreEnvironmentFlag and Py_NoUserSiteDirectory to 1. * pymain_read_conf() now saves/restores the configuration if the encoding changed
1 parent e65ec49 commit 0c90d6f

File tree

10 files changed

+781
-162
lines changed

10 files changed

+781
-162
lines changed

‎Include/pylifecycle.h‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,14 +51,24 @@ PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding,
5151
const char *errors);
5252

5353
/* PEP 432 Multi-phase initialization API (Private while provisional!) */
54-
PyAPI_FUNC(_PyInitError) _Py_InitializeCore(const _PyCoreConfig *);
54+
PyAPI_FUNC(_PyInitError) _Py_InitializeCore(
55+
PyInterpreterState **interp_p,
56+
const _PyCoreConfig *config);
5557
PyAPI_FUNC(int) _Py_IsCoreInitialized(void);
58+
PyAPI_FUNC(_PyInitError) _Py_InitializeFromConfig(
59+
const _PyCoreConfig *config);
60+
#ifdef Py_BUILD_CORE
61+
PyAPI_FUNC(void) _Py_Initialize_ReadEnvVarsNoAlloc(void);
62+
#endif
5663

5764
PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *);
5865
PyAPI_FUNC(void) _PyCoreConfig_Clear(_PyCoreConfig *);
5966
PyAPI_FUNC(int) _PyCoreConfig_Copy(
6067
_PyCoreConfig *config,
6168
const _PyCoreConfig *config2);
69+
PyAPI_FUNC(void) _PyCoreConfig_SetGlobalConfig(
70+
const _PyCoreConfig *config);
71+
6272

6373
PyAPI_FUNC(_PyInitError) _PyMainInterpreterConfig_Read(
6474
_PyMainInterpreterConfig *config,
@@ -68,14 +78,16 @@ PyAPI_FUNC(int) _PyMainInterpreterConfig_Copy(
6878
_PyMainInterpreterConfig *config,
6979
const _PyMainInterpreterConfig *config2);
7080

71-
PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *);
72-
#endif
81+
PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter(
82+
PyInterpreterState *interp,
83+
const _PyMainInterpreterConfig *config);
84+
#endif /* !defined(Py_LIMITED_API) */
85+
7386

7487
/* Initialization and finalization */
7588
PyAPI_FUNC(void) Py_Initialize(void);
7689
PyAPI_FUNC(void) Py_InitializeEx(int);
7790
#ifndef Py_LIMITED_API
78-
PyAPI_FUNC(_PyInitError) _Py_InitializeEx_Private(int, int);
7991
PyAPI_FUNC(void) _Py_FatalInitError(_PyInitError err) _Py_NO_RETURN;
8092
#endif
8193
PyAPI_FUNC(void) Py_Finalize(void);

‎Include/pystate.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,11 @@ typedef struct {
7979
#define _PyCoreConfig_INIT \
8080
(_PyCoreConfig){ \
8181
.install_signal_handlers = -1, \
82+
.ignore_environment = -1, \
8283
.use_hash_seed = -1, \
8384
.coerce_c_locale = -1, \
85+
.faulthandler = -1, \
86+
.tracemalloc = -1, \
8487
.utf8_mode = -1, \
8588
.argc = -1, \
8689
.nmodule_search_path = -1}

‎Lib/test/test_cmd_line.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -523,9 +523,7 @@ def run_xdev(self, *args, check_exitcode=True, xdev=True):
523523
env = dict(os.environ)
524524
env.pop('PYTHONWARNINGS', None)
525525
env.pop('PYTHONDEVMODE', None)
526-
# Force malloc() to disable the debug hooks which are enabled
527-
# by default for Python compiled in debug mode
528-
env['PYTHONMALLOC'] = 'malloc'
526+
env.pop('PYTHONMALLOC', None)
529527

530528
if xdev:
531529
args = (sys.executable, '-X', 'dev', *args)

‎Lib/test/test_embed.py‎

Lines changed: 147 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import sys
1010

1111

12-
class EmbeddingTests(unittest.TestCase):
12+
class EmbeddingTestsMixin:
1313
def setUp(self):
1414
here = os.path.abspath(__file__)
1515
basepath = os.path.dirname(os.path.dirname(os.path.dirname(here)))
@@ -110,6 +110,8 @@ def run_repeated_init_and_subinterpreters(self):
110110
yield current_run
111111
current_run = []
112112

113+
114+
class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase):
113115
def test_subinterps_main(self):
114116
for run in self.run_repeated_init_and_subinterpreters():
115117
main = run[0]
@@ -247,5 +249,149 @@ def test_initialize_pymain(self):
247249
self.assertEqual(err, '')
248250

249251

252+
class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
253+
maxDiff = 4096
254+
DEFAULT_CONFIG = {
255+
'install_signal_handlers': 1,
256+
'Py_IgnoreEnvironmentFlag': 0,
257+
'use_hash_seed': 0,
258+
'hash_seed': 0,
259+
'allocator': '(null)',
260+
'dev_mode': 0,
261+
'faulthandler': 0,
262+
'tracemalloc': 0,
263+
'import_time': 0,
264+
'show_ref_count': 0,
265+
'show_alloc_count': 0,
266+
'dump_refs': 0,
267+
'malloc_stats': 0,
268+
'utf8_mode': 0,
269+
270+
'coerce_c_locale': 0,
271+
'coerce_c_locale_warn': 0,
272+
273+
'program_name': './_testembed',
274+
'argc': 0,
275+
'argv': '[]',
276+
'program': '(null)',
277+
278+
'Py_IsolatedFlag': 0,
279+
'Py_NoSiteFlag': 0,
280+
'Py_BytesWarningFlag': 0,
281+
'Py_InspectFlag': 0,
282+
'Py_InteractiveFlag': 0,
283+
'Py_OptimizeFlag': 0,
284+
'Py_DebugFlag': 0,
285+
'Py_DontWriteBytecodeFlag': 0,
286+
'Py_VerboseFlag': 0,
287+
'Py_QuietFlag': 0,
288+
'Py_NoUserSiteDirectory': 0,
289+
'Py_UnbufferedStdioFlag': 0,
290+
291+
'_disable_importlib': 0,
292+
'Py_FrozenFlag': 0,
293+
}
294+
295+
def check_config(self, testname, expected):
296+
env = dict(os.environ)
297+
for key in list(env):
298+
if key.startswith('PYTHON'):
299+
del env[key]
300+
# Disable C locale coercion and UTF-8 mode to not depend
301+
# on the current locale
302+
env['PYTHONCOERCECLOCALE'] = '0'
303+
env['PYTHONUTF8'] = '0'
304+
out, err = self.run_embedded_interpreter(testname, env=env)
305+
# Ignore err
306+
307+
expected = dict(self.DEFAULT_CONFIG, **expected)
308+
for key, value in expected.items():
309+
expected[key] = str(value)
310+
311+
config = {}
312+
for line in out.splitlines():
313+
key, value = line.split(' = ', 1)
314+
config[key] = value
315+
self.assertEqual(config, expected)
316+
317+
def test_init_default_config(self):
318+
self.check_config("init_default_config", {})
319+
320+
def test_init_global_config(self):
321+
config = {
322+
'program_name': './globalvar',
323+
'Py_NoSiteFlag': 1,
324+
'Py_BytesWarningFlag': 1,
325+
'Py_InspectFlag': 1,
326+
'Py_InteractiveFlag': 1,
327+
'Py_OptimizeFlag': 2,
328+
'Py_DontWriteBytecodeFlag': 1,
329+
'Py_VerboseFlag': 1,
330+
'Py_QuietFlag': 1,
331+
'Py_UnbufferedStdioFlag': 1,
332+
'utf8_mode': 1,
333+
'Py_NoUserSiteDirectory': 1,
334+
'Py_FrozenFlag': 1,
335+
}
336+
self.check_config("init_global_config", config)
337+
338+
def test_init_from_config(self):
339+
config = {
340+
'install_signal_handlers': 0,
341+
'use_hash_seed': 1,
342+
'hash_seed': 123,
343+
'allocator': 'malloc_debug',
344+
'tracemalloc': 2,
345+
'import_time': 1,
346+
'show_ref_count': 1,
347+
'show_alloc_count': 1,
348+
'malloc_stats': 1,
349+
350+
'utf8_mode': 1,
351+
352+
'program_name': './conf_program_name',
353+
'program': 'conf_program',
354+
355+
'faulthandler': 1,
356+
}
357+
self.check_config("init_from_config", config)
358+
359+
def test_init_env(self):
360+
config = {
361+
'use_hash_seed': 1,
362+
'hash_seed': 42,
363+
'allocator': 'malloc_debug',
364+
'tracemalloc': 2,
365+
'import_time': 1,
366+
'malloc_stats': 1,
367+
'utf8_mode': 1,
368+
'Py_InspectFlag': 1,
369+
'Py_OptimizeFlag': 2,
370+
'Py_DontWriteBytecodeFlag': 1,
371+
'Py_VerboseFlag': 1,
372+
'Py_UnbufferedStdioFlag': 1,
373+
'Py_NoUserSiteDirectory': 1,
374+
'faulthandler': 1,
375+
'dev_mode': 1,
376+
}
377+
self.check_config("init_env", config)
378+
379+
def test_init_dev_mode(self):
380+
config = {
381+
'dev_mode': 1,
382+
'faulthandler': 1,
383+
'allocator': 'debug',
384+
}
385+
self.check_config("init_dev_mode", config)
386+
387+
def test_init_isolated(self):
388+
config = {
389+
'Py_IsolatedFlag': 1,
390+
'Py_IgnoreEnvironmentFlag': 1,
391+
'Py_NoUserSiteDirectory': 1,
392+
}
393+
self.check_config("init_isolated", config)
394+
395+
250396
if __name__ == "__main__":
251397
unittest.main()

‎Misc/NEWS.d/next/C API/2018-08-05-00-21-38.bpo-34247._Sn92u.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix Py_Initialize() regression introduced in 3.7.0: read environment
2+
variables like PYTHONOPTIMIZE.

‎Misc/NEWS.d/next/Core and Builtins/2018-08-03-21-59-06.bpo-34170.v1h_H2.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
-X dev: it is now possible to override the memory allocator using
2+
PYTHONMALLOC even if the developer mode is enabled.

0 commit comments

Comments
 (0)