From 0c90d6f75931da4fec84d06c2efe9dd94bb96b77 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 5 Aug 2018 12:31:59 +0200 Subject: [3.7] bpo-34247: Fix Python 3.7 initialization (#8659) * -X dev: it is now possible to override the memory allocator using PYTHONMALLOC even if the developer mode is enabled. * Add _Py_InitializeFromConfig() * Add _Py_Initialize_ReadEnvVars() to set global configuration variables from environment variables * Fix the code to initialize Python: Py_Initialize() now also reads environment variables * _Py_InitializeCore() can now be called twice: the second call only replaces the configuration. * Write unit tests on Py_Initialize() and the different ways to configure Python * The isolated mode now always sets Py_IgnoreEnvironmentFlag and Py_NoUserSiteDirectory to 1. * pymain_read_conf() now saves/restores the configuration if the encoding changed --- Programs/_freeze_importlib.c | 13 +- Programs/_testembed.c | 294 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 304 insertions(+), 3 deletions(-) (limited to 'Programs') diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c index b8b630cfed..7de641e1a5 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_importlib.c @@ -74,18 +74,25 @@ main(int argc, char *argv[]) } text[text_size] = '\0'; + _PyCoreConfig config = _PyCoreConfig_INIT; + config.program_name = L"./_freeze_importlib"; + /* Don't install importlib, since it could execute outdated bytecode. */ + config._disable_importlib = 1; + Py_NoUserSiteDirectory++; Py_NoSiteFlag++; Py_IgnoreEnvironmentFlag++; Py_FrozenFlag++; - Py_SetProgramName(L"./_freeze_importlib"); - /* Don't install importlib, since it could execute outdated bytecode. */ - _PyInitError err = _Py_InitializeEx_Private(1, 0); + + _PyInitError err = _Py_InitializeFromConfig(&config); + /* No need to call _PyCoreConfig_Clear() since we didn't allocate any + memory: program_name is a constant string. */ if (_Py_INIT_FAILED(err)) { _Py_FatalInitError(err); } + if (strstr(inpath, "_external") != NULL) { is_bootstrap = 0; } diff --git a/Programs/_testembed.c b/Programs/_testembed.c index b1be682f7a..6c35f9586b 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1,4 +1,5 @@ #include +#include "internal/import.h" #include "pythread.h" #include #include @@ -292,6 +293,293 @@ static int test_initialize_pymain(void) } +static void +dump_config(void) +{ +#define ASSERT_EQUAL(a, b) \ + if ((a) != (b)) { \ + printf("ERROR: %s != %s (%i != %i)\n", #a, #b, (a), (b)); \ + exit(1); \ + } +#define ASSERT_STR_EQUAL(a, b) \ + if ((a) == NULL || (b == NULL) || wcscmp((a), (b)) != 0) { \ + printf("ERROR: %s != %s ('%ls' != '%ls')\n", #a, #b, (a), (b)); \ + exit(1); \ + } + + PyInterpreterState *interp = PyThreadState_Get()->interp; + _PyCoreConfig *config = &interp->core_config; + + printf("install_signal_handlers = %i\n", config->install_signal_handlers); + + printf("Py_IgnoreEnvironmentFlag = %i\n", Py_IgnoreEnvironmentFlag); + + printf("use_hash_seed = %i\n", config->use_hash_seed); + printf("hash_seed = %lu\n", config->hash_seed); + + printf("allocator = %s\n", config->allocator); + + printf("dev_mode = %i\n", config->dev_mode); + printf("faulthandler = %i\n", config->faulthandler); + printf("tracemalloc = %i\n", config->tracemalloc); + printf("import_time = %i\n", config->import_time); + printf("show_ref_count = %i\n", config->show_ref_count); + printf("show_alloc_count = %i\n", config->show_alloc_count); + printf("dump_refs = %i\n", config->dump_refs); + printf("malloc_stats = %i\n", config->malloc_stats); + + printf("coerce_c_locale = %i\n", config->coerce_c_locale); + printf("coerce_c_locale_warn = %i\n", config->coerce_c_locale_warn); + printf("utf8_mode = %i\n", config->utf8_mode); + + printf("program_name = %ls\n", config->program_name); + ASSERT_STR_EQUAL(config->program_name, Py_GetProgramName()); + + printf("argc = %i\n", config->argc); + printf("argv = ["); + for (int i=0; i < config->argc; i++) { + if (i) { + printf(", "); + } + printf("\"%ls\"", config->argv[i]); + } + printf("]\n"); + + printf("program = %ls\n", config->program); + /* FIXME: test xoptions */ + /* FIXME: test warnoptions */ + /* FIXME: test module_search_path_env */ + /* FIXME: test home */ + /* FIXME: test module_search_paths */ + /* FIXME: test executable */ + /* FIXME: test prefix */ + /* FIXME: test base_prefix */ + /* FIXME: test exec_prefix */ + /* FIXME: test base_exec_prefix */ + /* FIXME: test dll_path */ + + printf("Py_IsolatedFlag = %i\n", Py_IsolatedFlag); + printf("Py_NoSiteFlag = %i\n", Py_NoSiteFlag); + printf("Py_BytesWarningFlag = %i\n", Py_BytesWarningFlag); + printf("Py_InspectFlag = %i\n", Py_InspectFlag); + printf("Py_InteractiveFlag = %i\n", Py_InteractiveFlag); + printf("Py_OptimizeFlag = %i\n", Py_OptimizeFlag); + printf("Py_DebugFlag = %i\n", Py_DebugFlag); + printf("Py_DontWriteBytecodeFlag = %i\n", Py_DontWriteBytecodeFlag); + printf("Py_VerboseFlag = %i\n", Py_VerboseFlag); + printf("Py_QuietFlag = %i\n", Py_QuietFlag); + printf("Py_NoUserSiteDirectory = %i\n", Py_NoUserSiteDirectory); + printf("Py_UnbufferedStdioFlag = %i\n", Py_UnbufferedStdioFlag); + /* FIXME: test legacy_windows_fs_encoding */ + /* FIXME: test legacy_windows_stdio */ + + printf("_disable_importlib = %i\n", config->_disable_importlib); + /* cannot test _Py_CheckHashBasedPycsMode: the symbol is not exported */ + printf("Py_FrozenFlag = %i\n", Py_FrozenFlag); + +#undef ASSERT_EQUAL +#undef ASSERT_STR_EQUAL +} + + +static int test_init_default_config(void) +{ + _testembed_Py_Initialize(); + dump_config(); + Py_Finalize(); + return 0; +} + + +static int test_init_global_config(void) +{ + /* FIXME: test Py_IgnoreEnvironmentFlag */ + + putenv("PYTHONUTF8=0"); + Py_UTF8Mode = 1; + + /* Test initialization from global configuration variables (Py_xxx) */ + Py_SetProgramName(L"./globalvar"); + + /* Py_IsolatedFlag is not tested */ + Py_NoSiteFlag = 1; + Py_BytesWarningFlag = 1; + + putenv("PYTHONINSPECT="); + Py_InspectFlag = 1; + + putenv("PYTHONOPTIMIZE=0"); + Py_InteractiveFlag = 1; + + putenv("PYTHONDEBUG=0"); + Py_OptimizeFlag = 2; + + /* Py_DebugFlag is not tested */ + + putenv("PYTHONDONTWRITEBYTECODE="); + Py_DontWriteBytecodeFlag = 1; + + putenv("PYTHONVERBOSE=0"); + Py_VerboseFlag = 1; + + Py_QuietFlag = 1; + Py_NoUserSiteDirectory = 1; + + putenv("PYTHONUNBUFFERED="); + Py_UnbufferedStdioFlag = 1; + + Py_FrozenFlag = 1; + + /* FIXME: test Py_LegacyWindowsFSEncodingFlag */ + /* FIXME: test Py_LegacyWindowsStdioFlag */ + + Py_Initialize(); + dump_config(); + Py_Finalize(); + return 0; +} + + +static int test_init_from_config(void) +{ + /* Test _Py_InitializeFromConfig() */ + _PyCoreConfig config = _PyCoreConfig_INIT; + config.install_signal_handlers = 0; + + /* FIXME: test ignore_environment */ + + putenv("PYTHONHASHSEED=42"); + config.use_hash_seed = 1; + config.hash_seed = 123; + + putenv("PYTHONMALLOC=malloc"); + config.allocator = "malloc_debug"; + + /* dev_mode=1 is tested in test_init_dev_mode() */ + + putenv("PYTHONFAULTHANDLER="); + config.faulthandler = 1; + + putenv("PYTHONTRACEMALLOC=0"); + config.tracemalloc = 2; + + putenv("PYTHONPROFILEIMPORTTIME=0"); + config.import_time = 1; + + config.show_ref_count = 1; + config.show_alloc_count = 1; + /* FIXME: test dump_refs: bpo-34223 */ + + putenv("PYTHONMALLOCSTATS=0"); + config.malloc_stats = 1; + + /* FIXME: test coerce_c_locale and coerce_c_locale_warn */ + + putenv("PYTHONUTF8=0"); + Py_UTF8Mode = 0; + config.utf8_mode = 1; + + Py_SetProgramName(L"./globalvar"); + config.program_name = L"./conf_program_name"; + + /* FIXME: test argc/argv */ + config.program = L"conf_program"; + /* FIXME: test xoptions */ + /* FIXME: test warnoptions */ + /* FIXME: test module_search_path_env */ + /* FIXME: test home */ + /* FIXME: test path config: module_search_path .. dll_path */ + + _PyInitError err = _Py_InitializeFromConfig(&config); + /* Don't call _PyCoreConfig_Clear() since all strings are static */ + if (_Py_INIT_FAILED(err)) { + _Py_FatalInitError(err); + } + dump_config(); + Py_Finalize(); + return 0; +} + + +static void test_init_env_putenvs(void) +{ + putenv("PYTHONHASHSEED=42"); + putenv("PYTHONMALLOC=malloc_debug"); + putenv("PYTHONTRACEMALLOC=2"); + putenv("PYTHONPROFILEIMPORTTIME=1"); + putenv("PYTHONMALLOCSTATS=1"); + putenv("PYTHONUTF8=1"); + putenv("PYTHONVERBOSE=1"); + putenv("PYTHONINSPECT=1"); + putenv("PYTHONOPTIMIZE=2"); + putenv("PYTHONDONTWRITEBYTECODE=1"); + putenv("PYTHONUNBUFFERED=1"); + putenv("PYTHONNOUSERSITE=1"); + putenv("PYTHONFAULTHANDLER=1"); + putenv("PYTHONDEVMODE=1"); + /* FIXME: test PYTHONWARNINGS */ + /* FIXME: test PYTHONEXECUTABLE */ + /* FIXME: test PYTHONHOME */ + /* FIXME: test PYTHONDEBUG */ + /* FIXME: test PYTHONDUMPREFS */ + /* FIXME: test PYTHONCOERCECLOCALE */ + /* FIXME: test PYTHONPATH */ +} + + +static int test_init_env(void) +{ + /* Test initialization from environment variables */ + Py_IgnoreEnvironmentFlag = 0; + test_init_env_putenvs(); + _testembed_Py_Initialize(); + dump_config(); + Py_Finalize(); + return 0; +} + + +static int test_init_isolated(void) +{ + /* Test _PyCoreConfig.isolated=1 */ + _PyCoreConfig config = _PyCoreConfig_INIT; + + /* Set coerce_c_locale and utf8_mode to not depend on the locale */ + config.coerce_c_locale = 0; + config.utf8_mode = 0; + /* Use path starting with "./" avoids a search along the PATH */ + config.program_name = L"./_testembed"; + + Py_IsolatedFlag = 1; + + test_init_env_putenvs(); + _PyInitError err = _Py_InitializeFromConfig(&config); + if (_Py_INIT_FAILED(err)) { + _Py_FatalInitError(err); + } + dump_config(); + Py_Finalize(); + return 0; +} + + +static int test_init_dev_mode(void) +{ + _PyCoreConfig config = _PyCoreConfig_INIT; + putenv("PYTHONFAULTHANDLER="); + putenv("PYTHONMALLOC="); + config.dev_mode = 1; + config.program_name = L"./_testembed"; + _PyInitError err = _Py_InitializeFromConfig(&config); + if (_Py_INIT_FAILED(err)) { + _Py_FatalInitError(err); + } + dump_config(); + Py_Finalize(); + return 0; +} + + /* ********************************************************* * List of test cases and the function that implements it. * @@ -318,6 +606,12 @@ static struct TestCase TestCases[] = { { "bpo20891", test_bpo20891 }, { "initialize_twice", test_initialize_twice }, { "initialize_pymain", test_initialize_pymain }, + { "init_default_config", test_init_default_config }, + { "init_global_config", test_init_global_config }, + { "init_from_config", test_init_from_config }, + { "init_env", test_init_env }, + { "init_dev_mode", test_init_dev_mode }, + { "init_isolated", test_init_isolated }, { NULL, NULL } }; -- cgit v1.2.1