bpo-36443: Disable C locale coercion and UTF-8 Mode by default (GH-12589)

bpo-36443, bpo-36202: Since Python 3.7.0, calling Py_DecodeLocale()
before Py_Initialize() produces mojibake if the LC_CTYPE locale is
coerced and/or if the UTF-8 Mode is enabled by the user
configuration. This change fix the issue by disabling LC_CTYPE
coercion and UTF-8 Mode by default. They must now be enabled
explicitly (opt-in) using the new _Py_PreInitialize() API with
_PyPreConfig.

When embedding Python, set coerce_c_locale and utf8_mode attributes
of _PyPreConfig to -1 to enable automatically these parameters
depending on the LC_CTYPE locale, environment variables and command
line arguments

Alternative: Setting Py_UTF8Mode to 1 always explicitly enables the
UTF-8 Mode.

Changes:

* _PyPreConfig_INIT now sets coerce_c_locale and utf8_mode to 0 by
  default.
* _Py_InitializeFromArgs() and _Py_InitializeFromWideArgs() can now
  be called with config=NULL.

1 files changed, 0 insertions, 21 deletions

diff --git a/Programs/_testembed.c b/Programs/_testembed.c
index 425954cdd4..d8e12cf3ff 100644
--- a/Programs/_testembed.c
+++ b/Programs/_testembed.c
@@ -441,8 +441,6 @@ static int test_init_from_config(void)
     putenv("PYTHONMALLOCSTATS=0");
     config.malloc_stats = 1;
 
-    /* FIXME: test coerce_c_locale and coerce_c_locale_warn */
-
     putenv("PYTHONPYCACHEPREFIX=env_pycache_prefix");
     config.pycache_prefix = L"conf_pycache_prefix";
 
@@ -617,17 +615,6 @@ static int test_init_isolated(void)
 {
     _PyInitError err;
 
-    _PyPreConfig preconfig = _PyPreConfig_INIT;
-
-    /* Set coerce_c_locale and utf8_mode to not depend on the locale */
-    preconfig.coerce_c_locale = 0;
-    preconfig.utf8_mode = 0;
-
-    err = _Py_PreInitialize(&preconfig);
-    if (_Py_INIT_FAILED(err)) {
-        _Py_ExitInitError(err);
-    }
-
     /* Test _PyCoreConfig.isolated=1 */
     _PyCoreConfig config = _PyCoreConfig_INIT;
 
@@ -654,10 +641,6 @@ static int test_preinit_isolated1(void)
     _PyInitError err;
 
     _PyPreConfig preconfig = _PyPreConfig_INIT;
-
-    /* Set coerce_c_locale and utf8_mode to not depend on the locale */
-    preconfig.coerce_c_locale = 0;
-    preconfig.utf8_mode = 0;
     preconfig.isolated = 1;
 
     err = _Py_PreInitialize(&preconfig);
@@ -685,10 +668,6 @@ static int test_preinit_isolated2(void)
     _PyInitError err;
 
     _PyPreConfig preconfig = _PyPreConfig_INIT;
-
-    /* Set coerce_c_locale and utf8_mode to not depend on the locale */
-    preconfig.coerce_c_locale = 0;
-    preconfig.utf8_mode = 0;
     preconfig.isolated = 0;
 
     err = _Py_PreInitialize(&preconfig);