bpo-36443: Disable C locale coercion and UTF-8 Mode by default (GH-12589)

bpo-36443, bpo-36202: Since Python 3.7.0, calling Py_DecodeLocale()
before Py_Initialize() produces mojibake if the LC_CTYPE locale is
coerced and/or if the UTF-8 Mode is enabled by the user
configuration. This change fix the issue by disabling LC_CTYPE
coercion and UTF-8 Mode by default. They must now be enabled
explicitly (opt-in) using the new _Py_PreInitialize() API with
_PyPreConfig.

When embedding Python, set coerce_c_locale and utf8_mode attributes
of _PyPreConfig to -1 to enable automatically these parameters
depending on the LC_CTYPE locale, environment variables and command
line arguments

Alternative: Setting Py_UTF8Mode to 1 always explicitly enables the
UTF-8 Mode.

Changes:

* _PyPreConfig_INIT now sets coerce_c_locale and utf8_mode to 0 by
  default.
* _Py_InitializeFromArgs() and _Py_InitializeFromWideArgs() can now
  be called with config=NULL.

1 files changed, 13 insertions, 6 deletions

diff --git a/Modules/main.c b/Modules/main.c
index ff79edbe43..766576939d 100644
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -52,23 +52,30 @@ pymain_init(const _PyArgv *args)
     fedisableexcept(FE_OVERFLOW);
 #endif
 
-    _PyCoreConfig config = _PyCoreConfig_INIT;
-
+    _PyPreConfig preconfig = _PyPreConfig_INIT;
+    /* Set to -1 to enable them depending on the LC_CTYPE locale and the
+       environment variables (PYTHONUTF8 and PYTHONCOERCECLOCALE)  */
+    preconfig.coerce_c_locale = -1;
+    preconfig.utf8_mode = -1;
     if (args->use_bytes_argv) {
-        err = _Py_PreInitializeFromArgs(NULL, args->argc, args->bytes_argv);
+        err = _Py_PreInitializeFromArgs(&preconfig,
+                                        args->argc, args->bytes_argv);
     }
     else {
-        err = _Py_PreInitializeFromWideArgs(NULL, args->argc, args->wchar_argv);
+        err = _Py_PreInitializeFromWideArgs(&preconfig,
+                                            args->argc, args->wchar_argv);
     }
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
 
+    /* pass NULL as the config: config is read from command line arguments,
+       environment variables, configuration files */
     if (args->use_bytes_argv) {
-        return _Py_InitializeFromArgs(&config, args->argc, args->bytes_argv);
+        return _Py_InitializeFromArgs(NULL, args->argc, args->bytes_argv);
     }
     else {
-        return _Py_InitializeFromWideArgs(&config, args->argc, args->wchar_argv);
+        return _Py_InitializeFromWideArgs(NULL, args->argc, args->wchar_argv);
     }
 }