patch 8.2.4317: MS-Windows: Vim exits when Python 3 initialisation failsv8.2.4317

Problem:    MS-Windows: Vim exits when Python 3 initialisation fails.
Solution:   Hook into the exit() function to recover from the failure.
            (Ken Takata, closes #9710)

5 files changed, 119 insertions, 14 deletions

diff --git a/src/errors.h b/src/errors.h
index 80a9a5aae..53bd0cd26 100644
--- a/src/errors.h
+++ b/src/errors.h
@@ -3224,3 +3224,7 @@ EXTERN char e_autoload_import_cannot_use_absolute_or_relative_path[]
 EXTERN char e_cannot_use_partial_here[]
 	INIT(= N_("E1265: Cannot use a partial here"));
 #endif
+#if defined(FEAT_PYTHON3) && defined(MSWIN)
+EXTERN char e_critical_error_in_python3_initialization_check_your_installation[]
+	INIT(= N_("E1266: Critical error in python3 initialization, check your python3 installation"));
+#endif
diff --git a/src/if_python3.c b/src/if_python3.c
index c8ca3c379..0b05857d5 100644
--- a/src/if_python3.c
+++ b/src/if_python3.c
@@ -112,12 +112,18 @@ typedef PyObject PySliceObject_T;
 typedef PySliceObject PySliceObject_T;
 #endif
 
+#ifndef MSWIN
+# define HINSTANCE void *
+#endif
+#if defined(DYNAMIC_PYTHON3) || defined(MSWIN)
+static HINSTANCE hinstPy3 = 0; // Instance of python.dll
+#endif
+
 #if defined(DYNAMIC_PYTHON3) || defined(PROTO)
 
 # ifndef MSWIN
 #  include <dlfcn.h>
 #  define FARPROC void*
-#  define HINSTANCE void*
 #  if defined(PY_NO_RTLD_GLOBAL) && defined(PY3_NO_RTLD_GLOBAL)
 #   define load_dll(n) dlopen((n), RTLD_LAZY)
 #  else
@@ -459,8 +465,6 @@ static void(*py3_PyObject_GC_Del)(void *);
 static void(*py3_PyObject_GC_UnTrack)(void *);
 static int (*py3_PyType_IsSubtype)(PyTypeObject *, PyTypeObject *);
 
-static HINSTANCE hinstPy3 = 0; // Instance of python.dll
-
 // Imported exception objects
 static PyObject *p3imp_PyExc_AttributeError;
 static PyObject *p3imp_PyExc_IndexError;
@@ -1032,13 +1036,8 @@ reset_stdin(void)
 {
     FILE *(*py__acrt_iob_func)(unsigned) = NULL;
     FILE *(*pyfreopen)(const char *, const char *, FILE *) = NULL;
-    HINSTANCE hinst;
+    HINSTANCE hinst = hinstPy3;
 
-# ifdef DYNAMIC_PYTHON3
-    hinst = hinstPy3;
-# else
-    hinst = GetModuleHandle(PYTHON3_DLL);
-# endif
     if (hinst == NULL || is_stdin_readable())
 	return;
 
@@ -1063,6 +1062,57 @@ reset_stdin(void)
 # define reset_stdin()
 #endif
 
+// Python 3.2 or later will abort inside Py_Initialize() when mandatory
+// modules cannot be loaded (e.g. 'pythonthreehome' is wrongly set.).
+// Install a hook to python dll's exit() and recover from it.
+#if defined(MSWIN) && (PY_VERSION_HEX >= 0x030200f0)
+# define HOOK_EXIT
+# include <setjmp.h>
+
+static jmp_buf exit_hook_jump_buf;
+static void *orig_exit = NULL;
+
+/*
+ * Function that replaces exit() while calling Py_Initialize().
+ */
+    static void
+hooked_exit(int ret)
+{
+    // Recover from exit.
+    longjmp(exit_hook_jump_buf, 1);
+}
+
+/*
+ * Install a hook to python dll's exit().
+ */
+    static void
+hook_py_exit(void)
+{
+    HINSTANCE hinst = hinstPy3;
+
+    if (hinst == NULL || orig_exit != NULL)
+	return;
+
+    orig_exit = hook_dll_import_func(hinst, "exit", (void *)hooked_exit);
+}
+
+/*
+ * Remove the hook installed by hook_py_exit().
+ */
+    static void
+restore_py_exit(void)
+{
+    HINSTANCE hinst = hinstPy3;
+
+    if (hinst == NULL)
+	return;
+
+    if (orig_exit != NULL)
+	hook_dll_import_func(hinst, "exit", orig_exit);
+    orig_exit = NULL;
+}
+#endif
+
     static int
 Python3_Init(void)
 {
@@ -1095,8 +1145,31 @@ Python3_Init(void)
 
 	PyImport_AppendInittab("vim", Py3Init_vim);
 
+#if !defined(DYNAMIC_PYTHON3) && defined(MSWIN)
+	hinstPy3 = GetModuleHandle(PYTHON3_DLL);
+#endif
 	reset_stdin();
-	Py_Initialize();
+
+#ifdef HOOK_EXIT
+	// Catch exit() called in Py_Initialize().
+	hook_py_exit();
+	if (setjmp(exit_hook_jump_buf) == 0)
+#endif
+	{
+	    Py_Initialize();
+#ifdef HOOK_EXIT
+	    restore_py_exit();
+#endif
+	}
+#ifdef HOOK_EXIT
+	else
+	{
+	    // exit() was called in Py_Initialize().
+	    restore_py_exit();
+	    emsg(_(e_critical_error_in_python3_initialization_check_your_installation));
+	    goto fail;
+	}
+#endif
 
 #if PY_VERSION_HEX < 0x03090000
 	// Initialise threads.  This is deprecated since Python 3.9.
diff --git a/src/os_win32.c b/src/os_win32.c
index a01cee5b8..682fdf2bc 100644
--- a/src/os_win32.c
+++ b/src/os_win32.c
@@ -572,14 +572,18 @@ mch_is_gui_executable(void)
 }
 #endif
 
-#if defined(DYNAMIC_ICONV) || defined(DYNAMIC_GETTEXT) || defined(PROTO)
+#if defined(DYNAMIC_ICONV) || defined(DYNAMIC_GETTEXT) \
+    || defined(FEAT_PYTHON3) || defined(PROTO)
 /*
  * Get related information about 'funcname' which is imported by 'hInst'.
  * If 'info' is 0, return the function address.
  * If 'info' is 1, return the module name which the function is imported from.
+ * If 'info' is 2, hook the function with 'ptr', and return the original
+ * function address.
  */
     static void *
-get_imported_func_info(HINSTANCE hInst, const char *funcname, int info)
+get_imported_func_info(HINSTANCE hInst, const char *funcname, int info,
+	const void *ptr)
 {
     PBYTE			pImage = (PBYTE)hInst;
     PIMAGE_DOS_HEADER		pDOS = (PIMAGE_DOS_HEADER)hInst;
@@ -611,12 +615,23 @@ get_imported_func_info(HINSTANCE hInst, const char *funcname, int info)
 					+ (UINT_PTR)(pINT->u1.AddressOfData));
 	    if (strcmp((char *)pImpName->Name, funcname) == 0)
 	    {
+		void *original;
+		DWORD old, new = PAGE_READWRITE;
+
 		switch (info)
 		{
 		    case 0:
 			return (void *)pIAT->u1.Function;
 		    case 1:
 			return (void *)(pImage + pImpDesc->Name);
+		    case 2:
+			original = (void *)pIAT->u1.Function;
+			VirtualProtect(&pIAT->u1.Function, sizeof(void *),
+				new, &old);
+			pIAT->u1.Function = (UINT_PTR)ptr;
+			VirtualProtect(&pIAT->u1.Function, sizeof(void *),
+				old, &new);
+			return original;
 		    default:
 			return NULL;
 		}
@@ -634,7 +649,7 @@ find_imported_module_by_funcname(HINSTANCE hInst, const char *funcname)
 {
     char    *modulename;
 
-    modulename = (char *)get_imported_func_info(hInst, funcname, 1);
+    modulename = (char *)get_imported_func_info(hInst, funcname, 1, NULL);
     if (modulename != NULL)
 	return GetModuleHandleA(modulename);
     return NULL;
@@ -646,7 +661,17 @@ find_imported_module_by_funcname(HINSTANCE hInst, const char *funcname)
     void *
 get_dll_import_func(HINSTANCE hInst, const char *funcname)
 {
-    return get_imported_func_info(hInst, funcname, 0);
+    return get_imported_func_info(hInst, funcname, 0, NULL);
+}
+
+/*
+ * Hook the function named 'funcname' which is imported by 'hInst' DLL,
+ * and return the original function address.
+ */
+    void *
+hook_dll_import_func(HINSTANCE hInst, const char *funcname, const void *hook)
+{
+    return get_imported_func_info(hInst, funcname, 2, hook);
 }
 #endif
 
diff --git a/src/proto/os_win32.pro b/src/proto/os_win32.pro
index 5c50816c6..dac2b7142 100644
--- a/src/proto/os_win32.pro
+++ b/src/proto/os_win32.pro
@@ -3,6 +3,7 @@ HINSTANCE vimLoadLib(char *name);
 int mch_is_gui_executable(void);
 HINSTANCE find_imported_module_by_funcname(HINSTANCE hInst, const char *funcname);
 void *get_dll_import_func(HINSTANCE hInst, const char *funcname);
+void *hook_dll_import_func(HINSTANCE hInst, const char *funcname, const void *hook);
 int dyn_libintl_init(void);
 void dyn_libintl_end(void);
 void PlatformId(void);
diff --git a/src/version.c b/src/version.c
index 3caad5f83..956df653a 100644
--- a/src/version.c
+++ b/src/version.c
@@ -747,6 +747,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    4317,
+/**/
     4316,
 /**/
     4315,