patch 9.0.0850: MS-Windows Terminal has unstable color controlv9.0.0850

Problem:    MS-Windows Terminal has unstable color control.
Solution:   Do not try to read the old command prompt colortable, use modern
            VT sequences. (Christopher Plewright, closes #11450,
            closes #11373)

3 files changed, 114 insertions, 80 deletions

diff --git a/src/os_win32.c b/src/os_win32.c
index b5ded95d5..a9d48633d 100644
--- a/src/os_win32.c
+++ b/src/os_win32.c
@@ -204,21 +204,17 @@ static void vtp_sgr_bulks(int argc, int *argv);
 static int wt_working = 0;
 static void wt_init();
 
-static guicolor_T save_console_bg_rgb;
-static guicolor_T save_console_fg_rgb;
-static guicolor_T store_console_bg_rgb;
-static guicolor_T store_console_fg_rgb;
-
 static int g_color_index_bg = 0;
 static int g_color_index_fg = 7;
 
 # ifdef FEAT_TERMGUICOLORS
+static guicolor_T save_console_bg_rgb;
+static guicolor_T save_console_fg_rgb;
+static guicolor_T store_console_bg_rgb;
+static guicolor_T store_console_fg_rgb;
 static int default_console_color_bg = 0x000000; // black
 static int default_console_color_fg = 0xc0c0c0; // white
-# endif
-
-# ifdef FEAT_TERMGUICOLORS
-#  define USE_VTP		(vtp_working && is_term_win32() && (p_tgc || (!p_tgc && t_colors >= 256)))
+#  define USE_VTP		(vtp_working && is_term_win32())
 #  define USE_WT		(wt_working)
 # else
 #  define USE_VTP		0
@@ -334,7 +330,7 @@ read_console_input(
 
     if (s_dwMax == 0)
     {
-	if (!USE_WT && nLength == -1)
+	if (!vtp_working && nLength == -1)
 	    return PeekConsoleInputW(hInput, lpBuffer, 1, lpEvents);
 	GetNumberOfConsoleInputEvents(hInput, &dwEvents);
 	if (dwEvents == 0 && nLength == -1)
@@ -1255,7 +1251,6 @@ mch_bevalterm_changed(void)
     static void
 decode_mouse_wheel(MOUSE_EVENT_RECORD *pmer)
 {
-    win_T   *wp;
     int	    horizontal = (pmer->dwEventFlags == MOUSE_HWHEELED);
     int	    zDelta = pmer->dwButtonState;
 
@@ -1265,7 +1260,7 @@ decode_mouse_wheel(MOUSE_EVENT_RECORD *pmer)
 #ifdef FEAT_PROP_POPUP
     int lcol = g_xMouse;
     int lrow = g_yMouse;
-    wp = mouse_find_win(&lrow, &lcol, FIND_POPUP);
+    win_T *wp = mouse_find_win(&lrow, &lcol, FIND_POPUP);
     if (wp != NULL && popup_is_popup(wp))
     {
 	g_nMouseClick = -1;
@@ -1605,7 +1600,7 @@ decode_mouse_event(
     static void
 mch_set_cursor_shape(int thickness)
 {
-    if (USE_VTP || USE_WT)
+    if (vtp_working)
     {
 	if (*T_CSI == NUL)
 	{
@@ -2977,6 +2972,9 @@ mch_init_c(void)
 	create_conin();
     g_hConOut = GetStdHandle(STD_OUTPUT_HANDLE);
 
+    wt_init();
+    vtp_flag_init();
+    vtp_init();
 # ifdef FEAT_RESTORE_ORIG_SCREEN
     // Save the initial console buffer for later restoration
     SaveConsoleBuffer(&g_cbOrig);
@@ -3033,10 +3031,6 @@ mch_init_c(void)
 # ifdef FEAT_CLIPBOARD
     win_clip_init();
 # endif
-
-    vtp_flag_init();
-    vtp_init();
-    wt_init();
 }
 
 /*
@@ -5854,7 +5848,7 @@ termcap_mode_end(void)
 
     g_fTermcapMode = FALSE;
 }
-#endif // FEAT_GUI_MSWIN
+#endif // !FEAT_GUI_MSWIN || VIMDLL
 
 
 #if defined(FEAT_GUI_MSWIN) && !defined(VIMDLL)
@@ -5876,7 +5870,7 @@ clear_chars(
     COORD coord,
     DWORD n)
 {
-    if (!USE_VTP)
+    if (!vtp_working)
     {
 	DWORD dwDummy;
 
@@ -5901,7 +5895,7 @@ clear_screen(void)
 {
     g_coord.X = g_coord.Y = 0;
 
-    if (!USE_VTP)
+    if (!vtp_working)
 	clear_chars(g_coord, Rows * Columns);
     else
     {
@@ -5920,7 +5914,7 @@ clear_to_end_of_display(void)
 {
     COORD save = g_coord;
 
-    if (!USE_VTP)
+    if (!vtp_working)
 	clear_chars(g_coord, (Rows - g_coord.Y - 1)
 					   * Columns + (Columns - g_coord.X));
     else
@@ -5943,7 +5937,7 @@ clear_to_end_of_line(void)
 {
     COORD save = g_coord;
 
-    if (!USE_VTP)
+    if (!vtp_working)
 	clear_chars(g_coord, Columns - g_coord.X);
     else
     {
@@ -6045,7 +6039,11 @@ insert_lines(unsigned cLines)
     clip.Bottom = g_srScrollRegion.Bottom;
 
     fill.Char.AsciiChar = ' ';
-    if (!USE_VTP)
+    if (!(vtp_working
+#ifdef FEAT_TERMGUICOLORS
+		&& (p_tgc || t_colors >= 256)
+#endif
+	 ))
 	fill.Attributes = g_attrCurrent;
     else
 	fill.Attributes = g_attrDefault;
@@ -6074,7 +6072,7 @@ insert_lines(unsigned cLines)
 	}
     }
 
-    if (USE_WT)
+    if (vtp_working)
     {
 	COORD coord;
 	int i;
@@ -6116,7 +6114,7 @@ delete_lines(unsigned cLines)
     clip.Bottom = g_srScrollRegion.Bottom;
 
     fill.Char.AsciiChar = ' ';
-    if (!USE_VTP)
+    if (!vtp_working)
 	fill.Attributes = g_attrCurrent;
     else
 	fill.Attributes = g_attrDefault;
@@ -6143,7 +6141,7 @@ delete_lines(unsigned cLines)
 	}
     }
 
-    if (USE_WT)
+    if (vtp_working)
     {
 	COORD coord;
 	int i;
@@ -6169,7 +6167,11 @@ gotoxy(
     if (x < 1 || x > (unsigned)Columns || y < 1 || y > (unsigned)Rows)
 	return;
 
-    if (!USE_VTP)
+    if (!(vtp_working
+#ifdef FEAT_TERMGUICOLORS
+		&& (p_tgc || t_colors >= 256)
+#endif
+	 ))
     {
 	// There are reports of double-width characters not displayed
 	// correctly.  This workaround should fix it, similar to how it's done
@@ -6215,7 +6217,7 @@ textcolor(WORD wAttr)
 {
     g_attrCurrent = (g_attrCurrent & 0xf0) + (wAttr & 0x0f);
 
-    if (!USE_VTP)
+    if (!vtp_working)
 	SetConsoleTextAttribute(g_hConOut, g_attrCurrent);
     else
 	vtp_sgr_bulk(wAttr);
@@ -6227,7 +6229,7 @@ textbackground(WORD wAttr)
 {
     g_attrCurrent = (g_attrCurrent & 0x0f) + ((wAttr & 0x0f) << 4);
 
-    if (!USE_VTP)
+    if (!vtp_working)
 	SetConsoleTextAttribute(g_hConOut, g_attrCurrent);
     else
 	vtp_sgr_bulk(wAttr);
@@ -6240,7 +6242,7 @@ textbackground(WORD wAttr)
     static void
 normvideo(void)
 {
-    if (!USE_VTP)
+    if (!vtp_working)
 	textattr(g_attrDefault);
     else
 	vtp_sgr_bulk(0);
@@ -6326,7 +6328,7 @@ visual_bell(void)
 			       coordOrigin, &dwDummy);
 
     Sleep(15);	    // wait for 15 msec
-    if (!USE_VTP)
+    if (!vtp_working)
 	WriteConsoleOutputAttribute(g_hConOut, oldattrs, Rows * Columns,
 				coordOrigin, &dwDummy);
     vim_free(oldattrs);
@@ -6341,7 +6343,7 @@ cursor_visible(BOOL fVisible)
 {
     s_cursor_visible = fVisible;
 
-    if (USE_VTP)
+    if (vtp_working)
 	vtp_printf("\033[?25%c", fVisible ? 'h' : 'l');
 
 # ifdef MCH_CURSOR_SHAPE
@@ -6413,7 +6415,11 @@ write_chars(
 	}
     }
 
-    if (!USE_VTP)
+    if (!(vtp_working
+#ifdef FEAT_TERMGUICOLORS
+	    && (p_tgc || t_colors >= 256)
+#endif
+	))
     {
 	FillConsoleOutputAttribute(g_hConOut, g_attrCurrent, cells,
 				    coord, &written);
@@ -6453,7 +6459,11 @@ write_chars(
     }
 
     // Cursor under VTP is always in the correct position, no need to reset.
-    if (!USE_VTP)
+    if (!(vtp_working
+#ifdef FEAT_TERMGUICOLORS
+		&& (p_tgc || t_colors >= 256)
+#endif
+	 ))
 	gotoxy(g_coord.X + 1, g_coord.Y + 1);
 
     return written;
@@ -6749,12 +6759,16 @@ notsgr:
 			normvideo();
 		    else if (argc == 1)
 		    {
-			if (USE_VTP)
-			    textcolor((WORD) arg1);
+			if (vtp_working
+# ifdef FEAT_TERMGUICOLORS
+				&& (p_tgc || t_colors >= 256)
+# endif
+			   )
+			    textcolor((WORD)arg1);
 			else
-			    textattr((WORD) arg1);
+			    textattr((WORD)arg1);
 		    }
-		    else if (USE_VTP)
+		    else if (vtp_working)
 			vtp_sgr_bulks(argc, args);
 		}
 		else if (argc == 2 && *p == 'H')
@@ -6893,7 +6907,7 @@ notsgr:
 	    if (s[l] == ' ' && s[l + 1] == 'q')
 	    {
 		// DECSCUSR (cursor style) sequences
-		if (USE_VTP || USE_WT)
+		if (vtp_working)
 		    vtp_printf("%.*s", l + 2, s);   // Pass through
 		s += l + 2;
 		len -= l + 1;
@@ -7932,6 +7946,8 @@ mch_setenv(char *var, char *value, int x UNUSED)
  * Not stable now.
  */
 #define CONPTY_STABLE_BUILD	    MAKE_VER(10, 0, 32767)  // T.B.D.
+// Note: Windows 11 (build >= 22000 means Windows 11, even though the major
+// version says 10!)
 
     static void
 vtp_flag_init(void)
@@ -7978,29 +7994,27 @@ vtp_flag_init(void)
     static void
 vtp_init(void)
 {
-    CONSOLE_SCREEN_BUFFER_INFOEX csbi;
 # ifdef FEAT_TERMGUICOLORS
-    COLORREF fg;
-# endif
-
-    csbi.cbSize = sizeof(csbi);
-    GetConsoleScreenBufferInfoEx(g_hConOut, &csbi);
-    save_console_bg_rgb = (guicolor_T)csbi.ColorTable[g_color_index_bg];
-    save_console_fg_rgb = (guicolor_T)csbi.ColorTable[g_color_index_fg];
-    store_console_bg_rgb = save_console_bg_rgb;
-    store_console_fg_rgb = save_console_fg_rgb;
-
-# ifdef FEAT_TERMGUICOLORS
-    if (!USE_WT)
+    if (!vtp_working)
     {
+	CONSOLE_SCREEN_BUFFER_INFOEX csbi;
+	csbi.cbSize = sizeof(csbi);
+	GetConsoleScreenBufferInfoEx(g_hConOut, &csbi);
+	save_console_bg_rgb = (guicolor_T)csbi.ColorTable[g_color_index_bg];
+	save_console_fg_rgb = (guicolor_T)csbi.ColorTable[g_color_index_fg];
+	store_console_bg_rgb = save_console_bg_rgb;
+	store_console_fg_rgb = save_console_fg_rgb;
+
 	COLORREF bg;
 	bg = (COLORREF)csbi.ColorTable[g_color_index_bg];
 	bg = (GetRValue(bg) << 16) | (GetGValue(bg) << 8) | GetBValue(bg);
 	default_console_color_bg = bg;
+
+	COLORREF fg;
+	fg = (COLORREF)csbi.ColorTable[g_color_index_fg];
+	fg = (GetRValue(fg) << 16) | (GetGValue(fg) << 8) | GetBValue(fg);
+	default_console_color_fg = fg;
     }
-    fg = (COLORREF)csbi.ColorTable[g_color_index_fg];
-    fg = (GetRValue(fg) << 16) | (GetGValue(fg) << 8) | GetBValue(fg);
-    default_console_color_fg = fg;
 # endif
 
     set_console_color_rgb();
@@ -8217,32 +8231,35 @@ set_console_color_rgb(void)
     guicolor_T	fg, bg;
     int		ctermfg, ctermbg;
 
-    if (!USE_VTP)
+    if (!vtp_working)
 	return;
 
     get_default_console_color(&ctermfg, &ctermbg, &fg, &bg);
 
-    if (USE_WT)
+    if (p_tgc || t_colors >= 256)
     {
 	term_fg_rgb_color(fg);
 	term_bg_rgb_color(bg);
 	return;
     }
 
-    fg = (GetRValue(fg) << 16) | (GetGValue(fg) << 8) | GetBValue(fg);
-    bg = (GetRValue(bg) << 16) | (GetGValue(bg) << 8) | GetBValue(bg);
+    if (!conpty_working)
+    {
+	fg = (GetRValue(fg) << 16) | (GetGValue(fg) << 8) | GetBValue(fg);
+	bg = (GetRValue(bg) << 16) | (GetGValue(bg) << 8) | GetBValue(bg);
 
-    csbi.cbSize = sizeof(csbi);
-    GetConsoleScreenBufferInfoEx(g_hConOut, &csbi);
+	csbi.cbSize = sizeof(csbi);
+	GetConsoleScreenBufferInfoEx(g_hConOut, &csbi);
 
-    csbi.cbSize = sizeof(csbi);
-    csbi.srWindow.Right += 1;
-    csbi.srWindow.Bottom += 1;
-    store_console_bg_rgb = csbi.ColorTable[g_color_index_bg];
-    store_console_fg_rgb = csbi.ColorTable[g_color_index_fg];
-    csbi.ColorTable[g_color_index_bg] = (COLORREF)bg;
-    csbi.ColorTable[g_color_index_fg] = (COLORREF)fg;
-    SetConsoleScreenBufferInfoEx(g_hConOut, &csbi);
+	csbi.cbSize = sizeof(csbi);
+	csbi.srWindow.Right += 1;
+	csbi.srWindow.Bottom += 1;
+	store_console_bg_rgb = csbi.ColorTable[g_color_index_bg];
+	store_console_fg_rgb = csbi.ColorTable[g_color_index_fg];
+	csbi.ColorTable[g_color_index_bg] = (COLORREF)bg;
+	csbi.ColorTable[g_color_index_fg] = (COLORREF)fg;
+	SetConsoleScreenBufferInfoEx(g_hConOut, &csbi);
+    }
 # endif
 }
 
@@ -8259,6 +8276,7 @@ get_default_console_color(
     guicolor_T guibg = INVALCOLOR;
     int ctermfg = 0;
     int ctermbg = 0;
+    int dummynull = 0;
 
     id = syn_name2id((char_u *)"Normal");
     if (id > 0 && p_tgc)
@@ -8267,18 +8285,27 @@ get_default_console_color(
     {
 	ctermfg = -1;
 	if (id > 0)
-	    syn_id2cterm_bg(id, &ctermfg, &ctermbg);
-	guifg = ctermfg != -1 ? ctermtoxterm(ctermfg)
+	    syn_id2cterm_bg(id, &ctermfg, &dummynull);
+	if (vtp_working)
+	{
+	    cterm_normal_fg_gui_color = guifg =
+			    ctermfg != -1 ? ctermtoxterm(ctermfg) : INVALCOLOR;
+	    ctermfg = ctermfg < 0 ? 0 : ctermfg;
+	}
+	else
+	{
+	    guifg = ctermfg != -1 ? ctermtoxterm(ctermfg)
 						    : default_console_color_fg;
-	cterm_normal_fg_gui_color = guifg;
-	ctermfg = ctermfg < 0 ? 0 : ctermfg;
+	    cterm_normal_fg_gui_color = guifg;
+	    ctermfg = ctermfg < 0 ? 0 : ctermfg;
+	}
     }
     if (guibg == INVALCOLOR)
     {
 	ctermbg = -1;
 	if (id > 0)
-	    syn_id2cterm_bg(id, &ctermfg, &ctermbg);
-	if (USE_WT)
+	    syn_id2cterm_bg(id, &dummynull, &ctermbg);
+	if (vtp_working)
 	{
 	    cterm_normal_bg_gui_color = guibg =
 			    ctermbg != -1 ? ctermtoxterm(ctermbg) : INVALCOLOR;
@@ -8308,11 +8335,12 @@ get_default_console_color(
 reset_console_color_rgb(void)
 {
 # ifdef FEAT_TERMGUICOLORS
-    CONSOLE_SCREEN_BUFFER_INFOEX csbi;
 
-    if (USE_WT)
+    if (vtp_working)
 	return;
 
+    CONSOLE_SCREEN_BUFFER_INFOEX csbi;
+
     csbi.cbSize = sizeof(csbi);
     GetConsoleScreenBufferInfoEx(g_hConOut, &csbi);
 
@@ -8332,6 +8360,9 @@ reset_console_color_rgb(void)
 restore_console_color_rgb(void)
 {
 # ifdef FEAT_TERMGUICOLORS
+    if (vtp_working)
+	return;
+
     CONSOLE_SCREEN_BUFFER_INFOEX csbi;
 
     csbi.cbSize = sizeof(csbi);
@@ -8349,7 +8380,7 @@ restore_console_color_rgb(void)
     void
 control_console_color_rgb(void)
 {
-    if (USE_VTP)
+    if (vtp_working)
 	set_console_color_rgb();
     else
 	reset_console_color_rgb();
diff --git a/src/term.c b/src/term.c
index e4b6587cd..4d43139e9 100644
--- a/src/term.c
+++ b/src/term.c
@@ -2987,7 +2987,7 @@ term_rgb_color(char_u *s, guicolor_T rgb)
     vim_snprintf(buf, MAX_COLOR_STR_LEN,
 				  (char *)s, RED(rgb), GREEN(rgb), BLUE(rgb));
 #ifdef FEAT_VTP
-    if (use_wt())
+    if (has_vtp_working())
     {
 	out_flush();
 	buf[1] = '[';
@@ -3001,7 +3001,8 @@ term_rgb_color(char_u *s, guicolor_T rgb)
     void
 term_fg_rgb_color(guicolor_T rgb)
 {
-    term_rgb_color(T_8F, rgb);
+    if (rgb != INVALCOLOR)
+	term_rgb_color(T_8F, rgb);
 }
 
     void
diff --git a/src/version.c b/src/version.c
index 6884514cd..3433ebce0 100644
--- a/src/version.c
+++ b/src/version.c
@@ -696,6 +696,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    850,
+/**/
     849,
 /**/
     848,