patch 8.0.1783: cannot use 256 colors in a MS-Windows consolev8.0.1783

Problem:    Cannot use 256 colors in a MS-Windows console.
Solution:   Add 256 color support. (Nobuhiro Takasaki, closes #2821)

8 files changed, 238 insertions, 112 deletions

diff --git a/src/misc1.c b/src/misc1.c
index de79c8e81..6c77f4b0d 100644
--- a/src/misc1.c
+++ b/src/misc1.c
@@ -3723,7 +3723,7 @@ vim_beep(
 		    /* No restore color information, refresh the screen. */
 		    if (has_vtp_working() != 0
 # ifdef FEAT_TERMGUICOLORS
-			    && p_tgc
+			    && (p_tgc || (!p_tgc && t_colors >= 256))
 # endif
 			)
 		    {
diff --git a/src/option.c b/src/option.c
index 807b1e043..b4a0e6b85 100644
--- a/src/option.c
+++ b/src/option.c
@@ -6082,6 +6082,9 @@ did_set_string_option(
     int		redraw_gui_only = FALSE;
 #endif
     int		ft_changed = FALSE;
+#if defined(FEAT_VTP) && defined(FEAT_TERMGUICOLORS)
+    int		did_swaptcap = FALSE;
+#endif
 
     /* Get the global option to compare with, otherwise we would have to check
      * two values for all local options. */
@@ -6821,6 +6824,13 @@ did_set_string_option(
 			vim_free(T_CCO);
 		    T_CCO = empty_option;
 		}
+#if defined(FEAT_VTP) && defined(FEAT_TERMGUICOLORS)
+		if (is_term_win32())
+		{
+		    swap_tcap();
+		    did_swaptcap = TRUE;
+		}
+#endif
 		/* We now have a different color setup, initialize it again. */
 		init_highlight(TRUE, FALSE);
 	    }
@@ -7674,6 +7684,16 @@ did_set_string_option(
 #endif
 	check_redraw(options[opt_idx].flags);
 
+#if defined(FEAT_VTP) && defined(FEAT_TERMGUICOLORS)
+    if (did_swaptcap)
+    {
+	if (t_colors < 256)
+	    p_tgc = 0;
+	set_termname((char_u *)"win32");
+	init_highlight(TRUE, FALSE);
+    }
+#endif
+
     return errmsg;
 }
 
@@ -8716,7 +8736,8 @@ set_bool_option(
 	    p_tgc = 0;
 	    return (char_u*)N_("E954: 24-bit colors are not supported on this environment");
 	}
-	swap_tcap();
+	if (is_term_win32())
+	    swap_tcap();
 # endif
 # ifdef FEAT_GUI
 	if (!gui.in_use && !gui.starting)
@@ -8725,7 +8746,7 @@ set_bool_option(
 # ifdef FEAT_VTP
 	control_console_color_rgb();
 	/* reset t_Co */
-	if (STRCMP(T_NAME, "win32") == 0)
+	if (is_term_win32())
 	    set_termname(T_NAME);
 # endif
     }
diff --git a/src/os_win32.c b/src/os_win32.c
index 0cb311337..8278ddd02 100644
--- a/src/os_win32.c
+++ b/src/os_win32.c
@@ -214,7 +214,7 @@ static guicolor_T save_console_bg_rgb;
 static guicolor_T save_console_fg_rgb;
 
 # ifdef FEAT_TERMGUICOLORS
-#  define USE_VTP		(vtp_working && p_tgc)
+#  define USE_VTP		(vtp_working && is_term_win32() && (p_tgc || (!p_tgc && t_colors >= 256)))
 # else
 #  define USE_VTP		0
 # endif
@@ -2630,7 +2630,6 @@ mch_init(void)
 
     /* set termcap codes to current text attributes */
     update_tcap(g_attrCurrent);
-    swap_tcap();
 
     GetConsoleCursorInfo(g_hConOut, &g_cci);
     GetConsoleMode(g_hConIn,  &g_cmodein);
@@ -5763,7 +5762,11 @@ clear_chars(
     if (!USE_VTP)
 	FillConsoleOutputAttribute(g_hConOut, g_attrCurrent, n, coord, &dwDummy);
     else
-	FillConsoleOutputAttribute(g_hConOut, 0, n, coord, &dwDummy);
+    {
+	set_console_color_rgb();
+	gotoxy(coord.X + 1, coord.Y + 1);
+	vtp_printf("\033[%dX", n);
+    }
 }
 
 
@@ -7653,6 +7656,16 @@ vtp_sgr_bulks(
     vtp_printf((char *)buf);
 }
 
+    static int
+ctermtoxterm(
+    int cterm)
+{
+    uint8_t r, g, b, idx;
+
+    cterm_color2rgb(cterm, &r, &g, &b, &idx);
+    return (((int)r << 16) | ((int)g << 8) | (int)b);
+}
+
     static void
 set_console_color_rgb(void)
 {
@@ -7661,6 +7674,8 @@ set_console_color_rgb(void)
     int id;
     guicolor_T fg = INVALCOLOR;
     guicolor_T bg = INVALCOLOR;
+    int ctermfg;
+    int ctermbg;
 
     if (!USE_VTP)
 	return;
@@ -7669,9 +7684,19 @@ set_console_color_rgb(void)
     if (id > 0)
 	syn_id2colors(id, &fg, &bg);
     if (fg == INVALCOLOR)
-	fg = 0xc0c0c0;	    /* white text */
+    {
+	ctermfg = -1;
+	if (id > 0)
+	    syn_id2cterm_bg(id, &ctermfg, &ctermbg);
+	fg = ctermfg != -1 ? ctermtoxterm(ctermfg) : 0xc0c0c0; /* white */
+    }
     if (bg == INVALCOLOR)
-	bg = 0x000000;	    /* black background */
+    {
+	ctermbg = -1;
+	if (id > 0)
+	    syn_id2cterm_bg(id, &ctermfg, &ctermbg);
+	bg = ctermbg != -1 ? ctermtoxterm(ctermbg) : 0x000000; /* black */
+    }
     fg = (GetRValue(fg) << 16) | (GetGValue(fg) << 8) | GetBValue(fg);
     bg = (GetRValue(bg) << 16) | (GetGValue(bg) << 8) | GetBValue(bg);
 
@@ -7730,4 +7755,10 @@ use_vtp(void)
     return USE_VTP;
 }
 
+    int
+is_term_win32(void)
+{
+    return T_NAME != NULL && STRCMP(T_NAME, "win32") == 0;
+}
+
 #endif
diff --git a/src/proto/os_win32.pro b/src/proto/os_win32.pro
index 1bed98aaf..7f45c5cf1 100644
--- a/src/proto/os_win32.pro
+++ b/src/proto/os_win32.pro
@@ -72,4 +72,5 @@ int mch_setenv(char *var, char *value, int x);
 void control_console_color_rgb(void);
 int has_vtp_working(void);
 int use_vtp(void);
+int is_term_win32(void);
 /* vim: set ft=c : */
diff --git a/src/proto/term.pro b/src/proto/term.pro
index 42405dda5..333df92d6 100644
--- a/src/proto/term.pro
+++ b/src/proto/term.pro
@@ -76,4 +76,5 @@ void update_tcap(int attr);
 void swap_tcap(void);
 guicolor_T gui_get_color_cmn(char_u *name);
 guicolor_T gui_get_rgb_color_cmn(int r, int g, int b);
+void cterm_color2rgb(int nr, uint8_t *r, uint8_t *g, uint8_t *b, uint8_t *ansi_idx);
 /* vim: set ft=c : */
diff --git a/src/term.c b/src/term.c
index d11729cb4..d2a6e4583 100644
--- a/src/term.c
+++ b/src/term.c
@@ -2009,11 +2009,6 @@ set_termname(char_u *term)
     may_req_termresponse();
 #endif
 
-#if defined(WIN3264) && !defined(FEAT_GUI) && defined(FEAT_TERMGUICOLORS)
-    if (STRCMP(term, "win32") == 0)
-	set_color_count((p_tgc) ? 256 : 16);
-#endif
-
     return OK;
 }
 
@@ -2851,7 +2846,11 @@ term_color(char_u *s, int n)
     /* Also accept "\e[3%dm" for TERMINFO, it is sometimes used */
     /* Also accept CSI instead of <Esc>[ */
     if (n >= 8 && t_colors >= 16
-	      && ((s[0] == ESC && s[1] == '[') || (s[0] == CSI && (i = 1) == 1))
+	      && ((s[0] == ESC && s[1] == '[')
+#if defined(FEAT_VTP) && defined(FEAT_TERMGUICOLORS)
+		  || (s[0] == ESC && s[1] == '|')
+#endif
+	          || (s[0] == CSI && (i = 1) == 1))
 	      && s[i] != NUL
 	      && (STRCMP(s + i + 1, "%p1%dm") == 0
 		  || STRCMP(s + i + 1, "%dm") == 0)
@@ -2863,7 +2862,11 @@ term_color(char_u *s, int n)
 	char *format = "%s%s%%dm";
 #endif
 	sprintf(buf, format,
-		i == 2 ? IF_EB("\033[", ESC_STR "[") : "\233",
+		i == 2 ?
+#if defined(FEAT_VTP) && defined(FEAT_TERMGUICOLORS)
+		s[1] == '|' ? IF_EB("\033|", ESC_STR "|") :
+#endif
+		IF_EB("\033[", ESC_STR "[") : "\233",
 		s[i] == '3' ? (n >= 16 ? "38;5;" : "9")
 			    : (n >= 16 ? "48;5;" : "10"));
 	OUT_STR(tgoto(buf, 0, n >= 16 ? n : n - 8));
@@ -6640,26 +6643,38 @@ update_tcap(int attr)
 }
 
 # ifdef FEAT_TERMGUICOLORS
+#  define KSSIZE 20
 struct ks_tbl_s
 {
-    int  code;	    /* value of KS_ */
-    char *vtp;	    /* code in vtp mode */
-    char *buf;	    /* buffer in non-vtp mode */
-    char *vbuf;	    /* buffer in vtp mode */
+    int  code;		/* value of KS_ */
+    char *vtp;		/* code in vtp mode */
+    char *vtp2;		/* code in vtp2 mode */
+    char buf[KSSIZE];   /* save buffer in non-vtp mode */
+    char vbuf[KSSIZE];  /* save buffer in vtp mode */
+    char v2buf[KSSIZE]; /* save buffer in vtp2 mode */
+    char arr[KSSIZE];   /* real buffer */
 };
 
 static struct ks_tbl_s ks_tbl[] =
 {
-    {(int)KS_ME,  "\033|0m" },	/* normal */
-    {(int)KS_MR,  "\033|7m" },	/* reverse */
-    {(int)KS_MD,  "\033|1m" },	/* bold */
-    {(int)KS_SO,  "\033|91m"},	/* standout: bright red text */
-    {(int)KS_SE,  "\033|39m"},	/* standout end: default color */
-    {(int)KS_CZH, "\033|95m"},	/* italic: bright magenta text */
-    {(int)KS_CZR, "\033|0m",},	/* italic end */
-    {(int)KS_US,  "\033|4m",},	/* underscore */
-    {(int)KS_UE,  "\033|24m"},	/* underscore end */
-    {(int)KS_NAME, NULL}
+    {(int)KS_ME,  "\033|0m",  "\033|0m"},   /* normal */
+    {(int)KS_MR,  "\033|7m",  "\033|7m"},   /* reverse */
+    {(int)KS_MD,  "\033|1m",  "\033|1m"},   /* bold */
+    {(int)KS_SO,  "\033|91m", "\033|91m"},  /* standout: bright red text */
+    {(int)KS_SE,  "\033|39m", "\033|39m"},  /* standout end: default color */
+    {(int)KS_CZH, "\033|95m", "\033|95m"},  /* italic: bright magenta text */
+    {(int)KS_CZR, "\033|0m",  "\033|0m"},   /* italic end */
+    {(int)KS_US,  "\033|4m",  "\033|4m"},   /* underscore */
+    {(int)KS_UE,  "\033|24m", "\033|24m"},  /* underscore end */
+#  ifdef TERMINFO
+    {(int)KS_CAB, "\033|%p1%db", "\033|%p14%dm"}, /* set background color */
+    {(int)KS_CAF, "\033|%p1%df", "\033|%p13%dm"}, /* set foreground color */
+#  else
+    {(int)KS_CAB, "\033|%db", "\033|4%dm"}, /* set background color */
+    {(int)KS_CAF, "\033|%df", "\033|3%dm"}, /* set foreground color */
+#  endif
+    {(int)KS_CCO, "16", "256"},     /* colors */
+    {(int)KS_NAME}		    /* terminator */
 };
 
     static struct builtin_term *
@@ -6684,57 +6699,85 @@ swap_tcap(void)
 {
 # ifdef FEAT_TERMGUICOLORS
     static int		init_done = FALSE;
-    static int		last_tgc;
+    static int		curr_mode;
     struct ks_tbl_s	*ks;
     struct builtin_term *bt;
+    int			mode;
+    enum
+    {
+	CMODEINDEX,
+	CMODE24,
+	CMODE256
+    };
 
     /* buffer initialization */
     if (!init_done)
     {
-	for (ks = ks_tbl; ks->vtp != NULL; ks++)
+	for (ks = ks_tbl; ks->code != (int)KS_NAME; ks++)
 	{
 	    bt = find_first_tcap(DEFAULT_TERM, ks->code);
 	    if (bt != NULL)
 	    {
-		ks->buf = bt->bt_string;
-		ks->vbuf = ks->vtp;
+		STRNCPY(ks->buf, bt->bt_string, KSSIZE);
+		STRNCPY(ks->vbuf, ks->vtp, KSSIZE);
+		STRNCPY(ks->v2buf, ks->vtp2, KSSIZE);
+
+		STRNCPY(ks->arr, bt->bt_string, KSSIZE);
+		bt->bt_string = &ks->arr[0];
 	    }
 	}
 	init_done = TRUE;
-	last_tgc = p_tgc;
-	return;
+	curr_mode = CMODEINDEX;
     }
 
-    if (last_tgc != p_tgc)
+    if (p_tgc)
+	mode = CMODE24;
+    else if (t_colors >= 256)
+	mode = CMODE256;
+    else
+	mode = CMODEINDEX;
+
+    for (ks = ks_tbl; ks->code != (int)KS_NAME; ks++)
     {
-	if (p_tgc)
+	bt = find_first_tcap(DEFAULT_TERM, ks->code);
+	if (bt != NULL)
 	{
-	    /* switch to special character sequence */
-	    for (ks = ks_tbl; ks->vtp != NULL; ks++)
+	    switch (curr_mode)
 	    {
-		bt = find_first_tcap(DEFAULT_TERM, ks->code);
-		if (bt != NULL)
-		{
-		    ks->buf = bt->bt_string;
-		    bt->bt_string = ks->vbuf;
-		}
+	    case CMODEINDEX:
+		STRNCPY(&ks->buf[0], bt->bt_string, KSSIZE);
+		break;
+	    case CMODE24:
+		STRNCPY(&ks->vbuf[0], bt->bt_string, KSSIZE);
+		break;
+	    default:
+		STRNCPY(&ks->v2buf[0], bt->bt_string, KSSIZE);
 	    }
 	}
-	else
+    }
+
+    if (mode != curr_mode)
+    {
+	for (ks = ks_tbl; ks->code != (int)KS_NAME; ks++)
 	{
-	    /* switch to index color */
-	    for (ks = ks_tbl; ks->vtp != NULL; ks++)
+	    bt = find_first_tcap(DEFAULT_TERM, ks->code);
+	    if (bt != NULL)
 	    {
-		bt = find_first_tcap(DEFAULT_TERM, ks->code);
-		if (bt != NULL)
+		switch (mode)
 		{
-		    ks->vbuf = bt->bt_string;
-		    bt->bt_string = ks->buf;
+		case CMODEINDEX:
+		    STRNCPY(bt->bt_string, &ks->buf[0], KSSIZE);
+		    break;
+		case CMODE24:
+		    STRNCPY(bt->bt_string, &ks->vbuf[0], KSSIZE);
+		    break;
+		default:
+		    STRNCPY(bt->bt_string, &ks->v2buf[0], KSSIZE);
 		}
 	    }
 	}
 
-	last_tgc = p_tgc;
+	curr_mode = mode;
     }
 # endif
 }
@@ -6920,3 +6963,79 @@ gui_get_rgb_color_cmn(int r, int g, int b)
     return color;
 }
 #endif
+
+#if (defined(WIN3264) && !defined(FEAT_GUI_W32)) || defined(FEAT_TERMINAL) \
+	|| defined(PROTO)
+static int cube_value[] = {
+    0x00, 0x5F, 0x87, 0xAF, 0xD7, 0xFF
+};
+
+static int grey_ramp[] = {
+    0x08, 0x12, 0x1C, 0x26, 0x30, 0x3A, 0x44, 0x4E, 0x58, 0x62, 0x6C, 0x76,
+    0x80, 0x8A, 0x94, 0x9E, 0xA8, 0xB2, 0xBC, 0xC6, 0xD0, 0xDA, 0xE4, 0xEE
+};
+
+# ifdef FEAT_TERMINAL
+#  include "libvterm/include/vterm.h"  // for VTERM_ANSI_INDEX_NONE
+# endif
+
+static uint8_t ansi_table[16][4] = {
+//   R    G    B   idx
+  {  0,   0,   0,  1}, // black
+  {224,   0,   0,  2}, // dark red
+  {  0, 224,   0,  3}, // dark green
+  {224, 224,   0,  4}, // dark yellow / brown
+  {  0,   0, 224,  5}, // dark blue
+  {224,   0, 224,  6}, // dark magenta
+  {  0, 224, 224,  7}, // dark cyan
+  {224, 224, 224,  8}, // light grey
+
+  {128, 128, 128,  9}, // dark grey
+  {255,  64,  64, 10}, // light red
+  { 64, 255,  64, 11}, // light green
+  {255, 255,  64, 12}, // yellow
+  { 64,  64, 255, 13}, // light blue
+  {255,  64, 255, 14}, // light magenta
+  { 64, 255, 255, 15}, // light cyan
+  {255, 255, 255, 16}, // white
+};
+
+    void
+cterm_color2rgb(int nr, uint8_t *r, uint8_t *g, uint8_t *b, uint8_t *ansi_idx)
+{
+    int idx;
+
+    if (nr < 16)
+    {
+	*r = ansi_table[nr][0];
+	*g = ansi_table[nr][1];
+	*b = ansi_table[nr][2];
+	*ansi_idx = ansi_table[nr][3];
+    }
+    else if (nr < 232)
+    {
+	/* 216 color cube */
+	idx = nr - 16;
+	*r = cube_value[idx / 36 % 6];
+	*g = cube_value[idx / 6  % 6];
+	*b = cube_value[idx      % 6];
+	*ansi_idx = VTERM_ANSI_INDEX_NONE;
+    }
+    else if (nr < 256)
+    {
+	/* 24 grey scale ramp */
+	idx = nr - 232;
+	*r = grey_ramp[idx];
+	*g = grey_ramp[idx];
+	*b = grey_ramp[idx];
+	*ansi_idx = VTERM_ANSI_INDEX_NONE;
+    }
+    else
+    {
+	*r = 0;
+	*g = 0;
+	*b = 0;
+	*ansi_idx = 0;
+    }
+}
+#endif
diff --git a/src/terminal.c b/src/terminal.c
index 7afc49dc1..9b336ee87 100644
--- a/src/terminal.c
+++ b/src/terminal.c
@@ -43,7 +43,6 @@
  * - Win32: Redirecting output does not work, Test_terminal_redir_file()
  *   is disabled.
  * - Add test for 'termwinkey'.
- * - libvterm: bringg back using // comments and trailing comma in enum
  * - When starting terminal window with shell in terminal, then using :gui to
  *   switch to GUI, shell stops working. Scrollback seems wrong, command
  *   running in shell is still running.
@@ -3016,66 +3015,14 @@ term_get_attr(buf_T *buf, linenr_T lnum, int col)
     return cell2attr(cellattr->attrs, cellattr->fg, cellattr->bg);
 }
 
-static VTermColor ansi_table[16] = {
-  {  0,   0,   0,  1}, /* black */
-  {224,   0,   0,  2}, /* dark red */
-  {  0, 224,   0,  3}, /* dark green */
-  {224, 224,   0,  4}, /* dark yellow / brown */
-  {  0,   0, 224,  5}, /* dark blue */
-  {224,   0, 224,  6}, /* dark magenta */
-  {  0, 224, 224,  7}, /* dark cyan */
-  {224, 224, 224,  8}, /* light grey */
-
-  {128, 128, 128,  9}, /* dark grey */
-  {255,  64,  64, 10}, /* light red */
-  { 64, 255,  64, 11}, /* light green */
-  {255, 255,  64, 12}, /* yellow */
-  { 64,  64, 255, 13}, /* light blue */
-  {255,  64, 255, 14}, /* light magenta */
-  { 64, 255, 255, 15}, /* light cyan */
-  {255, 255, 255, 16}, /* white */
-};
-
-static int cube_value[] = {
-    0x00, 0x5F, 0x87, 0xAF, 0xD7, 0xFF
-};
-
-static int grey_ramp[] = {
-    0x08, 0x12, 0x1C, 0x26, 0x30, 0x3A, 0x44, 0x4E, 0x58, 0x62, 0x6C, 0x76,
-    0x80, 0x8A, 0x94, 0x9E, 0xA8, 0xB2, 0xBC, 0xC6, 0xD0, 0xDA, 0xE4, 0xEE
-};
-
 /*
  * Convert a cterm color number 0 - 255 to RGB.
  * This is compatible with xterm.
  */
     static void
-cterm_color2rgb(int nr, VTermColor *rgb)
+cterm_color2vterm(int nr, VTermColor *rgb)
 {
-    int idx;
-
-    if (nr < 16)
-    {
-	*rgb = ansi_table[nr];
-    }
-    else if (nr < 232)
-    {
-	/* 216 color cube */
-	idx = nr - 16;
-	rgb->blue  = cube_value[idx      % 6];
-	rgb->green = cube_value[idx / 6  % 6];
-	rgb->red   = cube_value[idx / 36 % 6];
-	rgb->ansi_index = VTERM_ANSI_INDEX_NONE;
-    }
-    else if (nr < 256)
-    {
-	/* 24 grey scale ramp */
-	idx = nr - 232;
-	rgb->blue  = grey_ramp[idx];
-	rgb->green = grey_ramp[idx];
-	rgb->red   = grey_ramp[idx];
-	rgb->ansi_index = VTERM_ANSI_INDEX_NONE;
-    }
+    cterm_color2rgb(nr, &rgb->red, &rgb->green, &rgb->blue, &rgb->ansi_index);
 }
 
 /*
@@ -3120,6 +3067,10 @@ init_default_colors(term_T *term)
 # endif
 # ifdef FEAT_TERMGUICOLORS
 	    || p_tgc
+#  ifdef FEAT_VTP
+	    /* Finally get INVALCOLOR on this execution path */
+	    || (!p_tgc && t_colors >= 256)
+#  endif
 # endif
        )
     {
@@ -3171,9 +3122,9 @@ init_default_colors(term_T *term)
     if (id != 0 && t_colors >= 16)
     {
 	if (term_default_cterm_fg >= 0)
-	    cterm_color2rgb(term_default_cterm_fg, fg);
+	    cterm_color2vterm(term_default_cterm_fg, fg);
 	if (term_default_cterm_bg >= 0)
-	    cterm_color2rgb(term_default_cterm_bg, bg);
+	    cterm_color2vterm(term_default_cterm_bg, bg);
     }
     else
     {
@@ -3184,7 +3135,7 @@ init_default_colors(term_T *term)
 	/* In an MS-Windows console we know the normal colors. */
 	if (cterm_normal_fg_color > 0)
 	{
-	    cterm_color2rgb(cterm_normal_fg_color - 1, fg);
+	    cterm_color2vterm(cterm_normal_fg_color - 1, fg);
 # if defined(WIN3264) && !defined(FEAT_GUI_W32)
 	    tmp = fg->red;
 	    fg->red = fg->blue;
@@ -3198,7 +3149,7 @@ init_default_colors(term_T *term)
 
 	if (cterm_normal_bg_color > 0)
 	{
-	    cterm_color2rgb(cterm_normal_bg_color - 1, bg);
+	    cterm_color2vterm(cterm_normal_bg_color - 1, bg);
 # if defined(WIN3264) && !defined(FEAT_GUI_W32)
 	    tmp = bg->red;
 	    bg->red = bg->blue;
diff --git a/src/version.c b/src/version.c
index a02360ff7..76a4f2c37 100644
--- a/src/version.c
+++ b/src/version.c
@@ -762,6 +762,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    1783,
+/**/
     1782,
 /**/
     1781,