patch 8.0.1359: libvterm ANSI colors can not always be recognizedv8.0.1359

Problem:    Libvterm ANSI colors can not always be recognized from the RGB
            values. The default color is wrong when t_RB is empty.
Solution:   Add the ANSI color index to VTermColor.

4 files changed, 83 insertions, 105 deletions

diff --git a/src/libvterm/include/vterm.h b/src/libvterm/include/vterm.h
index 50a840a31..f731dcb8b 100644
--- a/src/libvterm/include/vterm.h
+++ b/src/libvterm/include/vterm.h
@@ -79,8 +79,16 @@ INLINE void vterm_rect_move(VTermRect *rect, int row_delta, int col_delta)
 }
 #endif
 
+/* The ansi_index is used for the lower 16 colors, which can be set to any
+ * color. */
+#define VTERM_ANSI_INDEX_DEFAULT 0	/* color cleared */
+#define VTERM_ANSI_INDEX_MIN 1
+#define VTERM_ANSI_INDEX_MAX 16
+#define VTERM_ANSI_INDEX_NONE 255	/* non-ANSI color, use red/green/blue */
+
 typedef struct {
   uint8_t red, green, blue;
+  uint8_t ansi_index;
 } VTermColor;
 
 typedef enum {
diff --git a/src/libvterm/src/pen.c b/src/libvterm/src/pen.c
index c533096a4..e9a6de9bd 100644
--- a/src/libvterm/src/pen.c
+++ b/src/libvterm/src/pen.c
@@ -3,25 +3,25 @@
 #include <stdio.h>
 
 static const VTermColor ansi_colors[] = {
-  /* R    G    B */
-  {   0,   0,   0 }, /* black */
-  { 224,   0,   0 }, /* red */
-  {   0, 224,   0 }, /* green */
-  { 224, 224,   0 }, /* yellow */
-  {   0,   0, 224 }, /* blue */
-  { 224,   0, 224 }, /* magenta */
-  {   0, 224, 224 }, /* cyan */
-  { 224, 224, 224 }, /* white == light grey */
+  /* R    G    B   index */
+  {   0,   0,   0,  1 }, /* black */
+  { 224,   0,   0,  2 }, /* red */
+  {   0, 224,   0,  3 }, /* green */
+  { 224, 224,   0,  4 }, /* yellow */
+  {   0,   0, 224,  5 }, /* blue */
+  { 224,   0, 224,  6 }, /* magenta */
+  {   0, 224, 224,  7 }, /* cyan */
+  { 224, 224, 224,  8 }, /* white == light grey */
 
   /* high intensity */
-  { 128, 128, 128 }, /* black */
-  { 255,  64,  64 }, /* red */
-  {  64, 255,  64 }, /* green */
-  { 255, 255,  64 }, /* yellow */
-  {  64,  64, 255 }, /* blue */
-  { 255,  64, 255 }, /* magenta */
-  {  64, 255, 255 }, /* cyan */
-  { 255, 255, 255 }, /* white for real */
+  { 128, 128, 128,  9 }, /* black */
+  { 255,  64,  64, 10 }, /* red */
+  {  64, 255,  64, 11 }, /* green */
+  { 255, 255,  64, 12 }, /* yellow */
+  {  64,  64, 255, 13 }, /* blue */
+  { 255,  64, 255, 14 }, /* magenta */
+  {  64, 255, 255, 15 }, /* cyan */
+  { 255, 255, 255, 16 }, /* white for real */
 };
 
 static int ramp6[] = {
@@ -57,6 +57,7 @@ static int lookup_colour_palette(const VTermState *state, long index, VTermColor
     col->blue  = ramp6[index     % 6];
     col->green = ramp6[index/6   % 6];
     col->red   = ramp6[index/6/6 % 6];
+    col->ansi_index = VTERM_ANSI_INDEX_NONE;
 
     return TRUE;
   }
@@ -67,6 +68,7 @@ static int lookup_colour_palette(const VTermState *state, long index, VTermColor
     col->blue  = ramp24[index];
     col->green = ramp24[index];
     col->red   = ramp24[index];
+    col->ansi_index = VTERM_ANSI_INDEX_NONE;
 
     return TRUE;
   }
@@ -84,6 +86,7 @@ static int lookup_colour(const VTermState *state, int palette, const long args[]
     col->red   = (uint8_t)CSI_ARG(args[0]);
     col->green = (uint8_t)CSI_ARG(args[1]);
     col->blue  = (uint8_t)CSI_ARG(args[2]);
+    col->ansi_index = VTERM_ANSI_INDEX_NONE;
 
     return 3;
 
@@ -152,7 +155,9 @@ INTERNAL void vterm_state_newpen(VTermState *state)
 
   /* 90% grey so that pure white is brighter */
   state->default_fg.red = state->default_fg.green = state->default_fg.blue = 240;
+  state->default_fg.ansi_index = VTERM_ANSI_INDEX_DEFAULT;
   state->default_bg.red = state->default_bg.green = state->default_bg.blue = 0;
+  state->default_bg.ansi_index = VTERM_ANSI_INDEX_DEFAULT;
 
   for(col = 0; col < 16; col++)
     state->colors[col] = ansi_colors[col];
@@ -208,13 +213,18 @@ void vterm_state_get_palette_color(const VTermState *state, int index, VTermColo
 void vterm_state_set_default_colors(VTermState *state, const VTermColor *default_fg, const VTermColor *default_bg)
 {
   state->default_fg = *default_fg;
+  state->default_fg.ansi_index = VTERM_ANSI_INDEX_DEFAULT;
   state->default_bg = *default_bg;
+  state->default_bg.ansi_index = VTERM_ANSI_INDEX_DEFAULT;
 }
 
 void vterm_state_set_palette_color(VTermState *state, int index, const VTermColor *col)
 {
   if(index >= 0 && index < 16)
+  {
     state->colors[index] = *col;
+    state->colors[index].ansi_index = index + VTERM_ANSI_INDEX_MIN;
+  }
 }
 
 void vterm_state_set_bold_highbright(VTermState *state, int bold_is_highbright)
diff --git a/src/terminal.c b/src/terminal.c
index 716d0b53b..2c3197473 100644
--- a/src/terminal.c
+++ b/src/terminal.c
@@ -38,6 +38,8 @@
  * in tl_scrollback are no longer used.
  *
  * TODO:
+ * - When using 'termguicolors' still use the 16 ANSI colors as-is.  Helps for
+ *   a job that uses 16 colors while Vim is using > 256.
  * - in GUI vertical split causes problems.  Cursor is flickering. (Hirohito
  *   Higashi, 2017 Sep 19)
  * - Shift-Tab does not work.
@@ -48,7 +50,6 @@
  * - When closing gvim with an active terminal buffer, the dialog suggests
  *   saving the buffer.  Should say something else. (Manas Thakur, #2215)
  *   Also: #2223
- * - implement term_setsize()
  * - Termdebug does not work when Vim build with mzscheme.  gdb hangs.
  * - MS-Windows GUI: WinBar has  tearoff item
  * - Adding WinBar to terminal window doesn't display, text isn't shifted down.
@@ -57,6 +58,7 @@
  * - What to store in a session file?  Shell at the prompt would be OK to
  *   restore, but others may not.  Open the window and let the user start the
  *   command?
+ * - implement term_setsize()
  * - add test for giving error for invalid 'termsize' value.
  * - support minimal size when 'termsize' is "rows*cols".
  * - support minimal size when 'termsize' is empty?
@@ -1707,7 +1709,7 @@ may_toggle_cursor(term_T *term)
 
 /*
  * Reverse engineer the RGB value into a cterm color index.
- * First color is 1.  Return 0 if no match found.
+ * First color is 1.  Return 0 if no match found (default color).
  */
     static int
 color2index(VTermColor *color, int fg, int *boldp)
@@ -1716,78 +1718,34 @@ color2index(VTermColor *color, int fg, int *boldp)
     int blue = color->blue;
     int green = color->green;
 
-    /* The argument for lookup_color() is for the color_names[] table. */
-    if (red == 0)
+    if (color->ansi_index != VTERM_ANSI_INDEX_NONE)
     {
-	if (green == 0)
-	{
-	    if (blue == 0)
-		return lookup_color(0, fg, boldp) + 1; /* black */
-	    if (blue == 224)
-		return lookup_color(1, fg, boldp) + 1; /* dark blue */
-	}
-	else if (green == 224)
+	/* First 16 colors and default: use the ANSI index, because these
+	 * colors can be redefined. */
+	if (t_colors >= 16)
+	    return color->ansi_index;
+	switch (color->ansi_index)
 	{
-	    if (blue == 0)
-		return lookup_color(2, fg, boldp) + 1; /* dark green */
-	    if (blue == 224)
-		return lookup_color(3, fg, boldp) + 1; /* dark cyan */
-	}
-    }
-    else if (red == 224)
-    {
-	if (green == 0)
-	{
-	    if (blue == 0)
-		return lookup_color(4, fg, boldp) + 1; /* dark red */
-	    if (blue == 224)
-		return lookup_color(5, fg, boldp) + 1; /* dark magenta */
-	}
-	else if (green == 224)
-	{
-	    if (blue == 0)
-		return lookup_color(6, fg, boldp) + 1; /* dark yellow / brown */
-	    if (blue == 224)
-		return lookup_color(8, fg, boldp) + 1; /* white / light grey */
-	}
-    }
-    else if (red == 128)
-    {
-	if (green == 128 && blue == 128)
-	    return lookup_color(12, fg, boldp) + 1; /* dark grey */
-    }
-    else if (red == 255)
-    {
-	if (green == 64)
-	{
-	    if (blue == 64)
-		return lookup_color(20, fg, boldp) + 1;  /* light red */
-	    if (blue == 255)
-		return lookup_color(22, fg, boldp) + 1;  /* light magenta */
-	}
-	else if (green == 255)
-	{
-	    if (blue == 64)
-		return lookup_color(24, fg, boldp) + 1;  /* yellow */
-	    if (blue == 255)
-		return lookup_color(26, fg, boldp) + 1;  /* white */
-	}
-    }
-    else if (red == 64)
-    {
-	if (green == 64)
-	{
-	    if (blue == 255)
-		return lookup_color(14, fg, boldp) + 1;  /* light blue */
-	}
-	else if (green == 255)
-	{
-	    if (blue == 64)
-		return lookup_color(16, fg, boldp) + 1;  /* light green */
-	    if (blue == 255)
-		return lookup_color(18, fg, boldp) + 1;  /* light cyan */
+	    case  0: return 0;
+	    case  1: return lookup_color( 0, fg, boldp) + 1;
+	    case  2: return lookup_color( 4, fg, boldp) + 1; /* dark red */
+	    case  3: return lookup_color( 2, fg, boldp) + 1; /* dark green */
+	    case  4: return lookup_color( 6, fg, boldp) + 1; /* brown */
+	    case  5: return lookup_color( 1, fg, boldp) + 1; /* dark blue*/
+	    case  6: return lookup_color( 5, fg, boldp) + 1; /* dark magenta */
+	    case  7: return lookup_color( 3, fg, boldp) + 1; /* dark cyan */
+	    case  8: return lookup_color( 8, fg, boldp) + 1; /* light grey */
+	    case  9: return lookup_color(12, fg, boldp) + 1; /* dark grey */
+	    case 10: return lookup_color(20, fg, boldp) + 1; /* red */
+	    case 11: return lookup_color(16, fg, boldp) + 1; /* green */
+	    case 12: return lookup_color(24, fg, boldp) + 1; /* yellow */
+	    case 13: return lookup_color(14, fg, boldp) + 1; /* blue */
+	    case 14: return lookup_color(22, fg, boldp) + 1; /* magenta */
+	    case 15: return lookup_color(18, fg, boldp) + 1; /* cyan */
+	    case 16: return lookup_color(26, fg, boldp) + 1; /* white */
 	}
     }
+
     if (t_colors >= 256)
     {
 	if (red == blue && red == green)
@@ -2447,23 +2405,23 @@ term_get_attr(buf_T *buf, linenr_T lnum, int col)
 }
 
 static VTermColor ansi_table[16] = {
-  {  0,   0,   0}, /* black */
-  {224,   0,   0}, /* dark red */
-  {  0, 224,   0}, /* dark green */
-  {224, 224,   0}, /* dark yellow / brown */
-  {  0,   0, 224}, /* dark blue */
-  {224,   0, 224}, /* dark magenta */
-  {  0, 224, 224}, /* dark cyan */
-  {224, 224, 224}, /* light grey */
-
-  {128, 128, 128}, /* dark grey */
-  {255,  64,  64}, /* light red */
-  { 64, 255,  64}, /* light green */
-  {255, 255,  64}, /* yellow */
-  { 64,  64, 255}, /* light blue */
-  {255,  64, 255}, /* light magenta */
-  { 64, 255, 255}, /* light cyan */
-  {255, 255, 255}, /* white */
+  {  0,   0,   0,  1}, /* black */
+  {224,   0,   0,  2}, /* dark red */
+  {  0, 224,   0,  3}, /* dark green */
+  {224, 224,   0,  4}, /* dark yellow / brown */
+  {  0,   0, 224,  5}, /* dark blue */
+  {224,   0, 224,  6}, /* dark magenta */
+  {  0, 224, 224,  7}, /* dark cyan */
+  {224, 224, 224,  8}, /* light grey */
+
+  {128, 128, 128,  9}, /* dark grey */
+  {255,  64,  64, 10}, /* light red */
+  { 64, 255,  64, 11}, /* light green */
+  {255, 255,  64, 12}, /* yellow */
+  { 64,  64, 255, 13}, /* light blue */
+  {255,  64, 255, 14}, /* light magenta */
+  { 64, 255, 255, 15}, /* light cyan */
+  {255, 255, 255, 16}, /* white */
 };
 
 static int cube_value[] = {
@@ -2549,7 +2507,7 @@ create_vterm(term_T *term, int rows, int cols)
     /* The "Terminal" highlight group overrules the defaults. */
     id = syn_name2id((char_u *)"Terminal");
 
-    /* Use the actual color for the GUI and when 'guitermcolors' is set. */
+    /* Use the actual color for the GUI and when 'termguicolors' is set. */
 #if defined(FEAT_GUI) || defined(FEAT_TERMGUICOLORS)
     if (0
 # ifdef FEAT_GUI
diff --git a/src/version.c b/src/version.c
index 3cbfe9ed7..075760fd3 100644
--- a/src/version.c
+++ b/src/version.c
@@ -772,6 +772,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    1359,
+/**/
     1358,
 /**/
     1357,