patch 8.2.1535: it is not possible to specify cell widths of charactersv8.2.1535

Problem:    It is not possible to specify cell widths of characters.
Solution:   Add setcellwidths().

9 files changed, 265 insertions, 4 deletions

diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt
index 09ef97963..129c1e2e2 100644
--- a/runtime/doc/eval.txt
+++ b/runtime/doc/eval.txt
@@ -2768,6 +2768,7 @@ setbufline({expr}, {lnum}, {text})
 					{expr}
 setbufvar({expr}, {varname}, {val})
 				none	set {varname} in buffer {expr} to {val}
+setcellwidths({list})		none	set character cell width overrides
 setcharsearch({dict})		Dict	set character search from {dict}
 setcmdpos({pos})		Number	set cursor position in command-line
 setenv({name}, {val})		none	set environment variable
@@ -8937,6 +8938,29 @@ setbufvar({expr}, {varname}, {val})			*setbufvar()*
 		third argument: >
 			GetValue()->setbufvar(buf, varname)
 
+
+setcellwidths({list})					*setcellwidths()*
+		Specify overrides for cell widths of character ranges.  This
+		tells Vim how wide characters are, counted in screen cells.
+		This overrides 'ambiwidth'.  Example: >
+		   setcellwidths([[0xad, 0xad, 1],
+		   		\ [0x2194, 0x2199, 2]]
+
+<					*E1109* *E1110* *E1111* *E1112* *E1113*
+		The {list} argument is a list of lists with each three
+		numbers. These three numbers are [low, high, width].  "low"
+		and "high" can be the same, in which case this refers to one
+		character. Otherwise it is the range of characters from "low"
+		to "high" (inclusive).  "width" is either 1 or 2, indicating
+		the character width in screen cells.
+		An error is given if the argument is invalid, also when a
+		range overlaps with another.
+		Only characters with value 0x100 and higher can be used.
+
+		To clear the overrides pass an empty list: >
+		   setcellwidths([]);
+
+
 setcharsearch({dict})					*setcharsearch()*
 		Set the current character search information to {dict},
 		which contains one or more of the following entries:
diff --git a/runtime/doc/options.txt b/runtime/doc/options.txt
index b463e75c4..de8dd487f 100644
--- a/runtime/doc/options.txt
+++ b/runtime/doc/options.txt
@@ -701,7 +701,9 @@ A jump table for the options with a short description can be found at |Q_op|.
 	"double":	Use twice the width of ASCII characters.
 							*E834* *E835*
 	The value "double" cannot be used if 'listchars' or 'fillchars'
-	contains a character that would be double width.
+
+	The values are overruled for characters specified with
+	|setcellwidths()|.
 
 	There are a number of CJK fonts for which the width of glyphs for
 	those characters are solely based on how many octets they take in
diff --git a/runtime/doc/usr_41.txt b/runtime/doc/usr_41.txt
index 459f6b016..bcd58c9fe 100644
--- a/runtime/doc/usr_41.txt
+++ b/runtime/doc/usr_41.txt
@@ -611,6 +611,7 @@ String manipulation:					*string-functions*
 	strchars()		length of a string in characters
 	strwidth()		size of string when displayed
 	strdisplaywidth()	size of string when displayed, deals with tabs
+	setcellwidths()		set character cell width overrides
 	substitute()		substitute a pattern match with a string
 	submatch()		get a specific match in ":s" and substitute()
 	strpart()		get part of a string using byte index
diff --git a/src/errors.h b/src/errors.h
index ffcbdefa3..4b3e5debe 100644
--- a/src/errors.h
+++ b/src/errors.h
@@ -238,4 +238,16 @@ EXTERN char e_string_list_dict_or_blob_required[]
 	INIT(= N_("E1107: String, List, Dict or Blob required"));
 EXTERN char e_item_not_found_str[]
 	INIT(= N_("E1108: Item not found: %s"));
+EXTERN char e_list_item_nr_is_not_list[]
+	INIT(= N_("E1109: List item %d is not a List"));
+EXTERN char e_list_item_nr_does_not_contain_3_numbers[]
+	INIT(= N_("E1110: List item %d does not contain 3 numbers"));
+EXTERN char e_list_item_nr_range_invalid[]
+	INIT(= N_("E1111: List item %d range invalid"));
+EXTERN char e_list_item_nr_cell_width_invalid[]
+	INIT(= N_("E1112: List item %d cell width invalid"));
+EXTERN char e_overlapping_ranges_for_nr[]
+	INIT(= N_("E1113: Overlapping ranges for %lx"));
+EXTERN char e_only_values_of_0x100_and_higher_supported[]
+	INIT(= N_("E1114: Only values of 0x100 and higher supported"));
 #endif
diff --git a/src/evalfunc.c b/src/evalfunc.c
index 270dcd2cd..567bbdfaa 100644
--- a/src/evalfunc.c
+++ b/src/evalfunc.c
@@ -886,6 +886,7 @@ static funcentry_T global_functions[] =
     {"serverlist",	0, 0, 0,	  ret_string,	f_serverlist},
     {"setbufline",	3, 3, FEARG_3,	  ret_number,	f_setbufline},
     {"setbufvar",	3, 3, FEARG_3,	  ret_void,	f_setbufvar},
+    {"setcellwidths",	1, 1, FEARG_1,	  ret_void,	f_setcellwidths},
     {"setcharsearch",	1, 1, FEARG_1,	  ret_void,	f_setcharsearch},
     {"setcmdpos",	1, 1, FEARG_1,	  ret_number,	f_setcmdpos},
     {"setenv",		2, 2, FEARG_2,	  ret_void,	f_setenv},
diff --git a/src/mbyte.c b/src/mbyte.c
index 4f4210a53..d72fb9191 100644
--- a/src/mbyte.c
+++ b/src/mbyte.c
@@ -132,6 +132,7 @@ static int dbcs_char2cells(int c);
 static int dbcs_ptr2cells_len(char_u *p, int size);
 static int dbcs_ptr2char(char_u *p);
 static int dbcs_head_off(char_u *base, char_u *p);
+static int cw_value(int c);
 
 /*
  * Lookup table to quickly get the length in bytes of a UTF-8 character from
@@ -1487,7 +1488,7 @@ utf_char2cells(int c)
     // Sorted list of non-overlapping intervals of Emoji characters that don't
     // have ambiguous or double width,
     // based on http://unicode.org/emoji/charts/emoji-list.html
-    static struct interval emoji_width[] =
+    static struct interval emoji_wide[] =
     {
 	{0x1f1e6, 0x1f1ff},
 	{0x1f321, 0x1f321},
@@ -1532,12 +1533,18 @@ utf_char2cells(int c)
 
     if (c >= 0x100)
     {
+	int	n;
+
+	n = cw_value(c);
+	if (n != 0)
+	    return n;
+
 #ifdef USE_WCHAR_FUNCTIONS
 	/*
 	 * Assume the library function wcwidth() works better than our own
 	 * stuff.  It should return 1 for ambiguous width chars!
 	 */
-	int	n = wcwidth(c);
+	n = wcwidth(c);
 
 	if (n < 0)
 	    return 6;		// unprintable, displays <xxxx>
@@ -1549,7 +1556,7 @@ utf_char2cells(int c)
 	if (intable(doublewidth, sizeof(doublewidth), c))
 	    return 2;
 #endif
-	if (p_emoji && intable(emoji_width, sizeof(emoji_width), c))
+	if (p_emoji && intable(emoji_wide, sizeof(emoji_wide), c))
 	    return 2;
     }
 
@@ -2570,6 +2577,8 @@ utf_printable(int c)
 
 // Sorted list of non-overlapping intervals of all Emoji characters,
 // based on http://unicode.org/emoji/charts/emoji-list.html
+// Generated by ../runtime/tools/unicode.vim.
+// Excludes 0x00a9 and 0x00ae because they are considered latin1.
 static struct interval emoji_all[] =
 {
     {0x203c, 0x203c},
@@ -5342,3 +5351,177 @@ string_convert_ext(
 
     return retval;
 }
+
+/*
+ * Table set by setcellwidths().
+ */
+typedef struct
+{
+    long    first;
+    long    last;
+    char    width;
+} cw_interval_T;
+
+static cw_interval_T	*cw_table = NULL;
+static size_t		cw_table_size = 0;
+
+/*
+ * Return 1 or 2 when "c" is in the cellwidth table.
+ * Return 0 if not.
+ */
+    static int
+cw_value(int c)
+{
+    int mid, bot, top;
+
+    if (cw_table == NULL)
+	return 0;
+
+    // first quick check for Latin1 etc. characters
+    if (c < cw_table[0].first)
+	return 0;
+
+    // binary search in table
+    bot = 0;
+    top = (int)cw_table_size - 1;
+    while (top >= bot)
+    {
+	mid = (bot + top) / 2;
+	if (cw_table[mid].last < c)
+	    bot = mid + 1;
+	else if (cw_table[mid].first > c)
+	    top = mid - 1;
+	else
+	    return cw_table[mid].width;
+    }
+    return 0;
+}
+
+    static int
+tv_nr_compare(const void *a1, const void *a2)
+{
+    listitem_T *li1 = (listitem_T *)a1;
+    listitem_T *li2 = (listitem_T *)a2;
+
+    return li1->li_tv.vval.v_number - li2->li_tv.vval.v_number;
+}
+
+    void
+f_setcellwidths(typval_T *argvars, typval_T *rettv UNUSED)
+{
+    list_T	    *l;
+    listitem_T	    *li;
+    int		    item;
+    int		    i;
+    listitem_T	    **ptrs;
+    cw_interval_T   *table;
+
+    if (argvars[0].v_type != VAR_LIST || argvars[0].vval.v_list == NULL)
+    {
+	emsg(_(e_listreq));
+	return;
+    }
+    l = argvars[0].vval.v_list;
+    if (l->lv_len == 0)
+    {
+	// Clearing the table.
+	vim_free(cw_table);
+	cw_table = NULL;
+	cw_table_size = 0;
+	return;
+    }
+
+    ptrs = ALLOC_MULT(listitem_T *, l->lv_len);
+    if (ptrs == NULL)
+	return;
+
+    // Check that all entries are a list with three numbers, the range is
+    // valid and the cell width is valid.
+    item = 0;
+    for (li = l->lv_first; li != NULL; li = li->li_next)
+    {
+	listitem_T *lili;
+	varnumber_T n1;
+
+	if (li->li_tv.v_type != VAR_LIST || li->li_tv.vval.v_list == NULL)
+	{
+	    semsg(_(e_list_item_nr_is_not_list), item);
+	    vim_free(ptrs);
+	    return;
+	}
+	for (lili = li->li_tv.vval.v_list->lv_first, i = 0; lili != NULL;
+						     lili = lili->li_next, ++i)
+	{
+	    if (lili->li_tv.v_type != VAR_NUMBER)
+		break;
+	    if (i == 0)
+	    {
+		n1 = lili->li_tv.vval.v_number;
+		if (n1 < 0x100)
+		{
+		    emsg(_(e_only_values_of_0x100_and_higher_supported));
+		    vim_free(ptrs);
+		    return;
+		}
+	    }
+	    else if (i == 1 && lili->li_tv.vval.v_number < n1)
+	    {
+		semsg(_(e_list_item_nr_range_invalid), item);
+		vim_free(ptrs);
+		return;
+	    }
+	    else if (i == 2 && (lili->li_tv.vval.v_number < 1
+					     || lili->li_tv.vval.v_number > 2))
+	    {
+		semsg(_(e_list_item_nr_cell_width_invalid), item);
+		vim_free(ptrs);
+		return;
+	    }
+	}
+	if (i != 3)
+	{
+	    semsg(_(e_list_item_nr_does_not_contain_3_numbers), item);
+	    vim_free(ptrs);
+	    return;
+	}
+	ptrs[item++] = lili;
+    }
+
+    // Sort the list on the first number.
+    qsort((void *)ptrs, (size_t)l->lv_len, sizeof(listitem_T *), tv_nr_compare);
+
+    table = ALLOC_MULT(cw_interval_T, l->lv_len);
+    if (table == NULL)
+    {
+	vim_free(ptrs);
+	return;
+    }
+
+    // Store the items in the new table.
+    item = 0;
+    for (li = l->lv_first; li != NULL; li = li->li_next)
+    {
+	listitem_T	*lili = li->li_tv.vval.v_list->lv_first;
+	varnumber_T	n1;
+
+	n1 = lili->li_tv.vval.v_number;
+	if (item > 0 && n1 <= table[item - 1].last)
+	{
+	    semsg(_(e_overlapping_ranges_for_nr), (long)n1);
+	    vim_free(ptrs);
+	    vim_free(table);
+	    return;
+	}
+	table[item].first = n1;
+	lili = lili->li_next;
+	table[item].last = lili->li_tv.vval.v_number;
+	lili = lili->li_next;
+	table[item].width = lili->li_tv.vval.v_number;
+	++item;
+    }
+
+    vim_free(ptrs);
+    vim_free(cw_table);
+    cw_table = table;
+    cw_table_size = l->lv_len;
+}
diff --git a/src/proto/mbyte.pro b/src/proto/mbyte.pro
index 79af974ec..9385d856d 100644
--- a/src/proto/mbyte.pro
+++ b/src/proto/mbyte.pro
@@ -84,4 +84,5 @@ int convert_input(char_u *ptr, int len, int maxlen);
 int convert_input_safe(char_u *ptr, int len, int maxlen, char_u **restp, int *restlenp);
 char_u *string_convert(vimconv_T *vcp, char_u *ptr, int *lenp);
 char_u *string_convert_ext(vimconv_T *vcp, char_u *ptr, int *lenp, int *unconvlenp);
+void f_setcellwidths(typval_T *argvars, typval_T *rettv);
 /* vim: set ft=c : */
diff --git a/src/testdir/test_utf8.vim b/src/testdir/test_utf8.vim
index f9f38780d..b2b74599d 100644
--- a/src/testdir/test_utf8.vim
+++ b/src/testdir/test_utf8.vim
@@ -145,4 +145,39 @@ func Test_screenchar_utf8()
   bwipe!
 endfunc
 
+func Test_setcellwidths()
+  call setcellwidths([
+        \ [0x1330, 0x1330, 2],
+        \ [0x1337, 0x1339, 2],
+        \ [9999, 10000, 1],
+        \])
+
+  call assert_equal(2, strwidth("\u1330"))
+  call assert_equal(1, strwidth("\u1336"))
+  call assert_equal(2, strwidth("\u1337"))
+  call assert_equal(2, strwidth("\u1339"))
+  call assert_equal(1, strwidth("\u133a"))
+
+  call setcellwidths([])
+
+  call assert_fails('call setcellwidths(1)', 'E714:')
+
+  call assert_fails('call setcellwidths([1, 2, 0])', 'E1109:')
+
+  call assert_fails('call setcellwidths([[0x101]])', 'E1110:')
+  call assert_fails('call setcellwidths([[0x101, 0x102]])', 'E1110:')
+  call assert_fails('call setcellwidths([[0x101, 0x102, 1, 4]])', 'E1110:')
+  call assert_fails('call setcellwidths([["a"]])', 'E1110:')
+
+  call assert_fails('call setcellwidths([[0x102, 0x101, 1]])', 'E1111:')
+
+  call assert_fails('call setcellwidths([[0x101, 0x102, 0]])', 'E1112:')
+  call assert_fails('call setcellwidths([[0x101, 0x102, 3]])', 'E1112:')
+
+  call assert_fails('call setcellwidths([[0x111, 0x122, 1], [0x115, 0x116, 2]])', 'E1113:')
+  call assert_fails('call setcellwidths([[0x111, 0x122, 1], [0x122, 0x123, 2]])', 'E1113:')
+
+  call assert_fails('call setcellwidths([[0x33, 0x44, 2]])', 'E1114:')
+endfunc
+
 " vim: shiftwidth=2 sts=2 expandtab
diff --git a/src/version.c b/src/version.c
index 21c6a18f2..88332a00f 100644
--- a/src/version.c
+++ b/src/version.c
@@ -755,6 +755,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    1535,
+/**/
     1534,
 /**/
     1533,