patch 8.0.0420: text garbled when the system encoding differs from 'encoding'v8.0.0420

Problem:    When running :make the output may be in the system encoding,
            different from 'encoding'.
Solution:   Add the 'makeencoding' option. (Ken Takata)

13 files changed, 287 insertions, 16 deletions

diff --git a/src/Makefile b/src/Makefile
index 0bdd1d470..6e7fdde3c 100644
--- a/src/Makefile
+++ b/src/Makefile
@@ -2157,6 +2157,7 @@ test_arglist \
 	test_listlbr \
 	test_listlbr_utf8 \
 	test_lua \
+	test_makeencoding \
 	test_man \
 	test_mapping \
 	test_marks \
diff --git a/src/buffer.c b/src/buffer.c
index 6edb48a6f..c25cb2f1a 100644
--- a/src/buffer.c
+++ b/src/buffer.c
@@ -2255,6 +2255,9 @@ free_buf_options(
     clear_string_option(&buf->b_p_lw);
 #endif
     clear_string_option(&buf->b_p_bkc);
+#ifdef FEAT_MBYTE
+    clear_string_option(&buf->b_p_menc);
+#endif
 }
 
 /*
diff --git a/src/if_cscope.c b/src/if_cscope.c
index 60480a5b7..5b384982e 100644
--- a/src/if_cscope.c
+++ b/src/if_cscope.c
@@ -1242,7 +1242,7 @@ cs_find_common(
 		wp = curwin;
 	    /* '-' starts a new error list */
 	    if (qf_init(wp, tmp, (char_u *)"%f%*\\t%l%*\\t%m",
-						  *qfpos == '-', cmdline) > 0)
+					  *qfpos == '-', cmdline, NULL) > 0)
 	    {
 # ifdef FEAT_WINDOWS
 		if (postponed_split != 0)
diff --git a/src/main.c b/src/main.c
index 0f9f3fff4..2f8d291bf 100644
--- a/src/main.c
+++ b/src/main.c
@@ -559,11 +559,16 @@ vim_main2(void)
      */
     if (params.edit_type == EDIT_QF)
     {
+	char_u	*enc = NULL;
+
+# ifdef FEAT_MBYTE
+	enc = p_menc;
+# endif
 	if (params.use_ef != NULL)
 	    set_string_option_direct((char_u *)"ef", -1,
 					   params.use_ef, OPT_FREE, SID_CARG);
 	vim_snprintf((char *)IObuff, IOSIZE, "cfile %s", p_ef);
-	if (qf_init(NULL, p_ef, p_efm, TRUE, IObuff) < 0)
+	if (qf_init(NULL, p_ef, p_efm, TRUE, IObuff, enc) < 0)
 	{
 	    out_char('\n');
 	    mch_exit(3);
diff --git a/src/option.c b/src/option.c
index 459179e2e..d4e760b89 100644
--- a/src/option.c
+++ b/src/option.c
@@ -139,6 +139,9 @@
 # define PV_LISP	OPT_BUF(BV_LISP)
 # define PV_LW		OPT_BOTH(OPT_BUF(BV_LW))
 #endif
+#ifdef FEAT_MBYTE
+# define PV_MENC	OPT_BOTH(OPT_BUF(BV_MENC))
+#endif
 #define PV_MA		OPT_BUF(BV_MA)
 #define PV_ML		OPT_BUF(BV_ML)
 #define PV_MOD		OPT_BUF(BV_MOD)
@@ -1900,6 +1903,15 @@ static struct vimoption options[] =
 			    {(char_u *)NULL, (char_u *)0L}
 #endif
 			    SCRIPTID_INIT},
+    {"makeencoding","menc", P_STRING|P_VI_DEF,
+#ifdef FEAT_MBYTE
+			    (char_u *)&p_menc, PV_MENC,
+			    {(char_u *)"", (char_u *)0L}
+#else
+			    (char_u *)NULL, PV_NONE,
+			    {(char_u *)0L, (char_u *)0L}
+#endif
+			    SCRIPTID_INIT},
     {"makeprg",	    "mp",   P_STRING|P_EXPAND|P_VI_DEF|P_SECURE,
 #ifdef FEAT_QUICKFIX
 			    (char_u *)&p_mp, PV_MP,
@@ -5686,6 +5698,9 @@ check_buf_options(buf_T *buf)
     check_string_option(&buf->b_p_lw);
 #endif
     check_string_option(&buf->b_p_bkc);
+#ifdef FEAT_MBYTE
+    check_string_option(&buf->b_p_menc);
+#endif
 }
 
 /*
@@ -6289,8 +6304,9 @@ did_set_string_option(
 #endif
 
 #ifdef FEAT_MBYTE
-    /* 'encoding' and 'fileencoding' */
-    else if (varp == &p_enc || gvarp == &p_fenc || varp == &p_tenc)
+    /* 'encoding', 'fileencoding', 'termencoding' and 'makeencoding' */
+    else if (varp == &p_enc || gvarp == &p_fenc || varp == &p_tenc
+							   || gvarp == &p_menc)
     {
 	if (gvarp == &p_fenc)
 	{
@@ -10425,6 +10441,11 @@ unset_global_local_option(char_u *name, void *from)
 	    clear_string_option(&buf->b_p_lw);
 	    break;
 #endif
+#ifdef FEAT_MBYTE
+	case PV_MENC:
+	    clear_string_option(&buf->b_p_menc);
+	    break;
+#endif
     }
 }
 
@@ -10478,6 +10499,9 @@ get_varp_scope(struct vimoption *p, int opt_flags)
 	    case PV_LW:   return (char_u *)&(curbuf->b_p_lw);
 #endif
 	    case PV_BKC:  return (char_u *)&(curbuf->b_p_bkc);
+#ifdef FEAT_MBYTE
+	    case PV_MENC: return (char_u *)&(curbuf->b_p_menc);
+#endif
 	}
 	return NULL; /* "cannot happen" */
     }
@@ -10553,6 +10577,10 @@ get_varp(struct vimoption *p)
 	case PV_LW:	return *curbuf->b_p_lw != NUL
 				    ? (char_u *)&(curbuf->b_p_lw) : p->var;
 #endif
+#ifdef FEAT_MBYTE
+	case PV_MENC:	return *curbuf->b_p_menc != NUL
+				    ? (char_u *)&(curbuf->b_p_menc) : p->var;
+#endif
 
 #ifdef FEAT_ARABIC
 	case PV_ARAB:	return (char_u *)&(curwin->w_p_arab);
@@ -11154,6 +11182,9 @@ buf_copy_options(buf_T *buf, int flags)
 #ifdef FEAT_LISP
 	    buf->b_p_lw = empty_option;
 #endif
+#ifdef FEAT_MBYTE
+	    buf->b_p_menc = empty_option;
+#endif
 
 	    /*
 	     * Don't copy the options set by ex_help(), use the saved values,
diff --git a/src/option.h b/src/option.h
index 62e66cd3d..2c6aeef03 100644
--- a/src/option.h
+++ b/src/option.h
@@ -630,6 +630,9 @@ EXTERN char_u	*p_luadll;	/* 'luadll' */
 EXTERN int	p_macatsui;	/* 'macatsui' */
 #endif
 EXTERN int	p_magic;	/* 'magic' */
+#ifdef FEAT_MBYTE
+EXTERN char_u	*p_menc;	/* 'makeencoding' */
+#endif
 #ifdef FEAT_QUICKFIX
 EXTERN char_u	*p_mef;		/* 'makeef' */
 EXTERN char_u	*p_mp;		/* 'makeprg' */
@@ -1065,6 +1068,9 @@ enum
     , BV_LISP
     , BV_LW
 #endif
+#ifdef FEAT_MBYTE
+    , BV_MENC
+#endif
     , BV_MA
     , BV_ML
     , BV_MOD
diff --git a/src/proto/quickfix.pro b/src/proto/quickfix.pro
index d48eb2532..995f41f03 100644
--- a/src/proto/quickfix.pro
+++ b/src/proto/quickfix.pro
@@ -1,5 +1,5 @@
 /* quickfix.c */
-int qf_init(win_T *wp, char_u *efile, char_u *errorformat, int newlist, char_u *qf_title);
+int qf_init(win_T *wp, char_u *efile, char_u *errorformat, int newlist, char_u *qf_title, char_u *enc);
 void qf_free_all(win_T *wp);
 void copy_loclist(win_T *from, win_T *to);
 void qf_jump(qf_info_T *qi, int dir, int errornr, int forceit);
diff --git a/src/quickfix.c b/src/quickfix.c
index df351634f..e4ea401ec 100644
--- a/src/quickfix.c
+++ b/src/quickfix.c
@@ -116,7 +116,7 @@ struct efm_S
 
 static efm_T	*fmt_start = NULL; /* cached across qf_parse_line() calls */
 
-static int	qf_init_ext(qf_info_T *qi, char_u *efile, buf_T *buf, typval_T *tv, char_u *errorformat, int newlist, linenr_T lnumfirst, linenr_T lnumlast, char_u *qf_title);
+static int	qf_init_ext(qf_info_T *qi, char_u *efile, buf_T *buf, typval_T *tv, char_u *errorformat, int newlist, linenr_T lnumfirst, linenr_T lnumlast, char_u *qf_title, char_u *enc);
 static void	qf_store_title(qf_info_T *qi, char_u *title);
 static void	qf_new_list(qf_info_T *qi, char_u *qf_title);
 static void	ll_free_all(qf_info_T **pqi);
@@ -167,7 +167,8 @@ qf_init(
     char_u	    *efile,
     char_u	    *errorformat,
     int		    newlist,		/* TRUE: start a new error list */
-    char_u	    *qf_title)
+    char_u	    *qf_title,
+    char_u	    *enc)
 {
     qf_info_T	    *qi = &ql_info;
 
@@ -180,7 +181,7 @@ qf_init(
 
     return qf_init_ext(qi, efile, curbuf, NULL, errorformat, newlist,
 						    (linenr_T)0, (linenr_T)0,
-						    qf_title);
+						    qf_title, enc);
 }
 
 /*
@@ -498,6 +499,7 @@ typedef struct {
     buf_T	*buf;
     linenr_T	buflnum;
     linenr_T	lnumlast;
+    vimconv_T	vc;
 } qfstate_T;
 
     static char_u *
@@ -713,6 +715,30 @@ qf_get_next_file_line(qfstate_T *state)
     else
 	state->linebuf = IObuff;
 
+#ifdef FEAT_MBYTE
+    /* Convert a line if it contains a non-ASCII character. */
+    if (state->vc.vc_type != CONV_NONE && has_non_ascii(state->linebuf)) {
+	char_u	*line;
+
+	line = string_convert(&state->vc, state->linebuf, &state->linelen);
+	if (line != NULL)
+	{
+	    if (state->linelen < IOSIZE)
+	    {
+		STRCPY(state->linebuf, line);
+		vim_free(line);
+	    }
+	    else
+	    {
+		vim_free(state->growbuf);
+		state->linebuf = state->growbuf = line;
+		state->growbufsiz = state->linelen < LINE_MAXLEN
+						? state->linelen : LINE_MAXLEN;
+	    }
+	}
+    }
+#endif
+
     return QF_OK;
 }
 
@@ -1105,11 +1131,11 @@ qf_init_ext(
     int		    newlist,		/* TRUE: start a new error list */
     linenr_T	    lnumfirst,		/* first line number to use */
     linenr_T	    lnumlast,		/* last line number to use */
-    char_u	    *qf_title)
+    char_u	    *qf_title,
+    char_u	    *enc)
 {
-    qfstate_T	    state = {NULL, 0, NULL, 0, NULL, NULL, NULL, NULL,
-			     NULL, 0, 0};
-    qffields_T	    fields = {NULL, NULL, 0, 0L, 0, FALSE, NULL, 0, 0, 0};
+    qfstate_T	    state;
+    qffields_T	    fields;
 #ifdef FEAT_WINDOWS
     qfline_T	    *old_last = NULL;
     int		    adding = FALSE;
@@ -1120,6 +1146,13 @@ qf_init_ext(
     int		    retval = -1;	/* default: return error flag */
     int		    status;
 
+    vim_memset(&state, 0, sizeof(state));
+    vim_memset(&fields, 0, sizeof(fields));
+#ifdef FEAT_MBYTE
+    state.vc.vc_type = CONV_NONE;
+    if (enc != NULL && *enc != NUL)
+	convert_setup(&state.vc, enc, p_enc);
+#endif
     fields.namebuf = alloc_id(CMDBUFFSIZE + 1, aid_qf_namebuf);
     fields.errmsglen = CMDBUFFSIZE + 1;
     fields.errmsg = alloc_id(fields.errmsglen, aid_qf_errmsg);
@@ -1286,6 +1319,10 @@ qf_init_end:
 #ifdef FEAT_WINDOWS
     qf_update_buffer(qi, old_last);
 #endif
+#ifdef FEAT_MBYTE
+    if (state.vc.vc_type != CONV_NONE)
+	convert_setup(&state.vc, NULL, NULL);
+#endif
 
     return retval;
 }
@@ -3431,6 +3468,7 @@ ex_make(exarg_T *eap)
 {
     char_u	*fname;
     char_u	*cmd;
+    char_u	*enc = NULL;
     unsigned	len;
     win_T	*wp = NULL;
     qf_info_T	*qi = &ql_info;
@@ -3464,6 +3502,9 @@ ex_make(exarg_T *eap)
 # endif
     }
 #endif
+#ifdef FEAT_MBYTE
+    enc = (*curbuf->b_p_menc != NUL) ? curbuf->b_p_menc : p_menc;
+#endif
 
     if (eap->cmdidx == CMD_lmake || eap->cmdidx == CMD_lgrep
 	|| eap->cmdidx == CMD_lgrepadd)
@@ -3511,7 +3552,7 @@ ex_make(exarg_T *eap)
 			    && eap->cmdidx != CMD_lmake) ? p_gefm : p_efm,
 					   (eap->cmdidx != CMD_grepadd
 					    && eap->cmdidx != CMD_lgrepadd),
-					   *eap->cmdlinep);
+					   *eap->cmdlinep, enc);
     if (wp != NULL)
 	qi = GET_LOC_LIST(wp);
 #ifdef FEAT_AUTOCMD
@@ -3850,6 +3891,7 @@ ex_cnext(exarg_T *eap)
     void
 ex_cfile(exarg_T *eap)
 {
+    char_u	*enc = NULL;
     win_T	*wp = NULL;
     qf_info_T	*qi = &ql_info;
 #ifdef FEAT_AUTOCMD
@@ -3874,6 +3916,9 @@ ex_cfile(exarg_T *eap)
     if (au_name != NULL)
 	apply_autocmds(EVENT_QUICKFIXCMDPRE, au_name, NULL, FALSE, curbuf);
 #endif
+#ifdef FEAT_MBYTE
+    enc = (*curbuf->b_p_menc != NUL) ? curbuf->b_p_menc : p_menc;
+#endif
 #ifdef FEAT_BROWSE
     if (cmdmod.browse)
     {
@@ -3901,7 +3946,7 @@ ex_cfile(exarg_T *eap)
      */
     if (qf_init(wp, p_ef, p_efm, (eap->cmdidx != CMD_caddfile
 				  && eap->cmdidx != CMD_laddfile),
-							   *eap->cmdlinep) > 0
+						       *eap->cmdlinep, enc) > 0
 				  && (eap->cmdidx == CMD_cfile
 					     || eap->cmdidx == CMD_lfile))
     {
@@ -4927,7 +4972,7 @@ ex_cbuffer(exarg_T *eap)
 			    (eap->cmdidx != CMD_caddbuffer
 			     && eap->cmdidx != CMD_laddbuffer),
 						   eap->line1, eap->line2,
-						   qf_title) > 0)
+						   qf_title, NULL) > 0)
 	    {
 #ifdef FEAT_AUTOCMD
 		if (au_name != NULL)
@@ -4995,7 +5040,8 @@ ex_cexpr(exarg_T *eap)
 	    if (qf_init_ext(qi, NULL, NULL, tv, p_efm,
 			    (eap->cmdidx != CMD_caddexpr
 			     && eap->cmdidx != CMD_laddexpr),
-				 (linenr_T)0, (linenr_T)0, *eap->cmdlinep) > 0)
+				 (linenr_T)0, (linenr_T)0, *eap->cmdlinep,
+				 NULL) > 0)
 	    {
 #ifdef FEAT_AUTOCMD
 		if (au_name != NULL)
diff --git a/src/structs.h b/src/structs.h
index 1e1b1da82..5745df2ae 100644
--- a/src/structs.h
+++ b/src/structs.h
@@ -2133,6 +2133,9 @@ struct file_buffer
 #ifdef FEAT_LISP
     int		b_p_lisp;	/* 'lisp' */
 #endif
+#ifdef FEAT_MBYTE
+    char_u	*b_p_menc;	/* 'makeencoding' */
+#endif
     char_u	*b_p_mps;	/* 'matchpairs' */
     int		b_p_ml;		/* 'modeline' */
     int		b_p_ml_nobin;	/* b_p_ml saved for binary mode */
diff --git a/src/testdir/Make_all.mak b/src/testdir/Make_all.mak
index 513fd8971..e1ce1518f 100644
--- a/src/testdir/Make_all.mak
+++ b/src/testdir/Make_all.mak
@@ -170,6 +170,7 @@ NEW_TESTS = test_arabic.res \
 	    test_listlbr.res \
 	    test_listlbr_utf8.res \
 	    test_lua.res \
+	    test_makeencoding.res \
 	    test_man.res \
 	    test_marks.res \
 	    test_matchadd_conceal.res \
diff --git a/src/testdir/test_makeencoding.py b/src/testdir/test_makeencoding.py
new file mode 100644
index 000000000..041edadc0
--- /dev/null
+++ b/src/testdir/test_makeencoding.py
@@ -0,0 +1,67 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Test program for :make, :grep and :cgetfile.
+
+from __future__ import print_function, unicode_literals
+import locale
+import io
+import sys
+
+def set_output_encoding(enc=None):
+    """Set the encoding of stdout and stderr
+
+    arguments:
+      enc -- Encoding name.
+             If omitted, locale.getpreferredencoding() is used.
+    """
+    if enc is None:
+        enc = locale.getpreferredencoding()
+
+    def get_text_writer(fo, **kwargs):
+        kw = dict(kwargs)
+        kw.setdefault('errors', 'backslashreplace') # use \uXXXX style
+        kw.setdefault('closefd', False)
+
+        if sys.version_info[0] < 3:
+            # Work around for Python 2.x
+            # New line conversion isn't needed here. Done in somewhere else.
+            writer = io.open(fo.fileno(), mode='w', newline='', **kw)
+            write = writer.write    # save the original write() function
+            enc = locale.getpreferredencoding()
+            def convwrite(s):
+                if isinstance(s, bytes):
+                    write(s.decode(enc))    # convert to unistr
+                else:
+                    write(s)
+                try:
+                    writer.flush()  # needed on Windows
+                except IOError:
+                    pass
+            writer.write = convwrite
+        else:
+            writer = io.open(fo.fileno(), mode='w', **kw)
+        return writer
+
+    sys.stdout = get_text_writer(sys.stdout, encoding=enc)
+    sys.stderr = get_text_writer(sys.stderr, encoding=enc)
+
+
+def main():
+    enc = 'utf-8'
+    if len(sys.argv) > 1:
+        enc = sys.argv[1]
+    set_output_encoding(enc)
+
+    message_tbl = {
+            'utf-8': 'ÀÈÌÒÙ こんにちは 你好',
+            'latin1': 'ÀÈÌÒÙ',
+            'cp932': 'こんにちは',
+            'cp936': '你好',
+            }
+
+    print('Xfoobar.c(10) : %s (%s)' % (message_tbl[enc], enc))
+
+
+if __name__ == "__main__":
+    main()
diff --git a/src/testdir/test_makeencoding.vim b/src/testdir/test_makeencoding.vim
new file mode 100644
index 000000000..a3d5538a4
--- /dev/null
+++ b/src/testdir/test_makeencoding.vim
@@ -0,0 +1,106 @@
+" Tests for 'makeencoding'.
+if !has('multi_byte')
+  finish
+endif
+
+source shared.vim
+
+let s:python = PythonProg()
+if s:python == ''
+  " Can't run this test.
+  finish
+endif
+
+let s:script = 'test_makeencoding.py'
+
+let s:message_tbl = {
+      \ 'utf-8': 'ÀÈÌÒÙ こんにちは 你好',
+      \ 'latin1': 'ÀÈÌÒÙ',
+      \ 'cp932': 'こんにちは',
+      \ 'cp936': '你好',
+      \}
+
+
+" Tests for :cgetfile and :lgetfile.
+func Test_getfile()
+  set errorfile=Xerror.txt
+  set errorformat=%f(%l)\ :\ %m
+
+  " :cgetfile
+  for enc in keys(s:message_tbl)
+    let &makeencoding = enc
+    exec "silent !" . s:python . " " . s:script . " " . enc . " > " . &errorfile
+    cgetfile
+    copen
+    call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
+          \ getline('.'))
+    cclose
+  endfor
+
+  " :lgetfile
+  for enc in keys(s:message_tbl)
+    let &makeencoding = enc
+    exec "silent !" . s:python . " " . s:script . " " . enc . " > " . &errorfile
+    lgetfile
+    lopen
+    call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
+          \ getline('.'))
+    lclose
+  endfor
+
+  call delete(&errorfile)
+endfunc
+
+
+" Tests for :grep and :lgrep.
+func Test_grep()
+  let &grepprg = s:python
+  set grepformat=%f(%l)\ :\ %m
+
+  " :grep
+  for enc in keys(s:message_tbl)
+    let &makeencoding = enc
+    exec "silent grep! " . s:script . " " . enc
+    copen
+    call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
+          \ getline('.'))
+    cclose
+  endfor
+
+  " :lgrep
+  for enc in keys(s:message_tbl)
+    let &makeencoding = enc
+    exec "silent lgrep! " . s:script . " " . enc
+    lopen
+    call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
+          \ getline('.'))
+    lclose
+  endfor
+endfunc
+
+
+" Tests for :make and :lmake.
+func Test_make()
+  let &makeprg = s:python
+  set errorformat=%f(%l)\ :\ %m
+
+  " :make
+  for enc in keys(s:message_tbl)
+    let &makeencoding = enc
+    exec "silent make! " . s:script . " " . enc
+    copen
+    call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
+          \ getline('.'))
+    cclose
+  endfor
+
+  " :lmake
+  for enc in keys(s:message_tbl)
+    let &makeencoding = enc
+    exec "silent lmake! " . s:script . " " . enc
+    lopen
+    call assert_equal("Xfoobar.c|10| " . s:message_tbl[enc] . " (" . enc . ")",
+          \ getline('.'))
+    lclose
+  endfor
+endfunc
diff --git a/src/version.c b/src/version.c
index 834f1442e..f3a0b6c90 100644
--- a/src/version.c
+++ b/src/version.c
@@ -765,6 +765,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    420,
+/**/
     419,
 /**/
     418,