patch 8.2.4804: expression in heredoc doesn't work for compiled functionv8.2.4804

Problem:    Expression in heredoc doesn't work for compiled function.
Solution:   Implement compiling the heredoc expressions. (Yegappan Lakshmanan,
            closes #10232)

8 files changed, 187 insertions, 58 deletions

diff --git a/runtime/doc/eval.txt b/runtime/doc/eval.txt
index 40f0bf85e..953227d21 100644
--- a/runtime/doc/eval.txt
+++ b/runtime/doc/eval.txt
@@ -3247,8 +3247,6 @@ text...
 			expression evaluation fails, then the assignment fails.
 			once the "`=" has been found {expr} and a backtick
 			must follow.  {expr} cannot be empty.
-			Currenty, in a compiled function {expr} is evaluated
-			when compiling the function, THIS WILL CHANGE.
 
 			{endmarker} must not contain white space.
 			{endmarker} cannot start with a lower case character.
diff --git a/src/evalvars.c b/src/evalvars.c
index c83aa61c9..ffa7e93f7 100644
--- a/src/evalvars.c
+++ b/src/evalvars.c
@@ -673,16 +673,21 @@ eval_all_expr_in_str(char_u *str)
  *
  * The {marker} is a string. If the optional 'trim' word is supplied before the
  * marker, then the leading indentation before the lines (matching the
- * indentation in the 'cmd' line) is stripped.
+ * indentation in the "cmd" line) is stripped.
  *
  * When getting lines for an embedded script (e.g. python, lua, perl, ruby,
- * tcl, mzscheme), script_get is set to TRUE. In this case, if the marker is
+ * tcl, mzscheme), "script_get" is set to TRUE. In this case, if the marker is
  * missing, then '.' is accepted as a marker.
  *
+ * When compiling a heredoc assignment to a variable in a Vim9 def function,
+ * "vim9compile" is set to TRUE. In this case, instead of generating a list of
+ * string values from the heredoc, vim9 instructions are generated.  On success
+ * the returned list will be empty.
+ *
  * Returns a List with {lines} or NULL on failure.
  */
     list_T *
-heredoc_get(exarg_T *eap, char_u *cmd, int script_get)
+heredoc_get(exarg_T *eap, char_u *cmd, int script_get, int vim9compile)
 {
     char_u	*theline = NULL;
     char_u	*marker;
@@ -696,6 +701,8 @@ heredoc_get(exarg_T *eap, char_u *cmd, int script_get)
     int		comment_char = in_vim9script() ? '#' : '"';
     int		evalstr = FALSE;
     int		eval_failed = FALSE;
+    cctx_T	*cctx = vim9compile ? eap->cookie : NULL;
+    int		count = 0;
 
     if (eap->getline == NULL)
     {
@@ -816,25 +823,41 @@ heredoc_get(exarg_T *eap, char_u *cmd, int script_get)
 		    break;
 
 	str = theline + ti;
-	if (evalstr)
+	if (vim9compile)
 	{
-	    str = eval_all_expr_in_str(str);
-	    if (str == NULL)
+	    if (compile_heredoc_string(str, evalstr, cctx) == FAIL)
 	    {
-		// expression evaluation failed
-		eval_failed = TRUE;
-		continue;
+		vim_free(theline);
+		vim_free(text_indent);
+		return FAIL;
 	    }
-	    vim_free(theline);
-	    theline = str;
+	    count++;
 	}
+	else
+	{
+	    if (evalstr)
+	    {
+		str = eval_all_expr_in_str(str);
+		if (str == NULL)
+		{
+		    // expression evaluation failed
+		    eval_failed = TRUE;
+		    continue;
+		}
+		vim_free(theline);
+		theline = str;
+	    }
 
-	if (list_append_string(l, str, -1) == FAIL)
-	    break;
+	    if (list_append_string(l, str, -1) == FAIL)
+		break;
+	}
     }
     vim_free(theline);
     vim_free(text_indent);
 
+    if (vim9compile && cctx->ctx_skip != SKIP_YES && !eval_failed)
+	generate_NEWLIST(cctx, count, FALSE);
+
     if (eval_failed)
     {
 	// expression evaluation in the heredoc failed
@@ -986,7 +1009,7 @@ ex_let(exarg_T *eap)
 	long	cur_lnum = SOURCING_LNUM;
 
 	// HERE document
-	l = heredoc_get(eap, expr + 3, FALSE);
+	l = heredoc_get(eap, expr + 3, FALSE, FALSE);
 	if (l != NULL)
 	{
 	    rettv_list_set(&rettv, l);
diff --git a/src/ex_getln.c b/src/ex_getln.c
index c351cf256..a97024b35 100644
--- a/src/ex_getln.c
+++ b/src/ex_getln.c
@@ -4605,7 +4605,7 @@ script_get(exarg_T *eap UNUSED, char_u *cmd UNUSED)
 	return NULL;
     cmd += 2;
 
-    l = heredoc_get(eap, cmd, TRUE);
+    l = heredoc_get(eap, cmd, TRUE, FALSE);
     if (l == NULL)
 	return NULL;
 
diff --git a/src/proto/evalvars.pro b/src/proto/evalvars.pro
index 9e08f667e..ab4320d32 100644
--- a/src/proto/evalvars.pro
+++ b/src/proto/evalvars.pro
@@ -13,7 +13,7 @@ list_T *eval_spell_expr(char_u *badword, char_u *expr);
 int get_spellword(list_T *list, char_u **pp);
 void prepare_vimvar(int idx, typval_T *save_tv);
 void restore_vimvar(int idx, typval_T *save_tv);
-list_T *heredoc_get(exarg_T *eap, char_u *cmd, int script_get);
+list_T *heredoc_get(exarg_T *eap, char_u *cmd, int script_get, int vim9compile);
 void ex_var(exarg_T *eap);
 void ex_let(exarg_T *eap);
 int ex_let_vars(char_u *arg_start, typval_T *tv, int copy, int semicolon, int var_count, int flags, char_u *op);
diff --git a/src/proto/vim9compile.pro b/src/proto/vim9compile.pro
index 0f3094f8c..1fe46f007 100644
--- a/src/proto/vim9compile.pro
+++ b/src/proto/vim9compile.pro
@@ -16,6 +16,7 @@ int may_get_next_line(char_u *whitep, char_u **arg, cctx_T *cctx);
 int may_get_next_line_error(char_u *whitep, char_u **arg, cctx_T *cctx);
 void fill_exarg_from_cctx(exarg_T *eap, cctx_T *cctx);
 int func_needs_compiling(ufunc_T *ufunc, compiletype_T compile_type);
+int compile_heredoc_string(char_u *str, int evalstr, cctx_T *cctx);
 int assignment_len(char_u *p, int *heredoc);
 void vim9_declare_error(char_u *name);
 int get_var_dest(char_u *name, assign_dest_T *dest, cmdidx_T cmdidx, int *option_scope, int *vimvaridx, type_T **type, cctx_T *cctx);
diff --git a/src/testdir/test_vim9_assign.vim b/src/testdir/test_vim9_assign.vim
index 02274b3c5..2c3db657e 100644
--- a/src/testdir/test_vim9_assign.vim
+++ b/src/testdir/test_vim9_assign.vim
@@ -1821,10 +1821,31 @@ def Test_assign_lambda()
 enddef
 
 def Test_heredoc()
-  var lines =<< trim END # comment
-    text
+  # simple heredoc
+  var lines =<< trim END
+    var text =<< trim TEXT # comment
+      abc
+    TEXT
+    assert_equal(['abc'], text)
   END
-  assert_equal(['text'], lines)
+  v9.CheckDefAndScriptSuccess(lines)
+
+  # empty heredoc
+  lines =<< trim END
+     var text =<< trim TEXT
+     TEXT
+     assert_equal([], text)
+  END
+  v9.CheckDefAndScriptSuccess(lines)
+
+  # heredoc with a single empty line
+  lines =<< trim END
+     var text =<< trim TEXT
+
+     TEXT
+     assert_equal([''], text)
+  END
+  v9.CheckDefAndScriptSuccess(lines)
 
   v9.CheckDefFailure(['var lines =<< trim END X', 'END'], 'E488:')
   v9.CheckDefFailure(['var lines =<< trim END " comment', 'END'], 'E488:')
@@ -2642,51 +2663,68 @@ let g:someVar = 'X'
 " Test for heredoc with Vim expressions.
 " This messes up highlighting, keep it near the end.
 def Test_heredoc_expr()
-  var code =<< trim eval END
-    var a = `=5 + 10`
-    var b = `=min([10, 6])` + `=max([4, 6])`
-  END
-  assert_equal(['var a = 15', 'var b = 6 + 6'], code)
+  var lines =<< trim CODE
+    var s = "local"
+    var a1 = "1"
+    var a2 = "2"
+    var a3 = "3"
+    var a4 = ""
+    var code =<< trim eval END
+      var a = `=5 + 10`
+      var b = `=min([10, 6])` + `=max([4, 6])`
+      var c = "`=s`"
+      var d = x`=a1`x`=a2`x`=a3`x`=a4`
+    END
+    assert_equal(['var a = 15', 'var b = 6 + 6', 'var c = "local"', 'var d = x1x2x3x'], code)
+  CODE
+  v9.CheckDefAndScriptSuccess(lines)
 
-  code =<< eval trim END
-    var s = "`=$SOME_ENV_VAR`"
-  END
-  assert_equal(['var s = "somemore"'], code)
+  lines =<< trim CODE
+    var code =<< eval trim END
+      var s = "`=$SOME_ENV_VAR`"
+    END
+    assert_equal(['var s = "somemore"'], code)
+  CODE
+  v9.CheckDefAndScriptSuccess(lines)
 
-  code =<< eval END
-    var s = "`=$SOME_ENV_VAR`"
-END
-  assert_equal(['    var s = "somemore"'], code)
+  lines =<< trim CODE
+    var code =<< eval END
+      var s = "`=$SOME_ENV_VAR`"
+    END
+    assert_equal(['  var s = "somemore"'], code)
+  CODE
+  v9.CheckDefAndScriptSuccess(lines)
 
-  code =<< eval trim END
-    let a = `abc`
-    let b = `=g:someVar`
-    let c = `
-  END
-  assert_equal(['let a = `abc`', 'let b = X', 'let c = `'], code)
+  lines =<< trim CODE
+    var code =<< eval trim END
+      let a = `abc`
+      let b = `=g:someVar`
+      let c = `
+    END
+    assert_equal(['let a = `abc`', 'let b = X', 'let c = `'], code)
+  CODE
+  v9.CheckDefAndScriptSuccess(lines)
 
-  var lines =<< trim LINES
+  lines =<< trim LINES
       var text =<< eval trim END
         let b = `=
       END
   LINES
-  v9.CheckDefAndScriptFailure(lines, 'E1083:')
+  v9.CheckDefAndScriptFailure(lines, ['E1143: Empty expression: ""', 'E1083: Missing backtick'])
 
   lines =<< trim LINES
       var text =<< eval trim END
         let b = `=abc
       END
   LINES
-  v9.CheckDefAndScriptFailure(lines, 'E1083:')
+  v9.CheckDefAndScriptFailure(lines, ['E1001: Variable not found: abc', 'E1083: Missing backtick'])
 
   lines =<< trim LINES
       var text =<< eval trim END
         let b = `=`
       END
   LINES
-  v9.CheckDefAndScriptFailure(lines, 'E15:')
+  v9.CheckDefAndScriptFailure(lines, ['E1015: Name expected: `', 'E15: Invalid expression: "`"'])
 enddef
 
-
-
 " vim: ts=8 sw=2 sts=2 expandtab tw=80 fdm=marker
diff --git a/src/version.c b/src/version.c
index e4e297e03..4c914af0d 100644
--- a/src/version.c
+++ b/src/version.c
@@ -747,6 +747,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    4804,
+/**/
     4803,
 /**/
     4802,
diff --git a/src/vim9compile.c b/src/vim9compile.c
index 205d9a9a6..88a4d57d7 100644
--- a/src/vim9compile.c
+++ b/src/vim9compile.c
@@ -595,6 +595,7 @@ find_imported_in_script(char_u *name, size_t len, int sid)
 
 /*
  * Find "name" in imported items of the current script.
+ * If "len" is 0 use any length that works.
  * If "load" is TRUE and the script was not loaded yet, load it now.
  */
     imported_T *
@@ -968,6 +969,83 @@ theend:
 }
 
 /*
+ * Compile a heredoc string "str" (either containing a literal string or a mix
+ * of literal strings and Vim expressions of the form `=<expr>`).  This is used
+ * when compiling a heredoc assignment to a variable in a Vim9 def function.
+ * Vim9 instructions are generated to push strings, evaluate expressions,
+ * concatenate them and create a list of lines.  When "evalstr" is TRUE, Vim
+ * expressions in "str" are evaluated.
+ */
+    int
+compile_heredoc_string(char_u *str, int evalstr, cctx_T *cctx)
+{
+    char_u	*p;
+    char_u	*val;
+
+    if (cctx->ctx_skip == SKIP_YES)
+	return OK;
+
+    if (evalstr && (p = (char_u *)strstr((char *)str, "`=")) != NULL)
+    {
+	char_u	*start = str;
+
+	// Need to evaluate expressions of the form `=<expr>` in the string.
+	// Split the string into literal strings and Vim expressions and
+	// generate instructions to concatenate the literal strings and the
+	// result of evaluating the Vim expressions.
+	val = vim_strsave((char_u *)"");
+	generate_PUSHS(cctx, &val);
+
+	for (;;)
+	{
+	    if (p > start)
+	    {
+		// literal string before the expression
+		val = vim_strnsave(start, p - start);
+		generate_PUSHS(cctx, &val);
+		generate_instr_drop(cctx, ISN_CONCAT, 1);
+	    }
+	    p += 2;
+
+	    // evaluate the Vim expression and convert the result to string.
+	    if (compile_expr0(&p, cctx) == FAIL)
+		return FAIL;
+	    may_generate_2STRING(-1, TRUE, cctx);
+	    generate_instr_drop(cctx, ISN_CONCAT, 1);
+
+	    p = skipwhite(p);
+	    if (*p != '`')
+	    {
+		emsg(_(e_missing_backtick));
+		return FAIL;
+	    }
+	    start = p + 1;
+
+	    p = (char_u *)strstr((char *)start, "`=");
+	    if (p == NULL)
+	    {
+		// no more Vim expressions left to process
+		if (*skipwhite(start) != NUL)
+		{
+		    val = vim_strsave(start);
+		    generate_PUSHS(cctx, &val);
+		    generate_instr_drop(cctx, ISN_CONCAT, 1);
+		}
+		break;
+	    }
+	}
+    }
+    else
+    {
+	// literal string
+	val = vim_strsave(str);
+	generate_PUSHS(cctx, &val);
+    }
+
+    return OK;
+}
+
+/*
  * Return the length of an assignment operator, or zero if there isn't one.
  */
     int
@@ -1946,25 +2024,14 @@ compile_assignment(char_u *arg, exarg_T *eap, cmdidx_T cmdidx, cctx_T *cctx)
     if (heredoc)
     {
 	list_T	   *l;
-	listitem_T *li;
 
 	// [let] varname =<< [trim] {end}
 	eap->getline = exarg_getline;
 	eap->cookie = cctx;
-	l = heredoc_get(eap, op + 3, FALSE);
+	l = heredoc_get(eap, op + 3, FALSE, TRUE);
 	if (l == NULL)
 	    return NULL;
 
-	if (cctx->ctx_skip != SKIP_YES)
-	{
-	    // Push each line and the create the list.
-	    FOR_ALL_LIST_ITEMS(l, li)
-	    {
-		generate_PUSHS(cctx, &li->li_tv.vval.v_string);
-		li->li_tv.vval.v_string = NULL;
-	    }
-	    generate_NEWLIST(cctx, l->lv_len, FALSE);
-	}
 	list_free(l);
 	p += STRLEN(p);
 	end = p;