patch 8.2.1491: Vim9: crash when compiling heredoc lines start with commentv8.2.1491

Problem:    Vim9: crash when compiling heredoc lines start with comment.
Solution:   Skip over NULL pointers. Do not remove comment and empty lines
            when fetching function lines. (closes #6743)

16 files changed, 81 insertions, 45 deletions

diff --git a/src/autocmd.c b/src/autocmd.c
index d5b1651b7..d5c61ca6a 100644
--- a/src/autocmd.c
+++ b/src/autocmd.c
@@ -2310,7 +2310,11 @@ auto_next_pat(
  * Returns allocated string, or NULL for end of autocommands.
  */
     char_u *
-getnextac(int c UNUSED, void *cookie, int indent UNUSED, int do_concat UNUSED)
+getnextac(
+	int c UNUSED,
+	void *cookie,
+	int indent UNUSED,
+	getline_opt_T options UNUSED)
 {
     AutoPatCmd	    *acp = (AutoPatCmd *)cookie;
     char_u	    *retval;
diff --git a/src/evalfunc.c b/src/evalfunc.c
index c759afcd2..c8747e2c1 100644
--- a/src/evalfunc.c
+++ b/src/evalfunc.c
@@ -2198,13 +2198,12 @@ execute_redir_str(char_u *value, int value_len)
  * Called by do_cmdline() to get the next line.
  * Returns allocated string, or NULL for end of function.
  */
-
     static char_u *
 get_list_line(
     int	    c UNUSED,
     void    *cookie,
     int	    indent UNUSED,
-    int	    do_concat UNUSED)
+    getline_opt_T options UNUSED)
 {
     listitem_T **p = (listitem_T **)cookie;
     listitem_T *item = *p;
diff --git a/src/ex_cmds.h b/src/ex_cmds.h
index b993ef21a..3977c4d6e 100644
--- a/src/ex_cmds.h
+++ b/src/ex_cmds.h
@@ -1867,7 +1867,7 @@ struct exarg
     int		bad_char;	// BAD_KEEP, BAD_DROP or replacement byte
     int		useridx;	// user command index
     char	*errmsg;	// returned error message
-    char_u	*(*getline)(int, void *, int, int);
+    char_u	*(*getline)(int, void *, int, getline_opt_T);
     void	*cookie;	// argument for getline()
 #ifdef FEAT_EVAL
     cstack_T	*cstack;	// condition stack for ":if" etc.
diff --git a/src/ex_docmd.c b/src/ex_docmd.c
index 612d478e9..4a6da4da0 100644
--- a/src/ex_docmd.c
+++ b/src/ex_docmd.c
@@ -612,7 +612,7 @@ do_cmdline_cmd(char_u *cmd)
     int
 do_cmdline(
     char_u	*cmdline,
-    char_u	*(*fgetline)(int, void *, int, int),
+    char_u	*(*fgetline)(int, void *, int, getline_opt_T),
     void	*cookie,		// argument for fgetline()
     int		flags)
 {
@@ -638,7 +638,7 @@ do_cmdline(
     msglist_T	*private_msg_list;
 
     // "fgetline" and "cookie" passed to do_one_cmd()
-    char_u	*(*cmd_getline)(int, void *, int, int);
+    char_u	*(*cmd_getline)(int, void *, int, getline_opt_T);
     void	*cmd_cookie;
     struct loop_cookie cmd_loop_cookie;
     void	*real_cookie;
@@ -1482,9 +1482,9 @@ free_cmdlines(garray_T *gap)
  */
     int
 getline_equal(
-    char_u	*(*fgetline)(int, void *, int, int),
+    char_u	*(*fgetline)(int, void *, int, getline_opt_T),
     void	*cookie UNUSED,		// argument for fgetline()
-    char_u	*(*func)(int, void *, int, int))
+    char_u	*(*func)(int, void *, int, getline_opt_T))
 {
 #ifdef FEAT_EVAL
     char_u		*(*gp)(int, void *, int, int);
@@ -1512,7 +1512,7 @@ getline_equal(
  */
     void *
 getline_cookie(
-    char_u	*(*fgetline)(int, void *, int, int) UNUSED,
+    char_u	*(*fgetline)(int, void *, int, getline_opt_T) UNUSED,
     void	*cookie)		// argument for fgetline()
 {
 #ifdef FEAT_EVAL
@@ -1541,7 +1541,7 @@ getline_cookie(
  */
     char_u *
 getline_peek(
-    char_u	*(*fgetline)(int, void *, int, int) UNUSED,
+    char_u	*(*fgetline)(int, void *, int, getline_opt_T) UNUSED,
     void	*cookie)		// argument for fgetline()
 {
     char_u		*(*gp)(int, void *, int, int);
diff --git a/src/ex_getln.c b/src/ex_getln.c
index 702b63db5..60a8a0c71 100644
--- a/src/ex_getln.c
+++ b/src/ex_getln.c
@@ -2705,12 +2705,12 @@ getexline(
     int		c,		// normally ':', NUL for ":append"
     void	*cookie UNUSED,
     int		indent,		// indent for inside conditionals
-    int		do_concat)
+    getline_opt_T options)
 {
     // When executing a register, remove ':' that's in front of each line.
     if (exec_from_reg && vpeekc() == ':')
 	(void)vgetc();
-    return getcmdline(c, 1L, indent, do_concat);
+    return getcmdline(c, 1L, indent, options);
 }
 
 /*
@@ -2725,7 +2725,7 @@ getexmodeline(
 				// :s prompt
     void	*cookie UNUSED,
     int		indent,		// indent for inside conditionals
-    int		do_concat UNUSED)
+    getline_opt_T options UNUSED)
 {
     garray_T	line_ga;
     char_u	*pend;
diff --git a/src/proto/autocmd.pro b/src/proto/autocmd.pro
index 65a63526a..88eae5a78 100644
--- a/src/proto/autocmd.pro
+++ b/src/proto/autocmd.pro
@@ -28,7 +28,7 @@ int has_completechanged(void);
 void block_autocmds(void);
 void unblock_autocmds(void);
 int is_autocmd_blocked(void);
-char_u *getnextac(int c, void *cookie, int indent, int do_concat);
+char_u *getnextac(int c, void *cookie, int indent, getline_opt_T options);
 int has_autocmd(event_T event, char_u *sfname, buf_T *buf);
 char_u *get_augroup_name(expand_T *xp, int idx);
 char_u *set_context_in_autocmd(expand_T *xp, char_u *arg, int doautocmd);
diff --git a/src/proto/ex_docmd.pro b/src/proto/ex_docmd.pro
index 2c70eb20f..1955ccf6c 100644
--- a/src/proto/ex_docmd.pro
+++ b/src/proto/ex_docmd.pro
@@ -1,10 +1,10 @@
 /* ex_docmd.c */
 void do_exmode(int improved);
 int do_cmdline_cmd(char_u *cmd);
-int do_cmdline(char_u *cmdline, char_u *(*fgetline)(int, void *, int, int), void *cookie, int flags);
-int getline_equal(char_u *(*fgetline)(int, void *, int, int), void *cookie, char_u *(*func)(int, void *, int, int));
-void *getline_cookie(char_u *(*fgetline)(int, void *, int, int), void *cookie);
-char_u *getline_peek(char_u *(*fgetline)(int, void *, int, int), void *cookie);
+int do_cmdline(char_u *cmdline, char_u *(*fgetline)(int, void *, int, getline_opt_T), void *cookie, int flags);
+int getline_equal(char_u *(*fgetline)(int, void *, int, getline_opt_T), void *cookie, char_u *(*func)(int, void *, int, getline_opt_T));
+void *getline_cookie(char_u *(*fgetline)(int, void *, int, getline_opt_T), void *cookie);
+char_u *getline_peek(char_u *(*fgetline)(int, void *, int, getline_opt_T), void *cookie);
 char *ex_errmsg(char *msg, char_u *arg);
 int parse_command_modifiers(exarg_T *eap, char **errormsg, int skip_only);
 void undo_cmdmod(exarg_T *eap, int save_msg_scroll);
diff --git a/src/proto/ex_getln.pro b/src/proto/ex_getln.pro
index f64bb1ff5..faecab2a7 100644
--- a/src/proto/ex_getln.pro
+++ b/src/proto/ex_getln.pro
@@ -9,8 +9,8 @@ char *get_text_locked_msg(void);
 int text_locked(void);
 int curbuf_locked(void);
 int allbuf_locked(void);
-char_u *getexline(int c, void *cookie, int indent, int do_concat);
-char_u *getexmodeline(int promptc, void *cookie, int indent, int do_concat);
+char_u *getexline(int c, void *cookie, int indent, getline_opt_T options);
+char_u *getexmodeline(int promptc, void *cookie, int indent, getline_opt_T options);
 int cmdline_overstrike(void);
 int cmdline_at_end(void);
 colnr_T cmdline_getvcol_cursor(void);
diff --git a/src/proto/scriptfile.pro b/src/proto/scriptfile.pro
index 00f1301ee..52df2ca82 100644
--- a/src/proto/scriptfile.pro
+++ b/src/proto/scriptfile.pro
@@ -29,13 +29,13 @@ void scriptnames_slash_adjust(void);
 char_u *get_scriptname(scid_T id);
 void free_scriptnames(void);
 void free_autoload_scriptnames(void);
-linenr_T get_sourced_lnum(char_u *(*fgetline)(int, void *, int, int), void *cookie);
-char_u *getsourceline(int c, void *cookie, int indent, int do_concat);
+linenr_T get_sourced_lnum(char_u *(*fgetline)(int, void *, int, getline_opt_T), void *cookie);
+char_u *getsourceline(int c, void *cookie, int indent, getline_opt_T options);
 void ex_scriptencoding(exarg_T *eap);
 void ex_scriptversion(exarg_T *eap);
 void ex_finish(exarg_T *eap);
 void do_finish(exarg_T *eap, int reanimate);
-int source_finished(char_u *(*fgetline)(int, void *, int, int), void *cookie);
+int source_finished(char_u *(*fgetline)(int, void *, int, getline_opt_T), void *cookie);
 char_u *autoload_name(char_u *name);
 int script_autoload(char_u *name, int reload);
 /* vim: set ft=c : */
diff --git a/src/proto/userfunc.pro b/src/proto/userfunc.pro
index 2c4cbd5d3..e6acc1886 100644
--- a/src/proto/userfunc.pro
+++ b/src/proto/userfunc.pro
@@ -46,7 +46,7 @@ void ex_call(exarg_T *eap);
 int do_return(exarg_T *eap, int reanimate, int is_cmd, void *rettv);
 void discard_pending_return(void *rettv);
 char_u *get_return_cmd(void *rettv);
-char_u *get_func_line(int c, void *cookie, int indent, int do_concat);
+char_u *get_func_line(int c, void *cookie, int indent, getline_opt_T options);
 int func_has_ended(void *cookie);
 int func_has_abort(void *cookie);
 dict_T *make_partial(dict_T *selfdict_in, typval_T *rettv);
diff --git a/src/scriptfile.c b/src/scriptfile.c
index 4df2c31fe..27d0eb2a5 100644
--- a/src/scriptfile.c
+++ b/src/scriptfile.c
@@ -1604,7 +1604,9 @@ free_autoload_scriptnames(void)
 #endif
 
     linenr_T
-get_sourced_lnum(char_u *(*fgetline)(int, void *, int, int), void *cookie)
+get_sourced_lnum(
+	char_u *(*fgetline)(int, void *, int, getline_opt_T),
+	void *cookie)
 {
     return fgetline == getsourceline
 			? ((struct source_cookie *)cookie)->sourcing_lnum
@@ -1724,7 +1726,11 @@ get_one_sourceline(struct source_cookie *sp)
  * Return NULL for end-of-file or some error.
  */
     char_u *
-getsourceline(int c UNUSED, void *cookie, int indent UNUSED, int do_concat)
+getsourceline(
+	int c UNUSED,
+	void *cookie,
+	int indent UNUSED,
+	getline_opt_T options)
 {
     struct source_cookie *sp = (struct source_cookie *)cookie;
     char_u		*line;
@@ -1765,7 +1771,8 @@ getsourceline(int c UNUSED, void *cookie, int indent UNUSED, int do_concat)
 
     // Only concatenate lines starting with a \ when 'cpoptions' doesn't
     // contain the 'C' flag.
-    if (line != NULL && do_concat && vim_strchr(p_cpo, CPO_CONCAT) == NULL)
+    if (line != NULL && options != GETLINE_NONE
+				      && vim_strchr(p_cpo, CPO_CONCAT) == NULL)
     {
 	// compensate for the one line read-ahead
 	--sp->sourcing_lnum;
@@ -1781,7 +1788,8 @@ getsourceline(int c UNUSED, void *cookie, int indent UNUSED, int do_concat)
 			      || (p[0] == '"' && p[1] == '\\' && p[2] == ' ')
 #ifdef FEAT_EVAL
 			      || (in_vim9script()
-				       && (*p == NUL || vim9_comment_start(p)))
+				      && options == GETLINE_CONCAT_ALL
+				      && (*p == NUL || vim9_comment_start(p)))
 #endif
 			      ))
 	{
@@ -1814,7 +1822,8 @@ getsourceline(int c UNUSED, void *cookie, int indent UNUSED, int do_concat)
 		else if (!(p[0] == '"' && p[1] == '\\' && p[2] == ' ')
 #ifdef FEAT_EVAL
 			&& !(in_vim9script()
-				       && (*p == NUL || vim9_comment_start(p)))
+				&& options == GETLINE_CONCAT_ALL
+				&& (*p == NUL || vim9_comment_start(p)))
 #endif
 			)
 		    break;
@@ -1968,7 +1977,7 @@ do_finish(exarg_T *eap, int reanimate)
  */
     int
 source_finished(
-    char_u	*(*fgetline)(int, void *, int, int),
+    char_u	*(*fgetline)(int, void *, int, getline_opt_T),
     void	*cookie)
 {
     return (getline_equal(fgetline, cookie, getsourceline)
diff --git a/src/structs.h b/src/structs.h
index c94fa94e2..d8545112b 100644
--- a/src/structs.h
+++ b/src/structs.h
@@ -1761,6 +1761,13 @@ typedef struct
 # endif
 } scriptitem_T;
 
+// type of getline() last argument
+typedef enum {
+    GETLINE_NONE,	    // do not concatenate any lines
+    GETLINE_CONCAT_CONT,    // concatenate continuation lines
+    GETLINE_CONCAT_ALL	    // concatenate continuation and Vim9 # comment lines
+} getline_opt_T;
+
 // Struct passed through eval() functions.
 // See EVALARG_EVALUATE for a fixed value with eval_flags set to EVAL_EVALUATE.
 typedef struct {
@@ -1768,7 +1775,7 @@ typedef struct {
     int		eval_break_count;   // nr of line breaks consumed
 
     // copied from exarg_T when "getline" is "getsourceline". Can be NULL.
-    char_u	*(*eval_getline)(int, void *, int, int);
+    char_u	*(*eval_getline)(int, void *, int, getline_opt_T);
     void	*eval_cookie;	    // argument for eval_getline()
 
     // used when compiling a :def function, NULL otherwise
diff --git a/src/testdir/test_vim9_script.vim b/src/testdir/test_vim9_script.vim
index 18e64105f..d442dc254 100644
--- a/src/testdir/test_vim9_script.vim
+++ b/src/testdir/test_vim9_script.vim
@@ -1141,6 +1141,17 @@ def Test_list_vimscript()
       assert_equal(['one', 'two', 'three'], mylist)
   END
   CheckScriptSuccess(lines)
+
+  # check all lines from heredoc are kept
+  lines =<< trim END
+      # comment 1
+      two
+      # comment 3
+
+      five
+      # comment 6
+  END
+  assert_equal(['# comment 1', 'two', '# comment 3', '', 'five', '# comment 6'], lines)
 enddef
 
 if has('channel')
diff --git a/src/userfunc.c b/src/userfunc.c
index 1b6eff893..0f6388d72 100644
--- a/src/userfunc.c
+++ b/src/userfunc.c
@@ -2651,7 +2651,7 @@ def_function(exarg_T *eap, char_u *name_arg)
     static int	func_nr = 0;	    // number for nameless function
     int		paren;
     hashitem_T	*hi;
-    int		do_concat = TRUE;
+    getline_opt_T getline_options = GETLINE_CONCAT_CONT;
     linenr_T	sourcing_lnum_off;
     linenr_T	sourcing_lnum_top;
     int		is_heredoc = FALSE;
@@ -3008,9 +3008,10 @@ def_function(exarg_T *eap, char_u *name_arg)
 	{
 	    vim_free(line_to_free);
 	    if (eap->getline == NULL)
-		theline = getcmdline(':', 0L, indent, do_concat);
+		theline = getcmdline(':', 0L, indent, getline_options);
 	    else
-		theline = eap->getline(':', eap->cookie, indent, do_concat);
+		theline = eap->getline(':', eap->cookie, indent,
+							      getline_options);
 	    line_to_free = theline;
 	}
 	if (KeyTyped)
@@ -3053,7 +3054,7 @@ def_function(exarg_T *eap, char_u *name_arg)
 		{
 		    VIM_CLEAR(skip_until);
 		    VIM_CLEAR(heredoc_trimmed);
-		    do_concat = TRUE;
+		    getline_options = GETLINE_CONCAT_CONT;
 		    is_heredoc = FALSE;
 		}
 	    }
@@ -3178,7 +3179,7 @@ def_function(exarg_T *eap, char_u *name_arg)
 		    skip_until = vim_strsave((char_u *)".");
 		else
 		    skip_until = vim_strnsave(p, skiptowhite(p) - p);
-		do_concat = FALSE;
+		getline_options = GETLINE_NONE;
 		is_heredoc = TRUE;
 	    }
 
@@ -3205,7 +3206,7 @@ def_function(exarg_T *eap, char_u *name_arg)
 						 skipwhite(theline) - theline);
 		    }
 		    skip_until = vim_strnsave(p, skiptowhite(p) - p);
-		    do_concat = FALSE;
+		    getline_options = GETLINE_NONE;
 		    is_heredoc = TRUE;
 		}
 	    }
@@ -4249,7 +4250,7 @@ get_func_line(
     int	    c UNUSED,
     void    *cookie,
     int	    indent UNUSED,
-    int	    do_concat UNUSED)
+    getline_opt_T options UNUSED)
 {
     funccall_T	*fcp = (funccall_T *)cookie;
     ufunc_T	*fp = fcp->func;
diff --git a/src/version.c b/src/version.c
index 8205b4d0f..c5ce04cba 100644
--- a/src/version.c
+++ b/src/version.c
@@ -755,6 +755,8 @@ static char *(features[]) =
 static int included_patches[] =
 {   /* Add new patch number below this line */
 /**/
+    1491,
+/**/
     1490,
 /**/
     1489,
diff --git a/src/vim9compile.c b/src/vim9compile.c
index 37ecf28c8..00572e316 100644
--- a/src/vim9compile.c
+++ b/src/vim9compile.c
@@ -4197,18 +4197,21 @@ exarg_getline(
 	int c UNUSED,
 	void *cookie,
 	int indent UNUSED,
-	int do_concat UNUSED)
+	getline_opt_T options UNUSED)
 {
     cctx_T  *cctx = (cctx_T *)cookie;
+    char_u  *p;
 
-    if (cctx->ctx_lnum == cctx->ctx_ufunc->uf_lines.ga_len)
+    for (;;)
     {
-	iemsg("Heredoc got to end");
-	return NULL;
+	if (cctx->ctx_lnum == cctx->ctx_ufunc->uf_lines.ga_len)
+	    return NULL;
+	++cctx->ctx_lnum;
+	p = ((char_u **)cctx->ctx_ufunc->uf_lines.ga_data)[cctx->ctx_lnum];
+	// Comment lines result in NULL pointers, skip them.
+	if (p != NULL)
+	    return vim_strsave(p);
     }
-    ++cctx->ctx_lnum;
-    return vim_strsave(((char_u **)cctx->ctx_ufunc->uf_lines.ga_data)
-							     [cctx->ctx_lnum]);
 }
 
 /*