Patch 8.2.4804

[Bram Moolenaar]

Patch 8.2.4804
Problem: Expression in heredoc doesn't work for compiled function.
Solution: Implement compiling the heredoc expressions. (Yegappan Lakshmanan,
closes #10232)
Files: runtime/doc/eval.txt, src/evalvars.c, src/proto/evalvars.pro,
src/ex_getln.c, src/vim9compile.c, src/proto/vim9compile.pro,
src/testdir/test_vim9_assign.vim


*** ../vim-8.2.4803/runtime/doc/eval.txt 2022-04-17 12:46:50.101294003 +0100
--- runtime/doc/eval.txt 2022-04-21 23:19:41.837086804 +0100
***************
*** 3210,3217 ****
expression evaluation fails, then the assignment fails.
once the "`=" has been found {expr} and a backtick
must follow. {expr} cannot be empty.
- Currenty, in a compiled function {expr} is evaluated
- when compiling the function, THIS WILL CHANGE.

{endmarker} must not contain white space.
{endmarker} cannot start with a lower case character.
--- 3247,3252 ----
*** ../vim-8.2.4803/src/evalvars.c 2022-04-18 15:45:19.704436521 +0100
--- src/evalvars.c 2022-04-21 23:25:57.708784208 +0100
***************
*** 673,688 ****
*
* The {marker} is a string. If the optional 'trim' word is supplied before the
* marker, then the leading indentation before the lines (matching the
! * indentation in the 'cmd' line) is stripped.
*
* When getting lines for an embedded script (e.g. python, lua, perl, ruby,
! * tcl, mzscheme), script_get is set to TRUE. In this case, if the marker is
* missing, then '.' is accepted as a marker.
*
* Returns a List with {lines} or NULL on failure.
*/
list_T *
! heredoc_get(exarg_T *eap, char_u *cmd, int script_get)
{
char_u *theline = NULL;
char_u *marker;
--- 673,693 ----
*
* The {marker} is a string. If the optional 'trim' word is supplied before the
* marker, then the leading indentation before the lines (matching the
! * indentation in the "cmd" line) is stripped.
*
* When getting lines for an embedded script (e.g. python, lua, perl, ruby,
! * tcl, mzscheme), "script_get" is set to TRUE. In this case, if the marker is
* missing, then '.' is accepted as a marker.
*
+ * When compiling a heredoc assignment to a variable in a Vim9 def function,
+ * "vim9compile" is set to TRUE. In this case, instead of generating a list of
+ * string values from the heredoc, vim9 instructions are generated. On success
+ * the returned list will be empty.
+ *
* Returns a List with {lines} or NULL on failure.
*/
list_T *
! heredoc_get(exarg_T *eap, char_u *cmd, int script_get, int vim9compile)
{
char_u *theline = NULL;
char_u *marker;
***************
*** 696,701 ****
--- 701,708 ----
int comment_char = in_vim9script() ? '#' : '"';
int evalstr = FALSE;
int eval_failed = FALSE;
+ cctx_T *cctx = vim9compile ? eap->cookie : NULL;
+ int count = 0;

if (eap->getline == NULL)
{
***************
*** 816,840 ****
break;

str = theline + ti;
! if (evalstr)
{
! str = eval_all_expr_in_str(str);
! if (str == NULL)
{
! // expression evaluation failed
! eval_failed = TRUE;
! continue;
}
! vim_free(theline);
! theline = str;
}

! if (list_append_string(l, str, -1) == FAIL)
! break;
}
vim_free(theline);
vim_free(text_indent);

if (eval_failed)
{
// expression evaluation in the heredoc failed
--- 823,863 ----
break;

str = theline + ti;
! if (vim9compile)
{
! if (compile_heredoc_string(str, evalstr, cctx) == FAIL)
{
! vim_free(theline);
! vim_free(text_indent);
! return FAIL;
}
! count++;
}
+ else
+ {
+ if (evalstr)
+ {
+ str = eval_all_expr_in_str(str);
+ if (str == NULL)
+ {
+ // expression evaluation failed
+ eval_failed = TRUE;
+ continue;
+ }
+ vim_free(theline);
+ theline = str;
+ }

! if (list_append_string(l, str, -1) == FAIL)
! break;
! }
}
vim_free(theline);
vim_free(text_indent);

+ if (vim9compile && cctx->ctx_skip != SKIP_YES && !eval_failed)
+ generate_NEWLIST(cctx, count, FALSE);
+
if (eval_failed)
{
// expression evaluation in the heredoc failed
***************
*** 986,992 ****
long cur_lnum = SOURCING_LNUM;

// HERE document
! l = heredoc_get(eap, expr + 3, FALSE);
if (l != NULL)
{
rettv_list_set(&rettv, l);
--- 1009,1015 ----
long cur_lnum = SOURCING_LNUM;

// HERE document
! l = heredoc_get(eap, expr + 3, FALSE, FALSE);
if (l != NULL)
{
rettv_list_set(&rettv, l);
*** ../vim-8.2.4803/src/proto/evalvars.pro 2022-03-20 17:46:01.797053490 +0000
--- src/proto/evalvars.pro 2022-04-21 23:19:41.837086804 +0100
***************
*** 13,19 ****
int get_spellword(list_T *list, char_u **pp);
void prepare_vimvar(int idx, typval_T *save_tv);
void restore_vimvar(int idx, typval_T *save_tv);
! list_T *heredoc_get(exarg_T *eap, char_u *cmd, int script_get);
void ex_var(exarg_T *eap);
void ex_let(exarg_T *eap);
int ex_let_vars(char_u *arg_start, typval_T *tv, int copy, int semicolon, int var_count, int flags, char_u *op);
--- 13,19 ----
int get_spellword(list_T *list, char_u **pp);
void prepare_vimvar(int idx, typval_T *save_tv);
void restore_vimvar(int idx, typval_T *save_tv);
! list_T *heredoc_get(exarg_T *eap, char_u *cmd, int script_get, int vim9compile);
void ex_var(exarg_T *eap);
void ex_let(exarg_T *eap);
int ex_let_vars(char_u *arg_start, typval_T *tv, int copy, int semicolon, int var_count, int flags, char_u *op);
*** ../vim-8.2.4803/src/ex_getln.c 2022-04-15 13:53:30.048708690 +0100
--- src/ex_getln.c 2022-04-21 23:19:41.837086804 +0100
***************
*** 4605,4611 ****
return NULL;
cmd += 2;

! l = heredoc_get(eap, cmd, TRUE);
if (l == NULL)
return NULL;

--- 4605,4611 ----
return NULL;
cmd += 2;

! l = heredoc_get(eap, cmd, TRUE, FALSE);
if (l == NULL)
return NULL;

*** ../vim-8.2.4803/src/vim9compile.c 2022-04-02 19:43:53.927491819 +0100
--- src/vim9compile.c 2022-04-21 23:28:38.244637908 +0100
***************
*** 595,600 ****
--- 595,601 ----

/*
* Find "name" in imported items of the current script.
+ * If "len" is 0 use any length that works.
* If "load" is TRUE and the script was not loaded yet, load it now.
*/
imported_T *
***************
*** 968,973 ****
--- 969,1051 ----
}

/*
+ * Compile a heredoc string "str" (either containing a literal string or a mix
+ * of literal strings and Vim expressions of the form `=<expr>`). This is used
+ * when compiling a heredoc assignment to a variable in a Vim9 def function.
+ * Vim9 instructions are generated to push strings, evaluate expressions,
+ * concatenate them and create a list of lines. When "evalstr" is TRUE, Vim
+ * expressions in "str" are evaluated.
+ */
+ int
+ compile_heredoc_string(char_u *str, int evalstr, cctx_T *cctx)
+ {
+ char_u *p;
+ char_u *val;
+
+ if (cctx->ctx_skip == SKIP_YES)
+ return OK;
+
+ if (evalstr && (p = (char_u *)strstr((char *)str, "`=")) != NULL)
+ {
+ char_u *start = str;
+
+ // Need to evaluate expressions of the form `=<expr>` in the string.
+ // Split the string into literal strings and Vim expressions and
+ // generate instructions to concatenate the literal strings and the
+ // result of evaluating the Vim expressions.
+ val = vim_strsave((char_u *)"");
+ generate_PUSHS(cctx, &val);
+
+ for (;;)
+ {
+ if (p > start)
+ {
+ // literal string before the expression
+ val = vim_strnsave(start, p - start);
+ generate_PUSHS(cctx, &val);
+ generate_instr_drop(cctx, ISN_CONCAT, 1);
+ }
+ p += 2;
+
+ // evaluate the Vim expression and convert the result to string.
+ if (compile_expr0(&p, cctx) == FAIL)
+ return FAIL;
+ may_generate_2STRING(-1, TRUE, cctx);
+ generate_instr_drop(cctx, ISN_CONCAT, 1);
+
+ p = skipwhite(p);
+ if (*p != '`')
+ {
+ emsg(_(e_missing_backtick));
+ return FAIL;
+ }
+ start = p + 1;
+
+ p = (char_u *)strstr((char *)start, "`=");
+ if (p == NULL)
+ {
+ // no more Vim expressions left to process
+ if (*skipwhite(start) != NUL)
+ {
+ val = vim_strsave(start);
+ generate_PUSHS(cctx, &val);
+ generate_instr_drop(cctx, ISN_CONCAT, 1);
+ }
+ break;
+ }
+ }
+ }
+ else
+ {
+ // literal string
+ val = vim_strsave(str);
+ generate_PUSHS(cctx, &val);
+ }
+
+ return OK;
+ }
+
+ /*
* Return the length of an assignment operator, or zero if there isn't one.
*/
int
***************
*** 1946,1970 ****
if (heredoc)
{
list_T *l;
- listitem_T *li;

// [let] varname =<< [trim] {end}
eap->getline = exarg_getline;
eap->cookie = cctx;
! l = heredoc_get(eap, op + 3, FALSE);
if (l == NULL)
return NULL;

- if (cctx->ctx_skip != SKIP_YES)
- {
- // Push each line and the create the list.
- FOR_ALL_LIST_ITEMS(l, li)
- {
- generate_PUSHS(cctx, &li->li_tv.vval.v_string);
- li->li_tv.vval.v_string = NULL;
- }
- generate_NEWLIST(cctx, l->lv_len, FALSE);
- }
list_free(l);
p += STRLEN(p);
end = p;
--- 2024,2037 ----
if (heredoc)
{
list_T *l;

// [let] varname =<< [trim] {end}
eap->getline = exarg_getline;
eap->cookie = cctx;
! l = heredoc_get(eap, op + 3, FALSE, TRUE);
if (l == NULL)
return NULL;

list_free(l);
p += STRLEN(p);
end = p;
*** ../vim-8.2.4803/src/proto/vim9compile.pro 2022-04-02 19:43:53.927491819 +0100
--- src/proto/vim9compile.pro 2022-04-21 23:19:41.837086804 +0100
***************
*** 16,21 ****
--- 16,22 ----
int may_get_next_line_error(char_u *whitep, char_u **arg, cctx_T *cctx);
void fill_exarg_from_cctx(exarg_T *eap, cctx_T *cctx);
int func_needs_compiling(ufunc_T *ufunc, compiletype_T compile_type);
+ int compile_heredoc_string(char_u *str, int evalstr, cctx_T *cctx);
int assignment_len(char_u *p, int *heredoc);
void vim9_declare_error(char_u *name);
int get_var_dest(char_u *name, assign_dest_T *dest, cmdidx_T cmdidx, int *option_scope, int *vimvaridx, type_T **type, cctx_T *cctx);
*** ../vim-8.2.4803/src/testdir/test_vim9_assign.vim 2022-04-17 12:46:50.101294003 +0100
--- src/testdir/test_vim9_assign.vim 2022-04-21 23:19:41.841086802 +0100
***************
*** 1821,1830 ****
enddef

def Test_heredoc()
! var lines =<< trim END # comment
! text
END
! assert_equal(['text'], lines)

v9.CheckDefFailure(['var lines =<< trim END X', 'END'], 'E488:')
v9.CheckDefFailure(['var lines =<< trim END " comment', 'END'], 'E488:')
--- 1821,1851 ----
enddef

def Test_heredoc()
! # simple heredoc
! var lines =<< trim END
! var text =<< trim TEXT # comment
! abc
! TEXT
! assert_equal(['abc'], text)
! END
! v9.CheckDefAndScriptSuccess(lines)
!
! # empty heredoc
! lines =<< trim END
! var text =<< trim TEXT
! TEXT
! assert_equal([], text)
END
! v9.CheckDefAndScriptSuccess(lines)
!
! # heredoc with a single empty line
! lines =<< trim END
! var text =<< trim TEXT
!
! TEXT
! assert_equal([''], text)
! END
! v9.CheckDefAndScriptSuccess(lines)

v9.CheckDefFailure(['var lines =<< trim END X', 'END'], 'E488:')
v9.CheckDefFailure(['var lines =<< trim END " comment', 'END'], 'E488:')
***************
*** 2642,2692 ****
" Test for heredoc with Vim expressions.
" This messes up highlighting, keep it near the end.
def Test_heredoc_expr()
! var code =<< trim eval END
! var a = `=5 + 10`
! var b = `=min([10, 6])` + `=max([4, 6])`
! END
! assert_equal(['var a = 15', 'var b = 6 + 6'], code)

! code =<< eval trim END
! var s = "`=$SOME_ENV_VAR`"
! END
! assert_equal(['var s = "somemore"'], code)

! code =<< eval END
! var s = "`=$SOME_ENV_VAR`"
! END
! assert_equal([' var s = "somemore"'], code)

! code =<< eval trim END
! let a = `abc`
! let b = `=g:someVar`
! let c = `
! END
! assert_equal(['let a = `abc`', 'let b = X', 'let c = `'], code)

! var lines =<< trim LINES
var text =<< eval trim END
let b = `=
END
LINES
! v9.CheckDefAndScriptFailure(lines, 'E1083:')

lines =<< trim LINES
var text =<< eval trim END
let b = `=abc
END
LINES
! v9.CheckDefAndScriptFailure(lines, 'E1083:')

lines =<< trim LINES
var text =<< eval trim END
let b = `=`
END
LINES
! v9.CheckDefAndScriptFailure(lines, 'E15:')
enddef

-
-
" vim: ts=8 sw=2 sts=2 expandtab tw=80 fdm=marker
--- 2663,2730 ----
" Test for heredoc with Vim expressions.
" This messes up highlighting, keep it near the end.
def Test_heredoc_expr()
! var lines =<< trim CODE
! var s = "local"
! var a1 = "1"
! var a2 = "2"
! var a3 = "3"
! var a4 = ""
! var code =<< trim eval END
! var a = `=5 + 10`
! var b = `=min([10, 6])` + `=max([4, 6])`
! var c = "`=s`"
! var d = x`=a1`x`=a2`x`=a3`x`=a4`
! END
! assert_equal(['var a = 15', 'var b = 6 + 6', 'var c = "local"', 'var d = x1x2x3x'], code)
! CODE
! v9.CheckDefAndScriptSuccess(lines)

! lines =<< trim CODE
! var code =<< eval trim END
! var s = "`=$SOME_ENV_VAR`"
! END
! assert_equal(['var s = "somemore"'], code)
! CODE
! v9.CheckDefAndScriptSuccess(lines)

! lines =<< trim CODE
! var code =<< eval END
! var s = "`=$SOME_ENV_VAR`"
! END
! assert_equal([' var s = "somemore"'], code)
! CODE
! v9.CheckDefAndScriptSuccess(lines)

! lines =<< trim CODE
! var code =<< eval trim END
! let a = `abc`
! let b = `=g:someVar`
! let c = `
! END
! assert_equal(['let a = `abc`', 'let b = X', 'let c = `'], code)
! CODE
! v9.CheckDefAndScriptSuccess(lines)

! lines =<< trim LINES
var text =<< eval trim END
let b = `=
END
LINES
! v9.CheckDefAndScriptFailure(lines, ['E1143: Empty expression: ""', 'E1083: Missing backtick'])

lines =<< trim LINES
var text =<< eval trim END
let b = `=abc
END
LINES
! v9.CheckDefAndScriptFailure(lines, ['E1001: Variable not found: abc', 'E1083: Missing backtick'])

lines =<< trim LINES
var text =<< eval trim END
let b = `=`
END
LINES
! v9.CheckDefAndScriptFailure(lines, ['E1015: Name expected: `', 'E15: Invalid expression: "`"'])
enddef

" vim: ts=8 sw=2 sts=2 expandtab tw=80 fdm=marker
*** ../vim-8.2.4803/src/version.c 2022-04-21 22:52:07.062317208 +0100
--- src/version.c 2022-04-21 23:22:38.984952863 +0100
***************
*** 748,749 ****
--- 748,751 ----
{ /* Add new patch number below this line */
+ /**/
+ 4804,
/**/

--
To define recursion, we must first define recursion.

/// Bram Moolenaar -- Br...@Moolenaar.net -- http://www.Moolenaar.net \\\
/// \\\
\\\ sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ ///
\\\ help me help AIDS victims -- http://ICCF-Holland.org ///