Patch 8.2.4770

[Bram Moolenaar]

Patch 8.2.4770
Problem: Cannot easily mix expression and heredoc.
Solution: Support `=expr` in heredoc. (Yegappan Lakshmanan, closes #10138)
Files: runtime/doc/eval.txt, src/evalvars.c, src/userfunc.c,
src/testdir/test_let.vim, src/testdir/test_vim9_assign.vim


*** ../vim-8.2.4769/runtime/doc/eval.txt 2022-04-10 11:26:00.941539608 +0100
--- runtime/doc/eval.txt 2022-04-17 12:42:58.353464163 +0100
***************
*** 3186,3200 ****
|List| item.

*:let=<<* *:let-heredoc*
! *E990* *E991* *E172* *E221*
! :let {var-name} =<< [trim] {endmarker}
text...
text...
{endmarker}
Set internal variable {var-name} to a |List|
containing the lines of text bounded by the string
! {endmarker}. The lines of text is used as a
! |literal-string|.
{endmarker} must not contain white space.
{endmarker} cannot start with a lower case character.
The last line should end only with the {endmarker}
--- 3223,3255 ----
|List| item.

*:let=<<* *:let-heredoc*
! *E990* *E991* *E172* *E221* *E1145*
! :let {var-name} =<< [trim] [eval] {endmarker}
text...
text...
{endmarker}
Set internal variable {var-name} to a |List|
containing the lines of text bounded by the string
! {endmarker}.
!
! If "eval" is not specified, then each line of text is
! used as a |literal-string|. If "eval" is specified,
! then any Vim expression in the form ``={expr}`` is
! evaluated and the result replaces the expression.
! Example where $HOME is expanded: >
! let lines =<< trim eval END
! some text
! See the file `=$HOME`/.vimrc
! more text
! END
! < There can be multiple Vim expressions in a single line
! but an expression cannot span multiple lines. If any
! expression evaluation fails, then the assignment fails.
! once the "`=" has been found {expr} and a backtick
! must follow. {expr} cannot be empty.
! Currenty, in a compiled function {expr} is evaluated
! when compiling the function, THIS WILL CHANGE.
!
{endmarker} must not contain white space.
{endmarker} cannot start with a lower case character.
The last line should end only with the {endmarker}
***************
*** 3244,3249 ****
--- 3299,3311 ----
1 2 3 4
5 6 7 8
DATA
+
+ let code =<< trim eval CODE
+ let v = `=10 + 20`
+ let h = "`=$HOME`"
+ let s = "`=Str1()` abc `=Str2()`"
+ let n = `=MyFunc(3, 4)`
+ CODE
<
*E121*
:let {var-name} .. List the value of variable {var-name}. Multiple
*** ../vim-8.2.4769/src/evalvars.c 2022-04-15 20:50:13.463235819 +0100
--- src/evalvars.c 2022-04-17 12:31:30.265976923 +0100
***************
*** 603,608 ****
--- 603,668 ----
}

/*
+ * Evaluate all the Vim expressions (`=expr`) in string "str" and return the
+ * resulting string. The caller must free the returned string.
+ */
+ static char_u *
+ eval_all_expr_in_str(char_u *str)
+ {
+ garray_T ga;
+ char_u *s;
+ char_u *p;
+ char_u save_c;
+ char_u *exprval;
+ int status;
+
+ ga_init2(&ga, 1, 80);
+ p = str;
+
+ // Look for `=expr`, evaluate the expression and replace `=expr` with the
+ // result.
+ while (*p != NUL)
+ {
+ s = p;
+ while (*p != NUL && (*p != '`' || p[1] != '='))
+ p++;
+ ga_concat_len(&ga, s, p - s);
+ if (*p == NUL)
+ break; // no backtick expression found
+
+ s = p;
+ p += 2; // skip `=
+
+ status = *p == NUL ? OK : skip_expr(&p, NULL);
+ if (status == FAIL || *p != '`')
+ {
+ // invalid expression or missing ending backtick
+ if (status != FAIL)
+ emsg(_(e_missing_backtick));
+ vim_free(ga.ga_data);
+ return NULL;
+ }
+ s += 2; // skip `=
+ save_c = *p;
+ *p = NUL;
+ exprval = eval_to_string(s, TRUE);
+ *p = save_c;
+ p++;
+ if (exprval == NULL)
+ {
+ // expression evaluation failed
+ vim_free(ga.ga_data);
+ return NULL;
+ }
+ ga_concat(&ga, exprval);
+ vim_free(exprval);
+ }
+ ga_append(&ga, NUL);
+
+ return ga.ga_data;
+ }
+
+ /*
* Get a list of lines from a HERE document. The here document is a list of
* lines surrounded by a marker.
* cmd << {marker}
***************
*** 619,625 ****
* tcl, mzscheme), script_get is set to TRUE. In this case, if the marker is
* missing, then '.' is accepted as a marker.
*
! * Returns a List with {lines} or NULL.
*/
list_T *
heredoc_get(exarg_T *eap, char_u *cmd, int script_get)
--- 679,685 ----
* tcl, mzscheme), script_get is set to TRUE. In this case, if the marker is
* missing, then '.' is accepted as a marker.
*
! * Returns a List with {lines} or NULL on failure.
*/
list_T *
heredoc_get(exarg_T *eap, char_u *cmd, int script_get)
***************
*** 628,638 ****
--- 688,701 ----
char_u *marker;
list_T *l;
char_u *p;
+ char_u *str;
int marker_indent_len = 0;
int text_indent_len = 0;
char_u *text_indent = NULL;
char_u dot[] = ".";
int comment_char = in_vim9script() ? '#' : '"';
+ int evalstr = FALSE;
+ int eval_failed = FALSE;

if (eap->getline == NULL)
{
***************
*** 642,662 ****

// Check for the optional 'trim' word before the marker
cmd = skipwhite(cmd);
! if (STRNCMP(cmd, "trim", 4) == 0 && (cmd[4] == NUL || VIM_ISWHITE(cmd[4])))
{
! cmd = skipwhite(cmd + 4);

! // Trim the indentation from all the lines in the here document.
! // The amount of indentation trimmed is the same as the indentation of
! // the first line after the :let command line. To find the end marker
! // the indent of the :let command line is trimmed.
! p = *eap->cmdlinep;
! while (VIM_ISWHITE(*p))
{
! p++;
! marker_indent_len++;
}
! text_indent_len = -1;
}

// The marker is the next word.
--- 705,740 ----

// Check for the optional 'trim' word before the marker
cmd = skipwhite(cmd);
!
! while (TRUE)
{
! if (STRNCMP(cmd, "trim", 4) == 0
! && (cmd[4] == NUL || VIM_ISWHITE(cmd[4])))
! {
! cmd = skipwhite(cmd + 4);
!
! // Trim the indentation from all the lines in the here document.
! // The amount of indentation trimmed is the same as the indentation
! // of the first line after the :let command line. To find the end
! // marker the indent of the :let command line is trimmed.
! p = *eap->cmdlinep;
! while (VIM_ISWHITE(*p))
! {
! p++;
! marker_indent_len++;
! }
! text_indent_len = -1;

! continue;
! }
! if (STRNCMP(cmd, "eval", 4) == 0
! && (cmd[4] == NUL || VIM_ISWHITE(cmd[4])))
{
! cmd = skipwhite(cmd + 4);
! evalstr = TRUE;
! continue;
}
! break;
}

// The marker is the next word.
***************
*** 716,721 ****
--- 794,807 ----
break;
}

+ // If expression evaluation failed in the heredoc, then skip till the
+ // end marker.
+ if (eval_failed)
+ {
+ vim_free(theline);
+ continue;
+ }
+
if (text_indent_len == -1 && *theline != NUL)
{
// set the text indent from the first line.
***************
*** 734,745 ****
if (theline[ti] != text_indent[ti])
break;

! if (list_append_string(l, theline + ti, -1) == FAIL)
break;
vim_free(theline);
}
vim_free(text_indent);

return l;
}

--- 820,852 ----
if (theline[ti] != text_indent[ti])
break;

! str = theline + ti;
! if (evalstr)
! {
! str = eval_all_expr_in_str(str);
! if (str == NULL)
! {
! // expression evaluation failed
! vim_free(theline);
! eval_failed = TRUE;
! continue;
! }
! vim_free(theline);
! theline = str;
! }
!
! if (list_append_string(l, str, -1) == FAIL)
break;
vim_free(theline);
}
vim_free(text_indent);

+ if (eval_failed)
+ {
+ // expression evaluation in the heredoc failed
+ list_free(l);
+ return NULL;
+ }
return l;
}

*** ../vim-8.2.4769/src/userfunc.c 2022-04-09 11:09:03.526052266 +0100
--- src/userfunc.c 2022-04-17 11:57:49.231398756 +0100
***************
*** 1077,1088 ****
|| checkforcmd(&p, "const", 5))))
{
p = skipwhite(arg + 3);
! if (STRNCMP(p, "trim", 4) == 0)
{
! // Ignore leading white space.
! p = skipwhite(p + 4);
! heredoc_trimmed = vim_strnsave(theline,
! skipwhite(theline) - theline);
}
skip_until = vim_strnsave(p, skiptowhite(p) - p);
getline_options = GETLINE_NONE;
--- 1077,1099 ----
|| checkforcmd(&p, "const", 5))))
{
p = skipwhite(arg + 3);
! while (TRUE)
{
! if (STRNCMP(p, "trim", 4) == 0)
! {
! // Ignore leading white space.
! p = skipwhite(p + 4);
! heredoc_trimmed = vim_strnsave(theline,
! skipwhite(theline) - theline);
! continue;
! }
! if (STRNCMP(p, "eval", 4) == 0)
! {
! // Ignore leading white space.
! p = skipwhite(p + 4);
! continue;
! }
! break;
}
skip_until = vim_strnsave(p, skiptowhite(p) - p);
getline_options = GETLINE_NONE;
*** ../vim-8.2.4769/src/testdir/test_let.vim 2021-08-14 13:27:26.451597297 +0100
--- src/testdir/test_let.vim 2022-04-17 12:37:56.821686886 +0100
***************
*** 1,5 ****
--- 1,7 ----
" Tests for the :let command.

+ import './vim9.vim' as v9
+
func Test_let()
" Test to not autoload when assigning. It causes internal error.
set runtimepath+=./sautest
***************
*** 379,385 ****
call assert_equal(['Text', 'with', 'indent'], text)
endfunc

! " Test for the setting a variable using the heredoc syntax
func Test_let_heredoc()
let var1 =<< END
Some sample text
--- 381,388 ----
call assert_equal(['Text', 'with', 'indent'], text)
endfunc

! " Test for the setting a variable using the heredoc syntax.
! " Keep near the end, this messes up highlighting.
func Test_let_heredoc()
let var1 =<< END
Some sample text
***************
*** 495,498 ****
--- 498,593 ----
call assert_equal([' x', ' \y', ' z'], [a, b, c])
endfunc

+ " Test for evaluating Vim expressions in a heredoc using `=expr`
+ " Keep near the end, this messes up highlighting.
+ func Test_let_heredoc_eval()
+ let str = ''
+ let code =<< trim eval END
+ let a = `=5 + 10`
+ let b = `=min([10, 6])` + `=max([4, 6])`
+ `=str`
+ let c = "abc`=str`d"
+ END
+ call assert_equal(['let a = 15', 'let b = 6 + 6', '', 'let c = "abcd"'], code)
+ let $TESTVAR = "Hello"
+ let code =<< eval trim END
+ let s = "`=$TESTVAR`"
+ END
+ call assert_equal(['let s = "Hello"'], code)
+ let code =<< eval END
+ let s = "`=$TESTVAR`"
+ END
+ call assert_equal([' let s = "Hello"'], code)
+ let a = 10
+ let data =<< eval END
+ `=a`
+ END
+ call assert_equal(['10'], data)
+ let x = 'X'
+ let code =<< eval trim END
+ let a = `abc`
+ let b = `=x`
+ let c = `
+ END
+ call assert_equal(['let a = `abc`', 'let b = X', 'let c = `'], code)
+ let code = 'xxx'
+ let code =<< eval trim END
+ let n = `=5 +
+ 6`
+ END
+ call assert_equal('xxx', code)
+ let code =<< eval trim END
+ let n = `=min([1, 2]` + `=max([3, 4])`
+ END
+ call assert_equal('xxx', code)
+
+ let lines =<< trim LINES
+ let text =<< eval trim END
+ let b = `=
+ END
+ LINES
+ call v9.CheckScriptFailure(lines, 'E1083:')
+
+ let lines =<< trim LINES
+ let text =<< eval trim END
+ let b = `=abc
+ END
+ LINES
+ call v9.CheckScriptFailure(lines, 'E1083:')
+
+ let lines =<< trim LINES
+ let text =<< eval trim END
+ let b = `=`
+ END
+ LINES
+ call v9.CheckScriptFailure(lines, 'E15:')
+
+ " Test for sourcing a script containing a heredoc with invalid expression.
+ " Variable assignment should fail, if expression evaluation fails
+ new
+ let g:Xvar = 'test'
+ let g:b = 10
+ let lines =<< trim END
+ let Xvar =<< eval CODE
+ let a = 1
+ let b = `=5+`
+ let c = 2
+ CODE
+ let g:Count += 1
+ END
+ call setline(1, lines)
+ let g:Count = 0
+ call assert_fails('source', 'E15:')
+ call assert_equal(1, g:Count)
+ call setline(3, 'let b = `=abc`')
+ call assert_fails('source', 'E121:')
+ call assert_equal(2, g:Count)
+ call setline(3, 'let b = `=abc` + `=min([9, 4])` + 2')
+ call assert_fails('source', 'E121:')
+ call assert_equal(3, g:Count)
+ call assert_equal('test', g:Xvar)
+ call assert_equal(10, g:b)
+ bw!
+ endfunc
+
" vim: shiftwidth=2 sts=2 expandtab
*** ../vim-8.2.4769/src/testdir/test_vim9_assign.vim 2022-04-05 21:40:32.107458262 +0100
--- src/testdir/test_vim9_assign.vim 2022-04-17 12:27:27.458163801 +0100
***************
*** 740,746 ****
enddef

def Test_extend_list()
! # using uninitilaized list assigns empty list
var lines =<< trim END
var l1: list<number>
var l2 = l1
--- 740,746 ----
enddef

def Test_extend_list()
! # using uninitialized list assigns empty list
var lines =<< trim END
var l1: list<number>
var l2 = l1
***************
*** 758,764 ****
END
v9.CheckDefAndScriptSuccess(lines)

! # appending to uninitialzed list from a function works
lines =<< trim END
vim9script
var list: list<string>
--- 758,764 ----
END
v9.CheckDefAndScriptSuccess(lines)

! # appending to uninitialized list from a function works
lines =<< trim END
vim9script
var list: list<string>
***************
*** 2637,2642 ****
--- 2637,2692 ----
v9.CheckScriptSuccess(lines)
enddef

+ let g:someVar = 'X'
+
+ " Test for heredoc with Vim expressions.
+ " This messes up highlighting, keep it near the end.
+ def Test_heredoc_expr()
+ var code =<< trim eval END
+ var a = `=5 + 10`
+ var b = `=min([10, 6])` + `=max([4, 6])`
+ END
+ assert_equal(['var a = 15', 'var b = 6 + 6'], code)
+
+ code =<< eval trim END
+ var s = "`=$SOME_ENV_VAR`"
+ END
+ assert_equal(['var s = "somemore"'], code)
+
+ code =<< eval END
+ var s = "`=$SOME_ENV_VAR`"
+ END
+ assert_equal([' var s = "somemore"'], code)
+
+ code =<< eval trim END
+ let a = `abc`
+ let b = `=g:someVar`
+ let c = `
+ END
+ assert_equal(['let a = `abc`', 'let b = X', 'let c = `'], code)
+
+ var lines =<< trim LINES
+ var text =<< eval trim END
+ let b = `=
+ END
+ LINES
+ v9.CheckDefAndScriptFailure(lines, 'E1083:')
+
+ lines =<< trim LINES
+ var text =<< eval trim END
+ let b = `=abc
+ END
+ LINES
+ v9.CheckDefAndScriptFailure(lines, 'E1083:')
+
+ lines =<< trim LINES
+ var text =<< eval trim END
+ let b = `=`
+ END
+ LINES
+ v9.CheckDefAndScriptFailure(lines, 'E15:')
+ enddef
+


" vim: ts=8 sw=2 sts=2 expandtab tw=80 fdm=marker
*** ../vim-8.2.4769/src/version.c 2022-04-17 10:57:41.100422501 +0100
--- src/version.c 2022-04-17 11:59:16.795357743 +0100
***************
*** 748,749 ****
--- 748,751 ----
{ /* Add new patch number below this line */
+ /**/
+ 4770,
/**/

--
hundred-and-one symptoms of being an internet addict:
5. You find yourself brainstorming for new subjects to search.

/// Bram Moolenaar -- Br...@Moolenaar.net -- http://www.Moolenaar.net \\\
/// \\\
\\\ sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ ///
\\\ help me help AIDS victims -- http://ICCF-Holland.org ///