Patch 8.1.1476

[Bram Moolenaar]

Patch 8.1.1476
Problem: No statistics displayed after running tests.
Solution: Summarize the test results. (Christian Brabandt, closes #4391)
Also make it possible to report a skipped file.
Files: src/Makefile, src/testdir/Makefile, src/testdir/summarize.vim,
src/testdir/runtest.vim, src/testdir/test_arabic.vim,
src/testdir/test_autochdir.vim, src/testdir/test_balloon.vim


*** ../vim-8.1.1475/src/Makefile 2019-05-31 20:02:47.567231180 +0200
--- src/Makefile 2019-06-06 13:12:56.701571974 +0200
***************
*** 2117,2123 ****
# TESTING
#
# Execute the test scripts and the unittests.
! test check: scripttests unittests test_libvterm

# Execute the test scripts. Run these after compiling Vim, before installing.
# This doesn't depend on $(VIMTARGET), because that won't work when configure
--- 2118,2125 ----
# TESTING
#
# Execute the test scripts and the unittests.
! # Do the scripttests first, so that the summary shows last.
! test check: unittests test_libvterm scripttests

# Execute the test scripts. Run these after compiling Vim, before installing.
# This doesn't depend on $(VIMTARGET), because that won't work when configure
*** ../vim-8.1.1475/src/testdir/Makefile 2019-02-21 21:50:42.603751312 +0100
--- src/testdir/Makefile 2019-06-06 12:56:44.810633118 +0200
***************
*** 12,18 ****
# Comment out this line to see the verbose output of tests.
#
# Catches SwapExists to avoid hanging at the ATTENTION prompt.
! REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null

# Uncomment this line to use valgrind for memory leaks and extra warnings.
# The output goes into a file "valgrind.testN"
--- 12,18 ----
# Comment out this line to see the verbose output of tests.
#
# Catches SwapExists to avoid hanging at the ATTENTION prompt.
! # REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null

# Uncomment this line to use valgrind for memory leaks and extra warnings.
# The output goes into a file "valgrind.testN"
***************
*** 49,58 ****
benchmark: $(SCRIPTS_BENCH)

report:
@echo
@echo 'Test results:'
@/bin/sh -c "if test -f test.log; \
! then cat test.log; echo TEST FAILURE; exit 1; \
else echo ALL DONE; \
fi"

--- 49,60 ----
benchmark: $(SCRIPTS_BENCH)

report:
+ $(RUN_VIMTEST) $(NO_INITS) -S summarize.vim messages $(REDIR_TEST_TO_NULL)
@echo
@echo 'Test results:'
+ @cat test_result.log
@/bin/sh -c "if test -f test.log; \
! then echo TEST FAILURE; exit 1; \
else echo ALL DONE; \
fi"

***************
*** 77,83 ****
RUN_VIM = VIMRUNTIME=$(SCRIPTSOURCE); export VIMRUNTIME; $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim $(NO_INITS) -s dotest.in

clean:
! -rm -rf *.out *.failed *.res *.rej *.orig opt_test.vim test.log messages $(RM_ON_RUN) $(RM_ON_START) valgrind.*

test1.out: test1.in
-rm -rf $*.failed $(RM_ON_RUN) $(RM_ON_START) wrongtermsize
--- 79,88 ----
RUN_VIM = VIMRUNTIME=$(SCRIPTSOURCE); export VIMRUNTIME; $(VALGRIND) $(VIMPROG) -f $(GUI_FLAG) -u unix.vim $(NO_INITS) -s dotest.in

clean:
! -rm -rf *.out *.failed *.res *.rej *.orig
! -rm opt_test.vim test.log test_result.log messages
! -rm $(RM_ON_RUN) $(RM_ON_START)
! -rm valgrind.*

test1.out: test1.in
-rm -rf $*.failed $(RM_ON_RUN) $(RM_ON_START) wrongtermsize
*** ../vim-8.1.1475/src/testdir/summarize.vim 2019-06-06 13:36:51.512235762 +0200
--- src/testdir/summarize.vim 2019-06-06 13:10:12.018902480 +0200
***************
*** 0 ****
--- 1,60 ----
+ if 1
+ " This is executed with the eval feature
+ set nocp
+ func Count(match, type)
+ if a:type ==# 'executed'
+ let g:executed += (a:match+0)
+ elseif a:type ==# 'failed'
+ let g:failed = a:match+0
+ elseif a:type ==# 'skipped'
+ let g:skipped += 1
+ call extend(g:skipped_output, ["\t".a:match])
+ endif
+ endfunc
+
+ let g:executed = 0
+ let g:skipped = 0
+ let g:failed = 0
+ let g:skipped_output = []
+ let g:failed_output = []
+ let output = [""]
+
+ try
+ " This uses the :s command to just fetch and process the output of the
+ " tests, it doesn't acutally replay anything
+ %s/^Executed\s\+\zs\d\+\ze\s\+tests/\=Count(submatch(0),'executed')/egn
+ %s/^SKIPPED \zs.*/\=Count(submatch(0), 'skipped')/egn
+ %s/^\(\d\+\)\s\+FAILED:/\=Count(submatch(1), 'failed')/egn
+
+ call extend(output, ["Skipped:"])
+ call extend(output, skipped_output)
+
+ call extend(output, [
+ \ "",
+ \ "-------------------------------",
+ \ printf("Executed: %5d Tests", g:executed),
+ \ printf(" Skipped: %5d Tests", g:skipped),
+ \ printf(" %s: %5d Tests", g:failed == 0 ? 'Failed' : 'FAILED', g:failed),
+ \ "",
+ \ ])
+ if filereadable('test.log')
+ " outputs and indents the failed test result
+ call extend(output, ["", "Failures: "])
+ let failed_output = filter(readfile('test.log'), { v,k -> !empty(k)})
+ call extend(output, map(failed_output, { v,k -> "\t".k}))
+ " Add a final newline
+ call extend(output, [""])
+ endif
+
+ catch " Catch-all
+ finally
+ call writefile(output, 'test_result.log') " overwrites an existing file
+ q!
+ endtry
+ endif
+
+ " This is executed without the eval feature
+ %d
+ r test.log
+ w test_result.log
+ q!
*** ../vim-8.1.1475/src/testdir/runtest.vim 2019-02-22 14:38:46.447452832 +0100
--- src/testdir/runtest.vim 2019-06-06 13:17:50.755349142 +0200
***************
*** 268,273 ****
--- 268,276 ----
else
try
source %
+ catch /^\cskipped/
+ call add(s:messages, ' Skipped')
+ call add(s:skipped, 'SKIPPED ' . expand('%') . ': ' . substitute(v:exception, '^\S*\s\+', '', ''))
catch
let s:fail += 1
call add(s:errors, 'Caught exception: ' . v:exception . ' @ ' . v:throwpoint)
*** ../vim-8.1.1475/src/testdir/test_arabic.vim 2019-03-23 12:59:36.262661906 +0100
--- src/testdir/test_arabic.vim 2019-06-06 13:24:06.741229849 +0200
***************
*** 3,9 ****
" functional tests that check the shaping works with real text.

if !has('arabic')
! finish
endif

source view_util.vim
--- 3,9 ----
" functional tests that check the shaping works with real text.

if !has('arabic')
! throw 'Skipped: arabic feature missing'
endif

source view_util.vim
*** ../vim-8.1.1475/src/testdir/test_autochdir.vim 2018-12-16 15:37:58.866807609 +0100
--- src/testdir/test_autochdir.vim 2019-06-06 13:25:39.432658711 +0200
***************
*** 1,7 ****
" Test 'autochdir' behavior

if !exists("+autochdir")
! finish
endif

func Test_set_filename()
--- 1,7 ----
" Test 'autochdir' behavior

if !exists("+autochdir")
! throw 'Skipped: autochdir feature missing'
endif

func Test_set_filename()
*** ../vim-8.1.1475/src/testdir/test_balloon.vim 2019-05-20 22:12:30.720442793 +0200
--- src/testdir/test_balloon.vim 2019-06-06 13:35:26.168814971 +0200
***************
*** 1,11 ****
" Tests for 'balloonevalterm'.

! " Tests that only work in the terminal.
! if has('balloon_eval_term') && !has('gui_running')

source screendump.vim
if !CanRunVimInTerminal()
! finish
endif

let s:common_script =<< [CODE]
--- 1,15 ----
" Tests for 'balloonevalterm'.

! if !has('balloon_eval_term')
! throw 'Skipped: balloon_eval_term feature missing'
! endif
!
! " A few tests only work in the terminal.
! if !has('gui_running')

source screendump.vim
if !CanRunVimInTerminal()
! throw 'Skipped: cannot run Vim in a terminal window'
endif

let s:common_script =<< [CODE]
*** ../vim-8.1.1475/src/version.c 2019-06-06 12:49:25.597017931 +0200
--- src/version.c 2019-06-06 12:53:55.112790029 +0200
***************
*** 769,770 ****
--- 769,772 ----
{ /* Add new patch number below this line */
+ /**/
+ 1476,
/**/

--
# echo reboot >universe
# chmod +x universe
# ./universe

/// Bram Moolenaar -- Br...@Moolenaar.net -- http://www.Moolenaar.net \\\
/// sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\
\\\ an exciting new programming language -- http://www.Zimbu.org ///
\\\ help me help AIDS victims -- http://ICCF-Holland.org ///

[Christian Brabandt]

Hi Bram,

On Do, 06 Jun 2019, Bram Moolenaar wrote:

> Patch 8.1.1476
> Problem: No statistics displayed after running tests.
> Solution: Summarize the test results. (Christian Brabandt, closes #4391)
> Also make it possible to report a skipped file.
> Files: src/Makefile, src/testdir/Makefile, src/testdir/summarize.vim,
> src/testdir/runtest.vim, src/testdir/test_arabic.vim,
> src/testdir/test_autochdir.vim, src/testdir/test_balloon.vim

Thanks!

> *** ../vim-8.1.1475/src/testdir/Makefile 2019-02-21 21:50:42.603751312 +0100
> --- src/testdir/Makefile 2019-06-06 12:56:44.810633118 +0200
> ***************
> *** 12,18 ****
> # Comment out this line to see the verbose output of tests.
> #
> # Catches SwapExists to avoid hanging at the ATTENTION prompt.
> ! REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null
>
> # Uncomment this line to use valgrind for memory leaks and extra warnings.
> # The output goes into a file "valgrind.testN"
> --- 12,18 ----
> # Comment out this line to see the verbose output of tests.
> #
> # Catches SwapExists to avoid hanging at the ATTENTION prompt.
> ! # REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null

That looks wrong. I think it is not necessary to comment it out.

Mit freundlichen Grüßen
Christian
--
Wer die Menschen kennt, liebt die Tiere. Und wer die Tiere kennt -
liebt die Pflanzen.

[Bram Moolenaar]

Christian wrote:

> On Do, 06 Jun 2019, Bram Moolenaar wrote:
>
> > Patch 8.1.1476
> > Problem: No statistics displayed after running tests.
> > Solution: Summarize the test results. (Christian Brabandt, closes #4391)
> > Also make it possible to report a skipped file.
> > Files: src/Makefile, src/testdir/Makefile, src/testdir/summarize.vim,
> > src/testdir/runtest.vim, src/testdir/test_arabic.vim,
> > src/testdir/test_autochdir.vim, src/testdir/test_balloon.vim
>
> Thanks!

Did some more tweaking, Travis showed more errors than what I got with
local testing.

We can also throw more "skipped" exceptions in various places, now that
the are listed together it's more useful.

> > *** ../vim-8.1.1475/src/testdir/Makefile 2019-02-21 21:50:42.603751312 +0100
> > --- src/testdir/Makefile 2019-06-06 12:56:44.810633118 +0200
> > ***************
> > *** 12,18 ****
> > # Comment out this line to see the verbose output of tests.
> > #
> > # Catches SwapExists to avoid hanging at the ATTENTION prompt.
> > ! REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null
> >
> > # Uncomment this line to use valgrind for memory leaks and extra warnings.
> > # The output goes into a file "valgrind.testN"
> > --- 12,18 ----
> > # Comment out this line to see the verbose output of tests.
> > #
> > # Catches SwapExists to avoid hanging at the ATTENTION prompt.
> > ! # REDIR_TEST_TO_NULL = --cmd 'au SwapExists * let v:swapchoice = "e"' > /dev/null
>
> That looks wrong. I think it is not necessary to comment it out.

Forgot to remove this from the patch, I'll send out a patch.

--
hundred-and-one symptoms of being an internet addict:
93. New mail alarm on your palmtop annoys other churchgoers.

[Christian Brabandt]

On Do, 06 Jun 2019, Bram Moolenaar wrote:

> Patch 8.1.1476
> Problem: No statistics displayed after running tests.
> Solution: Summarize the test results. (Christian Brabandt, closes #4391)
> Also make it possible to report a skipped file.

I found a mistake in this patch. Can you please update summarize.vim
like this:

diff --git a/src/testdir/summarize.vim b/src/testdir/summarize.vim
index 54cb5c184..220aa60f9 100644
--- a/src/testdir/summarize.vim
+++ b/src/testdir/summarize.vim
@@ -6,7 +6,7 @@ if 1

if a:type ==# 'executed'

let g:executed += (a:match+0)

elseif a:type ==# 'failed'

- let g:failed = a:match+0
+ let g:failed += a:match+0

elseif a:type ==# 'skipped'

let g:skipped += 1

call extend(g:skipped_output, ["\t".a:match])

Thanks,
Christian
--
Nicht die Sünde wurde geboren, als Eva den Apfel pflückte. Geboren
wurde an diesem Tag vielmehr eine großartige Tugend, Ungehorsam
genannt.
-- Oriana Fallaci

[Bram Moolenaar]

Christian wrote:

> On Do, 06 Jun 2019, Bram Moolenaar wrote:
>
> > Patch 8.1.1476
> > Problem: No statistics displayed after running tests.
> > Solution: Summarize the test results. (Christian Brabandt, closes #4391)
> > Also make it possible to report a skipped file.
>
> I found a mistake in this patch. Can you please update summarize.vim
> like this:
>
> diff --git a/src/testdir/summarize.vim b/src/testdir/summarize.vim
> index 54cb5c184..220aa60f9 100644
> --- a/src/testdir/summarize.vim
> +++ b/src/testdir/summarize.vim
> @@ -6,7 +6,7 @@ if 1
> if a:type ==# 'executed'
> let g:executed += (a:match+0)
> elseif a:type ==# 'failed'
> - let g:failed = a:match+0
> + let g:failed += a:match+0
> elseif a:type ==# 'skipped'
> let g:skipped += 1
> call extend(g:skipped_output, ["\t".a:match])

Will make a patch with that.

--
Nobody will ever need more than 640 kB RAM.
-- Bill Gates, 1983
Windows 98 requires 16 MB RAM.
-- Bill Gates, 1999
Logical conclusion: Nobody will ever need Windows 98.