diff --git a/ChangeLog b/ChangeLog index feaabe6f..28eb5486 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,14 @@ +2009-04-15 Eric Blake + + Teach AT_CHECK about hard failures. + * lib/autotest/general.m4 (AT_INIT) + : Handle hard + failures. + * doc/autoconf.texi (Writing Testsuites) : Document + AT_CHECK_NOESCAPE and exit status 99. + * NEWS: Likewise. + * tests/autotest.at (Hard fail, Cleanup): New tests. + 2009-04-14 Eric Blake Fix yesterday's regression in AS_IF. diff --git a/NEWS b/NEWS index 8eb3a068..d5d3c3c7 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,13 @@ GNU Autoconf NEWS - User visible changes. proper m4 quoting. For shell comments, this is a new feature; for non-shell comments, this fixes a regression introduced in 2.63b. +** The macro AT_CHECK now understands the concept of hard failure. If + a test exits with an unexpected status 99, cleanup actions for the + test are inhibited and the test is treated as a failure regardless + of AT_XFAIL_IF. + +** The autotest macro AT_CHECK_NOESCAPE is now documented. + ** The following documented m4sugar macros are new: m4_default_nblank m4_default_nblank_quoted m4_ifblank m4_ifnblank diff --git a/doc/autoconf.texi b/doc/autoconf.texi index 38b09720..5367ce6a 100644 --- a/doc/autoconf.texi +++ b/doc/autoconf.texi @@ -22385,34 +22385,72 @@ expansion. The contents must end with an end of line. @var{file} must be a single shell word that expands into a single file name. @end defmac -@defmac AT_CHECK (@var{commands}, @dvar{status, 0}, @dvar{stdout, }, @ - @dvar{stderr, }, @ovar{run-if-fail}, @ovar{run-if-pass}) +@defmac AT_CHECK (@var{commands}, @dvar{status, 0}, @ovar{stdout}, @ + @ovar{stderr}, @ovar{run-if-fail}, @ovar{run-if-pass}) +@defmacx AT_CHECK_NOESCAPE (@var{commands}, @dvar{status, 0}, @ovar{stdout}, @ + @ovar{stderr}, @ovar{run-if-fail}, @ovar{run-if-pass}) @atindex{CHECK} +@atindex{CHECK_NOESCAPE} Execute a test by performing given shell @var{commands}. These commands should normally exit with @var{status}, while producing expected @var{stdout} and @var{stderr} contents. If @var{commands} exit with -status 77, then the whole test group is skipped. Otherwise, if this test +unexpected status 77, then the rest of the test group is skipped. If +@var{commands} exit with unexpected status 99, then the test group is +immediately failed. Otherwise, if this test fails, run shell commands @var{run-if-fail} or, if this test passes, run shell commands @var{run-if-pass}. This macro must be invoked in between @code{AT_SETUP} and @code{AT_CLEANUP}. -@c Previously, we had this: -@c The @var{commands} @emph{must not} redirect the standard output, nor the -@c standard error. -@c to prevent trigerring the double redirect bug on Ultrix, see -@c `File Descriptors'. This was too restricting, and Ultrix is pretty -@c much dead, so we dropped the limitation; the obvious workaround on -@c Ultrix is to use a working shell there. +If @var{status} is the literal @samp{ignore}, then the corresponding +exit status is not checked, except for the special cases of 77 (skip) +and 99 (hard failure). The existence of hard failures allows one to +mark a test as an expected failure with @code{AT_XFAIL_IF} because a +feature has not yet been implemented, but to still distinguish between +gracefully handling the missing feature and dumping core. A hard +failure also inhibits post-test actions in @var{run-if-fail}. -If @var{status}, or @var{stdout}, or @var{stderr} is @samp{ignore}, then -the corresponding value is not checked. +If the value of the @var{stdout} or @var{stderr} parameter is one of the +literals in the following table, then the test treats the output +according to the rules of that literal. Otherwise, the value of the +parameter is treated as text that must exactly match the output given by +@var{commands} on standard out and standard error (including an empty +parameter for no output); any differences are captured in the testsuite +log and the test is failed. The difference between @code{AT_CHECK} and +@code{AT_CHECK_NOESCAPE} is that only the latter performs shell +expansions on comparison text given in the @var{stdout} and @var{stderr} +arguments. -The special value @samp{expout} for @var{stdout} means the expected -output of the @var{commands} is the content of the file @file{expout}. -If @var{stdout} is @samp{stdout}, then the standard output of the -@var{commands} is available for further tests in the file @file{stdout}. -Similarly for @var{stderr} with @samp{experr} and @samp{stderr}. +@table @samp +@item ignore +The content of the output is ignored, but still captured in the test +group log (if the test group later fails, the test group log is then +copied into the overall testsuite log). This is valid for both +@var{stdout} and @var{stderr}. + +@item stdout +For the @var{stdout} parameter, capture the content of standard output +to both the file @file{stdout} and the test group log. Subsequent +commands in the test group can then post-process the file. This action +is often used when it is desired to use @command{grep} to look for a +substring in the output, or when the output must be post-processed to +normalize error messages into a common form. + +@item stderr +Like @samp{stdout}, except that it only works for the @var{stderr} +parameter, and the standard error capture file will be named +@file{stderr}. + +@item expout +For the @var{stdout} parameter, compare standard output contents with +the previously created file @file{expout}, and list any differences in +the testsuite log. + +@item experr +Like @samp{expout}, except that it only works for the @var{stderr} +parameter, and the standard error contents are compared with +@file{experr}. +@end table @end defmac diff --git a/lib/autotest/general.m4 b/lib/autotest/general.m4 index 906e481b..80d7f66a 100644 --- a/lib/autotest/general.m4 +++ b/lib/autotest/general.m4 @@ -324,27 +324,33 @@ at_fn_log_failure () exit 1 } -AS_FUNCTION_DESCRIBE([at_fn_check_skip], [EXIT-CODE], -[Check whether EXIT-CODE is the special exit code 77, and if so exit the shell -with that same exit code.]) +AS_FUNCTION_DESCRIBE([at_fn_check_skip], [EXIT-CODE LINE], +[Check whether EXIT-CODE is a special exit code (77 or 99), and if so exit +the test group subshell with that same exit code. Use LINE in any report +about test failure.]) at_fn_check_skip () { case $[1] in + 99) echo 99 > "$at_status_file"; at_failed=: + AS_ECHO(["$[2]: hard failure"]); exit 99;; 77) echo 77 > "$at_status_file"; exit 77;; esac } AS_FUNCTION_DESCRIBE([at_fn_check_status], [EXPECTED EXIT-CODE LINE], -[Check whether EXIT-CODE is the expected exit code, and if so do nothing. -Otherwise, if it is 77 exit the shell with that same exit code; if it is -anything else print an error message and fail the test.]) +[Check whether EXIT-CODE is the EXPECTED exit code, and if so do nothing. +Otherwise, if it is 77 or 99, exit the test group subshell with that same +exit code; if it is anything else print an error message referring to LINE, +and fail the test.]) at_fn_check_status () { dnl This order ensures that we don't `skip' if we are precisely checking -dnl $? = 77. +dnl $? = 77 or $? = 99. case $[2] in $[1] ) ;; 77) echo 77 > "$at_status_file"; exit 77;; + 99) echo 99 > "$at_status_file"; at_failed=: + AS_ECHO(["$[3]: hard failure"]); exit 99;; *) AS_ECHO(["$[3]: exit code was $[2], expected $[1]"]) at_failed=:;; esac @@ -1130,11 +1136,16 @@ at_fn_group_postprocess () report this failure to . _ATEOF AS_ECHO(["$at_setup_line"]) >"$at_check_line_file" - at_xfail=no at_status=99 + at_status=99 fi $at_verbose AS_ECHO_N(["$at_group. $at_setup_line: "]) AS_ECHO_N(["$at_group. $at_setup_line: "]) >> "$at_group_log" case $at_xfail:$at_status in + *:99) + at_msg='FAILED ('`cat "$at_check_line_file"`')' + at_res=fail + at_errexit=$at_errexit_p + ;; yes:0) at_msg="UNEXPECTED PASS" at_res=xpass diff --git a/tests/autotest.at b/tests/autotest.at index d9c60819..89928521 100644 --- a/tests/autotest.at +++ b/tests/autotest.at @@ -253,6 +253,16 @@ AT_CHECK_AT_TEST([Skip], [], [], [], [], [], [AT_CHECK([grep skipped micro-suite.log], [], [ignore])]) +AT_CHECK_AT_TEST([Hard fail], + [AT_CHECK([exit 99]) + AT_CLEANUP + AT_SETUP([another test]) + AT_XFAIL_IF([:]) + AT_CHECK([exit 99])], + [], [1], [], [ignore], [], + [AT_CHECK([grep '2 failed unexpectedly' micro-suite.log], [], [ignore]) + AT_CHECK([grep ok micro-suite.log], [1])]) + AT_CHECK_AT_TEST([Syntax error], [AT_CHECK([:]) AT_CLEANUP @@ -277,6 +287,35 @@ AT_CHECK_AT_TEST([errexit], AT_CHECK([grep "1 .* inhibited subsequent" stderr], [], [ignore])], [--errexit]) +AT_CHECK_AT_TEST([Cleanup], + [AT_CHECK([test ! -f cleanup.success && test ! -f cleanup.failure]) + AT_CHECK_NOESCAPE([exit $value], [ignore], [$output], + [], [touch cleanup.failure], [touch cleanup.success])], + [], [], [], [], + [AT_KEYWORDS([AT@&t@_CHECK_NOESCAPE]) + output=; export output], + [AT_CHECK([test -d micro-suite.dir/1]) + AT_CHECK([test -f micro-suite.dir/1/cleanup.success]) + AT_CHECK([test ! -f micro-suite.dir/1/cleanup.failure]) + + AT_CHECK([$CONFIG_SHELL ./micro-suite -d value=1], [], [ignore]) + AT_CHECK([test -f micro-suite.dir/1/cleanup.success]) + AT_CHECK([test ! -f micro-suite.dir/1/cleanup.failure]) + + AT_CHECK([$CONFIG_SHELL ./micro-suite -d value=1 output=mismatch], + [1], [ignore], [ignore]) + AT_CHECK([test ! -f micro-suite.dir/1/cleanup.success]) + AT_CHECK([test -f micro-suite.dir/1/cleanup.failure]) + + AT_CHECK([$CONFIG_SHELL ./micro-suite -d value=77], [], [ignore]) + AT_CHECK([test ! -f micro-suite.dir/1/cleanup.success]) + AT_CHECK([test ! -f micro-suite.dir/1/cleanup.failure]) + + AT_CHECK([$CONFIG_SHELL ./micro-suite -d value=99], [1], [ignore], [ignore]) + AT_CHECK([test ! -f micro-suite.dir/1/cleanup.success]) + AT_CHECK([test ! -f micro-suite.dir/1/cleanup.failure]) + ], [-d value=0]) + ## ----------------------------------------------------- ## ## Newlines and command substitutions in test commands. ## ## ----------------------------------------------------- ##