# -*- Autotest -*- AT_BANNER([Autotest.]) # Copyright (C) 2004, 2005, 2006, 2007, 2008 Free Software Foundation, # Inc. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2, or (at your option) # any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA # 02110-1301, USA. # AT_CHECK_AT_PREP(NAME, SUITE-CODE, [STATUS = 0], [STDOUT], [STDERR], # [DIR = .]) # -------------------------------------------------------------------- # Create a new testsuite named NAME that runs a minimal Autotest test suite, # SUITE-CODE. Do not use `testsuite' for NAME, or the log file it generates # will overwrite the log that the Autoconf test produces when managing # this test case. STATUS, STDOUT, and STDERR pass directly to the AT_CHECK # that compiles the testsuite. DIR can specify a particular subdirectory # where the testsuite should live. m4_define([AT_CHECK_AT_PREP], [AT_KEYWORDS([autotest])dnl dnl overquote AT_dir, to make it easier to use m4_pushdef([AT_dir], m4_ifval([$6], [[[$6]]], [.]))dnl dnl AT_CAPTURE_FILE([$1.log])dnl dnl AT_DATA(AT_dir[/package.m4], [[m4_define([AT@&t@_PACKAGE_NAME], [GNU Nonsense]) m4_define([AT@&t@_PACKAGE_TARNAME], [nonsense]) m4_define([AT@&t@_PACKAGE_VERSION], [1.0]) m4_define([AT@&t@_PACKAGE_STRING], [GNU Nonsense 1.0]) m4_define([AT@&t@_PACKAGE_BUGREPORT], [bug-autoconf@gnu.org]) ]]) dnl AT_DATA(AT_dir[/$1.at], m4_bpatsubst([[$2]], [AT_], [AT@&t@_])) m4_ifval([$6], [(cd AT_dir]) AT_CHECK_AUTOM4TE([--language=autotest -o $1 $1.at], [$3], [$4], [$5]) m4_ifval([$6], [)]) m4_popdef([AT_dir])dnl ]) # AT_CHECK_AT_PREP # AT_CHECK_AT(TITLE, SUITE-CODE, [XFAIL-CONDITION], [STATUS = 0], # [STDOUT = ignore], STDERR, [POST-TEST-CODE], [SUITE-ARGS]) # ---------------------------------------------------------------------- # Create a new test named TITLE that runs a minimal Autotest test suite, # SUITE-CODE with additional SUITE-ARGS, once without and once with # '-v -x' added. Call AT_XFAIL_IF with XFAIL-CONDITION. Pass STDERR # directly to the AT_CHECK that calls the minimal test suite, STDOUT to # the AT_CHECK without '-v -x'; ignore stdout for the latter. # Run POST-TEST-CODE at the top level after the micro-suite has been run. m4_define([AT_CHECK_AT], [AT_SETUP([$1]) AT_XFAIL_IF([$3]) AT_CHECK_AT_PREP([micro-suite], [$2]) AT_CHECK([$CONFIG_SHELL ./micro-suite $8], m4_default([$4], 0), m4_default([$5], [ignore]), [$6]) AT_CHECK([$CONFIG_SHELL ./micro-suite -v -x $8], m4_default([$4], 0), [ignore], [$6]) $7 AT_CLEANUP ])# AT_CHECK_AT # AT_CHECK_AT_TEST(TITLE, SUITE-SNIPPET, ...) # ----------------------------------------------------------------------- # Wrapper for AT_CHECK_AT that surrounds SUITE-SNIPPET with a boilerplate # AT_INIT, AT_SETUP, and AT_CLEANUP and passes other arguments verbatim. m4_define([AT_CHECK_AT_TEST], [AT_CHECK_AT([$1], [[ AT_INIT([artificial test suite]) AT_SETUP([my only test]) $2 AT_CLEANUP ]], m4_shift2($@))]) # AT_CHECK_AT_TEST # AT_CHECK_AT_SYNTAX(TITLE, SUITE, MESSAGE) # ----------------------------------------- # Create a test named TITLE that tries compiling SUITE with syntax # errors with autotest. Expect a failed compilation, and grep for # MESSAGE occuring within the error output. m4_define([AT_CHECK_AT_SYNTAX], [AT_SETUP([$1]) AT_CHECK_AT_PREP([micro-suite], [$2], [1], [], [stderr]) AT_CHECK([grep '$3' stderr], [0], [ignore]) AT_CLEANUP ])# AT_CHECK_AT_SYNTAX # AT_NO_CMDSUBST # -------------- m4_define([AT_NO_CMDSUBST], [if (eval 'foo=$(echo bar) && test "$foo" = bar') >/dev/null 2>&1; then ]dnl [false; else :; fi]) # AT_CHECK_EGREP(PATTERN, STATUS, COUNT) # -------------------------------------- # Run grep -E, counting how many times PATTERN occurs in the file 'stdout', # and expecting exit STATUS and output COUNT. m4_define([AT_CHECK_EGREP], [AT_CHECK([$EGREP -c '$1' stdout], [$2], [$3 ], [ignore]) ]) ## ------------------ ## ## Empty test suite. ## ## ------------------ ## # This is not a sensible thing to do, but the user should not get an unhelpful # error message. AT_CHECK_AT([Empty test suite], [[AT_INIT([empty test suite]) ]]) AT_CHECK_AT([Banner-only test suite], [[AT_INIT([empty test suite]) AT_BANNER([banner]) ]]) # Next level of emptiness. AT_CHECK_AT_TEST([Empty test], []) # And finally, an empty check should not cause a syntax error. AT_CHECK_AT_TEST([Empty check], [AT_CHECK]) # Check for sensible error messages for common bugs. AT_CHECK_AT_SYNTAX([AT@&t@_SETUP without AT@&t@_INIT], [[AT_SETUP([only test]) AT_CHECK([:]) AT_CLEANUP ]], [AT@&t@_SETUP: missing AT@&t@_INIT detected]) AT_CHECK_AT_SYNTAX([AT@&t@_BANNER without AT@&t@_INIT], [[AT_BANNER([just a banner]) ]], [AT@&t@_BANNER: missing AT@&t@_INIT detected]) AT_CHECK_AT_SYNTAX([AT@&t@_CLEANUP without AT@&t@_INIT], [[AT_CLEANUP ]], [AT@&t@_CLEANUP: missing AT@&t@_INIT detected]) AT_CHECK_AT_SYNTAX([Missing AT@&t@_CLEANUP], [[AT_INIT([incomplete test suite]) AT_SETUP([only test]) AT_CHECK([:]) ]], [missing AT@&t@_CLEANUP detected]) AT_CHECK_AT_SYNTAX([AT@&t@_CHECK without AT@&t@_SETUP], [[AT_INIT([incomplete test suite]) AT_CHECK([:]) ]], [AT@&t@_CHECK: missing AT@&t@_SETUP detected]) AT_CHECK_AT_SYNTAX([AT@&t@_DATA without AT@&t@_SETUP], [[AT_INIT([incomplete test suite]) AT_DATA([file]) ]], [AT@&t@_DATA: missing AT@&t@_SETUP detected]) AT_CHECK_AT_SYNTAX([AT@&t@_XFAIL_IF without AT@&t@_SETUP], [[AT_INIT([incomplete test suite]) AT_XFAIL_IF([:]) ]], [AT@&t@_XFAIL_IF: missing AT@&t@_SETUP detected]) AT_CHECK_AT_SYNTAX([AT@&t@_KEYWORDS without AT@&t@_SETUP], [[AT_INIT([incomplete test suite]) AT_KEYWORDS([keyword]) ]], [AT@&t@_KEYWORDS: missing AT@&t@_SETUP detected]) AT_CHECK_AT_SYNTAX([AT@&t@_CLEANUP without AT@&t@_SETUP], [[AT_INIT([incomplete test suite]) AT_CLEANUP ]], [AT@&t@_CLEANUP: missing AT@&t@_SETUP detected]) AT_CHECK_AT_SYNTAX([AT@&t@_BANNER inside AT@&t@_SETUP], [[AT_INIT([incomplete test suite]) AT_SETUP([only test]) AT_BANNER([banner]) AT_CHECK([:]) AT_CLEANUP ]], [AT@&t@_BANNER: nested AT@&t@_SETUP detected]) AT_CHECK_AT_SYNTAX([AT@&t@_SETUP inside AT@&t@_SETUP], [[AT_INIT([incomplete test suite]) AT_SETUP([only test]) AT_SETUP([nested test]) AT_CHECK([:]) AT_CLEANUP AT_CHECK([:]) AT_CLEANUP ]], [AT@&t@_SETUP: nested AT@&t@_SETUP detected]) AT_CHECK_AT_SYNTAX([Multiple AT@&t@_INIT], [[AT_INIT([[suite, take one]]) AT_INIT([repeat]) ]], [AT@&t@_INIT: invoked multiple times]) # Check for tested programs. autoconf should only appear once. AT_CHECK_AT([Tested programs], [[AT_INIT([programs test suite]) AT_TESTED([autoconf autom4te]) AT_TESTED([autoconf]) ]], [], [], [], [], [AT_CHECK([[sed -n 's|.*/\([^ /]* --version\)|\1|p' micro-suite.log]], [], [[autoconf --version autom4te --version ]])]) AT_CHECK_AT([Startup error messages], [[AT_INIT([[suite]]) AT_SETUP([only test]) AT_CHECK([:]) AT_CLEANUP ]], [], [], [], [], [AT_CHECK([sed -n '/exec AS_MESSAGE_LOG_FD/q; />&AS_MESSAGE_LOG_FD/p' < micro-suite])]) ## ----------------------------------------------------- ## ## Newlines and command substitutions in test commands. ## ## ----------------------------------------------------- ## AT_CHECK_AT_TEST([Truth], [AT_CHECK([:], 0, [], [])]) AT_CHECK_AT_TEST([Fallacy], [AT_CHECK([false], [], [], [])], [], [1], [], [ignore], [AT_CHECK([grep failed micro-suite.log], [], [ignore])]) AT_CHECK_AT_TEST([Skip], [AT_CHECK([exit 77], 0, [], [])], [], [], [], [], [AT_CHECK([grep skipped micro-suite.log], [], [ignore])]) AT_CHECK_AT_TEST([errexit], [AT_CHECK([false]) AT_CLEANUP AT_SETUP([test that should not be run]) AT_CHECK([:])], [], [1], [stdout], [stderr], [AT_CHECK([test -f micro-suite.log], [1]) touch micro-suite.log # shut up AT_CAPTURE_FILE. AT_CHECK([grep "should not be run" stdout], [1]) AT_CHECK([grep "1 .* inhibited subsequent" stderr], [], [ignore])], [--errexit]) AT_CHECK_AT_TEST([Literal multiline command], [AT_CHECK([echo Auto' 'conf], 0, [Auto conf ], [])]) AT_CHECK_AT_TEST([Multiline parameter expansion], [FOO='one two' AT_CHECK([echo "$FOO"], 0, [one two ], [])]) AT_CHECK_AT_TEST([Backquote command substitution], [AT_CHECK([echo `echo hi`], 0, [hi ], [])]) AT_CHECK_AT_TEST([Multiline backquote command substitution], [AT_DATA([myfile],[foo bar ]) AT_CHECK([echo "`cat myfile`"], 0, [foo bar ], [])]) AT_CHECK_AT_TEST([Parenthetical command substitution], [AT_CHECK([echo $(echo hi)], 0, [hi ], [])], [AT_NO_CMDSUBST]) AT_CHECK_AT_TEST([Multiline parenthetical command substitution], [AT_DATA([myfile],[foo bar ]) AT_CHECK([echo "$(cat myfile)"], 0, [foo bar ], [])], [AT_NO_CMDSUBST]) ## ------------------------- ## ## ${...} in test commands. ## ## ------------------------- ## # If this invalid parameter expansion capsizes the test suite, the entire # AT_SETUP ... AT_CLEANUP subshell will exit, and the commands it runs will # appear to have succeeded. Therefore, we verify a failing test case. AT_CHECK_AT_TEST([Invalid brace-enclosed parameter expansion], [AT_CHECK([echo '${=invalid}'], 0, [wrong])], [false], 1, ignore, ignore) ## ---------------------------- ## ## M4 macros in test commands. ## ## ---------------------------- ## # The last paragaph in the comment above _AT_DECIDE_TRACEABLE illustrates why # this test fails (except with Korn shell-style quoting $'foo\nbar'). AT_CHECK_AT_TEST([Multiline command from M4 expansion], [m4_define([GNU], ['foo bar']) AT_CHECK([echo GNU], 0, [foo bar ], [])], [case `( set -x; echo 'foo bar') 2>&1` in *\$\'foo\\nbar\'*) false;; *) :;; esac]) AT_CHECK_AT_TEST([Double-M4-quoted command], [m4_define([GNU], ['foo bar']) AT_CHECK([[echo GNU]], 0, [[GNU ]], [])]) ## -------------------------------------- ## ## Backslash- in test commands. ## ## -------------------------------------- ## AT_CHECK_AT_TEST([BS-newline in command], [AT_CHECK([echo Auto"\ "conf], 0, [Autoconf ], [])]) AT_CHECK_AT_TEST([^BS-newline in command], [AT_CHECK([\ echo GNU], 0, [GNU ], [])]) AT_CHECK_AT_TEST([BSx641-newline in command], [AT_CHECK([printf '%s\n' Auto"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ "conf], 0, [Auto\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\conf ], [])]) AT_CHECK_AT_TEST([BS-BS-newline in command], [AT_CHECK([printf '%s\n' Auto"\\ "conf], 0, [Auto\ conf ], [])]) # A `^BS-BS-newline in command' test will run a command named `\'. No, thanks. AT_CHECK_AT_TEST([BSx640-newline in command], [AT_CHECK([printf '%s\n' Auto"\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ "conf], 0, [Auto\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ conf ], [])]) # This command has both escaped and unescaped newlines. AT_CHECK_AT_TEST([Newline-CODE-BS-newline in command], [AT_CHECK([printf '%s\n' Auto' 'co\ nf], 0, [Auto conf ], [])]) AT_CHECK_AT_TEST([Single-quote-BS-newline in command], [AT_CHECK([printf '%s\n' Auto'\ 'conf], 0, [Auto\ conf ], [])]) AT_CHECK_AT_TEST([Single-quote-newline-BS-newline in command], [AT_CHECK([printf '%s\n' Auto' \ 'conf], 0, [Auto \ conf ], [])]) ## ----------------- ## ## Input from stdin. ## ## ----------------- ## AT_SETUP([Input from stdin]) AT_CHECK_AT_PREP([micro-suite], [[AT_INIT AT_SETUP([please enter hello]) AT_CHECK([cat], [], [hello ]) AT_CLEANUP ]]) AT_CHECK([echo hello | $CONFIG_SHELL ./micro-suite], [], [ignore]) AT_CHECK([$CONFIG_SHELL ./micro-suite @], [@:>@]) AT_CHECK_AT_TITLE_CHAR([Pound], [[#]], [#]) AT_CHECK_AT_TITLE_CHAR([Quoted comma],[[,]], [,]) AT_CHECK_AT_TITLE_CHAR([Comma], [,], [,]) dnl this test also hits quadrigraphs for () AT_CHECK_AT_TITLE_CHAR([Parentheses], [(@{:@)@:}@], [(())]) AT_CHECK_AT_TITLE_CHAR([Left paren], [[(]], [(]) AT_CHECK_AT_TITLE_CHAR([Right paren], [[)]], [)]) AT_CHECK_AT_TITLE_CHAR([Quoted Macro], [[macro_name]], [macro_name]) AT_CHECK_AT_TITLE_CHAR([Macro], [macro_name], [macro_expanded]) AT_CHECK_AT_TITLE_CHAR([Macro with backquote], [macro_backquote], [`]) AT_CHECK_AT_TITLE_CHAR([Macro with single-quote], [macro_single_quote], [']) AT_CHECK_AT_TITLE_CHAR([Macro with double-quote], [macro_double_quote], ["]) dnl restore font-lock: " AT_CHECK_AT_TITLE_CHAR([Macro with backslash], [macro_backslash], [\]) AT_CHECK_AT_TITLE_CHAR([Macro echoing macro], [macro_echo([macro_name])], [macro_expanded]) AT_CHECK_AT_TITLE_CHAR([Macro echoing single-quote], [macro_echo(['])], [']) AT_CHECK_AT_TITLE_CHAR([Long test title], [0123456789012345678901234567890123]) AT_CHECK_AT_TITLE_CHAR([Longer test title], [01234567890123456789012345678901234], [], [], [54]) ## ----------------------- ## ## Long test source lines. ## ## ----------------------- ## # Create a test file that has more than 99 words in a line, for Solaris awk. # While at that, try out the limit of 2000 bytes in a text file line. AT_CHECK_AT_TEST([Long test source lines], [m4_for([nnn], [1], [999], [], [: ]) AT_CHECK([:]) ], [], [], [], [ignore], [AT_CHECK([$CONFIG_SHELL ./micro-suite -k skipalltests], [], [ignore], [ignore]) ]) ## ----------------- ## ## Debugging a test. ## ## ----------------- ## AT_CHECK_AT_TEST([Debugging a successful test], [AT_CHECK([:])], [], [], [], [ignore], [# Without options, when all tests pass, no test directory should exist. AT_CHECK([test -d micro-suite.dir/1 && exit 42 $CONFIG_SHELL ./micro-suite -d 1], [], [ignore], [ignore]) # Running with -d should leave a reproducible test group. # Also, running the test script from the test group locks the # directory from removal on some platforms; the script should still be # able to run even if rmdir fails. AT_CHECK([(cd micro-suite.dir/1 && ./run)], [], [ignore], [ignore]) # Running a debugging script implies -d. AT_CHECK([(cd micro-suite.dir/1 && ./run)], [], [ignore], [ignore]) ]) AT_CHECK_AT_TEST([Debugging script and environment], [AT_CHECK([test "$MY_VAR" = pass || exit 42])], [], [1], [], [ignore], [ # Changing environment outside of debugging script is not preserved. AT_CHECK([(cd micro-suite.dir/1 && MY_VAR=pass ./run)], [0], [ignore], [ignore]) AT_CHECK([(cd micro-suite.dir/1 && ./run)], [1], [ignore], [ignore]) # Changing environment as argument to debugging script is preserved. AT_CHECK([(cd micro-suite.dir/1; ./run MY_VAR=pass)], [0], [ignore], [ignore]) AT_CHECK([(cd micro-suite.dir/1; ./run)], [0], [ignore], [ignore]) ]) # The run script must still be valid when shell metacharacters are passed # in via an environment option. AT_CHECK_AT_TEST([Debugging a failed test], [AT_CHECK([test "$MY_VAR" = "one space" || exit 42])], [], [1], [], [ignore], [ AT_CHECK([(cd micro-suite.dir/1 && ./run MY_VAR='two spaces')], [1], [ignore], [ignore]) AT_CHECK([(cd micro-suite.dir/1 && ./run MY_VAR='one space')], [0], [ignore], [ignore]) ]) # Setting default variable values via atlocal. AT_CHECK_AT_TEST([Using atlocal], [AT_CHECK([test "x$MY_VAR" = "xodd; 'string" || exit 42])], [], [1], [ignore], [ignore], [ dnl check that command line can set variable AT_CHECK([$CONFIG_SHELL ./micro-suite MY_VAR="odd; 'string"], [0], [ignore]) dnl check that command line overrides environment AT_CHECK([MY_VAR="odd; 'string" $CONFIG_SHELL ./micro-suite MY_VAR=unset], [1], [ignore], [ignore]) dnl check that atlocal can give it a default AT_CHECK([cat <atlocal MY_VAR="odd; 'string" export MY_VAR dnl Also populate enough of atlocal to do what atconfig normally does. at_testdir=. abs_builddir='`pwd`' at_srcdir=. abs_srcdir='`pwd`' at_top_srcdir=. abs_top_srcdir='`pwd`' at_top_build_prefix= abs_top_builddir='`pwd`' EOF ]) AT_CHECK([$CONFIG_SHELL ./micro-suite], [0], [ignore]) dnl check that atlocal overrides environment AT_CHECK([MY_VAR=unset $CONFIG_SHELL ./micro-suite], [0], [ignore]) dnl check that command line overrides atlocal AT_CHECK([$CONFIG_SHELL ./micro-suite MY_VAR=], [1], [ignore], [ignore]) dnl check that syntax error is detected AT_CHECK([$CONFIG_SHELL ./micro-suite =], [1], [], [ignore], [ignore]) AT_CHECK([$CONFIG_SHELL ./micro-suite 1=2], [1], [], [ignore], [ignore]) ]) # Controlling where the testsuite is run. AT_CHECK_AT_TEST([Choosing where testsuite is run], [AT_CHECK([:])], [], [], [], [], [ dnl AT_CHECK_AT_TEST tests the default of running in `.'. AT_CHECK([$CONFIG_SHELL ./micro-suite --clean]) AT_CHECK([test -f micro-suite.log], [1]) AT_CHECK([test -d micro-suite.dir], [1]) AT_CHECK([mkdir sub1 sub2]) dnl check specifying a different relative path to run in. AT_CHECK([$CONFIG_SHELL ./micro-suite -C sub1], [0], [ignore], []) AT_CHECK([test -f micro-suite.log], [1]) AT_CHECK([test -f sub1/micro-suite.log], [0]) AT_CHECK([test -d micro-suite.dir], [1]) AT_CHECK([test -d sub1/micro-suite.dir], [0]) AT_CHECK([$CONFIG_SHELL ./micro-suite -C sub1 --clean]) AT_CHECK([test -f sub1/micro-suite.log], [1]) AT_CHECK([test -d sub1/micro-suite.dir], [1]) dnl check specifying an absolute path to run in. AT_CHECK([$CONFIG_SHELL ./micro-suite --directory="`pwd`/sub2"], [0], [ignore], []) AT_CHECK([test -f micro-suite.log], [1]) AT_CHECK([test -f sub2/micro-suite.log], [0]) AT_CHECK([$CONFIG_SHELL ./micro-suite --clean --directory="`pwd`/sub2"]) AT_CHECK([test -f sub2/micro-suite.log], [1]) AT_CHECK([test -f sub2/micro-suite.dir], [1]) dnl check for failure detection with bad, missing, or empty directory. AT_CHECK([$CONFIG_SHELL ./micro-suite -C nonesuch], [1], [ignore], [ignore]) AT_CHECK([$CONFIG_SHELL ./micro-suite -C ''], [1], [ignore], [ignore]) AT_CHECK([$CONFIG_SHELL ./micro-suite -C -], [1], [ignore], [ignore]) AT_CHECK([$CONFIG_SHELL ./micro-suite -C], [1], [ignore], [ignore]) dnl check that --help overrides bad directory selection. AT_CHECK([$CONFIG_SHELL ./micro-suite -C nonesuch --help], [0], [ignore], []) ]) ## -------- ## ## Banners. ## ## -------- ## AT_SETUP([Banners]) AT_CHECK_AT_PREP([b], [[AT_INIT AT_SETUP(zero)# 1 AT_CHECK(:) AT_CLEANUP AT_BANNER([first]) AT_SETUP(one a)# 2 AT_CHECK(:) AT_CLEANUP AT_SETUP(one b)# 3 AT_CHECK(:) AT_CLEANUP AT_BANNER() AT_SETUP(two)# 4 AT_CHECK(:) AT_CLEANUP AT_BANNER([second]) AT_SETUP(three a)# 5 AT_CHECK(:) AT_CLEANUP AT_SETUP(three b)# 6 AT_CHECK(:) AT_CLEANUP ]]) # AT_CHECK_BANNERS(TESTSUITE-OPTIONS, PATTERN1, COUNT1, PATTERN2, COUNT2) m4_define([AT_CHECK_BANNERS], [AT_CHECK([$CONFIG_SHELL ./b $1], [], [stdout]) AT_CHECK_EGREP([$2], m4_if([$3], [0], [1], [0]), [$3]) AT_CHECK_EGREP([$4], m4_if([$5], [0], [1], [0]), [$5]) ]) AT_CHECK_BANNERS([], [first], [1], [second], [1]) AT_CHECK_BANNERS([-k zero], [first], [0], [second], [0]) AT_CHECK_BANNERS([1], [first], [0], [second], [0]) AT_CHECK_BANNERS([-2], [first], [1], [second], [0]) AT_CHECK_BANNERS([-3], [first], [1], [second], [0]) AT_CHECK_BANNERS([-k one], [first], [1], [second], [0]) AT_CHECK_BANNERS([3-4], [first], [1], [second], [0]) AT_CHECK_BANNERS([3-5], [first], [1], [second], [1]) AT_CHECK_BANNERS([4-5], [first], [0], [second], [1]) AT_CHECK_BANNERS([3-], [first], [1], [second], [1]) AT_CHECK_BANNERS([-k a], [first], [1], [second], [1]) AT_CHECK_BANNERS([4], [first], [0], [second], [0]) AT_CHECK_BANNERS([4-], [first], [0], [second], [1]) AT_CHECK_BANNERS([-k two], [first], [0], [second], [0]) AT_CHECK_BANNERS([1 4], [first], [0], [second], [0]) AT_CHECK_BANNERS([-k three], [first], [0], [second], [1]) AT_CHECK_BANNERS([5], [first], [0], [second], [0]) AT_CHECK_BANNERS([5-], [first], [0], [second], [1]) AT_CLEANUP ## --------- ## ## Keywords. ## ## --------- ## AT_SETUP([Keywords and ranges]) AT_CHECK_AT_PREP([k], [[AT_INIT AT_SETUP(none) # 01 AT_CHECK(:) AT_CLEANUP AT_SETUP(first) # 02 AT_KEYWORDS(key1) AT_CHECK(:) AT_CLEANUP AT_SETUP(second) # 03 AT_KEYWORDS(key2) AT_CHECK(:) AT_CLEANUP AT_SETUP(both) # 04 AT_KEYWORDS([key1 key2]) AT_KEYWORDS([m4@&t@_echo([Key1])]) AT_CHECK(:) AT_CLEANUP AT_SETUP(test5) # 05 AT_CHECK(:) AT_CLEANUP AT_SETUP(test6) # 06 AT_CHECK(:) AT_CLEANUP AT_SETUP(test7) # 07 AT_CHECK(:) AT_CLEANUP AT_SETUP(test8) # 08 AT_CHECK(:) AT_CLEANUP AT_SETUP(test9) # 09 AT_CHECK(:) AT_CLEANUP AT_SETUP(test10) # 10 AT_CHECK(:) AT_CLEANUP ]]) dnl check that AT_KEYWORDS does not duplicate words AT_CHECK([grep -i 'key1.*key1' k], [1]) dnl check that -k requires an argument AT_CHECK([$CONFIG_SHELL ./k -k], [1], [], [ignore]) # AT_CHECK_KEYS(TESTSUITE-OPTIONS, PATTERN1, COUNT1, PATTERN2, COUNT2) m4_define([AT_CHECK_KEYS], [AT_CHECK([$CONFIG_SHELL ./k $1], 0, [stdout]) AT_CHECK_EGREP([$2], 0, [$3]) AT_CHECK_EGREP([$4], 1, [$5]) ]) AT_CHECK_KEYS([-k key1], [first|both], [2], [none|second], [0]) AT_CHECK_KEYS([-k key2], [second|both], [2], [none|first], [0]) AT_CHECK_KEYS([-k key1,key2], [both], [1], [none|first|second], [0]) AT_CHECK_KEYS([-k key1 -k key2], [first|second|both], [3], [none], [0]) AT_CHECK_KEYS([-k '!key1'], [none|second], [2], [first|both], [0]) AT_CHECK_KEYS([-k '!key2'], [none|first], [2], [second|both], [0]) AT_CHECK_KEYS([-k '!key1,key2'], [second], [1], [none|first|both], [0]) AT_CHECK_KEYS([-k 'key1,!key2'], [first], [1], [none|second|both], [0]) AT_CHECK_KEYS([-k '!key1,!key2'], [none], [1], [first|second|both], [0]) AT_CHECK_KEYS([-k '!key1' -k KEY2], [none|second|both], [3], [first], [0]) AT_CHECK_KEYS([-k key1 -k '!key2'], [none|first|both], [3], [second], [0]) AT_CHECK_KEYS([-k '!KEY1' -k '!key2'], [none|first|second], [3], [both], [0]) AT_CHECK_KEYS([-k none], [none], [1], [first|second|both], [0]) AT_CHECK_KEYS([-k key1,both], [both], [1], [none|first|second], [0]) AT_CHECK_KEYS([-k key1 -k both], [first|both], [2], [none|second], [0]) AT_CHECK_KEYS([-k none,first], [successful], [1], [none|first|second|both], [0]) AT_CHECK_KEYS([-k none,first,second,both], [successful], [1], [none|first|second|both], [0]) AT_CHECK_KEYS([-k !none,first], [first], [1], [none|second|both], [0]) AT_CHECK_KEYS([-k '.*eco.*'], [second], [1], [none|first|both], [0]) AT_CHECK_KEYS([-k 'ECO'], [successful], [1], [none|first|second|both], [0]) AT_CHECK_KEYS([-k '.*eco'], [successful], [1], [none|first|second|both], [0]) AT_CHECK_KEYS([-k 'eco.*'], [successful], [1], [none|first|second|both], [0]) AT_CHECK_KEYS([-k 'fir.*'], [first], [1], [none|second|both], [0]) AT_CHECK_KEYS([1-2], [none|first], [2], [second|both], [0]) AT_CHECK_KEYS([01-002 08], [none|first], [2], [second|both], [0]) AT_CHECK_KEYS([1-3 2-1], [none|first|second], [3], [both], [0]) AT_CHECK_KEYS([-3], [none|first|second], [3], [both], [0]) AT_CHECK_KEYS([4-], [both], [1], [none|first|second], [0]) AT_CHECK_KEYS([010], [test10], [1], [none|first|second|both], [0]) AT_CHECK_KEYS([-k second 4-], [second|both], [2], [none|first], [0]) AT_CHECK([$CONFIG_SHELL ./k 0], [1], [ignore], [ignore]) AT_CHECK([$CONFIG_SHELL ./k 0-], [1], [ignore], [ignore]) AT_CHECK([$CONFIG_SHELL ./k -0], [1], [ignore], [ignore]) AT_CHECK([$CONFIG_SHELL ./k 11], [1], [ignore], [ignore]) AT_CHECK([$CONFIG_SHELL ./k 11-], [1], [ignore], [ignore]) AT_CHECK([$CONFIG_SHELL ./k 1-011], [1], [ignore], [ignore]) AT_CHECK([$CONFIG_SHELL ./k -k nonexistent], [0], [ignore]) AT_CHECK_KEYS([--list -k nonexistent], [KEYWORDS], [1], [first|second|both], [0]) AT_CHECK_KEYS([--list 1], [none], [1], [first|second|both], [0]) AT_CHECK_KEYS([--list 01], [none], [1], [first|second|both], [0]) AT_CHECK_KEYS([--list -k none -k first], [none|first], [2], [second|both], [0]) AT_CLEANUP ## ----------------------- ## ## parallel test execution ## ## ----------------------- ## AT_SETUP([parallel test execution]) # This test tries to ensure that -j runs tests in parallel. # Such a test is inherently racy, because there are no real-time # guarantees about scheduling delays. So we try to minimize # the chance to lose the race. # The time needed for a micro-suite consisting of NTESTS tests each # sleeping for a second is estimated by # startup + ntests * (serial_overhead + 1 / njobs) # # in absence of major scheduling delays. This leads to side conditions: # - NTESTS should be high, so the STARTUP time is small compared to the # test run time, and scheduling delays can even out; it should not be # too high, to not slow down the testsuite unnecessarily, # - the number of concurrent jobs NJOBS should not be too low, so the # race is not lost so easily; it should not be too high, to avoid fork # failures on tightly limited systems. 4 seems a good compromise # here, considering that Autotest spawns several other processes. # - STARTUP is assumed to be the same for parallel and serial runs, so # the latter can estimate the former. # - To avoid unportable output from time measurement commands, spawn # both a parallel and a serial testsuite run; check that the former # completes before the latter has completed a fraction SERIAL_NTESTS # of the tests (the serial run is executed in a subdirectory), plus # some additional time to allow for compensation of SERIAL_OVERHEAD. # - when adding this time to the serial test execution, an initial delay # SERIAL_DELAY of the serial test helps to avoid unreliable scheduling # due to the startup burst of the suites. dnl total number of tests. m4_define([AT_PARALLEL_NTESTS], [16]) dnl number of jobs to run in parallel. m4_define([AT_PARALLEL_NJOBS], [4]) dnl number of tests to run serially, as comparison. m4_define([AT_PARALLEL_SERIAL_NTESTS], m4_eval(AT_PARALLEL_NTESTS / AT_PARALLEL_NJOBS)) dnl initial delay of serial run, to compensate for SERIAL_OVERHEAD. dnl This corresponds to 0.67 s of overhead per test. m4_define([AT_PARALLEL_SERIAL_DELAY], m4_eval((AT_PARALLEL_NTESTS - AT_PARALLEL_SERIAL_NTESTS + 1) * 2 / 3)) AT_CHECK_AT_PREP([micro-suite], [[AT_INIT([suite to test parallel execution]) m4_for([count], [1], ]]AT_PARALLEL_NTESTS[[, [], [AT_SETUP([test number count]) AT_CHECK([sleep 1]) AT_CLEANUP ]) ]]) AT_CHECK([$CONFIG_SHELL ./micro-suite --help | grep " --jobs"], [0], [ignore]) AT_CHECK([$CONFIG_SHELL ./micro-suite -j2foo], [1], [], [stderr]) AT_CHECK([grep 'non-numeric argument' stderr], [], [ignore]) AT_CHECK([$CONFIG_SHELL ./micro-suite --jobs=foo], [1], [], [stderr]) AT_CHECK([grep 'non-numeric argument' stderr], [], [ignore]) AT_CHECK([$CONFIG_SHELL ./micro-suite -j[]AT_PARALLEL_NJOBS], [], [stdout]) # Ensure that all tests run, and lines are not split. AT_CHECK([grep -c '^.\{53\}ok' stdout], [], [AT_PARALLEL_NTESTS ]) # Running one test with -j should produce correctly formatted output: AT_CHECK([$CONFIG_SHELL ./micro-suite -j 3], [], [stdout]) AT_CHECK([grep -c '^.\{53\}ok' stdout], [], [1 ]) # Specifying more jobs than tests should not hang: AT_CHECK([$CONFIG_SHELL ./micro-suite -j3 3], [], [stdout]) AT_CHECK([grep -c '^.\{53\}ok' stdout], [], [1 ]) # Not even with zero tests: AT_CHECK([$CONFIG_SHELL ./micro-suite -j -k nomatch], [], [ignore]) AT_CHECK([$CONFIG_SHELL ./micro-suite -j3 -k nomatch], [], [ignore]) # The parallel scheduler requires mkfifo to work. AT_CHECK([mkfifo fifo || exit 77]) mkdir serial # Unfortunately, the return value of wait is unreliable, # so we check that kill fails. AT_CHECK([$CONFIG_SHELL ./micro-suite --jobs=[]AT_PARALLEL_NJOBS & ]dnl [sleep AT_PARALLEL_SERIAL_DELAY && ]dnl [cd serial && $CONFIG_SHELL ../micro-suite -AT_PARALLEL_SERIAL_NTESTS >/dev/null && ]dnl [{ kill $! && exit 1; :; }], [], [stdout], [ignore]) AT_CHECK([grep -c '^.\{53\}ok' stdout], [], [AT_PARALLEL_NTESTS ]) AT_CHECK([grep 'AT_PARALLEL_NTESTS tests' stdout], [], [ignore]) AT_CLEANUP AT_CHECK_AT_TEST([parallel truth], [AT_CHECK([:], 0, [], [])], [], [], [], [], [], [-j]) AT_CHECK_AT_TEST([parallel fallacy], [AT_CHECK([false], [], [], [])], [], [1], [], [ignore], [AT_CHECK([grep failed micro-suite.log], [], [ignore])], [-j]) AT_CHECK_AT_TEST([parallel skip], [AT_CHECK([exit 77], 0, [], [])], [], [], [], [], [AT_CHECK([grep skipped micro-suite.log], [], [ignore])], [-j]) AT_CHECK_AT_TEST([parallel errexit], [AT_CHECK([false]) AT_CLEANUP AT_SETUP([barrier test]) AT_CHECK([sleep 4]) AT_CLEANUP AT_SETUP([test that should not be run]) AT_CHECK([:])], [], [1], [stdout], [stderr], [AT_CHECK([test -f micro-suite.log], [1]) touch micro-suite.log # shut up AT_CAPTURE_FILE. AT_CHECK([grep "should not be run" stdout], [1]) AT_CHECK([grep "[[12]] .* inhibited subsequent" stderr], [], [ignore])], [-j2 --errexit]) ## ------------------- ## ## srcdir propagation. ## ## ------------------- ## AT_SETUP([srcdir propagation]) mkdir pkg vpath-outside vpath-abs mkdir pkg/t pkg/vpath-inside AT_DATA([pkg/a]) AT_DATA([pkg/t/b]) AT_DATA([pkg/configure.ac], [[AC_INIT AC_CONFIG_TESTDIR([t]) AC_OUTPUT ]]) cp "$abs_top_srcdir/build-aux/install-sh" pkg cd pkg AT_CHECK_AUTOCONF cd .. AT_CHECK_AT_PREP([suite], [[AT_INIT([suite to check srcdir]) AT_SETUP([my only test]) AT_CHECK([test -f "$top_srcdir"/a && test -f "$srcdir"/b]) AT_CLEANUP ]], [], [], [], [pkg/t]) # Build directory totally outside source directory. cd vpath-outside AT_CHECK([../pkg/configure], [0], [ignore]) cd t AT_CHECK([../../pkg/t/suite], [0], [ignore]) cd ../.. # Build directory totally outside source directory (absolute). my_srcdir=`pwd`/pkg cd vpath-abs AT_CHECK(["$my_srcdir"/configure], [0], [ignore]) cd t AT_CHECK(["$my_srcdir"/t/suite], [0], [ignore]) cd ../.. # Build directory as subdirectory of source directory. cd pkg/vpath-inside AT_CHECK([../configure], [0], [ignore]) cd t AT_CHECK([../../t/suite], [0], [ignore]) cd ../../.. # Build directory as parent of source directory. AT_CHECK([pkg/configure], [0], [ignore]) cd t AT_CHECK([../pkg/t/suite], [0], [ignore]) cd .. # Build directory as source directory. cd pkg AT_CHECK_CONFIGURE cd t AT_CHECK([./suite], [0], [ignore]) cd ../.. AT_CLEANUP ## ------------------------------ ## ## whitespace in absolute testdir ## ## ------------------------------ ## AT_SETUP([whitespace in absolute testdir]) dir='dir with whitespace' mkdir "$dir" cd "$dir" wd=`pwd` AT_DATA([a]) AT_CHECK_AT_PREP([suite], [[AT_INIT([suite to check srcdir]) AT_SETUP([my only test]) AT_CHECK([test -f "$top_srcdir"/a]) AT_CLEANUP ]]) AT_CHECK([top_srcdir=$wd ./suite], [0], [ignore]) AT_CHECK([top_srcdir=$wd ./suite -d], [0], [ignore]) AT_CHECK([cd suite.dir/1 && ./run top_srcdir="$wd"], [0], [ignore], [ignore]) AT_CLEANUP ## ------------------ ## ## unusual file names ## ## ------------------ ## AT_SETUP([unusual file names]) AT_DATA_M4SUGAR([d@&t@nl.at], [[AT@&t@_SETUP([test one]) m4_pattern_allow([^dnl$]) AT@&t@_CHECK([test "]m4_dquote(AT@&t@_LINE)[" = dn[]l.at:3]) AT@&t@_CLEANUP ]]) mkdir sub AT_DATA_M4SUGAR([sub/"two spaces".at], [[AT@&t@_SETUP([test two]) AT@&t@_CHECK([test "]m4_dquote(AT@&t@_LINE)[" = "two spaces.at:2"]) AT@&t@_CLEANUP ]]) AT_CHECK_AT_PREP([suite], [[AT_INIT([suite to check included file names]) m4@&t@_include([d][nl.at]) m4@&t@_include([sub/two spaces.at]) ]]) AT_CHECK([$CONFIG_SHELL ./suite], [0], [ignore]) AT_CLEANUP