mirror/curl
2
0
Fork 0
mirror of https://github.com/curl/curl.git synced 2025-01-18 14:04:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

produce a skip-report at the end of all tests, and thus record and count

Browse Source 
them properly
This commit is contained in:
Daniel Stenberg 2003-06-26 11:44:01 +00:00
parent 4322c1106f
commit 26e17d89c9
1 changed files with 108 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

162
tests/runtests.pl
View File

@ -48,11 +48,8 @@ my $FTPSPIDFILE=".ftps.pid";
# invoke perl like this: # invoke perl like this:
my $perl="perl -I$srcdir"; my $perl="perl -I$srcdir";
# this gets set if curl is compiled with memory debugging: # this gets set if curl is compiled with debugging:
my $memory_debug=0; my $curl_debug=0;
# this gets set if curl is compiled with netrc debugging:
my $netrc_debug = 0;
# name of the file that the memory debugging creates: # name of the file that the memory debugging creates:
my $memdump="memdump"; my $memdump="memdump";
@ -65,7 +62,8 @@ my $checkstunnel = &checkstunnel;
my $ssl_version; # set if libcurl is built with SSL support my $ssl_version; # set if libcurl is built with SSL support
my $skipped=0; # number of tests skipped; reported in main loop my $skipped=0; # number of tests skipped; reported in main loop
my $problems=0; # number of tests that didn't run due to run-time problems my %skipped; # skipped{reason}=counter, reasons for skip
my @teststat; # teststat[testnum]=reason, reasons for skip
####################################################################### #######################################################################
# variables the command line options may set # variables the command line options may set
@ -190,7 +188,7 @@ sub runhttpserver {
} }
my $flag=$debugprotocol?"-v ":""; my $flag=$debugprotocol?"-v ":"";
my $cmd="$perl $srcdir/httpserver.pl $flag $HOSTPORT &"; $cmd="$perl $srcdir/httpserver.pl $flag $HOSTPORT &";
system($cmd); system($cmd);
if($verbose) { if($verbose) {
print "CMD: $cmd\n"; print "CMD: $cmd\n";
@ -439,18 +437,40 @@ sub compare {
####################################################################### #######################################################################
# display information about curl and the host the test suite runs on # display information about curl and the host the test suite runs on
# #
sub displaydata { sub checkcurl {
unlink($memdump); # remove this if there was one left unlink($memdump); # remove this if there was one left
my $curl;
my $libcurl;
my @version=`$CURL -V`; my @version=`$CURL -V`;
my $version=$version[0]; for(@version) {
chomp $version; chomp;
my $curl = $version; if($_ =~ /^curl/) {
$curl = $_;
$curl =~ s/^(.*)(libcurl.*)/$1/g; $curl =~ s/^(.*)(libcurl.*)/$1/g;
my $libcurl = $2; $libcurl = $2;
}
elsif($_ =~ /^Protocols: (.*)/i) {
# these are the supported protocols, we don't use this knowledge
# at this point
}
elsif($_ =~ /^Features: (.*)/i) {
my $feat = $1;
if($feat =~ /debug/i) {
# debug is a listed "feature", use that knowledge
$curl_debug = 1;
# set the NETRC debug env
$ENV{'CURL_DEBUG_NETRC'} = 'log/netrc';
}
if($feat =~ /SSL/i) {
# ssl enabled
$ssl_version=1;
}
}
}
my $hostname=`hostname`; my $hostname=`hostname`;
my $hosttype=`uname -a`; my $hosttype=`uname -a`;
@ -461,26 +481,9 @@ sub displaydata {
"* Host: $hostname", "* Host: $hostname",
"* System: $hosttype"; "* System: $hosttype";
if($libcurl =~ /SSL/i) { printf("* Server SSL: %s\n", $checkstunnel?"ON":"OFF");
$ssl_version=1;
}
if( -r $memdump) {
# if this exists, curl was compiled with memory debugging
# enabled and we shall verify that no memory leaks exist
# after each and every test!
$memory_debug=1;
# there's only one debug control in the configure script
# so hope netrc debugging is enabled and set it up
$netrc_debug = 1;
$ENV{'CURL_DEBUG_NETRC'} = 'log/netrc';
}
printf("* Memory debugging: %s\n", $memory_debug?"ON":"OFF");
printf("* Netrc debugging: %s\n", $netrc_debug?"ON":"OFF");
printf("* HTTPS server: %s\n", $checkstunnel?"ON":"OFF");
printf("* FTPS server: %s\n", $checkstunnel?"ON":"OFF");
printf("* libcurl SSL: %s\n", $ssl_version?"ON":"OFF"); printf("* libcurl SSL: %s\n", $ssl_version?"ON":"OFF");
printf("* libcurl debug: %s\n", $curl_debug?"ON":"OFF");
print "***************************************** \n"; print "***************************************** \n";
} }
@ -506,16 +509,24 @@ sub subVariables {
sub singletest { sub singletest {
my $testnum=$_[0]; my $testnum=$_[0];
my @what;
my $why;
my $serverproblem;
# load the test case file definition # load the test case file definition
if(loadtest("${TESTDIR}/test${testnum}")) { if(loadtest("${TESTDIR}/test${testnum}")) {
if($verbose) { if($verbose) {
# this is not a test # this is not a test
print "RUN: $testnum doesn't look like a test case!\n"; print "RUN: $testnum doesn't look like a test case!\n";
} }
return -1; $serverproblem = 100;
}
else {
@what = getpart("client", "features");
} }
my @what = getpart("client", "features"); printf("test %03d...", $testnum);
for(@what) { for(@what) {
my $f = $_; my $f = $_;
$f =~ s/\s//g; $f =~ s/\s//g;
@ -526,27 +537,45 @@ sub singletest {
} }
} }
elsif($f eq "netrc_debug") { elsif($f eq "netrc_debug") {
if($netrc_debug) { if($curl_debug) {
next; next;
} }
} }
warn "Test case $testnum requires the missing feature: $_"; $why = "lacks $f";
return -1; $serverproblem = 5; # set it here
last;
} }
my $serverproblem = serverfortest($testnum); if(!$serverproblem) {
$serverproblem = serverfortest($testnum);
}
if($serverproblem) { if($serverproblem) {
# there's a problem with the server, don't run # there's a problem with the server, don't run
# this particular server, but count it as "skipped" # this particular server, but count it as "skipped"
if($serverproblem> 1) { my $why;
print "RUN: test case $testnum couldn't run!\n"; if($serverproblem == 2) {
$problems++; $why = "server problems";
}
elsif($serverproblem == 100) {
$why = "no test";
}
elsif($serverproblem == 99) {
$why = "bad test";
} }
else { else {
$skipped++; $why = "unfulfilled requirements";
} }
$skipped++;
$skipped{$why}++;
$teststat[$testnum]=$why; # store reason for this test case
print "SKIPPED\n";
if(!$short) {
print "* Test $testnum: $why\n";
}
return -1; return -1;
} }
@ -591,7 +620,6 @@ sub singletest {
# name of the test # name of the test
my @testname= getpart("client", "name"); my @testname= getpart("client", "name");
printf("test %03d...", $testnum);
if(!$short) { if(!$short) {
my $name = $testname[0]; my $name = $testname[0];
$name =~ s/\n//g; $name =~ s/\n//g;
@ -653,7 +681,7 @@ sub singletest {
#$cmd =~ s/%HOSTNAME/$HOSTNAME/g; #$cmd =~ s/%HOSTNAME/$HOSTNAME/g;
if($memory_debug) { if($curl_debug) {
unlink($memdump); unlink($memdump);
} }
@ -874,7 +902,7 @@ sub singletest {
unlink($FTPDCMD); # remove the instructions for this test unlink($FTPDCMD); # remove the instructions for this test
my @what = getpart("client", "killserver"); @what = getpart("client", "killserver");
for(@what) { for(@what) {
my $serv = $_; my $serv = $_;
chomp $serv; chomp $serv;
@ -887,7 +915,7 @@ sub singletest {
} }
} }
if($memory_debug) { if($curl_debug) {
if(! -f $memdump) { if(! -f $memdump) {
print "\n** ALERT! memory debuggin without any output file?\n"; print "\n** ALERT! memory debuggin without any output file?\n";
} }
@ -925,6 +953,12 @@ sub singletest {
# This function makes sure the right set of server is running for the # This function makes sure the right set of server is running for the
# specified test case. This is a useful design when we run single tests as not # specified test case. This is a useful design when we run single tests as not
# all servers need to run then! # all servers need to run then!
#
# Returns:
# 100 if this is not a test case
# 99 if this test case has no servers specified
# 2 if one of the required servers couldn't be started
# 1 if this test is skipped due to unfulfilled requirements
sub serverfortest { sub serverfortest {
my ($testnum)=@_; my ($testnum)=@_;
@ -943,7 +977,7 @@ sub serverfortest {
if(!$what[0]) { if(!$what[0]) {
warn "Test case $testnum has no server(s) specified!"; warn "Test case $testnum has no server(s) specified!";
return 100; return 99;
} }
for(@what) { for(@what) {
@ -1018,7 +1052,6 @@ sub serverfortest {
warn "we don't support a server for $what"; warn "we don't support a server for $what";
} }
} }
return 0; # ok
} }
####################################################################### #######################################################################
@ -1105,7 +1138,7 @@ if($testthis[0] ne "") {
# #
if(!$listonly) { if(!$listonly) {
displaydata(); checkcurl();
} }
####################################################################### #######################################################################
@ -1150,11 +1183,14 @@ my $failed;
my $testnum; my $testnum;
my $ok=0; my $ok=0;
my $total=0; my $total=0;
my $lasttest;
foreach $testnum (split(" ", $TESTCASES)) { foreach $testnum (split(" ", $TESTCASES)) {
$lasttest = $testnum if($testnum > $lasttest);
my $error = singletest($testnum); my $error = singletest($testnum);
if(-1 == $error) { if($error < 0) {
# not a test we can run # not a test we can run
next; next;
} }
@ -1203,10 +1239,28 @@ else {
print "TESTFAIL: No tests were performed!\n"; print "TESTFAIL: No tests were performed!\n";
} }
if($skipped) { if($skipped) {
print "TESTINFO: $skipped tests were skipped due to restraints\n"; my $s=0;
} print "TESTINFO: $skipped tests were skipped due to these restraints:\n";
if($problems) {
print "TESTINFO: $problems tests didn't run due to run-time problems\n"; for(keys %skipped) {
print ", " if($s);
my $r = $_;
printf "TESTINFO: \"%s\" happened %d times (", $r, $skipped{$_};
# now show all test case numbers that had this reason for being
# skipped
my $c=0;
for(0 .. $lasttest) {
my $t = $_;
if($teststat[$_] eq $r) {
print ", " if($c);
print $_;
$c++;
}
}
print ")\n";
$s++;
}
} }
if($total && ($ok != $total)) { if($total && ($ok != $total)) {
exit 1; exit 1;