This is the mail archive of the
java-patches@gcc.gnu.org
mailing list for the Java project.
libjava test framework improvements
- To: gcc-patches at gcc dot gnu dot org, java-patches at gcc dot gnu dot org
- Subject: libjava test framework improvements
- From: "Zack Weinberg" <zackw at stanford dot edu>
- Date: Mon, 2 Apr 2001 14:45:50 -0700
This patch corrects two problems with the libjava test framework.
First, there was a typo in a result message in one place, which would
cause you to get confusing reports like
FAIL: Array_1 execution from source compiled test
XFAIL: Array_1 execution from source compiled test
The second 'execution' was meant to be 'output'.
Second, the framework generated XFAILs for all the tests that didn't
get run because a preceding test failed. For example,
FAIL: Divide_1 compilation from bytecode
XFAIL: Divide_1 execution from bytecode->native test
XFAIL: Divide_1 output from bytecode->native test
This obscures the _actual_ expected failures and makes it look like
the library is in much worse shape than it really is. The patch
causes it to generate UNTESTEDs instead. Compare these two summaries:
before:
=== libjava Summary ===
# of expected passes 840
# of unexpected failures 370
# of unexpected successes 14
# of expected failures 500
after:
=== libjava Summary ===
# of expected passes 846
# of unexpected failures 370
# of unexpected successes 8
# of expected failures 26
# of untested testcases 474
IMO, that makes it much clearer what's going on.
The number of unexpected passes went down because there was a
setup_xfail in the wrong place and it leaked into the next test.
It went away with this patch.
zw
* testsuite/lib/libjava.exp: Correct typo: 'output from source
compiled test', not 'execution from source compiled test'.
Use UNTESTED, not XFAIL, for tests which are not run because
they depend on a previous test which failed.
===================================================================
Index: testsuite/lib/libjava.exp
--- testsuite/lib/libjava.exp 2001/03/16 20:54:45 1.23
+++ testsuite/lib/libjava.exp 2001/04/02 21:38:31
@@ -374,10 +374,8 @@ proc test_libjava_from_source { options
fail "$errname compilation from source"
if {[info exists opts(xfail-gcj)] || ! [info exists opts(no-exec)]} {
- setup_xfail "*-*-*"
- fail "$errname execution from source compiled test"
- setup_xfail "*-*-*"
- fail "$errname output from source compiled test"
+ untested "$errname execution from source compiled test"
+ untested "$errname output from source compiled test"
}
return
}
@@ -400,8 +398,7 @@ proc test_libjava_from_source { options
}
$status "$errname execution from source compiled test"
if { $status != "pass" } {
- setup_xfail "*-*-*"
- fail "$errname execution from source compiled test"
+ untested "$errname output from source compiled test"
return;
}
@@ -488,13 +485,10 @@ proc test_libjava_from_javac { options s
return
}
fail "$errname byte compilation"
- setup_xfail "*-*-*"
- fail "$errname compilation from bytecode"
+ untested "$errname compilation from bytecode"
if {! [info exists opts(no-exec)]} {
- setup_xfail "*-*-*"
- fail "$errname execution from bytecode->native test"
- setup_xfail "*-*-*"
- fail "$errname output from bytecode->native test"
+ untested "$errname execution from bytecode->native test"
+ untested "$errname output from bytecode->native test"
}
return
}
@@ -520,14 +514,10 @@ proc test_libjava_from_javac { options s
if {[string match "*parse error*" $main_name]
|| [string match "*parse error*" $class_out]} {
- # Do the remaining fails.
- setup_xfail "*-*-*"
- fail "$errname compilation from bytecode"
+ untested "$errname compilation from bytecode"
if {! [info exists opts(no-exec)]} {
- setup_xfail "*-*-*"
- fail "$errname execution from bytecode->native test"
- setup_xfail "*-*-*"
- fail "$errname output from bytecode->native test"
+ untested "$errname execution from bytecode->native test"
+ untested "$errname output from bytecode->native test"
}
return
}
@@ -603,11 +593,9 @@ proc test_libjava_from_javac { options s
if { $x != "" } {
verbose "target_compile failed: $x" 2
fail "$errname compilation from bytecode"
- setup_xfail "*-*-*"
if {! [info exists opts(no-exec)]} {
- fail "$errname execution from bytecode->native test"
- setup_xfail "*-*-*"
- fail "$errname output from bytecode->native test"
+ untested "$errname execution from bytecode->native test"
+ untested "$errname output from bytecode->native test"
}
return;
}
@@ -625,8 +613,7 @@ proc test_libjava_from_javac { options s
}
$status "$errname execution from bytecode->native test"
if { $status != "pass" } {
- setup_xfail "*-*-*"
- fail "$errname output from bytecode->native test"
+ untested "$errname output from bytecode->native test"
return;
}