Patch: xfail support in test suite

[Tom Tromey <tromey@cygnus.com>]

I'm committing the appended patch.  It adds support for `.xfail' files
to the test suite.  These files let you (partially) express the
conditions under which a test is expected to fail.

The goal is to make it so the 2.95 release will run with 0 failures.
That will make it very easy to analyze test results posted by users.

I'll be checking in a similar patch on the branch.  It will be a bit
different because the branch compiler has a few more known failures
(which are fixed on the trunk).

1999-07-06  Tom Tromey  <tromey@cygnus.com>

	* libjava.compile/Case.xfail: New file.
	* libjava.compile/Where.xfail: New file.
	* libjava.compile/test.exp: Look for `.xfail' files.
	* libjava.lang/test.exp: Don't look for `.arg' files.
	Do look for `.xfail' files.
	* lib/libjava.exp (test_libjava_from_source): Changed meaning of
	`exec_args' argument.  Handle `xfail-*' arguments.
	(test_libjava_from_javac): Likewise.
	(test_libjava): Likewise.
	(libjava_read_xfail): New proc.

Tom

Index: ChangeLog
===================================================================
RCS file: /cvs/java/libgcj/libjava/testsuite/ChangeLog,v
retrieving revision 1.17
diff -u -r1.17 ChangeLog
--- ChangeLog	1999/07/02 15:55:44	1.17
+++ ChangeLog	1999/07/06 22:31:52
@@ -1,3 +1,16 @@
+1999-07-06  Tom Tromey  <tromey@cygnus.com>
+
+	* libjava.compile/Case.xfail: New file.
+	* libjava.compile/Where.xfail: New file.
+	* libjava.compile/test.exp: Look for `.xfail' files.
+	* libjava.lang/test.exp: Don't look for `.arg' files.
+	Do look for `.xfail' files.
+	* lib/libjava.exp (test_libjava_from_source): Changed meaning of
+	`exec_args' argument.  Handle `xfail-*' arguments.
+	(test_libjava_from_javac): Likewise.
+	(test_libjava): Likewise.
+	(libjava_read_xfail): New proc.
+
 1999-07-02  Tom Tromey  <tromey@cygnus.com>
 
 	* libjava.lang/Finalize_1.out: Removed.
Index: lib/libjava.exp
===================================================================
RCS file: /cvs/java/libgcj/libjava/testsuite/lib/libjava.exp,v
retrieving revision 1.8
diff -u -r1.8 libjava.exp
--- libjava.exp	1999/06/25 13:59:09	1.8
+++ libjava.exp	1999/07/06 22:31:59
@@ -10,6 +10,17 @@
     set tmpdir "/tmp"
 }
 
+# Read an `xfail' file if it exists.  Returns a list of xfail tokens.
+proc libjava_read_xfail {file} {
+    if {! [file exists $file]} {
+	return ""
+    }
+    set fd [open $file r]
+    set tokens [string trim [read $fd]]
+    close $fd
+    return $tokens
+}
+
 # Find `jv-scan'.  FIXME: this relies on DejaGnu internals.  These
 # should probably be exposed in a better way.
 proc find_jvscan {} {
@@ -221,8 +232,15 @@
 
 #
 # Run the test specified by srcfile and resultfile. compile_args and
-# exec_args are additional arguments to be passed in when compiling and
-# running the testcase, respectively.
+# exec_args are options telling this proc how to work.
+#   `no-exec'     don't try to run the test
+#   `xfail-gcj'   compilation from source will fail
+#   `xfail-javac' compilation with javac will fail
+#   `xfail-gcjC'  compilation with gcj -C will fail
+#   `xfail-byte'  compilation from bytecode will fail
+#   `xfail-exec'  exec will fail
+#   `xfail-output' output will be wrong
+#   
 #
 proc test_libjava_from_source { options srcfile compile_args inpfile resultfile exec_args } {
     global base_dir
@@ -234,6 +252,13 @@
     global tmpdir
     global runtests
 
+    # Make opts into an array.
+    set opts(_) x
+    unset opts(_)
+    foreach item $exec_args {
+	set opts($item) x
+    }
+
     set errname [file rootname [file tail $srcfile]]
     if {! [runtest_file_p $runtests $errname]} {
         return
@@ -253,10 +278,13 @@
     }
 
     set x [target_compile $srcfile "$executable" executable $args]
+    if {[info exists opts(xfail-gcj)]} {
+	setup_xfail *-*-*
+    }
     if { $x != "" } {
 	verbose "target_compile failed: $x" 2
 	fail "$errname compilation from source"
-	if {$exec_args != "no-exec"} {
+	if {[info exists opts(xfail-gcj)] || ! [info exists opts(no-exec)]} {
 	    setup_xfail "*-*-*"
 	    fail "$errname execution from source compiled test"
 	    setup_xfail "*-*-*"
@@ -266,17 +294,20 @@
     }
     pass "$errname compilation from source"
 
-    if { $exec_args == "no-exec" } {
+    if {[info exists opts(no-exec)]} {
 	return
     }
 
-    set result [libjava_load $executable "$exec_args" "$inpfile"];
+    set result [libjava_load $executable "" "$inpfile"];
     set status [lindex $result 0];
     set output [lindex $result 1];
+    if {[info exists opts(xfail-exec)]} then {
+	setup_xfail *-*-*
+    }
     $status "$errname execution from source compiled test"
     if { $status != "pass" } {
 	setup_xfail "*-*-*"
-	fail "$errname output from source compiled test"
+	fail "$errname execution from source compiled test"
 	return;
     }
 
@@ -305,6 +336,9 @@
 	    set passed 1;
 	}
     }
+    if {[info exists opts(xfail-output)]} {
+	setup_xfail *-*-*
+    }
     if { $passed == 1 } {
 	pass "$errname output from source compiled test"
     } else {
@@ -317,8 +351,14 @@
 
 #
 # Run the test specified by srcfile and resultfile. compile_args and
-# exec_args are additional arguments to be passed in when compiling and
-# running the testcase, respectively.
+# exec_args are options telling this proc how to work.
+#   `no-exec' don't try to run the test
+#   `xfail-gcj'   compilation from source will fail
+#   `xfail-javac' compilation with javac will fail
+#   `xfail-gcjC'  compilation with gcj -C will fail
+#   `xfail-byte'  compilation from bytecode will fail
+#   `xfail-exec'  exec will fail
+#   `xfail-output' output will be wrong
 #
 proc test_libjava_from_javac { options srcfile compile_args inpfile resultfile exec_args } {
     global base_dir
@@ -330,6 +370,13 @@
     global tmpdir
     global runtests
 
+    # Make opts into an array.
+    set opts(_) x
+    unset opts(_)
+    foreach item $exec_args {
+	set opts($item) x
+    }
+
     set errname [file rootname [file tail $srcfile]]
     if {! [runtest_file_p $runtests $errname]} {
         return
@@ -337,11 +384,16 @@
 
     # bytecompile files with Sun's compiler for now.
     set bc_ok [bytecompile_file $srcfile $objdir]
+    # FIXME: assumes we are using javac to compile to bytecode.
+    # This is not always the case.
+    if {[info exists opts(xfail-javac)]} {
+	setup_xfail *-*-*
+    }
     if {! $bc_ok} then {
 	fail "$errname byte compilation"
 	setup_xfail "*-*-*"
 	fail "$errname compilation from bytecode"
-	if {$exec_args != "no-exec"} {
+	if {! [info exists opts(no-exec)]} {
 	    setup_xfail "*-*-*"
 	    fail "$errname execution from bytecode->native test"
 	    setup_xfail "*-*-*"
@@ -380,7 +432,7 @@
     # method.  However, for no-exec tests it is ok.
     set largs {}
     if {$main_name == ""} {
-	if {$exec_args != "no-exec"} {
+	if {! [info exists opts(no-exec)]} {
 	    perror "No `main' given in program $errname"
 	    return
 	} else {
@@ -409,11 +461,14 @@
 
     verbose "compilation command = $args" 2
     set x [target_compile $class_files "$executable" $type $args]
+    if {[info exists opts(xfail-byte)]} {
+	setup_xfail *-*-*
+    }
     if { $x != "" } {
 	verbose "target_compile failed: $x" 2
 	fail "$errname compilation from bytecode"
 	setup_xfail "*-*-*"
-	if {$exec_args != "no-exec"} {
+	if {! [info exists opts(no-exec)]} {
 	    fail "$errname execution from bytecode->native test"
 	    setup_xfail "*-*-*"
 	    fail "$errname output from bytecode->native test"
@@ -422,13 +477,16 @@
     }
     pass "$errname compilation from bytecode"
 
-    if { $exec_args == "no-exec" } {
+    if {[info exists opts(no-exec)]} {
 	return
     }
 
-    set result [libjava_load $executable "$exec_args" "$inpfile"];
+    set result [libjava_load $executable "" "$inpfile"];
     set status [lindex $result 0];
     set output [lindex $result 1];
+    if {[info exists opts(xfail-exec)]} {
+	setup_xfail *-*-*
+    }
     $status "$errname execution from bytecode->native test"
     if { $status != "pass" } {
 	setup_xfail "*-*-*"
@@ -452,6 +510,9 @@
     verbose "expected is $expected"
     verbose "actual is $output"
     set passed 0;
+    if {[info exists opts(xfail-output)]} {
+	setup_xfail *-*-*
+    }
     if {$options == "regexp_match"} {
 	if [regexp $expected $output] {
 	    set passed 1;
@@ -473,8 +534,14 @@
 
 #
 # Run the test specified by srcfile and resultfile. compile_args and
-# exec_args are additional arguments to be passed in when compiling and
-# running the testcase, respectively.
+# exec_args are options telling this proc how to work.
+#   `no-exec' don't try to run the test
+#   `xfail-gcj'   compilation from source will fail
+#   `xfail-javac' compilation with javac will fail
+#   `xfail-gcjC'  compilation with gcj -C will fail
+#   `xfail-byte'  compilation from bytecode will fail
+#   `xfail-exec'  exec will fail
+#   `xfail-output' output will be wrong
 #
 proc test_libjava { options srcfile compile_args inpfile resultfile exec_args } {
     test_libjava_from_source $options $srcfile $compile_args $inpfile $resultfile $exec_args
@@ -489,3 +556,7 @@
 
 proc default_libjava_start { } {
 }
+
+# Local Variables:
+# tcl-indent-level:4
+# End:
Index: libjava.compile/Case.xfail
===================================================================
RCS file: Case.xfail
diff -N Case.xfail
--- /dev/null	Sat Dec  5 20:30:03 1998
+++ Case.xfail	Tue Jul  6 15:31:59 1999
@@ -0,0 +1 @@
+xfail-gcj
Index: libjava.compile/Where.xfail
===================================================================
RCS file: Where.xfail
diff -N Where.xfail
--- /dev/null	Sat Dec  5 20:30:03 1998
+++ Where.xfail	Tue Jul  6 15:31:59 1999
@@ -0,0 +1 @@
+xfail-gcj
Index: libjava.compile/test.exp
===================================================================
RCS file: /cvs/java/libgcj/libjava/testsuite/libjava.compile/test.exp,v
retrieving revision 1.1.1.1
diff -u -r1.1.1.1 test.exp
--- test.exp	1999/04/07 14:52:43	1.1.1.1
+++ test.exp	1999/07/06 22:31:59
@@ -5,6 +5,13 @@
 
 set prefix ""
 foreach x $srcfiles {
-    test_libjava $options "$x" "" "" "" "no-exec"
-    test_libjava $options "$x" "-O" "" "" "no-exec"
+    set args [libjava_read_xfail [file rootname $x].xfail]
+    lappend args no-exec
+
+    test_libjava $options "$x" "" "" "" $args
+    test_libjava $options "$x" "-O" "" "" $args
 }
+
+# Local Variables:
+# tcl-indent-level:4
+# End:
Index: libjava.lang/Array_1.xfail
===================================================================
RCS file: Array_1.xfail
diff -N Array_1.xfail
--- /dev/null	Sat Dec  5 20:30:03 1998
+++ Array_1.xfail	Tue Jul  6 15:31:59 1999
@@ -0,0 +1 @@
+xfail-gcj xfail-exec
Index: libjava.lang/test.exp
===================================================================
RCS file: /cvs/java/libgcj/libjava/testsuite/libjava.lang/test.exp,v
retrieving revision 1.1.1.1
diff -u -r1.1.1.1 test.exp
--- test.exp	1999/04/07 14:52:43	1.1.1.1
+++ test.exp	1999/07/06 22:31:59
@@ -7,12 +7,7 @@
 foreach x $srcfiles {
     regsub "\\.out$" $x "" prefix
     set bname [file tail $prefix]
-    set args ""
-    if [file exists $srcdir/$subdir/${bname}.arg] {
-	set id [open "$srcdir/$subdir/${bname}.arg" r];
-	set args [read -nonewline $id];
-	close $id;
-    }
+
     if [file exists $srcdir/$subdir/${bname}.xpo] {
 	set resfile "$srcdir/$subdir/${bname}.xpo"
 	set options "regexp_match"
@@ -27,8 +22,14 @@
 	set inpfile ""
     }
 
+    set args [libjava_read_xfail $srcdir/$subdir/$bname.xfail]
+
     verbose "inpfile is $inpfile"
 
     test_libjava $options "${prefix}.java" "" $inpfile $resfile $args
     test_libjava $options "${prefix}.java" "-O" $inpfile $resfile $args
 }
+
+# Local Variables:
+# tcl-indent-level:4
+# End: