[testsuite/RFC] dg-shouldfail test directive

[Janis Johnson <janis187 at us dot ibm dot com>]

This is an update of a patch I worked on last month to add testsuite
functionality that Jerry DeLisle say is needed for Fortran tests:
checking error handling at runtime, so that the test executable is
expected to fail.  The new directive dg-shouldfail gets its name from
similar functionality in the libjava testsuite.

This has been tested by a complete testrun of all languages but Ada
with both -m32 and -m64 on powerpc64-linux; the documentation changes
were tested with "make dvi", "make info", and "make html".  I also
tried some dummy tests in gfortran.dg to make sure it works there.

If this looks useful to the Fortran people I'll check it in.

2006-06-07  Janis Johnson  <janis187@us.ibm.com>

	* doc/sourcebuild.texi (Test Directives): Document dg-shouldfail.

testsuite:

	* lib/gcc-dg.exp (tool_load): Wrapper to support shouldfail tests.
	(dg-test): Support shouldfail tests.
	* lib/target-supports-dg.exp (dg-shouldfail): New.
	* gcc.test-framework/test-framework.awk: Handle shouldfail tests.
	* gcc.test-framework/dg-do-run-sf-exp-F.c: New test.
	* gcc.test-framework/dg-do-run-sf-exp-P.c: New test.
	* gcc.test-framework/dg-do-run-sft-exp-F.c: New test.
	* gcc.test-framework/dg-do-run-sft-exp-P.c: New test.
	* gcc.test-framework/dg-dox-run-sf-exp-XF.c: New test.
	* gcc.test-framework/dg-dox-run-sf-exp-XP.c: New test.

Index: gcc/doc/sourcebuild.texi
===================================================================
--- gcc/doc/sourcebuild.texi	(revision 114398)
+++ gcc/doc/sourcebuild.texi	(working copy)
@@ -999,6 +999,10 @@
 is not covered by the effective-target keyword.
 This directive must appear after any @code{dg-do} directive in the test.
 
+@item  @{ dg-shouldfail @var{comment} [@{ @var{selector} @} @{ @var{include-opts} @} @{ @var{exclude-opts} @}] @}
+Expect the test executable to return a nonzero exit status if the
+conditions (which are the same as for @code{dg-skip-if}) are met.
+
 @item @{ dg-error @var{regexp} [@var{comment} [@{ target/xfail @var{selector} @} [@var{line}] @}]] @}
 This DejaGnu directive appears on a source line that is expected to get
 an error message, or else specifies the source line associated with the
Index: gcc/testsuite/gcc.test-framework/test-framework.awk
===================================================================
--- gcc/testsuite/gcc.test-framework/test-framework.awk	(revision 114398)
+++ gcc/testsuite/gcc.test-framework/test-framework.awk	(working copy)
@@ -3,7 +3,7 @@
 # of passing tests.
 #
 #
-# Copyright (c) 2004, 2005 Free Software Foundation, Inc.
+# Copyright (c) 2004, 2005, 2006 Free Software Foundation, Inc.
 #
 # This file is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -45,6 +45,8 @@
 /^PASS.*sa.*-1.c.*(test for excess errors)/ { ignore(); next }
 # The other dox tests pass the compile step; ignore that message.
 /^PASS.*dox.*(test for excess errors)/ { ignore(); next }
+# The sf tests pass the compile step; ignore that message.
+/^PASS.*sf.*(test for excess errors)/ { ignore(); next }
 # Ignore lines that begin with comma.
 /^,/			{ ignore(); next }
 # For tests of dg-output, ignore successful compilation.
Index: gcc/testsuite/lib/gcc-dg.exp
===================================================================
--- gcc/testsuite/lib/gcc-dg.exp	(revision 114398)
+++ gcc/testsuite/lib/gcc-dg.exp	(working copy)
@@ -178,6 +178,28 @@
     return $text
 }
 
+# Replace ${tool}_load with a wrapper to provide for an expected nonzero
+# exit status.  Multiple languages include this file so this handles them
+# all, not just gcc.
+if { [info procs ${tool}_load] != [list] \
+      && [info procs saved_${tool}_load] == [list] } {
+    rename ${tool}_load saved_${tool}_load
+
+    proc ${tool}_load { program args } {
+	global tool
+	global shouldfail
+	set result [saved_${tool}_load $program $args]
+	if { $shouldfail != 0 } {
+	    switch [lindex $result 0] {
+		"pass" { set status "fail" }
+		"fail" { set status "pass" }
+	    }
+	    set result [list $status [lindex $result 1]]
+	}
+	return $result
+    }
+}
+
 # Utility routines.
 
 #
@@ -428,12 +450,14 @@
 	global additional_prunes
 	global errorInfo
 	global compiler_conditional_xfail_data
+	global shouldfail
 
 	if { [ catch { eval saved-dg-test $args } errmsg ] } {
 	    set saved_info $errorInfo
 	    set additional_files ""
 	    set additional_sources ""
 	    set additional_prunes ""
+	    set shouldfail 0
 	    if [info exists compiler_conditional_xfail_data] {
 		unset compiler_conditional_xfail_data
 	    }
@@ -442,6 +466,7 @@
 	set additional_files ""
 	set additional_sources ""
 	set additional_prunes ""
+	set shouldfail 0
 	if [info exists compiler_conditional_xfail_data] {
 	    unset compiler_conditional_xfail_data
 	}
Index: gcc/testsuite/lib/target-supports-dg.exp
===================================================================
--- gcc/testsuite/lib/target-supports-dg.exp	(revision 114398)
+++ gcc/testsuite/lib/target-supports-dg.exp	(working copy)
@@ -262,6 +262,37 @@
     }
 }
 
+# Record whether the program is expected to return a nonzero status.
+
+set shouldfail 0
+
+proc dg-shouldfail { args } {
+    # Don't bother if we're already skipping the test.
+    upvar dg-do-what dg-do-what
+    if { [lindex ${dg-do-what} 1] == "N" } {
+      return
+    }
+
+    global shouldfail
+
+    set args [lreplace $args 0 0]
+    if { [llength $args] > 1 } {
+	set selector [list target [lindex $args 1]]
+	if { [dg-process-target $selector] == "S" } {
+	    # The target matches, now check the flags.  These variables
+	    # are defined in DejaGnu's dg-test, needed by check-flags.
+	    upvar dg-extra-tool-flags dg-extra-tool-flags
+	    upvar tool_flags tool_flags
+
+	    if [check-flags $args] {
+		set shouldfail 1
+	    }
+	}
+    } else {
+	set shouldfail 1
+    }
+}
+
 # Intercept the call to the DejaGnu version of dg-process-target to
 # support use of an effective-target keyword in place of a list of
 # target triplets to xfail or skip a test.
--- /dev/null	2006-05-17 17:30:47.265893448 -0700
+++ gcc/testsuite/gcc.test-framework/dg-do-run-sf-exp-F.c	2006-06-05 11:18:17.000000000 -0700
@@ -0,0 +1,8 @@
+/* { dg-do run } */
+/* { dg-shouldfail "required comment" } */
+
+int
+main ()
+{
+    return 0;  /* We expect nonzero, so this fails.  */
+}
--- /dev/null	2006-05-17 17:30:47.265893448 -0700
+++ gcc/testsuite/gcc.test-framework/dg-do-run-sf-exp-P.c	2006-06-05 11:18:24.000000000 -0700
@@ -0,0 +1,10 @@
+/* { dg-do run } */
+/* { dg-shouldfail "required comment" } */
+
+extern void abort (void);
+
+int
+main ()
+{
+    abort ();  /* We expect nonzero, so this passes.  */
+}
--- /dev/null	2006-05-17 17:30:47.265893448 -0700
+++ gcc/testsuite/gcc.test-framework/dg-do-run-sft-exp-F.c	2006-06-05 11:19:25.000000000 -0700
@@ -0,0 +1,10 @@
+/* { dg-do run } */
+/* { dg-shouldfail "comment" { unknown-*-* } { "*" } { "" } } */
+
+extern void abort (void);
+
+int
+main ()
+{
+    abort ();  /* Directive is ignored so we expect zero; this fails.  */
+}
--- /dev/null	2006-05-17 17:30:47.265893448 -0700
+++ gcc/testsuite/gcc.test-framework/dg-do-run-sft-exp-P.c	2006-06-05 11:19:33.000000000 -0700
@@ -0,0 +1,10 @@
+/* { dg-do run } */
+/* { dg-shouldfail "comment" { *-*-* } { "*" } { "" } } */
+
+extern void abort (void);
+
+int
+main ()
+{
+    abort ();  /* We expect nonzero exit, so this passes.  */
+}
--- /dev/null	2006-05-17 17:30:47.265893448 -0700
+++ gcc/testsuite/gcc.test-framework/dg-dox-run-sf-exp-XF.c	2006-06-05 11:19:41.000000000 -0700
@@ -0,0 +1,8 @@
+/* { dg-do run { xfail *-*-* } } */
+/* { dg-shouldfail "required comment" } */
+
+int
+main ()
+{
+    return 0;  /* We want nonzero but expect to fail; XFAIL.  */
+}
--- /dev/null	2006-05-17 17:30:47.265893448 -0700
+++ gcc/testsuite/gcc.test-framework/dg-dox-run-sf-exp-XP.c	2006-06-05 11:20:31.000000000 -0700
@@ -0,0 +1,10 @@
+/* { dg-do run { xfail *-*-* } } */
+/* { dg-shouldfail "required comment" } */
+
+extern void abort (void);
+
+int
+main ()
+{
+    abort ();  /* We want nonzero, but expect to fail; XPASS.  */
+}