This is the mail archive of the gcc-patches@gcc.gnu.org mailing list for the GCC project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

[patch] contrib: tools to test the dg test framework


I've been using the following to test change to the test framework that
I'm developing and figured they might be useful to other people as well.
This patch would put them into a new directory under contrib/.  Any
interest in putting them there?

This depends on a fix to lib/gcc-dg.exp that I'm about to submit.

2004-11-02  Janis Johnson  <janis187@us.ibm.com>

	* test_framework/README: New file.
	* test_framework/test-framework.exp: New file.
	* test_framework/test-framework.awk: New file.
	* test_framework/generate_framework_tests: New file.

--- /dev/null	2004-06-24 11:06:20.000000000 -0700
+++ test_framework/README	2004-11-02 13:21:58.000000000 -0800
@@ -0,0 +1,28 @@
+Test dg- commands used in GCC's test framework by generating and running
+tests that use combinations of those commands.  Each test has two parts:
+a file ending in "-1.c" whose expected result is encoded in the filename
+and a file ending in "-2.c" which is always expected to pass.
+
+To run these tests using sources in $SRC:
+
+1. Set up the testsuite.
+
+     CTF=${SRC}/contrib/test_framework
+     mkdir ${SRC}/gcc/testsuite/gcc.test_framework
+     cd ${SRC}/gcc/testsuite/gcc.test_framework
+     cp ${CTF}/test-framework.exp .
+     ${CTF}/generate_framework_tests
+
+2. Export the environment variable CHECK_TEST_FRAMEWORK.  Define and
+   export whatever environment variables are needed for running the
+   testsuite, and run the tests:
+
+     cd <build_directory>
+     make -k check-gcc RUNTESTFLAGS="test-framework.exp"
+
+3. Check that the results are as expected:
+
+     awk -f ${CTF}/test-framework.awk gcc/testsuite/gcc.sum
+
+   The awk script prints unexpected results followed by the number of
+   tests that passed.
--- /dev/null	2004-06-24 11:06:20.000000000 -0700
+++ test_framework/test-framework.exp	2004-11-02 13:34:35.000000000 -0800
@@ -0,0 +1,38 @@
+# Copyright (c) 2002, 2003, 2004 Free Software Foundation, Inc.
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# For a copy of the GNU General Public License, write the the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# Based on gcc/testsuite/gcc.dg/dg.exp.
+
+# Don't run these tests unless an environment variable is defined.
+if { ![info exists env(CHECK_TEST_FRAMEWORK)] } {
+    return
+}
+
+load_lib gcc-dg.exp
+
+proc dg-require-true { args } {
+    verbose "dg-require-true" 2
+}
+
+proc dg-require-false { args } {
+    verbose "dg-require-false" 2
+    upvar dg-do-what dg-do-what
+    skip_test_and_clear_xfail
+}
+
+dg-init
+dg-runtest [lsort [find $srcdir/$subdir *.c]] "" ""
+dg-finish
--- /dev/null	2004-06-24 11:06:20.000000000 -0700
+++ test_framework/test-framework.awk	2004-11-02 13:34:28.000000000 -0800
@@ -0,0 +1,40 @@
+# Process the gcc.sum file for a run through gcc.test-framework.
+# Print result lines that show potential problems.  Report the number
+# of passing tests.
+#
+#
+# Copyright (c) 2004 Free Software Foundation, Inc.
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# For a copy of the GNU General Public License, write the the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+BEGIN			{ skip = 1; pass = 0 }
+/Running.*test-frame/	{ skip = 0; next }
+/gcc Summary/		{ skip = 1; next }
+			{ if (skip) next }
+/^$/			{ next }
+# The post tests are always expected to pass.
+/^PASS.*-2.c/		{ next }
+# dg-xfail-if applies to the compile step, these hould be XFAIL for the
+# compile step on dox tests, which are run tests.
+/^XPASS.*dox.*xiff.*-1.c.*(test for excess errors)/ { next }
+# The other dox tests pass the compile step; ignore that message.
+/^PASS.*dox.*(test for excess errors)/ { next }
+/^PASS/			{ if (match ($0, "exp-P")) { pass++; next } }
+/^FAIL/			{ if (match ($0, "exp-F")) { pass++; next } }
+/^XPASS/		{ if (match ($0, "exp-XP")) { pass++; next } }
+/^XFAIL/		{ if (match ($0, "exp-XF")) { pass++; next } }
+/^UNSUPPORTED/		{ if (match ($0, "exp-U")) { pass++; next } }
+			{ print }
+END			{ printf("%d tests passed\n", pass) }
--- /dev/null	2004-06-24 11:06:20.000000000 -0700
+++ test_framework/generate_framework_tests	2004-11-02 13:31:29.000000000 -0800
@@ -0,0 +1,304 @@
+#! /bin/sh
+
+########################################################################
+#
+# File:    generate_framework_tests
+# Author:  Janis Johnson
+# Date:    2004/11/02
+#
+# Generate tests of GCC's test framework.  Each test has encoded in
+# its name the dg commands that are used in the test and the expected
+# result of the test, *-1.c.  Each test is followed by a test ending
+# in *-2.c that is expected to pass.
+#
+#
+# Copyright (c) 2004 Free Software Foundation, Inc.
+#
+# This file is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# For a copy of the GNU General Public License, write the the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+#
+########################################################################
+
+GOOD_TARGET='*-*-*'
+BAD_TARGET='empty-empty-empty'
+
+# Programs used in the tests: good compile and run, bad compile, and
+# bad run.
+GOOD_PROG="int main () { return 0; }"
+BADC_PROG="int missing_body () }"
+BADR_PROG="extern void abort (void); int main () { abort (); }"
+
+# dg-do commands with a target list.
+DOT_S="{ dg-do compile { target \"${GOOD_TARGET}\" } }"
+DOT_N="{ dg-do compile { target \"${BAD_TARGET}\" } }"
+
+# dg-do commands with an xfail list; the xfail list is ignored for
+# everything but "run".
+DOX_P="{ dg-do run { xfail \"${BAD_TARGET}\" } }"
+DOX_F="{ dg-do run { xfail \"${GOOD_TARGET}\" } }"
+
+# dg-xfail-if commands.
+XIF_P="{ dg-xfail-if \"target not matched\" { \"${BAD_TARGET}\" } { \"*\" } { \"\" } }"
+XIF_Q="{ dg-xfail-if \"options not matched\" { \"${GOOD_TARGET}\" } { \"-bad-option\" } { \"\" } }"
+XIF_F="{ dg-xfail-if \"target matched\" { \"${GOOD_TARGET}\" } { \"*\" } { \"\" } }"
+
+# dg-require-* commands, using procedures defined for this set of tests.
+REQ_S="{ dg-require-true \"\" }"    # do not skip the test
+REQ_N="{ dg-require-false \"\" }"   # skip the test
+
+# Expected result of the *-1.c test, encoded into the file name.
+EXP_PASS="exp-P"
+EXP_XPASS="exp-XP"
+EXP_SKIP="exp-U"
+EXP_FAIL="exp-F"
+EXP_XFAIL="exp-XF"
+
+cmd() {
+    KIND=$1
+    FILE=$2
+
+    case $KIND in
+      "dots") echo '/*' "${DOT_S}" '*/' >> $FILE;;
+      "dotn") echo '/*' "${DOT_N}" '*/' >> $FILE;;
+      "doxp") echo '/*' "${DOX_P}" '*/' >> $FILE;;
+      "doxf") echo '/*' "${DOX_F}" '*/' >> $FILE;;
+      "xifp") echo '/*' "${XIF_P}" '*/' >> $FILE;;
+      "xifq") echo '/*' "${XIF_Q}" '*/' >> $FILE;;
+      "xiff") echo '/*' "${XIF_F}" '*/' >> $FILE;;
+      "reqs") echo '/*' "${REQ_S}" '*/' >> $FILE;;
+      "reqn") echo '/*' "${REQ_N}" '*/' >> $FILE;;
+      *) echo "oops: $KIND"; exit 1;;
+    esac
+}
+
+# Generate a test using a single dg- command.  If the test isn't skipped,
+# generate a second version that will fail.
+one() {
+    KIND=$1
+    EXP="$2"
+
+    NAME=${KIND}-${EXP}
+    NAME1=${NAME}-1.c
+    NAME2=${NAME}-2.c
+    rm -f $NAME1
+    touch $NAME1
+    cmd $KIND $NAME1
+    echo "${GOOD_PROG}" >> $NAME1
+    echo "${GOOD_PROG}" > $NAME2
+
+    if [ "${EXP}" != "${EXP_SKIP}" ]; then
+	if [ "${EXP}" == "${EXP_PASS}" ]; then
+	    NAME=${KIND}-${EXP_FAIL}
+	else
+	    NAME=${KIND}-${EXP_XFAIL}
+	fi
+
+	NAME1=${NAME}-1.c
+	NAME2=${NAME}-2.c
+	rm -f $NAME1
+	touch $NAME1
+	cmd $KIND $NAME1
+	if [ "${KIND}" == "doxp" -o "${KIND}" == "doxf" ]; then
+	    echo "${BADR_PROG}" >> $NAME1
+	else
+	    echo "${BADC_PROG}" >> $NAME1
+	fi
+	echo "${GOOD_PROG}" > $NAME2
+    fi
+}
+
+# Generate a test using two dg- commands.  If the test isn't skipped,
+# generate a second version that will fail.
+two() {
+    KIND1=$1
+    KIND2=$2
+    EXP="$3"
+
+    NAME=${KIND1}-${KIND2}-${EXP}
+    NAME1=${NAME}-1.c
+    NAME2=${NAME}-2.c
+
+    rm -f $NAME1
+    touch $NAME1
+    cmd $KIND1 $NAME1
+    cmd $KIND2 $NAME1
+    echo "${GOOD_PROG}" >> $NAME1
+    echo "${GOOD_PROG}" > $NAME2
+
+    if [ "${EXP}" != "${EXP_SKIP}" ]; then
+	if [ "${EXP}" == "${EXP_PASS}" ]; then
+	    NAME=${KIND1}-${KIND2}-${EXP_FAIL}
+	else
+	    NAME=${KIND1}-${KIND2}-${EXP_XFAIL}
+	fi
+
+	NAME1=${NAME}-1.c
+	NAME2=${NAME}-2.c
+	rm -f $NAME1
+	touch $NAME1
+	cmd $KIND1 $NAME1
+	cmd $KIND2 $NAME1
+	if [ "${KIND1}" == "doxp" -o "${KIND1}" == "doxf" \
+             -o "${KIND2}" == "doxp" -o "${KIND2}" == "doxf" ]; then
+	    echo "${BADR_PROG}" >> $NAME1
+	else
+	    echo "${BADC_PROG}" >> $NAME1
+	fi
+	echo "${GOOD_PROG}" > $NAME2
+    fi
+}
+
+# Generate a test using three dg- commands.  If the test isn't skipped,
+# generate a second version that will fail.
+three() {
+    KIND1=$1
+    KIND2=$2
+    KIND3=$3
+    EXP="$4"
+
+    NAME=${KIND1}-${KIND2}-${KIND3}-${EXP}
+    NAME1=${NAME}-1.c
+    NAME2=${NAME}-2.c
+    rm -f $NAME1
+    touch $NAME1
+    cmd $KIND1 $NAME1
+    cmd $KIND2 $NAME1
+    cmd $KIND3 $NAME1
+    echo "${GOOD_PROG}" >> $NAME1
+    echo "${GOOD_PROG}" > $NAME2
+
+    if [ "${EXP}" != "${EXP_SKIP}" ]; then
+	if [ "${EXP}" == "${EXP_PASS}" ]; then
+	    NAME=${KIND1}-${KIND2}-${KIND3}-${EXP_FAIL}
+	else
+	    NAME=${KIND1}-${KIND2}-${KIND3}-${EXP_XFAIL}
+	fi
+
+	NAME1=${NAME}-1.c
+	NAME2=${NAME}-2.c
+	rm -f $NAME1
+	touch $NAME1
+	cmd $KIND1 $NAME1
+	cmd $KIND2 $NAME1
+	cmd $KIND3 $NAME1
+	if [ "${KIND1}" == "doxp" -o "${KIND1}" == "doxf" \
+             -o "${KIND2}" == "doxp" -o "${KIND2}" == "doxf" \
+             -o "${KIND3}" == "doxp" -o "${KIND3}" == "doxf" ]; then
+	    echo "${BADR_PROG}" >> $NAME1
+	else
+	    echo "${BADC_PROG}" >> $NAME1
+	fi
+	echo "${GOOD_PROG}" > $NAME2
+    fi
+}
+
+# Generate tests.
+one dots $EXP_PASS
+one dotn $EXP_SKIP
+one doxp $EXP_PASS
+one doxf $EXP_XPASS
+one xifp $EXP_PASS
+one xifq $EXP_PASS
+one xiff $EXP_XPASS
+one reqs $EXP_PASS
+one reqn $EXP_SKIP
+
+two dots xifp $EXP_PASS
+two dots xifq $EXP_PASS
+two dots xiff $EXP_XPASS
+two dotn xifp $EXP_SKIP
+two dotn xifq $EXP_SKIP
+two dotn xiff $EXP_SKIP
+two dots reqs $EXP_PASS
+two dots reqn $EXP_SKIP
+two dotn reqs $EXP_SKIP
+two dotn reqn $EXP_SKIP
+
+two doxp xifp $EXP_PASS
+two doxp xifq $EXP_PASS
+two doxp xiff $EXP_PASS    # dg-xfail-if applies to compile, not run
+two doxf xifp $EXP_XPASS
+two doxf xifq $EXP_XPASS
+two doxf xiff $EXP_XPASS
+
+two doxp reqs $EXP_PASS
+two doxp reqn $EXP_SKIP
+two doxf reqs $EXP_XPASS
+two doxf reqn $EXP_SKIP
+
+two reqs xifp $EXP_PASS
+two reqs xifq $EXP_PASS
+two reqs xiff $EXP_XPASS
+two reqn xifp $EXP_SKIP
+two reqn xifq $EXP_SKIP
+two reqn xiff $EXP_SKIP
+
+two xifp reqs $EXP_PASS
+two xifq reqs $EXP_PASS
+two xiff reqs $EXP_XPASS
+two xifp reqn $EXP_SKIP
+two xifq reqn $EXP_SKIP
+two xiff reqn $EXP_SKIP
+
+three dots reqs xifp $EXP_PASS
+three dots reqs xifq $EXP_PASS
+three dots reqs xiff $EXP_XPASS
+three dots reqn xifp $EXP_SKIP
+three dots reqn xifq $EXP_SKIP
+three dots reqn xiff $EXP_SKIP
+three dotn reqs xifp $EXP_SKIP
+three dotn reqs xifq $EXP_SKIP
+three dotn reqs xiff $EXP_SKIP
+three dotn reqn xifp $EXP_SKIP
+three dotn reqn xifq $EXP_SKIP
+three dotn reqn xiff $EXP_SKIP
+
+three dots xifp reqs $EXP_PASS
+three dots xifq reqs $EXP_PASS
+three dots xiff reqs $EXP_XPASS
+three dots xifp reqn $EXP_SKIP
+three dots xifq reqn $EXP_SKIP
+three dots xiff reqn $EXP_SKIP
+three dotn xifp reqs $EXP_SKIP
+three dotn xifq reqs $EXP_SKIP
+three dotn xiff reqs $EXP_SKIP
+three dotn xifp reqn $EXP_SKIP
+three dotn xifq reqn $EXP_SKIP
+three dotn xiff reqn $EXP_SKIP
+
+three doxp reqs xifp $EXP_PASS
+three doxp reqs xifq $EXP_PASS
+three doxp reqs xiff $EXP_PASS   # dg-xfail-if applies to compile, not run
+three doxp reqn xifp $EXP_SKIP
+three doxp reqn xifq $EXP_SKIP
+three doxp reqn xiff $EXP_SKIP
+three doxf reqs xifp $EXP_XPASS
+three doxf reqs xifq $EXP_XPASS
+three doxf reqs xiff $EXP_XPASS
+three doxf reqn xifp $EXP_SKIP
+three doxf reqn xifq $EXP_SKIP
+three doxf reqn xiff $EXP_SKIP
+
+three doxp xifp reqs $EXP_PASS
+three doxp xifq reqs $EXP_PASS
+three doxp xiff reqs $EXP_PASS   # dg-xfail-if applies to compile, not run
+three doxp xifp reqn $EXP_SKIP
+three doxp xifq reqn $EXP_SKIP
+three doxp xiff reqn $EXP_SKIP
+three doxf xifp reqs $EXP_XPASS
+three doxf xifq reqs $EXP_XPASS
+three doxf xiff reqs $EXP_XPASS
+three doxf xifp reqn $EXP_SKIP
+three doxf xifq reqn $EXP_SKIP
+three doxf xiff reqn $EXP_SKIP


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]