[gcc.git] / contrib / compare_tests

#!/bin/sh
# This script automatically test the given tool with the tool's test cases,
# reporting anything of interest.

# Written by Mike Stump <mrs@cygnus.com>
# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>

usage()
{
	if [ -n "$1" ] ; then
		echo "$0: Error: $1" >&2
		echo >&2
	fi
	cat >&2 <<EOUSAGE
Usage: $0 [-strict] PREVIOUS CURRENT

Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.

	If PREVIOUS and CURRENT are directories, find and compare any *.sum files.

	Unless -strict is given, these discrepancies are not counted as errors:
		missing/extra .sum files when comparing directories
		tests that failed in PREVIOUS but pass in CURRENT
		tests that were not in PREVIOUS but appear in CURRENT
		tests in PREVIOUS that are missing in CURRENT

	Exit with the following values:
		0 if there is nothing of interest
		1 if there are errors when comparing single test case files
		N for the number of errors found when comparing directories
EOUSAGE
	exit 2
}

export LC_ALL=C

tool=gxx

tmp1=/tmp/$tool-testing.$$a
tmp2=/tmp/$tool-testing.$$b
now_s=/tmp/$tool-testing.$$d
before_s=/tmp/$tool-testing.$$e
lst1=/tmp/$tool-lst1.$$
lst2=/tmp/$tool-lst2.$$
lst3=/tmp/$tool-lst3.$$
lst4=/tmp/$tool-lst4.$$
lst5=/tmp/$tool-lst5.$$
sum1=/tmp/$tool-sum1.$$
sum2=/tmp/$tool-sum2.$$
tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"

[ "$1" = "-strict" ] && strict=$1 && shift
[ "$1" = "-?" ] && usage
[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"

trap "rm -f $tmps" 0 1 2 3 5 9 13 15
exit_status=0

if [ -d "$1" -a -d "$2" ] ; then
	find "$1/" -name '*.sum' >$lst1
	find "$2/" -name '*.sum' >$lst2
	echo "# Comparing directories"
	echo "## Dir1=$1: `cat $lst1 | wc -l` sum files"
	echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
	echo
	# remove leading directory components to compare
	sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
	sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
	comm -23 $lst3 $lst4 >$lst5
	if [ -s $lst5 ] ; then
		echo "# Extra sum files in Dir1=$1"
		sed -e "s|^|< $1/|" $lst5
		echo
		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
	fi
	comm -13 $lst3 $lst4 >$lst5
	if [ -s $lst5 ] ; then
		echo "# Extra sum files in Dir2=$2"
		sed -e "s|^|> $2/|" $lst5
		echo
		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
	fi
	comm -12 $lst3 $lst4 | sort -u >$lst5
	if [ ! -s $lst5 ] ; then
		echo "# No common sum files"
		exit_status=`expr $exit_status + 1`
		exit $exit_status
	fi
	cmnsums=`cat $lst5 | wc -l`
	echo "# Comparing $cmnsums common sum files"
	( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
	( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
	echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
	${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
	ret=$?
	if [ $ret -ne 0 ]; then
		exit_status=`expr $exit_status + 1`
		echo "## Differences found: $fname"
	fi
	if [ $exit_status -ne 0 ]; then
		echo "# $exit_status differences in $cmnsums common sum files found"
	else
		echo "# No differences found in $cmnsums common sum files"
	fi
	exit $exit_status
elif [ -d "$1" -o -d "$2" ] ; then
	usage "Must specify either two directories or two files"
fi

sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp1
sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp2

before=$tmp1
now=$tmp2


if sort -k 2 </dev/null >/dev/null 2>&1; then
  skip1='-k 2'
else
  skip1='+1'
fi

sort -t ':' $skip1 "$now" > "$now_s"
sort -t ':' $skip1 "$before" > "$before_s"

grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
grep '^PASS' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -12 $tmp1 - >$tmp2

grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
	echo "Tests that now fail, but worked before:"
	echo
	cat $tmp2
	echo
	exit_status=1
fi

grep '^PASS' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -12 $tmp1 - >$tmp2

grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
	echo "Tests that now work, but didn't before:"
	echo
	cat $tmp2
	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
	echo
fi

grep '^FAIL' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -23 $tmp1 - >$tmp2

grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
	echo "New tests that FAIL:"
	echo
	cat $tmp2
	echo
	exit_status=1
fi

grep '^PASS' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -23 $tmp1 - >$tmp2

grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
	echo "New tests that PASS:"
	echo
	cat $tmp2
	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
	echo
fi

grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
grep '^PASS' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -13 $tmp1 - >$tmp2

grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
	echo "Old tests that passed, that have disappeared: (Eeek!)"
	echo
	cat $tmp2
	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
	echo
fi

grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -13 $tmp1 - >$tmp2

grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
	echo "Old tests that failed, that have disappeared: (Eeek!)"
	echo
	cat $tmp2
	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
	echo
fi

exit $exit_status
Commit	Line	Data
	1	#!/bin/sh
	2	# This script automatically test the given tool with the tool's test cases,
	3	# reporting anything of interest.
	4
	5	# Written by Mike Stump <mrs@cygnus.com>
	6	# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
	7
	8	usage()
	9	{
	10	if [ -n "$1" ] ; then
	11	echo "$0: Error: $1" >&2
	12	echo >&2
	13	fi
	14	cat >&2 <<EOUSAGE
	15	Usage: $0 [-strict] PREVIOUS CURRENT
	16
	17	Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
	18
	19	If PREVIOUS and CURRENT are directories, find and compare any *.sum files.
	20
	21	Unless -strict is given, these discrepancies are not counted as errors:
	22	missing/extra .sum files when comparing directories
	23	tests that failed in PREVIOUS but pass in CURRENT
	24	tests that were not in PREVIOUS but appear in CURRENT
	25	tests in PREVIOUS that are missing in CURRENT
	26
	27	Exit with the following values:
	28	0 if there is nothing of interest
	29	1 if there are errors when comparing single test case files
	30	N for the number of errors found when comparing directories
	31	EOUSAGE
	32	exit 2
	33	}
	34
	35	export LC_ALL=C
	36
	37	tool=gxx
	38
	39	tmp1=/tmp/$tool-testing.$$a
	40	tmp2=/tmp/$tool-testing.$$b
	41	now_s=/tmp/$tool-testing.$$d
	42	before_s=/tmp/$tool-testing.$$e
	43	lst1=/tmp/$tool-lst1.$$
	44	lst2=/tmp/$tool-lst2.$$
	45	lst3=/tmp/$tool-lst3.$$
	46	lst4=/tmp/$tool-lst4.$$
	47	lst5=/tmp/$tool-lst5.$$
	48	sum1=/tmp/$tool-sum1.$$
	49	sum2=/tmp/$tool-sum2.$$
	50	tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
	51
	52	[ "$1" = "-strict" ] && strict=$1 && shift
	53	[ "$1" = "-?" ] && usage
	54	[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
	55
	56	trap "rm -f $tmps" 0 1 2 3 5 9 13 15
	57	exit_status=0
	58
	59	if [ -d "$1" -a -d "$2" ] ; then
	60	find "$1/" -name '*.sum' >$lst1
	61	find "$2/" -name '*.sum' >$lst2
	62	echo "# Comparing directories"
	63	echo "## Dir1=$1: `cat $lst1 \| wc -l` sum files"
	64	echo "## Dir2=$2: `cat $lst2 \| wc -l` sum files"
	65	echo
	66	# remove leading directory components to compare
	67	sed -e "s\|^$1[/]*\|\|" $lst1 \| sort >$lst3
	68	sed -e "s\|^$2[/]*\|\|" $lst2 \| sort >$lst4
	69	comm -23 $lst3 $lst4 >$lst5
	70	if [ -s $lst5 ] ; then
	71	echo "# Extra sum files in Dir1=$1"
	72	sed -e "s\|^\|< $1/\|" $lst5
	73	echo
	74	[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
	75	fi
	76	comm -13 $lst3 $lst4 >$lst5
	77	if [ -s $lst5 ] ; then
	78	echo "# Extra sum files in Dir2=$2"
	79	sed -e "s\|^\|> $2/\|" $lst5
	80	echo
	81	[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
	82	fi
	83	comm -12 $lst3 $lst4 \| sort -u >$lst5
	84	if [ ! -s $lst5 ] ; then
	85	echo "# No common sum files"
	86	exit_status=`expr $exit_status + 1`
	87	exit $exit_status
	88	fi
	89	cmnsums=`cat $lst5 \| wc -l`
	90	echo "# Comparing $cmnsums common sum files"
	91	( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
	92	( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
	93	echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
	94	${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
	95	ret=$?
	96	if [ $ret -ne 0 ]; then
	97	exit_status=`expr $exit_status + 1`
	98	echo "## Differences found: $fname"
	99	fi
	100	if [ $exit_status -ne 0 ]; then
	101	echo "# $exit_status differences in $cmnsums common sum files found"
	102	else
	103	echo "# No differences found in $cmnsums common sum files"
	104	fi
	105	exit $exit_status
	106	elif [ -d "$1" -o -d "$2" ] ; then
	107	usage "Must specify either two directories or two files"
	108	fi
	109
	110	sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$1" \| awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' \| cut -c1-2000 >$tmp1
	111	sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$2" \| awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' \| cut -c1-2000 >$tmp2
	112
	113	before=$tmp1
	114	now=$tmp2
	115
	116
	117	if sort -k 2 </dev/null >/dev/null 2>&1; then
	118	skip1='-k 2'
	119	else
	120	skip1='+1'
	121	fi
	122
	123	sort -t ':' $skip1 "$now" > "$now_s"
	124	sort -t ':' $skip1 "$before" > "$before_s"
	125
	126	grep '^FAIL:' "$now_s" \| sed 's/^[^:]*:[ ]//' >$tmp1
	127	grep '^PASS' "$before_s" \| sed 's/^[^:]*:[ ]//' \| comm -12 $tmp1 - >$tmp2
	128
	129	grep -s . $tmp2 >/dev/null
	130	if [ $? = 0 ]; then
	131	echo "Tests that now fail, but worked before:"
	132	echo
	133	cat $tmp2
	134	echo
	135	exit_status=1
	136	fi
	137
	138	grep '^PASS' "$now_s" \| sed 's/^[^:]*:[ ]//' >$tmp1
	139	grep '^FAIL' "$before_s" \| sed 's/^[^:]*:[ ]//' \| comm -12 $tmp1 - >$tmp2
	140
	141	grep -s . $tmp2 >/dev/null
	142	if [ $? = 0 ]; then
	143	echo "Tests that now work, but didn't before:"
	144	echo
	145	cat $tmp2
	146	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
	147	echo
	148	fi
	149
	150	grep '^FAIL' "$now_s" \| sed 's/^[^:]*:[ ]//' >$tmp1
	151	grep '^[PF]A[SI][SL]' "$before_s" \| sed 's/^[^:]*:[ ]//' \| comm -23 $tmp1 - >$tmp2
	152
	153	grep -s . $tmp2 >/dev/null
	154	if [ $? = 0 ]; then
	155	echo "New tests that FAIL:"
	156	echo
	157	cat $tmp2
	158	echo
	159	exit_status=1
	160	fi
	161
	162	grep '^PASS' "$now_s" \| sed 's/^[^:]*:[ ]//' >$tmp1
	163	grep '^[PF]A[SI][SL]' "$before_s" \| sed 's/^[^:]*:[ ]//' \| comm -23 $tmp1 - >$tmp2
	164
	165	grep -s . $tmp2 >/dev/null
	166	if [ $? = 0 ]; then
	167	echo "New tests that PASS:"
	168	echo
	169	cat $tmp2
	170	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
	171	echo
	172	fi
	173
	174	grep '^[PF]A[SI][SL]' "$now_s" \| sed 's/^[^:]*:[ ]//' >$tmp1
	175	grep '^PASS' "$before_s" \| sed 's/^[^:]*:[ ]//' \| comm -13 $tmp1 - >$tmp2
	176
	177	grep -s . $tmp2 >/dev/null
	178	if [ $? = 0 ]; then
	179	echo "Old tests that passed, that have disappeared: (Eeek!)"
	180	echo
	181	cat $tmp2
	182	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
	183	echo
	184	fi
	185
	186	grep '^[PF]A[SI][SL]' "$now_s" \| sed 's/^[^:]*:[ ]//' >$tmp1
	187	grep '^FAIL' "$before_s" \| sed 's/^[^:]*:[ ]//' \| comm -13 $tmp1 - >$tmp2
	188
	189	grep -s . $tmp2 >/dev/null
	190	if [ $? = 0 ]; then
	191	echo "Old tests that failed, that have disappeared: (Eeek!)"
	192	echo
	193	cat $tmp2
	194	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
	195	echo
	196	fi
	197
	198	exit $exit_status