]>
gcc.gnu.org Git - gcc.git/blob - contrib/compare_tests
e09fc4f113a3f096a2d6a7d398062c227a925670
2 # This script automatically test the given tool with the tool's test cases,
3 # reporting anything of interest.
5 # Written by Mike Stump <mrs@cygnus.com>
6 # Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
11 echo "$0: Error: $1" >&2
15 Usage: $0 [-strict] PREVIOUS CURRENT
17 Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
19 If PREVIOUS and CURRENT are directories, find and compare any *.sum files.
21 Unless -strict is given, these discrepancies are not counted as errors:
22 missing/extra .sum files when comparing directories
23 tests that failed in PREVIOUS but pass in CURRENT
24 tests that were not in PREVIOUS but appear in CURRENT
25 tests in PREVIOUS that are missing in CURRENT
27 Exit with the following values:
28 0 if there is nothing of interest
29 1 if there are errors when comparing single test case files
30 N for the number of errors found when comparing directories
39 TMPDIR
=${TMPDIR:-/tmp}
40 tmp1
=$TMPDIR/$tool-testing.$
$a
41 tmp2
=$TMPDIR/$tool-testing.$
$b
42 now_s
=$TMPDIR/$tool-testing.$
$d
43 before_s
=$TMPDIR/$tool-testing.$
$e
44 lst1
=$TMPDIR/$tool-lst1.$$
45 lst2
=$TMPDIR/$tool-lst2.$$
46 lst3
=$TMPDIR/$tool-lst3.$$
47 lst4
=$TMPDIR/$tool-lst4.$$
48 lst5
=$TMPDIR/$tool-lst5.$$
49 sum1
=$TMPDIR/$tool-sum1.$$
50 sum2
=$TMPDIR/$tool-sum2.$$
51 tmps
="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
53 [ "$1" = "-strict" ] && strict
=$1 && shift
54 [ "$1" = "-?" ] && usage
55 [ "$2" = "" ] && usage
"Must specify both PREVIOUS and CURRENT"
57 trap "rm -f $tmps" 0 1 2 3 5 9 13 15
60 if [ -d "$1" -a -d "$2" ] ; then
61 find "$1/" -name '*.sum' >$lst1
62 find "$2/" -name '*.sum' >$lst2
63 echo "# Comparing directories"
64 echo "## Dir1=$1: `cat $lst1 | wc -l` sum files"
65 echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
67 # remove leading directory components to compare
68 sed -e "s|^$1[/]*||" $lst1 |
sort >$lst3
69 sed -e "s|^$2[/]*||" $lst2 |
sort >$lst4
70 comm -23 $lst3 $lst4 >$lst5
71 if [ -s $lst5 ] ; then
72 echo "# Extra sum files in Dir1=$1"
73 sed -e "s|^|< $1/|" $lst5
75 [ -n "$strict" ] && exit_status
=`expr $exit_status + 1`
77 comm -13 $lst3 $lst4 >$lst5
78 if [ -s $lst5 ] ; then
79 echo "# Extra sum files in Dir2=$2"
80 sed -e "s|^|> $2/|" $lst5
82 [ -n "$strict" ] && exit_status
=`expr $exit_status + 1`
84 comm -12 $lst3 $lst4 |
sort -u >$lst5
85 if [ ! -s $lst5 ] ; then
86 echo "# No common sum files"
87 exit_status
=`expr $exit_status + 1`
90 cmnsums
=`cat $lst5 | wc -l`
91 echo "# Comparing $cmnsums common sum files"
92 ( for fname
in `cat $lst5`; do cat $1/$fname; done ) >$sum1
93 ( for fname
in `cat $lst5`; do cat $2/$fname; done ) >$sum2
94 echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
95 ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
97 if [ $ret -ne 0 ]; then
98 exit_status
=`expr $exit_status + 1`
99 echo "## Differences found"
101 if [ $exit_status -ne 0 ]; then
102 echo "# $exit_status differences in $cmnsums common sum files found"
104 echo "# No differences found in $cmnsums common sum files"
107 elif [ -d "$1" -o -d "$2" ] ; then
108 usage
"Must specify either two directories or two files"
111 sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$1" |
awk '/^[ ]*=== [^ ]* tests ===$/ {tool = $2} /^Running target / {target = $3} { if (tool != "") { sub(/:[ ]/, "&"tool": " ); }; if (target != "unix") { sub(/:[ ]/, "&"target": " ); }; print $0; }' | cut
-c1-2000 >$tmp1
112 sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$2" |
awk '/^[ ]*=== [^ ]* tests ===$/ {tool = $2} /^Running target / {target = $3} { if (tool != "") { sub(/:[ ]/, "&"tool": " ); }; if (target != "unix") { sub(/:[ ]/, "&"target": " ); }; print $0; }' | cut
-c1-2000 >$tmp2
118 if sort -k 2 </dev
/null
>/dev
/null
2>&1; then
124 sort -t ':' $skip1 "$now" > "$now_s"
125 sort -t ':' $skip1 "$before" > "$before_s"
127 grep '^FAIL:' "$now_s" |
sed 's/^[^:]*:[ ]//' >$tmp1
128 grep '^PASS' "$before_s" |
sed 's/^[^:]*:[ ]//' |
comm -12 $tmp1 - >$tmp2
130 grep -s .
$tmp2 >/dev
/null
132 num
=`cat $tmp2 | wc -l`
133 echo "Tests that now fail, but worked before ($num tests):"
140 grep '^PASS' "$now_s" |
sed 's/^[^:]*:[ ]//' >$tmp1
141 grep '^FAIL' "$before_s" |
sed 's/^[^:]*:[ ]//' |
comm -12 $tmp1 - >$tmp2
143 grep -s .
$tmp2 >/dev
/null
145 num
=`cat $tmp2 | wc -l`
146 echo "Tests that now work, but didn't before ($num tests):"
149 [ -n "$strict" ] && echo "Strict test fails" && exit_status
=1
153 grep '^FAIL' "$now_s" |
sed 's/^[^:]*:[ ]//' >$tmp1
154 grep '^[PF]A[SI][SL]' "$before_s" |
sed 's/^[^:]*:[ ]//' |
comm -23 $tmp1 - >$tmp2
156 grep -s .
$tmp2 >/dev
/null
158 num
=`cat $tmp2 | wc -l`
159 echo "New tests that FAIL ($num tests):"
166 grep '^PASS' "$now_s" |
sed 's/^[^:]*:[ ]//' >$tmp1
167 grep '^[PF]A[SI][SL]' "$before_s" |
sed 's/^[^:]*:[ ]//' |
comm -23 $tmp1 - >$tmp2
169 grep -s .
$tmp2 >/dev
/null
171 num
=`cat $tmp2 | wc -l`
172 echo "New tests that PASS ($num tests):"
175 [ -n "$strict" ] && echo "Strict test fails" && exit_status
=1
179 grep '^[PF]A[SI][SL]' "$now_s" |
sed 's/^[^:]*:[ ]//' >$tmp1
180 grep '^PASS' "$before_s" |
sed 's/^[^:]*:[ ]//' |
comm -13 $tmp1 - >$tmp2
182 grep -s .
$tmp2 >/dev
/null
184 num
=`cat $tmp2 | wc -l`
185 echo "Old tests that passed, that have disappeared ($num tests): (Eeek!)"
188 [ -n "$strict" ] && echo "Strict test fails" && exit_status
=1
192 grep '^[PF]A[SI][SL]' "$now_s" |
sed 's/^[^:]*:[ ]//' >$tmp1
193 grep '^FAIL' "$before_s" |
sed 's/^[^:]*:[ ]//' |
comm -13 $tmp1 - >$tmp2
195 grep -s .
$tmp2 >/dev
/null
197 num
=`cat $tmp2 | wc -l`
198 echo "Old tests that failed, that have disappeared ($num tests): (Eeek!)"
201 [ -n "$strict" ] && echo "Strict test fails" && exit_status
=1
This page took 0.062954 seconds and 6 git commands to generate.