]>
Commit | Line | Data |
---|---|---|
31e9c6ed MS |
1 | #!/bin/sh |
2 | # This script automatically test the given tool with the tool's test cases, | |
3 | # reporting anything of interest. | |
4 | ||
cf7716a2 QN |
5 | usage() |
6 | { | |
7 | if [ -n "$1" ] ; then | |
8 | echo "$0: Error: $1" >&2 | |
9 | echo >&2 | |
10 | fi | |
11 | cat >&2 <<EOUSAGE | |
12 | Usage: $0 [-strict] PREVIOUS CURRENT | |
13 | ||
14 | Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest. | |
15 | ||
16 | If PREVIOUS and CURRENT are directories, find and compare any *.sum files. | |
17 | ||
18 | Unless -strict is given, these discrepancies are not counted as errors: | |
19 | missing/extra .sum files when comparing directories | |
20 | tests that failed in PREVIOUS but pass in CURRENT | |
21 | tests that were not in PREVIOUS but appear in CURRENT | |
22 | tests in PREVIOUS that are missing in CURRENT | |
23 | ||
24 | Exit with the following values: | |
25 | 0 if there is nothing of interest | |
26 | 1 if there are errors when comparing single test case files | |
27 | N for the number of errors found when comparing directories | |
28 | EOUSAGE | |
29 | exit 2 | |
30 | } | |
31e9c6ed MS |
31 | |
32 | # Written by Mike Stump <mrs@cygnus.com> | |
cf7716a2 | 33 | # Subdir comparison added by Quentin Neill <quentin.neill@amd.com> |
31e9c6ed | 34 | |
112a0bfd MS |
35 | tool=gxx |
36 | ||
31e9c6ed MS |
37 | tmp1=/tmp/$tool-testing.$$a |
38 | tmp2=/tmp/$tool-testing.$$b | |
39 | now_s=/tmp/$tool-testing.$$d | |
40 | before_s=/tmp/$tool-testing.$$e | |
cf7716a2 QN |
41 | lst1=/tmp/$tool-lst1.$$ |
42 | lst2=/tmp/$tool-lst2.$$ | |
43 | lst3=/tmp/$tool-lst3.$$ | |
44 | lst4=/tmp/$tool-lst4.$$ | |
45 | lst5=/tmp/$tool-lst5.$$ | |
e53182b7 QN |
46 | sum1=/tmp/$tool-sum1.$$ |
47 | sum2=/tmp/$tool-sum2.$$ | |
48 | tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2" | |
cf7716a2 QN |
49 | |
50 | [ "$1" = "-strict" ] && strict=$1 && shift | |
51 | [ "$1" = "-?" ] && usage | |
52 | [ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT" | |
53 | ||
54 | trap "rm -f $tmps" 0 1 2 3 5 9 13 15 | |
55 | exit_status=0 | |
31e9c6ed | 56 | |
cf7716a2 QN |
57 | if [ -d "$1" -a -d "$2" ] ; then |
58 | find "$1" -name '*.sum' >$lst1 | |
59 | find "$2" -name '*.sum' >$lst2 | |
60 | echo "# Comparing directories" | |
61 | echo "## Dir1=$1: `cat $lst1 | wc -l` sum files" | |
62 | echo "## Dir2=$2: `cat $lst2 | wc -l` sum files" | |
63 | echo | |
64 | # remove leading directory components to compare | |
e53182b7 QN |
65 | sed -e "s|^$1[/]*||" $lst1 | sort >$lst3 |
66 | sed -e "s|^$2[/]*||" $lst2 | sort >$lst4 | |
cf7716a2 QN |
67 | comm -23 $lst3 $lst4 >$lst5 |
68 | if [ -s $lst5 ] ; then | |
69 | echo "# Extra sum files in Dir1=$1" | |
70 | sed -e "s|^|< $1/|" $lst5 | |
71 | echo | |
72 | [ -n "$strict" ] && exit_status=`expr $exit_status + 1` | |
73 | fi | |
74 | comm -13 $lst3 $lst4 >$lst5 | |
75 | if [ -s $lst5 ] ; then | |
76 | echo "# Extra sum files in Dir2=$2" | |
77 | sed -e "s|^|> $2/|" $lst5 | |
78 | echo | |
79 | [ -n "$strict" ] && exit_status=`expr $exit_status + 1` | |
80 | fi | |
81 | comm -12 $lst3 $lst4 | sort -u >$lst5 | |
82 | if [ ! -s $lst5 ] ; then | |
83 | echo "# No common sum files" | |
84 | exit_status=`expr $exit_status + 1` | |
85 | exit $exit_status | |
86 | fi | |
87 | cmnsums=`cat $lst5 | wc -l` | |
88 | echo "# Comparing $cmnsums common sum files" | |
89 | ( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1 | |
90 | ( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2 | |
91 | echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2" | |
92 | ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2 | |
93 | ret=$? | |
94 | if [ $ret -ne 0 ]; then | |
95 | exit_status=`expr $exit_status + 1` | |
96 | echo "## Differences found: $fname" | |
97 | fi | |
98 | if [ $exit_status -ne 0 ]; then | |
99 | echo "# $exit_status differences in $cmnsums common sum files found" | |
100 | else | |
101 | echo "# No differences found in $cmnsums common sum files" | |
102 | fi | |
103 | exit $exit_status | |
104 | elif [ -d "$1" -o -d "$2" ] ; then | |
105 | usage "Must specify either two directories or two files" | |
31e9c6ed MS |
106 | fi |
107 | ||
e58b87f7 RE |
108 | sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' >$tmp1 |
109 | sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' >$tmp2 | |
31e9c6ed MS |
110 | |
111 | before=$tmp1 | |
112 | now=$tmp2 | |
113 | ||
31e9c6ed | 114 | |
27b56ba6 PE |
115 | if sort -k 2 </dev/null >/dev/null 2>&1; then |
116 | skip1='-k 2' | |
117 | else | |
118 | skip1='+1' | |
119 | fi | |
120 | ||
121 | sort -t ':' $skip1 "$now" > "$now_s" | |
122 | sort -t ':' $skip1 "$before" > "$before_s" | |
31e9c6ed | 123 | |
112a0bfd MS |
124 | grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
125 | grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2 | |
31e9c6ed MS |
126 | |
127 | grep -s . $tmp2 >/dev/null | |
128 | if [ $? = 0 ]; then | |
129 | echo "Tests that now fail, but worked before:" | |
130 | echo | |
131 | cat $tmp2 | |
132 | echo | |
112a0bfd | 133 | exit_status=1 |
31e9c6ed MS |
134 | fi |
135 | ||
112a0bfd MS |
136 | grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
137 | grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2 | |
31e9c6ed MS |
138 | |
139 | grep -s . $tmp2 >/dev/null | |
140 | if [ $? = 0 ]; then | |
141 | echo "Tests that now work, but didn't before:" | |
142 | echo | |
143 | cat $tmp2 | |
cf7716a2 | 144 | [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 |
31e9c6ed MS |
145 | echo |
146 | fi | |
147 | ||
112a0bfd MS |
148 | grep '^FAIL' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
149 | grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2 | |
31e9c6ed MS |
150 | |
151 | grep -s . $tmp2 >/dev/null | |
152 | if [ $? = 0 ]; then | |
153 | echo "New tests that FAIL:" | |
154 | echo | |
155 | cat $tmp2 | |
156 | echo | |
112a0bfd | 157 | exit_status=1 |
31e9c6ed MS |
158 | fi |
159 | ||
112a0bfd MS |
160 | grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
161 | grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2 | |
31e9c6ed MS |
162 | |
163 | grep -s . $tmp2 >/dev/null | |
164 | if [ $? = 0 ]; then | |
165 | echo "New tests that PASS:" | |
166 | echo | |
167 | cat $tmp2 | |
cf7716a2 | 168 | [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 |
31e9c6ed MS |
169 | echo |
170 | fi | |
171 | ||
112a0bfd MS |
172 | grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
173 | grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2 | |
31e9c6ed MS |
174 | |
175 | grep -s . $tmp2 >/dev/null | |
176 | if [ $? = 0 ]; then | |
177 | echo "Old tests that passed, that have disappeared: (Eeek!)" | |
178 | echo | |
179 | cat $tmp2 | |
cf7716a2 | 180 | [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 |
31e9c6ed MS |
181 | echo |
182 | fi | |
183 | ||
112a0bfd MS |
184 | grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
185 | grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2 | |
31e9c6ed MS |
186 | |
187 | grep -s . $tmp2 >/dev/null | |
188 | if [ $? = 0 ]; then | |
189 | echo "Old tests that failed, that have disappeared: (Eeek!)" | |
190 | echo | |
191 | cat $tmp2 | |
cf7716a2 | 192 | [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 |
31e9c6ed MS |
193 | echo |
194 | fi | |
112a0bfd MS |
195 | |
196 | exit $exit_status |