]>
Commit | Line | Data |
---|---|---|
31e9c6ed MS |
1 | #!/bin/sh |
2 | # This script automatically test the given tool with the tool's test cases, | |
3 | # reporting anything of interest. | |
4 | ||
cf7716a2 QN |
5 | usage() |
6 | { | |
7 | if [ -n "$1" ] ; then | |
8 | echo "$0: Error: $1" >&2 | |
9 | echo >&2 | |
10 | fi | |
11 | cat >&2 <<EOUSAGE | |
12 | Usage: $0 [-strict] PREVIOUS CURRENT | |
13 | ||
14 | Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest. | |
15 | ||
16 | If PREVIOUS and CURRENT are directories, find and compare any *.sum files. | |
17 | ||
18 | Unless -strict is given, these discrepancies are not counted as errors: | |
19 | missing/extra .sum files when comparing directories | |
20 | tests that failed in PREVIOUS but pass in CURRENT | |
21 | tests that were not in PREVIOUS but appear in CURRENT | |
22 | tests in PREVIOUS that are missing in CURRENT | |
23 | ||
24 | Exit with the following values: | |
25 | 0 if there is nothing of interest | |
26 | 1 if there are errors when comparing single test case files | |
27 | N for the number of errors found when comparing directories | |
28 | EOUSAGE | |
29 | exit 2 | |
30 | } | |
31e9c6ed MS |
31 | |
32 | # Written by Mike Stump <mrs@cygnus.com> | |
cf7716a2 | 33 | # Subdir comparison added by Quentin Neill <quentin.neill@amd.com> |
31e9c6ed | 34 | |
112a0bfd MS |
35 | tool=gxx |
36 | ||
31e9c6ed MS |
37 | tmp1=/tmp/$tool-testing.$$a |
38 | tmp2=/tmp/$tool-testing.$$b | |
39 | now_s=/tmp/$tool-testing.$$d | |
40 | before_s=/tmp/$tool-testing.$$e | |
cf7716a2 QN |
41 | lst1=/tmp/$tool-lst1.$$ |
42 | lst2=/tmp/$tool-lst2.$$ | |
43 | lst3=/tmp/$tool-lst3.$$ | |
44 | lst4=/tmp/$tool-lst4.$$ | |
45 | lst5=/tmp/$tool-lst5.$$ | |
46 | tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5" | |
47 | ||
48 | [ "$1" = "-strict" ] && strict=$1 && shift | |
49 | [ "$1" = "-?" ] && usage | |
50 | [ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT" | |
51 | ||
52 | trap "rm -f $tmps" 0 1 2 3 5 9 13 15 | |
53 | exit_status=0 | |
31e9c6ed | 54 | |
cf7716a2 QN |
55 | if [ -d "$1" -a -d "$2" ] ; then |
56 | find "$1" -name '*.sum' >$lst1 | |
57 | find "$2" -name '*.sum' >$lst2 | |
58 | echo "# Comparing directories" | |
59 | echo "## Dir1=$1: `cat $lst1 | wc -l` sum files" | |
60 | echo "## Dir2=$2: `cat $lst2 | wc -l` sum files" | |
61 | echo | |
62 | # remove leading directory components to compare | |
63 | sed -e "s|^$1/||" $lst1 | sort >$lst3 | |
64 | sed -e "s|^$2/||" $lst2 | sort >$lst4 | |
65 | comm -23 $lst3 $lst4 >$lst5 | |
66 | if [ -s $lst5 ] ; then | |
67 | echo "# Extra sum files in Dir1=$1" | |
68 | sed -e "s|^|< $1/|" $lst5 | |
69 | echo | |
70 | [ -n "$strict" ] && exit_status=`expr $exit_status + 1` | |
71 | fi | |
72 | comm -13 $lst3 $lst4 >$lst5 | |
73 | if [ -s $lst5 ] ; then | |
74 | echo "# Extra sum files in Dir2=$2" | |
75 | sed -e "s|^|> $2/|" $lst5 | |
76 | echo | |
77 | [ -n "$strict" ] && exit_status=`expr $exit_status + 1` | |
78 | fi | |
79 | comm -12 $lst3 $lst4 | sort -u >$lst5 | |
80 | if [ ! -s $lst5 ] ; then | |
81 | echo "# No common sum files" | |
82 | exit_status=`expr $exit_status + 1` | |
83 | exit $exit_status | |
84 | fi | |
85 | cmnsums=`cat $lst5 | wc -l` | |
44414d2f MS |
86 | sum1="/tmp/$tool-sum-1" |
87 | sum2="/tmp/$tool-sum-2" | |
cf7716a2 QN |
88 | echo "# Comparing $cmnsums common sum files" |
89 | ( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1 | |
90 | ( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2 | |
91 | echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2" | |
92 | ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2 | |
44414d2f | 93 | rm -f $sum1 $sum2 |
cf7716a2 QN |
94 | ret=$? |
95 | if [ $ret -ne 0 ]; then | |
96 | exit_status=`expr $exit_status + 1` | |
97 | echo "## Differences found: $fname" | |
98 | fi | |
99 | if [ $exit_status -ne 0 ]; then | |
100 | echo "# $exit_status differences in $cmnsums common sum files found" | |
101 | else | |
102 | echo "# No differences found in $cmnsums common sum files" | |
103 | fi | |
104 | exit $exit_status | |
105 | elif [ -d "$1" -o -d "$2" ] ; then | |
106 | usage "Must specify either two directories or two files" | |
31e9c6ed MS |
107 | fi |
108 | ||
e58b87f7 RE |
109 | sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' >$tmp1 |
110 | sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' >$tmp2 | |
31e9c6ed MS |
111 | |
112 | before=$tmp1 | |
113 | now=$tmp2 | |
114 | ||
31e9c6ed | 115 | |
27b56ba6 PE |
116 | if sort -k 2 </dev/null >/dev/null 2>&1; then |
117 | skip1='-k 2' | |
118 | else | |
119 | skip1='+1' | |
120 | fi | |
121 | ||
122 | sort -t ':' $skip1 "$now" > "$now_s" | |
123 | sort -t ':' $skip1 "$before" > "$before_s" | |
31e9c6ed | 124 | |
112a0bfd MS |
125 | grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
126 | grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2 | |
31e9c6ed MS |
127 | |
128 | grep -s . $tmp2 >/dev/null | |
129 | if [ $? = 0 ]; then | |
130 | echo "Tests that now fail, but worked before:" | |
131 | echo | |
132 | cat $tmp2 | |
133 | echo | |
112a0bfd | 134 | exit_status=1 |
31e9c6ed MS |
135 | fi |
136 | ||
112a0bfd MS |
137 | grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
138 | grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2 | |
31e9c6ed MS |
139 | |
140 | grep -s . $tmp2 >/dev/null | |
141 | if [ $? = 0 ]; then | |
142 | echo "Tests that now work, but didn't before:" | |
143 | echo | |
144 | cat $tmp2 | |
cf7716a2 | 145 | [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 |
31e9c6ed MS |
146 | echo |
147 | fi | |
148 | ||
112a0bfd MS |
149 | grep '^FAIL' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
150 | grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2 | |
31e9c6ed MS |
151 | |
152 | grep -s . $tmp2 >/dev/null | |
153 | if [ $? = 0 ]; then | |
154 | echo "New tests that FAIL:" | |
155 | echo | |
156 | cat $tmp2 | |
157 | echo | |
112a0bfd | 158 | exit_status=1 |
31e9c6ed MS |
159 | fi |
160 | ||
112a0bfd MS |
161 | grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
162 | grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2 | |
31e9c6ed MS |
163 | |
164 | grep -s . $tmp2 >/dev/null | |
165 | if [ $? = 0 ]; then | |
166 | echo "New tests that PASS:" | |
167 | echo | |
168 | cat $tmp2 | |
cf7716a2 | 169 | [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 |
31e9c6ed MS |
170 | echo |
171 | fi | |
172 | ||
112a0bfd MS |
173 | grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
174 | grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2 | |
31e9c6ed MS |
175 | |
176 | grep -s . $tmp2 >/dev/null | |
177 | if [ $? = 0 ]; then | |
178 | echo "Old tests that passed, that have disappeared: (Eeek!)" | |
179 | echo | |
180 | cat $tmp2 | |
cf7716a2 | 181 | [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 |
31e9c6ed MS |
182 | echo |
183 | fi | |
184 | ||
112a0bfd MS |
185 | grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
186 | grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2 | |
31e9c6ed MS |
187 | |
188 | grep -s . $tmp2 >/dev/null | |
189 | if [ $? = 0 ]; then | |
190 | echo "Old tests that failed, that have disappeared: (Eeek!)" | |
191 | echo | |
192 | cat $tmp2 | |
cf7716a2 | 193 | [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 |
31e9c6ed MS |
194 | echo |
195 | fi | |
112a0bfd MS |
196 | |
197 | exit $exit_status |