]>
Commit | Line | Data |
---|---|---|
31e9c6ed MS |
1 | #!/bin/sh |
2 | # This script automatically test the given tool with the tool's test cases, | |
3 | # reporting anything of interest. | |
4 | ||
aa7e6aad MS |
5 | # Written by Mike Stump <mrs@cygnus.com> |
6 | # Subdir comparison added by Quentin Neill <quentin.neill@amd.com> | |
7 | ||
cf7716a2 QN |
8 | usage() |
9 | { | |
10 | if [ -n "$1" ] ; then | |
11 | echo "$0: Error: $1" >&2 | |
12 | echo >&2 | |
13 | fi | |
14 | cat >&2 <<EOUSAGE | |
15 | Usage: $0 [-strict] PREVIOUS CURRENT | |
16 | ||
17 | Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest. | |
18 | ||
19 | If PREVIOUS and CURRENT are directories, find and compare any *.sum files. | |
20 | ||
21 | Unless -strict is given, these discrepancies are not counted as errors: | |
22 | missing/extra .sum files when comparing directories | |
23 | tests that failed in PREVIOUS but pass in CURRENT | |
24 | tests that were not in PREVIOUS but appear in CURRENT | |
25 | tests in PREVIOUS that are missing in CURRENT | |
26 | ||
27 | Exit with the following values: | |
28 | 0 if there is nothing of interest | |
29 | 1 if there are errors when comparing single test case files | |
30 | N for the number of errors found when comparing directories | |
31 | EOUSAGE | |
32 | exit 2 | |
33 | } | |
31e9c6ed | 34 | |
a59a2c21 MS |
35 | export LC_ALL=C |
36 | ||
112a0bfd MS |
37 | tool=gxx |
38 | ||
9f88a07b YG |
39 | TMPDIR=${TMPDIR:-/tmp} |
40 | tmp1=$TMPDIR/$tool-testing.$$a | |
41 | tmp2=$TMPDIR/$tool-testing.$$b | |
42 | now_s=$TMPDIR/$tool-testing.$$d | |
43 | before_s=$TMPDIR/$tool-testing.$$e | |
44 | lst1=$TMPDIR/$tool-lst1.$$ | |
45 | lst2=$TMPDIR/$tool-lst2.$$ | |
46 | lst3=$TMPDIR/$tool-lst3.$$ | |
47 | lst4=$TMPDIR/$tool-lst4.$$ | |
48 | lst5=$TMPDIR/$tool-lst5.$$ | |
49 | sum1=$TMPDIR/$tool-sum1.$$ | |
50 | sum2=$TMPDIR/$tool-sum2.$$ | |
e53182b7 | 51 | tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2" |
cf7716a2 QN |
52 | |
53 | [ "$1" = "-strict" ] && strict=$1 && shift | |
54 | [ "$1" = "-?" ] && usage | |
55 | [ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT" | |
56 | ||
57 | trap "rm -f $tmps" 0 1 2 3 5 9 13 15 | |
58 | exit_status=0 | |
31e9c6ed | 59 | |
cf7716a2 | 60 | if [ -d "$1" -a -d "$2" ] ; then |
af13465f MLI |
61 | find "$1/" -name '*.sum' >$lst1 |
62 | find "$2/" -name '*.sum' >$lst2 | |
cf7716a2 QN |
63 | echo "# Comparing directories" |
64 | echo "## Dir1=$1: `cat $lst1 | wc -l` sum files" | |
65 | echo "## Dir2=$2: `cat $lst2 | wc -l` sum files" | |
66 | echo | |
67 | # remove leading directory components to compare | |
e53182b7 QN |
68 | sed -e "s|^$1[/]*||" $lst1 | sort >$lst3 |
69 | sed -e "s|^$2[/]*||" $lst2 | sort >$lst4 | |
cf7716a2 QN |
70 | comm -23 $lst3 $lst4 >$lst5 |
71 | if [ -s $lst5 ] ; then | |
72 | echo "# Extra sum files in Dir1=$1" | |
73 | sed -e "s|^|< $1/|" $lst5 | |
74 | echo | |
75 | [ -n "$strict" ] && exit_status=`expr $exit_status + 1` | |
76 | fi | |
77 | comm -13 $lst3 $lst4 >$lst5 | |
78 | if [ -s $lst5 ] ; then | |
79 | echo "# Extra sum files in Dir2=$2" | |
80 | sed -e "s|^|> $2/|" $lst5 | |
81 | echo | |
82 | [ -n "$strict" ] && exit_status=`expr $exit_status + 1` | |
83 | fi | |
84 | comm -12 $lst3 $lst4 | sort -u >$lst5 | |
85 | if [ ! -s $lst5 ] ; then | |
86 | echo "# No common sum files" | |
87 | exit_status=`expr $exit_status + 1` | |
88 | exit $exit_status | |
89 | fi | |
90 | cmnsums=`cat $lst5 | wc -l` | |
91 | echo "# Comparing $cmnsums common sum files" | |
92 | ( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1 | |
93 | ( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2 | |
94 | echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2" | |
95 | ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2 | |
96 | ret=$? | |
97 | if [ $ret -ne 0 ]; then | |
98 | exit_status=`expr $exit_status + 1` | |
586e3258 | 99 | echo "## Differences found" |
cf7716a2 QN |
100 | fi |
101 | if [ $exit_status -ne 0 ]; then | |
102 | echo "# $exit_status differences in $cmnsums common sum files found" | |
103 | else | |
104 | echo "# No differences found in $cmnsums common sum files" | |
105 | fi | |
106 | exit $exit_status | |
107 | elif [ -d "$1" -o -d "$2" ] ; then | |
108 | usage "Must specify either two directories or two files" | |
31e9c6ed MS |
109 | fi |
110 | ||
586e3258 AO |
111 | sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^[ ]*=== [^ ]* tests ===$/ {tool = $2} /^Running target / {target = $3} { if (tool != "") { sub(/:[ ]/, "&"tool": " ); }; if (target != "unix") { sub(/:[ ]/, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp1 |
112 | sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^[ ]*=== [^ ]* tests ===$/ {tool = $2} /^Running target / {target = $3} { if (tool != "") { sub(/:[ ]/, "&"tool": " ); }; if (target != "unix") { sub(/:[ ]/, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp2 | |
31e9c6ed MS |
113 | |
114 | before=$tmp1 | |
115 | now=$tmp2 | |
116 | ||
31e9c6ed | 117 | |
27b56ba6 PE |
118 | if sort -k 2 </dev/null >/dev/null 2>&1; then |
119 | skip1='-k 2' | |
120 | else | |
121 | skip1='+1' | |
122 | fi | |
123 | ||
124 | sort -t ':' $skip1 "$now" > "$now_s" | |
125 | sort -t ':' $skip1 "$before" > "$before_s" | |
31e9c6ed | 126 | |
112a0bfd MS |
127 | grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
128 | grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2 | |
31e9c6ed MS |
129 | |
130 | grep -s . $tmp2 >/dev/null | |
131 | if [ $? = 0 ]; then | |
e57aae39 CL |
132 | num=`cat $tmp2 | wc -l` |
133 | echo "Tests that now fail, but worked before ($num tests):" | |
31e9c6ed MS |
134 | echo |
135 | cat $tmp2 | |
136 | echo | |
112a0bfd | 137 | exit_status=1 |
31e9c6ed MS |
138 | fi |
139 | ||
112a0bfd MS |
140 | grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
141 | grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2 | |
31e9c6ed MS |
142 | |
143 | grep -s . $tmp2 >/dev/null | |
144 | if [ $? = 0 ]; then | |
e57aae39 CL |
145 | num=`cat $tmp2 | wc -l` |
146 | echo "Tests that now work, but didn't before ($num tests):" | |
31e9c6ed MS |
147 | echo |
148 | cat $tmp2 | |
cf7716a2 | 149 | [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 |
31e9c6ed MS |
150 | echo |
151 | fi | |
152 | ||
112a0bfd MS |
153 | grep '^FAIL' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
154 | grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2 | |
31e9c6ed MS |
155 | |
156 | grep -s . $tmp2 >/dev/null | |
157 | if [ $? = 0 ]; then | |
e57aae39 CL |
158 | num=`cat $tmp2 | wc -l` |
159 | echo "New tests that FAIL ($num tests):" | |
31e9c6ed MS |
160 | echo |
161 | cat $tmp2 | |
162 | echo | |
112a0bfd | 163 | exit_status=1 |
31e9c6ed MS |
164 | fi |
165 | ||
112a0bfd MS |
166 | grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
167 | grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2 | |
31e9c6ed MS |
168 | |
169 | grep -s . $tmp2 >/dev/null | |
170 | if [ $? = 0 ]; then | |
e57aae39 CL |
171 | num=`cat $tmp2 | wc -l` |
172 | echo "New tests that PASS ($num tests):" | |
31e9c6ed MS |
173 | echo |
174 | cat $tmp2 | |
cf7716a2 | 175 | [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 |
31e9c6ed MS |
176 | echo |
177 | fi | |
178 | ||
112a0bfd MS |
179 | grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
180 | grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2 | |
31e9c6ed MS |
181 | |
182 | grep -s . $tmp2 >/dev/null | |
183 | if [ $? = 0 ]; then | |
e57aae39 CL |
184 | num=`cat $tmp2 | wc -l` |
185 | echo "Old tests that passed, that have disappeared ($num tests): (Eeek!)" | |
31e9c6ed MS |
186 | echo |
187 | cat $tmp2 | |
cf7716a2 | 188 | [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 |
31e9c6ed MS |
189 | echo |
190 | fi | |
191 | ||
112a0bfd MS |
192 | grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 |
193 | grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2 | |
31e9c6ed MS |
194 | |
195 | grep -s . $tmp2 >/dev/null | |
196 | if [ $? = 0 ]; then | |
e57aae39 CL |
197 | num=`cat $tmp2 | wc -l` |
198 | echo "Old tests that failed, that have disappeared ($num tests): (Eeek!)" | |
31e9c6ed MS |
199 | echo |
200 | cat $tmp2 | |
cf7716a2 | 201 | [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 |
31e9c6ed MS |
202 | echo |
203 | fi | |
112a0bfd MS |
204 | |
205 | exit $exit_status |