]>
Commit | Line | Data |
---|---|---|
1 | #!/bin/sh | |
2 | # This script automatically test the given tool with the tool's test cases, | |
3 | # reporting anything of interest. | |
4 | ||
5 | # Written by Mike Stump <mrs@cygnus.com> | |
6 | # Subdir comparison added by Quentin Neill <quentin.neill@amd.com> | |
7 | ||
8 | usage() | |
9 | { | |
10 | if [ -n "$1" ] ; then | |
11 | echo "$0: Error: $1" >&2 | |
12 | echo >&2 | |
13 | fi | |
14 | cat >&2 <<EOUSAGE | |
15 | Usage: $0 [-strict] PREVIOUS CURRENT | |
16 | ||
17 | Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest. | |
18 | ||
19 | If PREVIOUS and CURRENT are directories, find and compare any *.sum files. | |
20 | ||
21 | Unless -strict is given, these discrepancies are not counted as errors: | |
22 | missing/extra .sum files when comparing directories | |
23 | tests that failed in PREVIOUS but pass in CURRENT | |
24 | tests that were not in PREVIOUS but appear in CURRENT | |
25 | tests in PREVIOUS that are missing in CURRENT | |
26 | ||
27 | Exit with the following values: | |
28 | 0 if there is nothing of interest | |
29 | 1 if there are errors when comparing single test case files | |
30 | N for the number of errors found when comparing directories | |
31 | EOUSAGE | |
32 | exit 2 | |
33 | } | |
34 | ||
35 | export LC_ALL=C | |
36 | ||
37 | tool=gxx | |
38 | ||
39 | tmp1=/tmp/$tool-testing.$$a | |
40 | tmp2=/tmp/$tool-testing.$$b | |
41 | now_s=/tmp/$tool-testing.$$d | |
42 | before_s=/tmp/$tool-testing.$$e | |
43 | lst1=/tmp/$tool-lst1.$$ | |
44 | lst2=/tmp/$tool-lst2.$$ | |
45 | lst3=/tmp/$tool-lst3.$$ | |
46 | lst4=/tmp/$tool-lst4.$$ | |
47 | lst5=/tmp/$tool-lst5.$$ | |
48 | sum1=/tmp/$tool-sum1.$$ | |
49 | sum2=/tmp/$tool-sum2.$$ | |
50 | tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2" | |
51 | ||
52 | [ "$1" = "-strict" ] && strict=$1 && shift | |
53 | [ "$1" = "-?" ] && usage | |
54 | [ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT" | |
55 | ||
56 | trap "rm -f $tmps" 0 1 2 3 5 9 13 15 | |
57 | exit_status=0 | |
58 | ||
59 | if [ -d "$1" -a -d "$2" ] ; then | |
60 | find "$1/" -name '*.sum' >$lst1 | |
61 | find "$2/" -name '*.sum' >$lst2 | |
62 | echo "# Comparing directories" | |
63 | echo "## Dir1=$1: `cat $lst1 | wc -l` sum files" | |
64 | echo "## Dir2=$2: `cat $lst2 | wc -l` sum files" | |
65 | echo | |
66 | # remove leading directory components to compare | |
67 | sed -e "s|^$1[/]*||" $lst1 | sort >$lst3 | |
68 | sed -e "s|^$2[/]*||" $lst2 | sort >$lst4 | |
69 | comm -23 $lst3 $lst4 >$lst5 | |
70 | if [ -s $lst5 ] ; then | |
71 | echo "# Extra sum files in Dir1=$1" | |
72 | sed -e "s|^|< $1/|" $lst5 | |
73 | echo | |
74 | [ -n "$strict" ] && exit_status=`expr $exit_status + 1` | |
75 | fi | |
76 | comm -13 $lst3 $lst4 >$lst5 | |
77 | if [ -s $lst5 ] ; then | |
78 | echo "# Extra sum files in Dir2=$2" | |
79 | sed -e "s|^|> $2/|" $lst5 | |
80 | echo | |
81 | [ -n "$strict" ] && exit_status=`expr $exit_status + 1` | |
82 | fi | |
83 | comm -12 $lst3 $lst4 | sort -u >$lst5 | |
84 | if [ ! -s $lst5 ] ; then | |
85 | echo "# No common sum files" | |
86 | exit_status=`expr $exit_status + 1` | |
87 | exit $exit_status | |
88 | fi | |
89 | cmnsums=`cat $lst5 | wc -l` | |
90 | echo "# Comparing $cmnsums common sum files" | |
91 | ( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1 | |
92 | ( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2 | |
93 | echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2" | |
94 | ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2 | |
95 | ret=$? | |
96 | if [ $ret -ne 0 ]; then | |
97 | exit_status=`expr $exit_status + 1` | |
98 | echo "## Differences found: $fname" | |
99 | fi | |
100 | if [ $exit_status -ne 0 ]; then | |
101 | echo "# $exit_status differences in $cmnsums common sum files found" | |
102 | else | |
103 | echo "# No differences found in $cmnsums common sum files" | |
104 | fi | |
105 | exit $exit_status | |
106 | elif [ -d "$1" -o -d "$2" ] ; then | |
107 | usage "Must specify either two directories or two files" | |
108 | fi | |
109 | ||
110 | sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp1 | |
111 | sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^Running target / {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp2 | |
112 | ||
113 | before=$tmp1 | |
114 | now=$tmp2 | |
115 | ||
116 | ||
117 | if sort -k 2 </dev/null >/dev/null 2>&1; then | |
118 | skip1='-k 2' | |
119 | else | |
120 | skip1='+1' | |
121 | fi | |
122 | ||
123 | sort -t ':' $skip1 "$now" > "$now_s" | |
124 | sort -t ':' $skip1 "$before" > "$before_s" | |
125 | ||
126 | grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 | |
127 | grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2 | |
128 | ||
129 | grep -s . $tmp2 >/dev/null | |
130 | if [ $? = 0 ]; then | |
131 | echo "Tests that now fail, but worked before:" | |
132 | echo | |
133 | cat $tmp2 | |
134 | echo | |
135 | exit_status=1 | |
136 | fi | |
137 | ||
138 | grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 | |
139 | grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -12 $tmp1 - >$tmp2 | |
140 | ||
141 | grep -s . $tmp2 >/dev/null | |
142 | if [ $? = 0 ]; then | |
143 | echo "Tests that now work, but didn't before:" | |
144 | echo | |
145 | cat $tmp2 | |
146 | [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 | |
147 | echo | |
148 | fi | |
149 | ||
150 | grep '^FAIL' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 | |
151 | grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2 | |
152 | ||
153 | grep -s . $tmp2 >/dev/null | |
154 | if [ $? = 0 ]; then | |
155 | echo "New tests that FAIL:" | |
156 | echo | |
157 | cat $tmp2 | |
158 | echo | |
159 | exit_status=1 | |
160 | fi | |
161 | ||
162 | grep '^PASS' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 | |
163 | grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -23 $tmp1 - >$tmp2 | |
164 | ||
165 | grep -s . $tmp2 >/dev/null | |
166 | if [ $? = 0 ]; then | |
167 | echo "New tests that PASS:" | |
168 | echo | |
169 | cat $tmp2 | |
170 | [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 | |
171 | echo | |
172 | fi | |
173 | ||
174 | grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 | |
175 | grep '^PASS' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2 | |
176 | ||
177 | grep -s . $tmp2 >/dev/null | |
178 | if [ $? = 0 ]; then | |
179 | echo "Old tests that passed, that have disappeared: (Eeek!)" | |
180 | echo | |
181 | cat $tmp2 | |
182 | [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 | |
183 | echo | |
184 | fi | |
185 | ||
186 | grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ ]//' >$tmp1 | |
187 | grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ ]//' | comm -13 $tmp1 - >$tmp2 | |
188 | ||
189 | grep -s . $tmp2 >/dev/null | |
190 | if [ $? = 0 ]; then | |
191 | echo "Old tests that failed, that have disappeared: (Eeek!)" | |
192 | echo | |
193 | cat $tmp2 | |
194 | [ -n "$strict" ] && echo "Strict test fails" && exit_status=1 | |
195 | echo | |
196 | fi | |
197 | ||
198 | exit $exit_status |