Re: PATCH: Add capability to contrib/compare_tests to handle directories

[Quentin Neill <quentin dot neill dot gnu at gmail dot com>]

Ping?

On Wed, Sep 7, 2011 at 12:21 PM, Quentin Neill
<quentin.neill.gnu@gmail.com> wrote:
> Hi,
>
> Should not change behavior for comparing two files (expect for usage
> output), and also should be POSIX compliant.
>
> Tested on x86_64 tests logs and test directories, would be interested
> in help testing on other platforms.
>
> Ok to commit?
> --
> Quentin Neill
>
>
> From 4d4fa9d094745ace0b6e51faadb2f3ea40cb7c7f Mon Sep 17 00:00:00 2001
> From: Quentin Neill <quentin.neill@amd.com>
> Date: Wed, 7 Sep 2011 12:04:35 -0500
> Subject: [PATCH] Add capability to compare test log directories.
>
> ---
> ?contrib/ChangeLog ? ? | ? ?4 ++
> ?contrib/compare_tests | ?107 ++++++++++++++++++++++++++++++++++++++++++++-----
> ?2 files changed, 101 insertions(+), 10 deletions(-)
>
> diff --git a/contrib/ChangeLog b/contrib/ChangeLog
> index 07adb58..e2007e7 100644
> --- a/contrib/ChangeLog
> +++ b/contrib/ChangeLog
> @@ -1,3 +1,7 @@
> +2011-09-07 ?Quentin Neill ?<quentin.neill@amd.com>
> +
> + ? ? ? * compare_tests: Add capability to compare test log directories.
> +
> ?2011-08-25 ?Rainer Orth ?<ro@CeBiTec.Uni-Bielefeld.DE>
>
> ? ? ? ?* gcc_update: Determine svn branch from hg convert_revision.
> diff --git a/contrib/compare_tests b/contrib/compare_tests
> index bed9742..b0e3321 100755
> --- a/contrib/compare_tests
> +++ b/contrib/compare_tests
> @@ -2,13 +2,36 @@
> ?# This script automatically test the given tool with the tool's test cases,
> ?# reporting anything of interest.
>
> -# exits with 0 if there is nothing of interest
> -# exits with 1 if there is something interesting
> -# exits with 2 if an error occurred
> -
> -# Give two .sum files to compare them
> +usage()
> +{
> + ? ? ? if [ -n "$1" ] ; then
> + ? ? ? ? ? ? ? echo "$0: Error: $1" >&2
> + ? ? ? ? ? ? ? echo >&2
> + ? ? ? fi
> + ? ? ? cat >&2 <<EOUSAGE
> +Usage: $0 [-strict] PREVIOUS CURRENT
> +
> + ? ? ? Compare the PREVIOUS and CURRENT test case logs, reporting anything
> of interest.
> +
> + ? ? ? If PREVIOUS and CURRENT are directories, find and compare any *.log files
> + ? ? ? therein (except config.log files).
> +
> + ? ? ? Unless -strict is given, these discrepancies are not counted as errors:
> + ? ? ? ? ? ? ? missing/extra log files when comparing directories
> + ? ? ? ? ? ? ? tests that failed in PREVIOUS but pass in CURRENT
> + ? ? ? ? ? ? ? tests that were not in PREVIOUS but appear in CURRENT
> + ? ? ? ? ? ? ? tests in PREVIOUS that are missing in CURRENT
> +
> + ? ? ? Exit with the following values:
> + ? ? ? ? ? ? ? 0 if there is nothing of interest
> + ? ? ? ? ? ? ? 1 if there are errors when comparing single test case files
> + ? ? ? ? ? ? ? N for the number of errors found when comparing directories
> +EOUSAGE
> + ? ? ? exit 2
> +}
>
> ?# Written by Mike Stump <mrs@cygnus.com>
> +# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
>
> ?tool=gxx
>
> @@ -16,10 +39,72 @@ tmp1=/tmp/$tool-testing.$$a
> ?tmp2=/tmp/$tool-testing.$$b
> ?now_s=/tmp/$tool-testing.$$d
> ?before_s=/tmp/$tool-testing.$$e
> +lst1=/tmp/$tool-lst1.$$
> +lst2=/tmp/$tool-lst2.$$
> +lst3=/tmp/$tool-lst3.$$
> +lst4=/tmp/$tool-lst4.$$
> +lst5=/tmp/$tool-lst5.$$
> +tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5"
> +
> +[ "$1" = "-strict" ] && strict=$1 && shift
> +[ "$1" = "-?" ] && usage
> +[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
> +
> +trap "rm -f $tmps" 0 1 2 3 5 9 13 15
> +exit_status=0
>
> -if [ "$2" = "" ]; then
> - ? ? ? echo "Usage: $0 previous current" >&2
> - ? ? ? exit 2
> +if [ -d "$1" -a -d "$2" ] ; then
> + ? ? ? find "$1" \( ! -name config.log \) -name '*.log' >$lst1
> + ? ? ? find "$2" \( ! -name config.log \) -name '*.log' >$lst2
> + ? ? ? echo "# Comparing directories"
> + ? ? ? echo "## Dir1=$1: `cat $lst1 | wc -l` log files"
> + ? ? ? echo "## Dir2=$2: `cat $lst2 | wc -l` log files"
> + ? ? ? echo
> + ? ? ? # remove leading directory components to compare
> + ? ? ? sed -e "s|^$1/||" $lst1 | sort >$lst3
> + ? ? ? sed -e "s|^$2/||" $lst2 | sort >$lst4
> + ? ? ? comm -23 $lst3 $lst4 >$lst5
> + ? ? ? if [ -s $lst5 ] ; then
> + ? ? ? ? ? ? ? echo "# Extra log files in Dir1=$1"
> + ? ? ? ? ? ? ? sed -e "s|^|< $1/|" $lst5
> + ? ? ? ? ? ? ? echo
> + ? ? ? ? ? ? ? [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
> + ? ? ? fi
> + ? ? ? comm -13 $lst3 $lst4 >$lst5
> + ? ? ? if [ -s $lst5 ] ; then
> + ? ? ? ? ? ? ? echo "# Extra log files in Dir2=$2"
> + ? ? ? ? ? ? ? sed -e "s|^|> $2/|" $lst5
> + ? ? ? ? ? ? ? echo
> + ? ? ? ? ? ? ? [ -n "$strict" ] && exit_status=`expr $exit_status + 1`
> + ? ? ? fi
> + ? ? ? comm -12 $lst3 $lst4 | sort -u >$lst5
> + ? ? ? if [ ! -s $lst5 ] ; then
> + ? ? ? ? ? ? ? echo "# No common log files"
> + ? ? ? ? ? ? ? exit_status=`expr $exit_status + 1`
> + ? ? ? ? ? ? ? exit $exit_status
> + ? ? ? fi
> + ? ? ? cmnlogs=`cat $lst5 | wc -l`
> + ? ? ? echo "# Comparing $cmnlogs common log files"
> + ? ? ? for fname in `cat $lst5`
> + ? ? ? do
> + ? ? ? ? ? ? ? f1="$1/$fname"
> + ? ? ? ? ? ? ? f2="$2/$fname"
> + ? ? ? ? ? ? ? echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $f1 $f2"
> + ? ? ? ? ? ? ? ${CONFIG_SHELL-/bin/sh} $0 $strict $f1 $f2
> + ? ? ? ? ? ? ? ret=$?
> + ? ? ? ? ? ? ? if [ $ret -ne 0 ]; then
> + ? ? ? ? ? ? ? ? ? ? ? exit_status=`expr $exit_status + 1`
> + ? ? ? ? ? ? ? ? ? ? ? echo "## Differences found: $fname"
> + ? ? ? ? ? ? ? fi
> + ? ? ? done
> + ? ? ? if [ $exit_status -ne 0 ]; then
> + ? ? ? ? ? ? ? echo "# $exit_status differences in $cmnlogs common log files found"
> + ? ? ? else
> + ? ? ? ? ? ? ? echo "# No differences found in $cmnlogs common log files"
> + ? ? ? fi
> + ? ? ? exit $exit_status
> +elif [ -d "$1" -o -d "$2" ] ; then
> + ? ? ? usage "Must specify either two directories or two files"
> ?fi
>
> ?sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^Running target /
> {target = $3} { if (target != "unix") { sub(/: /, "&"target": " ); };
> print $0; }' >$tmp1
> @@ -28,8 +113,6 @@ sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$2" | awk
> '/^Running target / {target =
> ?before=$tmp1
> ?now=$tmp2
>
> -exit_status=0
> -trap "rm -f $tmp1 $tmp2 $now_s $before_s" 0 1 2 3 5 9 13 15
>
> ?if sort -k 2 </dev/null >/dev/null 2>&1; then
> ? skip1='-k 2'
> @@ -60,6 +143,7 @@ if [ $? = 0 ]; then
> ? ? ? ?echo "Tests that now work, but didn't before:"
> ? ? ? ?echo
> ? ? ? ?cat $tmp2
> + ? ? ? [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
> ? ? ? ?echo
> ?fi
>
> @@ -83,6 +167,7 @@ if [ $? = 0 ]; then
> ? ? ? ?echo "New tests that PASS:"
> ? ? ? ?echo
> ? ? ? ?cat $tmp2
> + ? ? ? [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
> ? ? ? ?echo
> ?fi
>
> @@ -94,6 +179,7 @@ if [ $? = 0 ]; then
> ? ? ? ?echo "Old tests that passed, that have disappeared: (Eeek!)"
> ? ? ? ?echo
> ? ? ? ?cat $tmp2
> + ? ? ? [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
> ? ? ? ?echo
> ?fi
>
> @@ -105,6 +191,7 @@ if [ $? = 0 ]; then
> ? ? ? ?echo "Old tests that failed, that have disappeared: (Eeek!)"
> ? ? ? ?echo
> ? ? ? ?cat $tmp2
> + ? ? ? [ -n "$strict" ] && echo "Strict test fails" && exit_status=1
> ? ? ? ?echo
> ?fi
>
> --
> 1.7.1
>