[contrib] Compare against a clean build in validate_failures.py

[Diego Novillo <dnovillo at google dot com>]

Lawrence, this is the change I was referring to yesterday.  See
if it helps in comparing your results against clean builds.


Add a new option --clean_build to validate_failures.py

This is useful when you have two builds of the same compiler.  One with
your changes.  The other one, a clean build at the same revision.
Instead of using a manifest file, --clean_build will compare the
results it gather from the patched build against those it gathers from
the clean build.

Usage

$ cd /top/of/patched/gcc/bld
$ validate_failures.py --clean_build=clean/bld-gcc
Source directory: /usr/local/google/home/dnovillo/gcc/trunk
Build target:     x86_64-unknown-linux-gnu
Getting actual results from build directory .
        ./x86_64-unknown-linux-gnu/libstdc++-v3/testsuite/libstdc++.sum
        ./x86_64-unknown-linux-gnu/libffi/testsuite/libffi.sum
        ./x86_64-unknown-linux-gnu/libgomp/testsuite/libgomp.sum
        ./x86_64-unknown-linux-gnu/libgo/libgo.sum
        ./x86_64-unknown-linux-gnu/boehm-gc/testsuite/boehm-gc.sum
        ./x86_64-unknown-linux-gnu/libatomic/testsuite/libatomic.sum
        ./x86_64-unknown-linux-gnu/libmudflap/testsuite/libmudflap.sum
        ./x86_64-unknown-linux-gnu/libitm/testsuite/libitm.sum
        ./x86_64-unknown-linux-gnu/libjava/testsuite/libjava.sum
        ./gcc/testsuite/g++/g++.sum
        ./gcc/testsuite/gnat/gnat.sum
        ./gcc/testsuite/ada/acats/acats.sum
        ./gcc/testsuite/gcc/gcc.sum
        ./gcc/testsuite/gfortran/gfortran.sum
        ./gcc/testsuite/obj-c++/obj-c++.sum
        ./gcc/testsuite/go/go.sum
        ./gcc/testsuite/objc/objc.sum
Getting actual results from build directory clean/bld-gcc
        clean/bld-gcc/x86_64-unknown-linux-gnu/libstdc++-v3/testsuite/libstdc++.sum
        clean/bld-gcc/x86_64-unknown-linux-gnu/libffi/testsuite/libffi.sum
        clean/bld-gcc/x86_64-unknown-linux-gnu/libgomp/testsuite/libgomp.sum
        clean/bld-gcc/x86_64-unknown-linux-gnu/libgo/libgo.sum
        clean/bld-gcc/x86_64-unknown-linux-gnu/boehm-gc/testsuite/boehm-gc.sum
        clean/bld-gcc/x86_64-unknown-linux-gnu/libatomic/testsuite/libatomic.sum
        clean/bld-gcc/x86_64-unknown-linux-gnu/libmudflap/testsuite/libmudflap.sum
        clean/bld-gcc/x86_64-unknown-linux-gnu/libitm/testsuite/libitm.sum
        clean/bld-gcc/x86_64-unknown-linux-gnu/libjava/testsuite/libjava.sum
        clean/bld-gcc/gcc/testsuite/g++/g++.sum
        clean/bld-gcc/gcc/testsuite/gnat/gnat.sum
        clean/bld-gcc/gcc/testsuite/ada/acats/acats.sum
        clean/bld-gcc/gcc/testsuite/gcc/gcc.sum
        clean/bld-gcc/gcc/testsuite/gfortran/gfortran.sum
        clean/bld-gcc/gcc/testsuite/obj-c++/obj-c++.sum
        clean/bld-gcc/gcc/testsuite/go/go.sum
        clean/bld-gcc/gcc/testsuite/objc/objc.sum

SUCCESS: No unexpected failures.

2012-11-02  Diego Novillo  <dnovillo@google.com>

	* testsuite-management/validate_failures.py: Add option
	--clean_build to compare test results against another
	build.

diff --git a/contrib/testsuite-management/validate_failures.py b/contrib/testsuite-management/validate_failures.py
index be13cfd..7391937 100755
--- a/contrib/testsuite-management/validate_failures.py
+++ b/contrib/testsuite-management/validate_failures.py
@@ -292,7 +292,7 @@ def PrintSummary(msg, summary):
 
 def GetSumFiles(results, build_dir):
   if not results:
-    print 'Getting actual results from build'
+    print 'Getting actual results from build directory %s' % build_dir
     sum_files = CollectSumFiles(build_dir)
   else:
     print 'Getting actual results from user-provided results'
@@ -300,6 +300,27 @@ def GetSumFiles(results, build_dir):
   return sum_files
 
 
+def PerformComparison(expected, actual, ignore_missing_failures):
+  actual_vs_expected, expected_vs_actual = CompareResults(expected, actual)
+
+  tests_ok = True
+  if len(actual_vs_expected) > 0:
+    PrintSummary('Unexpected results in this build (new failures)',
+                 actual_vs_expected)
+    tests_ok = False
+
+  if not ignore_missing_failures and len(expected_vs_actual) > 0:
+    PrintSummary('Expected results not present in this build (fixed tests)'
+                 '\n\nNOTE: This is not a failure.  It just means that these '
+                 'tests were expected\nto fail, but they worked in this '
+                 'configuration.\n', expected_vs_actual)
+
+  if tests_ok:
+    print '\nSUCCESS: No unexpected failures.'
+
+  return tests_ok
+
+
 def CheckExpectedResults(options):
   if not options.manifest:
     (srcdir, target, valid_build) = GetBuildData(options)
@@ -320,24 +341,7 @@ def CheckExpectedResults(options):
     PrintSummary('Tests expected to fail', manifest)
     PrintSummary('\nActual test results', actual)
 
-  actual_vs_manifest, manifest_vs_actual = CompareResults(manifest, actual)
-
-  tests_ok = True
-  if len(actual_vs_manifest) > 0:
-    PrintSummary('Build results not in the manifest', actual_vs_manifest)
-    tests_ok = False
-
-  if not options.ignore_missing_failures and len(manifest_vs_actual) > 0:
-    PrintSummary('Manifest results not present in the build'
-                 '\n\nNOTE: This is not a failure.  It just means that the '
-                 'manifest expected\nthese tests to fail, '
-                 'but they worked in this configuration.\n',
-                 manifest_vs_actual)
-
-  if tests_ok:
-    print '\nSUCCESS: No unexpected failures.'
-
-  return tests_ok
+  return PerformComparison(manifest, actual, options.ignore_missing_failures)
 
 
 def ProduceManifest(options):
@@ -361,6 +365,20 @@ def ProduceManifest(options):
   return True
 
 
+def CompareBuilds(options):
+  (srcdir, target, valid_build) = GetBuildData(options)
+  if not valid_build:
+    return False
+
+  sum_files = GetSumFiles(options.results, options.build_dir)
+  actual = GetResults(sum_files)
+
+  clean_sum_files = GetSumFiles(None, options.clean_build)
+  clean = GetResults(clean_sum_files)
+
+  return PerformComparison(clean, actual, options.ignore_missing_failures)
+
+
 def Main(argv):
   parser = optparse.OptionParser(usage=__doc__)
 
@@ -368,6 +386,14 @@ def Main(argv):
   parser.add_option('--build_dir', action='store', type='string',
                     dest='build_dir', default='.',
                     help='Build directory to check (default = .)')
+  parser.add_option('--clean_build', action='store', type='string',
+                    dest='clean_build', default=None,
+                    help='Compare test results from this build against '
+                    'those of another (clean) build.  Use this option '
+                    'when comparing the test results of your patch versus '
+                    'the test results of a clean build without your patch. '
+                    'You must provide the path to the top directory of your '
+                    'clean build.')
   parser.add_option('--force', action='store_true', dest='force',
                     default=False, help='When used with --produce_manifest, '
                     'it will overwrite an existing manifest file '
@@ -400,6 +426,8 @@ def Main(argv):
 
   if options.produce_manifest:
     retval = ProduceManifest(options)
+  elif options.clean_build:
+    retval = CompareBuilds(options)
   else:
     retval = CheckExpectedResults(options)