Re: [PATCH] gcc parallel make check

[Jakub Jelinek <jakub at redhat dot com>]

On Thu, Sep 11, 2014 at 07:26:37PM +0200, Jakub Jelinek wrote:
> right now.  The patch below intends to serialize the content of the
> problematic *.exp tests (the first runtest to reach one of those will simply
> run all the tests from that *.exp file, others will skip it).

Forgotten patch below.  BTW, something will probably need to be done about
acats too, either similar approach or just splitting the chapters into
little more jobs, because otherwise in make -C check -j48 acats dominated
the testing time for me.

--- gcc/Makefile.in.jj	2014-09-08 22:12:56.000000000 +0200
+++ gcc/Makefile.in	2014-09-11 16:58:01.076371437 +0200
@@ -513,34 +513,10 @@ xm_include_list=@xm_include_list@
 xm_defines=@xm_defines@
 lang_checks=
 lang_checks_parallelized=
-dg_target_exps:=aarch64.exp,alpha.exp,arm.exp,avr.exp,bfin.exp,cris.exp
-dg_target_exps:=$(dg_target_exps),epiphany.exp,frv.exp,i386.exp,ia64.exp
-dg_target_exps:=$(dg_target_exps),m68k.exp,microblaze.exp,mips.exp,powerpc.exp
-dg_target_exps:=$(dg_target_exps),rx.exp,s390.exp,sh.exp,sparc.exp,spu.exp
-dg_target_exps:=$(dg_target_exps),tic6x.exp,xstormy16.exp
-# This lists a couple of test files that take most time during check-gcc.
-# When doing parallelized check-gcc, these can run in parallel with the
-# remaining tests.  Each word in this variable stands for work for one
-# make goal and one extra make goal is added to handle all the *.exp
-# files not handled explicitly already.  If multiple *.exp files
-# should be run in the same runtest invocation (usually if they aren't
-# very long running, but still should be split of from the check-parallel-$lang
-# remaining tests runtest invocation), they should be concatenated with commas.
-# Note that [a-zA-Z] wildcards need to have []s prefixed with \ (needed
-# by tcl) and as the *.exp arguments are mached both as is and with
-# */ prefixed to it in runtest_file_p, it is usually desirable to include
-# a subdirectory name.
-check_gcc_parallelize=execute.exp=execute/2* \
-		      execute.exp=execute/\[013-9a-fA-F\]* \
-		      execute.exp=execute/\[pP\]*,dg.exp \
-		      execute.exp=execute/\[g-oq-zG-OQ-Z\]*,compile.exp=compile/2* \
-		      compile.exp=compile/\[9pP\]*,builtins.exp \
-		      compile.exp=compile/\[013-8a-oq-zA-OQ-Z\]* \
-		      dg-torture.exp,ieee.exp \
-		      vect.exp,unsorted.exp \
-		      guality.exp \
-		      struct-layout-1.exp,stackalign.exp \
-		      $(dg_target_exps)
+# Upper limit to which it is useful to parallelize this lang target.
+# It doesn't make sense to try e.g. 128 goals for small testsuites
+# like objc or go.
+check_gcc_parallelize=10000
 lang_opt_files=@lang_opt_files@ $(srcdir)/c-family/c.opt $(srcdir)/common.opt
 lang_specs_files=@lang_specs_files@
 lang_tree_files=@lang_tree_files@
@@ -3631,27 +3607,32 @@ $(filter-out $(lang_checks_parallelized)
 	    export TCL_LIBRARY ; fi ; \
 	$(RUNTEST) --tool $* $(RUNTESTFLAGS))
 
-$(patsubst %,%-subtargets,$(filter-out $(lang_checks_parallelized),$(lang_checks))): check-%-subtargets:
+$(patsubst %,%-subtargets,$(lang_checks)): check-%-subtargets:
 	@echo check-$*
 
 check_p_tool=$(firstword $(subst _, ,$*))
-check_p_vars=$(check_$(check_p_tool)_parallelize)
+check_p_count=$(check_$(check_p_tool)_parallelize)
 check_p_subno=$(word 2,$(subst _, ,$*))
-check_p_comma=,
-check_p_subwork=$(subst $(check_p_comma), ,$(if $(check_p_subno),$(word $(check_p_subno),$(check_p_vars))))
-check_p_numbers=1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
+check_p_numbers0:=1 2 3 4 5 6 7 8 9
+check_p_numbers1:=0 $(check_p_numbers0)
+check_p_numbers2:=$(foreach i,$(check_p_numbers0),$(patsubst %,$(i)%,$(check_p_numbers1)))
+check_p_numbers3:=$(patsubst %,0%,$(check_p_numbers1)) $(check_p_numbers2)
+check_p_numbers4:=$(foreach i,$(check_p_numbers0),$(patsubst %,$(i)%,$(check_p_numbers3)))
+check_p_numbers5:=$(patsubst %,0%,$(check_p_numbers3)) $(check_p_numbers4)
+check_p_numbers6:=$(foreach i,$(check_p_numbers0),$(patsubst %,$(i)%,$(check_p_numbers5)))
+check_p_numbers:=$(check_p_numbers0) $(check_p_numbers2) $(check_p_numbers4) $(check_p_numbers6)
 check_p_subdir=$(subst _,,$*)
-check_p_subdirs=$(wordlist 1,$(words $(check_$*_parallelize)),$(check_p_numbers))
+check_p_subdirs=$(wordlist 1,$(check_p_count),$(wordlist 1,$(or $(GCC_TEST_PARALLEL_SLOTS),128),$(check_p_numbers)))
 
 # For parallelized check-% targets, this decides whether parallelization
 # is desirable (if -jN is used and RUNTESTFLAGS doesn't contain anything
 # but optional --target_board or --extra_opts arguments).  If desirable,
 # recursive make is run with check-parallel-$lang{,1,2,3,4,5} etc. goals,
 # which can be executed in parallel, as they are run in separate directories.
-# check-parallel-$lang{1,2,3,4,5} etc. goals invoke runtest with the longest
-# running *.exp files from the testsuite, as determined by check_$lang_parallelize
-# variable.  The check-parallel-$lang goal in that case invokes runtest with
-# all the remaining *.exp files not handled by the separate goals.
+# check-parallel-$lang{,1,2,3,4,5} etc. goals invoke runtest with
+# GCC_RUNTEST_PARALLELIZE_DIR var in the environment and runtest_file_p
+# dejaGNU procedure is overridden to additionally synchronize through
+# a $lang-parallel directory which tests will be run by which runtest instance.
 # Afterwards contrib/dg-extract-results.sh is used to merge the sum and log
 # files.  If parallelization isn't desirable, only one recursive make
 # is run with check-parallel-$lang goal and check_$lang_parallelize variable
@@ -3662,76 +3643,61 @@ check_p_subdirs=$(wordlist 1,$(words $(c
 # to lang_checks_parallelized variable and define check_$lang_parallelize
 # variable (see above check_gcc_parallelize description).
 $(lang_checks_parallelized): check-% : site.exp
+	-rm -rf $(TESTSUITEDIR)/$*-parallel
 	@if [ -z "$(filter-out --target_board=%,$(filter-out --extra_opts%,$(RUNTESTFLAGS)))" ] \
 	    && [ "$(filter -j, $(MFLAGS))" = "-j" ]; then \
+	  test -d $(TESTSUITEDIR) || mkdir $(TESTSUITEDIR) || true; \
+	  test -d $(TESTSUITEDIR)/$*-parallel || mkdir $(TESTSUITEDIR)/$*-parallel || true; \
+	  GCC_RUNTEST_PARALLELIZE_DIR=`${PWD_COMMAND}`/$(TESTSUITEDIR)/$(check_p_tool)-parallel ; \
+	  export GCC_RUNTEST_PARALLELIZE_DIR ; \
 	  $(MAKE) TESTSUITEDIR="$(TESTSUITEDIR)" RUNTESTFLAGS="$(RUNTESTFLAGS)" \
 	    check-parallel-$* \
 	    $(patsubst %,check-parallel-$*_%, $(check_p_subdirs)); \
-	  for file in $(TESTSUITEDIR)/$*/$* \
-		      $(patsubst %,$(TESTSUITEDIR)/$*%/$*,$(check_p_subdirs));\
+	  sums= ; logs= ; \
+	  for dir in $(TESTSUITEDIR)/$* \
+		     $(patsubst %,$(TESTSUITEDIR)/$*%,$(check_p_subdirs));\
 	  do \
-	    mv -f $$file.sum $$file.sum.sep; mv -f $$file.log $$file.log.sep; \
+	    if [ -d $$dir ]; then \
+	      mv -f $$dir/$*.sum $$dir/$*.sum.sep; mv -f $$dir/$*.log $$dir/$*.log.sep; \
+	      sums="$$sums $$dir/$*.sum.sep"; logs="$$logs $$dir/$*.log.sep"; \
+	    fi; \
 	  done; \
-	  $(SHELL) $(srcdir)/../contrib/dg-extract-results.sh \
-	    $(TESTSUITEDIR)/$*/$*.sum.sep \
-	    $(patsubst %,$(TESTSUITEDIR)/$*%/$*.sum.sep,$(check_p_subdirs)) \
+	  $(SHELL) $(srcdir)/../contrib/dg-extract-results.sh $$sums \
 	    > $(TESTSUITEDIR)/$*/$*.sum; \
-	  $(SHELL) $(srcdir)/../contrib/dg-extract-results.sh -L \
-	    $(TESTSUITEDIR)/$*/$*.log.sep \
-	    $(patsubst %,$(TESTSUITEDIR)/$*%/$*.log.sep,$(check_p_subdirs)) \
+	  $(SHELL) $(srcdir)/../contrib/dg-extract-results.sh -L $$logs \
 	    > $(TESTSUITEDIR)/$*/$*.log; \
+	  rm -rf $(TESTSUITEDIR)/$*-parallel || true; \
 	else \
 	  $(MAKE) TESTSUITEDIR="$(TESTSUITEDIR)" RUNTESTFLAGS="$(RUNTESTFLAGS)" \
 	    check_$*_parallelize= check-parallel-$*; \
 	fi
 
-# Just print the parallelized subtargets for those that want to split
-# the testing across machines.
-$(patsubst %,%-subtargets,$(lang_checks_parallelized)): check-%-subtargets:
-	@echo check-parallel-$* \
-	  $(patsubst %,check-parallel-$*_%, $(check_p_subdirs))
-
-# In the if [ -n "$(check_p_subno)" ] case runtest should be given the name of
-# the given *.exp file(s).  See comment above check_gcc_parallelize variable
-# for details on the content of these variables.
-#
-# In the elif [ -n "$(check_p_vars)" ] case runtest should be given
-# names of all the *.exp files for this tool that aren't already handled by
-# other goals.  First it finds all the *.exp files for this tool, then
-# prunes those already specified in check_$lang_parallelize or duplicates.
-#
-# Otherwise check-$lang isn't parallelized and runtest is invoked just with
-# the $(RUNTESTFLAGS) arguments.
 check-parallel-% : site.exp
 	-test -d plugin || mkdir plugin
 	-test -d $(TESTSUITEDIR) || mkdir $(TESTSUITEDIR)
 	test -d $(TESTSUITEDIR)/$(check_p_subdir) || mkdir $(TESTSUITEDIR)/$(check_p_subdir)
 	-(rootme=`${PWD_COMMAND}`; export rootme; \
 	srcdir=`cd ${srcdir}; ${PWD_COMMAND}` ; export srcdir ; \
-	cd $(TESTSUITEDIR)/$(check_p_subdir); \
-	rm -f tmp-site.exp; \
-	sed '/set tmpdir/ s|testsuite$$|$(TESTSUITEDIR)/$(check_p_subdir)|' \
+	if [ -n "$(check_p_subno)" \
+	     -a -n "$$GCC_RUNTEST_PARALLELIZE_DIR" \
+	     -a -f $(TESTSUITEDIR)/$(check_p_tool)-parallel/finished ]; then \
+	  rm -rf $(TESTSUITEDIR)/$(check_p_subdir); \
+	else \
+	  cd $(TESTSUITEDIR)/$(check_p_subdir); \
+	  rm -f tmp-site.exp; \
+	  sed '/set tmpdir/ s|testsuite$$|$(TESTSUITEDIR)/$(check_p_subdir)|' \
 		< ../../site.exp > tmp-site.exp; \
-	$(SHELL) $${srcdir}/../move-if-change tmp-site.exp site.exp; \
-	EXPECT=${EXPECT} ; export EXPECT ; \
-	if [ -f $${rootme}/../expect/expect ] ; then  \
-	   TCL_LIBRARY=`cd .. ; cd $${srcdir}/../tcl/library ; ${PWD_COMMAND}` ; \
-	    export TCL_LIBRARY ; fi ; \
-	runtestflags= ; \
-	if [ -n "$(check_p_subno)" ] ; then \
-	  runtestflags="$(check_p_subwork)"; \
-	elif [ -n "$(check_p_vars)" ] ; then \
-	  parts="`echo ' $(strip $(subst $(check_p_comma), ,$(check_p_vars))) ' \
-		  | sed 's/=[^ ]* / /g'`"; \
-	  for part in `find $$srcdir/testsuite/$(check_p_tool)* -name \*.exp` ; do \
-	    part=`basename $$part` ; \
-	    case " $$parts $$runtestflags " in \
-	      *" $$part "*) ;; \
-	      *) runtestflags="$$runtestflags $$part" ;; \
-	    esac ; \
-	  done ; \
-	fi ; \
-	$(RUNTEST) --tool $(check_p_tool) $(RUNTESTFLAGS) $$runtestflags)
+	  $(SHELL) $${srcdir}/../move-if-change tmp-site.exp site.exp; \
+	  EXPECT=${EXPECT} ; export EXPECT ; \
+	  if [ -f $${rootme}/../expect/expect ] ; then  \
+	    TCL_LIBRARY=`cd .. ; cd $${srcdir}/../tcl/library ; ${PWD_COMMAND}` ; \
+	    export TCL_LIBRARY ; \
+	  fi ; \
+	  $(RUNTEST) --tool $(check_p_tool) $(RUNTESTFLAGS); \
+	  if [ -n "$$GCC_RUNTEST_PARALLELIZE_DIR" ] ; then \
+	    touch $${rootme}/$(TESTSUITEDIR)/$(check_p_tool)-parallel/finished; \
+	  fi ; \
+	fi )
 
 # QMTest targets
 
--- gcc/go/Make-lang.in.jj	2014-05-02 15:39:49.766921376 +0200
+++ gcc/go/Make-lang.in	2014-09-11 16:08:50.049547349 +0200
@@ -131,11 +131,7 @@ go.srcman: doc/gccgo.1
 
 lang_checks += check-go
 lang_checks_parallelized += check-go
-check_go_parallelize = go-test.exp=*/test/\[0-57-9a-bd-hj-qs-zA-Z\]* \
-		       go-test.exp=*/test/c* \
-		       go-test.exp=*/test/i* \
-		       go-test.exp=*/test/r* \
-		       go-test.exp=*/test/6*
+check_go_parallelize = 10
 
 # Install hooks.
 
--- gcc/fortran/Make-lang.in.jj	2014-01-03 11:40:51.548352536 +0100
+++ gcc/fortran/Make-lang.in	2014-09-11 16:08:36.788612700 +0200
@@ -168,12 +168,7 @@ check-fortran-subtargets : check-gfortra
 lang_checks += check-gfortran
 lang_checks_parallelized += check-gfortran
 # For description see comment above check_gcc_parallelize in gcc/Makefile.in.
-check_gfortran_parallelize = dg.exp=gfortran.dg/\[adAD\]* \
-			     dg.exp=gfortran.dg/\[bcBC\]* \
-			     dg.exp=gfortran.dg/\[nopNOP\]* \
-			     dg.exp=gfortran.dg/\[isuvISUV\]* \
-			     dg.exp=gfortran.dg/\[efhkqrxzEFHKQRXZ\]* \
-			     dg.exp=gfortran.dg/\[0-9gjlmtwyGJLMTWY\]*
+check_gfortran_parallelize = 10000
 
 # GFORTRAN documentation.
 GFORTRAN_TEXI = \
--- gcc/cp/Make-lang.in.jj	2014-09-10 21:57:10.240320174 +0200
+++ gcc/cp/Make-lang.in	2014-09-11 16:08:27.367658600 +0200
@@ -156,11 +156,7 @@ check-c++-subtargets : check-g++-subtarg
 lang_checks += check-g++
 lang_checks_parallelized += check-g++
 # For description see comment above check_gcc_parallelize in gcc/Makefile.in.
-check_g++_parallelize = old-deja.exp \
-	dg.exp=g++.dg/[0-9A-Za-bd-su-z]* \
-	dg.exp=g++.dg/[ct]* \
-	dg.exp=c-c++-common/*,dg-torture.exp
-
+check_g++_parallelize = 10000
 #
 # Install hooks:
 # cc1plus is installed elsewhere as part of $(COMPILERS).
--- gcc/testsuite/g++.dg/compat/struct-layout-1.exp.jj	2014-08-01 09:23:40.000000000 +0200
+++ gcc/testsuite/g++.dg/compat/struct-layout-1.exp	2014-09-11 18:45:09.422267240 +0200
@@ -95,6 +95,13 @@ load_lib g++-dg.exp
 
 g++_init
 
+# These tests don't run runtest_file_p consistently if it
+# doesn't return the same values, so disable parallelization
+# of this *.exp file.  The first parallel runtest to reach
+# this will run all the tests serially.
+if [gcc_parallel_test_run_p struct_layout_1] {
+gcc_parallel_test_enable 0
+
 # Save variables for the C++ compiler under test, which each test will
 # change a couple of times.  This must be done after calling g++-init.
 set save_gxx_under_test $GXX_UNDER_TEST
@@ -168,3 +175,6 @@ if { $status == 0 } then {
 
 # Restore the original compiler under test.
 compat-use-tst-compiler
+
+gcc_parallel_test_enable 1
+}
--- gcc/testsuite/g++.dg/plugin/plugin.exp.jj	2014-03-10 10:49:57.000000000 +0100
+++ gcc/testsuite/g++.dg/plugin/plugin.exp	2014-09-11 18:47:53.920423064 +0200
@@ -44,6 +44,13 @@ if $tracelevel then {
 # Load support procs.
 load_lib plugin-support.exp
 
+# These tests don't run runtest_file_p consistently if it
+# doesn't return the same values, so disable parallelization
+# of this *.exp file.  The first parallel runtest to reach
+# this will run all the tests serially.
+if [gcc_parallel_test_run_p plugin] {
+gcc_parallel_test_enable 0
+
 # Specify the plugin source file and the associated test files in a list.
 # plugin_test_list={ {plugin1 test1 test2 ...} {plugin2 test1 ...} ... }
 set plugin_test_list [list \
@@ -68,3 +75,6 @@ foreach plugin_test $plugin_test_list {
     set plugin_input_tests [lreplace $plugin_test 0 0]
     plugin-test-execute $plugin_src $plugin_input_tests
 }
+
+gcc_parallel_test_enable 1
+}
--- gcc/testsuite/gcc.dg/compat/struct-layout-1.exp.jj	2014-01-03 11:40:41.000000000 +0100
+++ gcc/testsuite/gcc.dg/compat/struct-layout-1.exp	2014-09-11 18:43:05.234916416 +0200
@@ -53,6 +53,13 @@ load_lib compat.exp
 
 gcc_init
 
+# These tests don't run runtest_file_p consistently if it
+# doesn't return the same values, so disable parallelization
+# of this *.exp file.  The first parallel runtest to reach
+# this will run all the tests serially.
+if [gcc_parallel_test_run_p struct_layout_1] {
+gcc_parallel_test_enable 0
+
 # Save variables for the C compiler under test, which each test will
 # change a couple of times.  This must be done after calling gcc-init.
 set compat_save_gcc_under_test $GCC_UNDER_TEST
@@ -128,3 +135,6 @@ if { $status == 0 } then {
 
 # Restore the original compiler under test.
 compat-use-tst-compiler
+
+gcc_parallel_test_enable 1
+}
--- gcc/testsuite/gcc.dg/plugin/plugin.exp.jj	2014-01-03 11:40:42.000000000 +0100
+++ gcc/testsuite/gcc.dg/plugin/plugin.exp	2014-09-11 18:47:02.220691539 +0200
@@ -44,6 +44,13 @@ if $tracelevel then {
 # Load support procs.
 load_lib plugin-support.exp
 
+# These tests don't run runtest_file_p consistently if it
+# doesn't return the same values, so disable parallelization
+# of this *.exp file.  The first parallel runtest to reach
+# this will run all the tests serially.
+if [gcc_parallel_test_run_p plugin] {
+gcc_parallel_test_enable 0
+
 # Specify the plugin source file and the associated test files in a list.
 # plugin_test_list={ {plugin1 test1 test2 ...} {plugin2 test1 ...} ... }
 set plugin_test_list [list \
@@ -80,3 +87,6 @@ dg-runtest [lsort [glob -nocomplain $src
 
 # All done.
 dg-finish
+
+gcc_parallel_test_enable 1
+}
--- gcc/testsuite/lib/gcc-defs.exp.jj	2014-09-11 11:02:36.045944898 +0200
+++ gcc/testsuite/lib/gcc-defs.exp	2014-09-11 18:20:01.021037249 +0200
@@ -188,6 +188,56 @@ if { [info procs runtest_file_p] == "" }
     }
 }
 
+if { [info exists env(GCC_RUNTEST_PARALLELIZE_DIR)] \
+     && [info procs runtest_file_p] != [list] \
+     && [info procs gcc_parallelize_saved_runtest_file_p] == [list] } then {
+    global gcc_runtest_parallelize_counter
+    global gcc_runtest_parallelize_enable
+
+    set gcc_runtest_parallelize_counter 0
+    set gcc_runtest_parallelize_enable 1
+
+    proc gcc_parallel_test_run_p { testcase } {
+	global gcc_runtest_parallelize_counter
+	global gcc_runtest_parallelize_enable
+
+	if { $gcc_runtest_parallelize_enable == 0 } {
+	    return 1
+	}
+	set dir [getenv GCC_RUNTEST_PARALLELIZE_DIR]
+	set path $dir/$gcc_runtest_parallelize_counter
+	set gcc_runtest_parallelize_counter [expr {$gcc_runtest_parallelize_counter + 1}]
+	if {![catch {open $path {RDWR CREAT EXCL} 0600} fd]} {
+	    close $fd
+	    return 1
+	}
+	return 0
+    }
+
+    proc gcc_parallel_test_enable { val } {
+	global gcc_runtest_parallelize_enable
+	set gcc_runtest_parallelize_enable $val
+    }
+
+    rename runtest_file_p gcc_parallelize_saved_runtest_file_p
+    proc runtest_file_p { runtests testcase } {
+	if ![gcc_parallelize_saved_runtest_file_p $runtests $testcase] {
+	    return 0
+	}
+	return [gcc_parallel_test_run_p $testcase]
+    }
+
+} else {
+
+    proc gcc_parallel_test_run_p { testcase } {
+	return 1
+    }
+
+    proc gcc_parallel_test_enable { val } {
+    }
+
+}
+
 # Like dg-options, but adds to the default options rather than replacing them.
 
 proc dg-additional-options { args } {
--- gcc/testsuite/objc.dg/gnu-encoding/gnu-encoding.exp.jj	2014-01-03 11:40:43.000000000 +0100
+++ gcc/testsuite/objc.dg/gnu-encoding/gnu-encoding.exp	2014-09-11 18:12:05.575481502 +0200
@@ -28,6 +28,13 @@ if ![info exists DEFAULT_CFLAGS] then {
 # Initialize `dg'.
 dg-init
 
+# These tests don't run runtest_file_p consistently if it
+# doesn't return the same values, so disable parallelization
+# of this *.exp file.  The first parallel runtest to reach
+# this will run all the tests serially.
+if [gcc_parallel_test_run_p gnu_encoding] {
+gcc_parallel_test_enable 0
+
 #
 # gnu-encoding tests
 #
@@ -67,10 +74,8 @@ if { $status == 0 } then {
     warning "Could not compile objc.dg/gnu-encoding/struct-layout-encoding-1 generator"
 }
 
-
-
-
-
+gcc_parallel_test_enable 1
+}
 
 # All done.
 dg-finish
--- gcc/objc/Make-lang.in.jj	2014-01-03 11:40:35.471444709 +0100
+++ gcc/objc/Make-lang.in	2014-09-11 16:09:02.671483908 +0200
@@ -96,7 +96,7 @@ lang_checks += check-objc
 # The following allows you to do 'make check-objc -j2'.  The
 # execute.exp tests will be run in parallel with all the other ones.
 lang_checks_parallelized += check-objc
-check_objc_parallelize = gnu-encoding.exp execute.exp exceptions.exp
+check_objc_parallelize = 6
 
 #
 # Install hooks:

	Jakub