Re: [PATCH] gcc parallel make check

[Jakub Jelinek <jakub at redhat dot com>]

On Thu, Sep 11, 2014 at 10:06:40AM +0200, Jakub Jelinek wrote:
> There is an option to touch say *-parallel/finished file once any of the
> check-parallel-gcc-{1,2,...} goals is done (because when it finishes, it
> means all the tests for the particular check-$lang that are parallelizable
> have either finished, or at least touched their file) and not start runtest
> at all if finished already exists, but guess it would be still undesirable to have
> tens of thousands of goals by default, so perhaps we could go with say
> 128 subgoals by default and have some env var to override it, so on the
> really highly parallel boxes you'd specify
> make -j512 -k check GCC_TEST_PARALLEL_SLOTS=512
> or similar.

Here is a patch I'm testing now:

--- gcc/Makefile.in.jj	2014-09-08 22:12:56.000000000 +0200
+++ gcc/Makefile.in	2014-09-11 16:06:36.641219430 +0200
@@ -513,34 +513,10 @@ xm_include_list=@xm_include_list@
 xm_defines=@xm_defines@
 lang_checks=
 lang_checks_parallelized=
-dg_target_exps:=aarch64.exp,alpha.exp,arm.exp,avr.exp,bfin.exp,cris.exp
-dg_target_exps:=$(dg_target_exps),epiphany.exp,frv.exp,i386.exp,ia64.exp
-dg_target_exps:=$(dg_target_exps),m68k.exp,microblaze.exp,mips.exp,powerpc.exp
-dg_target_exps:=$(dg_target_exps),rx.exp,s390.exp,sh.exp,sparc.exp,spu.exp
-dg_target_exps:=$(dg_target_exps),tic6x.exp,xstormy16.exp
-# This lists a couple of test files that take most time during check-gcc.
-# When doing parallelized check-gcc, these can run in parallel with the
-# remaining tests.  Each word in this variable stands for work for one
-# make goal and one extra make goal is added to handle all the *.exp
-# files not handled explicitly already.  If multiple *.exp files
-# should be run in the same runtest invocation (usually if they aren't
-# very long running, but still should be split of from the check-parallel-$lang
-# remaining tests runtest invocation), they should be concatenated with commas.
-# Note that [a-zA-Z] wildcards need to have []s prefixed with \ (needed
-# by tcl) and as the *.exp arguments are mached both as is and with
-# */ prefixed to it in runtest_file_p, it is usually desirable to include
-# a subdirectory name.
-check_gcc_parallelize=execute.exp=execute/2* \
-		      execute.exp=execute/\[013-9a-fA-F\]* \
-		      execute.exp=execute/\[pP\]*,dg.exp \
-		      execute.exp=execute/\[g-oq-zG-OQ-Z\]*,compile.exp=compile/2* \
-		      compile.exp=compile/\[9pP\]*,builtins.exp \
-		      compile.exp=compile/\[013-8a-oq-zA-OQ-Z\]* \
-		      dg-torture.exp,ieee.exp \
-		      vect.exp,unsorted.exp \
-		      guality.exp \
-		      struct-layout-1.exp,stackalign.exp \
-		      $(dg_target_exps)
+# Upper limit to which it is useful to parallelize this lang target.
+# It doesn't make sense to try e.g. 128 goals for small testsuites
+# like objc or go.
+check_gcc_parallelize=10000
 lang_opt_files=@lang_opt_files@ $(srcdir)/c-family/c.opt $(srcdir)/common.opt
 lang_specs_files=@lang_specs_files@
 lang_tree_files=@lang_tree_files@
@@ -3631,27 +3607,32 @@ $(filter-out $(lang_checks_parallelized)
 	    export TCL_LIBRARY ; fi ; \
 	$(RUNTEST) --tool $* $(RUNTESTFLAGS))
 
-$(patsubst %,%-subtargets,$(filter-out $(lang_checks_parallelized),$(lang_checks))): check-%-subtargets:
+$(patsubst %,%-subtargets,$(lang_checks)): check-%-subtargets:
 	@echo check-$*
 
 check_p_tool=$(firstword $(subst _, ,$*))
-check_p_vars=$(check_$(check_p_tool)_parallelize)
+check_p_count=$(check_$(check_p_tool)_parallelize)
 check_p_subno=$(word 2,$(subst _, ,$*))
-check_p_comma=,
-check_p_subwork=$(subst $(check_p_comma), ,$(if $(check_p_subno),$(word $(check_p_subno),$(check_p_vars))))
-check_p_numbers=1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
+check_p_numbers0:=1 2 3 4 5 6 7 8 9
+check_p_numbers1:=0 $(check_p_numbers0)
+check_p_numbers2:=$(foreach i,$(check_p_numbers0),$(patsubst %,$(i)%,$(check_p_numbers1)))
+check_p_numbers3:=$(patsubst %,0%,$(check_p_numbers1)) $(check_p_numbers2)
+check_p_numbers4:=$(foreach i,$(check_p_numbers0),$(patsubst %,$(i)%,$(check_p_numbers3)))
+check_p_numbers5:=$(patsubst %,0%,$(check_p_numbers3)) $(check_p_numbers4)
+check_p_numbers6:=$(foreach i,$(check_p_numbers0),$(patsubst %,$(i)%,$(check_p_numbers5)))
+check_p_numbers:=$(check_p_numbers0) $(check_p_numbers2) $(check_p_numbers4) $(check_p_numbers6)
 check_p_subdir=$(subst _,,$*)
-check_p_subdirs=$(wordlist 1,$(words $(check_$*_parallelize)),$(check_p_numbers))
+check_p_subdirs=$(wordlist 1,$(check_p_count),$(wordlist 1,$(or $(GCC_TEST_PARALLEL_SLOTS),128),$(check_p_numbers)))
 
 # For parallelized check-% targets, this decides whether parallelization
 # is desirable (if -jN is used and RUNTESTFLAGS doesn't contain anything
 # but optional --target_board or --extra_opts arguments).  If desirable,
 # recursive make is run with check-parallel-$lang{,1,2,3,4,5} etc. goals,
 # which can be executed in parallel, as they are run in separate directories.
-# check-parallel-$lang{1,2,3,4,5} etc. goals invoke runtest with the longest
-# running *.exp files from the testsuite, as determined by check_$lang_parallelize
-# variable.  The check-parallel-$lang goal in that case invokes runtest with
-# all the remaining *.exp files not handled by the separate goals.
+# check-parallel-$lang{,1,2,3,4,5} etc. goals invoke runtest with
+# GCC_RUNTEST_PARALLELIZE_DIR var in the environment and runtest_file_p
+# dejaGNU procedure is overridden to additionally synchronize through
+# a $lang-parallel directory which tests will be run by which runtest instance.
 # Afterwards contrib/dg-extract-results.sh is used to merge the sum and log
 # files.  If parallelization isn't desirable, only one recursive make
 # is run with check-parallel-$lang goal and check_$lang_parallelize variable
@@ -3662,76 +3643,61 @@ check_p_subdirs=$(wordlist 1,$(words $(c
 # to lang_checks_parallelized variable and define check_$lang_parallelize
 # variable (see above check_gcc_parallelize description).
 $(lang_checks_parallelized): check-% : site.exp
+	-rm -rf $(TESTSUITEDIR)/$*-parallel
 	@if [ -z "$(filter-out --target_board=%,$(filter-out --extra_opts%,$(RUNTESTFLAGS)))" ] \
 	    && [ "$(filter -j, $(MFLAGS))" = "-j" ]; then \
+	  test -d $(TESTSUITEDIR) || mkdir $(TESTSUITEDIR) || true; \
+	  test -d $(TESTSUITEDIR)/$*-parallel || mkdir $(TESTSUITEDIR)/$*-parallel || true; \
+	  GCC_RUNTEST_PARALLELIZE_DIR=`${PWD_COMMAND}`/$(TESTSUITEDIR)/$(check_p_tool)-parallel ; \
+	  export GCC_RUNTEST_PARALLELIZE_DIR ; \
 	  $(MAKE) TESTSUITEDIR="$(TESTSUITEDIR)" RUNTESTFLAGS="$(RUNTESTFLAGS)" \
 	    check-parallel-$* \
 	    $(patsubst %,check-parallel-$*_%, $(check_p_subdirs)); \
-	  for file in $(TESTSUITEDIR)/$*/$* \
-		      $(patsubst %,$(TESTSUITEDIR)/$*%/$*,$(check_p_subdirs));\
+	  sums= ; logs= ; \
+	  for dir in $(TESTSUITEDIR)/$* \
+		     $(patsubst %,$(TESTSUITEDIR)/$*%,$(check_p_subdirs));\
 	  do \
-	    mv -f $$file.sum $$file.sum.sep; mv -f $$file.log $$file.log.sep; \
+	    if [ -d $$dir ]; then \
+	      mv -f $$dir/$*.sum $$dir/$*.sum.sep; mv -f $$dir/$*.log $$dir/$*.log.sep; \
+	      sums="$$sums $$dir/$*.sum.sep"; logs="$$logs $$dir/$*.log.sep"; \
+	    fi; \
 	  done; \
-	  $(SHELL) $(srcdir)/../contrib/dg-extract-results.sh \
-	    $(TESTSUITEDIR)/$*/$*.sum.sep \
-	    $(patsubst %,$(TESTSUITEDIR)/$*%/$*.sum.sep,$(check_p_subdirs)) \
+	  $(SHELL) $(srcdir)/../contrib/dg-extract-results.sh $$sums \
 	    > $(TESTSUITEDIR)/$*/$*.sum; \
-	  $(SHELL) $(srcdir)/../contrib/dg-extract-results.sh -L \
-	    $(TESTSUITEDIR)/$*/$*.log.sep \
-	    $(patsubst %,$(TESTSUITEDIR)/$*%/$*.log.sep,$(check_p_subdirs)) \
+	  $(SHELL) $(srcdir)/../contrib/dg-extract-results.sh -L $$logs \
 	    > $(TESTSUITEDIR)/$*/$*.log; \
+	  rm -rf $(TESTSUITEDIR)/$*-parallel || true; \
 	else \
 	  $(MAKE) TESTSUITEDIR="$(TESTSUITEDIR)" RUNTESTFLAGS="$(RUNTESTFLAGS)" \
 	    check_$*_parallelize= check-parallel-$*; \
 	fi
 
-# Just print the parallelized subtargets for those that want to split
-# the testing across machines.
-$(patsubst %,%-subtargets,$(lang_checks_parallelized)): check-%-subtargets:
-	@echo check-parallel-$* \
-	  $(patsubst %,check-parallel-$*_%, $(check_p_subdirs))
-
-# In the if [ -n "$(check_p_subno)" ] case runtest should be given the name of
-# the given *.exp file(s).  See comment above check_gcc_parallelize variable
-# for details on the content of these variables.
-#
-# In the elif [ -n "$(check_p_vars)" ] case runtest should be given
-# names of all the *.exp files for this tool that aren't already handled by
-# other goals.  First it finds all the *.exp files for this tool, then
-# prunes those already specified in check_$lang_parallelize or duplicates.
-#
-# Otherwise check-$lang isn't parallelized and runtest is invoked just with
-# the $(RUNTESTFLAGS) arguments.
 check-parallel-% : site.exp
 	-test -d plugin || mkdir plugin
 	-test -d $(TESTSUITEDIR) || mkdir $(TESTSUITEDIR)
 	test -d $(TESTSUITEDIR)/$(check_p_subdir) || mkdir $(TESTSUITEDIR)/$(check_p_subdir)
 	-(rootme=`${PWD_COMMAND}`; export rootme; \
 	srcdir=`cd ${srcdir}; ${PWD_COMMAND}` ; export srcdir ; \
-	cd $(TESTSUITEDIR)/$(check_p_subdir); \
-	rm -f tmp-site.exp; \
-	sed '/set tmpdir/ s|testsuite$$|$(TESTSUITEDIR)/$(check_p_subdir)|' \
+	if [ -n "$(check_p_subno)" \
+	     -a -n "$$GCC_RUNTEST_PARALLELIZE_DIR" \
+	     -a -f $(TESTSUITEDIR)/$(check_p_tool)-parallel/finished ]; then \
+	  rm -rf $(TESTSUITEDIR)/$(check_p_subdir); \
+	else \
+	  cd $(TESTSUITEDIR)/$(check_p_subdir); \
+	  rm -f tmp-site.exp; \
+	  sed '/set tmpdir/ s|testsuite$$|$(TESTSUITEDIR)/$(check_p_subdir)|' \
 		< ../../site.exp > tmp-site.exp; \
-	$(SHELL) $${srcdir}/../move-if-change tmp-site.exp site.exp; \
-	EXPECT=${EXPECT} ; export EXPECT ; \
-	if [ -f $${rootme}/../expect/expect ] ; then  \
-	   TCL_LIBRARY=`cd .. ; cd $${srcdir}/../tcl/library ; ${PWD_COMMAND}` ; \
-	    export TCL_LIBRARY ; fi ; \
-	runtestflags= ; \
-	if [ -n "$(check_p_subno)" ] ; then \
-	  runtestflags="$(check_p_subwork)"; \
-	elif [ -n "$(check_p_vars)" ] ; then \
-	  parts="`echo ' $(strip $(subst $(check_p_comma), ,$(check_p_vars))) ' \
-		  | sed 's/=[^ ]* / /g'`"; \
-	  for part in `find $$srcdir/testsuite/$(check_p_tool)* -name \*.exp` ; do \
-	    part=`basename $$part` ; \
-	    case " $$parts $$runtestflags " in \
-	      *" $$part "*) ;; \
-	      *) runtestflags="$$runtestflags $$part" ;; \
-	    esac ; \
-	  done ; \
-	fi ; \
-	$(RUNTEST) --tool $(check_p_tool) $(RUNTESTFLAGS) $$runtestflags)
+	  $(SHELL) $${srcdir}/../move-if-change tmp-site.exp site.exp; \
+	  EXPECT=${EXPECT} ; export EXPECT ; \
+	  if [ -f $${rootme}/../expect/expect ] ; then  \
+	    TCL_LIBRARY=`cd .. ; cd $${srcdir}/../tcl/library ; ${PWD_COMMAND}` ; \
+	    export TCL_LIBRARY ; \
+	  fi ; \
+	  $(RUNTEST) --tool $(check_p_tool) $(RUNTESTFLAGS); \
+	  if [ -n "$$GCC_RUNTEST_PARALLELIZE_DIR" ] ; then \
+	    touch $${rootme}/$(TESTSUITEDIR)/$(check_p_tool)-parallel/finished; \
+	  fi ; \
+	fi )
 
 # QMTest targets
 
--- gcc/go/Make-lang.in.jj	2014-05-02 15:39:49.766921376 +0200
+++ gcc/go/Make-lang.in	2014-09-11 16:08:50.049547349 +0200
@@ -131,11 +131,7 @@ go.srcman: doc/gccgo.1
 
 lang_checks += check-go
 lang_checks_parallelized += check-go
-check_go_parallelize = go-test.exp=*/test/\[0-57-9a-bd-hj-qs-zA-Z\]* \
-		       go-test.exp=*/test/c* \
-		       go-test.exp=*/test/i* \
-		       go-test.exp=*/test/r* \
-		       go-test.exp=*/test/6*
+check_go_parallelize = 10
 
 # Install hooks.
 
--- gcc/fortran/Make-lang.in.jj	2014-01-03 11:40:51.548352536 +0100
+++ gcc/fortran/Make-lang.in	2014-09-11 16:08:36.788612700 +0200
@@ -168,12 +168,7 @@ check-fortran-subtargets : check-gfortra
 lang_checks += check-gfortran
 lang_checks_parallelized += check-gfortran
 # For description see comment above check_gcc_parallelize in gcc/Makefile.in.
-check_gfortran_parallelize = dg.exp=gfortran.dg/\[adAD\]* \
-			     dg.exp=gfortran.dg/\[bcBC\]* \
-			     dg.exp=gfortran.dg/\[nopNOP\]* \
-			     dg.exp=gfortran.dg/\[isuvISUV\]* \
-			     dg.exp=gfortran.dg/\[efhkqrxzEFHKQRXZ\]* \
-			     dg.exp=gfortran.dg/\[0-9gjlmtwyGJLMTWY\]*
+check_gfortran_parallelize = 10000
 
 # GFORTRAN documentation.
 GFORTRAN_TEXI = \
--- gcc/cp/Make-lang.in.jj	2014-09-10 21:57:10.240320174 +0200
+++ gcc/cp/Make-lang.in	2014-09-11 16:08:27.367658600 +0200
@@ -156,11 +156,7 @@ check-c++-subtargets : check-g++-subtarg
 lang_checks += check-g++
 lang_checks_parallelized += check-g++
 # For description see comment above check_gcc_parallelize in gcc/Makefile.in.
-check_g++_parallelize = old-deja.exp \
-	dg.exp=g++.dg/[0-9A-Za-bd-su-z]* \
-	dg.exp=g++.dg/[ct]* \
-	dg.exp=c-c++-common/*,dg-torture.exp
-
+check_g++_parallelize = 10000
 #
 # Install hooks:
 # cc1plus is installed elsewhere as part of $(COMPILERS).
--- gcc/testsuite/lib/gcc-defs.exp.jj	2014-09-11 11:02:36.045944898 +0200
+++ gcc/testsuite/lib/gcc-defs.exp	2014-09-11 14:29:54.546328102 +0200
@@ -188,6 +188,30 @@ if { [info procs runtest_file_p] == "" }
     }
 }
 
+if { [info exists env(GCC_RUNTEST_PARALLELIZE_DIR)] \
+     && [info procs runtest_file_p] != [list] \
+     && [info procs gcc_parallelize_saved_runtest_file_p] == [list] } then {
+    rename runtest_file_p gcc_parallelize_saved_runtest_file_p
+    global gcc_runtest_parallelize_counter
+
+    set gcc_runtest_parallelize_counter 0
+    proc runtest_file_p { runtests testcase } {
+	global gcc_runtest_parallelize_counter
+	if ![gcc_parallelize_saved_runtest_file_p $runtests $testcase] {
+	    return 0
+	}
+
+	set dir [getenv GCC_RUNTEST_PARALLELIZE_DIR]
+	set path $dir/$gcc_runtest_parallelize_counter
+	set gcc_runtest_parallelize_counter [expr {$gcc_runtest_parallelize_counter + 1}]
+	if {![catch {open $path {RDWR CREAT EXCL} 0600} fd]} {
+	    close $fd
+	    return 1
+	}
+	return 0
+    }
+}
+
 # Like dg-options, but adds to the default options rather than replacing them.
 
 proc dg-additional-options { args } {
--- gcc/objc/Make-lang.in.jj	2014-01-03 11:40:35.471444709 +0100
+++ gcc/objc/Make-lang.in	2014-09-11 16:09:02.671483908 +0200
@@ -96,7 +96,7 @@ lang_checks += check-objc
 # The following allows you to do 'make check-objc -j2'.  The
 # execute.exp tests will be run in parallel with all the other ones.
 lang_checks_parallelized += check-objc
-check_objc_parallelize = gnu-encoding.exp execute.exp exceptions.exp
+check_objc_parallelize = 6
 
 #
 # Install hooks:


	Jakub