This is the mail archive of the gcc@gcc.gnu.org mailing list for the GCC project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

RE: [PATCH] RE: gcc parallel make check


Attached is a further revision of the patch, now dealing with check-c++. Roughly 50% speedup here at '-j32' (18m vs 12m). For my setup (--enable-languages=c,c++,fortran) I have now improved all targets called in 'make -j32 -k check'. The latter is now 30% faster (15m vs 20m). Note that there are +- 1m fluctuations in these numbers, easily.

I currently have no plans to work on other check targets before this patch is committed.

OK for trunk ?

Joost







Attachment: patch-speedup-check-v06-CL.txt
Description: patch-speedup-check-v06-CL.txt

Index: libstdc++-v3/testsuite/Makefile.am
===================================================================
--- libstdc++-v3/testsuite/Makefile.am	(revision 215017)
+++ libstdc++-v3/testsuite/Makefile.am	(working copy)
@@ -101,7 +101,7 @@ new-abi-baseline:
 	@test ! -f $*/site.exp || mv $*/site.exp $*/site.bak
 	@mv $*/site.exp.tmp $*/site.exp
 
-check_DEJAGNU_normal_targets = $(patsubst %,check-DEJAGNUnormal%,0 1 2 3 4 5 6 7 8 9 10)
+check_DEJAGNU_normal_targets = $(patsubst %,check-DEJAGNUnormal%,0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15)
 $(check_DEJAGNU_normal_targets): check-DEJAGNUnormal%: normal%/site.exp
 
 # Run the testsuite in normal mode.
@@ -111,7 +111,7 @@ check-DEJAGNU $(check_DEJAGNU_normal_tar
 	if [ -z "$*$(filter-out --target_board=%, $(RUNTESTFLAGS))" ] \
 	    && [ "$(filter -j, $(MFLAGS))" = "-j" ]; then \
 	  $(MAKE) $(AM_MAKEFLAGS) $(check_DEJAGNU_normal_targets); \
-	  for idx in 0 1 2 3 4 5 6 7 8 9 10; do \
+	  for idx in 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do \
 	    mv -f normal$$idx/libstdc++.sum normal$$idx/libstdc++.sum.sep; \
 	    mv -f normal$$idx/libstdc++.log normal$$idx/libstdc++.log.sep; \
 	  done; \
@@ -138,25 +138,35 @@ check-DEJAGNU $(check_DEJAGNU_normal_tar
 	    fi; \
 	    dirs="`cd $$srcdir; echo [013-9][0-9]_*/*`";; \
 	  normal1) \
-	    dirs="`cd $$srcdir; echo [ab]* de* [ep]*/*`";; \
+	    dirs="`cd $$srcdir; echo e*/*`";; \
 	  normal2) \
-	    dirs="`cd $$srcdir; echo 2[01]_*/*`";; \
+	    dirs="`cd $$srcdir; echo 28_*/a*`";; \
 	  normal3) \
-	    dirs="`cd $$srcdir; echo 22_*/*`";; \
+	    dirs="`cd $$srcdir; echo 23_*/[lu]*`";; \
 	  normal4) \
-	    dirs="`cd $$srcdir; echo 23_*/[a-km-tw-z]*`";; \
+	    dirs="`cd $$srcdir; echo 2[459]_*/*`";; \
 	  normal5) \
-	    dirs="`cd $$srcdir; echo 23_*/[luv]*`";; \
+	    dirs="`cd $$srcdir; echo 2[01]_*/*`";; \
 	  normal6) \
-	    dirs="`cd $$srcdir; echo 2[459]_*/*`";; \
+	    dirs="`cd $$srcdir; echo 23_*/[m-tw-z]*`";; \
 	  normal7) \
-	    dirs="`cd $$srcdir; echo 26_*/* 28_*/[c-z]*`";; \
+	    dirs="`cd $$srcdir; echo 26_*/*`";; \
 	  normal8) \
 	    dirs="`cd $$srcdir; echo 27_*/*`";; \
 	  normal9) \
-	    dirs="`cd $$srcdir; echo 28_*/[ab]*`";; \
+	    dirs="`cd $$srcdir; echo 22_*/*`";; \
 	  normal10) \
 	    dirs="`cd $$srcdir; echo t*/*`";; \
+	  normal11) \
+	    dirs="`cd $$srcdir; echo 28_*/b*`";; \
+	  normal12) \
+	    dirs="`cd $$srcdir; echo 28_*/[c-z]*`";; \
+	  normal13) \
+	    dirs="`cd $$srcdir; echo de* p*/*`";; \
+	  normal14) \
+	    dirs="`cd $$srcdir; echo [ab]* 23_*/v*`";; \
+	  normal15) \
+	    dirs="`cd $$srcdir; echo 23_*/[a-k]*`";; \
 	esac; \
 	if [ -n "$*" ]; then cd "$*"; fi; \
 	if $(SHELL) -c "$$runtest --version" > /dev/null 2>&1; then \
Index: libstdc++-v3/testsuite/Makefile.in
===================================================================
--- libstdc++-v3/testsuite/Makefile.in	(revision 215017)
+++ libstdc++-v3/testsuite/Makefile.in	(working copy)
@@ -301,7 +301,7 @@ lists_of_files = \
 
 extract_symvers = $(glibcxx_builddir)/scripts/extract_symvers
 baseline_subdir := $(shell $(CXX) $(baseline_subdir_switch))
-check_DEJAGNU_normal_targets = $(patsubst %,check-DEJAGNUnormal%,0 1 2 3 4 5 6 7 8 9 10)
+check_DEJAGNU_normal_targets = $(patsubst %,check-DEJAGNUnormal%,0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15)
 
 # Runs the testsuite, but in compile only mode.
 # Can be used to test sources with non-GNU FE's at various warning
@@ -562,7 +562,7 @@ check-DEJAGNU $(check_DEJAGNU_normal_tar
 	if [ -z "$*$(filter-out --target_board=%, $(RUNTESTFLAGS))" ] \
 	    && [ "$(filter -j, $(MFLAGS))" = "-j" ]; then \
 	  $(MAKE) $(AM_MAKEFLAGS) $(check_DEJAGNU_normal_targets); \
-	  for idx in 0 1 2 3 4 5 6 7 8 9 10; do \
+	  for idx in 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do \
 	    mv -f normal$$idx/libstdc++.sum normal$$idx/libstdc++.sum.sep; \
 	    mv -f normal$$idx/libstdc++.log normal$$idx/libstdc++.log.sep; \
 	  done; \
@@ -589,25 +589,35 @@ check-DEJAGNU $(check_DEJAGNU_normal_tar
 	    fi; \
 	    dirs="`cd $$srcdir; echo [013-9][0-9]_*/*`";; \
 	  normal1) \
-	    dirs="`cd $$srcdir; echo [ab]* de* [ep]*/*`";; \
+	    dirs="`cd $$srcdir; echo e*/*`";; \
 	  normal2) \
-	    dirs="`cd $$srcdir; echo 2[01]_*/*`";; \
+	    dirs="`cd $$srcdir; echo 28_*/a*`";; \
 	  normal3) \
-	    dirs="`cd $$srcdir; echo 22_*/*`";; \
+	    dirs="`cd $$srcdir; echo 23_*/[lu]*`";; \
 	  normal4) \
-	    dirs="`cd $$srcdir; echo 23_*/[a-km-tw-z]*`";; \
+	    dirs="`cd $$srcdir; echo 2[459]_*/*`";; \
 	  normal5) \
-	    dirs="`cd $$srcdir; echo 23_*/[luv]*`";; \
+	    dirs="`cd $$srcdir; echo 2[01]_*/*`";; \
 	  normal6) \
-	    dirs="`cd $$srcdir; echo 2[459]_*/*`";; \
+	    dirs="`cd $$srcdir; echo 23_*/[m-tw-z]*`";; \
 	  normal7) \
-	    dirs="`cd $$srcdir; echo 26_*/* 28_*/[c-z]*`";; \
+	    dirs="`cd $$srcdir; echo 26_*/*`";; \
 	  normal8) \
 	    dirs="`cd $$srcdir; echo 27_*/*`";; \
 	  normal9) \
-	    dirs="`cd $$srcdir; echo 28_*/[ab]*`";; \
+	    dirs="`cd $$srcdir; echo 22_*/*`";; \
 	  normal10) \
 	    dirs="`cd $$srcdir; echo t*/*`";; \
+	  normal11) \
+	    dirs="`cd $$srcdir; echo 28_*/b*`";; \
+	  normal12) \
+	    dirs="`cd $$srcdir; echo 28_*/[c-z]*`";; \
+	  normal13) \
+	    dirs="`cd $$srcdir; echo de* p*/*`";; \
+	  normal14) \
+	    dirs="`cd $$srcdir; echo [ab]* 23_*/v*`";; \
+	  normal15) \
+	    dirs="`cd $$srcdir; echo 23_*/[a-k]*`";; \
 	esac; \
 	if [ -n "$*" ]; then cd "$*"; fi; \
 	if $(SHELL) -c "$$runtest --version" > /dev/null 2>&1; then \
Index: contrib/generate_tcl_patterns.sh
===================================================================
--- contrib/generate_tcl_patterns.sh	(revision 0)
+++ contrib/generate_tcl_patterns.sh	(revision 0)
@@ -0,0 +1,114 @@
+#! /bin/sh
+
+#
+# based on a list of filenames as input, starting with [0-9A-Za-z],
+# generate regexps that match subsets trying to not exceed a
+# 'maxcount' parameter. Most useful to generate the
+# check_LANG_parallelize assignments needed to split
+# testsuite directories, defining prefix appropriately.
+#
+# Example usage:
+#   cd gcc/gcc/testsuite/gfortran.dg
+#   ls -1 | ../../../contrib/generate_tcl_patterns.sh 300 "dg.exp=gfortran.dg/"
+#
+# the first parameter is the maximum number of files.
+# the second parameter the prefix used for printing.
+#
+
+# Copyright (C) 2014 Free Software Foundation
+# Contributed by Joost VandeVondele <Joost.VandeVondele@mat.ethz.ch>
+#
+# This file is part of GCC.
+#
+# GCC is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# GCC is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GCC; see the file COPYING.  If not, write to
+# the Free Software Foundation, 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+
+gawk -v maxcount=$1 -v prefix=$2 '
+BEGIN{
+  # list of allowed starting chars for a file name in a dir to split
+  achars="0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+  ranget="11111111112222222222222222222222222233333333333333333333333333"
+}
+{
+  if (index(achars,substr($1,1,1))==0){
+     print "file : " $1 " does not start with an allowed character."
+     _assert_exit = 1
+     exit 1
+  }
+  nfiles++ ; files[nfiles]=$1
+}
+END{
+  if (_assert_exit) exit 1
+  for(i=1; i<=length(achars); i++) count[substr(achars,i,1)]=0
+  for(i=1; i<=nfiles; i++) {
+     if (length(files[i]>0)) { count[substr(files[i],1,1)]++ }
+  };
+  asort(count,ordered)
+  countsingle=0
+  groups=0
+  label=""
+  for(i=length(achars);i>=1;i--) {
+    countsingle=countsingle+ordered[i] 
+    for(j=1;j<=length(achars);j++) {
+       if(count[substr(achars,j,1)]==ordered[i]) found=substr(achars,j,1)
+    }
+    count[found]=-1
+    label=label found
+    if(i==1) { val=maxcount+1 } else { val=ordered[i-1] }
+    if(countsingle+val>maxcount) {
+      subset[label]=countsingle
+      print "Adding label: ", label, "matching files:" countsingle
+      groups++
+      countsingle=0
+      label=""
+    }
+  }
+  print "patterns:"
+  asort(subset,ordered)
+  for(i=groups;i>=1;i--) {
+    for(j in subset){
+      if(subset[j]==ordered[i]) found=j
+    }
+    subset[found]=-1
+    if (length(found)==1) {
+       printf("%s%s* \\\n",prefix,found)
+    } else {
+       sortandcompress()
+       printf("%s\\[%s\\]* \\\n",prefix,found)
+    }
+  }
+}
+function sortandcompress(i,n,tmp,bestj)
+{
+  n=length(found)
+  for(i=1; i<=n; i++) tmp[i]=substr(found,i,1) 
+  asort(tmp)
+  for(i=1;i<=n;i++){
+    ipos=index(achars,tmp[i])
+    for(j=i;j<=n;j++){
+      jpos=index(achars,tmp[j])
+      if (jpos-ipos==j-i && substr(ranget,ipos,1)==substr(ranget,jpos,1)) bestj=j
+    }
+    if (bestj-i>3) {
+      tmp[i+1]="-" 
+      for(j=i+2;j<bestj;j++) tmp[j]=" "
+    }
+    i=bestj 
+  }
+  found=""
+  for(i=1; i<=n; i++) found=found tmp[i]
+  gsub(/ /,"",found)
+}
+'
Index: gcc/fortran/Make-lang.in
===================================================================
--- gcc/fortran/Make-lang.in	(revision 215017)
+++ gcc/fortran/Make-lang.in	(working copy)
@@ -168,12 +168,22 @@ check-fortran-subtargets : check-gfortra
 lang_checks += check-gfortran
 lang_checks_parallelized += check-gfortran
 # For description see comment above check_gcc_parallelize in gcc/Makefile.in.
-check_gfortran_parallelize = dg.exp=gfortran.dg/\[adAD\]* \
-			     dg.exp=gfortran.dg/\[bcBC\]* \
-			     dg.exp=gfortran.dg/\[nopNOP\]* \
-			     dg.exp=gfortran.dg/\[isuvISUV\]* \
-			     dg.exp=gfortran.dg/\[efhkqrxzEFHKQRXZ\]* \
-			     dg.exp=gfortran.dg/\[0-9gjlmtwyGJLMTWY\]*
+check_gfortran_parallelize = execute.exp \
+			dg.exp=gfortran.dg/p* \
+			dg.exp=gfortran.dg/c* \
+			dg.exp=gfortran.dg/a* \
+			dg.exp=gfortran.dg/i* \
+			dg.exp=gfortran.dg/\[glow\]* \
+			dg.exp=gfortran.dg/\[mu\]* \
+			dg.exp=gfortran.dg/d* \
+			dg.exp=gfortran.dg/s* \
+			dg.exp=gfortran.dg/b* \
+			dg.exp=gfortran.dg/t* \
+			dg.exp=gfortran.dg/f* \
+			dg.exp=gfortran.dg/e* \
+			dg.exp=gfortran.dg/r* \
+			dg.exp=gfortran.dg/n* \
+			dg.exp=gfortran.dg/\[0-9A-Zhjkqvxyz\]*
 
 # GFORTRAN documentation.
 GFORTRAN_TEXI = \
Index: gcc/Makefile.in
===================================================================
--- gcc/Makefile.in	(revision 215017)
+++ gcc/Makefile.in	(working copy)
@@ -513,11 +513,17 @@ xm_include_list=@xm_include_list@
 xm_defines=@xm_defines@
 lang_checks=
 lang_checks_parallelized=
-dg_target_exps:=aarch64.exp,alpha.exp,arm.exp,avr.exp,bfin.exp,cris.exp
-dg_target_exps:=$(dg_target_exps),epiphany.exp,frv.exp,i386.exp,ia64.exp
-dg_target_exps:=$(dg_target_exps),m68k.exp,microblaze.exp,mips.exp,powerpc.exp
-dg_target_exps:=$(dg_target_exps),rx.exp,s390.exp,sh.exp,sparc.exp,spu.exp
-dg_target_exps:=$(dg_target_exps),tic6x.exp,xstormy16.exp
+dg_target_exps:=aarch64 alpha arm avr bfin cris epiphany frv i386 ia64 
+dg_target_exps:=$(dg_target_exps) m68k microblaze mips powerpc rx s390
+dg_target_exps:=$(dg_target_exps) sh sparc spu tic6x xstormy16
+# also parallelize the target exps, using one common choice (i386) of split
+null:=
+space:=$(null) $(null)
+comma:=$(null),$(null)
+dg_target_exps_p1:=$(subst $(space),$(comma),$(foreach target,$(dg_target_exps),$(target).exp=$(target)/a*))
+dg_target_exps_p2:=$(subst $(space),$(comma),$(foreach target,$(dg_target_exps),$(target).exp=$(target)/\[fs\]*))
+dg_target_exps_p3:=$(subst $(space),$(comma),$(foreach target,$(dg_target_exps),$(target).exp=$(target)/\[0-9A-Zbcdeg-oqrt-z\]*))
+dg_target_exps_p4:=$(subst $(space),$(comma),$(foreach target,$(dg_target_exps),$(target).exp=$(target)/p*))
 # This lists a couple of test files that take most time during check-gcc.
 # When doing parallelized check-gcc, these can run in parallel with the
 # remaining tests.  Each word in this variable stands for work for one
@@ -526,21 +532,42 @@ dg_target_exps:=$(dg_target_exps),tic6x.
 # should be run in the same runtest invocation (usually if they aren't
 # very long running, but still should be split of from the check-parallel-$lang
 # remaining tests runtest invocation), they should be concatenated with commas.
+# At most $(check_p_numbers) goals should be created.
 # Note that [a-zA-Z] wildcards need to have []s prefixed with \ (needed
 # by tcl) and as the *.exp arguments are mached both as is and with
 # */ prefixed to it in runtest_file_p, it is usually desirable to include
 # a subdirectory name.
-check_gcc_parallelize=execute.exp=execute/2* \
-		      execute.exp=execute/\[013-9a-fA-F\]* \
-		      execute.exp=execute/\[pP\]*,dg.exp \
-		      execute.exp=execute/\[g-oq-zG-OQ-Z\]*,compile.exp=compile/2* \
-		      compile.exp=compile/\[9pP\]*,builtins.exp \
-		      compile.exp=compile/\[013-8a-oq-zA-OQ-Z\]* \
-		      dg-torture.exp,ieee.exp \
-		      vect.exp,unsorted.exp \
-		      guality.exp \
-		      struct-layout-1.exp,stackalign.exp \
-		      $(dg_target_exps)
+check_gcc_parallelize=$(dg_target_exps_p1) \
+			$(dg_target_exps_p2) \
+			$(dg_target_exps_p3) \
+			$(dg_target_exps_p4) \
+			execute.exp=execute/2* \
+			execute.exp=execute/p* \
+			execute.exp=execute/9* \
+			execute.exp=execute/\[013-8A-Za-oq-z\]* \
+			compile.exp=compile/p* \
+			compile.exp=compile/2* \
+			compile.exp=compile/\[Babcdfilmsvx\]* \
+			compile.exp=compile/9* \
+			compile.exp=compile/\[013-8AC-Zeghjknoqrtuwyz\]* \
+			dg-torture.exp=torture/p* \
+			dg-torture.exp=torture/\[0-9A-Za-oq-z\]* \
+			vect.exp \
+			atomic.exp \
+			builtins.exp \
+			ubsan.exp \
+			struct-layout-1.exp \
+        		stackalign.exp \
+        		dg-exp.exp \
+        		ieee.exp \
+			asan.exp,tsan.exp \
+			lto.exp \
+			cilk-plus.exp \
+			noncompile.exp,dfp.exp,graphite.exp \
+			guality.exp,unsorted.exp \
+			debug.exp,tree-ssa.exp \
+			pch.exp,compat.exp,cpp.exp \
+			tls.exp,abi-x86_64.exp
 lang_opt_files=@lang_opt_files@ $(srcdir)/c-family/c.opt $(srcdir)/common.opt
 lang_specs_files=@lang_specs_files@
 lang_tree_files=@lang_tree_files@
@@ -3639,7 +3666,8 @@ check_p_vars=$(check_$(check_p_tool)_par
 check_p_subno=$(word 2,$(subst _, ,$*))
 check_p_comma=,
 check_p_subwork=$(subst $(check_p_comma), ,$(if $(check_p_subno),$(word $(check_p_subno),$(check_p_vars))))
-check_p_numbers=1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
+check_p_numbers=1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 \
+		21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
 check_p_subdir=$(subst _,,$*)
 check_p_subdirs=$(wordlist 1,$(words $(check_$*_parallelize)),$(check_p_numbers))
 
Index: gcc/cp/Make-lang.in
===================================================================
--- gcc/cp/Make-lang.in	(revision 215017)
+++ gcc/cp/Make-lang.in	(working copy)
@@ -156,10 +156,30 @@ check-c++-subtargets : check-g++-subtarg
 lang_checks += check-g++
 lang_checks_parallelized += check-g++
 # For description see comment above check_gcc_parallelize in gcc/Makefile.in.
-check_g++_parallelize = old-deja.exp \
-	dg.exp=g++.dg/[0-9A-Za-bd-su-z]* \
-	dg.exp=g++.dg/[ct]* \
-	dg.exp=c-c++-common/*,dg-torture.exp
+check_g++_parallelize = dg.exp=g++.dg/\[ac\]* \
+	dg.exp=g++.dg/\[eint\]* \
+	dg.exp=g++.dg/\[Wflmru\]* \
+	dg.exp=g++.dg/\[dsv\]* \
+	dg.exp=g++.dg/p* \
+	dg.exp=g++.dg/\[0-9A-VXYZbghjkoqwxyz\]* \
+	dg.exp=c-c++-common/* \
+	old-deja.exp=g++.old-deja/g++.p* \
+	old-deja.exp=g++.old-deja/g++.\[lm\]* \
+	old-deja.exp=g++.old-deja/g++.o* \
+	old-deja.exp=g++.old-deja/g++.b* \
+	old-deja.exp=g++.old-deja/g++.\[0-9A-Zac-iknq-z\]* \
+	old-deja.exp=g++.old-deja/g++.j* \
+	dg-torture.exp=torture/p* \
+	dg-torture.exp=torture/\[0-9A-Za-oq-z\]* \
+	asan.exp \
+	ubsan.exp \
+	cilk-plus.exp \
+	struct-layout-1.exp \
+	debug.exp \
+	dwarf2.exp \
+	lto.exp \
+	gomp.exp \
+	tsan.exp
 
 #
 # Install hooks:

Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]