This is the mail archive of the gcc@gcc.gnu.org mailing list for the GCC project.
Index Nav: | [Date Index] [Subject Index] [Author Index] [Thread Index] | |
---|---|---|
Message Nav: | [Date Prev] [Date Next] | [Thread Prev] [Thread Next] |
Other format: | [Raw text] |
> a newer patch (v8) I'll send soon attached with updated changelog. Compared to the previously posted v6, only the libstdc++-v3/testsuite/Makefile.am has been refined to split a little more the e*/* pattern, and two quickly running goal have been merged, in addition to fixing the pre-exisiting error in some of the patterns in that file. Checked comparing testsuite results before after. Obviously, if Jakub's patch can be made to work around the testsuite special cases, I believe it should be superior. If not, the attached patch is working as far as I can tell, and provides a significant improvement over current trunk. Joost
Attachment:
patch-speedup-check-v08-CL.txt
Description: patch-speedup-check-v08-CL.txt
Index: libstdc++-v3/testsuite/Makefile.in =================================================================== --- libstdc++-v3/testsuite/Makefile.in (revision 215147) +++ libstdc++-v3/testsuite/Makefile.in (working copy) @@ -301,7 +301,7 @@ lists_of_files = \ extract_symvers = $(glibcxx_builddir)/scripts/extract_symvers baseline_subdir := $(shell $(CXX) $(baseline_subdir_switch)) -check_DEJAGNU_normal_targets = $(patsubst %,check-DEJAGNUnormal%,0 1 2 3 4 5 6 7 8 9 10) +check_DEJAGNU_normal_targets = $(patsubst %,check-DEJAGNUnormal%,0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15) # Runs the testsuite, but in compile only mode. # Can be used to test sources with non-GNU FE's at various warning @@ -562,7 +562,7 @@ check-DEJAGNU $(check_DEJAGNU_normal_tar if [ -z "$*$(filter-out --target_board=%, $(RUNTESTFLAGS))" ] \ && [ "$(filter -j, $(MFLAGS))" = "-j" ]; then \ $(MAKE) $(AM_MAKEFLAGS) $(check_DEJAGNU_normal_targets); \ - for idx in 0 1 2 3 4 5 6 7 8 9 10; do \ + for idx in 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do \ mv -f normal$$idx/libstdc++.sum normal$$idx/libstdc++.sum.sep; \ mv -f normal$$idx/libstdc++.log normal$$idx/libstdc++.log.sep; \ done; \ @@ -589,25 +589,35 @@ check-DEJAGNU $(check_DEJAGNU_normal_tar fi; \ dirs="`cd $$srcdir; echo [013-9][0-9]_*/*`";; \ normal1) \ - dirs="`cd $$srcdir; echo [ab]* de* [ep]*/*`";; \ + dirs="`cd $$srcdir; echo experimental/* ext/[a-m]*`";; \ normal2) \ - dirs="`cd $$srcdir; echo 2[01]_*/*`";; \ + dirs="`cd $$srcdir; echo 28_*/a*`";; \ normal3) \ - dirs="`cd $$srcdir; echo 22_*/*`";; \ + dirs="`cd $$srcdir; echo 23_*/[lu]*`";; \ normal4) \ - dirs="`cd $$srcdir; echo 23_*/[a-km-tw-z]*`";; \ + dirs="`cd $$srcdir; echo 2[459]_*/*`";; \ normal5) \ - dirs="`cd $$srcdir; echo 23_*/[luv]*`";; \ + dirs="`cd $$srcdir; echo 2[01]_*/*`";; \ normal6) \ - dirs="`cd $$srcdir; echo 2[459]_*/*`";; \ + dirs="`cd $$srcdir; echo 23_*/[m-tw-z]*`";; \ normal7) \ - dirs="`cd $$srcdir; echo 26_*/* 28_*/[c-z]*`";; \ + dirs="`cd $$srcdir; echo 26_*/*`";; \ normal8) \ dirs="`cd $$srcdir; echo 27_*/*`";; \ normal9) \ - dirs="`cd $$srcdir; echo 28_*/[ab]*`";; \ + dirs="`cd $$srcdir; echo 22_*/*`";; \ normal10) \ dirs="`cd $$srcdir; echo t*/*`";; \ + normal11) \ + dirs="`cd $$srcdir; echo 28_*/b*`";; \ + normal12) \ + dirs="`cd $$srcdir; echo 28_*/[c-z]*`";; \ + normal13) \ + dirs="`cd $$srcdir; echo ext/[n-z]*`";; \ + normal14) \ + dirs="`cd $$srcdir; echo de*/* p*/* [ab]*/* 23_*/v*`";; \ + normal15) \ + dirs="`cd $$srcdir; echo 23_*/[a-k]*`";; \ esac; \ if [ -n "$*" ]; then cd "$*"; fi; \ if $(SHELL) -c "$$runtest --version" > /dev/null 2>&1; then \ Index: libstdc++-v3/testsuite/Makefile.am =================================================================== --- libstdc++-v3/testsuite/Makefile.am (revision 215147) +++ libstdc++-v3/testsuite/Makefile.am (working copy) @@ -101,7 +101,7 @@ new-abi-baseline: @test ! -f $*/site.exp || mv $*/site.exp $*/site.bak @mv $*/site.exp.tmp $*/site.exp -check_DEJAGNU_normal_targets = $(patsubst %,check-DEJAGNUnormal%,0 1 2 3 4 5 6 7 8 9 10) +check_DEJAGNU_normal_targets = $(patsubst %,check-DEJAGNUnormal%,0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15) $(check_DEJAGNU_normal_targets): check-DEJAGNUnormal%: normal%/site.exp # Run the testsuite in normal mode. @@ -111,7 +111,7 @@ check-DEJAGNU $(check_DEJAGNU_normal_tar if [ -z "$*$(filter-out --target_board=%, $(RUNTESTFLAGS))" ] \ && [ "$(filter -j, $(MFLAGS))" = "-j" ]; then \ $(MAKE) $(AM_MAKEFLAGS) $(check_DEJAGNU_normal_targets); \ - for idx in 0 1 2 3 4 5 6 7 8 9 10; do \ + for idx in 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15; do \ mv -f normal$$idx/libstdc++.sum normal$$idx/libstdc++.sum.sep; \ mv -f normal$$idx/libstdc++.log normal$$idx/libstdc++.log.sep; \ done; \ @@ -138,25 +138,35 @@ check-DEJAGNU $(check_DEJAGNU_normal_tar fi; \ dirs="`cd $$srcdir; echo [013-9][0-9]_*/*`";; \ normal1) \ - dirs="`cd $$srcdir; echo [ab]* de* [ep]*/*`";; \ + dirs="`cd $$srcdir; echo experimental/* ext/[a-m]*`";; \ normal2) \ - dirs="`cd $$srcdir; echo 2[01]_*/*`";; \ + dirs="`cd $$srcdir; echo 28_*/a*`";; \ normal3) \ - dirs="`cd $$srcdir; echo 22_*/*`";; \ + dirs="`cd $$srcdir; echo 23_*/[lu]*`";; \ normal4) \ - dirs="`cd $$srcdir; echo 23_*/[a-km-tw-z]*`";; \ + dirs="`cd $$srcdir; echo 2[459]_*/*`";; \ normal5) \ - dirs="`cd $$srcdir; echo 23_*/[luv]*`";; \ + dirs="`cd $$srcdir; echo 2[01]_*/*`";; \ normal6) \ - dirs="`cd $$srcdir; echo 2[459]_*/*`";; \ + dirs="`cd $$srcdir; echo 23_*/[m-tw-z]*`";; \ normal7) \ - dirs="`cd $$srcdir; echo 26_*/* 28_*/[c-z]*`";; \ + dirs="`cd $$srcdir; echo 26_*/*`";; \ normal8) \ dirs="`cd $$srcdir; echo 27_*/*`";; \ normal9) \ - dirs="`cd $$srcdir; echo 28_*/[ab]*`";; \ + dirs="`cd $$srcdir; echo 22_*/*`";; \ normal10) \ dirs="`cd $$srcdir; echo t*/*`";; \ + normal11) \ + dirs="`cd $$srcdir; echo 28_*/b*`";; \ + normal12) \ + dirs="`cd $$srcdir; echo 28_*/[c-z]*`";; \ + normal13) \ + dirs="`cd $$srcdir; echo ext/[n-z]*`";; \ + normal14) \ + dirs="`cd $$srcdir; echo de*/* p*/* [ab]*/* 23_*/v*`";; \ + normal15) \ + dirs="`cd $$srcdir; echo 23_*/[a-k]*`";; \ esac; \ if [ -n "$*" ]; then cd "$*"; fi; \ if $(SHELL) -c "$$runtest --version" > /dev/null 2>&1; then \ Index: contrib/generate_tcl_patterns.sh =================================================================== --- contrib/generate_tcl_patterns.sh (revision 0) +++ contrib/generate_tcl_patterns.sh (revision 0) @@ -0,0 +1,114 @@ +#! /bin/sh + +# +# based on a list of filenames as input, starting with [0-9A-Za-z], +# generate regexps that match subsets trying to not exceed a +# 'maxcount' parameter. Most useful to generate the +# check_LANG_parallelize assignments needed to split +# testsuite directories, defining prefix appropriately. +# +# Example usage: +# cd gcc/gcc/testsuite/gfortran.dg +# ls -1 | ../../../contrib/generate_tcl_patterns.sh 300 "dg.exp=gfortran.dg/" +# +# the first parameter is the maximum number of files. +# the second parameter the prefix used for printing. +# + +# Copyright (C) 2014 Free Software Foundation +# Contributed by Joost VandeVondele <Joost.VandeVondele@mat.ethz.ch> +# +# This file is part of GCC. +# +# GCC is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# GCC is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with GCC; see the file COPYING. If not, write to +# the Free Software Foundation, 51 Franklin Street, Fifth Floor, +# Boston, MA 02110-1301, USA. + +gawk -v maxcount=$1 -v prefix=$2 ' +BEGIN{ + # list of allowed starting chars for a file name in a dir to split + achars="0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + ranget="11111111112222222222222222222222222233333333333333333333333333" +} +{ + if (index(achars,substr($1,1,1))==0){ + print "file : " $1 " does not start with an allowed character." + _assert_exit = 1 + exit 1 + } + nfiles++ ; files[nfiles]=$1 +} +END{ + if (_assert_exit) exit 1 + for(i=1; i<=length(achars); i++) count[substr(achars,i,1)]=0 + for(i=1; i<=nfiles; i++) { + if (length(files[i]>0)) { count[substr(files[i],1,1)]++ } + }; + asort(count,ordered) + countsingle=0 + groups=0 + label="" + for(i=length(achars);i>=1;i--) { + countsingle=countsingle+ordered[i] + for(j=1;j<=length(achars);j++) { + if(count[substr(achars,j,1)]==ordered[i]) found=substr(achars,j,1) + } + count[found]=-1 + label=label found + if(i==1) { val=maxcount+1 } else { val=ordered[i-1] } + if(countsingle+val>maxcount) { + subset[label]=countsingle + print "Adding label: ", label, "matching files:" countsingle + groups++ + countsingle=0 + label="" + } + } + print "patterns:" + asort(subset,ordered) + for(i=groups;i>=1;i--) { + for(j in subset){ + if(subset[j]==ordered[i]) found=j + } + subset[found]=-1 + if (length(found)==1) { + printf("%s%s* \\\n",prefix,found) + } else { + sortandcompress() + printf("%s\\[%s\\]* \\\n",prefix,found) + } + } +} +function sortandcompress(i,n,tmp,bestj) +{ + n=length(found) + for(i=1; i<=n; i++) tmp[i]=substr(found,i,1) + asort(tmp) + for(i=1;i<=n;i++){ + ipos=index(achars,tmp[i]) + for(j=i;j<=n;j++){ + jpos=index(achars,tmp[j]) + if (jpos-ipos==j-i && substr(ranget,ipos,1)==substr(ranget,jpos,1)) bestj=j + } + if (bestj-i>3) { + tmp[i+1]="-" + for(j=i+2;j<bestj;j++) tmp[j]=" " + } + i=bestj + } + found="" + for(i=1; i<=n; i++) found=found tmp[i] + gsub(/ /,"",found) +} +' Property changes on: contrib/generate_tcl_patterns.sh ___________________________________________________________________ Added: svn:executable + * Index: gcc/cp/Make-lang.in =================================================================== --- gcc/cp/Make-lang.in (revision 215147) +++ gcc/cp/Make-lang.in (working copy) @@ -156,10 +156,30 @@ check-c++-subtargets : check-g++-subtarg lang_checks += check-g++ lang_checks_parallelized += check-g++ # For description see comment above check_gcc_parallelize in gcc/Makefile.in. -check_g++_parallelize = old-deja.exp \ - dg.exp=g++.dg/[0-9A-Za-bd-su-z]* \ - dg.exp=g++.dg/[ct]* \ - dg.exp=c-c++-common/*,dg-torture.exp +check_g++_parallelize = dg.exp=g++.dg/\[ac\]* \ + dg.exp=g++.dg/\[eint\]* \ + dg.exp=g++.dg/\[Wflmru\]* \ + dg.exp=g++.dg/\[dsv\]* \ + dg.exp=g++.dg/p* \ + dg.exp=g++.dg/\[0-9A-VXYZbghjkoqwxyz\]* \ + dg.exp=c-c++-common/* \ + old-deja.exp=g++.old-deja/g++.p* \ + old-deja.exp=g++.old-deja/g++.\[lm\]* \ + old-deja.exp=g++.old-deja/g++.o* \ + old-deja.exp=g++.old-deja/g++.b* \ + old-deja.exp=g++.old-deja/g++.\[0-9A-Zac-iknq-z\]* \ + old-deja.exp=g++.old-deja/g++.j* \ + dg-torture.exp=torture/p* \ + dg-torture.exp=torture/\[0-9A-Za-oq-z\]* \ + asan.exp \ + ubsan.exp \ + cilk-plus.exp \ + struct-layout-1.exp \ + debug.exp \ + dwarf2.exp \ + lto.exp \ + gomp.exp \ + tsan.exp # # Install hooks: Index: gcc/fortran/Make-lang.in =================================================================== --- gcc/fortran/Make-lang.in (revision 215147) +++ gcc/fortran/Make-lang.in (working copy) @@ -168,12 +168,22 @@ check-fortran-subtargets : check-gfortra lang_checks += check-gfortran lang_checks_parallelized += check-gfortran # For description see comment above check_gcc_parallelize in gcc/Makefile.in. -check_gfortran_parallelize = dg.exp=gfortran.dg/\[adAD\]* \ - dg.exp=gfortran.dg/\[bcBC\]* \ - dg.exp=gfortran.dg/\[nopNOP\]* \ - dg.exp=gfortran.dg/\[isuvISUV\]* \ - dg.exp=gfortran.dg/\[efhkqrxzEFHKQRXZ\]* \ - dg.exp=gfortran.dg/\[0-9gjlmtwyGJLMTWY\]* +check_gfortran_parallelize = execute.exp \ + dg.exp=gfortran.dg/p* \ + dg.exp=gfortran.dg/c* \ + dg.exp=gfortran.dg/a* \ + dg.exp=gfortran.dg/i* \ + dg.exp=gfortran.dg/\[glow\]* \ + dg.exp=gfortran.dg/\[mu\]* \ + dg.exp=gfortran.dg/d* \ + dg.exp=gfortran.dg/s* \ + dg.exp=gfortran.dg/b* \ + dg.exp=gfortran.dg/t* \ + dg.exp=gfortran.dg/f* \ + dg.exp=gfortran.dg/e* \ + dg.exp=gfortran.dg/r* \ + dg.exp=gfortran.dg/n* \ + dg.exp=gfortran.dg/\[0-9A-Zhjkqvxyz\]* # GFORTRAN documentation. GFORTRAN_TEXI = \ Index: gcc/Makefile.in =================================================================== --- gcc/Makefile.in (revision 215147) +++ gcc/Makefile.in (working copy) @@ -513,11 +513,17 @@ xm_include_list=@xm_include_list@ xm_defines=@xm_defines@ lang_checks= lang_checks_parallelized= -dg_target_exps:=aarch64.exp,alpha.exp,arm.exp,avr.exp,bfin.exp,cris.exp -dg_target_exps:=$(dg_target_exps),epiphany.exp,frv.exp,i386.exp,ia64.exp -dg_target_exps:=$(dg_target_exps),m68k.exp,microblaze.exp,mips.exp,powerpc.exp -dg_target_exps:=$(dg_target_exps),rx.exp,s390.exp,sh.exp,sparc.exp,spu.exp -dg_target_exps:=$(dg_target_exps),tic6x.exp,xstormy16.exp +dg_target_exps:=aarch64 alpha arm avr bfin cris epiphany frv i386 ia64 +dg_target_exps:=$(dg_target_exps) m68k microblaze mips powerpc rx s390 +dg_target_exps:=$(dg_target_exps) sh sparc spu tic6x xstormy16 +# also parallelize the target exps, using one common choice (i386) of split +null:= +space:=$(null) $(null) +comma:=$(null),$(null) +dg_target_exps_p1:=$(subst $(space),$(comma),$(foreach target,$(dg_target_exps),$(target).exp=$(target)/a*)) +dg_target_exps_p2:=$(subst $(space),$(comma),$(foreach target,$(dg_target_exps),$(target).exp=$(target)/\[fs\]*)) +dg_target_exps_p3:=$(subst $(space),$(comma),$(foreach target,$(dg_target_exps),$(target).exp=$(target)/\[0-9A-Zbcdeg-oqrt-z\]*)) +dg_target_exps_p4:=$(subst $(space),$(comma),$(foreach target,$(dg_target_exps),$(target).exp=$(target)/p*)) # This lists a couple of test files that take most time during check-gcc. # When doing parallelized check-gcc, these can run in parallel with the # remaining tests. Each word in this variable stands for work for one @@ -526,21 +532,42 @@ dg_target_exps:=$(dg_target_exps),tic6x. # should be run in the same runtest invocation (usually if they aren't # very long running, but still should be split of from the check-parallel-$lang # remaining tests runtest invocation), they should be concatenated with commas. +# At most $(check_p_numbers) goals should be created. # Note that [a-zA-Z] wildcards need to have []s prefixed with \ (needed # by tcl) and as the *.exp arguments are mached both as is and with # */ prefixed to it in runtest_file_p, it is usually desirable to include # a subdirectory name. -check_gcc_parallelize=execute.exp=execute/2* \ - execute.exp=execute/\[013-9a-fA-F\]* \ - execute.exp=execute/\[pP\]*,dg.exp \ - execute.exp=execute/\[g-oq-zG-OQ-Z\]*,compile.exp=compile/2* \ - compile.exp=compile/\[9pP\]*,builtins.exp \ - compile.exp=compile/\[013-8a-oq-zA-OQ-Z\]* \ - dg-torture.exp,ieee.exp \ - vect.exp,unsorted.exp \ - guality.exp \ - struct-layout-1.exp,stackalign.exp \ - $(dg_target_exps) +check_gcc_parallelize=$(dg_target_exps_p1) \ + $(dg_target_exps_p2) \ + $(dg_target_exps_p3) \ + $(dg_target_exps_p4) \ + execute.exp=execute/2* \ + execute.exp=execute/p* \ + execute.exp=execute/9* \ + execute.exp=execute/\[013-8A-Za-oq-z\]* \ + compile.exp=compile/p* \ + compile.exp=compile/2* \ + compile.exp=compile/\[Babcdfilmsvx\]* \ + compile.exp=compile/9* \ + compile.exp=compile/\[013-8AC-Zeghjknoqrtuwyz\]* \ + dg-torture.exp=torture/p* \ + dg-torture.exp=torture/\[0-9A-Za-oq-z\]* \ + vect.exp \ + atomic.exp \ + builtins.exp \ + ubsan.exp \ + struct-layout-1.exp \ + stackalign.exp \ + dg-exp.exp \ + ieee.exp \ + asan.exp,tsan.exp \ + lto.exp \ + cilk-plus.exp \ + noncompile.exp,dfp.exp,graphite.exp \ + guality.exp,unsorted.exp \ + debug.exp,tree-ssa.exp \ + pch.exp,compat.exp,cpp.exp \ + tls.exp,abi-x86_64.exp lang_opt_files=@lang_opt_files@ $(srcdir)/c-family/c.opt $(srcdir)/common.opt lang_specs_files=@lang_specs_files@ lang_tree_files=@lang_tree_files@ @@ -3639,7 +3666,8 @@ check_p_vars=$(check_$(check_p_tool)_par check_p_subno=$(word 2,$(subst _, ,$*)) check_p_comma=, check_p_subwork=$(subst $(check_p_comma), ,$(if $(check_p_subno),$(word $(check_p_subno),$(check_p_vars)))) -check_p_numbers=1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 +check_p_numbers=1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 \ + 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 check_p_subdir=$(subst _,,$*) check_p_subdirs=$(wordlist 1,$(words $(check_$*_parallelize)),$(check_p_numbers))
Index Nav: | [Date Index] [Subject Index] [Author Index] [Thread Index] | |
---|---|---|
Message Nav: | [Date Prev] [Date Next] | [Thread Prev] [Thread Next] |