RE: [PATCH] RE: gcc parallel make check

["VandeVondele Joost" <joost dot vandevondele at mat dot ethz dot ch>]

Attached is an extended version of the patch, it brings a 100% improvement in make -j32 -k check-gcc (down from 20min to <10min) by modification of check_gcc_parallelize.

It includes one non-trivial part, namely a split of the target exps. They are now all split using a common choice (based on i386), which I believe is reasonable as it is the target with most tests, and the patterns will be somewhat similar for other targets (e.g. split of p(rxxx)). The implementation of this in the makefile uses an odd looking technique to substitute spaces with commas in a variable, if this can be done more elegantly, I'm happy to make the change.

Bootstrap and testing revealed one issue, i386.exp hard-codes a loop for the testcase 'vect-args.c' in order to test 10 different combinations of options. With the current split (i.e. target x4) this test will thus be executed 4 times. There are two easy options

1) keep the current setup, overhead is small
2) keep the .exp file simple and just replicate this test 10x 

I've selected 1), but I can update a patch with 2). Ideally dg-options in the testcase file itself could be repeated, but I haven't found an example of this. 

The script now includes sorting and compression of the ranges, and an additional sanity check on the input, i.e. that file names start with [0-9A-Za-z]. Some (few) files seem to start with _ or # (in ./gcc.dg/cpp/).

I'll follow up with a separate patch to improve check_g++_parallelize.

Full 'make -j k32 check' is now dominated by libstdc++ testing, which contains single goals that run ~1100s (e.g. regex related tests). These uses a slightly different syntax (see gcc/libstdc++-v3/testsuite/Makefile.am) and I'm not yet sure how to deal with the .am files.

current patch OK for trunk ?

Joost

Attachment: patch-speedup-checkfortran-v05.CL
Description: patch-speedup-checkfortran-v05.CL

Index: contrib/generate_tcl_patterns.sh
===================================================================
--- contrib/generate_tcl_patterns.sh	(revision 0)
+++ contrib/generate_tcl_patterns.sh	(revision 0)
@@ -0,0 +1,114 @@
+#! /bin/sh
+
+#
+# based on a list of filenames as input, starting with [0-9A-Za-z],
+# generate regexps that match subsets trying to not exceed a
+# 'maxcount' parameter. Most useful to generate the
+# check_LANG_parallelize assignments needed to split
+# testsuite directories, defining prefix appropriately.
+#
+# Example usage:
+#   cd gcc/gcc/testsuite/gfortran.dg
+#   ls -1 | ../../../contrib/generate_tcl_patterns.sh 300 "dg.exp=gfortran.dg/"
+#
+# the first parameter is the maximum number of files.
+# the second parameter the prefix used for printing.
+#
+
+# Copyright (C) 2014 Free Software Foundation
+# Contributed by Joost VandeVondele <Joost.VandeVondele@mat.ethz.ch>
+#
+# This file is part of GCC.
+#
+# GCC is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# GCC is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GCC; see the file COPYING.  If not, write to
+# the Free Software Foundation, 51 Franklin Street, Fifth Floor,
+# Boston, MA 02110-1301, USA.
+
+gawk -v maxcount=$1 -v prefix=$2 '
+BEGIN{
+  # list of allowed starting chars for a file name in a dir to split
+  achars="0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+  ranget="11111111112222222222222222222222222233333333333333333333333333"
+}
+{
+  if (index(achars,substr($1,1,1))==0){
+     print "file : " $1 " does not start with an allowed character."
+     _assert_exit = 1
+     exit 1
+  }
+  nfiles++ ; files[nfiles]=$1
+}
+END{
+  if (_assert_exit) exit 1
+  for(i=1; i<=length(achars); i++) count[substr(achars,i,1)]=0
+  for(i=1; i<=nfiles; i++) {
+     if (length(files[i]>0)) { count[substr(files[i],1,1)]++ }
+  };
+  asort(count,ordered)
+  countsingle=0
+  groups=0
+  label=""
+  for(i=length(achars);i>=1;i--) {
+    countsingle=countsingle+ordered[i] 
+    for(j=1;j<=length(achars);j++) {
+       if(count[substr(achars,j,1)]==ordered[i]) found=substr(achars,j,1)
+    }
+    count[found]=-1
+    label=label found
+    if(i==1) { val=maxcount+1 } else { val=ordered[i-1] }
+    if(countsingle+val>maxcount) {
+      subset[label]=countsingle
+      print "Adding label: ", label, "matching files:" countsingle
+      groups++
+      countsingle=0
+      label=""
+    }
+  }
+  print "patterns:"
+  asort(subset,ordered)
+  for(i=groups;i>=1;i--) {
+    for(j in subset){
+      if(subset[j]==ordered[i]) found=j
+    }
+    subset[found]=-1
+    if (length(found)==1) {
+       printf("%s%s* \\\n",prefix,found)
+    } else {
+       sortandcompress()
+       printf("%s\\[%s\\]* \\\n",prefix,found)
+    }
+  }
+}
+function sortandcompress(i,n,tmp,bestj)
+{
+  n=length(found)
+  for(i=1; i<=n; i++) tmp[i]=substr(found,i,1) 
+  asort(tmp)
+  for(i=1;i<=n;i++){
+    ipos=index(achars,tmp[i])
+    for(j=i;j<=n;j++){
+      jpos=index(achars,tmp[j])
+      if (jpos-ipos==j-i && substr(ranget,ipos,1)==substr(ranget,jpos,1)) bestj=j
+    }
+    if (bestj-i>3) {
+      tmp[i+1]="-" 
+      for(j=i+2;j<bestj;j++) tmp[j]=" "
+    }
+    i=bestj 
+  }
+  found=""
+  for(i=1; i<=n; i++) found=found tmp[i]
+  gsub(/ /,"",found)
+}
+'

Property changes on: contrib/generate_tcl_patterns.sh
___________________________________________________________________
Added: svn:executable
   + *

Index: gcc/fortran/Make-lang.in
===================================================================
--- gcc/fortran/Make-lang.in	(revision 215008)
+++ gcc/fortran/Make-lang.in	(working copy)
@@ -168,12 +168,22 @@ check-fortran-subtargets : check-gfortra
 lang_checks += check-gfortran
 lang_checks_parallelized += check-gfortran
 # For description see comment above check_gcc_parallelize in gcc/Makefile.in.
-check_gfortran_parallelize = dg.exp=gfortran.dg/\[adAD\]* \
-			     dg.exp=gfortran.dg/\[bcBC\]* \
-			     dg.exp=gfortran.dg/\[nopNOP\]* \
-			     dg.exp=gfortran.dg/\[isuvISUV\]* \
-			     dg.exp=gfortran.dg/\[efhkqrxzEFHKQRXZ\]* \
-			     dg.exp=gfortran.dg/\[0-9gjlmtwyGJLMTWY\]*
+check_gfortran_parallelize = execute.exp \
+			dg.exp=gfortran.dg/p* \
+			dg.exp=gfortran.dg/c* \
+			dg.exp=gfortran.dg/a* \
+			dg.exp=gfortran.dg/i* \
+			dg.exp=gfortran.dg/\[glow\]* \
+			dg.exp=gfortran.dg/\[mu\]* \
+			dg.exp=gfortran.dg/d* \
+			dg.exp=gfortran.dg/s* \
+			dg.exp=gfortran.dg/b* \
+			dg.exp=gfortran.dg/t* \
+			dg.exp=gfortran.dg/f* \
+			dg.exp=gfortran.dg/e* \
+			dg.exp=gfortran.dg/r* \
+			dg.exp=gfortran.dg/n* \
+			dg.exp=gfortran.dg/\[0-9A-Zhjkqvxyz\]*
 
 # GFORTRAN documentation.
 GFORTRAN_TEXI = \
Index: gcc/Makefile.in
===================================================================
--- gcc/Makefile.in	(revision 215008)
+++ gcc/Makefile.in	(working copy)
@@ -513,11 +513,17 @@ xm_include_list=@xm_include_list@
 xm_defines=@xm_defines@
 lang_checks=
 lang_checks_parallelized=
-dg_target_exps:=aarch64.exp,alpha.exp,arm.exp,avr.exp,bfin.exp,cris.exp
-dg_target_exps:=$(dg_target_exps),epiphany.exp,frv.exp,i386.exp,ia64.exp
-dg_target_exps:=$(dg_target_exps),m68k.exp,microblaze.exp,mips.exp,powerpc.exp
-dg_target_exps:=$(dg_target_exps),rx.exp,s390.exp,sh.exp,sparc.exp,spu.exp
-dg_target_exps:=$(dg_target_exps),tic6x.exp,xstormy16.exp
+dg_target_exps:=aarch64 alpha arm avr bfin cris epiphany frv i386 ia64 
+dg_target_exps:=$(dg_target_exps) m68k microblaze mips powerpc rx s390
+dg_target_exps:=$(dg_target_exps) sh sparc spu tic6x xstormy16
+# also parallelize the target exps, using one common choice (i386) of split
+null:=
+space:=$(null) $(null)
+comma:=$(null),$(null)
+dg_target_exps_p1:=$(subst $(space),$(comma),$(foreach target,$(dg_target_exps),$(target).exp=$(target)/a*))
+dg_target_exps_p2:=$(subst $(space),$(comma),$(foreach target,$(dg_target_exps),$(target).exp=$(target)/\[fs\]*))
+dg_target_exps_p3:=$(subst $(space),$(comma),$(foreach target,$(dg_target_exps),$(target).exp=$(target)/\[0-9A-Zbcdeg-oqrt-z\]*))
+dg_target_exps_p4:=$(subst $(space),$(comma),$(foreach target,$(dg_target_exps),$(target).exp=$(target)/p*))
 # This lists a couple of test files that take most time during check-gcc.
 # When doing parallelized check-gcc, these can run in parallel with the
 # remaining tests.  Each word in this variable stands for work for one
@@ -526,21 +532,42 @@ dg_target_exps:=$(dg_target_exps),tic6x.
 # should be run in the same runtest invocation (usually if they aren't
 # very long running, but still should be split of from the check-parallel-$lang
 # remaining tests runtest invocation), they should be concatenated with commas.
+# At most $(check_p_numbers) goals should be created.
 # Note that [a-zA-Z] wildcards need to have []s prefixed with \ (needed
 # by tcl) and as the *.exp arguments are mached both as is and with
 # */ prefixed to it in runtest_file_p, it is usually desirable to include
 # a subdirectory name.
-check_gcc_parallelize=execute.exp=execute/2* \
-		      execute.exp=execute/\[013-9a-fA-F\]* \
-		      execute.exp=execute/\[pP\]*,dg.exp \
-		      execute.exp=execute/\[g-oq-zG-OQ-Z\]*,compile.exp=compile/2* \
-		      compile.exp=compile/\[9pP\]*,builtins.exp \
-		      compile.exp=compile/\[013-8a-oq-zA-OQ-Z\]* \
-		      dg-torture.exp,ieee.exp \
-		      vect.exp,unsorted.exp \
-		      guality.exp \
-		      struct-layout-1.exp,stackalign.exp \
-		      $(dg_target_exps)
+check_gcc_parallelize=$(dg_target_exps_p1) \
+			$(dg_target_exps_p2) \
+			$(dg_target_exps_p3) \
+			$(dg_target_exps_p4) \
+			execute.exp=execute/2* \
+			execute.exp=execute/p* \
+			execute.exp=execute/9* \
+			execute.exp=execute/\[013-8A-Za-oq-z\]* \
+			compile.exp=compile/p* \
+			compile.exp=compile/2* \
+			compile.exp=compile/\[Babcdfilmsvx\]* \
+			compile.exp=compile/9* \
+			compile.exp=compile/\[013-8AC-Zeghjknoqrtuwyz\]* \
+			dg-torture.exp=torture/p* \
+			dg-torture.exp=torture/\[0-9A-Za-oq-z\]* \
+			vect.exp \
+			atomic.exp \
+			builtins.exp \
+			ubsan.exp \
+			struct-layout-1.exp \
+        		stackalign.exp \
+        		dg-exp.exp \
+        		ieee.exp \
+			asan.exp,tsan.exp \
+			lto.exp \
+			cilk-plus.exp \
+			noncompile.exp,dfp.exp,graphite.exp \
+			guality.exp,unsorted.exp \
+			debug.exp,tree-ssa.exp \
+			pch.exp,compat.exp,cpp.exp \
+			tls.exp,abi-x86_64.exp
 lang_opt_files=@lang_opt_files@ $(srcdir)/c-family/c.opt $(srcdir)/common.opt
 lang_specs_files=@lang_specs_files@
 lang_tree_files=@lang_tree_files@
@@ -3639,7 +3666,8 @@ check_p_vars=$(check_$(check_p_tool)_par
 check_p_subno=$(word 2,$(subst _, ,$*))
 check_p_comma=,
 check_p_subwork=$(subst $(check_p_comma), ,$(if $(check_p_subno),$(word $(check_p_subno),$(check_p_vars))))
-check_p_numbers=1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
+check_p_numbers=1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 \
+		21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40
 check_p_subdir=$(subst _,,$*)
 check_p_subdirs=$(wordlist 1,$(words $(check_$*_parallelize)),$(check_p_numbers))