[libsanitizer] Enable libsanitizer on Solaris (PR sanitizer/80953)

[Rainer Orth <ro at CeBiTec dot Uni-Bielefeld dot DE>]

Now that the Solaris sanitizer port (asan and ubsan only for now) has
been upstream for a while and some last issues fixed, I'd finally like
to enable the sanitizers on both sparc and x86.

The asan port is 32-bit only for now (on sparc because 64-bit
Solaris/SPARC uses the full address space with a large virtual address
hole in the middle whose exact location is machine-dependent and not
easily determined at runtime; on x86 the situation was the same before
Solaris 11.4).

The patch itself is pretty trivial, mostly duplicating what Linux does.

The only parts I need approval for are

* The SPARC implementation of TARGET_ASAN_SHADOW_OFFSET, so far only
  tested on 32-bit Solaris/SPARC.

* The change to c-c++-common/asan/alloca_loop_unpoisoning.c is necessary
  since Solaris doesn't have an alloca prototype by default:

FAIL: c-c++-common/asan/alloca_loop_unpoisoning.c   -O0  (test for excess errors)
WARNING: c-c++-common/asan/alloca_loop_unpoisoning.c   -O0  compilation failed to produce executable

Excess errors:
/vol/gcc/src/hg/trunk/solaris-asan/gcc/testsuite/c-c++-common/asan/alloca_loop_unpoisoning.c:19:3: warning: implicit declaration of function 'alloca' [-Wimplicit-function-declaration]
/vol/gcc/src/hg/trunk/solaris-asan/gcc/testsuite/c-c++-common/asan/alloca_loop_unpoisoning.c:19:3: warning: incompatible implicit declaration of built-in function 'alloca'

  This isn't an issue on Linux where glibc <stdlib.h> includes
  <alloca.h> which defines alloca as __builtin_alloca.

* I'm only enabling libsanitizer on Solaris 11 right now: on Solaris 10
  there are a couple of libsanitizer compilation failures and I don't
  know if I have much time or inclination to fix them only to rip the
  changes out again in GCC 10.

Tested on i386-pc-solaris2.11 and sparc-sun-solaris2.11.  x86 testsuite
results are decent with only c-c++-common/asan/swapcontext-test-1.c
failing and g++.dg/asan/default-options-1.C failing on Solaris 11.4+
(11.3 is fine; I suspect this is just another instance of PR c++/52477).
On sparc there are quite some more which I intend to investigate
subsequently.

Ok for mainline?

	Rainer

-- 
-----------------------------------------------------------------------------
Rainer Orth, Center for Biotechnology, Bielefeld University


2018-10-31  Rainer Orth  <ro@CeBiTec.Uni-Bielefeld.DE>

	gcc:
	PR sanitizer/80953
	* config/sol2.h (ASAN_CC1_SPEC): Define.
	(LD_WHOLE_ARCHIVE_OPTION): Define.
	(LD_NO_WHOLE_ARCHIVE_OPTION): Define.
	(ASAN_REJECT_SPEC): Provide default.
	(LIBASAN_EARLY_SPEC): Define.
	(LIBTSAN_EARLY_SPEC): Define.
	(LIBLSAN_EARLY_SPEC): Define.
	* config/i386/sol2.h (CC1_SPEC): Redefine.
	(ASAN_REJECT_SPEC): Define.

	* config/sparc/sparc.c (sparc_asan_shadow_offset): Declare.
	(TARGET_ASAN_SHADOW_OFFSET): Define.
	(sparc_asan_shadow_offset): New function.
	* config/sparc/sol2.h (CC1_SPEC): Append ASAN_CC1_SPEC.
	(ASAN_REJECT_SPEC): Define.

	gcc/testsuite:
	PR sanitizer/80953
	* c-c++-common/asan/alloca_loop_unpoisoning.c: Require alloca
	support.
	(foo): Use __builtin_alloca.

	libsanitizer:
	PR sanitizer/80953
	* configure.tgt (sparc*-*-solaris2.11*): Enable.
	(x86_64-*-solaris2.11* | i?86-*-solaris2.11*): Enable.

# HG changeset patch
# Parent  7a0019c2c3d32fe0c18f4708f0099e2816f05fa6
Enable libsanitizer on Solaris (PR sanitizer/80953)

diff --git a/gcc/config/i386/sol2.h b/gcc/config/i386/sol2.h
--- a/gcc/config/i386/sol2.h
+++ b/gcc/config/i386/sol2.h
@@ -54,6 +54,9 @@ along with GCC; see the file COPYING3.  
 #undef CPP_SPEC
 #define CPP_SPEC "%(cpp_subtarget)"
 
+#undef CC1_SPEC
+#define CC1_SPEC "%(cc1_cpu) " ASAN_CC1_SPEC
+
 /* GNU as understands --32 and --64, but the native Solaris
    assembler requires -xarch=generic or -xarch=generic64 instead.  */
 #ifdef USE_GAS
@@ -241,6 +244,10 @@ along with GCC; see the file COPYING3.  
 #define LARGECOMM_SECTION_ASM_OP "\t.lbcomm\t"
 #endif
 
+/* -fsanitize=address is currently only supported for 32-bit.  */
+#define ASAN_REJECT_SPEC \
+  DEF_ARCH64_SPEC("%e:-fsanitize=address is not supported in this configuration")
+
 #define USE_IX86_FRAME_POINTER 1
 #define USE_X86_64_FRAME_POINTER 1
 
diff --git a/gcc/config/sol2.h b/gcc/config/sol2.h
--- a/gcc/config/sol2.h
+++ b/gcc/config/sol2.h
@@ -138,6 +138,9 @@ along with GCC; see the file COPYING3.  
 #define DEF_ARCH64_SPEC(__str) "%{!m32:" __str "}"
 #endif
 
+/* Solaris needs -fasynchronous-unwind-tables to generate unwind info.  */
+#define ASAN_CC1_SPEC "%{%:sanitize(address):-fasynchronous-unwind-tables}"
+
 /* It's safe to pass -s always, even if -g is not used.  Those options are
    handled by both Sun as and GNU as.  */
 #define ASM_SPEC_BASE \
@@ -231,6 +234,36 @@ along with GCC; see the file COPYING3.  
 #define ENDFILE_VTV_SPEC ""
 #endif /* !ENABLE_VTABLE_VERIFY */
 
+/* Link -lasan early on the command line.  For -static-libasan, don't link
+   it for -shared link, the executable should be compiled with -static-libasan
+   in that case, and for executable link with --{,no-}whole-archive around
+   it to force everything into the executable.  */
+
+#ifndef USE_GNU_LD
+#define LD_WHOLE_ARCHIVE_OPTION "-z allextract"
+#define LD_NO_WHOLE_ARCHIVE_OPTION "-z defaultextract"
+#else
+#define LD_WHOLE_ARCHIVE_OPTION "--whole-archive"
+#define LD_NO_WHOLE_ARCHIVE_OPTION "--no-whole-archive"
+#endif
+
+/* Allow rejecting -fsanitize=address, e.g. for specific multilibs.  */
+#ifndef ASAN_REJECT_SPEC
+#define ASAN_REJECT_SPEC ""
+#endif
+
+#define LIBASAN_EARLY_SPEC ASAN_REJECT_SPEC \
+  " %{!shared:libasan_preinit%O%s} \
+    %{static-libasan:%{!shared: -Bstatic "\
+    LD_WHOLE_ARCHIVE_OPTION " -lasan " LD_NO_WHOLE_ARCHIVE_OPTION \
+    "-Bdynamic}}%{!static-libasan:-lasan}"
+
+/* Error out on -fsanitize=thread|leak.  */
+#define LIBTSAN_EARLY_SPEC "\
+  %e:-fsanitize=thread is not supported in this configuration"
+#define LIBLSAN_EARLY_SPEC "\
+  %e:-fsanitize=leak is not supported in this configuration"
+
 /* We don't use the standard svr4 STARTFILE_SPEC because it's wrong for us.  */
 #undef STARTFILE_SPEC
 #ifdef HAVE_SOLARIS_CRTS
diff --git a/gcc/config/sparc/sol2.h b/gcc/config/sparc/sol2.h
--- a/gcc/config/sparc/sol2.h
+++ b/gcc/config/sparc/sol2.h
@@ -237,7 +237,7 @@ extern const char *host_detect_local_cpu
 %{m64:%{m32:%emay not use both -m32 and -m64}} \
 %{m64:-mptr64 -mstack-bias -mno-v8plus \
   %{!mcpu*:-%{!mv8plus:mcpu=v9}}} \
-"
+" ASAN_CC1_SPEC
 #else
 #define CC1_SPEC "\
 %{m32:%{m64:%emay not use both -m32 and -m64}} \
@@ -245,7 +245,7 @@ extern const char *host_detect_local_cpu
   %{!mcpu*:%{!mv8plus:-mcpu=v9}}} \
 %{mv8plus:-m32 -mptr32 -mno-stack-bias \
   %{!mcpu*:-mcpu=v9}} \
-"
+" ASAN_CC1_SPEC
 #endif
 
 /* Support for a compile-time default CPU, et cetera.  The rules are:
@@ -322,6 +322,9 @@ extern const char *host_detect_local_cpu
 
 #define ENDFILE_ARCH_SPEC ""
 
+/* -fsanitize=address is currently only supported for 32-bit.  */
+#define ASAN_REJECT_SPEC \
+  DEF_ARCH64_SPEC("%e:-fsanitize=address is not supported in this configuration")
 
 
 /* Register the Solaris-specific #pragma directives.  */
diff --git a/gcc/config/sparc/sparc.c b/gcc/config/sparc/sparc.c
--- a/gcc/config/sparc/sparc.c
+++ b/gcc/config/sparc/sparc.c
@@ -674,6 +674,7 @@ static unsigned int sparc_function_arg_b
 						 const_tree);
 static int sparc_arg_partial_bytes (cumulative_args_t,
 				    machine_mode, tree, bool);
+static unsigned HOST_WIDE_INT sparc_asan_shadow_offset (void);
 static void sparc_output_dwarf_dtprel (FILE *, int, rtx) ATTRIBUTE_UNUSED;
 static void sparc_file_end (void);
 static bool sparc_frame_pointer_required (void);
@@ -839,6 +840,9 @@ char sparc_hard_reg_printed[8];
 #undef TARGET_STRICT_ARGUMENT_NAMING
 #define TARGET_STRICT_ARGUMENT_NAMING sparc_strict_argument_naming
 
+#undef TARGET_ASAN_SHADOW_OFFSET
+#define TARGET_ASAN_SHADOW_OFFSET sparc_asan_shadow_offset
+
 #undef TARGET_EXPAND_BUILTIN_VA_START
 #define TARGET_EXPAND_BUILTIN_VA_START sparc_va_start
 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
@@ -12490,7 +12494,15 @@ sparc_init_machine_status (void)
 {
   return ggc_cleared_alloc<machine_function> ();
 }
-
+
+/* Implement the TARGET_ASAN_SHADOW_OFFSET hook.  */
+
+static unsigned HOST_WIDE_INT
+sparc_asan_shadow_offset (void)
+{
+  return TARGET_ARCH64 ? HOST_WIDE_INT_C (0x7fff8000) : (HOST_WIDE_INT_1 << 29);
+}
+
 /* This is called from dwarf2out.c via TARGET_ASM_OUTPUT_DWARF_DTPREL.
    We need to emit DTP-relative relocations.  */
 
diff --git a/gcc/testsuite/c-c++-common/asan/alloca_loop_unpoisoning.c b/gcc/testsuite/c-c++-common/asan/alloca_loop_unpoisoning.c
--- a/gcc/testsuite/c-c++-common/asan/alloca_loop_unpoisoning.c
+++ b/gcc/testsuite/c-c++-common/asan/alloca_loop_unpoisoning.c
@@ -1,4 +1,5 @@
 /* { dg-do run } */
+/* { dg-require-effective-target alloca } */
 
 /* This testcase checks that allocas and VLAs inside loop are correctly unpoisoned.  */
 
@@ -16,7 +17,7 @@ volatile int thirty_two = 32;
   top = &x;
   volatile char array[len];
   assert(!((uintptr_t) array & 31L));
-  alloca(len);
+  __builtin_alloca(len);
   for (int i = 0; i < thirty_two; ++i) {
     char array[i];
     bot = array;
diff --git a/libsanitizer/configure.tgt b/libsanitizer/configure.tgt
--- a/libsanitizer/configure.tgt
+++ b/libsanitizer/configure.tgt
@@ -43,6 +43,8 @@ case "${target}" in
 	;;
   s390*-*-linux*)
 	;;
+  sparc*-*-solaris2.11*)
+	;;
   arm*-*-linux*)
 	;;
   mips*64*-*-linux*)
@@ -68,6 +70,8 @@ case "${target}" in
 	esac
 	TSAN_SUPPORTED=no
 	;;
+  x86_64-*-solaris2.11* | i?86-*-solaris2.11*)
+	;;
   *)
 	UNSUPPORTED=1
 	;;