[PATCH] Stack protector i386/x86_64 fixes

[Jakub Jelinek]

Hi!

This patch (on top of
http://gcc.gnu.org/ml/gcc-patches/2005-06/msg01867.html
) fixes a bug in the x86-64 stack_protect_set pattern,
which generates invalid xorl %rax, %rax instruction.
Also, it only uses __stack_chk_fail_local for 32-bit code in i386/*
(x86-64 doesn't need to set up any PIC pointer and using
__stack_chk_fail_local makes programs/libraries tiny bit bigger),
likewise it doesn't use it if visibility is not supported or
for non-pic code.
There is also a tiny optimization, we know that the scratch
register after stack_protect_set will be zero, so it is desirable
to tell the compiler about that, otherwise we ended up e.g. with
xorl %eax, %eax
xorl %eax, %eax
etc. on some testcases.
I have tried to do something similar for stack_protect_test, but
failed.  Telling the compiler it is always 0 is lying, what if the compiler
(unlikely, but who knows) decided to need zero somewhere on the
path to the noreturn function where the register does not
contain zero.  I have tried to write the pattern so that
the compiler could know if the branch does not jump, it is zero,
but it was probably too complicated and did not result in
any code improvements.

	Jakub
-------------- next part --------------
2005-06-24  Jakub Jelinek  <jakub@redhat.com>

	* targhooks.c (default_hidden_stack_protect_fail): Fall back to
	default_external_stack_protect_fail if visibility is not supported
	or not flag_pic.
	* config/i386/i386.c (ix86_stack_protect_fail): New function.
	(TARGET_STACK_PROTECT_FAIL): Define.
	* config/i386/i386.md (stack_protect_si): Change CLOBBER into
	SET to zero.
	(stack_protect_di): Likewise.  Use %k2 instead of %2 to avoid
	invalid instruction xorl %rax, %rax.

--- gcc/config/i386/i386.c.jj	2005-06-23 11:54:21.000000000 +0200
+++ gcc/config/i386/i386.c	2005-06-23 11:54:21.000000000 +0200
@@ -906,6 +906,7 @@ static bool ix86_pass_by_reference (CUMU
 static void ix86_init_builtins (void);
 static rtx ix86_expand_builtin (tree, rtx, rtx, enum machine_mode, int);
 static const char *ix86_mangle_fundamental_type (tree);
+static tree ix86_stack_protect_fail (void);
 
 /* This function is only used on Solaris.  */
 static void i386_solaris_elf_named_section (const char *, unsigned int, tree)
@@ -1082,7 +1083,7 @@ static void init_ext_80387_constants (vo
 #define TARGET_MANGLE_FUNDAMENTAL_TYPE ix86_mangle_fundamental_type
 
 #undef TARGET_STACK_PROTECT_FAIL
-#define TARGET_STACK_PROTECT_FAIL default_hidden_stack_protect_fail
+#define TARGET_STACK_PROTECT_FAIL ix86_stack_protect_fail
 
 struct gcc_target targetm = TARGET_INITIALIZER;
 
@@ -17564,4 +17565,17 @@ ix86_mangle_fundamental_type (tree type)
     }
 }
 
+/* For 32-bit code we can save PIC register setup by using
+   __stack_chk_fail_local hidden function instead of calling
+   __stack_chk_fail directly.  64-bit code doesn't need to setup any PIC
+   register, so it is better to call __stack_chk_fail directly.  */
+
+static tree
+ix86_stack_protect_fail (void)
+{
+  return TARGET_64BIT
+	 ? default_external_stack_protect_fail ()
+	 : default_hidden_stack_protect_fail ();
+}
+
 #include "gt-i386.h"
--- gcc/config/i386/i386.md.jj	2005-06-22 18:26:10.000000000 +0200
+++ gcc/config/i386/i386.md	2005-06-22 23:05:22.000000000 +0200
@@ -19613,7 +19613,7 @@
 (define_insn "stack_protect_set_si"
   [(set (match_operand:SI 0 "memory_operand" "=m")
 	(unspec:SI [(match_operand:SI 1 "memory_operand" "m")] UNSPEC_SP_SET))
-   (clobber (match_scratch:SI 2 "=r"))
+   (set (match_scratch:SI 2 "=&r") (const_int 0))
    (clobber (reg:CC FLAGS_REG))]
   ""
   "mov{l}\t{%1, %2|%2, %1}\;mov{l}\t{%2, %0|%0, %2}\;xor{l}\t%2, %2"
@@ -19622,10 +19622,10 @@
 (define_insn "stack_protect_set_di"
   [(set (match_operand:DI 0 "memory_operand" "=m")
 	(unspec:DI [(match_operand:DI 1 "memory_operand" "m")] UNSPEC_SP_SET))
-   (clobber (match_scratch:DI 2 "=r"))
+   (set (match_scratch:DI 2 "=&r") (const_int 0))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_64BIT"
-  "mov{q}\t{%1, %2|%2, %1}\;mov{q}\t{%2, %0|%0, %2}\;xor{l}\t%2, %2"
+  "mov{q}\t{%1, %2|%2, %1}\;mov{q}\t{%2, %0|%0, %2}\;xor{l}\t%k2, %k2"
   [(set_attr "type" "multi")])
 
 (define_expand "stack_protect_test"
--- gcc/targhooks.c.jj	2005-06-22 18:26:10.000000000 +0200
+++ gcc/targhooks.c	2005-06-23 11:57:50.000000000 +0200
@@ -380,9 +380,15 @@ default_external_stack_protect_fail (voi
 tree
 default_hidden_stack_protect_fail (void)
 {
+#ifndef HAVE_GAS_HIDDEN
+  return default_external_stack_protect_fail ();
+#else
   tree t = stack_chk_fail_decl;
 
-  if (stack_chk_fail_decl == NULL_TREE)
+  if (!flag_pic)
+    return default_external_stack_protect_fail ();
+
+  if (t == NULL_TREE)
     {
       t = build_function_type_list (void_type_node, NULL_TREE);
       t = build_decl (FUNCTION_DECL,
@@ -402,6 +408,7 @@ default_hidden_stack_protect_fail (void)
     }
 
   return build_function_call_expr (t, NULL_TREE);
+#endif
 }
 
 #include "gt-targhooks.h"