[PATCH] Add support for the Win32 hook prologue

[Stefan Dösinger <stefan at codeweavers dot com>]

A number of Windows programs(Steam, Xfire and others) try to hook Win32 API 
functions by replacing the first 5 bytes in the function. This causes 
problems for Wine because the functions generated by gcc usually start with a 
different opcode sequence than the one expected by these applications.

Starting with Windows XP SP2, Microsoft starts Win32 functions with this 
sequence:

8b ff		mov %edi, %edi
55		push %ebp
8b ec	mov %esp, %ebp

The attached patch implements a function attribute that allows Wine to request 
the same 5 bytes at the beginning of a function.

I tested the testuite on x86_64-pc-linux-gnu.

2009-09-09: Stefan Dösinger <stefan@codeweavers.com
	* i386.c i386.md: Add a new function
	attribute msvc_prologue that starts functions with the same opcode sequence
	used in most Win32 API functions

	* gcc.misc-tests/msvc_prologue.c msvc_prologue.exp: New testcase

	* configure.ac: Test for swap suffix support in as

Index: gcc/doc/extend.texi
===================================================================
--- gcc/doc/extend.texi	(revision 151558)
+++ gcc/doc/extend.texi	(working copy)
@@ -2672,6 +2672,14 @@ when targeting Windows.  On all other systems, the
 
 Note, This feature is currently sorried out for Windows targets trying to
 
+@item msvc_prologue
+@cindex @code{msvc_prologue} attribute
+
+On 32 bit i[34567]86-*-* targets, you can use this function attribute to make
+gcc generate the "hot-patching" function prologue used in Win32 API
+functions in Microsoft Windows XP Service Pack 2 and newer. This requires
+support for the swap suffix in the assembler. (GNU Binutils 2.19.51 or later)
+
 @item naked
 @cindex function without a prologue/epilogue code
 Use this attribute on the ARM, AVR, IP2K and SPU ports to indicate that
Index: gcc/configure.ac
===================================================================
--- gcc/configure.ac	(revision 151558)
+++ gcc/configure.ac	(working copy)
@@ -3036,6 +3036,12 @@ foo:	nop
       [AC_DEFINE(HAVE_AS_IX86_SAHF, 1,
         [Define if your assembler supports the sahf mnemonic.])])
 
+    gcc_GAS_CHECK_FEATURE([swap suffix],
+      gcc_cv_as_ix86_swap,,,
+      [movl.s %esp, %ebp],,
+      [AC_DEFINE(HAVE_AS_IX86_SWAP, 1,
+        [Define if your assembler supports the swap suffix.])])
+
     gcc_GAS_CHECK_FEATURE([different section symbol subtraction],
       gcc_cv_as_ix86_diff_sect_delta,,,
       [.section .rodata
Index: gcc/config/i386/i386.h
===================================================================
--- gcc/config/i386/i386.h	(revision 151558)
+++ gcc/config/i386/i386.h	(working copy)
@@ -2388,6 +2388,9 @@ struct GTY(()) machine_function {
      to be used. MS_ABI means ms abi. Otherwise SYSV_ABI means sysv abi.  */
   enum calling_abi call_abi;
   struct machine_cfa_state cfa;
+  /* This value is used for i386 targets and specifies if the function
+   * should start with the hooking-friendly Win32 function prologue       */
+  int msvc_prologue;
 };
 #endif
 
Index: gcc/config/i386/i386.md
===================================================================
--- gcc/config/i386/i386.md	(revision 151558)
+++ gcc/config/i386/i386.md	(working copy)
@@ -237,6 +237,7 @@
    (UNSPECV_RDTSC		18)
    (UNSPECV_RDTSCP		19)
    (UNSPECV_RDPMC		20)
+   (UNSPECV_VSWAPMOV	21)
   ])
 
 ;; Constants to represent pcomtrue/pcomfalse variants
@@ -15747,6 +15748,16 @@
    (set_attr "length_immediate" "0")
    (set_attr "modrm" "0")])
 
+(define_insn "vswapmov"
+  [(set (match_operand:SI 0 "register_operand" "=r")
+        (match_operand:SI 1 "register_operand" "r"))
+   (unspec_volatile [(const_int 0)] UNSPECV_VSWAPMOV)]
+  ""
+  "movl.s\t%1,%0"
+  [(set_attr "length" "2")
+   (set_attr "length_immediate" "0")
+   (set_attr "modrm" "0")])
+
 ;; Pad to 16-byte boundary, max skip in op0.  Used to avoid
 ;; branch prediction penalty for the third jump in a 16-byte
 ;; block on K8.
Index: gcc/config/i386/i386.c
===================================================================
--- gcc/config/i386/i386.c	(revision 151558)
+++ gcc/config/i386/i386.c	(working copy)
@@ -4777,6 +4777,24 @@ ix86_function_type_abi (const_tree fntype)
   return ix86_abi;
 }
 
+static int
+ix86_function_msvc_prologue (const_tree fntype)
+{
+  if (!TARGET_64BIT && fntype != NULL)
+    {
+      if(lookup_attribute ("msvc_prologue", TYPE_ATTRIBUTES (fntype)))
+        {
+#ifdef HAVE_AS_IX86_SWAP
+          return 1;
+#else
+          sorry ("msvc_prologue needs swap suffix support in as");
+          return 0;
+#endif
+        }
+    }
+  return 0;
+}
+
 static enum calling_abi
 ix86_function_abi (const_tree fndecl)
 {
@@ -4808,6 +4826,11 @@ ix86_call_abi_override (const_tree fndecl)
     cfun->machine->call_abi = ix86_abi;
   else
     cfun->machine->call_abi = ix86_function_type_abi (TREE_TYPE (fndecl));
+
+  if (fndecl == NULL_TREE)
+    cfun->machine->msvc_prologue = 0;
+  else
+    cfun->machine->msvc_prologue = ix86_function_msvc_prologue (TREE_TYPE (fndecl));
 }
 
 /* MS and SYSV ABI have different set of call used registers.  Avoid expensive
@@ -8316,6 +8339,7 @@ ix86_expand_prologue (void)
   bool pic_reg_used;
   struct ix86_frame frame;
   HOST_WIDE_INT allocate;
+  int gen_frame_pointer = frame_pointer_needed;
 
   ix86_finalize_stack_realign_flags ();
 
@@ -8328,6 +8352,43 @@ ix86_expand_prologue (void)
 
   ix86_compute_frame_layout (&frame);
 
+  if(cfun->machine->msvc_prologue)
+  {
+    rtx push, mov;
+    /* Make sure the function starts with
+       8b ff     movl.s %edi,%edi
+       55        push   %ebp
+       8b ec     movl.s %esp,%ebp
+
+       This matches the hookable function prologue in Win32 API functions in Microsoft Windows
+       XP Service Pack 2 and newer. Wine uses this to enable Windows apps to hook the Win32 API
+       functions provided by Wine */
+    insn = emit_insn(gen_vswapmov(gen_rtx_REG (Pmode, DI_REG), gen_rtx_REG (Pmode, DI_REG)));
+    push = emit_insn (gen_push (hard_frame_pointer_rtx));
+    mov = emit_insn(gen_vswapmov(hard_frame_pointer_rtx, stack_pointer_rtx));
+
+    if(frame_pointer_needed && !(crtl->drap_reg && crtl->stack_realign_needed))
+    {
+      /* The push %ebp and movl.s %esp, %ebp already set up the frame pointer. No need to do
+         this again. */
+      gen_frame_pointer = 0;
+      RTX_FRAME_RELATED_P (push) = 1;
+      RTX_FRAME_RELATED_P (mov) = 1;
+      if (ix86_cfa_state->reg == stack_pointer_rtx)
+        ix86_cfa_state->reg = hard_frame_pointer_rtx;
+    }
+    else
+    {
+      /* If the frame pointer is not needed, pop %ebp again. This could be optimized for cases where
+         ebp needs to be backed up for some other reason.
+
+         If stack realignment is needed, pop the base pointer again, align the stack, and later
+         regenerate the frame pointer setup. The frame pointer generated by the msvc prologue
+         is not aligned, so it can't be used */
+      insn = emit_insn ((*ix86_gen_pop1) (hard_frame_pointer_rtx));
+    }
+  }
+
   /* Emit prologue code to adjust stack alignment and setup DRAP, in case
      of DRAP is needed and stack realignment is really needed after reload */
   if (crtl->drap_reg && crtl->stack_realign_needed)
@@ -8377,7 +8438,7 @@ ix86_expand_prologue (void)
   /* Note: AT&T enter does NOT have reversed args.  Enter is probably
      slower on all targets.  Also sdb doesn't like it.  */
 
-  if (frame_pointer_needed)
+  if (gen_frame_pointer)
     {
       insn = emit_insn (gen_push (hard_frame_pointer_rtx));
       RTX_FRAME_RELATED_P (insn) = 1;
@@ -26067,10 +26128,12 @@ ix86_handle_abi_attribute (tree *node, tree name,
       *no_add_attrs = true;
       return NULL_TREE;
     }
-  if (!TARGET_64BIT)
+  if (TARGET_64BIT
+      ? is_attribute_p ("msvc_prologue", name)
+      : !is_attribute_p ("msvc_prologue", name))
     {
-      warning (OPT_Wattributes, "%qE attribute only available for 64-bit",
-	       name);
+      warning (OPT_Wattributes, "%qE attribute only available for %d-bit",
+        name, TARGET_64BIT ? 32 : 64);
       *no_add_attrs = true;
       return NULL_TREE;
     }
@@ -28983,6 +29046,7 @@ static const struct attribute_spec ix86_attribute_
   /* ms_abi and sysv_abi calling convention function attributes.  */
   { "ms_abi", 0, 0, false, true, true, ix86_handle_abi_attribute },
   { "sysv_abi", 0, 0, false, true, true, ix86_handle_abi_attribute },
+  { "msvc_prologue", 0, 0, false, true, true, ix86_handle_abi_attribute },
   /* End element.  */
   { NULL,        0, 0, false, false, false, NULL }
 };
--- /dev/null	2009-09-09 12:15:38.779548517 +0200
+++ gcc/testsuite/gcc.misc-tests/msvc_prologue.c	2009-09-09 16:04:45.000000000 +0200
@@ -0,0 +1,19 @@
+/* Test that the msvc_prologue attribute generates the correct code.  */
+
+/* { dg-do run } */
+
+int __attribute__((__msvc_prologue__)) foo()
+{
+    unsigned char *ptr = (unsigned char *) foo;
+    if(*ptr++ != 0x8b) return 1;
+    if(*ptr++ != 0xff) return 1;
+    if(*ptr++ != 0x55) return 1;
+    if(*ptr++ != 0x8b) return 1;
+    if(*ptr++ != 0xec) return 1;
+    return 0;
+}
+
+int main ()
+{
+   return foo();
+}
--- /dev/null	2009-09-09 12:15:38.779548517 +0200
+++ gcc/testsuite/gcc.misc-tests/msvc_prologue.exp	2009-09-09 16:04:00.000000000 +0200
@@ -0,0 +1,32 @@
+#   Copyright (C) 2001, 2007, 2008 Free Software Foundation, Inc.
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GCC; see the file COPYING3.  If not see
+# <http://www.gnu.org/licenses/>.
+
+# This file is based on a bug report submitted to gcc-bugs:
+# http://gcc.gnu.org/ml/gcc-patches/2001-12/msg02158.html
+
+# Load support procs.
+load_lib gcc-dg.exp
+
+dg-init
+set native_cflags ""
+
+if { [istarget "i?86-*-*"] || [istarget "x86_64-*-linux*"] } {
+  if { [istarget "x86_64-*-linux*"] } {
+    set native_cflags "-m32"
+  }
+  dg-runtest "$srcdir/$subdir/msvc_prologue.c" "$native_cflags" ""
+}
+dg-finish