Stack cleanup after CALL misses a DWORD on x86

[Phil Jerkins <jpjerkins at yahoo dot com>]

To Whom It Concerns:

First, a word of thanks.  You have done a great work
for free, and it is greatly appreciated.  This
compiler has contributed perhaps more than Linux to
the changes we're seeing in the software landscape. 
And they're good changes.

THE SHORT VERSION

If an external function that returns a structure is
called through a function pointer, an extra DWORD gets
left on the stack after each call, leading to possible
stack overflow.

THE CONTEXT

I'm using MinGW and NASM.  I'm writing a function in
NASM that I'm calling using a FUNCTION POINTER in C. 
The function takes and returns a structure, and does
absolutely nothing - see the code.  The code generated
for the call sets up the stack:
 1 LEA the return struct's address into EAX
 2 Reserve 12 bytes on the stack (?) 12
 3 Push the input structure +16 = 28
 4 Push EAX (the return struct's address) +4 = 32
 5 Load the function pointer into EAX
 6 CALL EAX.

THE PROBLEM

The code that immediately follows the CALL EAX
instruction is the problem.  Remember - it has pushed
a 16-byte structure, reserved an additional 12 bytes,
then pushed the return struct's address - 32 bytes
total.  After the CALL returns, it only adds 28 bytes
to the stack pointer - leaving out 4 bytes, or 1
DWORD.

I found the problem because I'm calling my ASM
function in a loop 200 million times (comparing it's
performance to that of a comparable C# program).  With
that many DWORDs missed, it eventually ran out of
stack space.  The program executes normally until it
runs out of stack space.

I'm including the source files.  I'm using MinGW32
2.0.0.3 (which includes gcc 3.2 mingw special
20020817-1 - www.mingw.org), and NASM 0.98.34 for
Win32 (nasm.sourceforge.net) on Windows XP SP1. Note
how little is NOT commented out in the ASM file.  You
can use a hex editor to change the byte at offset
0x0A02 from 0x1C to 0x20, and the program runs
correctly.  I used PEBrowse Pro Interactive (freeware
at www.smidgeonsoft.com) to disassemble and debug the
EXE.

The command lines I'm using to compile:
 nasmw -f coff VectorSSE.asm
 gcc -fpcc-struct-return -c SpeedTests2.c
 gcc -o SpeedTests2 SpeedTests2.o VectorSSE.o


Feel free to contact me for more info.

Thanks again!

Phil Jerkins
jpjerkins@yahoo.com



__________________________________________________
Do you Yahoo!?
New DSL Internet Access from SBC & Yahoo!
http://sbc.yahoo.com
#include <stdio.h>
#include <time.h>
#include <float.h>

struct VectorStruct {
  float x;
  float y;
  float z;
  float unused;
} __attribute__((aligned(16)));

// Alignment attribute doesn't seem to be working - movaps causes an exception
typedef struct VectorStruct Vector __attribute__((aligned(16)));

Vector NormalizeVectorSSE(Vector v);
// Vector NormalizeVectorSSE(Vector v) {}

main() {
  register int count = 0;
  double top = 1.2;
  double bottom = 3.4;
  double result = 0.0;
  register clock_t t = clock();
  Vector (*NormalizeVector)(Vector) = NormalizeVectorSSE;
  
  Vector v;
  v.x = 1.0;
  v.y = 2.0;
  v.z = 3.0;
  v.unused = 1.0;
  printf("%f %f %f\n", v.x, v.y, v.z);
  // v = NormalizeVectorSSE(v);
  count = 200000000;
  v = (*NormalizeVector)(v);
  printf("%f %f %f\n", v.x, v.y, v.z);
  printf("%x %d\n", &v, sizeof(v));
  printf("%d\n", count);
  
  count = 200000000;
  // count = 1000000;
  
  v.x = 3.0;
  v.y = 2.0;
  v.z = 1.0;
  v.unused = 1.0;
  v = NormalizeVectorSSE(v);
  printf("%f %f %f\n", v.x, v.y, v.z);

  v.x = 1.0;
  v.y = 2.0;
  v.z = 3.0;
  v.unused = 1.0;
  v = (*NormalizeVector)(v);
  printf("%f %f %f\n", v.x, v.y, v.z);

  v.x = 3.0;
  v.y = 2.0;
  v.z = 1.0;
  v.unused = 1.0;
  v = (*NormalizeVector)(v);
  printf("%f %f %f\n", v.x, v.y, v.z);

  v.x = 1.0;
  v.y = 2.0;
  v.z = 3.0;
  v.unused = 1.0;
  v = (*NormalizeVector)(v);
  printf("%f %f %f\n", v.x, v.y, v.z);

  v.x = 3.0;
  v.y = 2.0;
  v.z = 1.0;
  v.unused = 1.0;
  v = (*NormalizeVector)(v);
  printf("%f %f %f\n", v.x, v.y, v.z);

  while(count-- > 0) {
    // result = top / bottom;
    v.x = 3.0;
    v.y = 2.0;
    v.z = 1.0;
    v.unused = 1.0;
    v = (*NormalizeVector)(v);
    // if(200000000 - count < 15) { // Prints fine; program aborts when printing stops
    //   printf("%d\n", count);
    //   printf("%f %f %f\n", v.x, v.y, v.z);
    // }
    // count--;
  }
  printf("%d\n", count);
  printf("%f %f %f\n", v.x, v.y, v.z);
  
  printf("%d %d %d\n", t, clock(), CLOCKS_PER_SEC);
  
  // while(clock() - t < CLOCKS_PER_SEC) {
  //   result = top / bottom;
  //   count++;
  // }
  // 
  // printf("Executed the test %d times in 1 second.\n", count);
  printf("Done!\n");
}
section .text

  global _NormalizeVectorSSE

; Vector NormalizeVectorSSE(Vector vin);
_NormalizeVectorSSE:

  ; push EBP
  ; mov EBP, ESP
  ; 
  ; ; float length = (float)Math.Sqrt(X*X + Y*Y + Z*Z);
  ; ; X /= length;
  ; ; Y /= length;
  ; ; Z /= length;
  ; ; return this;
  ; 
  ; movups XMM0, [EBP+12]           ; Get the input Vector into XMM0
  ; 
  ; movaps XMM1, XMM0               ; Copy the input vector to XMM1
  ; mulps XMM1, XMM1                ; Square all 4 values
  ; movaps XMM2, XMM1               ; Copy everything to XMM2
  ; 
  ; ; Now have:
  ; ;  XMM1[ 0- 31] = X^2
  ; ;  XMM1[32- 63] = Y^2
  ; ;  XMM1[64- 95] = Z^2
  ; ;  XMM1[96-127] = unused^2
  ; 
  ; ; Add the 3 squares
  ; shufps XMM2, XMM2, 11100001b    ; Move Y^2 from XMM2[32-63] into XMM2[0-31]
  ; addss XMM1, XMM2                ; XMM1[0-31] = X^2 + Y^2
  ; ;movaps XMM2, XMM1               ; Copy everything to XMM2
  ; shufps XMM2, XMM1, 11100010b    ; Move Z^2 from XMM2[64-95] into XMM2[0-31]
  ; addss XMM1, XMM2                ; XMM1[0-31] = X^2 + Y^2 + Z^2
  ; 
  ; ; Now have X^2 + Y^2 + Z^2 in XMM1[0-31]
  ; 
  ; ; Take the square root
  ; sqrtss XMM1, XMM1
  ; 
  ; ; Now have length in XMM1[0-31] and original vector in XMM0
  ; 
  ; ; Perform the division
  ; shufps XMM1, XMM1, 00000000b    ; Copy length to all 4 DWORDs in XMM1
  ; divps XMM0, XMM1                ; Perform the division
  ; 
  ; ; Now have the return vector in XMM0
  ; 
  ; push EDX
  ; mov EDX, [EBP+8]                ; Get the address of the return Vector
  ; movups [EDX], XMM0              ; Copy the return vector out
  ; pop EDX
  ; 
  ; pop EBP
  ret

Attachment: SpeedTests2.exe
Description: SpeedTests2.exe