[Bug c/11491] New: Problem with optimized inline caller of alloca()

["tingjun dot wen at idt dot com" <gcc-bugzilla at gcc dot gnu dot org>]

PLEASE REPLY TO gcc-bugzilla@gcc.gnu.org ONLY, *NOT* gcc-bugs@gcc.gnu.org.

http://gcc.gnu.org/bugzilla/show_bug.cgi?id=11491

           Summary: Problem with optimized inline caller of alloca()
           Product: gcc
           Version: 3.2.2
            Status: UNCONFIRMED
          Severity: normal
          Priority: P1
         Component: c
        AssignedTo: unassigned at gcc dot gnu dot org
        ReportedBy: tingjun dot wen at idt dot com
                CC: gcc-bugs at gcc dot gnu dot org
 GCC build triplet: gcc-3.2_2.13.2/i586-pc-linux-gnu/bin/gcc
  GCC host triplet: Linux 2.4.9-13 i686
GCC target triplet: Linux 2.4.9-13 i686

The reproducible snippet code I used is the code with a static 
function, say alloca_caller(), which calls alloca(). The main
function calls it in a loop. When I compiled it using gcc 3.2.2
with options -O -finline-functions, alloca_caller() will be
optimized inline. The optimized assembly code "alloca"s
the stack memory blocks inside the loop but will not "release"
them. Since the alloca() is inside the loop, the stack memory
will soon be used up. (code follows).

The non-optimized code is fine since it will not inline the
function inside the loop and will automatically "release"
the memory when the caller returns.

Here is the script I used to reproduce this problem:
====================================================
#!/bin/sh -x

GCC=/tools/sw_build/gcc-3.2_2.13.2/i586-pc-linux-gnu/bin/gcc

$GCC -v

uname -a

cat inline-alloca.c

$GCC -O -finline-functions -S -o inline-alloca.s inline-alloca.c
cat inline-alloca.s

$GCC -O -finline-functions -o inline-alloca inline-alloca.c

ulimit -S -s256
ulimit -a

./inline-alloca


Here is the C code and the result:
===============================================================
+ /tools/sw_build/gcc-3.2_2.13.2/i586-pc-linux-gnu/bin/gcc -v
Reading specs from
/tools/sw_build/gcc-3.2_2.13.2/i586-pc-linux-gnu/lib/gcc-lib/i586-pc-linux-gnu/3.2/specs
Configured with: /tools/sw_build/src/gcc-3.2/configure
--prefix=/tools/sw_build/gcc-3.2_2.13.2
--exec-prefix=/tools/sw_build/gcc-3.2_2.13.2/i586-pc-linux-gnu
--host=i586-pc-linux-gnu --target=i586-pc-linux-gnu --enable-shared
--enable-languages=c,c++,java --disable-multilib --with-gnu-as --with-gnu-ld
--with-as=/tools/sw_build/gcc-3.2_2.13.2/i586-pc-linux-gnu/bin/as
--with-ld=/tools/sw_build/gcc-3.2_2.13.2/i586-pc-linux-gnu/bin/ld
Thread model: posix
gcc version 3.2
+ uname -a
Linux altair.ott.idt.com 2.4.9-13 #1 Tue Oct 30 20:05:14 EST 2001 i686 unknown
+ cat inline-alloca.c
#include <stdio.h>
#include <stdlib.h>

static void
alloca_caller()
{
        int * ptr;
        ptr = alloca(16384);
        fprintf(stderr, "%p\n", ptr);
        return;
}


int
main(int argc, char ** argv)
{
        int i;

        for (i = 0; i < 16384; i++) {
                alloca_caller();
        }

        return 0;
}
+ /tools/sw_build/gcc-3.2_2.13.2/i586-pc-linux-gnu/bin/gcc -O -finline-functions
-S -o
inline-alloca.s inline-alloca.c
+ cat inline-alloca.s
        .file   "inline-alloca.c"
        .section        .rodata.str1.1,"aMS",@progbits,1
.LC0:
        .string "%p\n"
        .text
        .align 2
.globl main
        .type   main,@function
main:
        pushl   %ebp
        movl    %esp, %ebp
        pushl   %ebx
        subl    $4, %esp
        andl    $-16, %esp
        movl    $0, %ebx
.L7:
        subl    $16384, %esp
        movl    %esp, %eax
        subl    $4, %esp
        pushl   %eax
        pushl   $.LC0
        pushl   stderr
        call    fprintf
        addl    $16, %esp
        incl    %ebx
        cmpl    $16383, %ebx
        jle     .L7
        movl    $0, %eax
        movl    -4(%ebp), %ebx
        leave
        ret
.Lfe1:
        .size   main,.Lfe1-main
        .ident  "GCC: (GNU) 3.2"
+ /tools/sw_build/gcc-3.2_2.13.2/i586-pc-linux-gnu/bin/gcc -O -finline-functions
-o inline-alloca inline-alloca.c
+ ulimit -S -s256
+ ulimit -a
core file size (blocks)     0
data seg size (kbytes)      unlimited
file size (blocks)          unlimited
max locked memory (kbytes)  unlimited
max memory size (kbytes)    unlimited
open files                  1024
pipe size (512 bytes)       8
stack size (kbytes)         256
cpu time (seconds)          unlimited
max user processes          1023
virtual memory (kbytes)     unlimited
+ ./inline-alloca
0xbfffb600
0xbfff7600
0xbfff3600
0xbffef600
0xbffeb600
0xbffe7600
0xbffe3600
0xbffdf600
0xbffdb600
0xbffd7600
0xbffd3600
0xbffcf600
0xbffcb600
0xbffc7600
0xbffc3600
./runit.sh: line 19: 14658 Segmentation fault      ./inline-alloca