[Bug sanitizer/77538] segmentation fault: thread sanitizer shadow stack overflow

["coollpe at hotmail dot com" <gcc-bugzilla at gcc dot gnu dot org>]

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77538

--- Comment #14 from peien luo <coollpe at hotmail dot com> ---
(In reply to Dmitry Vyukov from comment #12)
> The crash in gdb looks like stack overflow (unsurprising if there are 1MB
> frames). Does increasing thread stack size or reducing frame size (there
> must something very big on the stack) help?

I tried gcc 4.9.4, 4.9.3, 4.9.2, 4.9.1, 4.9.0 today and found in this case, the
problem began to occur compiled with 4.9.0.

I tried to replace libsanitizer in 4.9.0 with the one in 4.8.5, no issue found.

The difference between the disassemble code at that function entry is:

4.8.5:
   0x00007f224dab0620 <+0>:     push   %r15
   0x00007f224dab0622 <+2>:     mov    %r9d,%r15d
   0x00007f224dab0625 <+5>:     push   %r14
   0x00007f224dab0627 <+7>:     push   %r13
   0x00007f224dab0629 <+9>:     mov    %rsi,%r13
   0x00007f224dab062c <+12>:    push   %r12
   0x00007f224dab062e <+14>:    push   %rbp
   0x00007f224dab062f <+15>:    mov    %rdi,%rbp
   0x00007f224dab0632 <+18>:    lea    0x30(%rbp),%r14
   0x00007f224dab0636 <+22>:    push   %rbx
   0x00007f224dab0637 <+23>:    sub    $0x1000f8,%rsp
   0x00007f224dab063e <+30>:    mov    0x100128(%rsp),%rdi
   0x00007f224dab0646 <+38>:    lea    0x50(%rsp),%rbx
   0x00007f224dab064b <+43>:    mov    %rdx,0x28(%rsp)
   0x00007f224dab0650 <+48>:    mov    %rcx,0x38(%rsp)
   0x00007f224dab0655 <+53>:    mov    %r8,0x30(%rsp)
   0x00007f224dab065a <+58>:    mov    %fs:0x28,%rax
   0x00007f224dab0663 <+67>:    mov    %rax,0x1000e8(%rsp)
   0x00007f224dab066b <+75>:    xor    %eax,%eax
   0x00007f224dab066d <+77>:    callq  0x7f224d69ae50
<__tsan_func_entry(void*)>


4.9.0
   0x00007fc63563a710 <+0>:     push   %rbp
   0x00007fc63563a711 <+1>:     mov    %rsp,%rbp
   0x00007fc63563a714 <+4>:     push   %r15
   0x00007fc63563a716 <+6>:     push   %r14
   0x00007fc63563a718 <+8>:     push   %r13
   0x00007fc63563a71a <+10>:    push   %r12
   0x00007fc63563a71c <+12>:    mov    %rdi,%r15
   0x00007fc63563a71f <+15>:    push   %rbx
   0x00007fc63563a720 <+16>:    mov    %rsi,%r13
   0x00007fc63563a723 <+19>:    mov    %r9d,%r14d
   0x00007fc63563a726 <+22>:    lea    -0x1000d0(%rbp),%rbx
   0x00007fc63563a72d <+29>:    sub    $0x1000e8,%rsp
=> 0x00007fc63563a734 <+36>:    mov    %rdi,-0x1000e8(%rbp)
   0x00007fc63563a73b <+43>:    mov    0x8(%rbp),%rdi
   0x00007fc63563a73f <+47>:    mov    %rdx,-0x1000f0(%rbp)
   0x00007fc63563a746 <+54>:    mov    %rcx,-0x100100(%rbp)
   0x00007fc63563a74d <+61>:    mov    %r8,-0x1000f8(%rbp)
   0x00007fc63563a754 <+68>:    mov    %fs:0x28,%rax
   0x00007fc63563a75d <+77>:    mov    %rax,-0x38(%rbp)
   0x00007fc63563a761 <+81>:    xor    %eax,%eax
   0x00007fc63563a763 <+83>:    callq  0x7fc63527d1e0
<__tsan_func_entry(void*)>