[Bug other/36339] New: gcc-4.3 miscompiles tagged pointer code

[mikpe at it dot uu dot se]

/* gcc430-tagged-ptr-bug.c
 *
 * This program illustrates a miscompilation with gcc-4.3.0.
 *
 * The failing code (try_a() below) initialises a Lisp-like
 * CONS cell in an on-stack mini-heap, then passes a tagged
 * pointer to this object in a tail-recursive call.
 *
 * With gcc-4.3.0, the reservation of space on the stack and
 * the initialisation of it is REMOVED. Instead the code just
 * constructs a pointer as an offset from the stack pointer
 * and passes that in the call, causing failures in the callee.
 *
 * This error has been observed on x86_64, sparc64, and powerpc.
 * It still occurs with the gcc-4.3-20080515 snapshot.
 *
 * Compiling with -fno-strict-aliasing does NOT fix the problem.
 *
 * This code works with all older gccs up to and including 4.2.4.
 *
 * The functions try_a(), try_b(), and try_c() explore different
 * ways of passing the object reference:
 * - try_c() passes it as a tagged (offset by 3) void*
 *   this works
 * - try_b() passes it as an untagged uintptr_t
 *   this works
 * - try_a() passes it as a tagged uintptr_t
 *   this fails
 *
 * The original code which broke due to this problem is in the
 * virtual machine for the Erlang functional programming language,
 * specifically the ets_match_2() and ets_match_3() functions
 * in the erl_db.c module.
 */
#include <stdio.h>

typedef unsigned long my_uintptr_t;

int check_a(my_uintptr_t tagged_ptr);
int check_b(my_uintptr_t untagged_ptr);
int check_c(void *tagged_ptr);

int __attribute__((noinline)) try_a(my_uintptr_t x)
{
    my_uintptr_t heap[2];
    my_uintptr_t *hp = heap;

    hp[0] = x;
    hp[1] = 0;
    return check_a((my_uintptr_t)(void*)((char*)hp + 1));
}

int __attribute__((noinline)) check_a(my_uintptr_t tagged_ptr)
{
    my_uintptr_t *hp = (my_uintptr_t*)(void*)((char*)tagged_ptr - 1);

    if (hp[0] == 42 && hp[1] == 0)
        return 0;
    printf("try_a() is broken\n");
    return -1;
}

int __attribute__((noinline)) try_b(my_uintptr_t x)
{
    my_uintptr_t heap[2];
    my_uintptr_t *hp = heap;

    hp[0] = x;
    hp[1] = 0;
    return check_b((my_uintptr_t)(void*)hp + 0);
}

int __attribute__((noinline)) check_b(my_uintptr_t untagged_ptr)
{
    my_uintptr_t *hp = (my_uintptr_t*)(void*)(untagged_ptr - 0);

    if (hp[0] == 43 && hp[1] == 0)
        return 0;
    printf("try_b() is broken\n");
    return -1;
}

int __attribute__((noinline)) try_c(my_uintptr_t x)
{
    my_uintptr_t heap[2];
    my_uintptr_t *hp = heap;

    hp[0] = x;
    hp[1] = 0;
    return check_c((void*)((char*)hp + 3));
}

int __attribute__((noinline)) check_c(void *tagged_ptr)
{
    my_uintptr_t *hp = (my_uintptr_t*)((char*)tagged_ptr - 3);

    if (hp[0] == 27 && hp[1] == 0)
        return 0;
    printf("try_c() is broken\n");
    return -1;
}

int main(void)
{
    int ret = 0;

    if (try_a(42) < 0)  /* fails with gcc-4.3.0 */
        ret += 1;
    if (try_b(43) < 0)  /* works */
        ret += 2;
    if (try_c(27) < 0)  /* works */
        ret += 4;
    if (ret == 0)
        printf("OK\n");
    return ret;
}


-- 
           Summary: gcc-4.3 miscompiles tagged pointer code
           Product: gcc
           Version: 4.3.0
            Status: UNCONFIRMED
          Severity: critical
          Priority: P3
         Component: other
        AssignedTo: unassigned at gcc dot gnu dot org
        ReportedBy: mikpe at it dot uu dot se
 GCC build triplet: x86_64-unknown-linux-gnu
  GCC host triplet: x86_64-unknown-linux-gnu
GCC target triplet: x86_64-unknown-linux-gnu


http://gcc.gnu.org/bugzilla/show_bug.cgi?id=36339