[Bug tree-optimization/84188] New: assume non-null malloc pointers are distinct

["msebor at gcc dot gnu.org" <gcc-bugzilla at gcc dot gnu dot org>]

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84188

            Bug ID: 84188
           Summary: assume non-null malloc pointers are distinct
           Product: gcc
           Version: 8.0
            Status: UNCONFIRMED
          Severity: normal
          Priority: P3
         Component: tree-optimization
          Assignee: unassigned at gcc dot gnu.org
          Reporter: msebor at gcc dot gnu.org
  Target Milestone: ---

The manual documents the effect of the malloc attribute like so:

    This tells the compiler that a function is malloc-like, i.e., that the
pointer P returned by the function cannot alias any other pointer valid when
the function returns, and moreover no pointers to valid objects occur in any
storage addressed by P.

The following test case shows that GCC doesn't take advantage of this property
nearly to the extent it could.  (To enable GCC to make full use of this
property implementations of malloc functions would of course have to avoid such
comparisons internally, but presumably they do that already since that's the
documentation makes it clear that such comparisons must be false.)

$ cat t.c && gcc -O2 -S -Wall -fdump-tree-optimized=/dev/stdout t.c
void* __attribute__ ((malloc, returns_nonnull))
f (unsigned);

void g0 (unsigned n)
{
  extern char a[];

  void *p = f (n);        // p is non-null
  if (p == a)             // must be false
    __builtin_abort ();   // can be eliminated but isn't
}

void g1 (unsigned n)
{
  char a[1];

  void *p = f (n);
  if (p == a)             // folded into false
    __builtin_abort ();   // eliminated (good)
}

void g2 (unsigned n, void *p)
{
  // here, p must be valid when f() returns regardless what
  // it points to otherwise the equality would be undefined
  // therefore p cannot equal q

  void *q = f (n);
  if (p == q)             // must be false
    __builtin_abort ();   // can be eliminated but isn't
}

void g3 (unsigned n)
{
  void *p = f (n);
  void *q = f (n);

  // the relationship between p and q is the same as in g2

  if (p == q)             // must be false
    __builtin_abort ();   // can be eliminated but isn't
}

;; Function g0 (g0, funcdef_no=0, decl_uid=1952, cgraph_uid=0, symbol_order=0)

g0 (unsigned int n)
{
  void * p;

  <bb 2> [local count: 1073741825]:
  p_4 = f (n_2(D));
  if (p_4 == &a)
    goto <bb 3>; [0.00%]
  else
    goto <bb 4>; [99.96%]

  <bb 3> [count: 0]:
  __builtin_abort ();

  <bb 4> [local count: 1073312327]:
  return;

}



;; Function g1 (g1, funcdef_no=1, decl_uid=1958, cgraph_uid=1, symbol_order=1)

g1 (unsigned int n)
{
  <bb 2> [local count: 1073741825]:
  f (n_2(D)); [tail call]
  return;

}



;; Function g2 (g2, funcdef_no=2, decl_uid=1964, cgraph_uid=2, symbol_order=2)

g2 (unsigned int n, void * p)
{
  void * q;

  <bb 2> [local count: 1073741825]:
  q_4 = f (n_2(D));
  if (q_4 == p_5(D))
    goto <bb 3>; [0.00%]
  else
    goto <bb 4>; [99.96%]

  <bb 3> [count: 0]:
  __builtin_abort ();

  <bb 4> [local count: 1073312327]:
  return;

}



;; Function g3 (g3, funcdef_no=3, decl_uid=1968, cgraph_uid=3, symbol_order=3)

g3 (unsigned int n)
{
  void * q;
  void * p;

  <bb 2> [local count: 1073741825]:
  p_4 = f (n_2(D));
  q_6 = f (n_2(D));
  if (p_4 == q_6)
    goto <bb 3>; [0.00%]
  else
    goto <bb 4>; [99.96%]

  <bb 3> [count: 0]:
  __builtin_abort ();

  <bb 4> [local count: 1073312327]:
  return;

}