[Bug tree-optimization/17552] New: Infinite recursion in analyze_scalar_evolution

["jakub at gcc dot gnu dot org" <gcc-bugzilla at gcc dot gnu dot org>]

/* { dg-do compile } */
/* { dg-options "-Os" } */

typedef __SIZE_TYPE__ size_t;

struct S1
{
  void *s1a;
  void **s1b;
};

struct S2
{
  void *s2a;
  unsigned int s2b;
  struct S1 *s2c;
};

extern void *memset (void *__s, int __c, size_t __n);
extern void *fn1 (void *, void *);
void fn2 (unsigned long *);
void *fn3 (void *, size_t);
extern int fn4 (void *, void *);

static int
f1 (struct S2 *x, unsigned int y)
{
  register struct S1 *t;
  unsigned int v;
  struct S1 *w;

  if (y >= x->s2b)
    {
      v = x->s2b;
      if (v == 0)
        v = 4;
      while (y >= v)
        v *= 2;
      x->s2c = ((struct S1 *) fn3 (x->s2c, v * sizeof *x->s2c));
      memset (x->s2c + x->s2b, 0,
              (v - x->s2b) * sizeof *x->s2c);
      w = x->s2c + v;
      for (t = x->s2c + x->s2b; t < w; t++)
        t->s1a = 0;
      x->s2b = v;
    }
  return 1;
}

int
test (struct S2 *x, const unsigned char **y)
{
  unsigned char u;

  u = **y;
  ++*y;
  if (u == 4)
    {
      unsigned long k;
      void *l;

      fn2 (&k);
      k -= 256;
      if (!f1 (x, k))
        return 0;
      l = x->s2c[k].s1a;
      if (fn4 (x->s2a, l) == 10)
        l = fn1 (x->s2a, l);
    }
  return 1;
}

causes segfault on x86-64 at -Os, apparently because of infinite recursion
eats all of stack.
#0  0x0000000000957153 in eq_scev_info (e1=Cannot access memory at address
0x7fbf3ffff8
) at ../../gcc/tree-scalar-evolution.c:317
#1  0x0000000000a0719a in htab_find_slot_with_hash (htab=0xd364d0,
element=0x7fbf400070, hash=104, insert=INSERT)
    at ../../libiberty/hashtab.c:660
#2  0x00000000009571c2 in find_var_scev_info (var=0x2a97d80be0) at
../../gcc/tree-scalar-evolution.c:344
#3  0x0000000000958351 in get_scalar_evolution (scalar=0x2a97d80be0) at
../../gcc/tree-scalar-evolution.c:655
#4  0x000000000095c558 in analyze_scalar_evolution (loop=0xd34e30,
var=0x2a97d80be0) at ../../gcc/tree-scalar-evolution.c:1913
#5  0x000000000095c00a in interpret_rhs_modify_expr (loop=0xd34e30,
opnd1=0x2a97d7c370, type=0x2a97c0ba80)
    at ../../gcc/tree-scalar-evolution.c:1757
#6  0x000000000095c433 in analyze_scalar_evolution_1 (loop=0xd34e30,
var=0x2a97d8a320, res=0x0)
    at ../../gcc/tree-scalar-evolution.c:1856
#7  0x000000000095c568 in analyze_scalar_evolution (loop=0xd34e30,
var=0x2a97d8a320) at ../../gcc/tree-scalar-evolution.c:1913
#8  0x000000000095b96e in interpret_condition_phi (loop=0xd34e30,
condition_phi=0x2a97c0d800)
    at ../../gcc/tree-scalar-evolution.c:1700
#9  0x000000000095c466 in analyze_scalar_evolution_1 (loop=0xd34e30,
var=0x2a97d899b0, res=0x0)
    at ../../gcc/tree-scalar-evolution.c:1863
#10 0x000000000095c568 in analyze_scalar_evolution (loop=0xd34e30,
var=0x2a97d899b0) at ../../gcc/tree-scalar-evolution.c:1913
#11 0x000000000095b96e in interpret_condition_phi (loop=0xd34e30,
condition_phi=0x2a97c0da00)
    at ../../gcc/tree-scalar-evolution.c:1700

-- 
           Summary: Infinite recursion in analyze_scalar_evolution
           Product: gcc
           Version: 4.0.0
            Status: UNCONFIRMED
          Severity: normal
          Priority: P2
         Component: tree-optimization
        AssignedTo: unassigned at gcc dot gnu dot org
        ReportedBy: jakub at gcc dot gnu dot org
                CC: gcc-bugs at gcc dot gnu dot org
GCC target triplet: x86_64-redhat-linux


http://gcc.gnu.org/bugzilla/show_bug.cgi?id=17552