[gcc r13-8165] tree-object-size: Robustify alloc_size attribute handling [PR113013]

[Jakub Jelinek]

https://gcc.gnu.org/g:2ea5a22b3e594cdf88e916b3df774883ce9dbc3a

commit r13-8165-g2ea5a22b3e594cdf88e916b3df774883ce9dbc3a
Author: Jakub Jelinek <jakub@redhat.com>
Date:   Mon Dec 18 09:59:28 2023 +0100

    tree-object-size: Robustify alloc_size attribute handling [PR113013]
    
    The following testcase ICEs because we aren't careful enough with
    alloc_size attribute.  We do check that such an argument exists
    (although wouldn't handle correctly functions with more than INT_MAX
    arguments), but didn't check that it is scalar integer, the ICE is
    trying to fold_convert a structure to sizetype.
    
    Given that the attribute can also appear on non-prototyped functions
    where the arguments aren't known, I don't see how the FE could diagnose
    that and because we already handle the case where argument doesn't exist,
    I think we should also verify the argument is scalar integer convertible
    to sizetype.  Furthermore, given this is not just in diagnostics but
    used for code generation, I think it is better to punt on arguments with
    larger precision then sizetype, the upper bits are then truncated.
    
    The patch also fixes some formatting issues and avoids duplication of the
    fold_convert, plus removes unnecessary check for if (arg1 >= 0), that is
    always the case after if (arg1 < 0) return ...;
    
    2023-12-18  Jakub Jelinek  <jakub@redhat.com>
    
            PR tree-optimization/113013
            * tree-object-size.cc (alloc_object_size): Return size_unknown if
            corresponding argument(s) don't have integral type or have integral
            type with higher precision than sizetype.  Don't check arg1 >= 0
            uselessly.  Compare argument indexes against gimple_call_num_args
            in unsigned type rather than int.  Formatting fixes.
    
            * gcc.dg/pr113013.c: New test.
    
    (cherry picked from commit 5347263b347d02e875879ca40ca6e289ac178919)

Diff:
---
 gcc/testsuite/gcc.dg/pr113013.c | 14 ++++++++++++++
 gcc/tree-object-size.cc         | 30 +++++++++++++++++++++---------
 2 files changed, 35 insertions(+), 9 deletions(-)

diff --git a/gcc/testsuite/gcc.dg/pr113013.c b/gcc/testsuite/gcc.dg/pr113013.c
new file mode 100644
index 00000000000..b34964071b7
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/pr113013.c
@@ -0,0 +1,14 @@
+/* PR tree-optimization/113013 */
+/* { dg-do compile } */
+/* { dg-options "-std=gnu99 -O2" } */
+
+struct S { short x; } s;
+void *foo () __attribute__((__alloc_size__(1)));
+struct S *p;
+
+__SIZE_TYPE__
+bar (void)
+{
+  p = foo (s);
+  return __builtin_dynamic_object_size (p, 0);
+}
diff --git a/gcc/tree-object-size.cc b/gcc/tree-object-size.cc
index 9a936a91983..64b530bbbf9 100644
--- a/gcc/tree-object-size.cc
+++ b/gcc/tree-object-size.cc
@@ -773,21 +773,33 @@ alloc_object_size (const gcall *call, int object_size_type)
         arg2 = TREE_INT_CST_LOW (TREE_VALUE (TREE_CHAIN (p)))-1;
     }
   else if (gimple_call_builtin_p (call, BUILT_IN_NORMAL)
-	   && callfn && ALLOCA_FUNCTION_CODE_P (DECL_FUNCTION_CODE (callfn)))
-  arg1 = 0;
+	   && callfn
+	   && ALLOCA_FUNCTION_CODE_P (DECL_FUNCTION_CODE (callfn)))
+    arg1 = 0;
 
   /* Non-const arguments are OK here, let the caller handle constness.  */
-  if (arg1 < 0 || arg1 >= (int) gimple_call_num_args (call)
-      || arg2 >= (int) gimple_call_num_args (call))
+  if (arg1 < 0
+      || (unsigned) arg1 >= gimple_call_num_args (call)
+      || (arg2 >= 0 && (unsigned) arg2 >= gimple_call_num_args (call)))
     return size_unknown (object_size_type);
 
+  tree targ1 = gimple_call_arg (call, arg1);
+  if (!INTEGRAL_TYPE_P (TREE_TYPE (targ1))
+      || TYPE_PRECISION (TREE_TYPE (targ1)) > TYPE_PRECISION (sizetype))
+    return size_unknown (object_size_type);
+  targ1 = fold_convert (sizetype, targ1);
   tree bytes = NULL_TREE;
   if (arg2 >= 0)
-    bytes = size_binop (MULT_EXPR,
-	fold_convert (sizetype, gimple_call_arg (call, arg1)),
-	fold_convert (sizetype, gimple_call_arg (call, arg2)));
-  else if (arg1 >= 0)
-    bytes = fold_convert (sizetype, gimple_call_arg (call, arg1));
+    {
+      tree targ2 = gimple_call_arg (call, arg2);
+      if (!INTEGRAL_TYPE_P (TREE_TYPE (targ2))
+	  || TYPE_PRECISION (TREE_TYPE (targ2)) > TYPE_PRECISION (sizetype))
+	return size_unknown (object_size_type);
+      targ2 = fold_convert (sizetype, targ2);
+      bytes = size_binop (MULT_EXPR, targ1, targ2);
+    }
+  else
+    bytes = targ1;
 
   return bytes ? bytes : size_unknown (object_size_type);
 }