[gcc r12-5714] [PR103437] Process multiplication overflow in priority calculation for allocno assignments

[Vladimir Makarov]

https://gcc.gnu.org/g:d47393d0b4d0d498795c4ae1353e6c156c1c4500

commit r12-5714-gd47393d0b4d0d498795c4ae1353e6c156c1c4500
Author: Vladimir N. Makarov <vmakarov@redhat.com>
Date:   Thu Dec 2 08:29:45 2021 -0500

    [PR103437] Process multiplication overflow in priority calculation for allocno assignments
    
    We process overflows in cost calculations but for huge functions
    priority calculation can overflow as priority can be bigger the cost
    used for it.  The patch fixes the problem.
    
    gcc/ChangeLog:
    
            PR rtl-optimization/103437
            * ira-color.c (setup_allocno_priorities): Process multiplication
            overflow.

Diff:
---
 gcc/ira-color.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/gcc/ira-color.c b/gcc/ira-color.c
index 3d01c60800c..1f80cbea0e2 100644
--- a/gcc/ira-color.c
+++ b/gcc/ira-color.c
@@ -2796,7 +2796,7 @@ static int *allocno_priorities;
 static void
 setup_allocno_priorities (ira_allocno_t *consideration_allocnos, int n)
 {
-  int i, length, nrefs, priority, max_priority, mult;
+  int i, length, nrefs, priority, max_priority, mult, diff;
   ira_allocno_t a;
 
   max_priority = 0;
@@ -2807,11 +2807,14 @@ setup_allocno_priorities (ira_allocno_t *consideration_allocnos, int n)
       ira_assert (nrefs >= 0);
       mult = floor_log2 (ALLOCNO_NREFS (a)) + 1;
       ira_assert (mult >= 0);
-      allocno_priorities[ALLOCNO_NUM (a)]
-	= priority
-	= (mult
-	   * (ALLOCNO_MEMORY_COST (a) - ALLOCNO_CLASS_COST (a))
-	   * ira_reg_class_max_nregs[ALLOCNO_CLASS (a)][ALLOCNO_MODE (a)]);
+      mult *= ira_reg_class_max_nregs[ALLOCNO_CLASS (a)][ALLOCNO_MODE (a)];
+      diff = ALLOCNO_MEMORY_COST (a) - ALLOCNO_CLASS_COST (a);
+      /* Multiplication can overflow for very large functions.
+	 Check the overflow and constrain the result if necessary: */
+      if (__builtin_smul_overflow (mult, diff, &priority)
+	  || priority <= -INT_MAX)
+	priority = diff >= 0 ? INT_MAX : -INT_MAX;
+      allocno_priorities[ALLOCNO_NUM (a)] = priority;
       if (priority < 0)
 	priority = -priority;
       if (max_priority < priority)