[PATCH v2] rs6000: ICE when using an MMA type as a function param or return value [PR96506]

[Peter Bergner]

On 8/12/20 8:00 PM, Segher Boessenkool wrote:
> On Wed, Aug 12, 2020 at 03:32:18PM -0500, Peter Bergner wrote:
>> --- a/gcc/config/rs6000/rs6000-call.c
>> +++ b/gcc/config/rs6000/rs6000-call.c
>> @@ -6444,8 +6444,26 @@ machine_mode
>>  rs6000_promote_function_mode (const_tree type ATTRIBUTE_UNUSED,
>>  			      machine_mode mode,
>>  			      int *punsignedp ATTRIBUTE_UNUSED,
>> -			      const_tree, int)
>> +			      const_tree, int for_return)
>>  {
>> +  /* Warning: this is a static local variable and not always NULL!  */
>> +  static struct function *fn = NULL;
> 
> It may be just me that always misses "static" on locals, heh.  But
> please comment what this is *for*: to warn only once per function.  You
> could choose a better variable name to say that, too.

Ok, how about this comment then?

@@ -6444,8 +6444,30 @@ machine_mode
 rs6000_promote_function_mode (const_tree type ATTRIBUTE_UNUSED,
                              machine_mode mode,
                              int *punsignedp ATTRIBUTE_UNUSED,
-                             const_tree, int)
+                             const_tree, int for_return)
 {
+  /* Warning: this is a static local variable and not always NULL!
+     This function is called multiple times for the same function
+     and return value.  PREV_FUNC is used to keep track of the
+     first time we encounter a function's return value in order
+     to not report an error with that return value multiple times.  */
+  static struct function *prev_func = NULL;
+
+  /* We do not allow MMA types being used as return values.  Only report
+     the invalid return value usage the first time we encounter it.  */
+  if (for_return
+      && prev_func != cfun
+      && (mode == POImode || mode == PXImode))
+    {
+      /* Record we have now handled function CFUN, so the next time we
+        are called, we do not re-report the same error.  */
+      prev_func = cfun;
+      if (TYPE_CANONICAL (type) != NULL_TREE)
+       type = TYPE_CANONICAL (type);
+      error ("invalid use of MMA type %qs as a function return value",
+            IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type))));
+    }
+
   PROMOTE_MODE (mode, *punsignedp, type);
 
   return mode;

Peter