[PATCH] complex type canonicalization

[Nathan Sidwell <nathan at acm dot org>]

This patch fixes PR 83187, a C++ ICE with alias sets.  As Jakub observed 
in the bug, we ended up with a type whose main variant's canonical 
type's main variant was not itself.  That should be an invariant.

In build_complex_type, we create a probe type:

  tree probe = make_node (COMPLEX_TYPE);
  TREE_TYPE (probe) = TYPE_MAIN_VARIANT (component_type);

notice setting TREE_TYPE to main_variant of the component type.  We then 
insert this into the canonical type hash (or find a match).

I think my type hashing clean up exposed an underlying problem, when I 
changed:
 hstate.add_object (TYPE_HASH (component_type));
to
 hashval_t hash = type_hash_canon_hash (t);

because that's abstractly better.  However, build_complex_type then 
proceeds to do two strange things.

1) regardless of whether it added a new type to the hash table, it 
checks for t == CANONICAL (t) and if equal goes to recreate the 
canonical type.

2) but it checks the canonicalness of component_type, not 
TYPE_MAIN_VARIANT (component_type), which is what the newly created 
complex's TREE_TYPE was set to.

#2 is where I think things are getting confused.  Anyway, this patch 
simply moves all of the post-insertion setting after a check to see 
whether the insertion inserted the new type, (rather than find an 
existing variant).  It also consistently uses TREE_TYPE (new_type), when 
looking at the component type in that context.

booted on x86_64-linux-gnu.  no C or C++ regressions.  ok?

nathan
--
Nathan Sidwell

2017-11-28  Nathan Sidwell  <nathan@acm.org>

	PR c++/83817
	* tree.c (build_complex_type): Fix canonicalization.  Only fill in
	type if it is new.

	PR c++/83187
	* g++.dg/opt/pr83187.C: New.

Index: testsuite/g++.dg/opt/pr83187.C
===================================================================
--- testsuite/g++.dg/opt/pr83187.C	(revision 0)
+++ testsuite/g++.dg/opt/pr83187.C	(working copy)
@@ -0,0 +1,32 @@
+// { dg-do compile { target c++11 } }
+// { dg-additional-options "-O1 -Wno-pedantic" }
+// PR c++/83187 ICE in get_alias_set due to canonical type confusion.
+
+extern "C" {
+  double cos (double);
+  double sin (double);
+}
+
+template <typename> class COMPLEX;
+
+template <>
+struct COMPLEX<double>
+{
+  COMPLEX(double r, double i);
+
+  __complex__ mem;
+};
+
+COMPLEX<double>::COMPLEX (double r, double i)
+  : mem {r, i} {}
+
+typedef double dbl_t;
+
+dbl_t var;
+
+void foo (COMPLEX<double> *ptr)
+{
+  const dbl_t unused = var;
+
+  *ptr = COMPLEX<double> (cos (var), sin (var));
+}
Index: tree.c
===================================================================
--- tree.c	(revision 255202)
+++ tree.c	(working copy)
@@ -8077,65 +8077,66 @@ build_offset_type (tree basetype, tree t
 tree
 build_complex_type (tree component_type, bool named)
 {
-  tree t;
-
   gcc_assert (INTEGRAL_TYPE_P (component_type)
 	      || SCALAR_FLOAT_TYPE_P (component_type)
 	      || FIXED_POINT_TYPE_P (component_type));
 
   /* Make a node of the sort we want.  */
-  t = make_node (COMPLEX_TYPE);
+  tree probe = make_node (COMPLEX_TYPE);
 
-  TREE_TYPE (t) = TYPE_MAIN_VARIANT (component_type);
+  TREE_TYPE (probe) = TYPE_MAIN_VARIANT (component_type);
 
   /* If we already have such a type, use the old one.  */
-  hashval_t hash = type_hash_canon_hash (t);
-  t = type_hash_canon (hash, t);
-
-  if (!COMPLETE_TYPE_P (t))
-    layout_type (t);
+  hashval_t hash = type_hash_canon_hash (probe);
+  tree t = type_hash_canon (hash, probe);
 
-  if (TYPE_CANONICAL (t) == t)
+  if (t == probe)
     {
-      if (TYPE_STRUCTURAL_EQUALITY_P (component_type))
+      /* We created a new type.  The hash insertion will have laid
+	 out the type.  We need to check the canonicalization and
+	 maybe set the name.  */
+      gcc_checking_assert (COMPLETE_TYPE_P (t)
+			   && !TYPE_NAME (t)
+			   && TYPE_CANONICAL (t) == t);
+
+      if (TYPE_STRUCTURAL_EQUALITY_P (TREE_TYPE (t)))
 	SET_TYPE_STRUCTURAL_EQUALITY (t);
-      else if (TYPE_CANONICAL (component_type) != component_type)
+      else if (TYPE_CANONICAL (TREE_TYPE (t)) != TREE_TYPE (t))
 	TYPE_CANONICAL (t)
-	  = build_complex_type (TYPE_CANONICAL (component_type), named);
-    }
+	  = build_complex_type (TYPE_CANONICAL (TREE_TYPE (t)), named);
 
-  /* We need to create a name, since complex is a fundamental type.  */
-  if (!TYPE_NAME (t) && named)
-    {
-      const char *name;
-      if (component_type == char_type_node)
-	name = "complex char";
-      else if (component_type == signed_char_type_node)
-	name = "complex signed char";
-      else if (component_type == unsigned_char_type_node)
-	name = "complex unsigned char";
-      else if (component_type == short_integer_type_node)
-	name = "complex short int";
-      else if (component_type == short_unsigned_type_node)
-	name = "complex short unsigned int";
-      else if (component_type == integer_type_node)
-	name = "complex int";
-      else if (component_type == unsigned_type_node)
-	name = "complex unsigned int";
-      else if (component_type == long_integer_type_node)
-	name = "complex long int";
-      else if (component_type == long_unsigned_type_node)
-	name = "complex long unsigned int";
-      else if (component_type == long_long_integer_type_node)
-	name = "complex long long int";
-      else if (component_type == long_long_unsigned_type_node)
-	name = "complex long long unsigned int";
-      else
-	name = 0;
-
-      if (name != 0)
-	TYPE_NAME (t) = build_decl (UNKNOWN_LOCATION, TYPE_DECL,
-	    			    get_identifier (name), t);
+      /* We need to create a name, since complex is a fundamental type.  */
+      if (named)
+	{
+	  const char *name = NULL;
+
+	  if (TREE_TYPE (t) == char_type_node)
+	    name = "complex char";
+	  else if (TREE_TYPE (t) == signed_char_type_node)
+	    name = "complex signed char";
+	  else if (TREE_TYPE (t) == unsigned_char_type_node)
+	    name = "complex unsigned char";
+	  else if (TREE_TYPE (t) == short_integer_type_node)
+	    name = "complex short int";
+	  else if (TREE_TYPE (t) == short_unsigned_type_node)
+	    name = "complex short unsigned int";
+	  else if (TREE_TYPE (t) == integer_type_node)
+	    name = "complex int";
+	  else if (TREE_TYPE (t) == unsigned_type_node)
+	    name = "complex unsigned int";
+	  else if (TREE_TYPE (t) == long_integer_type_node)
+	    name = "complex long int";
+	  else if (TREE_TYPE (t) == long_unsigned_type_node)
+	    name = "complex long unsigned int";
+	  else if (TREE_TYPE (t) == long_long_integer_type_node)
+	    name = "complex long long int";
+	  else if (TREE_TYPE (t) == long_long_unsigned_type_node)
+	    name = "complex long long unsigned int";
+
+	  if (name != NULL)
+	    TYPE_NAME (t) = build_decl (UNKNOWN_LOCATION, TYPE_DECL,
+					get_identifier (name), t);
+	}
     }
 
   return build_qualified_type (t, TYPE_QUALS (component_type));