Fix fir PR71696 in Libiberty Demangler (6)

[Marcel BÃhme <boehme dot marcel at gmail dot com>]

Hi,

The attached patch fixes the stack overflow in the demangler due to cycles in the references of ârememberedâ mangled types (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71696).

The methods demangle_signature and do_arg in cplus-dem.c allow to ârememberâ mangled type names that can later be referenced and will also be demangled. The method demangle_args demangles those types following any references. So, if there is a cycle in the referencing (or in the simplest case a self-reference), the method enters infinite recursion.

The patch tracks the mangled types that are currently being demangled in a new variable called work->proctypevec. If a referenced type is currently being demangled, the demangling is marked as not successful.

Bootstrapped and regression tested on x86_64-pc-linux-gnu. Test case added to libiberty/testsuite/demangler-expected and checked PR71696 is resolved.


Index: libiberty/ChangeLog
===================================================================
--- libiberty/ChangeLog	(revision 237852)
+++ libiberty/ChangeLog	(working copy)
@@ -1,3 +1,21 @@
+2016-06-30  Marcel BÃhme <boehme.marcel@gmail.com>
+
+	* cplus-dem.c: Prevent infinite recursion when there is a cycle
+	in the referencing of remembered mangled types.
+	(work_stuff): New stack to keep track of the remembered mangled
+	types that are currently being processed. 
+	(push_processed_type): New method to push currently processed
+	remembered type onto the stack.
+	(pop_processed_type): New method to pop currently processed
+	remembered type from the stack.
+	(work_stuff_copy_to_from): Copy values of new variables.
+	(delete_non_B_K_work_stuff): Free stack memory.
+	(demangle_args): Push/Pop currently processed remembered type.
+	(do_type): Do not demangle a cyclic reference and push/pop
+	referenced remembered type.
+
+	* testsuite/demangle-expected: Add tests.
+
 2016-05-31  Alan Modra  <amodra@gmail.com>
 
 	* xmemdup.c (xmemdup): Use xmalloc rather than xcalloc.
Index: libiberty/cplus-dem.c
===================================================================
--- libiberty/cplus-dem.c	(revision 237852)
+++ libiberty/cplus-dem.c	(working copy)
@@ -144,6 +144,9 @@ struct work_stuff
   string* previous_argument; /* The last function argument demangled.  */
   int nrepeats;         /* The number of times to repeat the previous
 			   argument.  */
+  int *proctypevec;     /* Indices of currently processed remembered typevecs.  */
+  int proctypevec_size;
+  int nproctypes;
 };
 
 #define PRINT_ANSI_QUALIFIERS (work -> options & DMGL_ANSI)
@@ -436,6 +439,10 @@ iterate_demangle_function (struct work_stuff *,
 
 static void remember_type (struct work_stuff *, const char *, int);
 
+static void push_processed_type (struct work_stuff *, int);
+
+static void pop_processed_type (struct work_stuff *);
+
 static void remember_Btype (struct work_stuff *, const char *, int, int);
 
 static int register_Btype (struct work_stuff *);
@@ -1302,6 +1309,13 @@ work_stuff_copy_to_from (struct work_stuff *to, st
       memcpy (to->btypevec[i], from->btypevec[i], len);
     }
 
+  if (from->proctypevec)
+    {
+      to->proctypevec = XNEWVEC (int, from->proctypevec_size);
+      memcpy (to->proctypevec, from->proctypevec, 
+	      from->proctypevec_size * sizeof(int));
+    }
+
   if (from->ntmpl_args)
     to->tmpl_argvec = XNEWVEC (char *, from->ntmpl_args);
 
@@ -1336,6 +1350,12 @@ delete_non_B_K_work_stuff (struct work_stuff *work
       work -> typevec = NULL;
       work -> typevec_size = 0;
     }
+  if (work -> proctypevec != NULL)
+    {
+      free (work -> proctypevec);
+      work -> proctypevec = NULL;
+      work -> proctypevec_size = 0;
+    }
   if (work->tmpl_argvec)
     {
       int i;
@@ -3554,6 +3574,8 @@ static int
 do_type (struct work_stuff *work, const char **mangled, string *result)
 {
   int n;
+  int i;
+  int is_proctypevec;
   int done;
   int success;
   string decl;
@@ -3566,6 +3588,7 @@ do_type (struct work_stuff *work, const char **man
 
   done = 0;
   success = 1;
+  is_proctypevec = 0;
   while (success && !done)
     {
       int member;
@@ -3626,7 +3649,14 @@ do_type (struct work_stuff *work, const char **man
 	      success = 0;
 	    }
 	  else
-	    {
+	    for (i = 0; i < work -> nproctypes; i++)
+	      if (work -> proctypevec [i] == n)
+	        success = 0;
+	  
+	  if (success)
+	    {    
+	      is_proctypevec = 1;
+	      push_processed_type (work, n);
 	      remembered_type = work -> typevec[n];
 	      mangled = &remembered_type;
 	    }
@@ -3849,6 +3879,9 @@ do_type (struct work_stuff *work, const char **man
     string_delete (result);
   string_delete (&decl);
 
+  if (is_proctypevec)
+    pop_processed_type (work); 
+
   if (success)
     /* Assume an integral type, if we're not sure.  */
     return (int) ((tk == tk_none) ? tk_integral : tk);
@@ -4261,6 +4294,34 @@ do_arg (struct work_stuff *work, const char **mang
 }
 
 static void
+push_processed_type (struct work_stuff *work, int typevec_index)
+{
+  if (work -> nproctypes >= work -> proctypevec_size)
+    {
+      if (work -> proctypevec_size == 0)
+	{
+	  work -> proctypevec_size = 3;
+	  work -> proctypevec = XNEWVEC (int, work -> proctypevec_size);
+	}
+      else
+	{
+	  if (work -> proctypevec_size > INT_MAX / 2)
+            xmalloc_failed (INT_MAX);
+          work -> proctypevec_size *= 2;
+          work -> proctypevec
+            = XRESIZEVEC (int, work->proctypevec, work->proctypevec_size);
+        }
+    }
+    work -> proctypevec [work -> nproctypes ++] = typevec_index;
+}
+
+static void
+pop_processed_type (struct work_stuff *work)
+{
+  work -> nproctypes --;
+}
+
+static void
 remember_type (struct work_stuff *work, const char *start, int len)
 {
   char *tem;
@@ -4524,10 +4585,13 @@ demangle_args (struct work_stuff *work, const char
 		{
 		  string_append (declp, ", ");
 		}
+	      push_processed_type (work, t);  
 	      if (!do_arg (work, &tem, &arg))
 		{
+		  pop_processed_type (work);
 		  return (0);
 		}
+	      pop_processed_type (work);
 	      if (PRINT_ARG_TYPES)
 		{
 		  string_appends (declp, &arg);
Index: libiberty/testsuite/demangle-expected
===================================================================
--- libiberty/testsuite/demangle-expected	(revision 237852)
+++ libiberty/testsuite/demangle-expected	(working copy)
@@ -4556,3 +4556,8 @@ __vt_90000000000cafebabe
 
 _Z80800000000000000000000
 _Z80800000000000000000000
+#
+# Tests stack overflow PR71696
+
+__10%0__S4_0T0T0
+%0<>::%0(%0<>)