re PR c++/71696 (Libiberty Demangler segfaults (6))

2016-08-04  Marcel Böhme  <boehme.marcel@gmail.com>

	PR c++/71696
	* cplus-dem.c: Prevent infinite recursion when there is a cycle
	in the referencing of remembered mangled types.
	(work_stuff): New stack to keep track of the remembered mangled
	types that are currently being processed.
	(push_processed_type): New method to push currently processed
	remembered type onto the stack.
	(pop_processed_type): New method to pop currently processed
	remembered type from the stack.
	(work_stuff_copy_to_from): Copy values of new variables.
	(delete_non_B_K_work_stuff): Free stack memory.
	(demangle_args): Push/Pop currently processed remembered type.
	(do_type): Do not demangle a cyclic reference and push/pop
	referenced remembered type.

From-SVN: r239143

libiberty/ChangeLog

@@ -1,3 +1,20 @@
+2016-08-04  Marcel Böhme  <boehme.marcel@gmail.com>
+
+	PR c++/71696
+	* cplus-dem.c: Prevent infinite recursion when there is a cycle
+	in the referencing of remembered mangled types.
+	(work_stuff): New stack to keep track of the remembered mangled
+	types that are currently being processed.
+	(push_processed_type): New method to push currently processed
+	remembered type onto the stack.
+	(pop_processed_type): New method to pop currently processed
+	remembered type from the stack.
+	(work_stuff_copy_to_from): Copy values of new variables.
+	(delete_non_B_K_work_stuff): Free stack memory.
+	(demangle_args): Push/Pop currently processed remembered type.
+	(do_type): Do not demangle a cyclic reference and push/pop
+	referenced remembered type.
+
 2016-07-29  Aldy Hernandez  <aldyh@redhat.com>
 
 	* make-relative-prefix.c (make_relative_prefix_1): Fall back to
@@ -16,7 +33,7 @@
 	(d_template_args_1): Split out from d_template_args.
 	(d_args_length): New.
 
-2016-07-13  Marcel BÃhme  <boehme.marcel@gmail.com>
+2016-07-13  Marcel Böhme  <boehme.marcel@gmail.com>
 
 	PR c++/70926
 	* cplus-dem.c: Handle large values and overflow when demangling

libiberty/cplus-dem.c

@@ -144,6 +144,9 @@ struct work_stuff
   string* previous_argument; /* The last function argument demangled.  */
   int nrepeats;         /* The number of times to repeat the previous
 			   argument.  */
+  int *proctypevec;     /* Indices of currently processed remembered typevecs.  */
+  int proctypevec_size;
+  int nproctypes;
 };
 
 #define PRINT_ANSI_QUALIFIERS (work -> options & DMGL_ANSI)
@@ -436,6 +439,10 @@ iterate_demangle_function (struct work_stuff *,
 
 static void remember_type (struct work_stuff *, const char *, int);
 
+static void push_processed_type (struct work_stuff *, int);
+
+static void pop_processed_type (struct work_stuff *);
+
 static void remember_Btype (struct work_stuff *, const char *, int, int);
 
 static int register_Btype (struct work_stuff *);
@@ -1302,6 +1309,10 @@ work_stuff_copy_to_from (struct work_stuff *to, struct work_stuff *from)
       memcpy (to->btypevec[i], from->btypevec[i], len);
     }
 
+  if (from->proctypevec)
+    to->proctypevec =
+      XDUPVEC (int, from->proctypevec, from->proctypevec_size);
+
   if (from->ntmpl_args)
     to->tmpl_argvec = XNEWVEC (char *, from->ntmpl_args);
 
@@ -1330,11 +1341,17 @@ delete_non_B_K_work_stuff (struct work_stuff *work)
   /* Discard the remembered types, if any.  */
 
   forget_types (work);
-  if (work -> typevec != NULL)
+  if (work->typevec != NULL)
     {
-      free ((char *) work -> typevec);
+      free ((char *) work->typevec);
-      work -> typevec = NULL;
+      work->typevec = NULL;
-      work -> typevec_size = 0;
+      work->typevec_size = 0;
+    }
+  if (work->proctypevec != NULL)
+    {
+      free (work->proctypevec);
+      work->proctypevec = NULL;
+      work->proctypevec_size = 0;
     }
   if (work->tmpl_argvec)
     {
@@ -3555,6 +3572,8 @@ static int
 do_type (struct work_stuff *work, const char **mangled, string *result)
 {
   int n;
+  int i;
+  int is_proctypevec;
   int done;
   int success;
   string decl;
@@ -3567,6 +3586,7 @@ do_type (struct work_stuff *work, const char **mangled, string *result)
 
   done = 0;
   success = 1;
+  is_proctypevec = 0;
   while (success && !done)
     {
       int member;
@@ -3627,8 +3647,15 @@ do_type (struct work_stuff *work, const char **mangled, string *result)
 	      success = 0;
 	    }
 	  else
+	    for (i = 0; i < work->nproctypes; i++)
+	      if (work -> proctypevec [i] == n)
+	        success = 0;
+
+	  if (success)
 	    {    
-	      remembered_type = work -> typevec[n];
+	      is_proctypevec = 1;
+	      push_processed_type (work, n);
+	      remembered_type = work->typevec[n];
 	      mangled = &remembered_type;
 	    }
 	  break;
@@ -3850,6 +3877,9 @@ do_type (struct work_stuff *work, const char **mangled, string *result)
     string_delete (result);
   string_delete (&decl);
 
+  if (is_proctypevec)
+    pop_processed_type (work); 
+
   if (success)
     /* Assume an integral type, if we're not sure.  */
     return (int) ((tk == tk_none) ? tk_integral : tk);
@@ -4262,6 +4292,41 @@ do_arg (struct work_stuff *work, const char **mangled, string *result)
   return 1;
 }
 
+static void
+push_processed_type (struct work_stuff *work, int typevec_index)
+{
+  if (work->nproctypes >= work->proctypevec_size)
+    {
+      if (!work->proctypevec_size)
+	{
+	  work->proctypevec_size = 4;
+	  work->proctypevec = XNEWVEC (int, work->proctypevec_size);
+	}
+      else 
+	{
+	  if (work->proctypevec_size < 16)
+	    /* Double when small.  */
+	    work->proctypevec_size *= 2;
+	  else
+	    {
+	      /* Grow slower when large.  */
+	      if (work->proctypevec_size > (INT_MAX / 3) * 2)
+                xmalloc_failed (INT_MAX);
+              work->proctypevec_size = (work->proctypevec_size * 3 / 2);
+	    }   
+          work->proctypevec
+            = XRESIZEVEC (int, work->proctypevec, work->proctypevec_size);
+	}
+    }
+    work->proctypevec [work->nproctypes++] = typevec_index;
+}
+
+static void
+pop_processed_type (struct work_stuff *work)
+{
+  work->nproctypes--;
+}
+
 static void
 remember_type (struct work_stuff *work, const char *start, int len)
 {
@@ -4526,10 +4591,13 @@ demangle_args (struct work_stuff *work, const char **mangled,
 		{
 		  string_append (declp, ", ");
 		}
+	      push_processed_type (work, t);  
 	      if (!do_arg (work, &tem, &arg))
 		{
+		  pop_processed_type (work);
 		  return (0);
 		}
+	      pop_processed_type (work);
 	      if (PRINT_ARG_TYPES)
 		{
 		  string_appends (declp, &arg);

libiberty/testsuite/demangle-expected

@@ -4587,3 +4587,8 @@ _Z80800000000000000000000
 
 __t2m05B500000000000000000_
 __t2m05B500000000000000000_
+#
+# Tests stack overflow PR71696
+
+__10%0__S4_0T0T0
+%0<>::%0(%0<>)