[RFC] GC: external thread suspension

[Keith Seitz <keiths at redhat dot com>]

Hi,

At long last I am returning to JDWP work, and one of the most pressing 
issues is still modification of the GC to allow for external thread 
suspension.

I've attached a patch that I've been using for my JDWP testing. It seems 
to work. I'm sure there are problems, and I'm sure that I am just plain 
misinformed about some of what's going on in pthread_stop_world.c, but I 
think it's close. More knowledgeable eyes needed.

I sent an earlier version of this patch against our older GC sources to 
the GC mailing list, but no one ever responded. So now I am submitting 
the patches here in hopes of someone seeing it, reviewing it, and 
something going somewhere.

For a description of the changes, please see the GC list message I left 
many months ago (which includes my requirements for this functionality):

http://www.hpl.hp.com/hosted/linux/mail-archives/gc/2005-December/001061.html

Keith
Index: include/private/pthread_support.h
===================================================================
--- include/private/pthread_support.h	(revision 112226)
+++ include/private/pthread_support.h	(working copy)
@@ -33,6 +33,8 @@
 #	define FINISHED 1   	/* Thread has exited.	*/
 #	define DETACHED 2	/* Thread is intended to be detached.	*/
 #	define MAIN_THREAD 4	/* True for the original thread only.	*/
+#       define SUSPENDED 8      /* True if thread was suspended externally */
+#       define SUSPENDED_SELF 16 /* True if thread suspended itself */
     short thread_blocked;	/* Protected by GC lock.		*/
     				/* Treated as a boolean value.  If set,	*/
     				/* thread will acquire GC lock before	*/
Index: pthread_stop_world.c
===================================================================
--- pthread_stop_world.c	(revision 112226)
+++ pthread_stop_world.c	(working copy)
@@ -185,11 +185,17 @@
 	me -> backing_store_ptr = (ptr_t)GC_save_regs_in_stack();
 #   endif
 
-    /* Tell the thread that wants to stop the world that this   */
-    /* thread has been stopped.  Note that sem_post() is  	*/
-    /* the only async-signal-safe primitive in LinuxThreads.    */
-    sem_post(&GC_suspend_ack_sem);
-    me -> stop_info.last_stop_count = my_stop_count;
+	if ((me -> flags & SUSPENDED_SELF)) {
+	  /* If this thread is suspending itself, we must release */
+	  /* the allocation lock.                                 */
+	  UNLOCK();
+	} else {
+	  /* Tell the thread that wants to stop the world that this   */
+	  /* thread has been stopped.  Note that sem_post() is  	*/
+	  /* the only async-signal-safe primitive in LinuxThreads.    */
+	  sem_post(&GC_suspend_ack_sem);
+	  me -> stop_info.last_stop_count = my_stop_count;
+	}
 
     /* Wait until that thread tells us to restart by sending    */
     /* this thread a SIG_THR_RESTART signal.			*/
@@ -203,7 +209,8 @@
     /* really safe to proceed.  Under normal circumstances,	*/
     /* this code should not be executed.			*/
     sigsuspend(&suspend_handler_mask);        /* Wait for signal */
-    while (GC_world_is_stopped && GC_stop_count == my_stop_count) {
+    while ((!(me -> flags & SUSPENDED) && GC_world_is_stopped)
+	   && GC_stop_count == my_stop_count) {
         GC_brief_async_signal_safe_sleep();
 #       if DEBUG_THREADS
 	  GC_err_printf0("Sleeping in signal handler");
@@ -314,6 +321,33 @@
       ABORT("Collecting from unknown thread.");
 }
 
+static int suspend_thread(GC_thread thread) {
+  int result;
+
+  if (thread -> flags & SUSPENDED) return 0;
+  if (thread -> flags & FINISHED) return 0;
+  if (thread -> stop_info.last_stop_count == GC_stop_count) return 0;
+  if (thread -> thread_blocked) /* Will wait */ return 0;
+
+#if DEBUG_THREADS
+  GC_printf1("Sending suspend signal to 0x%lx\n", thread -> id);
+#endif
+  
+  result = pthread_kill(thread -> id, SIG_SUSPEND);
+  switch(result) {
+  case ESRCH:
+    /* Not really there anymore.  Possible? */
+    return 0;
+    break;
+  case 0:
+    break;
+  default:
+    ABORT("pthread_kill failed");
+  }
+
+  return 1;
+}
+
 /* There seems to be a very rare thread stopping problem.  To help us  */
 /* debug that, we save the ids of the stopping thread. */
 pthread_t GC_stopping_thread;
@@ -335,40 +369,42 @@
     for (i = 0; i < THREAD_TABLE_SZ; i++) {
       for (p = GC_threads[i]; p != 0; p = p -> next) {
         if (p -> id != my_thread) {
-            if (p -> flags & FINISHED) continue;
-            if (p -> stop_info.last_stop_count == GC_stop_count) continue;
-	    if (p -> thread_blocked) /* Will wait */ continue;
-            n_live_threads++;
-	    #if DEBUG_THREADS
-	      GC_printf1("Sending suspend signal to 0x%lx\n", p -> id);
-	    #endif
-        
-        result = pthread_kill(p -> id, SIG_SUSPEND);
-	    switch(result) {
-                case ESRCH:
-                    /* Not really there anymore.  Possible? */
-                    n_live_threads--;
-                    break;
-                case 0:
-                    break;
-                default:
-                    ABORT("pthread_kill failed");
-            }
+	  n_live_threads +=  suspend_thread(p);
         }
       }
     }
     return n_live_threads;
 }
 
-/* Caller holds allocation lock.	*/
-void GC_stop_world()
+/* This function is meant to be called by a thread (known by the GC) to */
+/* suspend itself, i.e., a thread suspending itself because it hit a    */
+/* breakpoint in an interpreted environment.                            */
+void GC_suspend_self()
 {
+  pthread_t thread = pthread_self();
+  GC_thread me = GC_lookup_thread(thread);
+  if (me != 0) {
+    LOCK();
+    me -> flags |= (SUSPENDED_SELF | SUSPENDED);
+    GC_suspend_handler(SIG_SUSPEND);
+#ifdef DEBUG_THREADS
+  } else {
+    GC_err_printf0("Unknown thread 0x%lx trying to suspend self\n", thread);
+#endif
+  }
+}
+
+/* Caller holds allocation lock. */
+void GC_stop_world_inner (GC_thread thread)
+{
     int i;
     int n_live_threads;
     int code;
 
     #if DEBUG_THREADS
-    GC_printf1("Stopping the world from 0x%lx\n", pthread_self());
+    GC_printf1("Stopping the %s from 0x%lx\n",
+	       (thread == NULL) ? "thread" : "world",
+	       pthread_self());
     #endif
        
     /* Make sure all free list construction has stopped before we start. */
@@ -382,7 +418,10 @@
 #   endif /* PARALLEL_MARK */
     ++GC_stop_count;
     GC_world_is_stopped = TRUE;
-    n_live_threads = GC_suspend_all();
+    if (thread != 0)
+      n_live_threads = suspend_thread(thread);
+    else
+      n_live_threads = GC_suspend_all();
 
       if (GC_retry_signals) {
 	  unsigned long wait_usecs = 0;  /* Total wait since retry.	*/
@@ -394,7 +433,11 @@
 	      sem_getvalue(&GC_suspend_ack_sem, &ack_count);
 	      if (ack_count == n_live_threads) break;
 	      if (wait_usecs > RETRY_INTERVAL) {
-		  int newly_sent = GC_suspend_all();
+		  int newly_sent;
+		  if (thread != 0)
+		    newly_sent = suspend_thread(thread);
+		  else
+		    newly_sent = GC_suspend_all();
 
 #                 ifdef CONDPRINT
 		    if (GC_print_stats) {
@@ -425,11 +468,69 @@
       GC_release_mark_lock();
 #   endif
     #if DEBUG_THREADS
-      GC_printf1("World stopped from 0x%lx\n", pthread_self());
+      GC_printf1("%s stopped from 0x%lx\n",
+	       (thread == NULL) ? "Thread" : "World",
+		 pthread_self());
     #endif
     GC_stopping_thread = 0;  /* debugging only */
 }
 
+/* Caller holds allocation lock.	*/
+void GC_stop_world() {
+  GC_stop_world_inner(NULL);
+}
+
+/* Allows a thread to suspend another thread. Returns boolean indicating */
+/* success of suspension.                                                */
+int GC_suspend_thread(pthread_t thread) {
+  GC_thread t = GC_lookup_thread(thread);
+#if DEBUG_THREADS
+  if (t == 0) {
+    GC_err_printf0 ("Thread 0x%lx is not under GC control\n", thread);
+  }
+#endif
+  if (t != 0 && !(t -> flags & SUSPENDED)) {
+    GC_stopping_thread = pthread_self(); /* debugging only.      */
+    GC_stopping_pid = getpid();                /* debugging only.      */
+    LOCK();
+    GC_stop_world_inner(t);
+    t -> flags |= SUSPENDED;
+#if DEBUG_THREADS
+    GC_printf0 ("Suspended thread 0x%lx\n", thread);
+#endif
+    UNLOCK();
+    return 1;
+  }
+
+  return 0;
+}
+
+static int resume_thread (GC_thread thread)
+{
+  int result;
+
+  if (thread -> flags & FINISHED) return 0;
+  if (thread -> thread_blocked) return 0;
+  if (thread -> flags & SUSPENDED) return 0; /* okay -- GC_resume_thread will clear this flag before getting here */
+
+#if DEBUG_THREADS
+  GC_printf1("Sending restart signal to 0x%lx\n", p -> id);
+#endif
+  
+  result = pthread_kill(thread -> id, SIG_THR_RESTART);
+  switch(result) {
+  case ESRCH:
+    /* Not really there anymore.  Possible? */
+    return 0;
+  case 0:
+    break;
+  default:
+    ABORT("pthread_kill failed");
+  }
+
+  return 1;
+}
+
 /* Caller holds allocation lock, and has held it continuously since	*/
 /* the world stopped.							*/
 void GC_start_world()
@@ -438,41 +539,40 @@
     register int i;
     register GC_thread p;
     register int n_live_threads = 0;
-    register int result;
 
 #   if DEBUG_THREADS
       GC_printf0("World starting\n");
 #   endif
 
-    GC_world_is_stopped = FALSE;
-    for (i = 0; i < THREAD_TABLE_SZ; i++) {
-      for (p = GC_threads[i]; p != 0; p = p -> next) {
-        if (p -> id != my_thread) {
-            if (p -> flags & FINISHED) continue;
-	    if (p -> thread_blocked) continue;
-            n_live_threads++;
-	    #if DEBUG_THREADS
-	      GC_printf1("Sending restart signal to 0x%lx\n", p -> id);
-	    #endif
-            result = pthread_kill(p -> id, SIG_THR_RESTART);
-	    switch(result) {
-                case ESRCH:
-                    /* Not really there anymore.  Possible? */
-                    n_live_threads--;
-                    break;
-                case 0:
-                    break;
-                default:
-                    ABORT("pthread_kill failed");
-            }
-        }
-      }
-    }
+     GC_world_is_stopped = FALSE;
+     for (i = 0; i < THREAD_TABLE_SZ; i++) {
+       for (p = GC_threads[i]; p != 0; p = p -> next) {
+	 if (p -> id != my_thread) {
+	   n_live_threads += resume_thread (p);
+	 }
+       }
+     }
+
     #if DEBUG_THREADS
-      GC_printf0("World started\n");
+       GC_printf0("World started\n");
     #endif
 }
 
+/* This function resumes a thread that was suspended (either by another */
+/* thread via GC_suspend_thread or itself via GC_suspend_self).         */
+void GC_resume_thread (pthread_t thread) {
+  GC_thread t = GC_lookup_thread(thread);
+  if (t != 0) {
+    LOCK();
+    t -> flags &= ~(SUSPENDED | SUSPENDED_SELF);
+    resume_thread (t);
+#if DEBUG_THREADS
+    GC_printf0 ("Resumed thread 0x%lx\n", thread);
+#endif
+    UNLOCK();
+  }
+}
+
 void GC_stop_init() {
     struct sigaction act;