[PATCH 3/3] [gomp] Add thread attribute customization

[Sebastian Huber <sebastian dot huber at embedded-brains dot de>]

libgomp/ChangeLog
2015-07-28  Sebastian Huber  <sebastian.huber@embedded-brains.de>

	* config/posix/pool.h (gomp_adjust_thread_attr): New.
	* config/rtems/pool.h (gomp_adjust_thread_attr): Likewise.
	(gomp_thread_pool_reservoir): Add priority member.
	* confi/rtems/proc.c (allocate_thread_pool_reservoir): Add
	priority.
	(parse_thread_pools): Likewise.
	* team.c (gomp_team_start): Rename thread_attr to mutable_attr.
	Call configuration provided gomp_adjust_thread_attr(). Destroy
	mutable attributes if necessary.
	* libgomp.texi: Document GOMP_RTEMS_THREAD_POOLS.
---
 libgomp/config/posix/pool.h |  7 +++++
 libgomp/config/rtems/pool.h | 29 ++++++++++++++++++
 libgomp/config/rtems/proc.c | 23 +++++++++++---
 libgomp/libgomp.texi        | 75 +++++++++++++++++++++++++++++++++++----------
 libgomp/team.c              | 15 ++++-----
 5 files changed, 121 insertions(+), 28 deletions(-)

diff --git a/libgomp/config/posix/pool.h b/libgomp/config/posix/pool.h
index 0d127a0..a8e2eec 100644
--- a/libgomp/config/posix/pool.h
+++ b/libgomp/config/posix/pool.h
@@ -57,4 +57,11 @@ gomp_release_thread_pool (struct gomp_thread_pool *pool)
   /* Do nothing in the default implementation.  */
 }
 
+static inline pthread_attr_t *
+gomp_adjust_thread_attr (pthread_attr_t *attr, pthread_attr_t *mutable_attr)
+{
+  /* Do nothing in the default implementation.  */
+  return attr;
+}
+
 #endif /* GOMP_POOL_H */
diff --git a/libgomp/config/rtems/pool.h b/libgomp/config/rtems/pool.h
index 5c989d0..facac05 100644
--- a/libgomp/config/rtems/pool.h
+++ b/libgomp/config/rtems/pool.h
@@ -41,6 +41,7 @@ struct gomp_thread_pool_reservoir {
   gomp_sem_t available;
   gomp_mutex_t lock;
   size_t index;
+  int priority;
   struct gomp_thread_pool *pools[];
 };
 
@@ -125,4 +126,32 @@ gomp_release_thread_pool (struct gomp_thread_pool *pool)
     }
 }
 
+static inline pthread_attr_t *
+gomp_adjust_thread_attr (pthread_attr_t *attr, pthread_attr_t *mutable_attr)
+{
+  struct gomp_thread_pool_reservoir *res = gomp_get_thread_pool_reservoir ();
+  if (res != NULL && res->priority > 0)
+    {
+      struct sched_param param;
+      int err;
+      if (attr != mutable_attr)
+	{
+	  attr = mutable_attr;
+	  pthread_attr_init (attr);
+	}
+      memset (&param, 0, sizeof (param));
+      param.sched_priority = res->priority;
+      err = pthread_attr_setschedparam (attr, &param);
+      if (err != 0)
+	gomp_fatal ("Thread attribute set scheduler parameters failed: %s", strerror (err));
+      err = pthread_attr_setschedpolicy (attr, SCHED_FIFO);
+      if (err != 0)
+	gomp_fatal ("Thread attribute set scheduler policy failed: %s", strerror (err));
+      err = pthread_attr_setinheritsched (attr, PTHREAD_EXPLICIT_SCHED);
+      if (err != 0)
+	gomp_fatal ("Thread attribute set explicit scheduler failed: %s", strerror (err));
+    }
+  return attr;
+}
+
 #endif /* GOMP_POOL_H */
diff --git a/libgomp/config/rtems/proc.c b/libgomp/config/rtems/proc.c
index 9c36dcb..2939928 100644
--- a/libgomp/config/rtems/proc.c
+++ b/libgomp/config/rtems/proc.c
@@ -48,7 +48,8 @@ allocate_thread_pool_reservoirs (void)
 }
 
 static void
-allocate_thread_pool_reservoir (unsigned long count, unsigned long scheduler)
+allocate_thread_pool_reservoir (unsigned long count, unsigned long priority,
+				unsigned long scheduler)
 {
   struct gomp_thread_pool_reservoir *res;
   struct gomp_thread_pool *pools;
@@ -63,6 +64,7 @@ allocate_thread_pool_reservoir (unsigned long count, unsigned long scheduler)
   memset (pools, 0, size);
   res = (struct gomp_thread_pool_reservoir *) (pools + count);
   res->index = count;
+  res->priority = priority;
   gomp_sem_init (&res->available, count);
   gomp_mutex_init (&res->lock);
   for (i = 0; i < count; ++i)
@@ -71,7 +73,8 @@ allocate_thread_pool_reservoir (unsigned long count, unsigned long scheduler)
 }
 
 static char *
-parse_thread_pools (char *env, unsigned long *count, unsigned long *scheduler)
+parse_thread_pools (char *env, unsigned long *count, unsigned long *priority,
+		    unsigned long *scheduler)
 {
   size_t len;
   int i;
@@ -84,6 +87,17 @@ parse_thread_pools (char *env, unsigned long *count, unsigned long *scheduler)
   if (errno != 0)
     gomp_fatal ("Invalid thread pool count");
 
+  if (*env == '$')
+    {
+      ++env;
+      errno = 0;
+      *priority = strtoul (env, &env, 10);
+      if (errno != 0)
+	gomp_fatal ("Invalid thread pool priority");
+    }
+  else
+    *priority = -1;
+
   if (*env != '@')
     gomp_fatal ("Invalid thread pool scheduler prefix");
   ++env;
@@ -110,9 +124,10 @@ init_thread_pool_reservoirs (void)
       while (*env != '\0')
 	{
 	  unsigned long count;
+	  unsigned long priority;
 	  unsigned long scheduler;
-	  env = parse_thread_pools (env, &count, &scheduler);
-	  allocate_thread_pool_reservoir (count, scheduler);
+	  env = parse_thread_pools (env, &count, &priority, &scheduler);
+	  allocate_thread_pool_reservoir (count, priority, scheduler);
 	}
     }
 }
diff --git a/libgomp/libgomp.texi b/libgomp/libgomp.texi
index 6c7f1ae..06b1c67 100644
--- a/libgomp/libgomp.texi
+++ b/libgomp/libgomp.texi
@@ -1306,23 +1306,24 @@ section 4 of the OpenMP specification in version 4.0, while those
 beginning with @env{GOMP_} are GNU extensions.
 
 @menu
-* OMP_CANCELLATION::      Set whether cancellation is activated
-* OMP_DISPLAY_ENV::       Show OpenMP version and environment variables
-* OMP_DEFAULT_DEVICE::    Set the device used in target regions
-* OMP_DYNAMIC::           Dynamic adjustment of threads
-* OMP_MAX_ACTIVE_LEVELS:: Set the maximum number of nested parallel regions
-* OMP_NESTED::            Nested parallel regions
-* OMP_NUM_THREADS::       Specifies the number of threads to use
-* OMP_PROC_BIND::         Whether theads may be moved between CPUs
-* OMP_PLACES::            Specifies on which CPUs the theads should be placed
-* OMP_STACKSIZE::         Set default thread stack size
-* OMP_SCHEDULE::          How threads are scheduled
-* OMP_THREAD_LIMIT::      Set the maximum number of threads
-* OMP_WAIT_POLICY::       How waiting threads are handled
-* GOMP_CPU_AFFINITY::     Bind threads to specific CPUs
-* GOMP_DEBUG::            Enable debugging output
-* GOMP_STACKSIZE::        Set default thread stack size
-* GOMP_SPINCOUNT::        Set the busy-wait spin count
+* OMP_CANCELLATION::        Set whether cancellation is activated
+* OMP_DISPLAY_ENV::         Show OpenMP version and environment variables
+* OMP_DEFAULT_DEVICE::      Set the device used in target regions
+* OMP_DYNAMIC::             Dynamic adjustment of threads
+* OMP_MAX_ACTIVE_LEVELS::   Set the maximum number of nested parallel regions
+* OMP_NESTED::              Nested parallel regions
+* OMP_NUM_THREADS::         Specifies the number of threads to use
+* OMP_PROC_BIND::           Whether theads may be moved between CPUs
+* OMP_PLACES::              Specifies on which CPUs the theads should be placed
+* OMP_STACKSIZE::           Set default thread stack size
+* OMP_SCHEDULE::            How threads are scheduled
+* OMP_THREAD_LIMIT::        Set the maximum number of threads
+* OMP_WAIT_POLICY::         How waiting threads are handled
+* GOMP_CPU_AFFINITY::       Bind threads to specific CPUs
+* GOMP_DEBUG::              Enable debugging output
+* GOMP_STACKSIZE::          Set default thread stack size
+* GOMP_SPINCOUNT::          Set the busy-wait spin count
+* GOMP_RTEMS_THREAD_POOLS:: Set the RTEMS specific thread pools
 @end menu
 
 
@@ -1705,6 +1706,46 @@ or @env{OMP_WAIT_POLICY} is @code{PASSIVE}.
 
 
 
+@node GOMP_RTEMS_THREAD_POOLS
+@section @env{GOMP_RTEMS_THREAD_POOLS} -- Set the RTEMS specific thread pools
+@cindex Environment Variable
+@cindex Implementation specific setting
+@table @asis
+@item @emph{Description}:
+This environment variable is only used on the RTEMS real-time operating system.
+It determines the scheduler instance specific thread pools.  The format for
+@env{GOMP_RTEMS_THREAD_POOLS} is a list of optional
+@code{<thread-pool-count>[$<priority>]@@<scheduler-name>} configurations
+separated by @code{:} where:
+@itemize @bullet
+@item @code{<thread-pool-count>} is the thread pool count for this scheduler
+instance.
+@item @code{$<priority>} is an optional priority for the worker threads of a
+thread pool according to @code{pthread_setschedparam}.  In case a priority
+value is omitted, then a worker thread will inherit the priority of the OpenMP
+master thread that created it.  The priority of the worker thread is not
+changed after creation, even if a new OpenMP master thread using the worker has
+a different priority.
+@item @code{@@<scheduler-name>} is the scheduler instance name according to the
+RTEMS application configuration.
+@end itemize
+In case no thread pool configuration is specified for a scheduler instance,
+then each OpenMP master thread of this scheduler instance will use its own
+dynamically allocated thread pool.  To limit the worker thread count of the
+thread pools, each OpenMP master thread must call @code{omp_set_num_threads}.
+@item @emph{Example}:
+Lets suppose we have three scheduler instances @code{IO}, @code{WRK0}, and
+@code{WRK1} with @env{GOMP_RTEMS_THREAD_POOLS} set to
+@code{"1@@WRK0:3$4@@WRK1"}.  Then there are no thread pool restrictions for
+scheduler instance @code{IO}.  In the scheduler instance @code{WRK0} there is
+one thread pool available.  Since no priority is specified for this scheduler
+instance, the worker thread inherits the priority of the OpenMP master thread
+that created it.  In the scheduler instance @code{WRK1} there are three thread
+pools available and their worker threads run at priority four.
+@end table
+
+
+
 @c ---------------------------------------------------------------------
 @c The libgomp ABI
 @c ---------------------------------------------------------------------
diff --git a/libgomp/team.c b/libgomp/team.c
index 5edae07..2a51b66 100644
--- a/libgomp/team.c
+++ b/libgomp/team.c
@@ -292,7 +292,7 @@ gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads,
   bool nested;
   struct gomp_thread_pool *pool;
   unsigned i, n, old_threads_used = 0;
-  pthread_attr_t thread_attr, *attr;
+  pthread_attr_t mutable_attr, *attr;
   unsigned long nthreads_var;
   char bind, bind_var;
   unsigned int s = 0, rest = 0, p = 0, k = 0;
@@ -697,11 +697,11 @@ gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads,
   if (__builtin_expect (gomp_places_list != NULL, 0))
     {
       size_t stacksize;
-      pthread_attr_init (&thread_attr);
-      pthread_attr_setdetachstate (&thread_attr, PTHREAD_CREATE_DETACHED);
+      attr = &mutable_attr;
+      pthread_attr_init (attr);
+      pthread_attr_setdetachstate (attr, PTHREAD_CREATE_DETACHED);
       if (! pthread_attr_getstacksize (&gomp_thread_attr, &stacksize))
-	pthread_attr_setstacksize (&thread_attr, stacksize);
-      attr = &thread_attr;
+	pthread_attr_setstacksize (attr, stacksize);
     }
 
   start_data = gomp_alloca (sizeof (struct gomp_thread_start_data)
@@ -799,13 +799,14 @@ gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads,
       start_data->thread_pool = pool;
       start_data->nested = nested;
 
+      attr = gomp_adjust_thread_attr (attr, &mutable_attr);
       err = pthread_create (&pt, attr, gomp_thread_start, start_data++);
       if (err != 0)
 	gomp_fatal ("Thread creation failed: %s", strerror (err));
     }
 
-  if (__builtin_expect (gomp_places_list != NULL, 0))
-    pthread_attr_destroy (&thread_attr);
+  if (__builtin_expect (attr == &mutable_attr, 0))
+    pthread_attr_destroy (attr);
 
  do_release:
   gomp_barrier_wait (nested ? &team->barrier : &pool->threads_dock);
-- 
1.8.4.5