This is the mail archive of the gcc-patches@gcc.gnu.org mailing list for the GCC project.
| Index Nav: | [Date Index] [Subject Index] [Author Index] [Thread Index] | |
|---|---|---|
| Message Nav: | [Date Prev] [Date Next] | [Thread Prev] [Thread Next] |
| Other format: | [Raw text] | |
Hi!
All futexes used explicitly in libgomp are process private,
and when kernel supports it (2.6.22 and higher), using private futexes
gives measurable speedups, on the attached microbenchmark primarily
on the parallel bench, single bench and static bench.
Similar speedups can be seen on EPCC microbenchmark.
If the kernel doesn't support FUTEX_PRIVATE_FLAG, the futex syscalls
should return ENOSYS error and libgomp should fall back to normal
futexes.
OMP_SCHEDULE=static,1 GOMP_BLOCKTIME=infinity LD_LIBRARY_PATH=~/libgomp-vanilla/ ./micro
barrier bench 0.993256 seconds
parallel bench 0.693095 seconds
static bench 0.0363882 seconds
dynamic bench 0.182232 seconds
guided bench 0.000468774 seconds
runtime bench 0.0134134 seconds
single bench 1.47163 seconds
OMP_SCHEDULE=static,1 GOMP_BLOCKTIME=infinity LD_LIBRARY_PATH=~/libgomp-privatefutex/ ./micro
barrier bench 0.990152 seconds
parallel bench 0.563748 seconds
static bench 0.0314544 seconds
dynamic bench 0.182004 seconds
guided bench 0.000466495 seconds
runtime bench 0.0133739 seconds
single bench 1.09475 seconds
Regtested on x86_64-linux, i686-linux and ppc-linux.
2008-03-19 Jakub Jelinek <jakub@redhat.com>
* config/linux/wait.h: Include errno.h.
(FUTEX_WAIT, FUTEX_WAKE, FUTEX_PRIVATE_FLAG): Define.
(gomp_futex_wake, gomp_futex_wait): New extern decls.
* config/linux/mutex.c (gomp_futex_wake, gomp_futex_wait): New
variables.
* config/linux/powerpc/futex.h (FUTEX_WAIT, FUTEX_WAKE): Remove.
(sys_futex0): Return error code.
(futex_wake, futex_wait): If ENOSYS was returned, clear
FUTEX_PRIVATE_FLAG in gomp_futex_wa{ke,it} and retry.
* config/linux/alpha/futex.h (FUTEX_WAIT, FUTEX_WAKE): Remove.
(futex_wake, futex_wait): If ENOSYS was returned, clear
FUTEX_PRIVATE_FLAG in gomp_futex_wa{ke,it} and retry.
* config/linux/x86/futex.h (FUTEX_WAIT, FUTEX_WAKE): Remove.
(sys_futex0): Return error code.
(futex_wake, futex_wait): If ENOSYS was returned, clear
FUTEX_PRIVATE_FLAG in gomp_futex_wa{ke,it} and retry.
* config/linux/s390/futex.h (FUTEX_WAIT, FUTEX_WAKE): Remove.
(sys_futex0): Return error code.
(futex_wake, futex_wait): If ENOSYS was returned, clear
FUTEX_PRIVATE_FLAG in gomp_futex_wa{ke,it} and retry.
* config/linux/ia64/futex.h (FUTEX_WAIT, FUTEX_WAKE): Remove.
(sys_futex0): Return error code.
(futex_wake, futex_wait): If ENOSYS was returned, clear
FUTEX_PRIVATE_FLAG in gomp_futex_wa{ke,it} and retry.
* config/linux/sparc/futex.h (FUTEX_WAIT, FUTEX_WAKE): Remove.
(sys_futex0): Return error code.
(futex_wake, futex_wait): If ENOSYS was returned, clear
FUTEX_PRIVATE_FLAG in gomp_futex_wa{ke,it} and retry.
--- libgomp/config/linux/powerpc/futex.h (revision 133291)
+++ libgomp/config/linux/powerpc/futex.h (working copy)
@@ -28,10 +28,8 @@
/* Provide target-specific access to the futex system call. */
#include <sys/syscall.h>
-#define FUTEX_WAIT 0
-#define FUTEX_WAKE 1
-static inline void
+static inline long
sys_futex0 (int *addr, int op, int val)
{
register long int r0 __asm__ ("r0");
@@ -50,23 +48,38 @@ sys_futex0 (int *addr, int op, int val)
doesn't. It doesn't much matter for us. In the interest of unity,
go ahead and clobber it always. */
- __asm volatile ("sc"
+ __asm volatile ("sc; mfcr %0"
: "=r"(r0), "=r"(r3), "=r"(r4), "=r"(r5), "=r"(r6)
: "r"(r0), "r"(r3), "r"(r4), "r"(r5), "r"(r6)
: "r7", "r8", "r9", "r10", "r11", "r12",
"cr0", "ctr", "memory");
+ if (__builtin_expect (r0 & (1 << 28), 0))
+ return r3;
+ return 0;
}
static inline void
futex_wait (int *addr, int val)
{
- sys_futex0 (addr, FUTEX_WAIT, val);
+ long err = sys_futex0 (addr, gomp_futex_wait, val);
+ if (__builtin_expect (err == ENOSYS, 0))
+ {
+ gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG;
+ gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG;
+ sys_futex0 (addr, gomp_futex_wait, val);
+ }
}
static inline void
futex_wake (int *addr, int count)
{
- sys_futex0 (addr, FUTEX_WAKE, count);
+ long err = sys_futex0 (addr, gomp_futex_wake, count);
+ if (__builtin_expect (err == ENOSYS, 0))
+ {
+ gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG;
+ gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG;
+ sys_futex0 (addr, gomp_futex_wake, count);
+ }
}
static inline void
--- libgomp/config/linux/mutex.c (revision 133291)
+++ libgomp/config/linux/mutex.c (working copy)
@@ -31,6 +31,8 @@
#include "wait.h"
+long int gomp_futex_wake = FUTEX_WAKE | FUTEX_PRIVATE_FLAG;
+long int gomp_futex_wait = FUTEX_WAIT | FUTEX_PRIVATE_FLAG;
void
gomp_mutex_lock_slow (gomp_mutex_t *mutex)
--- libgomp/config/linux/alpha/futex.h (revision 133291)
+++ libgomp/config/linux/alpha/futex.h (working copy)
@@ -30,8 +30,6 @@
#ifndef SYS_futex
#define SYS_futex 394
#endif
-#define FUTEX_WAIT 0
-#define FUTEX_WAKE 1
static inline void
@@ -45,7 +43,7 @@ futex_wait (int *addr, int val)
sc_0 = SYS_futex;
sc_16 = (long) addr;
- sc_17 = FUTEX_WAIT;
+ sc_17 = gomp_futex_wait;
sc_18 = val;
sc_19 = 0;
__asm volatile ("callsys"
@@ -53,6 +51,20 @@ futex_wait (int *addr, int val)
: "0"(sc_0), "r" (sc_16), "r"(sc_17), "r"(sc_18), "1"(sc_19)
: "$1", "$2", "$3", "$4", "$5", "$6", "$7", "$8",
"$22", "$23", "$24", "$25", "$27", "$28", "memory");
+ if (__builtin_expect (sc_19, 0) && sc_0 == ENOSYS)
+ {
+ gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG;
+ gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG;
+ sc_0 = SYS_futex;
+ sc_17 &= ~FUTEX_PRIVATE_FLAG;
+ sc_19 = 0;
+ __asm volatile ("callsys"
+ : "=r" (sc_0), "=r"(sc_19)
+ : "0"(sc_0), "r" (sc_16), "r"(sc_17), "r"(sc_18),
+ "1"(sc_19)
+ : "$1", "$2", "$3", "$4", "$5", "$6", "$7", "$8",
+ "$22", "$23", "$24", "$25", "$27", "$28", "memory");
+ }
}
static inline void
@@ -66,13 +78,25 @@ futex_wake (int *addr, int count)
sc_0 = SYS_futex;
sc_16 = (long) addr;
- sc_17 = FUTEX_WAKE;
+ sc_17 = gomp_futex_wake;
sc_18 = count;
__asm volatile ("callsys"
: "=r" (sc_0), "=r"(sc_19)
: "0"(sc_0), "r" (sc_16), "r"(sc_17), "r"(sc_18)
: "$1", "$2", "$3", "$4", "$5", "$6", "$7", "$8",
"$22", "$23", "$24", "$25", "$27", "$28", "memory");
+ if (__builtin_expect (sc_19, 0) && sc_0 == ENOSYS)
+ {
+ gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG;
+ gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG;
+ sc_0 = SYS_futex;
+ sc_17 &= ~FUTEX_PRIVATE_FLAG;
+ __asm volatile ("callsys"
+ : "=r" (sc_0), "=r"(sc_19)
+ : "0"(sc_0), "r" (sc_16), "r"(sc_17), "r"(sc_18)
+ : "$1", "$2", "$3", "$4", "$5", "$6", "$7", "$8",
+ "$22", "$23", "$24", "$25", "$27", "$28", "memory");
+ }
}
static inline void
--- libgomp/config/linux/x86/futex.h (revision 133291)
+++ libgomp/config/linux/x86/futex.h (working copy)
@@ -27,9 +27,6 @@
/* Provide target-specific access to the futex system call. */
-#define FUTEX_WAIT 0
-#define FUTEX_WAKE 1
-
#ifdef __LP64__
# ifndef SYS_futex
# define SYS_futex 202
@@ -38,14 +35,26 @@
static inline void
futex_wait (int *addr, int val)
{
- register long r10 __asm__("%r10") = 0;
+ register long r10 __asm__("%r10");
long res;
+ r10 = 0;
__asm volatile ("syscall"
: "=a" (res)
- : "0"(SYS_futex), "D" (addr), "S"(FUTEX_WAIT),
- "d"(val), "r"(r10)
+ : "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wait),
+ "d" (val), "r" (r10)
: "r11", "rcx", "memory");
+ if (__builtin_expect (res == -ENOSYS, 0))
+ {
+ gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG;
+ gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG;
+ r10 = 0;
+ __asm volatile ("syscall"
+ : "=a" (res)
+ : "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wait),
+ "d" (val), "r" (r10)
+ : "r11", "rcx", "memory");
+ }
}
static inline void
@@ -55,8 +64,19 @@ futex_wake (int *addr, int count)
__asm volatile ("syscall"
: "=a" (res)
- : "0"(SYS_futex), "D" (addr), "S"(FUTEX_WAKE), "d"(count)
+ : "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wake),
+ "d" (count)
: "r11", "rcx", "memory");
+ if (__builtin_expect (res == -ENOSYS, 0))
+ {
+ gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG;
+ gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG;
+ __asm volatile ("syscall"
+ : "=a" (res)
+ : "0" (SYS_futex), "D" (addr), "S" (gomp_futex_wake),
+ "d" (count)
+ : "r11", "rcx", "memory");
+ }
}
#else
# ifndef SYS_futex
@@ -65,7 +85,7 @@ futex_wake (int *addr, int count)
# ifdef __PIC__
-static inline void
+static inline long
sys_futex0 (int *addr, int op, int val)
{
long res;
@@ -77,11 +97,12 @@ sys_futex0 (int *addr, int op, int val)
: "0"(SYS_futex), "r" (addr), "c"(op),
"d"(val), "S"(0)
: "memory");
+ return res;
}
# else
-static inline void
+static inline long
sys_futex0 (int *addr, int op, int val)
{
long res;
@@ -91,6 +112,7 @@ sys_futex0 (int *addr, int op, int val)
: "0"(SYS_futex), "b" (addr), "c"(op),
"d"(val), "S"(0)
: "memory");
+ return res;
}
# endif /* __PIC__ */
@@ -98,13 +120,25 @@ sys_futex0 (int *addr, int op, int val)
static inline void
futex_wait (int *addr, int val)
{
- sys_futex0 (addr, FUTEX_WAIT, val);
+ long res = sys_futex0 (addr, gomp_futex_wait, val);
+ if (__builtin_expect (res == -ENOSYS, 0))
+ {
+ gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG;
+ gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG;
+ sys_futex0 (addr, gomp_futex_wait, val);
+ }
}
static inline void
futex_wake (int *addr, int count)
{
- sys_futex0 (addr, FUTEX_WAKE, count);
+ long res = sys_futex0 (addr, gomp_futex_wake, count);
+ if (__builtin_expect (res == -ENOSYS, 0))
+ {
+ gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG;
+ gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG;
+ sys_futex0 (addr, gomp_futex_wake, count);
+ }
}
#endif /* __LP64__ */
--- libgomp/config/linux/wait.h (revision 133291)
+++ libgomp/config/linux/wait.h (working copy)
@@ -33,6 +33,18 @@
#define GOMP_WAIT_H 1
#include "libgomp.h"
+#include <errno.h>
+
+#define FUTEX_WAIT 0
+#define FUTEX_WAKE 1
+#define FUTEX_PRIVATE_FLAG 128L
+
+#ifdef HAVE_ATTRIBUTE_VISIBILITY
+# pragma GCC visibility push(hidden)
+#endif
+
+extern long int gomp_futex_wait, gomp_futex_wake;
+
#include "futex.h"
static inline void do_wait (int *addr, int val)
@@ -47,4 +59,8 @@ static inline void do_wait (int *addr, i
futex_wait (addr, val);
}
+#ifdef HAVE_ATTRIBUTE_VISIBILITY
+# pragma GCC visibility pop
+#endif
+
#endif /* GOMP_WAIT_H */
--- libgomp/config/linux/s390/futex.h (revision 133291)
+++ libgomp/config/linux/s390/futex.h (working copy)
@@ -28,10 +28,8 @@
/* Provide target-specific access to the futex system call. */
#include <sys/syscall.h>
-#define FUTEX_WAIT 0
-#define FUTEX_WAKE 1
-static inline void
+static inline long
sys_futex0 (int *addr, int op, int val)
{
register long int gpr2 __asm__ ("2");
@@ -49,18 +47,31 @@ sys_futex0 (int *addr, int op, int val)
: "i" (SYS_futex),
"0" (gpr2), "d" (gpr3), "d" (gpr4), "d" (gpr5)
: "memory");
+ return gpr2;
}
static inline void
futex_wait (int *addr, int val)
{
- sys_futex0 (addr, FUTEX_WAIT, val);
+ long err = sys_futex0 (addr, gomp_futex_wait, val);
+ if (__builtin_expect (err == -ENOSYS, 0))
+ {
+ gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG;
+ gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG;
+ sys_futex0 (addr, gomp_futex_wait, val);
+ }
}
static inline void
futex_wake (int *addr, int count)
{
- sys_futex0 (addr, FUTEX_WAKE, count);
+ long err = sys_futex0 (addr, gomp_futex_wake, count);
+ if (__builtin_expect (err == -ENOSYS, 0))
+ {
+ gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG;
+ gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG;
+ sys_futex0 (addr, gomp_futex_wake, count);
+ }
}
static inline void
--- libgomp/config/linux/ia64/futex.h (revision 133291)
+++ libgomp/config/linux/ia64/futex.h (working copy)
@@ -29,23 +29,24 @@
#include <sys/syscall.h>
-#define FUTEX_WAIT 0
-#define FUTEX_WAKE 1
-static inline void
-sys_futex0(int *addr, int op, int val)
+static inline long
+sys_futex0(int *addr, long op, int val)
{
register long out0 asm ("out0") = (long) addr;
register long out1 asm ("out1") = op;
register long out2 asm ("out2") = val;
register long out3 asm ("out3") = 0;
+ register long r8 asm ("r8");
+ register long r10 asm ("r10");
register long r15 asm ("r15") = SYS_futex;
__asm __volatile ("break 0x100000"
- : "=r"(r15), "=r"(out0), "=r"(out1), "=r"(out2), "=r"(out3)
+ : "=r"(r15), "=r"(out0), "=r"(out1), "=r"(out2), "=r"(out3),
+ "=r"(r8), "=r"(r10)
: "r"(r15), "r"(out0), "r"(out1), "r"(out2), "r"(out3)
- : "memory", "r8", "r10", "out4", "out5", "out6", "out7",
+ : "memory", "out4", "out5", "out6", "out7",
/* Non-stacked integer registers, minus r8, r10, r15. */
"r2", "r3", "r9", "r11", "r12", "r13", "r14", "r16", "r17", "r18",
"r19", "r20", "r21", "r22", "r23", "r24", "r25", "r26", "r27",
@@ -56,18 +57,31 @@ sys_futex0(int *addr, int op, int val)
"f6", "f7", "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15",
/* Branch registers. */
"b6");
+ return r8 & r10;
}
static inline void
futex_wait (int *addr, int val)
{
- sys_futex0 (addr, FUTEX_WAIT, val);
+ long err = sys_futex0 (addr, gomp_futex_wait, val);
+ if (__builtin_expect (err == ENOSYS, 0))
+ {
+ gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG;
+ gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG;
+ sys_futex0 (addr, gomp_futex_wait, val);
+ }
}
static inline void
futex_wake (int *addr, int count)
{
- sys_futex0 (addr, FUTEX_WAKE, count);
+ long err = sys_futex0 (addr, gomp_futex_wake, count);
+ if (__builtin_expect (err == ENOSYS, 0))
+ {
+ gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG;
+ gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG;
+ sys_futex0 (addr, gomp_futex_wake, count);
+ }
}
static inline void
--- libgomp/config/linux/sparc/futex.h (revision 133291)
+++ libgomp/config/linux/sparc/futex.h (working copy)
@@ -28,10 +28,8 @@
/* Provide target-specific access to the futex system call. */
#include <sys/syscall.h>
-#define FUTEX_WAIT 0
-#define FUTEX_WAKE 1
-static inline void
+static inline long
sys_futex0 (int *addr, int op, int val)
{
register long int g1 __asm__ ("g1");
@@ -47,9 +45,9 @@ sys_futex0 (int *addr, int op, int val)
o3 = 0;
#ifdef __arch64__
-# define SYSCALL_STRING "ta\t0x6d"
+# define SYSCALL_STRING "ta\t0x6d; bcs,a,pt %%xcc, 1f; sub %%g0, %%o0, %%o0; 1:"
#else
-# define SYSCALL_STRING "ta\t0x10"
+# define SYSCALL_STRING "ta\t0x10; bcs,a 1f; sub %%g0, %%o0, %%o0; 1:"
#endif
__asm volatile (SYSCALL_STRING
@@ -65,18 +63,31 @@ sys_futex0 (int *addr, int op, int val)
"f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62",
#endif
"cc", "memory");
+ return o0;
}
static inline void
futex_wait (int *addr, int val)
{
- sys_futex0 (addr, FUTEX_WAIT, val);
+ long err = sys_futex0 (addr, gomp_futex_wait, val);
+ if (__builtin_expect (err == ENOSYS, 0))
+ {
+ gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG;
+ gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG;
+ sys_futex0 (addr, gomp_futex_wait, val);
+ }
}
static inline void
futex_wake (int *addr, int count)
{
- sys_futex0 (addr, FUTEX_WAKE, count);
+ long err = sys_futex0 (addr, gomp_futex_wake, count);
+ if (__builtin_expect (err == ENOSYS, 0))
+ {
+ gomp_futex_wait &= ~FUTEX_PRIVATE_FLAG;
+ gomp_futex_wake &= ~FUTEX_PRIVATE_FLAG;
+ sys_futex0 (addr, gomp_futex_wake, count);
+ }
}
static inline void
Jakub
Attachment:
micro.c
Description: Text document
| Index Nav: | [Date Index] [Subject Index] [Author Index] [Thread Index] | |
|---|---|---|
| Message Nav: | [Date Prev] [Date Next] | [Thread Prev] [Thread Next] |