From a032a6587011cbdac8c2f7e11f15dc4e592bbb55 Mon Sep 17 00:00:00 2001 From: Austin Foxley Date: Tue, 16 Feb 2010 12:27:18 -0800 Subject: mass sync with glibc nptl Signed-off-by: Austin Foxley --- .../nptl/sysdeps/unix/sysv/linux/x86_64/Versions | 7 - .../unix/sysv/linux/x86_64/bits/semaphore.h | 3 - .../sysdeps/unix/sysv/linux/x86_64/lowlevellock.S | 334 +++++++-- .../sysdeps/unix/sysv/linux/x86_64/lowlevellock.h | 718 ++++++++++++------ .../nptl/sysdeps/unix/sysv/linux/x86_64/pt-vfork.S | 2 + .../unix/sysv/linux/x86_64/pthread_barrier_wait.S | 40 +- .../sysv/linux/x86_64/pthread_cond_broadcast.S | 81 +- .../unix/sysv/linux/x86_64/pthread_cond_signal.S | 93 ++- .../sysv/linux/x86_64/pthread_cond_timedwait.S | 819 +++++++++++++++------ .../unix/sysv/linux/x86_64/pthread_cond_wait.S | 470 +++++++----- .../sysdeps/unix/sysv/linux/x86_64/pthread_once.S | 152 ++-- .../unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S | 47 +- .../sysv/linux/x86_64/pthread_rwlock_timedrdlock.S | 121 ++- .../sysv/linux/x86_64/pthread_rwlock_timedwrlock.S | 120 ++- .../unix/sysv/linux/x86_64/pthread_rwlock_unlock.S | 43 +- .../unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S | 47 +- .../nptl/sysdeps/unix/sysv/linux/x86_64/sem_post.S | 57 +- .../sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S | 327 ++++++-- .../sysdeps/unix/sysv/linux/x86_64/sem_trywait.S | 11 +- .../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S | 187 +++-- 20 files changed, 2484 insertions(+), 1195 deletions(-) delete mode 100644 libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/Versions (limited to 'libpthread/nptl/sysdeps/unix/sysv/linux/x86_64') diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/Versions b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/Versions deleted file mode 100644 index 3b111ddb5..000000000 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/Versions +++ /dev/null @@ -1,7 +0,0 @@ -librt { - GLIBC_2.3.3 { - # Changed timer_t. - timer_create; timer_delete; timer_getoverrun; timer_gettime; - timer_settime; - } -} diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/bits/semaphore.h b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/bits/semaphore.h index 57edbbbfb..e973bc5bf 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/bits/semaphore.h +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/bits/semaphore.h @@ -33,9 +33,6 @@ /* Value returned if `sem_open' failed. */ #define SEM_FAILED ((sem_t *) 0) -/* Maximum value the semaphore can have. */ -#define SEM_VALUE_MAX (2147483647) - typedef union { diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S index 1e461ad41..b0d04c75b 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002-2006, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -19,39 +19,74 @@ #include #include +#include +#include .text -#ifndef LOCK -# ifdef UP -# define LOCK +#ifdef __ASSUME_PRIVATE_FUTEX +# define LOAD_PRIVATE_FUTEX_WAIT(reg) \ + movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg +# define LOAD_PRIVATE_FUTEX_WAKE(reg) \ + movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg +# define LOAD_FUTEX_WAIT(reg) \ + xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg +# define LOAD_FUTEX_WAIT_ABS(reg) \ + xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg +# define LOAD_FUTEX_WAKE(reg) \ + xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg +#else +# if FUTEX_WAIT == 0 +# define LOAD_PRIVATE_FUTEX_WAIT(reg) \ + movl %fs:PRIVATE_FUTEX, reg +# else +# define LOAD_PRIVATE_FUTEX_WAIT(reg) \ + movl %fs:PRIVATE_FUTEX, reg ; \ + orl $FUTEX_WAIT, reg +# endif +# define LOAD_PRIVATE_FUTEX_WAKE(reg) \ + movl %fs:PRIVATE_FUTEX, reg ; \ + orl $FUTEX_WAKE, reg +# if FUTEX_WAIT == 0 +# define LOAD_FUTEX_WAIT(reg) \ + xorl $FUTEX_PRIVATE_FLAG, reg ; \ + andl %fs:PRIVATE_FUTEX, reg # else -# define LOCK lock +# define LOAD_FUTEX_WAIT(reg) \ + xorl $FUTEX_PRIVATE_FLAG, reg ; \ + andl %fs:PRIVATE_FUTEX, reg ; \ + orl $FUTEX_WAIT, reg # endif +# define LOAD_FUTEX_WAIT_ABS(reg) \ + xorl $FUTEX_PRIVATE_FLAG, reg ; \ + andl %fs:PRIVATE_FUTEX, reg ; \ + orl $FUTEX_WAIT_BITSET | FUTEX_CLOCK_REALTIME, reg +# define LOAD_FUTEX_WAKE(reg) \ + xorl $FUTEX_PRIVATE_FLAG, reg ; \ + andl %fs:PRIVATE_FUTEX, reg ; \ + orl $FUTEX_WAKE, reg #endif -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 /* For the calculation see asm/vsyscall.h. */ #define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 - .globl __lll_mutex_lock_wait - .type __lll_mutex_lock_wait,@function - .hidden __lll_mutex_lock_wait + .globl __lll_lock_wait_private + .type __lll_lock_wait_private,@function + .hidden __lll_lock_wait_private .align 16 -__lll_mutex_lock_wait: +__lll_lock_wait_private: + cfi_startproc pushq %r10 + cfi_adjust_cfa_offset(8) pushq %rdx - + cfi_adjust_cfa_offset(8) + cfi_offset(%r10, -16) + cfi_offset(%rdx, -24) xorq %r10, %r10 /* No timeout. */ movl $2, %edx -#if FUTEX_WAIT == 0 - xorl %esi, %esi -#else - movl $FUTEX_WAIT, %esi -#endif + LOAD_PRIVATE_FUTEX_WAIT (%esi) cmpl %edx, %eax /* NB: %edx == 2 */ jne 2f @@ -66,33 +101,144 @@ __lll_mutex_lock_wait: jnz 1b popq %rdx + cfi_adjust_cfa_offset(-8) + cfi_restore(%rdx) popq %r10 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r10) retq - .size __lll_mutex_lock_wait,.-__lll_mutex_lock_wait - + cfi_endproc + .size __lll_lock_wait_private,.-__lll_lock_wait_private #ifdef NOT_IN_libc - .globl __lll_mutex_timedlock_wait - .type __lll_mutex_timedlock_wait,@function - .hidden __lll_mutex_timedlock_wait + .globl __lll_lock_wait + .type __lll_lock_wait,@function + .hidden __lll_lock_wait + .align 16 +__lll_lock_wait: + cfi_startproc + pushq %r10 + cfi_adjust_cfa_offset(8) + pushq %rdx + cfi_adjust_cfa_offset(8) + cfi_offset(%r10, -16) + cfi_offset(%rdx, -24) + xorq %r10, %r10 /* No timeout. */ + movl $2, %edx + LOAD_FUTEX_WAIT (%esi) + + cmpl %edx, %eax /* NB: %edx == 2 */ + jne 2f + +1: movl $SYS_futex, %eax + syscall + +2: movl %edx, %eax + xchgl %eax, (%rdi) /* NB: lock is implied */ + + testl %eax, %eax + jnz 1b + + popq %rdx + cfi_adjust_cfa_offset(-8) + cfi_restore(%rdx) + popq %r10 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r10) + retq + cfi_endproc + .size __lll_lock_wait,.-__lll_lock_wait + + /* %rdi: futex + %rsi: flags + %rdx: timeout + %eax: futex value + */ + .globl __lll_timedlock_wait + .type __lll_timedlock_wait,@function + .hidden __lll_timedlock_wait .align 16 -__lll_mutex_timedlock_wait: +__lll_timedlock_wait: + cfi_startproc +# ifndef __ASSUME_FUTEX_CLOCK_REALTIME +# ifdef PIC + cmpl $0, __have_futex_clock_realtime(%rip) +# else + cmpl $0, __have_futex_clock_realtime +# endif + je .Lreltmo +# endif + + pushq %r9 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r9, 0) + movq %rdx, %r10 + movl $0xffffffff, %r9d + LOAD_FUTEX_WAIT_ABS (%esi) + + movl $2, %edx + cmpl %edx, %eax + jne 2f + +1: movl $SYS_futex, %eax + movl $2, %edx + syscall + +2: xchgl %edx, (%rdi) /* NB: lock is implied */ + + testl %edx, %edx + jz 3f + + cmpl $-ETIMEDOUT, %eax + je 4f + cmpl $-EINVAL, %eax + jne 1b +4: movl %eax, %edx + negl %edx + +3: movl %edx, %eax + popq %r9 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r9) + retq + +# ifndef __ASSUME_FUTEX_CLOCK_REALTIME +.Lreltmo: /* Check for a valid timeout value. */ cmpq $1000000000, 8(%rdx) jae 3f pushq %r8 + cfi_adjust_cfa_offset(8) pushq %r9 + cfi_adjust_cfa_offset(8) pushq %r12 + cfi_adjust_cfa_offset(8) pushq %r13 + cfi_adjust_cfa_offset(8) pushq %r14 + cfi_adjust_cfa_offset(8) + cfi_offset(%r8, -16) + cfi_offset(%r9, -24) + cfi_offset(%r12, -32) + cfi_offset(%r13, -40) + cfi_offset(%r14, -48) + pushq %rsi + cfi_adjust_cfa_offset(8) /* Stack frame for the timespec and timeval structs. */ - subq $16, %rsp + subq $24, %rsp + cfi_adjust_cfa_offset(24) movq %rdi, %r12 movq %rdx, %r13 + movl $2, %edx + xchgl %edx, (%r12) + + testl %edx, %edx + je 6f + 1: /* Get current time. */ movq %rsp, %rdi @@ -114,118 +260,137 @@ __lll_mutex_timedlock_wait: addq $1000000000, %rsi decq %rdi 4: testq %rdi, %rdi - js 5f /* Time is already up. */ + js 2f /* Time is already up. */ - /* Futex call. */ - movq %rdi, (%rsp) /* Store relative timeout. */ + /* Store relative timeout. */ + movq %rdi, (%rsp) movq %rsi, 8(%rsp) - movl $1, %eax + /* Futex call. */ movl $2, %edx - LOCK - cmpxchgl %edx, (%r12) - - testl %eax, %eax - je 8f - + movl $1, %eax movq %rsp, %r10 -#if FUTEX_WAIT == 0 - xorl %esi, %esi -#else - movl $FUTEX_WAIT, %esi -#endif + movl 24(%rsp), %esi + LOAD_FUTEX_WAIT (%esi) movq %r12, %rdi movl $SYS_futex, %eax syscall - movq %rax, %rcx -8: /* NB: %edx == 2 */ - xorl %eax, %eax - LOCK - cmpxchgl %edx, (%rdi) - jnz 7f + /* NB: %edx == 2 */ + xchgl %edx, (%r12) + + testl %edx, %edx + je 6f + + cmpl $-ETIMEDOUT, %eax + jne 1b +2: movl $ETIMEDOUT, %edx -6: addq $16, %rsp +6: addq $32, %rsp + cfi_adjust_cfa_offset(-32) popq %r14 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r14) popq %r13 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r13) popq %r12 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r12) popq %r9 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r9) popq %r8 - retq - - /* Check whether the time expired. */ -7: cmpq $-ETIMEDOUT, %rcx - je 5f - - /* Make sure the current holder knows we are going to sleep. */ + cfi_adjust_cfa_offset(-8) + cfi_restore(%r8) movl %edx, %eax - xchgl %eax, (%rdi) - testl %eax, %eax - jz 6b - jmp 1b + retq 3: movl $EINVAL, %eax retq - -5: movl $ETIMEDOUT, %eax - jmp 6b - .size __lll_mutex_timedlock_wait,.-__lll_mutex_timedlock_wait +# endif + cfi_endproc + .size __lll_timedlock_wait,.-__lll_timedlock_wait #endif -#ifdef NOT_IN_libc - .globl lll_unlock_wake_cb - .type lll_unlock_wake_cb,@function - .hidden lll_unlock_wake_cb + .globl __lll_unlock_wake_private + .type __lll_unlock_wake_private,@function + .hidden __lll_unlock_wake_private .align 16 -lll_unlock_wake_cb: +__lll_unlock_wake_private: + cfi_startproc pushq %rsi + cfi_adjust_cfa_offset(8) pushq %rdx + cfi_adjust_cfa_offset(8) + cfi_offset(%rsi, -16) + cfi_offset(%rdx, -24) - LOCK - addl $1, (%rdi) - jng 1f + movl $0, (%rdi) + LOAD_PRIVATE_FUTEX_WAKE (%esi) + movl $1, %edx /* Wake one thread. */ + movl $SYS_futex, %eax + syscall popq %rdx + cfi_adjust_cfa_offset(-8) + cfi_restore(%rdx) popq %rsi + cfi_adjust_cfa_offset(-8) + cfi_restore(%rsi) retq - .size lll_unlock_wake_cb,.-lll_unlock_wake_cb -#endif - + cfi_endproc + .size __lll_unlock_wake_private,.-__lll_unlock_wake_private - .globl __lll_mutex_unlock_wake - .type __lll_mutex_unlock_wake,@function - .hidden __lll_mutex_unlock_wake +#ifdef NOT_IN_libc + .globl __lll_unlock_wake + .type __lll_unlock_wake,@function + .hidden __lll_unlock_wake .align 16 -__lll_mutex_unlock_wake: +__lll_unlock_wake: + cfi_startproc pushq %rsi + cfi_adjust_cfa_offset(8) pushq %rdx + cfi_adjust_cfa_offset(8) + cfi_offset(%rsi, -16) + cfi_offset(%rdx, -24) movl $0, (%rdi) - movl $FUTEX_WAKE, %esi + LOAD_FUTEX_WAKE (%esi) movl $1, %edx /* Wake one thread. */ movl $SYS_futex, %eax syscall popq %rdx + cfi_adjust_cfa_offset(-8) + cfi_restore(%rdx) popq %rsi + cfi_adjust_cfa_offset(-8) + cfi_restore(%rsi) retq - .size __lll_mutex_unlock_wake,.-__lll_mutex_unlock_wake + cfi_endproc + .size __lll_unlock_wake,.-__lll_unlock_wake - -#ifdef NOT_IN_libc .globl __lll_timedwait_tid .type __lll_timedwait_tid,@function .hidden __lll_timedwait_tid .align 16 __lll_timedwait_tid: + cfi_startproc pushq %r12 + cfi_adjust_cfa_offset(8) pushq %r13 + cfi_adjust_cfa_offset(8) + cfi_offset(%r12, -16) + cfi_offset(%r13, -24) movq %rdi, %r12 movq %rsi, %r13 subq $16, %rsp + cfi_adjust_cfa_offset(16) /* Get current time. */ 2: movq %rsp, %rdi @@ -255,6 +420,8 @@ __lll_timedwait_tid: jz 4f movq %rsp, %r10 + /* XXX The kernel so far uses global futex for the wakeup at + all times. */ #if FUTEX_WAIT == 0 xorl %esi, %esi #else @@ -269,14 +436,21 @@ __lll_timedwait_tid: 4: xorl %eax, %eax 8: addq $16, %rsp + cfi_adjust_cfa_offset(-16) popq %r13 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r13) popq %r12 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r12) retq + cfi_adjust_cfa_offset(32) 1: cmpq $-ETIMEDOUT, %rax jne 2b 6: movl $ETIMEDOUT, %eax jmp 8b + cfi_endproc .size __lll_timedwait_tid,.-__lll_timedwait_tid #endif diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h index c9f30e962..7c042fc80 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004 Free Software Foundation, Inc. +/* Copyright (C) 2002-2004, 2006-2008, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -20,266 +20,541 @@ #ifndef _LOWLEVELLOCK_H #define _LOWLEVELLOCK_H 1 -#include -#include -#include -#include -#include - -#ifndef LOCK_INSTR -# ifdef UP -# define LOCK_INSTR /* nothing */ -# else -# define LOCK_INSTR "lock;" +#ifndef __ASSEMBLER__ +# include +# include +# include +# include +# include + +# ifndef LOCK_INSTR +# ifdef UP +# define LOCK_INSTR /* nothing */ +# else +# define LOCK_INSTR "lock;" +# endif +# endif +#else +# ifndef LOCK +# ifdef UP +# define LOCK +# else +# define LOCK lock +# endif # endif #endif #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 +#define FUTEX_CMP_REQUEUE 4 +#define FUTEX_WAKE_OP 5 +#define FUTEX_LOCK_PI 6 +#define FUTEX_UNLOCK_PI 7 +#define FUTEX_TRYLOCK_PI 8 +#define FUTEX_WAIT_BITSET 9 +#define FUTEX_WAKE_BITSET 10 +#define FUTEX_WAIT_REQUEUE_PI 11 +#define FUTEX_CMP_REQUEUE_PI 12 +#define FUTEX_PRIVATE_FLAG 128 +#define FUTEX_CLOCK_REALTIME 256 + +#define FUTEX_BITSET_MATCH_ANY 0xffffffff + +#define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE ((4 << 24) | 1) + +/* Values for 'private' parameter of locking macros. Yes, the + definition seems to be backwards. But it is not. The bit will be + reversed before passing to the system call. */ +#define LLL_PRIVATE 0 +#define LLL_SHARED FUTEX_PRIVATE_FLAG + +#ifndef __ASSEMBLER__ + +#if !defined NOT_IN_libc || defined IS_IN_rtld +/* In libc.so or ld.so all futexes are private. */ +# ifdef __ASSUME_PRIVATE_FUTEX +# define __lll_private_flag(fl, private) \ + ((fl) | FUTEX_PRIVATE_FLAG) +# else +# define __lll_private_flag(fl, private) \ + ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex)) +# endif +#else +# ifdef __ASSUME_PRIVATE_FUTEX +# define __lll_private_flag(fl, private) \ + (((fl) | FUTEX_PRIVATE_FLAG) ^ (private)) +# else +# define __lll_private_flag(fl, private) \ + (__builtin_constant_p (private) \ + ? ((private) == 0 \ + ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex)) \ + : (fl)) \ + : ({ unsigned int __fl = ((private) ^ FUTEX_PRIVATE_FLAG); \ + __asm__ ("andl %%fs:%P1, %0" : "+r" (__fl) \ + : "i" (offsetof (struct pthread, header.private_futex))); \ + __fl | (fl); })) +# endif +#endif - -/* Initializer for compatibility lock. */ -#define LLL_MUTEX_LOCK_INITIALIZER (0) -#define LLL_MUTEX_LOCK_INITIALIZER_LOCKED (1) -#define LLL_MUTEX_LOCK_INITIALIZER_WAITERS (2) +/* Initializer for lock. */ +#define LLL_LOCK_INITIALIZER (0) +#define LLL_LOCK_INITIALIZER_LOCKED (1) +#define LLL_LOCK_INITIALIZER_WAITERS (2) /* Delay in spinlock loop. */ -#define BUSY_WAIT_NOP __asm__ ("rep; nop") - - -#define lll_futex_wait(futex, val) \ - do { \ - int __ignore; \ +#define BUSY_WAIT_NOP __asm__ ("rep; nop") + + +#define LLL_STUB_UNWIND_INFO_START \ + ".section .eh_frame,\"a\",@progbits\n" \ +"7:\t" ".long 9f-8f # Length of Common Information Entry\n" \ +"8:\t" ".long 0x0 # CIE Identifier Tag\n\t" \ + ".byte 0x1 # CIE Version\n\t" \ + ".ascii \"zR\\0\" # CIE Augmentation\n\t" \ + ".uleb128 0x1 # CIE Code Alignment Factor\n\t" \ + ".sleb128 -8 # CIE Data Alignment Factor\n\t" \ + ".byte 0x10 # CIE RA Column\n\t" \ + ".uleb128 0x1 # Augmentation size\n\t" \ + ".byte 0x1b # FDE Encoding (pcrel sdata4)\n\t" \ + ".byte 0x12 # DW_CFA_def_cfa_sf\n\t" \ + ".uleb128 0x7\n\t" \ + ".sleb128 16\n\t" \ + ".align 8\n" \ +"9:\t" ".long 23f-10f # FDE Length\n" \ +"10:\t" ".long 10b-7b # FDE CIE offset\n\t" \ + ".long 1b-. # FDE initial location\n\t" \ + ".long 6b-1b # FDE address range\n\t" \ + ".uleb128 0x0 # Augmentation size\n\t" \ + ".byte 0x16 # DW_CFA_val_expression\n\t" \ + ".uleb128 0x10\n\t" \ + ".uleb128 12f-11f\n" \ +"11:\t" ".byte 0x80 # DW_OP_breg16\n\t" \ + ".sleb128 4b-1b\n" +#define LLL_STUB_UNWIND_INFO_END \ + ".byte 0x16 # DW_CFA_val_expression\n\t" \ + ".uleb128 0x10\n\t" \ + ".uleb128 14f-13f\n" \ +"13:\t" ".byte 0x80 # DW_OP_breg16\n\t" \ + ".sleb128 4b-2b\n" \ +"14:\t" ".byte 0x40 + (3b-2b) # DW_CFA_advance_loc\n\t" \ + ".byte 0x0e # DW_CFA_def_cfa_offset\n\t" \ + ".uleb128 0\n\t" \ + ".byte 0x16 # DW_CFA_val_expression\n\t" \ + ".uleb128 0x10\n\t" \ + ".uleb128 16f-15f\n" \ +"15:\t" ".byte 0x80 # DW_OP_breg16\n\t" \ + ".sleb128 4b-3b\n" \ +"16:\t" ".byte 0x40 + (4b-3b-1) # DW_CFA_advance_loc\n\t" \ + ".byte 0x0e # DW_CFA_def_cfa_offset\n\t" \ + ".uleb128 128\n\t" \ + ".byte 0x16 # DW_CFA_val_expression\n\t" \ + ".uleb128 0x10\n\t" \ + ".uleb128 20f-17f\n" \ +"17:\t" ".byte 0x80 # DW_OP_breg16\n\t" \ + ".sleb128 19f-18f\n\t" \ + ".byte 0x0d # DW_OP_const4s\n" \ +"18:\t" ".4byte 4b-.\n\t" \ + ".byte 0x1c # DW_OP_minus\n\t" \ + ".byte 0x0d # DW_OP_const4s\n" \ +"19:\t" ".4byte 24f-.\n\t" \ + ".byte 0x22 # DW_OP_plus\n" \ +"20:\t" ".byte 0x40 + (5b-4b+1) # DW_CFA_advance_loc\n\t" \ + ".byte 0x13 # DW_CFA_def_cfa_offset_sf\n\t" \ + ".sleb128 16\n\t" \ + ".byte 0x16 # DW_CFA_val_expression\n\t" \ + ".uleb128 0x10\n\t" \ + ".uleb128 22f-21f\n" \ +"21:\t" ".byte 0x80 # DW_OP_breg16\n\t" \ + ".sleb128 4b-5b\n" \ +"22:\t" ".align 8\n" \ +"23:\t" ".previous\n" + +/* Unwind info for + 1: leaq ..., %rdi + 2: subq $128, %rsp + 3: callq ... + 4: addq $128, %rsp + 5: jmp 24f + 6: + snippet. */ +#define LLL_STUB_UNWIND_INFO_5 \ +LLL_STUB_UNWIND_INFO_START \ +"12:\t" ".byte 0x40 + (2b-1b) # DW_CFA_advance_loc\n\t" \ +LLL_STUB_UNWIND_INFO_END + +/* Unwind info for + 1: leaq ..., %rdi + 0: movq ..., %rdx + 2: subq $128, %rsp + 3: callq ... + 4: addq $128, %rsp + 5: jmp 24f + 6: + snippet. */ +#define LLL_STUB_UNWIND_INFO_6 \ +LLL_STUB_UNWIND_INFO_START \ +"12:\t" ".byte 0x40 + (0b-1b) # DW_CFA_advance_loc\n\t" \ + ".byte 0x16 # DW_CFA_val_expression\n\t" \ + ".uleb128 0x10\n\t" \ + ".uleb128 26f-25f\n" \ +"25:\t" ".byte 0x80 # DW_OP_breg16\n\t" \ + ".sleb128 4b-0b\n" \ +"26:\t" ".byte 0x40 + (2b-0b) # DW_CFA_advance_loc\n\t" \ +LLL_STUB_UNWIND_INFO_END + + +#define lll_futex_wait(futex, val, private) \ + lll_futex_timed_wait(futex, val, NULL, private) + + +#define lll_futex_timed_wait(futex, val, timeout, private) \ + ({ \ + register const struct timespec *__to __asm__ ("r10") = timeout; \ + int __status; \ register __typeof (val) _val __asm__ ("edx") = (val); \ - __asm__ __volatile ("xorq %%r10, %%r10\n\t" \ - "syscall" \ - : "=a" (__ignore) \ - : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAIT), \ - "d" (_val) \ - : "memory", "cc", "r10", "r11", "cx"); \ - } while (0) + __asm__ __volatile ("syscall" \ + : "=a" (__status) \ + : "0" (SYS_futex), "D" (futex), \ + "S" (__lll_private_flag (FUTEX_WAIT, private)), \ + "d" (_val), "r" (__to) \ + : "memory", "cc", "r11", "cx"); \ + __status; \ + }) -#define lll_futex_wake(futex, nr) \ +#define lll_futex_wake(futex, nr, private) \ do { \ int __ignore; \ register __typeof (nr) _nr __asm__ ("edx") = (nr); \ __asm__ __volatile ("syscall" \ : "=a" (__ignore) \ - : "0" (SYS_futex), "D" (futex), "S" (FUTEX_WAKE), \ + : "0" (SYS_futex), "D" (futex), \ + "S" (__lll_private_flag (FUTEX_WAKE, private)), \ "d" (_nr) \ : "memory", "cc", "r10", "r11", "cx"); \ } while (0) -/* Does not preserve %eax and %ecx. */ -extern int __lll_mutex_lock_wait (int *__futex, int __val) attribute_hidden; -/* Does not preserver %eax, %ecx, and %edx. */ -extern int __lll_mutex_timedlock_wait (int *__futex, int __val, - const struct timespec *__abstime) - attribute_hidden; -/* Preserves all registers but %eax. */ -extern int __lll_mutex_unlock_wait (int *__futex) attribute_hidden; - - -/* NB: in the lll_mutex_trylock macro we simply return the value in %eax +/* NB: in the lll_trylock macro we simply return the value in %eax after the cmpxchg instruction. In case the operation succeded this value is zero. In case the operation failed, the cmpxchg instruction has loaded the current value of the memory work which is guaranteed to be nonzero. */ -#define lll_mutex_trylock(futex) \ +#if defined NOT_IN_libc || defined UP +# define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1" +#else +# define __lll_trylock_asm "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ + "je 0f\n\t" \ + "lock; cmpxchgl %2, %1\n\t" \ + "jmp 1f\n\t" \ + "0:\tcmpxchgl %2, %1\n\t" \ + "1:" +#endif + +#define lll_trylock(futex) \ ({ int ret; \ - __asm__ __volatile (LOCK_INSTR "cmpxchgl %2, %1" \ + __asm__ __volatile (__lll_trylock_asm \ : "=a" (ret), "=m" (futex) \ - : "r" (LLL_MUTEX_LOCK_INITIALIZER_LOCKED), "m" (futex),\ - "0" (LLL_MUTEX_LOCK_INITIALIZER) \ + : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex), \ + "0" (LLL_LOCK_INITIALIZER) \ : "memory"); \ ret; }) - -#define lll_mutex_cond_trylock(futex) \ +#define lll_robust_trylock(futex, id) \ ({ int ret; \ __asm__ __volatile (LOCK_INSTR "cmpxchgl %2, %1" \ : "=a" (ret), "=m" (futex) \ - : "r" (LLL_MUTEX_LOCK_INITIALIZER_WAITERS), \ - "m" (futex), "0" (LLL_MUTEX_LOCK_INITIALIZER) \ + : "r" (id), "m" (futex), "0" (LLL_LOCK_INITIALIZER) \ : "memory"); \ ret; }) +#define lll_cond_trylock(futex) \ + ({ int ret; \ + __asm__ __volatile (LOCK_INSTR "cmpxchgl %2, %1" \ + : "=a" (ret), "=m" (futex) \ + : "r" (LLL_LOCK_INITIALIZER_WAITERS), \ + "m" (futex), "0" (LLL_LOCK_INITIALIZER) \ + : "memory"); \ + ret; }) -#define lll_mutex_lock(futex) \ - (void) ({ int ignore1, ignore2, ignore3; \ - __asm__ __volatile (LOCK_INSTR "cmpxchgl %0, %2\n\t" \ - "jnz 1f\n\t" \ - ".subsection 1\n" \ - "1:\tleaq %2, %%rdi\n\t" \ - "subq $128, %%rsp\n\t" \ - "callq __lll_mutex_lock_wait\n\t" \ - "addq $128, %%rsp\n\t" \ - "jmp 2f\n\t" \ - ".previous\n" \ - "2:" \ - : "=S" (ignore1), "=&D" (ignore2), "=m" (futex),\ - "=a" (ignore3) \ - : "0" (1), "m" (futex), "3" (0) \ - : "cx", "r11", "cc", "memory"); }) - - -#define lll_mutex_cond_lock(futex) \ - (void) ({ int ignore1, ignore2, ignore3; \ - __asm__ __volatile (LOCK_INSTR "cmpxchgl %0, %2\n\t" \ +#if defined NOT_IN_libc || defined UP +# define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %4, %2\n\t" \ + "jnz 1f\n\t" +#else +# define __lll_lock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ + "je 0f\n\t" \ + "lock; cmpxchgl %4, %2\n\t" \ "jnz 1f\n\t" \ - ".subsection 1\n" \ - "1:\tleaq %2, %%rdi\n\t" \ - "subq $128, %%rsp\n\t" \ - "callq __lll_mutex_lock_wait\n\t" \ - "addq $128, %%rsp\n\t" \ - "jmp 2f\n\t" \ - ".previous\n" \ - "2:" \ - : "=S" (ignore1), "=&D" (ignore2), "=m" (futex),\ - "=a" (ignore3) \ - : "0" (2), "m" (futex), "3" (0) \ - : "cx", "r11", "cc", "memory"); }) - - -#define lll_mutex_timedlock(futex, timeout) \ - ({ int _result, ignore1, ignore2, ignore3; \ - __asm__ __volatile (LOCK_INSTR "cmpxchgl %2, %4\n\t" \ + "jmp 24f\n" \ + "0:\tcmpxchgl %4, %2\n\t" \ + "jnz 1f\n\t" +#endif + +#define lll_lock(futex, private) \ + (void) \ + ({ int ignore1, ignore2, ignore3; \ + if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ + __asm__ __volatile (__lll_lock_asm_start \ + ".subsection 1\n\t" \ + ".type _L_lock_%=, @function\n" \ + "_L_lock_%=:\n" \ + "1:\tleaq %2, %%rdi\n" \ + "2:\tsubq $128, %%rsp\n" \ + "3:\tcallq __lll_lock_wait_private\n" \ + "4:\taddq $128, %%rsp\n" \ + "5:\tjmp 24f\n" \ + "6:\t.size _L_lock_%=, 6b-1b\n\t" \ + ".previous\n" \ + LLL_STUB_UNWIND_INFO_5 \ + "24:" \ + : "=S" (ignore1), "=&D" (ignore2), "=m" (futex), \ + "=a" (ignore3) \ + : "0" (1), "m" (futex), "3" (0) \ + : "cx", "r11", "cc", "memory"); \ + else \ + __asm__ __volatile (__lll_lock_asm_start \ + ".subsection 1\n\t" \ + ".type _L_lock_%=, @function\n" \ + "_L_lock_%=:\n" \ + "1:\tleaq %2, %%rdi\n" \ + "2:\tsubq $128, %%rsp\n" \ + "3:\tcallq __lll_lock_wait\n" \ + "4:\taddq $128, %%rsp\n" \ + "5:\tjmp 24f\n" \ + "6:\t.size _L_lock_%=, 6b-1b\n\t" \ + ".previous\n" \ + LLL_STUB_UNWIND_INFO_5 \ + "24:" \ + : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \ + "=a" (ignore3) \ + : "1" (1), "m" (futex), "3" (0), "0" (private) \ + : "cx", "r11", "cc", "memory"); \ + }) \ + +#define lll_robust_lock(futex, id, private) \ + ({ int result, ignore1, ignore2; \ + __asm__ __volatile (LOCK_INSTR "cmpxchgl %4, %2\n\t" \ + "jnz 1f\n\t" \ + ".subsection 1\n\t" \ + ".type _L_robust_lock_%=, @function\n" \ + "_L_robust_lock_%=:\n" \ + "1:\tleaq %2, %%rdi\n" \ + "2:\tsubq $128, %%rsp\n" \ + "3:\tcallq __lll_robust_lock_wait\n" \ + "4:\taddq $128, %%rsp\n" \ + "5:\tjmp 24f\n" \ + "6:\t.size _L_robust_lock_%=, 6b-1b\n\t" \ + ".previous\n" \ + LLL_STUB_UNWIND_INFO_5 \ + "24:" \ + : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \ + "=a" (result) \ + : "1" (id), "m" (futex), "3" (0), "0" (private) \ + : "cx", "r11", "cc", "memory"); \ + result; }) + +#define lll_cond_lock(futex, private) \ + (void) \ + ({ int ignore1, ignore2, ignore3; \ + __asm__ __volatile (LOCK_INSTR "cmpxchgl %4, %2\n\t" \ + "jnz 1f\n\t" \ + ".subsection 1\n\t" \ + ".type _L_cond_lock_%=, @function\n" \ + "_L_cond_lock_%=:\n" \ + "1:\tleaq %2, %%rdi\n" \ + "2:\tsubq $128, %%rsp\n" \ + "3:\tcallq __lll_lock_wait\n" \ + "4:\taddq $128, %%rsp\n" \ + "5:\tjmp 24f\n" \ + "6:\t.size _L_cond_lock_%=, 6b-1b\n\t" \ + ".previous\n" \ + LLL_STUB_UNWIND_INFO_5 \ + "24:" \ + : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \ + "=a" (ignore3) \ + : "1" (2), "m" (futex), "3" (0), "0" (private) \ + : "cx", "r11", "cc", "memory"); \ + }) + +#define lll_robust_cond_lock(futex, id, private) \ + ({ int result, ignore1, ignore2; \ + __asm__ __volatile (LOCK_INSTR "cmpxchgl %4, %2\n\t" \ + "jnz 1f\n\t" \ + ".subsection 1\n\t" \ + ".type _L_robust_cond_lock_%=, @function\n" \ + "_L_robust_cond_lock_%=:\n" \ + "1:\tleaq %2, %%rdi\n" \ + "2:\tsubq $128, %%rsp\n" \ + "3:\tcallq __lll_robust_lock_wait\n" \ + "4:\taddq $128, %%rsp\n" \ + "5:\tjmp 24f\n" \ + "6:\t.size _L_robust_cond_lock_%=, 6b-1b\n\t" \ + ".previous\n" \ + LLL_STUB_UNWIND_INFO_5 \ + "24:" \ + : "=S" (ignore1), "=D" (ignore2), "=m" (futex), \ + "=a" (result) \ + : "1" (id | FUTEX_WAITERS), "m" (futex), "3" (0), \ + "0" (private) \ + : "cx", "r11", "cc", "memory"); \ + result; }) + +#define lll_timedlock(futex, timeout, private) \ + ({ int result, ignore1, ignore2, ignore3; \ + __asm__ __volatile (LOCK_INSTR "cmpxchgl %1, %4\n\t" \ "jnz 1f\n\t" \ - ".subsection 1\n" \ - "1:\tleaq %4, %%rdi\n\t" \ - "movq %8, %%rdx\n\t" \ - "subq $128, %%rsp\n\t" \ - "callq __lll_mutex_timedlock_wait\n\t" \ - "addq $128, %%rsp\n\t" \ - "jmp 2f\n\t" \ + ".subsection 1\n\t" \ + ".type _L_timedlock_%=, @function\n" \ + "_L_timedlock_%=:\n" \ + "1:\tleaq %4, %%rdi\n" \ + "0:\tmovq %8, %%rdx\n" \ + "2:\tsubq $128, %%rsp\n" \ + "3:\tcallq __lll_timedlock_wait\n" \ + "4:\taddq $128, %%rsp\n" \ + "5:\tjmp 24f\n" \ + "6:\t.size _L_timedlock_%=, 6b-1b\n\t" \ ".previous\n" \ - "2:" \ - : "=a" (_result), "=&D" (ignore1), "=S" (ignore2), \ + LLL_STUB_UNWIND_INFO_6 \ + "24:" \ + : "=a" (result), "=D" (ignore1), "=S" (ignore2), \ "=&d" (ignore3), "=m" (futex) \ - : "0" (0), "2" (1), "m" (futex), "m" (timeout) \ + : "0" (0), "1" (1), "m" (futex), "m" (timeout), \ + "2" (private) \ : "memory", "cx", "cc", "r10", "r11"); \ - _result; }) - - -#define lll_mutex_unlock(futex) \ - (void) ({ int ignore; \ - __asm__ __volatile (LOCK_INSTR "decl %0\n\t" \ - "jne 1f\n\t" \ - ".subsection 1\n" \ - "1:\tleaq %0, %%rdi\n\t" \ - "subq $128, %%rsp\n\t" \ - "callq __lll_mutex_unlock_wake\n\t" \ - "addq $128, %%rsp\n\t" \ - "jmp 2f\n\t" \ - ".previous\n" \ - "2:" \ - : "=m" (futex), "=&D" (ignore) \ - : "m" (futex) \ - : "ax", "cx", "r11", "cc", "memory"); }) - - -#define lll_mutex_islocked(futex) \ - (futex != LLL_MUTEX_LOCK_INITIALIZER) - - -/* We have a separate internal lock implementation which is not tied - to binary compatibility. */ - -/* Type for lock object. */ -typedef int lll_lock_t; - -/* Initializers for lock. */ -#define LLL_LOCK_INITIALIZER (0) -#define LLL_LOCK_INITIALIZER_LOCKED (1) - - -extern int lll_unlock_wake_cb (int *__futex) attribute_hidden; - - -/* The states of a lock are: - 0 - untaken - 1 - taken by one user - 2 - taken by more users */ + result; }) +#define lll_robust_timedlock(futex, timeout, id, private) \ + ({ int result, ignore1, ignore2, ignore3; \ + __asm__ __volatile (LOCK_INSTR "cmpxchgl %1, %4\n\t" \ + "jnz 1f\n\t" \ + ".subsection 1\n\t" \ + ".type _L_robust_timedlock_%=, @function\n" \ + "_L_robust_timedlock_%=:\n" \ + "1:\tleaq %4, %%rdi\n" \ + "0:\tmovq %8, %%rdx\n" \ + "2:\tsubq $128, %%rsp\n" \ + "3:\tcallq __lll_robust_timedlock_wait\n" \ + "4:\taddq $128, %%rsp\n" \ + "5:\tjmp 24f\n" \ + "6:\t.size _L_robust_timedlock_%=, 6b-1b\n\t" \ + ".previous\n" \ + LLL_STUB_UNWIND_INFO_6 \ + "24:" \ + : "=a" (result), "=D" (ignore1), "=S" (ignore2), \ + "=&d" (ignore3), "=m" (futex) \ + : "0" (0), "1" (id), "m" (futex), "m" (timeout), \ + "2" (private) \ + : "memory", "cx", "cc", "r10", "r11"); \ + result; }) #if defined NOT_IN_libc || defined UP -# define lll_trylock(futex) lll_mutex_trylock (futex) -# define lll_lock(futex) lll_mutex_lock (futex) -# define lll_unlock(futex) lll_mutex_unlock (futex) +# define __lll_unlock_asm_start LOCK_INSTR "decl %0\n\t" \ + "jne 1f\n\t" #else -/* Special versions of the macros for use in libc itself. They avoid - the lock prefix when the thread library is not used. - - The code sequence to avoid unnecessary lock prefixes is what the AMD - guys suggested. If you do not like it, bring it up with AMD. - - XXX In future we might even want to avoid it on UP machines. */ - -# define lll_trylock(futex) \ - ({ unsigned char ret; \ - __asm__ __volatile ("cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ - "je 0f\n\t" \ - "lock; cmpxchgl %2, %1\n\t" \ - "jmp 1f\n" \ - "0:\tcmpxchgl %2, %1\n\t" \ - "1:setne %0" \ - : "=a" (ret), "=m" (futex) \ - : "r" (LLL_MUTEX_LOCK_INITIALIZER_LOCKED), "m" (futex),\ - "0" (LLL_MUTEX_LOCK_INITIALIZER) \ - : "memory"); \ - ret; }) - - -# define lll_lock(futex) \ - (void) ({ int ignore1, ignore2, ignore3; \ - __asm__ __volatile ("cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ - "je 0f\n\t" \ - "lock; cmpxchgl %0, %2\n\t" \ - "jnz 1f\n\t" \ - "jmp 2f\n" \ - "0:\tcmpxchgl %0, %2\n\t" \ - "jnz 1f\n\t" \ - ".subsection 1\n" \ - "1:\tleaq %2, %%rdi\n\t" \ - "subq $128, %%rsp\n\t" \ - "callq __lll_mutex_lock_wait\n\t" \ - "addq $128, %%rsp\n\t" \ - "jmp 2f\n\t" \ - ".previous\n" \ - "2:" \ - : "=S" (ignore1), "=&D" (ignore2), "=m" (futex),\ - "=a" (ignore3) \ - : "0" (1), "m" (futex), "3" (0) \ - : "cx", "r11", "cc", "memory"); }) - - -# define lll_unlock(futex) \ - (void) ({ int ignore; \ - __asm__ __volatile ("cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ - "je 0f\n\t" \ - "lock; decl %0\n\t" \ - "jne 1f\n\t" \ - "jmp 2f\n" \ - "0:\tdecl %0\n\t" \ - "jne 1f\n\t" \ - ".subsection 1\n" \ - "1:\tleaq %0, %%rdi\n\t" \ - "subq $128, %%rsp\n\t" \ - "callq __lll_mutex_unlock_wake\n\t" \ - "addq $128, %%rsp\n\t" \ - "jmp 2f\n\t" \ - ".previous\n" \ - "2:" \ - : "=m" (futex), "=&D" (ignore) \ - : "m" (futex) \ - : "ax", "cx", "r11", "cc", "memory"); }) +# define __lll_unlock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \ + "je 0f\n\t" \ + "lock; decl %0\n\t" \ + "jne 1f\n\t" \ + "jmp 24f\n\t" \ + "0:\tdecl %0\n\t" \ + "jne 1f\n\t" #endif +#define lll_unlock(futex, private) \ + (void) \ + ({ int ignore; \ + if (__builtin_constant_p (private) && (private) == LLL_PRIVATE) \ + __asm__ __volatile (__lll_unlock_asm_start \ + ".subsection 1\n\t" \ + ".type _L_unlock_%=, @function\n" \ + "_L_unlock_%=:\n" \ + "1:\tleaq %0, %%rdi\n" \ + "2:\tsubq $128, %%rsp\n" \ + "3:\tcallq __lll_unlock_wake_private\n" \ + "4:\taddq $128, %%rsp\n" \ + "5:\tjmp 24f\n" \ + "6:\t.size _L_unlock_%=, 6b-1b\n\t" \ + ".previous\n" \ + LLL_STUB_UNWIND_INFO_5 \ + "24:" \ + : "=m" (futex), "=&D" (ignore) \ + : "m" (futex) \ + : "ax", "cx", "r11", "cc", "memory"); \ + else \ + __asm__ __volatile (__lll_unlock_asm_start \ + ".subsection 1\n\t" \ + ".type _L_unlock_%=, @function\n" \ + "_L_unlock_%=:\n" \ + "1:\tleaq %0, %%rdi\n" \ + "2:\tsubq $128, %%rsp\n" \ + "3:\tcallq __lll_unlock_wake\n" \ + "4:\taddq $128, %%rsp\n" \ + "5:\tjmp 24f\n" \ + "6:\t.size _L_unlock_%=, 6b-1b\n\t" \ + ".previous\n" \ + LLL_STUB_UNWIND_INFO_5 \ + "24:" \ + : "=m" (futex), "=&D" (ignore) \ + : "m" (futex), "S" (private) \ + : "ax", "cx", "r11", "cc", "memory"); \ + }) + +#define lll_robust_unlock(futex, private) \ + do \ + { \ + int ignore; \ + __asm__ __volatile (LOCK_INSTR "andl %2, %0\n\t" \ + "jne 1f\n\t" \ + ".subsection 1\n\t" \ + ".type _L_robust_unlock_%=, @function\n" \ + "_L_robust_unlock_%=:\n" \ + "1:\tleaq %0, %%rdi\n" \ + "2:\tsubq $128, %%rsp\n" \ + "3:\tcallq __lll_unlock_wake\n" \ + "4:\taddq $128, %%rsp\n" \ + "5:\tjmp 24f\n" \ + "6:\t.size _L_robust_unlock_%=, 6b-1b\n\t" \ + ".previous\n" \ + LLL_STUB_UNWIND_INFO_5 \ + "24:" \ + : "=m" (futex), "=&D" (ignore) \ + : "i" (FUTEX_WAITERS), "m" (futex), \ + "S" (private) \ + : "ax", "cx", "r11", "cc", "memory"); \ + } \ + while (0) + +#define lll_robust_dead(futex, private) \ + do \ + { \ + int ignore; \ + __asm__ __volatile (LOCK_INSTR "orl %3, (%2)\n\t" \ + "syscall" \ + : "=m" (futex), "=a" (ignore) \ + : "D" (&(futex)), "i" (FUTEX_OWNER_DIED), \ + "S" (__lll_private_flag (FUTEX_WAKE, private)), \ + "1" (__NR_futex), "d" (1) \ + : "cx", "r11", "cc", "memory"); \ + } \ + while (0) + +/* Returns non-zero if error happened, zero if success. */ +#define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val, private) \ + ({ int __res; \ + register int __nr_move __asm__ ("r10") = nr_move; \ + register void *__mutex __asm__ ("r8") = mutex; \ + register int __val __asm__ ("r9") = val; \ + __asm__ __volatile ("syscall" \ + : "=a" (__res) \ + : "0" (__NR_futex), "D" ((void *) ftx), \ + "S" (__lll_private_flag (FUTEX_CMP_REQUEUE, \ + private)), "d" (nr_wake), \ + "r" (__nr_move), "r" (__mutex), "r" (__val) \ + : "cx", "r11", "cc", "memory"); \ + __res < 0; }) #define lll_islocked(futex) \ - (futex != LLL_MUTEX_LOCK_INITIALIZER) + (futex != LLL_LOCK_INITIALIZER) /* The kernel notifies a process with uses CLONE_CLEARTID via futex @@ -318,25 +593,6 @@ extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime) } \ __result; }) - -/* Conditional variable handling. */ - -extern void __lll_cond_wait (pthread_cond_t *cond) attribute_hidden; -extern int __lll_cond_timedwait (pthread_cond_t *cond, - const struct timespec *abstime) - attribute_hidden; -extern void __lll_cond_wake (pthread_cond_t *cond) attribute_hidden; -extern void __lll_cond_broadcast (pthread_cond_t *cond) attribute_hidden; - - -#define lll_cond_wait(cond) \ - __lll_cond_wait (cond) -#define lll_cond_timedwait(cond, abstime) \ - __lll_cond_timedwait (cond, abstime) -#define lll_cond_wake(cond) \ - __lll_cond_wake (cond) -#define lll_cond_broadcast(cond) \ - __lll_cond_broadcast (cond) - +#endif /* !__ASSEMBLER__ */ #endif /* lowlevellock.h */ diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pt-vfork.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pt-vfork.S index c20ef73e7..df4949615 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pt-vfork.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pt-vfork.S @@ -16,6 +16,8 @@ Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. */ +#include + #define SAVE_PID \ movl %fs:PID, %esi; \ movl %esi, %edx; \ diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S index f6e15a2d7..15ad534fa 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_barrier_wait.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2004, 2005, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -18,17 +18,9 @@ 02111-1307 USA. */ #include +#include #include -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 - -#ifndef UP -# define LOCK lock -#else -# define LOCK -#endif - .text @@ -64,9 +56,10 @@ pthread_barrier_wait: if the CURR_EVENT memory has meanwhile been changed. */ 7: #if FUTEX_WAIT == 0 - xorl %esi, %esi + movl PRIVATE(%rdi), %esi #else movl $FUTEX_WAIT, %esi + orl PRIVATE(%rdi), %esi #endif xorq %r10, %r10 8: movl $SYS_futex, %eax @@ -115,6 +108,7 @@ pthread_barrier_wait: so 0x7fffffff is the highest value. */ movl $0x7fffffff, %edx movl $FUTEX_WAKE, %esi + orl PRIVATE(%rdi), %esi movl $SYS_futex, %eax syscall @@ -139,21 +133,29 @@ pthread_barrier_wait: retq -1: addq $MUTEX, %rdi - callq __lll_mutex_lock_wait +1: movl PRIVATE(%rdi), %esi + addq $MUTEX, %rdi + xorl $LLL_SHARED, %esi + callq __lll_lock_wait subq $MUTEX, %rdi jmp 2b -4: addq $MUTEX, %rdi - callq __lll_mutex_unlock_wake +4: movl PRIVATE(%rdi), %esi + addq $MUTEX, %rdi + xorl $LLL_SHARED, %esi + callq __lll_unlock_wake jmp 5b -6: addq $MUTEX, %rdi - callq __lll_mutex_unlock_wake +6: movl PRIVATE(%rdi), %esi + addq $MUTEX, %rdi + xorl $LLL_SHARED, %esi + callq __lll_unlock_wake subq $MUTEX, %rdi jmp 7b -9: addq $MUTEX, %rdi - callq __lll_mutex_unlock_wake +9: movl PRIVATE(%rdi), %esi + addq $MUTEX, %rdi + xorl $LLL_SHARED, %esi + callq __lll_unlock_wake jmp 10b .size pthread_barrier_wait,.-pthread_barrier_wait diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S index d8ebdfab8..0f8037ba2 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_broadcast.S @@ -1,4 +1,5 @@ -/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2009 + Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -18,21 +19,11 @@ 02111-1307 USA. */ #include +#include #include #include - -#ifdef UP -# define LOCK -#else -# define LOCK lock -#endif - -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 -#define FUTEX_REQUEUE 3 -#define FUTEX_CMP_REQUEUE 4 - -#define EINVAL 22 +#include +#include .text @@ -78,8 +69,23 @@ __pthread_cond_broadcast: 8: cmpq $-1, %r8 je 9f + /* Do not use requeue for pshared condvars. */ + testl $PS_BIT, MUTEX_KIND(%r8) + jne 9f + + /* Requeue to a PI mutex if the PI bit is set. */ + movl MUTEX_KIND(%r8), %eax + andl $(ROBUST_BIT|PI_BIT), %eax + cmpl $PI_BIT, %eax + je 81f + /* Wake up all threads. */ - movl $FUTEX_CMP_REQUEUE, %esi +#ifdef __ASSUME_PRIVATE_FUTEX + movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %esi +#else + movl %fs:PRIVATE_FUTEX, %esi + orl $FUTEX_CMP_REQUEUE, %esi +#endif movl $SYS_futex, %eax movl $1, %edx movl $0x7fffffff, %r10d @@ -94,6 +100,20 @@ __pthread_cond_broadcast: 10: xorl %eax, %eax retq + /* Wake up all threads. */ +81: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi + movl $SYS_futex, %eax + movl $1, %edx + movl $0x7fffffff, %r10d + syscall + + /* For any kind of error, which mainly is EAGAIN, we try again + with WAKE. The general test also covers running on old + kernels. */ + cmpq $-4095, %rax + jb 10b + jmp 9f + .align 16 /* Unlock. */ 4: LOCK @@ -108,7 +128,11 @@ __pthread_cond_broadcast: #if cond_lock != 0 addq $cond_lock, %rdi #endif - callq __lll_mutex_lock_wait + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_lock_wait #if cond_lock != 0 subq $cond_lock, %rdi #endif @@ -116,21 +140,38 @@ __pthread_cond_broadcast: /* Unlock in loop requires wakeup. */ 5: addq $cond_lock-cond_futex, %rdi - callq __lll_mutex_unlock_wake + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_unlock_wake jmp 6b /* Unlock in loop requires wakeup. */ 7: addq $cond_lock-cond_futex, %rdi - callq __lll_mutex_unlock_wake + cmpq $-1, %r8 + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_unlock_wake subq $cond_lock-cond_futex, %rdi jmp 8b 9: /* The futex requeue functionality is not available. */ + cmpq $-1, %r8 movl $0x7fffffff, %edx - movl $FUTEX_WAKE, %esi +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif movl $SYS_futex, %eax syscall jmp 10b .size __pthread_cond_broadcast, .-__pthread_cond_broadcast weak_alias(__pthread_cond_broadcast, pthread_cond_broadcast) - diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S index c7cc3ddd8..568c98470 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_signal.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002-2005, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -18,20 +18,11 @@ 02111-1307 USA. */ #include +#include #include +#include #include - -#ifdef UP -# define LOCK -#else -# define LOCK lock -#endif - -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 -#define FUTEX_REQUEUE 3 - -#define EINVAL 22 +#include .text @@ -64,9 +55,66 @@ __pthread_cond_signal: addl $1, (%rdi) /* Wake up one thread. */ - movl $FUTEX_WAKE, %esi - movl $SYS_futex, %eax + cmpq $-1, dep_mutex(%r8) + movl $FUTEX_WAKE_OP, %esi movl $1, %edx + movl $SYS_futex, %eax + je 8f + + /* Get the address of the mutex used. */ + movq dep_mutex(%r8), %rcx + movl MUTEX_KIND(%rcx), %r11d + andl $(ROBUST_BIT|PI_BIT), %r11d + cmpl $PI_BIT, %r11d + je 9f + +#ifdef __ASSUME_PRIVATE_FUTEX + movl $(FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG), %esi +#else + orl %fs:PRIVATE_FUTEX, %esi +#endif + +8: movl $1, %r10d +#if cond_lock != 0 + addq $cond_lock, %r8 +#endif + movl $FUTEX_OP_CLEAR_WAKE_IF_GT_ONE, %r9d + syscall +#if cond_lock != 0 + subq $cond_lock, %r8 +#endif + /* For any kind of error, we try again with WAKE. + The general test also covers running on old kernels. */ + cmpq $-4095, %rax + jae 7f + + xorl %eax, %eax + retq + + /* Wake up one thread and requeue none in the PI Mutex case. */ +9: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi + movq %rcx, %r8 + xorq %r10, %r10 + movl (%rdi), %r9d // XXX Can this be right? + syscall + + leaq -cond_futex(%rdi), %r8 + + /* For any kind of error, we try again with WAKE. + The general test also covers running on old kernels. */ + cmpq $-4095, %rax + jb 4f + +7: +#ifdef __ASSUME_PRIVATE_FUTEX + andl $FUTEX_PRIVATE_FLAG, %esi +#else + andl %fs:PRIVATE_FUTEX, %esi +#endif + orl $FUTEX_WAKE, %esi + movl $SYS_futex, %eax + /* %rdx should be 1 already from $FUTEX_WAKE_OP syscall. + movl $1, %edx */ syscall /* Unlock. */ @@ -86,7 +134,11 @@ __pthread_cond_signal: #if cond_lock != 0 addq $cond_lock, %rdi #endif - callq __lll_mutex_lock_wait + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_lock_wait #if cond_lock != 0 subq $cond_lock, %rdi #endif @@ -95,7 +147,14 @@ __pthread_cond_signal: /* Unlock in loop requires wakeup. */ 5: movq %r8, %rdi - callq __lll_mutex_unlock_wake +#if cond_lock != 0 + addq $cond_lock, %rdi +#endif + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_unlock_wake jmp 6b .size __pthread_cond_signal, .-__pthread_cond_signal weak_alias(__pthread_cond_signal, pthread_cond_signal) diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S index f0dcdb750..427a723cb 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_timedwait.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002-2005, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -18,18 +18,12 @@ 02111-1307 USA. */ #include +#include #include +#include #include -#include -#ifdef UP -# define LOCK -#else -# define LOCK lock -#endif - -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 +#include /* For the calculation see asm/vsyscall.h. */ #define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 @@ -37,6 +31,7 @@ .text + /* int pthread_cond_timedwait (pthread_cond_t *cond, pthread_mutex_t *mutex, const struct timespec *abstime) */ .globl __pthread_cond_timedwait @@ -44,38 +39,57 @@ .align 16 __pthread_cond_timedwait: .LSTARTCODE: + cfi_startproc +#ifdef SHARED + cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect, + DW.ref.__gcc_personality_v0) + cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART) +#else + cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0) + cfi_lsda(DW_EH_PE_udata4, .LexceptSTART) +#endif + pushq %r12 -.Lpush_r12: + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r12, 0) pushq %r13 -.Lpush_r13: + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r13, 0) pushq %r14 -.Lpush_r14: -#define FRAME_SIZE 80 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r14, 0) + pushq %r15 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r15, 0) +#ifdef __ASSUME_FUTEX_CLOCK_REALTIME +# define FRAME_SIZE 32 +#else +# define FRAME_SIZE 48 +#endif subq $FRAME_SIZE, %rsp -.Lsubq: + cfi_adjust_cfa_offset(FRAME_SIZE) + cfi_remember_state cmpq $1000000000, 8(%rdx) movl $EINVAL, %eax - jae 18f + jae 48f /* Stack frame: - rsp + 80 - +--------------------------+ - rsp + 48 | cleanup buffer | - +--------------------------+ - rsp + 40 | old wake_seq value | - +--------------------------+ - rsp + 24 | timeout value | - +--------------------------+ + rsp + 48 + +--------------------------+ + rsp + 32 | timeout value | + +--------------------------+ + rsp + 24 | old wake_seq value | + +--------------------------+ rsp + 16 | mutex pointer | - +--------------------------+ + +--------------------------+ rsp + 8 | condvar pointer | - +--------------------------+ + +--------------------------+ rsp + 4 | old broadcast_seq value | - +--------------------------+ + +--------------------------+ rsp + 0 | old cancellation mode | - +--------------------------+ + +--------------------------+ */ cmpq $-1, dep_mutex(%rdi) @@ -88,8 +102,18 @@ __pthread_cond_timedwait: je 22f movq %rsi, dep_mutex(%rdi) +22: +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME +# ifdef PIC + cmpl $0, __have_futex_clock_realtime(%rip) +# else + cmpl $0, __have_futex_clock_realtime +# endif + je .Lreltmo +#endif + /* Get internal lock. */ -22: movl $1, %esi + movl $1, %esi xorl %eax, %eax LOCK #if cond_lock == 0 @@ -97,89 +121,29 @@ __pthread_cond_timedwait: #else cmpxchgl %esi, cond_lock(%rdi) #endif - jnz 1f + jnz 31f /* Unlock the mutex. */ -2: movq 16(%rsp), %rdi +32: movq 16(%rsp), %rdi xorl %esi, %esi callq __pthread_mutex_unlock_usercnt testl %eax, %eax - jne 16f + jne 46f movq 8(%rsp), %rdi incq total_seq(%rdi) incl cond_futex(%rdi) - addl $(1 << clock_bits), cond_nwaiters(%rdi) - - /* Install cancellation handler. */ -#ifdef __PIC__ - leaq __condvar_cleanup(%rip), %rsi -#else - leaq __condvar_cleanup, %rsi -#endif - leaq 48(%rsp), %rdi - movq %rsp, %rdx - callq __pthread_cleanup_push + addl $(1 << nwaiters_shift), cond_nwaiters(%rdi) /* Get and store current wakeup_seq value. */ movq 8(%rsp), %rdi movq wakeup_seq(%rdi), %r9 movl broadcast_seq(%rdi), %edx - movq %r9, 40(%rsp) + movq %r9, 24(%rsp) movl %edx, 4(%rsp) - /* Get the current time. */ -8: -#ifdef __NR_clock_gettime - /* Get the clock number. Note that the field in the condvar - structure stores the number minus 1. */ - movq 8(%rsp), %rdi - movl cond_nwaiters(%rdi), %edi - andl $((1 << clock_bits) - 1), %edi - /* Only clocks 0 and 1 are allowed so far. Both are handled in the - kernel. */ - leaq 24(%rsp), %rsi - movl $__NR_clock_gettime, %eax - syscall -# ifndef __ASSUME_POSIX_TIMERS - cmpq $-ENOSYS, %rax - je 19f -# endif - - /* Compute relative timeout. */ - movq (%r13), %rcx - movq 8(%r13), %rdx - subq 24(%rsp), %rcx - subq 32(%rsp), %rdx -#else - leaq 24(%rsp), %rdi - xorl %esi, %esi - movq $VSYSCALL_ADDR_vgettimeofday, %rax - callq *%rax - - /* Compute relative timeout. */ - movq 32(%rsp), %rax - movl $1000, %edx - mul %rdx /* Milli seconds to nano seconds. */ - movq (%r13), %rcx - movq 8(%r13), %rdx - subq 24(%rsp), %rcx - subq %rax, %rdx -#endif - jns 12f - addq $1000000000, %rdx - decq %rcx -12: testq %rcx, %rcx - movq 8(%rsp), %rdi - movq $-ETIMEDOUT, %r14 - js 6f - - /* Store relative timeout. */ -21: movq %rcx, 24(%rsp) - movq %rdx, 32(%rsp) - - movl cond_futex(%rdi), %r12d +38: movl cond_futex(%rdi), %r12d /* Unlock. */ LOCK @@ -188,25 +152,67 @@ __pthread_cond_timedwait: #else decl cond_lock(%rdi) #endif - jne 3f + jne 33f -4: callq __pthread_enable_asynccancel +.LcleanupSTART1: +34: callq __pthread_enable_asynccancel movl %eax, (%rsp) - leaq 24(%rsp), %r10 -#if FUTEX_WAIT == 0 - xorl %esi, %esi + movq %r13, %r10 + movl $FUTEX_WAIT_BITSET, %esi + cmpq $-1, dep_mutex(%rdi) + je 60f + + movq dep_mutex(%rdi), %r8 + /* Requeue to a non-robust PI mutex if the PI bit is set and + the robust bit is not set. */ + movl MUTEX_KIND(%r8), %eax + andl $(ROBUST_BIT|PI_BIT), %eax + cmpl $PI_BIT, %eax + jne 61f + + movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi + xorl %eax, %eax + /* The following only works like this because we only support + two clocks, represented using a single bit. */ + testl $1, cond_nwaiters(%rdi) + movl $FUTEX_CLOCK_REALTIME, %edx + cmove %edx, %eax + orl %eax, %esi + movq %r12, %rdx + addq $cond_futex, %rdi + movl $SYS_futex, %eax + syscall + + movl $1, %r15d +#ifdef __ASSUME_REQUEUE_PI + jmp 62f #else - movl $FUTEX_WAIT, %esi + cmpq $-4095, %rax + jnae 62f + + subq $cond_futex, %rdi #endif + +61: movl $(FUTEX_WAIT_BITSET|FUTEX_PRIVATE_FLAG), %esi +60: xorl %r15d, %r15d + xorl %eax, %eax + /* The following only works like this because we only support + two clocks, represented using a single bit. */ + testl $1, cond_nwaiters(%rdi) + movl $FUTEX_CLOCK_REALTIME, %edx + movl $0xffffffff, %r9d + cmove %edx, %eax + orl %eax, %esi movq %r12, %rdx addq $cond_futex, %rdi movl $SYS_futex, %eax syscall - movq %rax, %r14 +62: movq %rax, %r14 movl (%rsp), %edi callq __pthread_disable_asynccancel +.LcleanupEND1: /* Lock. */ movq 8(%rsp), %rdi @@ -218,120 +224,158 @@ __pthread_cond_timedwait: #else cmpxchgl %esi, cond_lock(%rdi) #endif - jne 5f + jne 35f -6: movl broadcast_seq(%rdi), %edx +36: movl broadcast_seq(%rdi), %edx movq woken_seq(%rdi), %rax movq wakeup_seq(%rdi), %r9 cmpl 4(%rsp), %edx - jne 23f + jne 53f - cmpq 40(%rsp), %r9 - jbe 15f + cmpq 24(%rsp), %r9 + jbe 45f cmpq %rax, %r9 - ja 9f + ja 39f -15: cmpq $-ETIMEDOUT, %r14 - jne 8b +45: cmpq $-ETIMEDOUT, %r14 + jne 38b -13: incq wakeup_seq(%rdi) +99: incq wakeup_seq(%rdi) incl cond_futex(%rdi) movl $ETIMEDOUT, %r14d - jmp 14f + jmp 44f -23: xorq %r14, %r14 - jmp 24f +53: xorq %r14, %r14 + jmp 54f -9: xorq %r14, %r14 -14: incq woken_seq(%rdi) +39: xorq %r14, %r14 +44: incq woken_seq(%rdi) -24: subl $(1 << clock_bits), cond_nwaiters(%rdi) +54: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi) /* Wake up a thread which wants to destroy the condvar object. */ cmpq $0xffffffffffffffff, total_seq(%rdi) - jne 25f + jne 55f movl cond_nwaiters(%rdi), %eax - andl $~((1 << clock_bits) - 1), %eax - jne 25f + andl $~((1 << nwaiters_shift) - 1), %eax + jne 55f addq $cond_nwaiters, %rdi - movl $SYS_futex, %eax - movl $FUTEX_WAKE, %esi + cmpq $-1, dep_mutex-cond_nwaiters(%rdi) movl $1, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif + movl $SYS_futex, %eax syscall subq $cond_nwaiters, %rdi -25: LOCK +55: LOCK #if cond_lock == 0 decl (%rdi) #else decl cond_lock(%rdi) #endif - jne 10f + jne 40f - /* Remove cancellation handler. */ -11: movq 48+CLEANUP_PREV(%rsp), %rdx - movq %rdx, %fs:CLEANUP + /* If requeue_pi is used the kernel performs the locking of the + mutex. */ +41: movq 16(%rsp), %rdi + testl %r15d, %r15d + jnz 64f - movq 16(%rsp), %rdi callq __pthread_mutex_cond_lock - testq %rax, %rax +63: testq %rax, %rax cmoveq %r14, %rax -18: addq $FRAME_SIZE, %rsp -.Laddq: +48: addq $FRAME_SIZE, %rsp + cfi_adjust_cfa_offset(-FRAME_SIZE) + popq %r15 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r15) popq %r14 -.Lpop_r14: + cfi_adjust_cfa_offset(-8) + cfi_restore(%r14) popq %r13 -.Lpop_r13: + cfi_adjust_cfa_offset(-8) + cfi_restore(%r13) popq %r12 -.Lpop_r12: + cfi_adjust_cfa_offset(-8) + cfi_restore(%r12) retq + cfi_restore_state + +64: callq __pthread_mutex_cond_lock_adjust + movq %r14, %rax + jmp 48b + /* Initial locking failed. */ -1: -.LSbl1: +31: #if cond_lock != 0 addq $cond_lock, %rdi #endif - callq __lll_mutex_lock_wait - jmp 2b + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_lock_wait + jmp 32b /* Unlock in loop requires wakeup. */ -3: +33: #if cond_lock != 0 addq $cond_lock, %rdi #endif - callq __lll_mutex_unlock_wake - jmp 4b + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_unlock_wake + jmp 34b /* Locking in loop failed. */ -5: +35: #if cond_lock != 0 addq $cond_lock, %rdi #endif - callq __lll_mutex_lock_wait + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_lock_wait #if cond_lock != 0 subq $cond_lock, %rdi #endif - jmp 6b + jmp 36b /* Unlock after loop requires wakeup. */ -10: +40: #if cond_lock != 0 addq $cond_lock, %rdi #endif - callq __lll_mutex_unlock_wake - jmp 11b + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_unlock_wake + jmp 41b /* The initial unlocking of the mutex failed. */ -16: movq 8(%rsp), %rdi +46: movq 8(%rsp), %rdi movq %rax, (%rsp) LOCK #if cond_lock == 0 @@ -339,30 +383,239 @@ __pthread_cond_timedwait: #else decl cond_lock(%rdi) #endif - jne 17f + jne 47f #if cond_lock != 0 addq $cond_lock, %rdi #endif - callq __lll_mutex_unlock_wake + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_unlock_wake + +47: movq (%rsp), %rax + jmp 48b + -17: movq (%rsp), %rax - jmp 18b +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME +.Lreltmo: + xorl %r15d, %r15d -#if defined __NR_clock_gettime && !defined __ASSUME_POSIX_TIMERS + /* Get internal lock. */ + movl $1, %esi + xorl %eax, %eax + LOCK +# if cond_lock == 0 + cmpxchgl %esi, (%rdi) +# else + cmpxchgl %esi, cond_lock(%rdi) +# endif + jnz 1f + + /* Unlock the mutex. */ +2: movq 16(%rsp), %rdi + xorl %esi, %esi + callq __pthread_mutex_unlock_usercnt + + testl %eax, %eax + jne 46b + + movq 8(%rsp), %rdi + incq total_seq(%rdi) + incl cond_futex(%rdi) + addl $(1 << nwaiters_shift), cond_nwaiters(%rdi) + + /* Get and store current wakeup_seq value. */ + movq 8(%rsp), %rdi + movq wakeup_seq(%rdi), %r9 + movl broadcast_seq(%rdi), %edx + movq %r9, 24(%rsp) + movl %edx, 4(%rsp) + + /* Get the current time. */ +8: +# ifdef __NR_clock_gettime + /* Get the clock number. Note that the field in the condvar + structure stores the number minus 1. */ + movq 8(%rsp), %rdi + movl cond_nwaiters(%rdi), %edi + andl $((1 << nwaiters_shift) - 1), %edi + /* Only clocks 0 and 1 are allowed so far. Both are handled in the + kernel. */ + leaq 32(%rsp), %rsi +# ifdef SHARED + movq __vdso_clock_gettime@GOTPCREL(%rip), %rax + movq (%rax), %rax + PTR_DEMANGLE (%rax) + jz 26f + call *%rax + jmp 27f +# endif +26: movl $__NR_clock_gettime, %eax + syscall +27: +# ifndef __ASSUME_POSIX_TIMERS + cmpq $-ENOSYS, %rax + je 19f +# endif + + /* Compute relative timeout. */ + movq (%r13), %rcx + movq 8(%r13), %rdx + subq 32(%rsp), %rcx + subq 40(%rsp), %rdx +# else + leaq 24(%rsp), %rdi + xorl %esi, %esi + movq $VSYSCALL_ADDR_vgettimeofday, %rax + callq *%rax + + /* Compute relative timeout. */ + movq 40(%rsp), %rax + movl $1000, %edx + mul %rdx /* Milli seconds to nano seconds. */ + movq (%r13), %rcx + movq 8(%r13), %rdx + subq 32(%rsp), %rcx + subq %rax, %rdx +# endif + jns 12f + addq $1000000000, %rdx + decq %rcx +12: testq %rcx, %rcx + movq 8(%rsp), %rdi + movq $-ETIMEDOUT, %r14 + js 6f + + /* Store relative timeout. */ +21: movq %rcx, 32(%rsp) + movq %rdx, 40(%rsp) + + movl cond_futex(%rdi), %r12d + + /* Unlock. */ + LOCK +# if cond_lock == 0 + decl (%rdi) +# else + decl cond_lock(%rdi) +# endif + jne 3f + +.LcleanupSTART2: +4: callq __pthread_enable_asynccancel + movl %eax, (%rsp) + + leaq 32(%rsp), %r10 + cmpq $-1, dep_mutex(%rdi) + movq %r12, %rdx +# ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAIT, %eax + movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +# else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi +# if FUTEX_WAIT != 0 + orl $FUTEX_WAIT, %esi +# endif +# endif + addq $cond_futex, %rdi + movl $SYS_futex, %eax + syscall + movq %rax, %r14 + + movl (%rsp), %edi + callq __pthread_disable_asynccancel +.LcleanupEND2: + + /* Lock. */ + movq 8(%rsp), %rdi + movl $1, %esi + xorl %eax, %eax + LOCK +# if cond_lock == 0 + cmpxchgl %esi, (%rdi) +# else + cmpxchgl %esi, cond_lock(%rdi) +# endif + jne 5f + +6: movl broadcast_seq(%rdi), %edx + + movq woken_seq(%rdi), %rax + + movq wakeup_seq(%rdi), %r9 + + cmpl 4(%rsp), %edx + jne 53b + + cmpq 24(%rsp), %r9 + jbe 15f + + cmpq %rax, %r9 + ja 39b + +15: cmpq $-ETIMEDOUT, %r14 + jne 8b + + jmp 99b + + /* Initial locking failed. */ +1: +# if cond_lock != 0 + addq $cond_lock, %rdi +# endif + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_lock_wait + jmp 2b + + /* Unlock in loop requires wakeup. */ +3: +# if cond_lock != 0 + addq $cond_lock, %rdi +# endif + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_unlock_wake + jmp 4b + + /* Locking in loop failed. */ +5: +# if cond_lock != 0 + addq $cond_lock, %rdi +# endif + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_lock_wait +# if cond_lock != 0 + subq $cond_lock, %rdi +# endif + jmp 6b + +# if defined __NR_clock_gettime && !defined __ASSUME_POSIX_TIMERS /* clock_gettime not available. */ -19: leaq 24(%rsp), %rdi +19: leaq 32(%rsp), %rdi xorl %esi, %esi movq $VSYSCALL_ADDR_vgettimeofday, %rax callq *%rax /* Compute relative timeout. */ - movq 32(%rsp), %rax + movq 40(%rsp), %rax movl $1000, %edx mul %rdx /* Milli seconds to nano seconds. */ movq (%r13), %rcx movq 8(%r13), %rdx - subq 24(%rsp), %rcx + subq 32(%rsp), %rcx subq %rax, %rdx jns 20f addq $1000000000, %rdx @@ -372,97 +625,187 @@ __pthread_cond_timedwait: movq $-ETIMEDOUT, %r14 js 6b jmp 21b +# endif #endif -.LENDCODE: .size __pthread_cond_timedwait, .-__pthread_cond_timedwait weak_alias(__pthread_cond_timedwait, pthread_cond_timedwait) - .section .eh_frame,"a",@progbits -.LSTARTFRAME: - .long .LENDCIE-.LSTARTCIE # Length of the CIE. -.LSTARTCIE: - .long 0 # CIE ID. - .byte 1 # Version number. -#ifdef SHARED - .string "zR" # NUL-terminated augmentation - # string. + .align 16 + .type __condvar_cleanup2, @function +__condvar_cleanup2: + /* Stack frame: + + rsp + 72 + +--------------------------+ + rsp + 64 | %r12 | + +--------------------------+ + rsp + 56 | %r13 | + +--------------------------+ + rsp + 48 | %r14 | + +--------------------------+ + rsp + 24 | unused | + +--------------------------+ + rsp + 16 | mutex pointer | + +--------------------------+ + rsp + 8 | condvar pointer | + +--------------------------+ + rsp + 4 | old broadcast_seq value | + +--------------------------+ + rsp + 0 | old cancellation mode | + +--------------------------+ + */ + + movq %rax, 24(%rsp) + + /* Get internal lock. */ + movq 8(%rsp), %rdi + movl $1, %esi + xorl %eax, %eax + LOCK +#if cond_lock == 0 + cmpxchgl %esi, (%rdi) #else - .ascii "\0" # NUL-terminated augmentation - # string. + cmpxchgl %esi, cond_lock(%rdi) #endif - .uleb128 1 # Code alignment factor. - .sleb128 -8 # Data alignment factor. - .byte 16 # Return address register - # column. -#ifdef SHARED - .uleb128 1 # Augmentation value length. - .byte 0x1b # Encoding: DW_EH_PE_pcrel - # + DW_EH_PE_sdata4. + jz 1f + +#if cond_lock != 0 + addq $cond_lock, %rdi #endif - .byte 0x0c # DW_CFA_def_cfa - .uleb128 7 - .uleb128 8 - .byte 0x90 # DW_CFA_offset, column 0x8 - .uleb128 1 - .align 8 -.LENDCIE: - - .long .LENDFDE-.LSTARTFDE # Length of the FDE. -.LSTARTFDE: - .long .LSTARTFDE-.LSTARTFRAME # CIE pointer. -#ifdef SHARED - .long .LSTARTCODE-. # PC-relative start address - # of the code + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_lock_wait +#if cond_lock != 0 + subq $cond_lock, %rdi +#endif + +1: movl broadcast_seq(%rdi), %edx + cmpl 4(%rsp), %edx + jne 3f + + /* We increment the wakeup_seq counter only if it is lower than + total_seq. If this is not the case the thread was woken and + then canceled. In this case we ignore the signal. */ + movq total_seq(%rdi), %rax + cmpq wakeup_seq(%rdi), %rax + jbe 6f + incq wakeup_seq(%rdi) + incl cond_futex(%rdi) +6: incq woken_seq(%rdi) + +3: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi) + + /* Wake up a thread which wants to destroy the condvar object. */ + xorq %r12, %r12 + cmpq $0xffffffffffffffff, total_seq(%rdi) + jne 4f + movl cond_nwaiters(%rdi), %eax + andl $~((1 << nwaiters_shift) - 1), %eax + jne 4f + + cmpq $-1, dep_mutex(%rdi) + leaq cond_nwaiters(%rdi), %rdi + movl $1, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi #else - .long .LSTARTCODE # Start address of the code. + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi #endif - .long .LENDCODE-.LSTARTCODE # Length of the code. -#ifdef SHARED - .uleb128 0 # No augmentation data. + movl $SYS_futex, %eax + syscall + subq $cond_nwaiters, %rdi + movl $1, %r12d + +4: LOCK +#if cond_lock == 0 + decl (%rdi) +#else + decl cond_lock(%rdi) +#endif + je 2f +#if cond_lock != 0 + addq $cond_lock, %rdi +#endif + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_unlock_wake + + /* Wake up all waiters to make sure no signal gets lost. */ +2: testq %r12, %r12 + jnz 5f + addq $cond_futex, %rdi + cmpq $-1, dep_mutex-cond_futex(%rdi) + movl $0x7fffffff, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif + movl $SYS_futex, %eax + syscall + +5: movq 16(%rsp), %rdi + callq __pthread_mutex_cond_lock + + movq 24(%rsp), %rdi + movq FRAME_SIZE(%rsp), %r15 + movq FRAME_SIZE+8(%rsp), %r14 + movq FRAME_SIZE+16(%rsp), %r13 + movq FRAME_SIZE+24(%rsp), %r12 +.LcallUR: + call _Unwind_Resume@PLT + hlt +.LENDCODE: + cfi_endproc + .size __condvar_cleanup2, .-__condvar_cleanup2 + + + .section .gcc_except_table,"a",@progbits +.LexceptSTART: + .byte DW_EH_PE_omit # @LPStart format + .byte DW_EH_PE_omit # @TType format + .byte DW_EH_PE_uleb128 # call-site format + .uleb128 .Lcstend-.Lcstbegin +.Lcstbegin: + .uleb128 .LcleanupSTART1-.LSTARTCODE + .uleb128 .LcleanupEND1-.LcleanupSTART1 + .uleb128 __condvar_cleanup2-.LSTARTCODE + .uleb128 0 +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME + .uleb128 .LcleanupSTART2-.LSTARTCODE + .uleb128 .LcleanupEND2-.LcleanupSTART2 + .uleb128 __condvar_cleanup2-.LSTARTCODE + .uleb128 0 #endif - .byte 0x40+.Lpush_r12-.LSTARTCODE # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 - .byte 0x8c # DW_CFA_offset %r12 - .uleb128 2 - .byte 0x40+.Lpush_r13-.Lpush_r12 # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 24 - .byte 0x8d # DW_CFA_offset %r13 - .uleb128 3 - .byte 0x40+.Lpush_r14-.Lpush_r13 # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 32 - .byte 0x84 # DW_CFA_offset %r14 - .uleb128 4 - .byte 0x40+.Lsubq-.Lpush_r14 # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 32+FRAME_SIZE - .byte 3 # DW_CFA_advance_loc2 - .2byte .Laddq-.Lsubq - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 32 - .byte 0x40+.Lpop_r14-.Laddq # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 24 - .byte 0xce # DW_CFA_restore %r14 - .byte 0x40+.Lpop_r13-.Lpop_r14 # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 - .byte 0xcd # DW_CFA_restore %r13 - .byte 0x40+.Lpop_r12-.Lpop_r13 # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 8 - .byte 0xcc # DW_CFA_restore %r12 - .byte 0x40+.LSbl1-.Lpop_r12 # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 32+FRAME_SIZE - .byte 0x8c # DW_CFA_offset %r12 - .uleb128 2 - .byte 0x8d # DW_CFA_offset %r13 - .uleb128 3 - .byte 0x84 # DW_CFA_offset %r14 - .uleb128 4 + .uleb128 .LcallUR-.LSTARTCODE + .uleb128 .LENDCODE-.LcallUR + .uleb128 0 + .uleb128 0 +.Lcstend: + + +#ifdef SHARED + .hidden DW.ref.__gcc_personality_v0 + .weak DW.ref.__gcc_personality_v0 + .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits .align 8 -.LENDFDE: + .type DW.ref.__gcc_personality_v0, @object + .size DW.ref.__gcc_personality_v0, 8 +DW.ref.__gcc_personality_v0: + .quad __gcc_personality_v0 +#endif diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S index 544118eb7..7c488f261 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002-2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -18,121 +18,39 @@ 02111-1307 USA. */ #include +#include #include #include +#include -#ifdef UP -# define LOCK -#else -# define LOCK lock -#endif - -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 +#include .text - .align 16 - .type __condvar_cleanup, @function - .globl __condvar_cleanup - .hidden __condvar_cleanup -__condvar_cleanup: - pushq %r12 - - /* Get internal lock. */ - movq %rdi, %r8 - movq 8(%rdi), %rdi - movl $1, %esi - xorl %eax, %eax - LOCK -#if cond_lock == 0 - cmpxchgl %esi, (%rdi) -#else - cmpxchgl %esi, cond_lock(%rdi) -#endif - jz 1f - -#if cond_lock != 0 - addq $cond_lock, %rdi -#endif - callq __lll_mutex_lock_wait -#if cond_lock != 0 - subq $cond_lock, %rdi -#endif - -1: movl broadcast_seq(%rdi), %edx - cmpl 4(%r8), %edx - jne 3f - - incq wakeup_seq(%rdi) - incq woken_seq(%rdi) - incl cond_futex(%rdi) - -3: subl $(1 << clock_bits), cond_nwaiters(%rdi) - - /* Wake up a thread which wants to destroy the condvar object. */ - xorq %r12, %r12 - cmpq $0xffffffffffffffff, total_seq(%rdi) - jne 4f - movl cond_nwaiters(%rdi), %eax - andl $~((1 << clock_bits) - 1), %eax - jne 4f - - addq $cond_nwaiters, %rdi - movl $SYS_futex, %eax - movl $FUTEX_WAKE, %esi - movl $1, %edx - syscall - subq $cond_nwaiters, %rdi - movl $1, %r12d - -4: LOCK -#if cond_lock == 0 - decl (%rdi) -#else - decl cond_lock(%rdi) -#endif - je 2f -#if cond_lock != 0 - addq $cond_lock, %rdi -#endif - callq __lll_mutex_unlock_wake - - /* Wake up all waiters to make sure no signal gets lost. */ -2: testq %r12, %r12 - jnz 5f - addq $cond_futex, %rdi - movl $FUTEX_WAKE, %esi - movl $0x7fffffff, %edx - movl $SYS_futex, %eax - syscall - -5: movq 16(%r8), %rdi - callq __pthread_mutex_cond_lock - - popq %r12 - - retq - .size __condvar_cleanup, .-__condvar_cleanup - - /* int pthread_cond_wait (pthread_cond_t *cond, pthread_mutex_t *mutex) */ .globl __pthread_cond_wait .type __pthread_cond_wait, @function .align 16 __pthread_cond_wait: .LSTARTCODE: - pushq %r12 -.Lpush_r12: -#define FRAME_SIZE 64 - subq $FRAME_SIZE, %rsp -.Lsubq: + cfi_startproc +#ifdef SHARED + cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect, + DW.ref.__gcc_personality_v0) + cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART) +#else + cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0) + cfi_lsda(DW_EH_PE_udata4, .LexceptSTART) +#endif + +#define FRAME_SIZE 32 + leaq -FRAME_SIZE(%rsp), %rsp + cfi_adjust_cfa_offset(FRAME_SIZE) + /* Stack frame: - rsp + 64 - +--------------------------+ - rsp + 32 | cleanup buffer | + rsp + 32 +--------------------------+ rsp + 24 | old wake_seq value | +--------------------------+ @@ -177,17 +95,7 @@ __pthread_cond_wait: movq 8(%rsp), %rdi incq total_seq(%rdi) incl cond_futex(%rdi) - addl $(1 << clock_bits), cond_nwaiters(%rdi) - - /* Install cancellation handler. */ -#ifdef __PIC__ - leaq __condvar_cleanup(%rip), %rsi -#else - leaq __condvar_cleanup, %rsi -#endif - leaq 32(%rsp), %rdi - movq %rsp, %rdx - callq __pthread_cleanup_push + addl $(1 << nwaiters_shift), cond_nwaiters(%rdi) /* Get and store current wakeup_seq value. */ movq 8(%rsp), %rdi @@ -197,7 +105,7 @@ __pthread_cond_wait: movl %edx, 4(%rsp) /* Unlock. */ -8: movl cond_futex(%rdi), %r12d +8: movl cond_futex(%rdi), %edx LOCK #if cond_lock == 0 decl (%rdi) @@ -206,23 +114,53 @@ __pthread_cond_wait: #endif jne 3f +.LcleanupSTART: 4: callq __pthread_enable_asynccancel movl %eax, (%rsp) - movq 8(%rsp), %rdi xorq %r10, %r10 - movq %r12, %rdx - addq $cond_futex-cond_lock, %rdi + cmpq $-1, dep_mutex(%rdi) + leaq cond_futex(%rdi), %rdi + movl $FUTEX_WAIT, %esi + je 60f + + movq dep_mutex-cond_futex(%rdi), %r8 + /* Requeue to a non-robust PI mutex if the PI bit is set and + the robust bit is not set. */ + movl MUTEX_KIND(%r8), %eax + andl $(ROBUST_BIT|PI_BIT), %eax + cmpl $PI_BIT, %eax + jne 61f + + movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %esi movl $SYS_futex, %eax -#if FUTEX_WAIT == 0 - xorl %esi, %esi + syscall + + movl $1, %r8d +#ifdef __ASSUME_REQUEUE_PI + jmp 62f #else + cmpq $-4095, %rax + jnae 62f + +# ifndef __ASSUME_PRIVATE_FUTEX movl $FUTEX_WAIT, %esi +# endif #endif + +61: +#ifdef __ASSUME_PRIVATE_FUTEX + movl $(FUTEX_WAIT|FUTEX_PRIVATE_FLAG), %esi +#else + orl %fs:PRIVATE_FUTEX, %esi +#endif +60: xorl %r8d, %r8d + movl $SYS_futex, %eax syscall - movl (%rsp), %edi +62: movl (%rsp), %edi callq __pthread_disable_asynccancel +.LcleanupEND: /* Lock. */ movq 8(%rsp), %rdi @@ -254,19 +192,29 @@ __pthread_cond_wait: incq woken_seq(%rdi) /* Unlock */ -16: subl $(1 << clock_bits), cond_nwaiters(%rdi) +16: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi) /* Wake up a thread which wants to destroy the condvar object. */ cmpq $0xffffffffffffffff, total_seq(%rdi) jne 17f movl cond_nwaiters(%rdi), %eax - andl $~((1 << clock_bits) - 1), %eax + andl $~((1 << nwaiters_shift) - 1), %eax jne 17f addq $cond_nwaiters, %rdi - movl $SYS_futex, %eax - movl $FUTEX_WAKE, %esi + cmpq $-1, dep_mutex-cond_nwaiters(%rdi) movl $1, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi +#endif + movl $SYS_futex, %eax syscall subq $cond_nwaiters, %rdi @@ -278,28 +226,36 @@ __pthread_cond_wait: #endif jne 10f - /* Remove cancellation handler. */ -11: movq 32+CLEANUP_PREV(%rsp), %rdx - movq %rdx, %fs:CLEANUP + /* If requeue_pi is used the kernel performs the locking of the + mutex. */ +11: movq 16(%rsp), %rdi + testl %r8d, %r8d + jnz 18f - movq 16(%rsp), %rdi callq __pthread_mutex_cond_lock -14: addq $FRAME_SIZE, %rsp -.Laddq: - popq %r12 -.Lpop_r12: +14: leaq FRAME_SIZE(%rsp), %rsp + cfi_adjust_cfa_offset(-FRAME_SIZE) /* We return the result of the mutex_lock operation. */ retq + cfi_adjust_cfa_offset(FRAME_SIZE) + +18: callq __pthread_mutex_cond_lock_adjust + xorl %eax, %eax + jmp 14b + /* Initial locking failed. */ 1: -.LSbl1: #if cond_lock != 0 addq $cond_lock, %rdi #endif - callq __lll_mutex_lock_wait + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_lock_wait jmp 2b /* Unlock in loop requires wakeup. */ @@ -307,7 +263,15 @@ __pthread_cond_wait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - callq __lll_mutex_unlock_wake + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + /* The call preserves %rdx. */ + callq __lll_unlock_wake +#if cond_lock != 0 + subq $cond_lock, %rdi +#endif jmp 4b /* Locking in loop failed. */ @@ -315,7 +279,11 @@ __pthread_cond_wait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - callq __lll_mutex_lock_wait + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_lock_wait #if cond_lock != 0 subq $cond_lock, %rdi #endif @@ -326,7 +294,11 @@ __pthread_cond_wait: #if cond_lock != 0 addq $cond_lock, %rdi #endif - callq __lll_mutex_unlock_wake + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_unlock_wake jmp 11b /* The initial unlocking of the mutex failed. */ @@ -338,83 +310,185 @@ __pthread_cond_wait: #else decl cond_lock(%rdi) #endif - jne 13f + je 13f #if cond_lock != 0 addq $cond_lock, %rdi #endif - callq __lll_mutex_unlock_wake + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_unlock_wake 13: movq %r10, %rax jmp 14b -.LENDCODE: .size __pthread_cond_wait, .-__pthread_cond_wait weak_alias(__pthread_cond_wait, pthread_cond_wait) - .section .eh_frame,"a",@progbits -.LSTARTFRAME: - .long .LENDCIE-.LSTARTCIE # Length of the CIE. -.LSTARTCIE: - .long 0 # CIE ID. - .byte 1 # Version number. -#ifdef SHARED - .string "zR" # NUL-terminated augmentation - # string. + .align 16 + .type __condvar_cleanup1, @function + .globl __condvar_cleanup1 + .hidden __condvar_cleanup1 +__condvar_cleanup1: + /* Stack frame: + + rsp + 32 + +--------------------------+ + rsp + 24 | unused | + +--------------------------+ + rsp + 16 | mutex pointer | + +--------------------------+ + rsp + 8 | condvar pointer | + +--------------------------+ + rsp + 4 | old broadcast_seq value | + +--------------------------+ + rsp + 0 | old cancellation mode | + +--------------------------+ + */ + + movq %rax, 24(%rsp) + + /* Get internal lock. */ + movq 8(%rsp), %rdi + movl $1, %esi + xorl %eax, %eax + LOCK +#if cond_lock == 0 + cmpxchgl %esi, (%rdi) #else - .ascii "\0" # NUL-terminated augmentation - # string. + cmpxchgl %esi, cond_lock(%rdi) #endif - .uleb128 1 # Code alignment factor. - .sleb128 -8 # Data alignment factor. - .byte 16 # Return address register - # column. -#ifdef SHARED - .uleb128 1 # Augmentation value length. - .byte 0x1b # Encoding: DW_EH_PE_pcrel - # + DW_EH_PE_sdata4. + jz 1f + +#if cond_lock != 0 + addq $cond_lock, %rdi #endif - .byte 0x0c # DW_CFA_def_cfa - .uleb128 7 - .uleb128 8 - .byte 0x90 # DW_CFA_offset, column 0x8 - .uleb128 1 - .align 8 -.LENDCIE: - - .long .LENDFDE-.LSTARTFDE # Length of the FDE. -.LSTARTFDE: - .long .LSTARTFDE-.LSTARTFRAME # CIE pointer. -#ifdef SHARED - .long .LSTARTCODE-. # PC-relative start address - # of the code + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + callq __lll_lock_wait +#if cond_lock != 0 + subq $cond_lock, %rdi +#endif + +1: movl broadcast_seq(%rdi), %edx + cmpl 4(%rsp), %edx + jne 3f + + /* We increment the wakeup_seq counter only if it is lower than + total_seq. If this is not the case the thread was woken and + then canceled. In this case we ignore the signal. */ + movq total_seq(%rdi), %rax + cmpq wakeup_seq(%rdi), %rax + jbe 6f + incq wakeup_seq(%rdi) + incl cond_futex(%rdi) +6: incq woken_seq(%rdi) + +3: subl $(1 << nwaiters_shift), cond_nwaiters(%rdi) + + /* Wake up a thread which wants to destroy the condvar object. */ + xorl %ecx, %ecx + cmpq $0xffffffffffffffff, total_seq(%rdi) + jne 4f + movl cond_nwaiters(%rdi), %eax + andl $~((1 << nwaiters_shift) - 1), %eax + jne 4f + + cmpq $-1, dep_mutex(%rdi) + leaq cond_nwaiters(%rdi), %rdi + movl $1, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi #else - .long .LSTARTCODE # Start address of the code. + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi #endif - .long .LENDCODE-.LSTARTCODE # Length of the code. -#ifdef SHARED - .uleb128 0 # No augmentation data. + movl $SYS_futex, %eax + syscall + subq $cond_nwaiters, %rdi + movl $1, %ecx + +4: LOCK +#if cond_lock == 0 + decl (%rdi) +#else + decl cond_lock(%rdi) +#endif + je 2f +#if cond_lock != 0 + addq $cond_lock, %rdi +#endif + cmpq $-1, dep_mutex-cond_lock(%rdi) + movl $LLL_PRIVATE, %eax + movl $LLL_SHARED, %esi + cmovne %eax, %esi + /* The call preserves %rcx. */ + callq __lll_unlock_wake + + /* Wake up all waiters to make sure no signal gets lost. */ +2: testl %ecx, %ecx + jnz 5f + addq $cond_futex, %rdi + cmpq $-1, dep_mutex-cond_futex(%rdi) + movl $0x7fffffff, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE, %eax + movl $(FUTEX_WAKE|FUTEX_PRIVATE_FLAG), %esi + cmove %eax, %esi +#else + movl $0, %eax + movl %fs:PRIVATE_FUTEX, %esi + cmove %eax, %esi + orl $FUTEX_WAKE, %esi #endif - .byte 0x40+.Lpush_r12-.LSTARTCODE # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 - .byte 0x8c # DW_CFA_offset %r12 - .uleb128 2 - .byte 0x40+.Lsubq-.Lpush_r12 # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16+FRAME_SIZE - .byte 3 # DW_CFA_advance_loc2 - .2byte .Laddq-.Lsubq - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 - .byte 0x40+.Lpop_r12-.Laddq # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 8 - .byte 0xcc # DW_CFA_restore %r12 - .byte 0x40+.LSbl1-.Lpop_r12 # DW_CFA_advance_loc+N - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 80 - .byte 0x8c # DW_CFA_offset %r12 - .uleb128 2 + movl $SYS_futex, %eax + syscall + +5: movq 16(%rsp), %rdi + callq __pthread_mutex_cond_lock + + movq 24(%rsp), %rdi +.LcallUR: + call _Unwind_Resume@PLT + hlt +.LENDCODE: + cfi_endproc + .size __condvar_cleanup1, .-__condvar_cleanup1 + + + .section .gcc_except_table,"a",@progbits +.LexceptSTART: + .byte DW_EH_PE_omit # @LPStart format + .byte DW_EH_PE_omit # @TType format + .byte DW_EH_PE_uleb128 # call-site format + .uleb128 .Lcstend-.Lcstbegin +.Lcstbegin: + .uleb128 .LcleanupSTART-.LSTARTCODE + .uleb128 .LcleanupEND-.LcleanupSTART + .uleb128 __condvar_cleanup1-.LSTARTCODE + .uleb128 0 + .uleb128 .LcallUR-.LSTARTCODE + .uleb128 .LENDCODE-.LcallUR + .uleb128 0 + .uleb128 0 +.Lcstend: + + +#ifdef SHARED + .hidden DW.ref.__gcc_personality_v0 + .weak DW.ref.__gcc_personality_v0 + .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits .align 8 -.LENDFDE: + .type DW.ref.__gcc_personality_v0, @object + .size DW.ref.__gcc_personality_v0, 8 +DW.ref.__gcc_personality_v0: + .quad __gcc_personality_v0 +#endif diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S index d8bfa26c6..0ac952b66 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -18,15 +18,10 @@ 02111-1307 USA. */ #include +#include +#include +#include -#ifndef UP -# define LOCK lock -#else -# define LOCK -#endif - -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 .comm __fork_generation, 4, 4 @@ -38,6 +33,15 @@ .align 16 __pthread_once: .LSTARTCODE: + cfi_startproc +#ifdef SHARED + cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect, + DW.ref.__gcc_personality_v0) + cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART) +#else + cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0) + cfi_lsda(DW_EH_PE_udata4, .LexceptSTART) +#endif testl $2, (%rdi) jz 1f xorl %eax, %eax @@ -45,7 +49,7 @@ __pthread_once: /* Preserve the function pointer. */ 1: pushq %rsi -.Lpush_rsi: + cfi_adjust_cfa_offset(8) xorq %r10, %r10 /* Not yet initialized or initialization in progress. @@ -76,10 +80,15 @@ __pthread_once: jnz 3f /* Different for generation -> run initializer. */ /* Somebody else got here first. Wait. */ -#if FUTEX_WAIT == 0 - xorl %esi, %esi +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %esi #else +# if FUTEX_WAIT == 0 + movl %fs:PRIVATE_FUTEX, %esi +# else movl $FUTEX_WAIT, %esi + orl %fs:PRIVATE_FUTEX, %esi +# endif #endif movl $SYS_futex, %eax syscall @@ -87,31 +96,40 @@ __pthread_once: /* Preserve the pointer to the control variable. */ 3: pushq %rdi -.Lpush_rdi: + cfi_adjust_cfa_offset(8) + pushq %rdi + cfi_adjust_cfa_offset(8) .LcleanupSTART: - callq *8(%rsp) + callq *16(%rsp) .LcleanupEND: /* Get the control variable address back. */ popq %rdi -.Lpop_rdi: + cfi_adjust_cfa_offset(-8) /* Sucessful run of the initializer. Signal that we are done. */ LOCK incl (%rdi) + addq $8, %rsp + cfi_adjust_cfa_offset(-8) + /* Wake up all other threads. */ movl $0x7fffffff, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi +#else movl $FUTEX_WAKE, %esi + orl %fs:PRIVATE_FUTEX, %esi +#endif movl $SYS_futex, %eax syscall 4: addq $8, %rsp -.Ladd: + cfi_adjust_cfa_offset(-8) xorl %eax, %eax retq - .size __pthread_once,.-__pthread_once @@ -125,12 +143,18 @@ pthread_once = __pthread_once .type clear_once_control,@function .align 16 clear_once_control: + cfi_adjust_cfa_offset(3 * 8) movq (%rsp), %rdi movq %rax, %r8 movl $0, (%rdi) movl $0x7fffffff, %edx +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi +#else movl $FUTEX_WAKE, %esi + orl %fs:PRIVATE_FUTEX, %esi +#endif movl $SYS_futex, %eax syscall @@ -139,15 +163,15 @@ clear_once_control: call _Unwind_Resume@PLT hlt .LENDCODE: + cfi_endproc .size clear_once_control,.-clear_once_control .section .gcc_except_table,"a",@progbits .LexceptSTART: - .byte 0xff # @LPStart format (omit) - .byte 0xff # @TType format (omit) - .byte 0x01 # call-site format - # DW_EH_PE_uleb128 + .byte DW_EH_PE_omit # @LPStart format + .byte DW_EH_PE_omit # @TType format + .byte DW_EH_PE_uleb128 # call-site format .uleb128 .Lcstend-.Lcstbegin .Lcstbegin: .uleb128 .LcleanupSTART-.LSTARTCODE @@ -161,92 +185,6 @@ clear_once_control: .Lcstend: - .section .eh_frame,"a",@progbits -.LSTARTFRAME: - .long .LENDCIE-.LSTARTCIE # Length of the CIE. -.LSTARTCIE: - .long 0 # CIE ID. - .byte 1 # Version number. -#ifdef SHARED - .string "zPLR" # NUL-terminated augmentation - # string. -#else - .string "zPL" # NUL-terminated augmentation - # string. -#endif - .uleb128 1 # Code alignment factor. - .sleb128 -8 # Data alignment factor. - .byte 16 # Return address register - # column. -#ifdef SHARED - .uleb128 7 # Augmentation value length. - .byte 0x9b # Personality: DW_EH_PE_pcrel - # + DW_EH_PE_sdata4 - # + DW_EH_PE_indirect - .long DW.ref.__gcc_personality_v0-. - .byte 0x1b # LSDA Encoding: DW_EH_PE_pcrel - # + DW_EH_PE_sdata4. - .byte 0x1b # FDE Encoding: DW_EH_PE_pcrel - # + DW_EH_PE_sdata4. -#else - .uleb128 10 # Augmentation value length. - .byte 0x0 # Personality: absolute - .quad __gcc_personality_v0 - .byte 0x0 # LSDA Encoding: absolute -#endif - .byte 0x0c # DW_CFA_def_cfa - .uleb128 7 - .uleb128 8 - .byte 0x90 # DW_CFA_offset, column 0x10 - .uleb128 1 - .align 8 -.LENDCIE: - - .long .LENDFDE-.LSTARTFDE # Length of the FDE. -.LSTARTFDE: - .long .LSTARTFDE-.LSTARTFRAME # CIE pointer. -#ifdef SHARED - .long .LSTARTCODE-. # PC-relative start address - # of the code. - .long .LENDCODE-.LSTARTCODE # Length of the code. - .uleb128 4 # Augmentation size - .long .LexceptSTART-. -#else - .quad .LSTARTCODE # Start address of the code. - .quad .LENDCODE-.LSTARTCODE # Length of the code. - .uleb128 8 # Augmentation size - .quad .LexceptSTART -#endif - .byte 4 # DW_CFA_advance_loc4 - .long .Lpush_rsi-.LSTARTCODE - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 - .byte 4 # DW_CFA_advance_loc4 - .long .Lpush_rdi-.Lpush_rsi - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 24 - .byte 4 # DW_CFA_advance_loc4 - .long .Lpop_rdi-.Lpush_rdi - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 - .byte 4 # DW_CFA_advance_loc4 - .long .Ladd-.Lpop_rdi - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 8 - .byte 4 # DW_CFA_advance_loc4 - .long clear_once_control-.Ladd - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 24 -#if 0 - .byte 4 # DW_CFA_advance_loc4 - .long .Lpop_rdi2-clear_once_control - .byte 14 # DW_CFA_def_cfa_offset - .uleb128 16 -#endif - .align 8 -.LENDFDE: - - #ifdef SHARED .hidden DW.ref.__gcc_personality_v0 .weak DW.ref.__gcc_personality_v0 diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S index d7543572a..9b8408b69 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -18,19 +18,10 @@ 02111-1307 USA. */ #include +#include #include #include -#include - - -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 - -#ifndef UP -# define LOCK lock -#else -# define LOCK -#endif +#include .text @@ -39,6 +30,7 @@ .type __pthread_rwlock_rdlock,@function .align 16 __pthread_rwlock_rdlock: + cfi_startproc xorq %r10, %r10 /* Get the lock. */ @@ -73,12 +65,20 @@ __pthread_rwlock_rdlock: #endif jne 10f -11: addq $READERS_WAKEUP, %rdi -#if FUTEX_WAIT == 0 - xorl %esi, %esi +11: +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %esi + xorl PSHARED(%rdi), %esi #else +# if FUTEX_WAIT == 0 + movl PSHARED(%rdi), %esi +# else movl $FUTEX_WAIT, %esi + orl PSHARED(%rdi), %esi +# endif + xorl %fs:PRIVATE_FUTEX, %esi #endif + addq $READERS_WAKEUP, %rdi movl $SYS_futex, %eax syscall @@ -113,11 +113,11 @@ __pthread_rwlock_rdlock: movq %rdx, %rax retq -1: +1: movl PSHARED(%rdi), %esi #if MUTEX != 0 addq $MUTEX, %rdi #endif - callq __lll_mutex_lock_wait + callq __lll_lock_wait #if MUTEX != 0 subq $MUTEX, %rdi #endif @@ -129,11 +129,11 @@ __pthread_rwlock_rdlock: movl $EDEADLK, %edx jmp 9b -6: +6: movl PSHARED(%rdi), %esi #if MUTEX != 0 addq $MUTEX, %rdi #endif - callq __lll_mutex_unlock_wake + callq __lll_unlock_wake #if MUTEX != 0 subq $MUTEX, %rdi #endif @@ -149,25 +149,26 @@ __pthread_rwlock_rdlock: movl $EAGAIN, %edx jmp 9b -10: +10: movl PSHARED(%rdi), %esi #if MUTEX != 0 addq $MUTEX, %rdi #endif - callq __lll_mutex_unlock_wake + callq __lll_unlock_wake #if MUTEX != 0 subq $MUTEX, %rdi #endif jmp 11b -12: +12: movl PSHARED(%rdi), %esi #if MUTEX == 0 addq $MUTEX, %rdi #endif - callq __lll_mutex_lock_wait + callq __lll_lock_wait #if MUTEX != 0 subq $MUTEX, %rdi #endif jmp 13b + cfi_endproc .size __pthread_rwlock_rdlock,.-__pthread_rwlock_rdlock .globl pthread_rwlock_rdlock diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S index f044842e0..bb12d4941 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2004, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002-2005, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -18,35 +18,39 @@ 02111-1307 USA. */ #include +#include #include #include -#include +#include -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 - /* For the calculation see asm/vsyscall.h. */ #define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 - -#ifndef UP -# define LOCK lock -#else -# define LOCK -#endif - - .text .globl pthread_rwlock_timedrdlock .type pthread_rwlock_timedrdlock,@function .align 16 pthread_rwlock_timedrdlock: + cfi_startproc pushq %r12 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r12, 0) pushq %r13 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r13, 0) +#ifdef __ASSUME_FUTEX_CLOCK_REALTIME +# define VALREG %edx +#else pushq %r14 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r14, 0) + subq $16, %rsp + cfi_adjust_cfa_offset(16) +# define VALREG %r14d +#endif movq %rdi, %r12 movq %rsi, %r13 @@ -77,7 +81,7 @@ pthread_rwlock_timedrdlock: incl READERS_QUEUED(%r12) je 4f - movl READERS_WAKEUP(%r12), %r14d + movl READERS_WAKEUP(%r12), VALREG /* Unlock. */ LOCK @@ -88,8 +92,33 @@ pthread_rwlock_timedrdlock: #endif jne 10f +11: +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME +# ifdef PIC + cmpl $0, __have_futex_clock_realtime(%rip) +# else + cmpl $0, __have_futex_clock_realtime +# endif + je .Lreltmo +#endif + + movl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT_BITSET|FUTEX_CLOCK_REALTIME, %esi + xorl PSHARED(%r12), %esi + movq %r13, %r10 + movl $0xffffffff, %r9d +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME + movl %r14d, %edx +#endif +21: leaq READERS_WAKEUP(%r12), %rdi + movl $SYS_futex, %eax + syscall + movq %rax, %rdx + +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME + .subsection 2 +.Lreltmo: /* Get current time. */ -11: movq %rsp, %rdi + movq %rsp, %rdi xorl %esi, %esi movq $VSYSCALL_ADDR_vgettimeofday, %rax callq *%rax @@ -112,20 +141,26 @@ pthread_rwlock_timedrdlock: movq %rcx, (%rsp) /* Store relative timeout. */ movq %rdi, 8(%rsp) -#if FUTEX_WAIT == 0 - xorl %esi, %esi -#else +# ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %esi + xorl PSHARED(%r12), %esi +# else +# if FUTEX_WAIT == 0 + movl PSHARED(%r12), %esi +# else movl $FUTEX_WAIT, %esi -#endif + orl PSHARED(%r12), %esi +# endif + xorl %fs:PRIVATE_FUTEX, %esi +# endif movq %rsp, %r10 movl %r14d, %edx - leaq READERS_WAKEUP(%r12), %rdi - movl $SYS_futex, %eax - syscall - movq %rax, %rdx -17: - /* Reget the lock. */ + jmp 21b + .previous +#endif + +17: /* Reget the lock. */ movl $1, %esi xorl %eax, %eax LOCK @@ -157,17 +192,36 @@ pthread_rwlock_timedrdlock: 7: movq %rdx, %rax +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME addq $16, %rsp + cfi_adjust_cfa_offset(-16) popq %r14 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r14) +#endif popq %r13 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r13) popq %r12 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r12) retq -1: +#ifdef __ASSUME_PRIVATE_FUTEX + cfi_adjust_cfa_offset(16) + cfi_rel_offset(%r12, 8) + cfi_rel_offset(%r13, 0) +#else + cfi_adjust_cfa_offset(40) + cfi_offset(%r12, -16) + cfi_offset(%r13, -24) + cfi_offset(%r14, -32) +#endif +1: movl PSHARED(%rdi), %esi #if MUTEX != 0 addq $MUTEX, %rdi #endif - callq __lll_mutex_lock_wait + callq __lll_lock_wait jmp 2b 14: cmpl %fs:TID, %eax @@ -175,13 +229,13 @@ pthread_rwlock_timedrdlock: movl $EDEADLK, %edx jmp 9b -6: +6: movl PSHARED(%r12), %esi #if MUTEX == 0 movq %r12, %rdi #else leal MUTEX(%r12), %rdi #endif - callq __lll_mutex_unlock_wake + callq __lll_unlock_wake jmp 7b /* Overflow. */ @@ -194,22 +248,22 @@ pthread_rwlock_timedrdlock: movl $EAGAIN, %edx jmp 9b -10: +10: movl PSHARED(%r12), %esi #if MUTEX == 0 movq %r12, %rdi #else leaq MUTEX(%r12), %rdi #endif - callq __lll_mutex_unlock_wake + callq __lll_unlock_wake jmp 11b -12: +12: movl PSHARED(%r12), %esi #if MUTEX == 0 movq %r12, %rdi #else leaq MUTEX(%r12), %rdi #endif - callq __lll_mutex_lock_wait + callq __lll_lock_wait jmp 13b 16: movq $-ETIMEDOUT, %rdx @@ -217,4 +271,5 @@ pthread_rwlock_timedrdlock: 19: movl $EINVAL, %edx jmp 9b + cfi_endproc .size pthread_rwlock_timedrdlock,.-pthread_rwlock_timedrdlock diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S index b479da727..401bbc5d9 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -18,34 +18,39 @@ 02111-1307 USA. */ #include +#include #include #include -#include +#include -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 - /* For the calculation see asm/vsyscall.h. */ #define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 -#ifndef UP -# define LOCK lock -#else -# define LOCK -#endif - - .text .globl pthread_rwlock_timedwrlock .type pthread_rwlock_timedwrlock,@function .align 16 pthread_rwlock_timedwrlock: + cfi_startproc pushq %r12 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r12, 0) pushq %r13 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r13, 0) +#ifdef __ASSUME_FUTEX_CLOCK_REALTIME +# define VALREG %edx +#else pushq %r14 + cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r14, 0) + subq $16, %rsp + cfi_adjust_cfa_offset(16) +# define VALREG %r14d +#endif movq %rdi, %r12 movq %rsi, %r13 @@ -74,7 +79,7 @@ pthread_rwlock_timedwrlock: incl WRITERS_QUEUED(%r12) je 4f - movl WRITERS_WAKEUP(%r12), %r14d + movl WRITERS_WAKEUP(%r12), VALREG LOCK #if MUTEX == 0 @@ -84,8 +89,33 @@ pthread_rwlock_timedwrlock: #endif jne 10f +11: +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME +# ifdef PIC + cmpl $0, __have_futex_clock_realtime(%rip) +# else + cmpl $0, __have_futex_clock_realtime +# endif + je .Lreltmo +#endif + + movl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT_BITSET|FUTEX_CLOCK_REALTIME, %esi + xorl PSHARED(%r12), %esi + movq %r13, %r10 + movl $0xffffffff, %r9d +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME + movl %r14d, %edx +#endif +21: leaq WRITERS_WAKEUP(%r12), %rdi + movl $SYS_futex, %eax + syscall + movq %rax, %rdx + +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME + .subsection 2 +.Lreltmo: /* Get current time. */ -11: movq %rsp, %rdi + movq %rsp, %rdi xorl %esi, %esi movq $VSYSCALL_ADDR_vgettimeofday, %rax callq *%rax @@ -108,20 +138,26 @@ pthread_rwlock_timedwrlock: movq %rcx, (%rsp) /* Store relative timeout. */ movq %rdi, 8(%rsp) -#if FUTEX_WAIT == 0 - xorl %esi, %esi -#else +# ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %esi + xorl PSHARED(%r12), %esi +# else +# if FUTEX_WAIT == 0 + movl PSHARED(%r12), %esi +# else movl $FUTEX_WAIT, %esi -#endif + orl PSHARED(%r12), %esi +# endif + xorl %fs:PRIVATE_FUTEX, %esi +# endif movq %rsp, %r10 movl %r14d, %edx - leaq WRITERS_WAKEUP(%r12), %rdi - movl $SYS_futex, %eax - syscall - movq %rax, %rdx -17: - /* Reget the lock. */ + jmp 21b + .previous +#endif + +17: /* Reget the lock. */ movl $1, %esi xorl %eax, %eax LOCK @@ -153,17 +189,36 @@ pthread_rwlock_timedwrlock: 7: movq %rdx, %rax +#ifndef __ASSUME_PRIVATE_FUTEX addq $16, %rsp + cfi_adjust_cfa_offset(-16) popq %r14 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r14) +#endif popq %r13 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r13) popq %r12 + cfi_adjust_cfa_offset(-8) + cfi_restore(%r12) retq -1: +#ifdef __ASSUME_PRIVATE_FUTEX + cfi_adjust_cfa_offset(16) + cfi_rel_offset(%r12, 8) + cfi_rel_offset(%r13, 0) +#else + cfi_adjust_cfa_offset(40) + cfi_offset(%r12, -16) + cfi_offset(%r13, -24) + cfi_offset(%r14, -32) +#endif +1: movl PSHARED(%rdi), %esi #if MUTEX != 0 addq $MUTEX, %rdi #endif - callq __lll_mutex_lock_wait + callq __lll_lock_wait jmp 2b 14: cmpl %fs:TID, %eax @@ -171,13 +226,13 @@ pthread_rwlock_timedwrlock: 20: movl $EDEADLK, %edx jmp 9b -6: +6: movl PSHARED(%r12), %esi #if MUTEX == 0 movq %r12, %rdi #else leal MUTEX(%r12), %rdi #endif - callq __lll_mutex_unlock_wake + callq __lll_unlock_wake jmp 7b /* Overflow. */ @@ -185,22 +240,22 @@ pthread_rwlock_timedwrlock: movl $EAGAIN, %edx jmp 9b -10: +10: movl PSHARED(%r12), %esi #if MUTEX == 0 movq %r12, %rdi #else leaq MUTEX(%r12), %rdi #endif - callq __lll_mutex_unlock_wake + callq __lll_unlock_wake jmp 11b -12: +12: movl PSHARED(%r12), %esi #if MUTEX == 0 movq %r12, %rdi #else leaq MUTEX(%r12), %rdi #endif - callq __lll_mutex_lock_wait + callq __lll_lock_wait jmp 13b 16: movq $-ETIMEDOUT, %rdx @@ -208,4 +263,5 @@ pthread_rwlock_timedwrlock: 19: movl $EINVAL, %edx jmp 9b + cfi_endproc .size pthread_rwlock_timedwrlock,.-pthread_rwlock_timedwrlock diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S index a0f75226a..cfcc7a18c 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -18,17 +18,9 @@ 02111-1307 USA. */ #include +#include #include - - -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 - -#ifndef UP -# define LOCK lock -#else -# define LOCK -#endif +#include .text @@ -37,6 +29,7 @@ .type __pthread_rwlock_unlock,@function .align 16 __pthread_rwlock_unlock: + cfi_startproc /* Get the lock. */ movl $1, %esi xorl %eax, %eax @@ -55,9 +48,8 @@ __pthread_rwlock_unlock: 5: movl $0, WRITER(%rdi) - movl $1, %esi + movl $1, %edx leaq WRITERS_WAKEUP(%rdi), %r10 - movq %rsi, %rdx cmpl $0, WRITERS_QUEUED(%rdi) jne 0f @@ -77,7 +69,16 @@ __pthread_rwlock_unlock: #endif jne 7f -8: movl $SYS_futex, %eax +8: +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_PRIVATE_FLAG|FUTEX_WAKE, %esi + xorl PSHARED(%rdi), %esi +#else + movl $FUTEX_WAKE, %esi + orl PSHARED(%rdi), %esi + xorl %fs:PRIVATE_FUTEX, %esi +#endif + movl $SYS_futex, %eax movq %r10, %rdi syscall @@ -96,30 +97,30 @@ __pthread_rwlock_unlock: 4: xorl %eax, %eax retq -1: +1: movl PSHARED(%rdi), %esi #if MUTEX != 0 addq $MUTEX, %rdi #endif - callq __lll_mutex_lock_wait + callq __lll_lock_wait #if MUTEX != 0 subq $MUTEX, %rdi #endif jmp 2b -3: +3: movl PSHARED(%rdi), %esi #if MUTEX != 0 addq $MUTEX, %rdi #endif - callq __lll_mutex_unlock_wake + callq __lll_unlock_wake jmp 4b -7: +7: movl PSHARED(%rdi), %esi #if MUTEX != 0 addq $MUTEX, %rdi #endif - callq __lll_mutex_unlock_wake + callq __lll_unlock_wake jmp 8b - + cfi_endproc .size __pthread_rwlock_unlock,.-__pthread_rwlock_unlock .globl pthread_rwlock_unlock diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S index 39b54dc6b..b7bc8522d 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -18,19 +18,10 @@ 02111-1307 USA. */ #include +#include #include #include -#include - - -#define FUTEX_WAIT 0 -#define FUTEX_WAKE 1 - -#ifndef UP -# define LOCK lock -#else -# define LOCK -#endif +#include .text @@ -39,6 +30,7 @@ .type __pthread_rwlock_wrlock,@function .align 16 __pthread_rwlock_wrlock: + cfi_startproc xorq %r10, %r10 /* Get the lock. */ @@ -71,12 +63,20 @@ __pthread_rwlock_wrlock: #endif jne 10f -11: addq $WRITERS_WAKEUP, %rdi -#if FUTEX_WAIT == 0 - xorl %esi, %esi +11: +#ifdef __ASSUME_PRIVATE_FUTEX + movl $FUTEX_PRIVATE_FLAG|FUTEX_WAIT, %esi + xorl PSHARED(%rdi), %esi #else +# if FUTEX_WAIT == 0 + movl PSHARED(%rdi), %esi +# else movl $FUTEX_WAIT, %esi + orl PSHARED(%rdi), %esi +# endif + xorl %fs:PRIVATE_FUTEX, %esi #endif + addq $WRITERS_WAKEUP, %rdi movl $SYS_futex, %eax syscall @@ -111,11 +111,11 @@ __pthread_rwlock_wrlock: movq %rdx, %rax retq -1: +1: movl PSHARED(%rdi), %esi #if MUTEX != 0 addq $MUTEX, %rdi #endif - callq __lll_mutex_lock_wait + callq __lll_lock_wait #if MUTEX != 0 subq $MUTEX, %rdi #endif @@ -126,36 +126,37 @@ __pthread_rwlock_wrlock: movl $EDEADLK, %edx jmp 9b -6: +6: movl PSHARED(%rdi), %esi #if MUTEX != 0 addq $MUTEX, %rdi #endif - callq __lll_mutex_unlock_wake + callq __lll_unlock_wake jmp 7b 4: decl WRITERS_QUEUED(%rdi) movl $EAGAIN, %edx jmp 9b -10: +10: movl PSHARED(%rdi), %esi #if MUTEX != 0 addq $MUTEX, %rdi #endif - callq __lll_mutex_unlock_wake + callq __lll_unlock_wake #if MUTEX != 0 subq $MUTEX, %rdi #endif jmp 11b -12: +12: movl PSHARED(%rdi), %esi #if MUTEX != 0 addq $MUTEX, %rdi #endif - callq __lll_mutex_lock_wait + callq __lll_lock_wait #if MUTEX != 0 subq $MUTEX, %rdi #endif jmp 13b + cfi_endproc .size __pthread_rwlock_wrlock,.-__pthread_rwlock_wrlock .globl pthread_rwlock_wrlock diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_post.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_post.S index 5c8a858ad..7af6524fe 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_post.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_post.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2007, 2008 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -18,15 +18,9 @@ 02111-1307 USA. */ #include +#include #include - -#ifndef UP -# define LOCK lock -#else -# define -#endif - -#define FUTEX_WAKE 1 +#include .text @@ -35,30 +29,61 @@ .type sem_post,@function .align 16 sem_post: - movl $1, %edx +#if VALUE == 0 + movl (%rdi), %eax +#else + movl VALUE(%rdi), %eax +#endif +0: cmpl $SEM_VALUE_MAX, %eax + je 3f + leal 1(%rax), %esi LOCK - xaddl %edx, (%rdi) +#if VALUE == 0 + cmpxchgl %esi, (%rdi) +#else + cmpxchgl %esi, VALUE(%rdi) +#endif + jnz 0b + + cmpq $0, NWAITERS(%rdi) + je 2f movl $SYS_futex, %eax movl $FUTEX_WAKE, %esi - incl %edx + orl PRIVATE(%rdi), %esi + movl $1, %edx syscall testq %rax, %rax js 1f - xorl %eax, %eax +2: xorl %eax, %eax retq 1: #if USE___THREAD - movq errno@gottpoff(%rip), %rdx - movl $EINVAL, %fs:(%rdx) + movl $EINVAL, %eax +#else + callq __errno_location@plt + movl $EINVAL, %edx +#endif + jmp 4f + +3: +#if USE___THREAD + movl $EOVERFLOW, %eax #else callq __errno_location@plt - movl $EINVAL, (%rax) + movl $EOVERFLOW, %edx #endif +4: +#if USE___THREAD + movq errno@gottpoff(%rip), %rdx + movl %eax, %fs:(%rdx) +#else + movl %edx, (%rax) +#endif orl $-1, %eax retq .size sem_post,.-sem_post diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S index 64e168099..f9af8ecc1 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_timedwait.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -18,70 +18,183 @@ 02111-1307 USA. */ #include +#include +#include #include -#include -#include +#include -#ifndef UP -# define LOCK lock -#else -# define -#endif /* For the calculation see asm/vsyscall.h. */ #define VSYSCALL_ADDR_vgettimeofday 0xffffffffff600000 - .text .globl sem_timedwait .type sem_timedwait,@function .align 16 - cfi_startproc sem_timedwait: - /* First check for cancellation. */ - movl %fs:CANCELHANDLING, %eax - andl $0xfffffff9, %eax - cmpl $8, %eax - je 11f - +.LSTARTCODE: + cfi_startproc +#ifdef SHARED + cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect, + DW.ref.__gcc_personality_v0) + cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART) +#else + cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0) + cfi_lsda(DW_EH_PE_udata4, .LexceptSTART) +#endif +#if VALUE == 0 movl (%rdi), %eax +#else + movl VALUE(%rdi), %eax +#endif 2: testl %eax, %eax je 1f leaq -1(%rax), %rdx LOCK +#if VALUE == 0 cmpxchgl %edx, (%rdi) +#else + cmpxchgl %edx, VALUE(%rdi) +#endif jne 2b xorl %eax, %eax retq /* Check whether the timeout value is valid. */ -1: pushq %r12 +1: cmpq $1000000000, 8(%rsi) + jae 6f + +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME +# ifdef PIC + cmpl $0, __have_futex_clock_realtime(%rip) +# else + cmpl $0, __have_futex_clock_realtime +# endif + je .Lreltmo +#endif + + /* This push is only needed to store the sem_t pointer for the + exception handler. */ + pushq %rdi + cfi_adjust_cfa_offset(8) + + movq %rsi, %r10 + + LOCK + addq $1, NWAITERS(%rdi) + +.LcleanupSTART: +13: call __pthread_enable_asynccancel + movl %eax, %r8d + +#if VALUE != 0 + leaq VALUE(%rdi), %rdi +#endif + movl $0xffffffff, %r9d + movl $FUTEX_WAIT_BITSET|FUTEX_CLOCK_REALTIME, %esi + orl PRIVATE(%rdi), %esi + movl $SYS_futex, %eax + xorl %edx, %edx + syscall + movq %rax, %r9 +#if VALUE != 0 + leaq -VALUE(%rdi), %rdi +#endif + + xchgq %r8, %rdi + call __pthread_disable_asynccancel +.LcleanupEND: + movq %r8, %rdi + + testq %r9, %r9 + je 11f + cmpq $-EWOULDBLOCK, %r9 + jne 3f + +11: +#if VALUE == 0 + movl (%rdi), %eax +#else + movl VALUE(%rdi), %eax +#endif +14: testl %eax, %eax + je 13b + + leaq -1(%rax), %rcx + LOCK +#if VALUE == 0 + cmpxchgl %ecx, (%rdi) +#else + cmpxchgl %ecx, VALUE(%rdi) +#endif + jne 14b + + xorl %eax, %eax + +15: LOCK + subq $1, NWAITERS(%rdi) + + leaq 8(%rsp), %rsp + cfi_adjust_cfa_offset(-8) + retq + + cfi_adjust_cfa_offset(8) +3: negq %r9 +#if USE___THREAD + movq errno@gottpoff(%rip), %rdx + movl %r9d, %fs:(%rdx) +#else + callq __errno_location@plt + movl %r9d, (%rax) +#endif + + orl $-1, %eax + jmp 15b + + cfi_adjust_cfa_offset(-8) +6: +#if USE___THREAD + movq errno@gottpoff(%rip), %rdx + movl $EINVAL, %fs:(%rdx) +#else + callq __errno_location@plt + movl $EINVAL, (%rax) +#endif + + orl $-1, %eax + + retq + +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME +.Lreltmo: + pushq %r12 cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r12, 0) pushq %r13 cfi_adjust_cfa_offset(8) + cfi_rel_offset(%r13, 0) pushq %r14 cfi_adjust_cfa_offset(8) - subq $24, %rsp - cfi_adjust_cfa_offset(24) + cfi_rel_offset(%r14, 0) + +#ifdef __ASSUME_FUTEX_CLOCK_REALTIME +# define STACKFRAME 8 +#else +# define STACKFRAME 24 +#endif + subq $STACKFRAME, %rsp + cfi_adjust_cfa_offset(STACKFRAME) movq %rdi, %r12 - cfi_offset(12, -16) /* %r12 */ movq %rsi, %r13 - cfi_offset(13, -24) /* %r13 */ - /* Check for invalid nanosecond field. */ - cmpq $1000000000, 8(%r13) - movl $EINVAL, %r14d - cfi_offset(14, -24) /* %r14 */ - jae 6f - -7: call __pthread_enable_asynccancel - movl %eax, 16(%rsp) + LOCK + addq $1, NWAITERS(%r12) - xorl %esi, %esi +7: xorl %esi, %esi movq %rsp, %rdi movq $VSYSCALL_ADDR_vgettimeofday, %rax callq *%rax @@ -99,14 +212,27 @@ sem_timedwait: decq %rdi 5: testq %rdi, %rdi movl $ETIMEDOUT, %r14d - js 6f /* Time is already up. */ + js 36f /* Time is already up. */ movq %rdi, (%rsp) /* Store relative timeout. */ movq %rsi, 8(%rsp) +.LcleanupSTART2: + call __pthread_enable_asynccancel + movl %eax, 16(%rsp) + movq %rsp, %r10 +# if VALUE == 0 movq %r12, %rdi - xorl %esi, %esi +# else + leaq VALUE(%r12), %rdi +# endif +# if FUTEX_WAIT == 0 + movl PRIVATE(%rdi), %esi +# else + movl $FUTEX_WAIT, %esi + orl PRIVATE(%rdi), %esi +# endif movl $SYS_futex, %eax xorl %edx, %edx syscall @@ -114,41 +240,55 @@ sem_timedwait: movl 16(%rsp), %edi call __pthread_disable_asynccancel +.LcleanupEND2: testq %r14, %r14 je 9f cmpq $-EWOULDBLOCK, %r14 - jne 3f + jne 33f -9: movl (%r12), %eax +9: +# if VALUE == 0 + movl (%r12), %eax +# else + movl VALUE(%r12), %eax +# endif 8: testl %eax, %eax je 7b leaq -1(%rax), %rcx LOCK +# if VALUE == 0 cmpxchgl %ecx, (%r12) +# else + cmpxchgl %ecx, VALUE(%r12) +# endif jne 8b xorl %eax, %eax -10: addq $24, %rsp - cfi_adjust_cfa_offset(-24) + +45: LOCK + subq $1, NWAITERS(%r12) + + addq $STACKFRAME, %rsp + cfi_adjust_cfa_offset(-STACKFRAME) popq %r14 cfi_adjust_cfa_offset(-8) - cfi_restore(14) + cfi_restore(%r14) popq %r13 cfi_adjust_cfa_offset(-8) - cfi_restore(13) + cfi_restore(%r13) popq %r12 cfi_adjust_cfa_offset(-8) - cfi_restore(12) + cfi_restore(%r12) retq - cfi_adjust_cfa_offset(48) - cfi_offset(12, -16) /* %r12 */ - cfi_offset(13, -24) /* %r13 */ - cfi_offset(14, -32) /* %r14 */ -3: negq %r14 -6: + cfi_adjust_cfa_offset(STACKFRAME + 3 * 8) + cfi_rel_offset(%r12, STACKFRAME + 2 * 8) + cfi_rel_offset(%r13, STACKFRAME + 1 * 8) + cfi_rel_offset(%r14, STACKFRAME) +33: negq %r14 +36: #if USE___THREAD movq errno@gottpoff(%rip), %rdx movl %r14d, %fs:(%rdx) @@ -158,17 +298,90 @@ sem_timedwait: #endif orl $-1, %eax - jmp 10b - cfi_adjust_cfa_offset(-48) - cfi_restore(14) - cfi_restore(13) - cfi_restore(12) - -11: /* Canceled. */ - movq $0xffffffffffffffff, %fs:RESULT - LOCK - orl $0x10, %fs:CANCELHANDLING - movq %fs:CLEANUP_JMP_BUF, %rdi - jmp HIDDEN_JUMPTARGET (__pthread_unwind) + jmp 45b +#endif cfi_endproc .size sem_timedwait,.-sem_timedwait + + + .type sem_timedwait_cleanup,@function +sem_timedwait_cleanup: + cfi_startproc + cfi_adjust_cfa_offset(8) + + movq (%rsp), %rdi + LOCK + subq $1, NWAITERS(%rdi) + movq %rax, %rdi +.LcallUR: + call _Unwind_Resume@PLT + hlt +.LENDCODE: + cfi_endproc + .size sem_timedwait_cleanup,.-sem_timedwait_cleanup + + +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME + .type sem_timedwait_cleanup2,@function +sem_timedwait_cleanup2: + cfi_startproc + cfi_adjust_cfa_offset(STACKFRAME + 3 * 8) + cfi_rel_offset(%r12, STACKFRAME + 2 * 8) + cfi_rel_offset(%r13, STACKFRAME + 1 * 8) + cfi_rel_offset(%r14, STACKFRAME) + + LOCK + subq $1, NWAITERS(%r12) + movq %rax, %rdi + movq STACKFRAME(%rsp), %r14 + movq STACKFRAME+8(%rsp), %r13 + movq STACKFRAME+16(%rsp), %r12 +.LcallUR2: + call _Unwind_Resume@PLT + hlt +.LENDCODE2: + cfi_endproc + .size sem_timedwait_cleanup2,.-sem_timedwait_cleanup2 +#endif + + + .section .gcc_except_table,"a",@progbits +.LexceptSTART: + .byte DW_EH_PE_omit # @LPStart format + .byte DW_EH_PE_omit # @TType format + .byte DW_EH_PE_uleb128 # call-site format + .uleb128 .Lcstend-.Lcstbegin +.Lcstbegin: + .uleb128 .LcleanupSTART-.LSTARTCODE + .uleb128 .LcleanupEND-.LcleanupSTART + .uleb128 sem_timedwait_cleanup-.LSTARTCODE + .uleb128 0 +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME + .uleb128 .LcleanupSTART2-.LSTARTCODE + .uleb128 .LcleanupEND2-.LcleanupSTART2 + .uleb128 sem_timedwait_cleanup2-.LSTARTCODE + .uleb128 0 +#endif + .uleb128 .LcallUR-.LSTARTCODE + .uleb128 .LENDCODE-.LcallUR + .uleb128 0 + .uleb128 0 +#ifndef __ASSUME_FUTEX_CLOCK_REALTIME + .uleb128 .LcallUR2-.LSTARTCODE + .uleb128 .LENDCODE2-.LcallUR2 + .uleb128 0 + .uleb128 0 +#endif +.Lcstend: + + +#ifdef SHARED + .hidden DW.ref.__gcc_personality_v0 + .weak DW.ref.__gcc_personality_v0 + .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits + .align 8 + .type DW.ref.__gcc_personality_v0, @object + .size DW.ref.__gcc_personality_v0, 8 +DW.ref.__gcc_personality_v0: + .quad __gcc_personality_v0 +#endif diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_trywait.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_trywait.S index 08edc390c..7b7f63ddb 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_trywait.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_trywait.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2007 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -18,14 +18,9 @@ 02111-1307 USA. */ #include +#include #include -#ifndef UP -# define LOCK lock -#else -# define -#endif - .text .globl sem_trywait @@ -36,7 +31,7 @@ sem_trywait: 2: testl %eax, %eax jz 1f - leaq -1(%rax), %rdx + leal -1(%rax), %edx LOCK cmpxchgl %edx, (%rdi) jne 2b diff --git a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S index c2f94d47f..73d1d1633 100644 --- a/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S +++ b/libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S @@ -1,4 +1,4 @@ -/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc. +/* Copyright (C) 2002, 2003, 2005, 2007, 2009 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper , 2002. @@ -18,15 +18,9 @@ 02111-1307 USA. */ #include +#include #include -#include -#include - -#ifndef UP -# define LOCK lock -#else -# define -#endif +#include .text @@ -34,86 +28,155 @@ .globl sem_wait .type sem_wait,@function .align 16 - cfi_startproc sem_wait: - /* First check for cancellation. */ - movl %fs:CANCELHANDLING, %eax - andl $0xfffffff9, %eax - cmpl $8, %eax - je 4f - - pushq %r12 - cfi_adjust_cfa_offset(8) - cfi_offset(12, -16) - pushq %r13 - cfi_adjust_cfa_offset(8) - movq %rdi, %r13 - cfi_offset(13, -24) +.LSTARTCODE: + cfi_startproc +#ifdef SHARED + cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect, + DW.ref.__gcc_personality_v0) + cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART) +#else + cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0) + cfi_lsda(DW_EH_PE_udata4, .LexceptSTART) +#endif -3: movl (%r13), %eax +#if VALUE == 0 + movl (%rdi), %eax +#else + movl VALUE(%rdi), %eax +#endif 2: testl %eax, %eax je 1f - leaq -1(%rax), %rdx + leal -1(%rax), %edx LOCK - cmpxchgl %edx, (%r13) +#if VALUE == 0 + cmpxchgl %edx, (%rdi) +#else + cmpxchgl %edx, VALUE(%rdi) +#endif jne 2b + xorl %eax, %eax + retq - popq %r13 - cfi_adjust_cfa_offset(-8) - cfi_restore(13) - popq %r12 - cfi_adjust_cfa_offset(-8) - cfi_restore(12) + /* This push is only needed to store the sem_t pointer for the + exception handler. */ +1: pushq %rdi + cfi_adjust_cfa_offset(8) - retq + LOCK + addq $1, NWAITERS(%rdi) - cfi_adjust_cfa_offset(16) - cfi_offset(12, -16) - cfi_offset(13, -24) -1: call __pthread_enable_asynccancel +.LcleanupSTART: +6: call __pthread_enable_asynccancel movl %eax, %r8d xorq %r10, %r10 movl $SYS_futex, %eax - movq %r13, %rdi - movq %r10, %rsi - movq %r10, %rdx +#if FUTEX_WAIT == 0 + movl PRIVATE(%rdi), %esi +#else + movl $FUTEX_WAIT, %esi + orl PRIVATE(%rdi), %esi +#endif + xorl %edx, %edx syscall - movq %rax, %r12 + movq %rax, %rcx - movl %r8d, %edi + xchgq %r8, %rdi call __pthread_disable_asynccancel +.LcleanupEND: + movq %r8, %rdi - testq %r12, %r12 - je 3b - cmpq $-EWOULDBLOCK, %r12 - je 3b - negq %r12 + testq %rcx, %rcx + je 3f + cmpq $-EWOULDBLOCK, %rcx + jne 4f + +3: +#if VALUE == 0 + movl (%rdi), %eax +#else + movl VALUE(%rdi), %eax +#endif +5: testl %eax, %eax + je 6b + + leal -1(%rax), %edx + LOCK +#if VALUE == 0 + cmpxchgl %edx, (%rdi) +#else + cmpxchgl %edx, VALUE(%rdi) +#endif + jne 5b + + xorl %eax, %eax + +9: LOCK + subq $1, NWAITERS(%rdi) + + leaq 8(%rsp), %rsp + cfi_adjust_cfa_offset(-8) + + retq + + cfi_adjust_cfa_offset(8) +4: negq %rcx #if USE___THREAD movq errno@gottpoff(%rip), %rdx - movl %r12d, %fs:(%rdx) + movl %ecx, %fs:(%rdx) #else +# error "not supported. %rcx and %rdi must be preserved" callq __errno_location@plt - movl %r12d, (%rax) + movl %ecx, (%rax) #endif orl $-1, %eax - popq %r13 - cfi_adjust_cfa_offset(-8) - cfi_restore(13) - popq %r12 - cfi_adjust_cfa_offset(-8) - cfi_restore(12) + jmp 9b + .size sem_wait,.-sem_wait - retq -4: /* Canceled. */ - movq $0xffffffffffffffff, %fs:RESULT + .type sem_wait_cleanup,@function +sem_wait_cleanup: + movq (%rsp), %rdi LOCK - orl $0x10, %fs:CANCELHANDLING - movq %fs:CLEANUP_JMP_BUF, %rdi - jmp HIDDEN_JUMPTARGET (__pthread_unwind) + subq $1, NWAITERS(%rdi) + movq %rax, %rdi +.LcallUR: + call _Unwind_Resume@PLT + hlt +.LENDCODE: cfi_endproc - .size sem_wait,.-sem_wait + .size sem_wait_cleanup,.-sem_wait_cleanup + + + .section .gcc_except_table,"a",@progbits +.LexceptSTART: + .byte DW_EH_PE_omit # @LPStart format + .byte DW_EH_PE_omit # @TType format + .byte DW_EH_PE_uleb128 # call-site format + .uleb128 .Lcstend-.Lcstbegin +.Lcstbegin: + .uleb128 .LcleanupSTART-.LSTARTCODE + .uleb128 .LcleanupEND-.LcleanupSTART + .uleb128 sem_wait_cleanup-.LSTARTCODE + .uleb128 0 + .uleb128 .LcallUR-.LSTARTCODE + .uleb128 .LENDCODE-.LcallUR + .uleb128 0 + .uleb128 0 +.Lcstend: + + +#ifdef SHARED + .hidden DW.ref.__gcc_personality_v0 + .weak DW.ref.__gcc_personality_v0 + .section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits + .align 8 + .type DW.ref.__gcc_personality_v0, @object + .size DW.ref.__gcc_personality_v0, 8 +DW.ref.__gcc_personality_v0: + .quad __gcc_personality_v0 +#endif -- cgit v1.2.3