pthread_once: Add fast path and remove x86 variants.

4 files changed, 30 insertions, 378 deletions

diff --git a/ChangeLog b/ChangeLog
index 8e3373cc06..f6a7e43c32 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,14 @@
 2014-10-20  Torvald Riegel  <triegel@redhat.com>
 
 	[BZ #15215]
+	* nptl/pthread_once.c (__pthread_once): Split out fast path to ...
+	(__pthread_once_slow): ... here.
+	* sysdeps/unix/sysv/linux/i386/pthread_once.S: Remove file.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_once.S: Remove file.
+
+2014-10-20  Torvald Riegel  <triegel@redhat.com>
+
+	[BZ #15215]
 	* nptl/pthreadP.h (__PTHREAD_ONCE_INPROGRESS, __PTHREAD_ONCE_DONE,
 	__PTHREAD_ONCE_FORK_GEN_INCR): New.
 	* sysdeps/nptl/fork.c (__libc_fork): Use them.
diff --git a/nptl/pthread_once.c b/nptl/pthread_once.c
index 595bd7e298..05aea84d38 100644
--- a/nptl/pthread_once.c
+++ b/nptl/pthread_once.c
@@ -58,11 +58,13 @@ clear_once_control (void *arg)
    initialization is interrupted, we then fork 2^30 times (30 bits of
    once_control are used for the fork generation), and try to initialize
    again, we can deadlock because we can't distinguish the in-progress and
-   interrupted cases anymore.  */
-int
-__pthread_once (once_control, init_routine)
-     pthread_once_t *once_control;
-     void (*init_routine) (void);
+   interrupted cases anymore.
+   XXX: We split out this slow path because current compilers do not generate
+   as efficient code when the fast path in __pthread_once below is not in a
+   separate function.  */
+static int
+__attribute__ ((noinline))
+__pthread_once_slow (pthread_once_t *once_control, void (*init_routine) (void))
 {
   while (1)
     {
@@ -72,7 +74,7 @@ __pthread_once (once_control, init_routine)
          signals that initialization has finished, we need to see any
          data modifications done during initialization.  */
       val = *once_control;
-      atomic_read_barrier();
+      atomic_read_barrier ();
       do
 	{
 	  /* Check if the initialization has already been done.  */
@@ -120,7 +122,7 @@ __pthread_once (once_control, init_routine)
       /* Mark *once_control as having finished the initialization.  We need
          release memory order here because we need to synchronize with other
          threads that want to use the initialized data.  */
-      atomic_write_barrier();
+      atomic_write_barrier ();
       *once_control = __PTHREAD_ONCE_DONE;
 
       /* Wake up all other threads.  */
@@ -130,5 +132,18 @@ __pthread_once (once_control, init_routine)
 
   return 0;
 }
+
+int
+__pthread_once (pthread_once_t *once_control, void (*init_routine) (void))
+{
+  /* Fast path.  See __pthread_once_slow.  */
+  int val;
+  val = *once_control;
+  atomic_read_barrier ();
+  if (__glibc_likely ((val & __PTHREAD_ONCE_DONE) != 0))
+    return 0;
+  else
+    return __pthread_once_slow (once_control, init_routine);
+}
 weak_alias (__pthread_once, pthread_once)
 hidden_def (__pthread_once)
diff --git a/sysdeps/unix/sysv/linux/i386/pthread_once.S b/sysdeps/unix/sysv/linux/i386/pthread_once.S
deleted file mode 100644
index dacd724d8b..0000000000
--- a/sysdeps/unix/sysv/linux/i386/pthread_once.S
+++ /dev/null
@@ -1,178 +0,0 @@
-/* Copyright (C) 2002-2014 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#include <unwindbuf.h>
-#include <sysdep.h>
-#include <kernel-features.h>
-#include <lowlevellock.h>
-
-
-	.comm	__fork_generation, 4, 4
-
-	.text
-
-
-	.globl	__pthread_once
-	.type	__pthread_once,@function
-	.align	16
-	cfi_startproc
-__pthread_once:
-	movl	4(%esp), %ecx
-	testl	$2, (%ecx)
-	jz	1f
-	xorl	%eax, %eax
-	ret
-
-1:	pushl	%ebx
-	cfi_adjust_cfa_offset (4)
-	cfi_rel_offset (3, 0)
-	pushl	%esi
-	cfi_adjust_cfa_offset (4)
-	cfi_rel_offset (6, 0)
-	movl	%ecx, %ebx
-	xorl	%esi, %esi
-
-	/* Not yet initialized or initialization in progress.
-	   Get the fork generation counter now.  */
-6:	movl	(%ebx), %eax
-#ifdef PIC
-	LOAD_PIC_REG(cx)
-#endif
-
-5:	movl	%eax, %edx
-
-	testl	$2, %eax
-	jnz	4f
-
-	andl	$3, %edx
-#ifdef PIC
-	orl	__fork_generation@GOTOFF(%ecx), %edx
-#else
-	orl	__fork_generation, %edx
-#endif
-	orl	$1, %edx
-
-	LOCK
-	cmpxchgl %edx, (%ebx)
-	jnz	5b
-
-	/* Check whether another thread already runs the initializer.  */
-	testl	$1, %eax
-	jz	3f	/* No -> do it.  */
-
-	/* Check whether the initializer execution was interrupted
-	   by a fork.  */
-	xorl	%edx, %eax
-	testl	$0xfffffffc, %eax
-	jnz	3f	/* Different for generation -> run initializer.  */
-
-	/* Somebody else got here first.  Wait.  */
-#ifdef __ASSUME_PRIVATE_FUTEX
-	movl	$FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %ecx
-#else
-# if FUTEX_WAIT == 0
-	movl	%gs:PRIVATE_FUTEX, %ecx
-# else
-	movl	$FUTEX_WAIT, %ecx
-	orl	%gs:PRIVATE_FUTEX, %ecx
-# endif
-#endif
-	movl	$SYS_futex, %eax
-	ENTER_KERNEL
-	jmp	6b
-
-3:	/* Call the initializer function after setting up the
-	   cancellation handler.  Note that it is not possible here
-	   to use the unwind-based cleanup handling.  This would require
-	   that the user-provided function and all the code it calls
-	   is compiled with exceptions.  Unfortunately this cannot be
-	   guaranteed.  */
-	subl	$UNWINDBUFSIZE+8, %esp
-	cfi_adjust_cfa_offset (UNWINDBUFSIZE+8)
-	movl	%ecx, %ebx		/* PIC register value.  */
-
-	leal	8+UWJMPBUF(%esp), %eax
-	movl	$0, 4(%esp)
-	movl	%eax, (%esp)
-	call	__sigsetjmp@PLT
-	testl	%eax, %eax
-	jne	7f
-
-	leal	8(%esp), %eax
-	call	HIDDEN_JUMPTARGET(__pthread_register_cancel)
-
-	/* Call the user-provided initialization function.  */
-	call	*24+UNWINDBUFSIZE(%esp)
-
-	/* Pop the cleanup handler.  */
-	leal	8(%esp), %eax
-	call	HIDDEN_JUMPTARGET(__pthread_unregister_cancel)
-	addl	$UNWINDBUFSIZE+8, %esp
-	cfi_adjust_cfa_offset (-UNWINDBUFSIZE-8)
-
-	/* Sucessful run of the initializer.  Signal that we are done.  */
-	movl	12(%esp), %ebx
-	LOCK
-	addl	$1, (%ebx)
-
-	/* Wake up all other threads.  */
-	movl	$0x7fffffff, %edx
-#ifdef __ASSUME_PRIVATE_FUTEX
-	movl	$FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx
-#else
-	movl	$FUTEX_WAKE, %ecx
-	orl	%gs:PRIVATE_FUTEX, %ecx
-#endif
-	movl	$SYS_futex, %eax
-	ENTER_KERNEL
-
-4:	popl	%esi
-	cfi_adjust_cfa_offset (-4)
-	cfi_restore (6)
-	popl	%ebx
-	cfi_adjust_cfa_offset (-4)
-	cfi_restore (3)
-	xorl	%eax, %eax
-	ret
-
-7:	/* __sigsetjmp returned for the second time.  */
-	movl	20+UNWINDBUFSIZE(%esp), %ebx
-	cfi_adjust_cfa_offset (UNWINDBUFSIZE+16)
-	cfi_offset (3, -8)
-	cfi_offset (6, -12)
-	movl	$0, (%ebx)
-
-	movl	$0x7fffffff, %edx
-#ifdef __ASSUME_PRIVATE_FUTEX
-	movl	$FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx
-#else
-	movl	$FUTEX_WAKE, %ecx
-	orl	%gs:PRIVATE_FUTEX, %ecx
-#endif
-	movl	$SYS_futex, %eax
-	ENTER_KERNEL
-
-	leal	8(%esp), %eax
-	call	HIDDEN_JUMPTARGET (__pthread_unwind_next)
-	/* NOTREACHED */
-	hlt
-	cfi_endproc
-	.size	__pthread_once,.-__pthread_once
-
-hidden_def (__pthread_once)
-strong_alias (__pthread_once, pthread_once)
diff --git a/sysdeps/unix/sysv/linux/x86_64/pthread_once.S b/sysdeps/unix/sysv/linux/x86_64/pthread_once.S
deleted file mode 100644
index 2cbe2fae62..0000000000
--- a/sysdeps/unix/sysv/linux/x86_64/pthread_once.S
+++ /dev/null
@@ -1,193 +0,0 @@
-/* Copyright (C) 2002-2014 Free Software Foundation, Inc.
-   This file is part of the GNU C Library.
-   Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
-
-   The GNU C Library is free software; you can redistribute it and/or
-   modify it under the terms of the GNU Lesser General Public
-   License as published by the Free Software Foundation; either
-   version 2.1 of the License, or (at your option) any later version.
-
-   The GNU C Library is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Lesser General Public License for more details.
-
-   You should have received a copy of the GNU Lesser General Public
-   License along with the GNU C Library; if not, see
-   <http://www.gnu.org/licenses/>.  */
-
-#include <sysdep.h>
-#include <kernel-features.h>
-#include <tcb-offsets.h>
-#include <lowlevellock.h>
-
-
-	.comm	__fork_generation, 4, 4
-
-	.text
-
-
-	.globl	__pthread_once
-	.type	__pthread_once,@function
-	.align	16
-__pthread_once:
-.LSTARTCODE:
-	cfi_startproc
-#ifdef SHARED
-	cfi_personality(DW_EH_PE_pcrel | DW_EH_PE_sdata4 | DW_EH_PE_indirect,
-			DW.ref.__gcc_personality_v0)
-	cfi_lsda(DW_EH_PE_pcrel | DW_EH_PE_sdata4, .LexceptSTART)
-#else
-	cfi_personality(DW_EH_PE_udata4, __gcc_personality_v0)
-	cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
-#endif
-	testl	$2, (%rdi)
-	jz	1f
-	xorl	%eax, %eax
-	retq
-
-	/* Preserve the function pointer.  */
-1:	pushq	%rsi
-	cfi_adjust_cfa_offset(8)
-	xorq	%r10, %r10
-
-	/* Not yet initialized or initialization in progress.
-	   Get the fork generation counter now.  */
-6:	movl	(%rdi), %eax
-
-5:	movl	%eax, %edx
-
-	testl	$2, %eax
-	jnz	4f
-
-	andl	$3, %edx
-	orl	__fork_generation(%rip), %edx
-	orl	$1, %edx
-
-	LOCK
-	cmpxchgl %edx, (%rdi)
-	jnz	5b
-
-	/* Check whether another thread already runs the initializer.  */
-	testl	$1, %eax
-	jz	3f	/* No -> do it.  */
-
-	/* Check whether the initializer execution was interrupted
-	   by a fork.  */
-	xorl	%edx, %eax
-	testl	$0xfffffffc, %eax
-	jnz	3f	/* Different for generation -> run initializer.  */
-
-	/* Somebody else got here first.  Wait.  */
-#ifdef __ASSUME_PRIVATE_FUTEX
-	movl	$FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %esi
-#else
-# if FUTEX_WAIT == 0
-	movl	%fs:PRIVATE_FUTEX, %esi
-# else
-	movl	$FUTEX_WAIT, %esi
-	orl	%fs:PRIVATE_FUTEX, %esi
-# endif
-#endif
-	movl	$SYS_futex, %eax
-	syscall
-	jmp	6b
-
-	/* Preserve the pointer to the control variable.  */
-3:	pushq	%rdi
-	cfi_adjust_cfa_offset(8)
-	pushq	%rdi
-	cfi_adjust_cfa_offset(8)
-
-.LcleanupSTART:
-	callq	*16(%rsp)
-.LcleanupEND:
-
-	/* Get the control variable address back.  */
-	popq	%rdi
-	cfi_adjust_cfa_offset(-8)
-
-	/* Sucessful run of the initializer.  Signal that we are done.  */
-	LOCK
-	incl	(%rdi)
-
-	addq	$8, %rsp
-	cfi_adjust_cfa_offset(-8)
-
-	/* Wake up all other threads.  */
-	movl	$0x7fffffff, %edx
-#ifdef __ASSUME_PRIVATE_FUTEX
-	movl	$FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
-#else
-	movl	$FUTEX_WAKE, %esi
-	orl	%fs:PRIVATE_FUTEX, %esi
-#endif
-	movl	$SYS_futex, %eax
-	syscall
-
-4:	addq	$8, %rsp
-	cfi_adjust_cfa_offset(-8)
-	xorl	%eax, %eax
-	retq
-	.size	__pthread_once,.-__pthread_once
-
-
-hidden_def (__pthread_once)
-strong_alias (__pthread_once, pthread_once)
-
-
-	.type	clear_once_control,@function
-	.align	16
-clear_once_control:
-	cfi_adjust_cfa_offset(3 * 8)
-	movq	(%rsp), %rdi
-	movq	%rax, %r8
-	movl	$0, (%rdi)
-
-	movl	$0x7fffffff, %edx
-#ifdef __ASSUME_PRIVATE_FUTEX
-	movl	$FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
-#else
-	movl	$FUTEX_WAKE, %esi
-	orl	%fs:PRIVATE_FUTEX, %esi
-#endif
-	movl	$SYS_futex, %eax
-	syscall
-
-	movq	%r8, %rdi
-.LcallUR:
-	call	_Unwind_Resume@PLT
-	hlt
-.LENDCODE:
-	cfi_endproc
-	.size	clear_once_control,.-clear_once_control
-
-
-	.section .gcc_except_table,"a",@progbits
-.LexceptSTART:
-	.byte	DW_EH_PE_omit			# @LPStart format
-	.byte	DW_EH_PE_omit			# @TType format
-	.byte	DW_EH_PE_uleb128		# call-site format
-	.uleb128 .Lcstend-.Lcstbegin
-.Lcstbegin:
-	.uleb128 .LcleanupSTART-.LSTARTCODE
-	.uleb128 .LcleanupEND-.LcleanupSTART
-	.uleb128 clear_once_control-.LSTARTCODE
-	.uleb128  0
-	.uleb128 .LcallUR-.LSTARTCODE
-	.uleb128 .LENDCODE-.LcallUR
-	.uleb128 0
-	.uleb128  0
-.Lcstend:
-
-
-#ifdef SHARED
-	.hidden	DW.ref.__gcc_personality_v0
-	.weak	DW.ref.__gcc_personality_v0
-	.section .gnu.linkonce.d.DW.ref.__gcc_personality_v0,"aw",@progbits
-	.align	LP_SIZE
-	.type	DW.ref.__gcc_personality_v0, @object
-	.size	DW.ref.__gcc_personality_v0, LP_SIZE
-DW.ref.__gcc_personality_v0:
-	ASM_ADDR __gcc_personality_v0
-#endif