diff options
| -rw-r--r-- | ChangeLog | 12 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/Makefile | 1 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/ifunc-impl-list.c | 4 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/ifunc-memcmp.h | 45 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/memcmp-sse2.S | 31 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/memcmp.S | 78 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/memcmp.c | 38 | ||||
| -rw-r--r-- | sysdeps/x86_64/multiarch/wmemcmp.c (renamed from sysdeps/x86_64/multiarch/wmemcmp.S) | 42 |
8 files changed, 138 insertions, 113 deletions
@@ -2,6 +2,18 @@ Erich Elsen <eriche@google.com> * sysdeps/x86_64/multiarch/Makefile (sysdep_routines): Add + memcmp-sse2. + * sysdeps/x86_64/multiarch/ifunc-memcmp.h: New file. + * sysdeps/x86_64/multiarch/memcmp-sse2.S: Likewise. + * sysdeps/x86_64/multiarch/memcmp.c: Likewise. + * sysdeps/x86_64/multiarch/wmemcmp.c: Likewise. + * sysdeps/x86_64/multiarch/memcmp.S: Removed. + * sysdeps/x86_64/multiarch/wmemcmp.S: Likewise. + +2017-06-15 H.J. Lu <hongjiu.lu@intel.com> + Erich Elsen <eriche@google.com> + + * sysdeps/x86_64/multiarch/Makefile (sysdep_routines): Add memset-sse2-unaligned-erms, and memset_chk-nonshared. * sysdeps/x86_64/multiarch/ifunc-impl-list.c (__libc_ifunc_impl_list): Add test for __memset_chk_erms. diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile index f577175f3d..ff6c7f43de 100644 --- a/sysdeps/x86_64/multiarch/Makefile +++ b/sysdeps/x86_64/multiarch/Makefile @@ -8,6 +8,7 @@ sysdep_routines += strncat-c stpncpy-c strncpy-c strcmp-ssse3 \ strcmp-sse2-unaligned strncmp-ssse3 \ memchr-sse2 rawmemchr-sse2 memchr-avx2 rawmemchr-avx2 \ memrchr-sse2 memrchr-avx2 \ + memcmp-sse2 \ memcmp-avx2-movbe \ memcmp-sse4 memcpy-ssse3 \ memmove-ssse3 \ diff --git a/sysdeps/x86_64/multiarch/ifunc-impl-list.c b/sysdeps/x86_64/multiarch/ifunc-impl-list.c index 52ea44018e..7912dd0003 100644 --- a/sysdeps/x86_64/multiarch/ifunc-impl-list.c +++ b/sysdeps/x86_64/multiarch/ifunc-impl-list.c @@ -45,7 +45,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, __memchr_avx2) IFUNC_IMPL_ADD (array, i, memchr, 1, __memchr_sse2)) - /* Support sysdeps/x86_64/multiarch/memcmp.S. */ + /* Support sysdeps/x86_64/multiarch/memcmp.c. */ IFUNC_IMPL (i, name, memcmp, IFUNC_IMPL_ADD (array, i, memcmp, (HAS_ARCH_FEATURE (AVX2_Usable) @@ -390,7 +390,7 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array, __wmemchr_avx2) IFUNC_IMPL_ADD (array, i, wmemchr, 1, __wmemchr_sse2)) - /* Support sysdeps/x86_64/multiarch/wmemcmp.S. */ + /* Support sysdeps/x86_64/multiarch/wmemcmp.c. */ IFUNC_IMPL (i, name, wmemcmp, IFUNC_IMPL_ADD (array, i, wmemcmp, (HAS_ARCH_FEATURE (AVX2_Usable) diff --git a/sysdeps/x86_64/multiarch/ifunc-memcmp.h b/sysdeps/x86_64/multiarch/ifunc-memcmp.h new file mode 100644 index 0000000000..6f76a93bb8 --- /dev/null +++ b/sysdeps/x86_64/multiarch/ifunc-memcmp.h @@ -0,0 +1,45 @@ +/* Common definition for memcmp/wmemcmp ifunc selections. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +# include <init-arch.h> + +extern __typeof (REDIRECT_NAME) OPTIMIZE (sse2) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (ssse3) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (sse4_1) attribute_hidden; +extern __typeof (REDIRECT_NAME) OPTIMIZE (avx2_movbe) attribute_hidden; + +static inline void * +IFUNC_SELECTOR (void) +{ + const struct cpu_features* cpu_features = __get_cpu_features (); + + if (!CPU_FEATURES_ARCH_P (cpu_features, Prefer_No_VZEROUPPER) + && CPU_FEATURES_ARCH_P (cpu_features, AVX2_Usable) + && CPU_FEATURES_CPU_P (cpu_features, MOVBE) + && CPU_FEATURES_ARCH_P (cpu_features, AVX_Fast_Unaligned_Load)) + return OPTIMIZE (avx2_movbe); + + if (CPU_FEATURES_CPU_P (cpu_features, SSE4_1)) + return OPTIMIZE (sse4_1); + + if (CPU_FEATURES_CPU_P (cpu_features, SSSE3)) + return OPTIMIZE (ssse3); + + return OPTIMIZE (sse2); +} diff --git a/sysdeps/x86_64/multiarch/memcmp-sse2.S b/sysdeps/x86_64/multiarch/memcmp-sse2.S new file mode 100644 index 0000000000..5172d991bb --- /dev/null +++ b/sysdeps/x86_64/multiarch/memcmp-sse2.S @@ -0,0 +1,31 @@ +/* memcmp with SSE2. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#if IS_IN (libc) +# define memcmp __memcmp_sse2 + +# ifdef SHARED +# undef libc_hidden_builtin_def +# define libc_hidden_builtin_def(name) +# endif + +# undef weak_alias +# define weak_alias(ignored1, ignored2) +#endif + +#include <sysdeps/x86_64/memcmp.S> diff --git a/sysdeps/x86_64/multiarch/memcmp.S b/sysdeps/x86_64/multiarch/memcmp.S deleted file mode 100644 index 0c9804b7e9..0000000000 --- a/sysdeps/x86_64/multiarch/memcmp.S +++ /dev/null @@ -1,78 +0,0 @@ -/* Multiple versions of memcmp - All versions must be listed in ifunc-impl-list.c. - Copyright (C) 2010-2017 Free Software Foundation, Inc. - Contributed by Intel Corporation. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <http://www.gnu.org/licenses/>. */ - -#include <sysdep.h> -#include <init-arch.h> - -/* Define multiple versions only for the definition in libc. */ -#if IS_IN (libc) - .text -ENTRY(memcmp) - .type memcmp, @gnu_indirect_function - LOAD_RTLD_GLOBAL_RO_RDX - HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER) - jnz 1f - HAS_ARCH_FEATURE (AVX2_Usable) - jz 1f - HAS_CPU_FEATURE (MOVBE) - jz 1f - HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load) - jz 1f - leaq __memcmp_avx2_movbe(%rip), %rax - ret - -1: HAS_CPU_FEATURE (SSSE3) - jnz 2f - leaq __memcmp_sse2(%rip), %rax - ret - -2: HAS_CPU_FEATURE (SSE4_1) - jz 3f - leaq __memcmp_sse4_1(%rip), %rax - ret - -3: leaq __memcmp_ssse3(%rip), %rax - ret - -END(memcmp) - -# undef ENTRY -# define ENTRY(name) \ - .type __memcmp_sse2, @function; \ - .p2align 4; \ - .globl __memcmp_sse2; \ - .hidden __memcmp_sse2; \ - __memcmp_sse2: cfi_startproc; \ - CALL_MCOUNT -# undef END -# define END(name) \ - cfi_endproc; .size __memcmp_sse2, .-__memcmp_sse2 - -# ifdef SHARED -# undef libc_hidden_builtin_def -/* It doesn't make sense to send libc-internal memcmp calls through a PLT. - The speedup we get from using SSE4 instructions is likely eaten away - by the indirect call in the PLT. */ -# define libc_hidden_builtin_def(name) \ - .globl __GI_memcmp; __GI_memcmp = __memcmp_sse2 -# endif -#endif - -#include "../memcmp.S" diff --git a/sysdeps/x86_64/multiarch/memcmp.c b/sysdeps/x86_64/multiarch/memcmp.c new file mode 100644 index 0000000000..b3e4ac36ff --- /dev/null +++ b/sysdeps/x86_64/multiarch/memcmp.c @@ -0,0 +1,38 @@ +/* Multiple versions of memcmp. + All versions must be listed in ifunc-impl-list.c. + Copyright (C) 2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +/* Define multiple versions only for the definition in lib and for + DSO. */ +#if IS_IN (libc) +# define memcmp __redirect_memcmp +# include <string.h> +# undef memcmp + +# define SYMBOL_NAME memcmp +# include "ifunc-memcmp.h" + +libc_ifunc_redirected (__redirect_memcmp, memcmp, IFUNC_SELECTOR ()); +# undef bcmp +weak_alias (memcmp, bcmp) + +# ifdef SHARED +__hidden_ver1 (memcmp, __GI_memcmp, __redirect_memcmp) + __attribute__ ((visibility ("hidden"))); +# endif +#endif diff --git a/sysdeps/x86_64/multiarch/wmemcmp.S b/sysdeps/x86_64/multiarch/wmemcmp.c index 94b25a214c..bbf6c836e9 100644 --- a/sysdeps/x86_64/multiarch/wmemcmp.S +++ b/sysdeps/x86_64/multiarch/wmemcmp.c @@ -1,7 +1,6 @@ /* Multiple versions of wmemcmp All versions must be listed in ifunc-impl-list.c. - Copyright (C) 2011-2017 Free Software Foundation, Inc. - Contributed by Intel Corporation. + Copyright (C) 2017 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -18,38 +17,15 @@ License along with the GNU C Library; if not, see <http://www.gnu.org/licenses/>. */ -#include <sysdep.h> -#include <init-arch.h> - -/* Define multiple versions only for the definition in libc. */ +/* Define multiple versions only for the definition in lib and for + DSO. */ #if IS_IN (libc) - .text -ENTRY(wmemcmp) - .type wmemcmp, @gnu_indirect_function - LOAD_RTLD_GLOBAL_RO_RDX - HAS_ARCH_FEATURE (Prefer_No_VZEROUPPER) - jnz 1f - HAS_ARCH_FEATURE (AVX2_Usable) - jz 1f - HAS_CPU_FEATURE (MOVBE) - jz 1f - HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load) - jz 1f - leaq __wmemcmp_avx2_movbe(%rip), %rax - ret - -1: HAS_CPU_FEATURE (SSSE3) - jnz 2f - leaq __wmemcmp_sse2(%rip), %rax - ret - -2: HAS_CPU_FEATURE (SSE4_1) - jz 3f - leaq __wmemcmp_sse4_1(%rip), %rax - ret +# define wmemcmp __redirect_wmemcmp +# include <wchar.h> +# undef wmemcmp -3: leaq __wmemcmp_ssse3(%rip), %rax - ret +# define SYMBOL_NAME wmemcmp +# include "ifunc-memcmp.h" -END(wmemcmp) +libc_ifunc_redirected (__redirect_wmemcmp, wmemcmp, IFUNC_SELECTOR ()); #endif |