diff options
Diffstat (limited to 'REORG.TODO/sysdeps/powerpc/powerpc64/power4')
10 files changed, 2334 insertions, 0 deletions
diff --git a/REORG.TODO/sysdeps/powerpc/powerpc64/power4/Implies b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/Implies new file mode 100644 index 0000000000..a372141bb7 --- /dev/null +++ b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/Implies @@ -0,0 +1,2 @@ +powerpc/power4/fpu +powerpc/power4 diff --git a/REORG.TODO/sysdeps/powerpc/powerpc64/power4/Makefile b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/Makefile new file mode 100644 index 0000000000..ba06adb5d0 --- /dev/null +++ b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/Makefile @@ -0,0 +1,6 @@ +# Makefile fragment for POWER4/5/5+. + +ifeq ($(subdir),string) +CFLAGS-wordcopy.c += --param max-variable-expansions-in-unroller=2 --param max-unroll-times=2 -funroll-loops -fpeel-loops +CFLAGS-memmove.c += --param max-variable-expansions-in-unroller=2 --param max-unroll-times=2 -funroll-loops -fpeel-loops +endif diff --git a/REORG.TODO/sysdeps/powerpc/powerpc64/power4/fpu/Implies b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/fpu/Implies new file mode 100644 index 0000000000..c1f617b7da --- /dev/null +++ b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/fpu/Implies @@ -0,0 +1 @@ +powerpc/powerpc64/fpu diff --git a/REORG.TODO/sysdeps/powerpc/powerpc64/power4/fpu/multiarch/Implies b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/fpu/multiarch/Implies new file mode 100644 index 0000000000..8d6531a174 --- /dev/null +++ b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/fpu/multiarch/Implies @@ -0,0 +1 @@ +powerpc/powerpc64/fpu/multiarch diff --git a/REORG.TODO/sysdeps/powerpc/powerpc64/power4/memcmp.S b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/memcmp.S new file mode 100644 index 0000000000..6ca98e909c --- /dev/null +++ b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/memcmp.S @@ -0,0 +1,1369 @@ +/* Optimized memcmp implementation for PowerPC64. + Copyright (C) 2003-2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> + +/* int [r3] memcmp (const char *s1 [r3], + const char *s2 [r4], + size_t size [r5]) */ + +#ifndef MEMCMP +# define MEMCMP memcmp +#endif + + .machine power4 +EALIGN (MEMCMP, 4, 0) + CALL_MCOUNT 3 + +#define rRTN r3 +#define rSTR1 r3 /* first string arg */ +#define rSTR2 r4 /* second string arg */ +#define rN r5 /* max string length */ +#define rWORD1 r6 /* current word in s1 */ +#define rWORD2 r7 /* current word in s2 */ +#define rWORD3 r8 /* next word in s1 */ +#define rWORD4 r9 /* next word in s2 */ +#define rWORD5 r10 /* next word in s1 */ +#define rWORD6 r11 /* next word in s2 */ +#define rWORD7 r30 /* next word in s1 */ +#define rWORD8 r31 /* next word in s2 */ + + xor r0, rSTR2, rSTR1 + cmpldi cr6, rN, 0 + cmpldi cr1, rN, 12 + clrldi. r0, r0, 61 + clrldi r12, rSTR1, 61 + cmpldi cr5, r12, 0 + beq- cr6, L(zeroLength) + dcbt 0, rSTR1 + dcbt 0, rSTR2 +/* If less than 8 bytes or not aligned, use the unaligned + byte loop. */ + blt cr1, L(bytealigned) + std rWORD8, -8(r1) + std rWORD7, -16(r1) + cfi_offset(rWORD8, -8) + cfi_offset(rWORD7, -16) + bne L(unaligned) +/* At this point we know both strings have the same alignment and the + compare length is at least 8 bytes. r12 contains the low order + 3 bits of rSTR1 and cr5 contains the result of the logical compare + of r12 to 0. If r12 == 0 then we are already double word + aligned and can perform the DW aligned loop. + + Otherwise we know the two strings have the same alignment (but not + yet DW). So we force the string addresses to the next lower DW + boundary and special case this first DW using shift left to + eliminate bits preceding the first byte. Since we want to join the + normal (DW aligned) compare loop, starting at the second double word, + we need to adjust the length (rN) and special case the loop + versioning for the first DW. This ensures that the loop count is + correct and the first DW (shifted) is in the expected register pair. */ + .align 4 +L(samealignment): + clrrdi rSTR1, rSTR1, 3 + clrrdi rSTR2, rSTR2, 3 + beq cr5, L(DWaligned) + add rN, rN, r12 + sldi rWORD6, r12, 3 + srdi r0, rN, 5 /* Divide by 32 */ + andi. r12, rN, 24 /* Get the DW remainder */ +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD1, 0, rSTR1 + ldbrx rWORD2, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD1, 0(rSTR1) + ld rWORD2, 0(rSTR2) +#endif + cmpldi cr1, r12, 16 + cmpldi cr7, rN, 32 + clrldi rN, rN, 61 + beq L(dPs4) + mtctr r0 /* Power4 wants mtctr 1st in dispatch group */ + bgt cr1, L(dPs3) + beq cr1, L(dPs2) + +/* Remainder is 8 */ + .align 3 +L(dsP1): + sld rWORD5, rWORD1, rWORD6 + sld rWORD6, rWORD2, rWORD6 + cmpld cr5, rWORD5, rWORD6 + blt cr7, L(dP1x) +/* Do something useful in this cycle since we have to branch anyway. */ +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD1, 0, rSTR1 + ldbrx rWORD2, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD1, 8(rSTR1) + ld rWORD2, 8(rSTR2) +#endif + cmpld cr7, rWORD1, rWORD2 + b L(dP1e) +/* Remainder is 16 */ + .align 4 +L(dPs2): + sld rWORD5, rWORD1, rWORD6 + sld rWORD6, rWORD2, rWORD6 + cmpld cr6, rWORD5, rWORD6 + blt cr7, L(dP2x) +/* Do something useful in this cycle since we have to branch anyway. */ +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD7, 0, rSTR1 + ldbrx rWORD8, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD7, 8(rSTR1) + ld rWORD8, 8(rSTR2) +#endif + cmpld cr5, rWORD7, rWORD8 + b L(dP2e) +/* Remainder is 24 */ + .align 4 +L(dPs3): + sld rWORD3, rWORD1, rWORD6 + sld rWORD4, rWORD2, rWORD6 + cmpld cr1, rWORD3, rWORD4 + b L(dP3e) +/* Count is a multiple of 32, remainder is 0 */ + .align 4 +L(dPs4): + mtctr r0 /* Power4 wants mtctr 1st in dispatch group */ + sld rWORD1, rWORD1, rWORD6 + sld rWORD2, rWORD2, rWORD6 + cmpld cr7, rWORD1, rWORD2 + b L(dP4e) + +/* At this point we know both strings are double word aligned and the + compare length is at least 8 bytes. */ + .align 4 +L(DWaligned): + andi. r12, rN, 24 /* Get the DW remainder */ + srdi r0, rN, 5 /* Divide by 32 */ + cmpldi cr1, r12, 16 + cmpldi cr7, rN, 32 + clrldi rN, rN, 61 + beq L(dP4) + bgt cr1, L(dP3) + beq cr1, L(dP2) + +/* Remainder is 8 */ + .align 4 +L(dP1): + mtctr r0 /* Power4 wants mtctr 1st in dispatch group */ +/* Normally we'd use rWORD7/rWORD8 here, but since we might exit early + (8-15 byte compare), we want to use only volatile registers. This + means we can avoid restoring non-volatile registers since we did not + change any on the early exit path. The key here is the non-early + exit path only cares about the condition code (cr5), not about which + register pair was used. */ +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD5, 0, rSTR1 + ldbrx rWORD6, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD5, 0(rSTR1) + ld rWORD6, 0(rSTR2) +#endif + cmpld cr5, rWORD5, rWORD6 + blt cr7, L(dP1x) +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD1, 0, rSTR1 + ldbrx rWORD2, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD1, 8(rSTR1) + ld rWORD2, 8(rSTR2) +#endif + cmpld cr7, rWORD1, rWORD2 +L(dP1e): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD3, 0, rSTR1 + ldbrx rWORD4, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD3, 16(rSTR1) + ld rWORD4, 16(rSTR2) +#endif + cmpld cr1, rWORD3, rWORD4 +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD5, 0, rSTR1 + ldbrx rWORD6, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD5, 24(rSTR1) + ld rWORD6, 24(rSTR2) +#endif + cmpld cr6, rWORD5, rWORD6 + bne cr5, L(dLcr5x) + bne cr7, L(dLcr7x) + +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD7, 0, rSTR1 + ldbrx rWORD8, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ldu rWORD7, 32(rSTR1) + ldu rWORD8, 32(rSTR2) +#endif + bne cr1, L(dLcr1) + cmpld cr5, rWORD7, rWORD8 + bdnz L(dLoop) + bne cr6, L(dLcr6) + ld rWORD8, -8(r1) + ld rWORD7, -16(r1) + .align 3 +L(dP1x): + sldi. r12, rN, 3 + bne cr5, L(dLcr5x) + subfic rN, r12, 64 /* Shift count is 64 - (rN * 8). */ + bne L(d00) + li rRTN, 0 + blr + +/* Remainder is 16 */ + .align 4 +L(dP2): + mtctr r0 /* Power4 wants mtctr 1st in dispatch group */ +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD5, 0, rSTR1 + ldbrx rWORD6, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD5, 0(rSTR1) + ld rWORD6, 0(rSTR2) +#endif + cmpld cr6, rWORD5, rWORD6 + blt cr7, L(dP2x) +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD7, 0, rSTR1 + ldbrx rWORD8, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD7, 8(rSTR1) + ld rWORD8, 8(rSTR2) +#endif + cmpld cr5, rWORD7, rWORD8 +L(dP2e): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD1, 0, rSTR1 + ldbrx rWORD2, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD1, 16(rSTR1) + ld rWORD2, 16(rSTR2) +#endif + cmpld cr7, rWORD1, rWORD2 +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD3, 0, rSTR1 + ldbrx rWORD4, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD3, 24(rSTR1) + ld rWORD4, 24(rSTR2) +#endif + cmpld cr1, rWORD3, rWORD4 +#ifndef __LITTLE_ENDIAN__ + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#endif + bne cr6, L(dLcr6) + bne cr5, L(dLcr5) + b L(dLoop2) +/* Again we are on a early exit path (16-23 byte compare), we want to + only use volatile registers and avoid restoring non-volatile + registers. */ + .align 4 +L(dP2x): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD3, 0, rSTR1 + ldbrx rWORD4, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD3, 8(rSTR1) + ld rWORD4, 8(rSTR2) +#endif + cmpld cr1, rWORD3, rWORD4 + sldi. r12, rN, 3 + bne cr6, L(dLcr6x) +#ifndef __LITTLE_ENDIAN__ + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#endif + bne cr1, L(dLcr1x) + subfic rN, r12, 64 /* Shift count is 64 - (rN * 8). */ + bne L(d00) + li rRTN, 0 + blr + +/* Remainder is 24 */ + .align 4 +L(dP3): + mtctr r0 /* Power4 wants mtctr 1st in dispatch group */ +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD3, 0, rSTR1 + ldbrx rWORD4, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD3, 0(rSTR1) + ld rWORD4, 0(rSTR2) +#endif + cmpld cr1, rWORD3, rWORD4 +L(dP3e): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD5, 0, rSTR1 + ldbrx rWORD6, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD5, 8(rSTR1) + ld rWORD6, 8(rSTR2) +#endif + cmpld cr6, rWORD5, rWORD6 + blt cr7, L(dP3x) +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD7, 0, rSTR1 + ldbrx rWORD8, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD7, 16(rSTR1) + ld rWORD8, 16(rSTR2) +#endif + cmpld cr5, rWORD7, rWORD8 +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD1, 0, rSTR1 + ldbrx rWORD2, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD1, 24(rSTR1) + ld rWORD2, 24(rSTR2) +#endif + cmpld cr7, rWORD1, rWORD2 +#ifndef __LITTLE_ENDIAN__ + addi rSTR1, rSTR1, 16 + addi rSTR2, rSTR2, 16 +#endif + bne cr1, L(dLcr1) + bne cr6, L(dLcr6) + b L(dLoop1) +/* Again we are on a early exit path (24-31 byte compare), we want to + only use volatile registers and avoid restoring non-volatile + registers. */ + .align 4 +L(dP3x): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD1, 0, rSTR1 + ldbrx rWORD2, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD1, 16(rSTR1) + ld rWORD2, 16(rSTR2) +#endif + cmpld cr7, rWORD1, rWORD2 + sldi. r12, rN, 3 + bne cr1, L(dLcr1x) +#ifndef __LITTLE_ENDIAN__ + addi rSTR1, rSTR1, 16 + addi rSTR2, rSTR2, 16 +#endif + bne cr6, L(dLcr6x) + subfic rN, r12, 64 /* Shift count is 64 - (rN * 8). */ + bne cr7, L(dLcr7x) + bne L(d00) + li rRTN, 0 + blr + +/* Count is a multiple of 32, remainder is 0 */ + .align 4 +L(dP4): + mtctr r0 /* Power4 wants mtctr 1st in dispatch group */ +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD1, 0, rSTR1 + ldbrx rWORD2, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD1, 0(rSTR1) + ld rWORD2, 0(rSTR2) +#endif + cmpld cr7, rWORD1, rWORD2 +L(dP4e): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD3, 0, rSTR1 + ldbrx rWORD4, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD3, 8(rSTR1) + ld rWORD4, 8(rSTR2) +#endif + cmpld cr1, rWORD3, rWORD4 +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD5, 0, rSTR1 + ldbrx rWORD6, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD5, 16(rSTR1) + ld rWORD6, 16(rSTR2) +#endif + cmpld cr6, rWORD5, rWORD6 +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD7, 0, rSTR1 + ldbrx rWORD8, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ldu rWORD7, 24(rSTR1) + ldu rWORD8, 24(rSTR2) +#endif + cmpld cr5, rWORD7, rWORD8 + bne cr7, L(dLcr7) + bne cr1, L(dLcr1) + bdz- L(d24) /* Adjust CTR as we start with +4 */ +/* This is the primary loop */ + .align 4 +L(dLoop): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD1, 0, rSTR1 + ldbrx rWORD2, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD1, 8(rSTR1) + ld rWORD2, 8(rSTR2) +#endif + cmpld cr1, rWORD3, rWORD4 + bne cr6, L(dLcr6) +L(dLoop1): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD3, 0, rSTR1 + ldbrx rWORD4, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD3, 16(rSTR1) + ld rWORD4, 16(rSTR2) +#endif + cmpld cr6, rWORD5, rWORD6 + bne cr5, L(dLcr5) +L(dLoop2): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD5, 0, rSTR1 + ldbrx rWORD6, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD5, 24(rSTR1) + ld rWORD6, 24(rSTR2) +#endif + cmpld cr5, rWORD7, rWORD8 + bne cr7, L(dLcr7) +L(dLoop3): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD7, 0, rSTR1 + ldbrx rWORD8, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ldu rWORD7, 32(rSTR1) + ldu rWORD8, 32(rSTR2) +#endif + bne- cr1, L(dLcr1) + cmpld cr7, rWORD1, rWORD2 + bdnz+ L(dLoop) + +L(dL4): + cmpld cr1, rWORD3, rWORD4 + bne cr6, L(dLcr6) + cmpld cr6, rWORD5, rWORD6 + bne cr5, L(dLcr5) + cmpld cr5, rWORD7, rWORD8 +L(d44): + bne cr7, L(dLcr7) +L(d34): + bne cr1, L(dLcr1) +L(d24): + bne cr6, L(dLcr6) +L(d14): + sldi. r12, rN, 3 + bne cr5, L(dLcr5) +L(d04): + ld rWORD8, -8(r1) + ld rWORD7, -16(r1) + subfic rN, r12, 64 /* Shift count is 64 - (rN * 8). */ + beq L(zeroLength) +/* At this point we have a remainder of 1 to 7 bytes to compare. Since + we are aligned it is safe to load the whole double word, and use + shift right double to eliminate bits beyond the compare length. */ +L(d00): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD1, 0, rSTR1 + ldbrx rWORD2, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD1, 8(rSTR1) + ld rWORD2, 8(rSTR2) +#endif + srd rWORD1, rWORD1, rN + srd rWORD2, rWORD2, rN + cmpld cr7, rWORD1, rWORD2 + bne cr7, L(dLcr7x) + li rRTN, 0 + blr + + .align 4 +L(dLcr7): + ld rWORD8, -8(r1) + ld rWORD7, -16(r1) +L(dLcr7x): + li rRTN, 1 + bgtlr cr7 + li rRTN, -1 + blr + .align 4 +L(dLcr1): + ld rWORD8, -8(r1) + ld rWORD7, -16(r1) +L(dLcr1x): + li rRTN, 1 + bgtlr cr1 + li rRTN, -1 + blr + .align 4 +L(dLcr6): + ld rWORD8, -8(r1) + ld rWORD7, -16(r1) +L(dLcr6x): + li rRTN, 1 + bgtlr cr6 + li rRTN, -1 + blr + .align 4 +L(dLcr5): + ld rWORD8, -8(r1) + ld rWORD7, -16(r1) +L(dLcr5x): + li rRTN, 1 + bgtlr cr5 + li rRTN, -1 + blr + + .align 4 +L(bytealigned): + mtctr rN /* Power4 wants mtctr 1st in dispatch group */ +#if 0 +/* Huh? We've already branched on cr6! */ + beq- cr6, L(zeroLength) +#endif + +/* We need to prime this loop. This loop is swing modulo scheduled + to avoid pipe delays. The dependent instruction latencies (load to + compare to conditional branch) is 2 to 3 cycles. In this loop each + dispatch group ends in a branch and takes 1 cycle. Effectively + the first iteration of the loop only serves to load operands and + branches based on compares are delayed until the next loop. + + So we must precondition some registers and condition codes so that + we don't exit the loop early on the first iteration. */ + + lbz rWORD1, 0(rSTR1) + lbz rWORD2, 0(rSTR2) + bdz- L(b11) + cmpld cr7, rWORD1, rWORD2 + lbz rWORD3, 1(rSTR1) + lbz rWORD4, 1(rSTR2) + bdz- L(b12) + cmpld cr1, rWORD3, rWORD4 + lbzu rWORD5, 2(rSTR1) + lbzu rWORD6, 2(rSTR2) + bdz- L(b13) + .align 4 +L(bLoop): + lbzu rWORD1, 1(rSTR1) + lbzu rWORD2, 1(rSTR2) + bne- cr7, L(bLcr7) + + cmpld cr6, rWORD5, rWORD6 + bdz- L(b3i) + + lbzu rWORD3, 1(rSTR1) + lbzu rWORD4, 1(rSTR2) + bne- cr1, L(bLcr1) + + cmpld cr7, rWORD1, rWORD2 + bdz- L(b2i) + + lbzu rWORD5, 1(rSTR1) + lbzu rWORD6, 1(rSTR2) + bne- cr6, L(bLcr6) + + cmpld cr1, rWORD3, rWORD4 + bdnz+ L(bLoop) + +/* We speculatively loading bytes before we have tested the previous + bytes. But we must avoid overrunning the length (in the ctr) to + prevent these speculative loads from causing a segfault. In this + case the loop will exit early (before the all pending bytes are + tested. In this case we must complete the pending operations + before returning. */ +L(b1i): + bne- cr7, L(bLcr7) + bne- cr1, L(bLcr1) + b L(bx56) + .align 4 +L(b2i): + bne- cr6, L(bLcr6) + bne- cr7, L(bLcr7) + b L(bx34) + .align 4 +L(b3i): + bne- cr1, L(bLcr1) + bne- cr6, L(bLcr6) + b L(bx12) + .align 4 +L(bLcr7): + li rRTN, 1 + bgtlr cr7 + li rRTN, -1 + blr +L(bLcr1): + li rRTN, 1 + bgtlr cr1 + li rRTN, -1 + blr +L(bLcr6): + li rRTN, 1 + bgtlr cr6 + li rRTN, -1 + blr + +L(b13): + bne- cr7, L(bx12) + bne- cr1, L(bx34) +L(bx56): + sub rRTN, rWORD5, rWORD6 + blr + nop +L(b12): + bne- cr7, L(bx12) +L(bx34): + sub rRTN, rWORD3, rWORD4 + blr +L(b11): +L(bx12): + sub rRTN, rWORD1, rWORD2 + blr + .align 4 +L(zeroLength): + li rRTN, 0 + blr + + .align 4 +/* At this point we know the strings have different alignment and the + compare length is at least 8 bytes. r12 contains the low order + 3 bits of rSTR1 and cr5 contains the result of the logical compare + of r12 to 0. If r12 == 0 then rStr1 is double word + aligned and can perform the DWunaligned loop. + + Otherwise we know that rSTR1 is not already DW aligned yet. + So we can force the string addresses to the next lower DW + boundary and special case this first DW using shift left to + eliminate bits preceding the first byte. Since we want to join the + normal (DWaligned) compare loop, starting at the second double word, + we need to adjust the length (rN) and special case the loop + versioning for the first DW. This ensures that the loop count is + correct and the first DW (shifted) is in the expected resister pair. */ +#define rSHL r29 /* Unaligned shift left count. */ +#define rSHR r28 /* Unaligned shift right count. */ +#define rWORD8_SHIFT r27 /* Left rotation temp for rWORD2. */ +#define rWORD2_SHIFT r26 /* Left rotation temp for rWORD4. */ +#define rWORD4_SHIFT r25 /* Left rotation temp for rWORD6. */ +#define rWORD6_SHIFT r24 /* Left rotation temp for rWORD8. */ +L(unaligned): + std rSHL, -24(r1) + cfi_offset(rSHL, -24) + clrldi rSHL, rSTR2, 61 + beq- cr6, L(duzeroLength) + std rSHR, -32(r1) + cfi_offset(rSHR, -32) + beq cr5, L(DWunaligned) + std rWORD8_SHIFT, -40(r1) + cfi_offset(rWORD8_SHIFT, -40) +/* Adjust the logical start of rSTR2 to compensate for the extra bits + in the 1st rSTR1 DW. */ + sub rWORD8_SHIFT, rSTR2, r12 +/* But do not attempt to address the DW before that DW that contains + the actual start of rSTR2. */ + clrrdi rSTR2, rSTR2, 3 + std rWORD2_SHIFT, -48(r1) +/* Compute the left/right shift counts for the unaligned rSTR2, + compensating for the logical (DW aligned) start of rSTR1. */ + clrldi rSHL, rWORD8_SHIFT, 61 + clrrdi rSTR1, rSTR1, 3 + std rWORD4_SHIFT, -56(r1) + sldi rSHL, rSHL, 3 + cmpld cr5, rWORD8_SHIFT, rSTR2 + add rN, rN, r12 + sldi rWORD6, r12, 3 + std rWORD6_SHIFT, -64(r1) + cfi_offset(rWORD2_SHIFT, -48) + cfi_offset(rWORD4_SHIFT, -56) + cfi_offset(rWORD6_SHIFT, -64) + subfic rSHR, rSHL, 64 + srdi r0, rN, 5 /* Divide by 32 */ + andi. r12, rN, 24 /* Get the DW remainder */ +/* We normally need to load 2 DWs to start the unaligned rSTR2, but in + this special case those bits may be discarded anyway. Also we + must avoid loading a DW where none of the bits are part of rSTR2 as + this may cross a page boundary and cause a page fault. */ + li rWORD8, 0 + blt cr5, L(dus0) +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD8, 0, rSTR2 + addi rSTR2, rSTR2, 8 +#else + ld rWORD8, 0(rSTR2) + addi rSTR2, rSTR2, 8 +#endif + sld rWORD8, rWORD8, rSHL + +L(dus0): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD1, 0, rSTR1 + ldbrx rWORD2, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD1, 0(rSTR1) + ld rWORD2, 0(rSTR2) +#endif + cmpldi cr1, r12, 16 + cmpldi cr7, rN, 32 + srd r12, rWORD2, rSHR + clrldi rN, rN, 61 + beq L(duPs4) + mtctr r0 /* Power4 wants mtctr 1st in dispatch group */ + or rWORD8, r12, rWORD8 + bgt cr1, L(duPs3) + beq cr1, L(duPs2) + +/* Remainder is 8 */ + .align 4 +L(dusP1): + sld rWORD8_SHIFT, rWORD2, rSHL + sld rWORD7, rWORD1, rWORD6 + sld rWORD8, rWORD8, rWORD6 + bge cr7, L(duP1e) +/* At this point we exit early with the first double word compare + complete and remainder of 0 to 7 bytes. See L(du14) for details on + how we handle the remaining bytes. */ + cmpld cr5, rWORD7, rWORD8 + sldi. rN, rN, 3 + bne cr5, L(duLcr5) + cmpld cr7, rN, rSHR + beq L(duZeroReturn) + li r0, 0 + ble cr7, L(dutrim) +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD2, 0, rSTR2 + addi rSTR2, rSTR2, 8 +#else + ld rWORD2, 8(rSTR2) +#endif + srd r0, rWORD2, rSHR + b L(dutrim) +/* Remainder is 16 */ + .align 4 +L(duPs2): + sld rWORD6_SHIFT, rWORD2, rSHL + sld rWORD5, rWORD1, rWORD6 + sld rWORD6, rWORD8, rWORD6 + b L(duP2e) +/* Remainder is 24 */ + .align 4 +L(duPs3): + sld rWORD4_SHIFT, rWORD2, rSHL + sld rWORD3, rWORD1, rWORD6 + sld rWORD4, rWORD8, rWORD6 + b L(duP3e) +/* Count is a multiple of 32, remainder is 0 */ + .align 4 +L(duPs4): + mtctr r0 /* Power4 wants mtctr 1st in dispatch group */ + or rWORD8, r12, rWORD8 + sld rWORD2_SHIFT, rWORD2, rSHL + sld rWORD1, rWORD1, rWORD6 + sld rWORD2, rWORD8, rWORD6 + b L(duP4e) + +/* At this point we know rSTR1 is double word aligned and the + compare length is at least 8 bytes. */ + .align 4 +L(DWunaligned): + std rWORD8_SHIFT, -40(r1) + clrrdi rSTR2, rSTR2, 3 + std rWORD2_SHIFT, -48(r1) + srdi r0, rN, 5 /* Divide by 32 */ + std rWORD4_SHIFT, -56(r1) + andi. r12, rN, 24 /* Get the DW remainder */ + std rWORD6_SHIFT, -64(r1) + cfi_offset(rWORD8_SHIFT, -40) + cfi_offset(rWORD2_SHIFT, -48) + cfi_offset(rWORD4_SHIFT, -56) + cfi_offset(rWORD6_SHIFT, -64) + sldi rSHL, rSHL, 3 +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD6, 0, rSTR2 + addi rSTR2, rSTR2, 8 + ldbrx rWORD8, 0, rSTR2 + addi rSTR2, rSTR2, 8 +#else + ld rWORD6, 0(rSTR2) + ldu rWORD8, 8(rSTR2) +#endif + cmpldi cr1, r12, 16 + cmpldi cr7, rN, 32 + clrldi rN, rN, 61 + subfic rSHR, rSHL, 64 + sld rWORD6_SHIFT, rWORD6, rSHL + beq L(duP4) + mtctr r0 /* Power4 wants mtctr 1st in dispatch group */ + bgt cr1, L(duP3) + beq cr1, L(duP2) + +/* Remainder is 8 */ + .align 4 +L(duP1): + srd r12, rWORD8, rSHR +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD7, 0, rSTR1 + addi rSTR1, rSTR1, 8 +#else + ld rWORD7, 0(rSTR1) +#endif + sld rWORD8_SHIFT, rWORD8, rSHL + or rWORD8, r12, rWORD6_SHIFT + blt cr7, L(duP1x) +L(duP1e): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD1, 0, rSTR1 + ldbrx rWORD2, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD1, 8(rSTR1) + ld rWORD2, 8(rSTR2) +#endif + cmpld cr5, rWORD7, rWORD8 + srd r0, rWORD2, rSHR + sld rWORD2_SHIFT, rWORD2, rSHL + or rWORD2, r0, rWORD8_SHIFT +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD3, 0, rSTR1 + ldbrx rWORD4, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD3, 16(rSTR1) + ld rWORD4, 16(rSTR2) +#endif + cmpld cr7, rWORD1, rWORD2 + srd r12, rWORD4, rSHR + sld rWORD4_SHIFT, rWORD4, rSHL + bne cr5, L(duLcr5) + or rWORD4, r12, rWORD2_SHIFT +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD5, 0, rSTR1 + ldbrx rWORD6, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD5, 24(rSTR1) + ld rWORD6, 24(rSTR2) +#endif + cmpld cr1, rWORD3, rWORD4 + srd r0, rWORD6, rSHR + sld rWORD6_SHIFT, rWORD6, rSHL + bne cr7, L(duLcr7) + or rWORD6, r0, rWORD4_SHIFT + cmpld cr6, rWORD5, rWORD6 + b L(duLoop3) + .align 4 +/* At this point we exit early with the first double word compare + complete and remainder of 0 to 7 bytes. See L(du14) for details on + how we handle the remaining bytes. */ +L(duP1x): + cmpld cr5, rWORD7, rWORD8 + sldi. rN, rN, 3 + bne cr5, L(duLcr5) + cmpld cr7, rN, rSHR + beq L(duZeroReturn) + li r0, 0 + ble cr7, L(dutrim) +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD2, 0, rSTR2 + addi rSTR2, rSTR2, 8 +#else + ld rWORD2, 8(rSTR2) +#endif + srd r0, rWORD2, rSHR + b L(dutrim) +/* Remainder is 16 */ + .align 4 +L(duP2): + srd r0, rWORD8, rSHR +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD5, 0, rSTR1 + addi rSTR1, rSTR1, 8 +#else + ld rWORD5, 0(rSTR1) +#endif + or rWORD6, r0, rWORD6_SHIFT + sld rWORD6_SHIFT, rWORD8, rSHL +L(duP2e): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD7, 0, rSTR1 + ldbrx rWORD8, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD7, 8(rSTR1) + ld rWORD8, 8(rSTR2) +#endif + cmpld cr6, rWORD5, rWORD6 + srd r12, rWORD8, rSHR + sld rWORD8_SHIFT, rWORD8, rSHL + or rWORD8, r12, rWORD6_SHIFT + blt cr7, L(duP2x) +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD1, 0, rSTR1 + ldbrx rWORD2, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD1, 16(rSTR1) + ld rWORD2, 16(rSTR2) +#endif + cmpld cr5, rWORD7, rWORD8 + bne cr6, L(duLcr6) + srd r0, rWORD2, rSHR + sld rWORD2_SHIFT, rWORD2, rSHL + or rWORD2, r0, rWORD8_SHIFT +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD3, 0, rSTR1 + ldbrx rWORD4, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD3, 24(rSTR1) + ld rWORD4, 24(rSTR2) +#endif + cmpld cr7, rWORD1, rWORD2 + bne cr5, L(duLcr5) + srd r12, rWORD4, rSHR + sld rWORD4_SHIFT, rWORD4, rSHL + or rWORD4, r12, rWORD2_SHIFT +#ifndef __LITTLE_ENDIAN__ + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#endif + cmpld cr1, rWORD3, rWORD4 + b L(duLoop2) + .align 4 +L(duP2x): + cmpld cr5, rWORD7, rWORD8 +#ifndef __LITTLE_ENDIAN__ + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#endif + bne cr6, L(duLcr6) + sldi. rN, rN, 3 + bne cr5, L(duLcr5) + cmpld cr7, rN, rSHR + beq L(duZeroReturn) + li r0, 0 + ble cr7, L(dutrim) +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD2, 0, rSTR2 + addi rSTR2, rSTR2, 8 +#else + ld rWORD2, 8(rSTR2) +#endif + srd r0, rWORD2, rSHR + b L(dutrim) + +/* Remainder is 24 */ + .align 4 +L(duP3): + srd r12, rWORD8, rSHR +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD3, 0, rSTR1 + addi rSTR1, rSTR1, 8 +#else + ld rWORD3, 0(rSTR1) +#endif + sld rWORD4_SHIFT, rWORD8, rSHL + or rWORD4, r12, rWORD6_SHIFT +L(duP3e): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD5, 0, rSTR1 + ldbrx rWORD6, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD5, 8(rSTR1) + ld rWORD6, 8(rSTR2) +#endif + cmpld cr1, rWORD3, rWORD4 + srd r0, rWORD6, rSHR + sld rWORD6_SHIFT, rWORD6, rSHL + or rWORD6, r0, rWORD4_SHIFT +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD7, 0, rSTR1 + ldbrx rWORD8, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD7, 16(rSTR1) + ld rWORD8, 16(rSTR2) +#endif + cmpld cr6, rWORD5, rWORD6 + bne cr1, L(duLcr1) + srd r12, rWORD8, rSHR + sld rWORD8_SHIFT, rWORD8, rSHL + or rWORD8, r12, rWORD6_SHIFT + blt cr7, L(duP3x) +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD1, 0, rSTR1 + ldbrx rWORD2, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD1, 24(rSTR1) + ld rWORD2, 24(rSTR2) +#endif + cmpld cr5, rWORD7, rWORD8 + bne cr6, L(duLcr6) + srd r0, rWORD2, rSHR + sld rWORD2_SHIFT, rWORD2, rSHL + or rWORD2, r0, rWORD8_SHIFT +#ifndef __LITTLE_ENDIAN__ + addi rSTR1, rSTR1, 16 + addi rSTR2, rSTR2, 16 +#endif + cmpld cr7, rWORD1, rWORD2 + b L(duLoop1) + .align 4 +L(duP3x): +#ifndef __LITTLE_ENDIAN__ + addi rSTR1, rSTR1, 16 + addi rSTR2, rSTR2, 16 +#endif +#if 0 +/* Huh? We've already branched on cr1! */ + bne cr1, L(duLcr1) +#endif + cmpld cr5, rWORD7, rWORD8 + bne cr6, L(duLcr6) + sldi. rN, rN, 3 + bne cr5, L(duLcr5) + cmpld cr7, rN, rSHR + beq L(duZeroReturn) + li r0, 0 + ble cr7, L(dutrim) +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD2, 0, rSTR2 + addi rSTR2, rSTR2, 8 +#else + ld rWORD2, 8(rSTR2) +#endif + srd r0, rWORD2, rSHR + b L(dutrim) + +/* Count is a multiple of 32, remainder is 0 */ + .align 4 +L(duP4): + mtctr r0 /* Power4 wants mtctr 1st in dispatch group */ + srd r0, rWORD8, rSHR +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD1, 0, rSTR1 + addi rSTR1, rSTR1, 8 +#else + ld rWORD1, 0(rSTR1) +#endif + sld rWORD2_SHIFT, rWORD8, rSHL + or rWORD2, r0, rWORD6_SHIFT +L(duP4e): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD3, 0, rSTR1 + ldbrx rWORD4, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD3, 8(rSTR1) + ld rWORD4, 8(rSTR2) +#endif + cmpld cr7, rWORD1, rWORD2 + srd r12, rWORD4, rSHR + sld rWORD4_SHIFT, rWORD4, rSHL + or rWORD4, r12, rWORD2_SHIFT +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD5, 0, rSTR1 + ldbrx rWORD6, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD5, 16(rSTR1) + ld rWORD6, 16(rSTR2) +#endif + cmpld cr1, rWORD3, rWORD4 + bne cr7, L(duLcr7) + srd r0, rWORD6, rSHR + sld rWORD6_SHIFT, rWORD6, rSHL + or rWORD6, r0, rWORD4_SHIFT +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD7, 0, rSTR1 + ldbrx rWORD8, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ldu rWORD7, 24(rSTR1) + ldu rWORD8, 24(rSTR2) +#endif + cmpld cr6, rWORD5, rWORD6 + bne cr1, L(duLcr1) + srd r12, rWORD8, rSHR + sld rWORD8_SHIFT, rWORD8, rSHL + or rWORD8, r12, rWORD6_SHIFT + cmpld cr5, rWORD7, rWORD8 + bdz- L(du24) /* Adjust CTR as we start with +4 */ +/* This is the primary loop */ + .align 4 +L(duLoop): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD1, 0, rSTR1 + ldbrx rWORD2, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD1, 8(rSTR1) + ld rWORD2, 8(rSTR2) +#endif + cmpld cr1, rWORD3, rWORD4 + bne cr6, L(duLcr6) + srd r0, rWORD2, rSHR + sld rWORD2_SHIFT, rWORD2, rSHL + or rWORD2, r0, rWORD8_SHIFT +L(duLoop1): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD3, 0, rSTR1 + ldbrx rWORD4, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD3, 16(rSTR1) + ld rWORD4, 16(rSTR2) +#endif + cmpld cr6, rWORD5, rWORD6 + bne cr5, L(duLcr5) + srd r12, rWORD4, rSHR + sld rWORD4_SHIFT, rWORD4, rSHL + or rWORD4, r12, rWORD2_SHIFT +L(duLoop2): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD5, 0, rSTR1 + ldbrx rWORD6, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ld rWORD5, 24(rSTR1) + ld rWORD6, 24(rSTR2) +#endif + cmpld cr5, rWORD7, rWORD8 + bne cr7, L(duLcr7) + srd r0, rWORD6, rSHR + sld rWORD6_SHIFT, rWORD6, rSHL + or rWORD6, r0, rWORD4_SHIFT +L(duLoop3): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD7, 0, rSTR1 + ldbrx rWORD8, 0, rSTR2 + addi rSTR1, rSTR1, 8 + addi rSTR2, rSTR2, 8 +#else + ldu rWORD7, 32(rSTR1) + ldu rWORD8, 32(rSTR2) +#endif + cmpld cr7, rWORD1, rWORD2 + bne- cr1, L(duLcr1) + srd r12, rWORD8, rSHR + sld rWORD8_SHIFT, rWORD8, rSHL + or rWORD8, r12, rWORD6_SHIFT + bdnz+ L(duLoop) + +L(duL4): +#if 0 +/* Huh? We've already branched on cr1! */ + bne cr1, L(duLcr1) +#endif + cmpld cr1, rWORD3, rWORD4 + bne cr6, L(duLcr6) + cmpld cr6, rWORD5, rWORD6 + bne cr5, L(duLcr5) + cmpld cr5, rWORD7, rWORD8 +L(du44): + bne cr7, L(duLcr7) +L(du34): + bne cr1, L(duLcr1) +L(du24): + bne cr6, L(duLcr6) +L(du14): + sldi. rN, rN, 3 + bne cr5, L(duLcr5) +/* At this point we have a remainder of 1 to 7 bytes to compare. We use + shift right double to eliminate bits beyond the compare length. + + However it may not be safe to load rWORD2 which may be beyond the + string length. So we compare the bit length of the remainder to + the right shift count (rSHR). If the bit count is less than or equal + we do not need to load rWORD2 (all significant bits are already in + rWORD8_SHIFT). */ + cmpld cr7, rN, rSHR + beq L(duZeroReturn) + li r0, 0 + ble cr7, L(dutrim) +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD2, 0, rSTR2 + addi rSTR2, rSTR2, 8 +#else + ld rWORD2, 8(rSTR2) +#endif + srd r0, rWORD2, rSHR + .align 4 +L(dutrim): +#ifdef __LITTLE_ENDIAN__ + ldbrx rWORD1, 0, rSTR1 +#else + ld rWORD1, 8(rSTR1) +#endif + ld rWORD8, -8(r1) + subfic rN, rN, 64 /* Shift count is 64 - (rN * 8). */ + or rWORD2, r0, rWORD8_SHIFT + ld rWORD7, -16(r1) + ld rSHL, -24(r1) + srd rWORD1, rWORD1, rN + srd rWORD2, rWORD2, rN + ld rSHR, -32(r1) + ld rWORD8_SHIFT, -40(r1) + li rRTN, 0 + cmpld cr7, rWORD1, rWORD2 + ld rWORD2_SHIFT, -48(r1) + ld rWORD4_SHIFT, -56(r1) + beq cr7, L(dureturn24) + li rRTN, 1 + ld rWORD6_SHIFT, -64(r1) + bgtlr cr7 + li rRTN, -1 + blr + .align 4 +L(duLcr7): + ld rWORD8, -8(r1) + ld rWORD7, -16(r1) + li rRTN, 1 + bgt cr7, L(dureturn29) + ld rSHL, -24(r1) + ld rSHR, -32(r1) + li rRTN, -1 + b L(dureturn27) + .align 4 +L(duLcr1): + ld rWORD8, -8(r1) + ld rWORD7, -16(r1) + li rRTN, 1 + bgt cr1, L(dureturn29) + ld rSHL, -24(r1) + ld rSHR, -32(r1) + li rRTN, -1 + b L(dureturn27) + .align 4 +L(duLcr6): + ld rWORD8, -8(r1) + ld rWORD7, -16(r1) + li rRTN, 1 + bgt cr6, L(dureturn29) + ld rSHL, -24(r1) + ld rSHR, -32(r1) + li rRTN, -1 + b L(dureturn27) + .align 4 +L(duLcr5): + ld rWORD8, -8(r1) + ld rWORD7, -16(r1) + li rRTN, 1 + bgt cr5, L(dureturn29) + ld rSHL, -24(r1) + ld rSHR, -32(r1) + li rRTN, -1 + b L(dureturn27) + .align 3 +L(duZeroReturn): + li rRTN, 0 + .align 4 +L(dureturn): + ld rWORD8, -8(r1) + ld rWORD7, -16(r1) +L(dureturn29): + ld rSHL, -24(r1) + ld rSHR, -32(r1) +L(dureturn27): + ld rWORD8_SHIFT, -40(r1) +L(dureturn26): + ld rWORD2_SHIFT, -48(r1) +L(dureturn25): + ld rWORD4_SHIFT, -56(r1) +L(dureturn24): + ld rWORD6_SHIFT, -64(r1) + blr +L(duzeroLength): + li rRTN, 0 + blr + +END (MEMCMP) +libc_hidden_builtin_def (memcmp) +weak_alias (memcmp, bcmp) diff --git a/REORG.TODO/sysdeps/powerpc/powerpc64/power4/memcopy.h b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/memcopy.h new file mode 100644 index 0000000000..9a4ff79f4a --- /dev/null +++ b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/memcopy.h @@ -0,0 +1 @@ +#include "../../powerpc32/power4/memcopy.h" diff --git a/REORG.TODO/sysdeps/powerpc/powerpc64/power4/memcpy.S b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/memcpy.S new file mode 100644 index 0000000000..2e96376b9f --- /dev/null +++ b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/memcpy.S @@ -0,0 +1,477 @@ +/* Optimized memcpy implementation for PowerPC64. + Copyright (C) 2003-2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> + +/* __ptr_t [r3] memcpy (__ptr_t dst [r3], __ptr_t src [r4], size_t len [r5]); + Returns 'dst'. + + Memcpy handles short copies (< 32-bytes) using a binary move blocks + (no loops) of lwz/stw. The tail (remaining 1-3) bytes is handled + with the appropriate combination of byte and halfword load/stores. + There is minimal effort to optimize the alignment of short moves. + The 64-bit implementations of POWER3 and POWER4 do a reasonable job + of handling unaligned load/stores that do not cross 32-byte boundaries. + + Longer moves (>= 32-bytes) justify the effort to get at least the + destination doubleword (8-byte) aligned. Further optimization is + possible when both source and destination are doubleword aligned. + Each case has a optimized unrolled loop. */ + +#ifndef MEMCPY +# define MEMCPY memcpy +#endif + .machine power4 +EALIGN (MEMCPY, 5, 0) + CALL_MCOUNT 3 + + cmpldi cr1,5,31 + neg 0,3 + std 3,-16(1) + std 31,-8(1) + cfi_offset(31,-8) + andi. 11,3,7 /* check alignment of dst. */ + clrldi 0,0,61 /* Number of bytes until the 1st doubleword of dst. */ + clrldi 10,4,61 /* check alignment of src. */ + cmpldi cr6,5,8 + ble- cr1,.L2 /* If move < 32 bytes use short move code. */ + cmpld cr6,10,11 + mr 12,4 + srdi 9,5,3 /* Number of full double words remaining. */ + mtcrf 0x01,0 + mr 31,5 + beq .L0 + + subf 31,0,5 + /* Move 0-7 bytes as needed to get the destination doubleword aligned. */ +1: bf 31,2f + lbz 6,0(12) + addi 12,12,1 + stb 6,0(3) + addi 3,3,1 +2: bf 30,4f + lhz 6,0(12) + addi 12,12,2 + sth 6,0(3) + addi 3,3,2 +4: bf 29,0f + lwz 6,0(12) + addi 12,12,4 + stw 6,0(3) + addi 3,3,4 +0: + clrldi 10,12,61 /* check alignment of src again. */ + srdi 9,31,3 /* Number of full double words remaining. */ + + /* Copy doublewords from source to destination, assuming the + destination is aligned on a doubleword boundary. + + At this point we know there are at least 25 bytes left (32-7) to copy. + The next step is to determine if the source is also doubleword aligned. + If not branch to the unaligned move code at .L6. which uses + a load, shift, store strategy. + + Otherwise source and destination are doubleword aligned, and we can + the optimized doubleword copy loop. */ +.L0: + clrldi 11,31,61 + mtcrf 0x01,9 + cmpldi cr1,11,0 + bne- cr6,.L6 /* If source is not DW aligned. */ + + /* Move doublewords where destination and source are DW aligned. + Use a unrolled loop to copy 4 doubleword (32-bytes) per iteration. + If the copy is not an exact multiple of 32 bytes, 1-3 + doublewords are copied as needed to set up the main loop. After + the main loop exits there may be a tail of 1-7 bytes. These byte are + copied a word/halfword/byte at a time as needed to preserve alignment. */ + + srdi 8,31,5 + cmpldi cr1,9,4 + cmpldi cr6,11,0 + mr 11,12 + + bf 30,1f + ld 6,0(12) + ld 7,8(12) + addi 11,12,16 + mtctr 8 + std 6,0(3) + std 7,8(3) + addi 10,3,16 + bf 31,4f + ld 0,16(12) + std 0,16(3) + blt cr1,3f + addi 11,12,24 + addi 10,3,24 + b 4f + .align 4 +1: + mr 10,3 + mtctr 8 + bf 31,4f + ld 6,0(12) + addi 11,12,8 + std 6,0(3) + addi 10,3,8 + + .align 4 +4: + ld 6,0(11) + ld 7,8(11) + ld 8,16(11) + ld 0,24(11) + addi 11,11,32 +2: + std 6,0(10) + std 7,8(10) + std 8,16(10) + std 0,24(10) + addi 10,10,32 + bdnz 4b +3: + + rldicr 0,31,0,60 + mtcrf 0x01,31 + beq cr6,0f +.L9: + add 3,3,0 + add 12,12,0 + +/* At this point we have a tail of 0-7 bytes and we know that the + destination is double word aligned. */ +4: bf 29,2f + lwz 6,0(12) + addi 12,12,4 + stw 6,0(3) + addi 3,3,4 +2: bf 30,1f + lhz 6,0(12) + addi 12,12,2 + sth 6,0(3) + addi 3,3,2 +1: bf 31,0f + lbz 6,0(12) + stb 6,0(3) +0: + /* Return original dst pointer. */ + ld 31,-8(1) + ld 3,-16(1) + blr + +/* Copy up to 31 bytes. This divided into two cases 0-8 bytes and 9-31 + bytes. Each case is handled without loops, using binary (1,2,4,8) + tests. + + In the short (0-8 byte) case no attempt is made to force alignment + of either source or destination. The hardware will handle the + unaligned load/stores with small delays for crossing 32- 64-byte, and + 4096-byte boundaries. Since these short moves are unlikely to be + unaligned or cross these boundaries, the overhead to force + alignment is not justified. + + The longer (9-31 byte) move is more likely to cross 32- or 64-byte + boundaries. Since only loads are sensitive to the 32-/64-byte + boundaries it is more important to align the source then the + destination. If the source is not already word aligned, we first + move 1-3 bytes as needed. Since we are only word aligned we don't + use double word load/stores to insure that all loads are aligned. + While the destination and stores may still be unaligned, this + is only an issue for page (4096 byte boundary) crossing, which + should be rare for these short moves. The hardware handles this + case automatically with a small delay. */ + + .align 4 +.L2: + mtcrf 0x01,5 + neg 8,4 + clrrdi 11,4,2 + andi. 0,8,3 + ble cr6,.LE8 /* Handle moves of 0-8 bytes. */ +/* At least 9 bytes left. Get the source word aligned. */ + cmpldi cr1,5,16 + mr 10,5 + mr 12,4 + cmpldi cr6,0,2 + beq .L3 /* If the source is already word aligned skip this. */ +/* Copy 1-3 bytes to get source address word aligned. */ + lwz 6,0(11) + subf 10,0,5 + add 12,4,0 + blt cr6,5f + srdi 7,6,16 + bgt cr6,3f +#ifdef __LITTLE_ENDIAN__ + sth 7,0(3) +#else + sth 6,0(3) +#endif + b 7f + .align 4 +3: +#ifdef __LITTLE_ENDIAN__ + rotlwi 6,6,24 + stb 6,0(3) + sth 7,1(3) +#else + stb 7,0(3) + sth 6,1(3) +#endif + b 7f + .align 4 +5: +#ifdef __LITTLE_ENDIAN__ + rotlwi 6,6,8 +#endif + stb 6,0(3) +7: + cmpldi cr1,10,16 + add 3,3,0 + mtcrf 0x01,10 + .align 4 +.L3: +/* At least 6 bytes left and the source is word aligned. */ + blt cr1,8f +16: /* Move 16 bytes. */ + lwz 6,0(12) + lwz 7,4(12) + stw 6,0(3) + lwz 6,8(12) + stw 7,4(3) + lwz 7,12(12) + addi 12,12,16 + stw 6,8(3) + stw 7,12(3) + addi 3,3,16 +8: /* Move 8 bytes. */ + bf 28,4f + lwz 6,0(12) + lwz 7,4(12) + addi 12,12,8 + stw 6,0(3) + stw 7,4(3) + addi 3,3,8 +4: /* Move 4 bytes. */ + bf 29,2f + lwz 6,0(12) + addi 12,12,4 + stw 6,0(3) + addi 3,3,4 +2: /* Move 2-3 bytes. */ + bf 30,1f + lhz 6,0(12) + sth 6,0(3) + bf 31,0f + lbz 7,2(12) + stb 7,2(3) + ld 3,-16(1) + blr +1: /* Move 1 byte. */ + bf 31,0f + lbz 6,0(12) + stb 6,0(3) +0: + /* Return original dst pointer. */ + ld 3,-16(1) + blr + +/* Special case to copy 0-8 bytes. */ + .align 4 +.LE8: + mr 12,4 + bne cr6,4f +/* Would have liked to use use ld/std here but the 630 processors are + slow for load/store doubles that are not at least word aligned. + Unaligned Load/Store word execute with only a 1 cycle penalty. */ + lwz 6,0(4) + lwz 7,4(4) + stw 6,0(3) + stw 7,4(3) + /* Return original dst pointer. */ + ld 3,-16(1) + blr + .align 4 +4: bf 29,2b + lwz 6,0(4) + stw 6,0(3) +6: + bf 30,5f + lhz 7,4(4) + sth 7,4(3) + bf 31,0f + lbz 8,6(4) + stb 8,6(3) + ld 3,-16(1) + blr + .align 4 +5: + bf 31,0f + lbz 6,4(4) + stb 6,4(3) + .align 4 +0: + /* Return original dst pointer. */ + ld 3,-16(1) + blr + + .align 4 +.L6: + + /* Copy doublewords where the destination is aligned but the source is + not. Use aligned doubleword loads from the source, shifted to realign + the data, to allow aligned destination stores. */ + addi 11,9,-1 /* loop DW count is one less than total */ + subf 5,10,12 + sldi 10,10,3 + mr 4,3 + srdi 8,11,2 /* calculate the 32 byte loop count */ + ld 6,0(5) + mtcrf 0x01,11 + cmpldi cr6,9,4 + mtctr 8 + ld 7,8(5) + subfic 9,10,64 + bf 30,1f + + /* there are at least two DWs to copy */ +#ifdef __LITTLE_ENDIAN__ + srd 0,6,10 + sld 8,7,9 +#else + sld 0,6,10 + srd 8,7,9 +#endif + or 0,0,8 + ld 6,16(5) + std 0,0(4) +#ifdef __LITTLE_ENDIAN__ + srd 0,7,10 + sld 8,6,9 +#else + sld 0,7,10 + srd 8,6,9 +#endif + or 0,0,8 + ld 7,24(5) + std 0,8(4) + addi 4,4,16 + addi 5,5,32 + blt cr6,8f /* if total DWs = 3, then bypass loop */ + bf 31,4f + /* there is a third DW to copy */ +#ifdef __LITTLE_ENDIAN__ + srd 0,6,10 + sld 8,7,9 +#else + sld 0,6,10 + srd 8,7,9 +#endif + or 0,0,8 + std 0,0(4) + mr 6,7 + ld 7,0(5) + addi 5,5,8 + addi 4,4,8 + beq cr6,8f /* if total DWs = 4, then bypass loop */ + b 4f + .align 4 +1: +#ifdef __LITTLE_ENDIAN__ + srd 0,6,10 + sld 8,7,9 +#else + sld 0,6,10 + srd 8,7,9 +#endif + addi 5,5,16 + or 0,0,8 + bf 31,4f + mr 6,7 + ld 7,0(5) + addi 5,5,8 + std 0,0(4) + addi 4,4,8 + .align 4 +/* copy 32 bytes at a time */ +4: +#ifdef __LITTLE_ENDIAN__ + srd 0,6,10 + sld 8,7,9 +#else + sld 0,6,10 + srd 8,7,9 +#endif + or 0,0,8 + ld 6,0(5) + std 0,0(4) +#ifdef __LITTLE_ENDIAN__ + srd 0,7,10 + sld 8,6,9 +#else + sld 0,7,10 + srd 8,6,9 +#endif + or 0,0,8 + ld 7,8(5) + std 0,8(4) +#ifdef __LITTLE_ENDIAN__ + srd 0,6,10 + sld 8,7,9 +#else + sld 0,6,10 + srd 8,7,9 +#endif + or 0,0,8 + ld 6,16(5) + std 0,16(4) +#ifdef __LITTLE_ENDIAN__ + srd 0,7,10 + sld 8,6,9 +#else + sld 0,7,10 + srd 8,6,9 +#endif + or 0,0,8 + ld 7,24(5) + std 0,24(4) + addi 5,5,32 + addi 4,4,32 + bdnz+ 4b + .align 4 +8: + /* calculate and store the final DW */ +#ifdef __LITTLE_ENDIAN__ + srd 0,6,10 + sld 8,7,9 +#else + sld 0,6,10 + srd 8,7,9 +#endif + or 0,0,8 + std 0,0(4) +3: + rldicr 0,31,0,60 + mtcrf 0x01,31 + bne cr1,.L9 /* If the tail is 0 bytes we are done! */ + /* Return original dst pointer. */ + ld 31,-8(1) + ld 3,-16(1) + blr +END_GEN_TB (MEMCPY,TB_TOCLESS) +libc_hidden_builtin_def (memcpy) diff --git a/REORG.TODO/sysdeps/powerpc/powerpc64/power4/memset.S b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/memset.S new file mode 100644 index 0000000000..a57214e0b0 --- /dev/null +++ b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/memset.S @@ -0,0 +1,251 @@ +/* Optimized memset implementation for PowerPC64. + Copyright (C) 1997-2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> + +/* __ptr_t [r3] memset (__ptr_t s [r3], int c [r4], size_t n [r5])); + Returns 's'. + + The memset is done in three sizes: byte (8 bits), word (32 bits), + cache line (256 bits). There is a special case for setting cache lines + to 0, to take advantage of the dcbz instruction. */ + +#ifndef MEMSET +# define MEMSET memset +#endif + .machine power4 +EALIGN (MEMSET, 5, 0) + CALL_MCOUNT 3 + +#define rTMP r0 +#define rRTN r3 /* Initial value of 1st argument. */ +#define rMEMP0 r3 /* Original value of 1st arg. */ +#define rCHR r4 /* Char to set in each byte. */ +#define rLEN r5 /* Length of region to set. */ +#define rMEMP r6 /* Address at which we are storing. */ +#define rALIGN r7 /* Number of bytes we are setting now (when aligning). */ +#define rMEMP2 r8 + +#define rNEG64 r8 /* Constant -64 for clearing with dcbz. */ +#define rCLS r8 /* Cache line size obtained from static. */ +#define rCLM r9 /* Cache line size mask to check for cache alignment. */ +L(_memset): +/* Take care of case for size <= 4. */ + cmpldi cr1, rLEN, 8 + andi. rALIGN, rMEMP0, 7 + mr rMEMP, rMEMP0 + ble- cr1, L(small) + +/* Align to doubleword boundary. */ + cmpldi cr5, rLEN, 31 + insrdi rCHR, rCHR, 8, 48 /* Replicate byte to halfword. */ + beq+ L(aligned2) + mtcrf 0x01, rMEMP0 + subfic rALIGN, rALIGN, 8 + cror 28,30,31 /* Detect odd word aligned. */ + add rMEMP, rMEMP, rALIGN + sub rLEN, rLEN, rALIGN + insrdi rCHR, rCHR, 16, 32 /* Replicate halfword to word. */ + bt 29, L(g4) +/* Process the even word of doubleword. */ + bf+ 31, L(g2) + stb rCHR, 0(rMEMP0) + bt 30, L(g4x) +L(g2): + sth rCHR, -6(rMEMP) +L(g4x): + stw rCHR, -4(rMEMP) + b L(aligned) +/* Process the odd word of doubleword. */ +L(g4): + bf 28, L(g4x) /* If false, word aligned on odd word. */ + bf+ 31, L(g0) + stb rCHR, 0(rMEMP0) + bt 30, L(aligned) +L(g0): + sth rCHR, -2(rMEMP) + +/* Handle the case of size < 31. */ +L(aligned2): + insrdi rCHR, rCHR, 16, 32 /* Replicate halfword to word. */ +L(aligned): + mtcrf 0x01, rLEN + ble cr5, L(medium) +/* Align to 32-byte boundary. */ + andi. rALIGN, rMEMP, 0x18 + subfic rALIGN, rALIGN, 0x20 + insrdi rCHR, rCHR, 32, 0 /* Replicate word to double word. */ + beq L(caligned) + mtcrf 0x01, rALIGN + add rMEMP, rMEMP, rALIGN + sub rLEN, rLEN, rALIGN + cmplwi cr1, rALIGN, 0x10 + mr rMEMP2, rMEMP + bf 28, L(a1) + stdu rCHR, -8(rMEMP2) +L(a1): blt cr1, L(a2) + std rCHR, -8(rMEMP2) + stdu rCHR, -16(rMEMP2) +L(a2): + +/* Now aligned to a 32 byte boundary. */ +L(caligned): + cmpldi cr1, rCHR, 0 + clrrdi. rALIGN, rLEN, 5 + mtcrf 0x01, rLEN + beq cr1, L(zloopstart) /* Special case for clearing memory using dcbz. */ +L(nondcbz): + srdi rTMP, rALIGN, 5 + mtctr rTMP + beq L(medium) /* We may not actually get to do a full line. */ + clrldi. rLEN, rLEN, 59 + add rMEMP, rMEMP, rALIGN + li rNEG64, -0x40 + bdz L(cloopdone) + +L(c3): dcbtst rNEG64, rMEMP + std rCHR, -8(rMEMP) + std rCHR, -16(rMEMP) + std rCHR, -24(rMEMP) + stdu rCHR, -32(rMEMP) + bdnz L(c3) +L(cloopdone): + std rCHR, -8(rMEMP) + std rCHR, -16(rMEMP) + cmpldi cr1, rLEN, 16 + std rCHR, -24(rMEMP) + stdu rCHR, -32(rMEMP) + beqlr + add rMEMP, rMEMP, rALIGN + b L(medium_tail2) + + .align 5 +/* Clear lines of memory in 128-byte chunks. */ +L(zloopstart): +/* If the remaining length is less the 32 bytes, don't bother getting + the cache line size. */ + beq L(medium) + li rCLS,128 /* cache line size is 128 */ + +/* Now we know the cache line size, and it is not 32-bytes, but + we may not yet be aligned to the cache line. May have a partial + line to fill, so touch it 1st. */ + dcbt 0,rMEMP +L(getCacheAligned): + cmpldi cr1,rLEN,32 + andi. rTMP,rMEMP,127 + blt cr1,L(handletail32) + beq L(cacheAligned) + addi rMEMP,rMEMP,32 + addi rLEN,rLEN,-32 + std rCHR,-32(rMEMP) + std rCHR,-24(rMEMP) + std rCHR,-16(rMEMP) + std rCHR,-8(rMEMP) + b L(getCacheAligned) + +/* Now we are aligned to the cache line and can use dcbz. */ +L(cacheAligned): + cmpld cr1,rLEN,rCLS + blt cr1,L(handletail32) + dcbz 0,rMEMP + subf rLEN,rCLS,rLEN + add rMEMP,rMEMP,rCLS + b L(cacheAligned) + +/* We are here because the cache line size was set and was not 32-bytes + and the remainder (rLEN) is less than the actual cache line size. + So set up the preconditions for L(nondcbz) and go there. */ +L(handletail32): + clrrwi. rALIGN, rLEN, 5 + b L(nondcbz) + + .align 5 +L(small): +/* Memset of 8 bytes or less. */ + cmpldi cr6, rLEN, 4 + cmpldi cr5, rLEN, 1 + ble cr6,L(le4) + subi rLEN, rLEN, 4 + stb rCHR,0(rMEMP) + stb rCHR,1(rMEMP) + stb rCHR,2(rMEMP) + stb rCHR,3(rMEMP) + addi rMEMP,rMEMP, 4 + cmpldi cr5, rLEN, 1 +L(le4): + cmpldi cr1, rLEN, 3 + bltlr cr5 + stb rCHR, 0(rMEMP) + beqlr cr5 + stb rCHR, 1(rMEMP) + bltlr cr1 + stb rCHR, 2(rMEMP) + beqlr cr1 + stb rCHR, 3(rMEMP) + blr + +/* Memset of 0-31 bytes. */ + .align 5 +L(medium): + insrdi rCHR, rCHR, 32, 0 /* Replicate word to double word. */ + cmpldi cr1, rLEN, 16 +L(medium_tail2): + add rMEMP, rMEMP, rLEN +L(medium_tail): + bt- 31, L(medium_31t) + bt- 30, L(medium_30t) +L(medium_30f): + bt- 29, L(medium_29t) +L(medium_29f): + bge- cr1, L(medium_27t) + bflr- 28 + std rCHR, -8(rMEMP) + blr + +L(medium_31t): + stbu rCHR, -1(rMEMP) + bf- 30, L(medium_30f) +L(medium_30t): + sthu rCHR, -2(rMEMP) + bf- 29, L(medium_29f) +L(medium_29t): + stwu rCHR, -4(rMEMP) + blt- cr1, L(medium_27f) +L(medium_27t): + std rCHR, -8(rMEMP) + stdu rCHR, -16(rMEMP) +L(medium_27f): + bflr- 28 +L(medium_28t): + std rCHR, -8(rMEMP) + blr +END_GEN_TB (MEMSET,TB_TOCLESS) +libc_hidden_builtin_def (memset) + +/* Copied from bzero.S to prevent the linker from inserting a stub + between bzero and memset. */ +ENTRY (__bzero) + CALL_MCOUNT 3 + mr r5,r4 + li r4,0 + b L(_memset) +END (__bzero) +#ifndef __bzero +weak_alias (__bzero, bzero) +#endif diff --git a/REORG.TODO/sysdeps/powerpc/powerpc64/power4/multiarch/Implies b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/multiarch/Implies new file mode 100644 index 0000000000..30edcf7f9d --- /dev/null +++ b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/multiarch/Implies @@ -0,0 +1 @@ +powerpc/powerpc64/multiarch diff --git a/REORG.TODO/sysdeps/powerpc/powerpc64/power4/strncmp.S b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/strncmp.S new file mode 100644 index 0000000000..2b0c00dfb2 --- /dev/null +++ b/REORG.TODO/sysdeps/powerpc/powerpc64/power4/strncmp.S @@ -0,0 +1,225 @@ +/* Optimized strcmp implementation for PowerPC64. + Copyright (C) 2003-2017 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <http://www.gnu.org/licenses/>. */ + +#include <sysdep.h> + +#ifndef STRNCMP +# define STRNCMP strncmp +#endif + +/* See strlen.s for comments on how the end-of-string testing works. */ + +/* int [r3] strncmp (const char *s1 [r3], const char *s2 [r4], size_t size [r5]) */ + +EALIGN (STRNCMP, 4, 0) + CALL_MCOUNT 3 + +#define rTMP2 r0 +#define rRTN r3 +#define rSTR1 r3 /* first string arg */ +#define rSTR2 r4 /* second string arg */ +#define rN r5 /* max string length */ +#define rWORD1 r6 /* current word in s1 */ +#define rWORD2 r7 /* current word in s2 */ +#define rWORD3 r10 +#define rWORD4 r11 +#define rFEFE r8 /* constant 0xfefefefefefefeff (-0x0101010101010101) */ +#define r7F7F r9 /* constant 0x7f7f7f7f7f7f7f7f */ +#define rNEG r10 /* ~(word in s1 | 0x7f7f7f7f7f7f7f7f) */ +#define rBITDIF r11 /* bits that differ in s1 & s2 words */ +#define rTMP r12 + + dcbt 0,rSTR1 + or rTMP, rSTR2, rSTR1 + lis r7F7F, 0x7f7f + dcbt 0,rSTR2 + clrldi. rTMP, rTMP, 61 + cmpldi cr1, rN, 0 + lis rFEFE, -0x101 + bne L(unaligned) +/* We are doubleword aligned so set up for two loops. first a double word + loop, then fall into the byte loop if any residual. */ + srdi. rTMP, rN, 3 + clrldi rN, rN, 61 + addi rFEFE, rFEFE, -0x101 + addi r7F7F, r7F7F, 0x7f7f + cmpldi cr1, rN, 0 + beq L(unaligned) + + mtctr rTMP /* Power4 wants mtctr 1st in dispatch group. */ + ld rWORD1, 0(rSTR1) + ld rWORD2, 0(rSTR2) + sldi rTMP, rFEFE, 32 + insrdi r7F7F, r7F7F, 32, 0 + add rFEFE, rFEFE, rTMP + b L(g1) + +L(g0): + ldu rWORD1, 8(rSTR1) + bne- cr1, L(different) + ldu rWORD2, 8(rSTR2) +L(g1): add rTMP, rFEFE, rWORD1 + nor rNEG, r7F7F, rWORD1 + bdz L(tail) + and. rTMP, rTMP, rNEG + cmpd cr1, rWORD1, rWORD2 + beq+ L(g0) + +/* OK. We've hit the end of the string. We need to be careful that + we don't compare two strings as different because of gunk beyond + the end of the strings... */ + +#ifdef __LITTLE_ENDIAN__ +L(endstring): + addi rTMP2, rTMP, -1 + beq cr1, L(equal) + andc rTMP2, rTMP2, rTMP + rldimi rTMP2, rTMP2, 1, 0 + and rWORD2, rWORD2, rTMP2 /* Mask off gunk. */ + and rWORD1, rWORD1, rTMP2 + cmpd cr1, rWORD1, rWORD2 + beq cr1, L(equal) + xor rBITDIF, rWORD1, rWORD2 /* rBITDIF has bits that differ. */ + neg rNEG, rBITDIF + and rNEG, rNEG, rBITDIF /* rNEG has LS bit that differs. */ + cntlzd rNEG, rNEG /* bitcount of the bit. */ + andi. rNEG, rNEG, 56 /* bitcount to LS byte that differs. */ + sld rWORD1, rWORD1, rNEG /* shift left to clear MS bytes. */ + sld rWORD2, rWORD2, rNEG + xor. rBITDIF, rWORD1, rWORD2 + sub rRTN, rWORD1, rWORD2 + blt- L(highbit) + sradi rRTN, rRTN, 63 /* must return an int. */ + ori rRTN, rRTN, 1 + blr +L(equal): + li rRTN, 0 + blr + +L(different): + ld rWORD1, -8(rSTR1) + xor rBITDIF, rWORD1, rWORD2 /* rBITDIF has bits that differ. */ + neg rNEG, rBITDIF + and rNEG, rNEG, rBITDIF /* rNEG has LS bit that differs. */ + cntlzd rNEG, rNEG /* bitcount of the bit. */ + andi. rNEG, rNEG, 56 /* bitcount to LS byte that differs. */ + sld rWORD1, rWORD1, rNEG /* shift left to clear MS bytes. */ + sld rWORD2, rWORD2, rNEG + xor. rBITDIF, rWORD1, rWORD2 + sub rRTN, rWORD1, rWORD2 + blt- L(highbit) + sradi rRTN, rRTN, 63 + ori rRTN, rRTN, 1 + blr +L(highbit): + sradi rRTN, rWORD2, 63 + ori rRTN, rRTN, 1 + blr + +#else +L(endstring): + and rTMP, r7F7F, rWORD1 + beq cr1, L(equal) + add rTMP, rTMP, r7F7F + xor. rBITDIF, rWORD1, rWORD2 + andc rNEG, rNEG, rTMP + blt- L(highbit) + cntlzd rBITDIF, rBITDIF + cntlzd rNEG, rNEG + addi rNEG, rNEG, 7 + cmpd cr1, rNEG, rBITDIF + sub rRTN, rWORD1, rWORD2 + blt- cr1, L(equal) + sradi rRTN, rRTN, 63 /* must return an int. */ + ori rRTN, rRTN, 1 + blr +L(equal): + li rRTN, 0 + blr + +L(different): + ld rWORD1, -8(rSTR1) + xor. rBITDIF, rWORD1, rWORD2 + sub rRTN, rWORD1, rWORD2 + blt- L(highbit) + sradi rRTN, rRTN, 63 + ori rRTN, rRTN, 1 + blr +L(highbit): + sradi rRTN, rWORD2, 63 + ori rRTN, rRTN, 1 + blr +#endif + +/* Oh well. In this case, we just do a byte-by-byte comparison. */ + .align 4 +L(tail): + and. rTMP, rTMP, rNEG + cmpd cr1, rWORD1, rWORD2 + bne- L(endstring) + addi rSTR1, rSTR1, 8 + bne- cr1, L(different) + addi rSTR2, rSTR2, 8 + cmpldi cr1, rN, 0 +L(unaligned): + mtctr rN /* Power4 wants mtctr 1st in dispatch group */ + ble cr1, L(ux) +L(uz): + lbz rWORD1, 0(rSTR1) + lbz rWORD2, 0(rSTR2) + .align 4 +L(u1): + cmpdi cr1, rWORD1, 0 + bdz L(u4) + cmpd rWORD1, rWORD2 + beq- cr1, L(u4) + bne- L(u4) + lbzu rWORD3, 1(rSTR1) + lbzu rWORD4, 1(rSTR2) + cmpdi cr1, rWORD3, 0 + bdz L(u3) + cmpd rWORD3, rWORD4 + beq- cr1, L(u3) + bne- L(u3) + lbzu rWORD1, 1(rSTR1) + lbzu rWORD2, 1(rSTR2) + cmpdi cr1, rWORD1, 0 + bdz L(u4) + cmpd rWORD1, rWORD2 + beq- cr1, L(u4) + bne- L(u4) + lbzu rWORD3, 1(rSTR1) + lbzu rWORD4, 1(rSTR2) + cmpdi cr1, rWORD3, 0 + bdz L(u3) + cmpd rWORD3, rWORD4 + beq- cr1, L(u3) + bne- L(u3) + lbzu rWORD1, 1(rSTR1) + lbzu rWORD2, 1(rSTR2) + b L(u1) + +L(u3): sub rRTN, rWORD3, rWORD4 + blr +L(u4): sub rRTN, rWORD1, rWORD2 + blr +L(ux): + li rRTN, 0 + blr +END (STRNCMP) +libc_hidden_builtin_def (strncmp) |