diff options
Diffstat (limited to 'src/arch/loong64/core/loong64_string.c')
-rw-r--r-- | src/arch/loong64/core/loong64_string.c | 266 |
1 files changed, 266 insertions, 0 deletions
diff --git a/src/arch/loong64/core/loong64_string.c b/src/arch/loong64/core/loong64_string.c new file mode 100644 index 00000000..941b7e2b --- /dev/null +++ b/src/arch/loong64/core/loong64_string.c @@ -0,0 +1,266 @@ +/* + * Copyright (C) 2016 Michael Brown <mbrown@fensystems.co.uk>. + * Copyright (c) 2023, Xiaotian Wu <wuxiaotian@loongson.cn> + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + +/** @file + * + * Optimised string operations + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <string.h> + +/** + * Copy memory area + * + * @v dest Destination address + * @v src Source address + * @v len Length + * @ret dest Destination address + */ +void loong64_memcpy ( void *dest, const void *src, size_t len ) { + void *discard_dest; + void *discard_end; + const void *discard_src; + size_t discard_offset; + unsigned long discard_data; + unsigned long discard_low; + unsigned long discard_high; + + /* If length is too short, then just copy individual bytes. + */ + if ( len < 16 ) { + __asm__ __volatile__ ( "beqz %0, 2f\n\t" + "\n1:\n\t" + "addi.d %0, %0, -1\n\t" + "ldx.b %1, %3, %0\n\t" + "stx.b %1, %2, %0\n\t" + "bnez %0, 1b\n\t" + "\n2:\n\t" + : "=&r" ( discard_offset ), + "=&r" ( discard_data ) + : "r" ( dest ), "r" ( src ), "0" ( len ) + : "memory", "t0" ); + return; + } + + /* Copy 16 bytes at a time: one initial + * potentially unaligned access, multiple destination-aligned + * accesses, one final potentially unaligned access. + */ + __asm__ __volatile__ ( "ld.d %3, %1, 0\n\t" + "ld.d %4, %1, 8\n\t" + "addi.d %1, %1, 16\n\t" + "st.d %3, %0, 0\n\t" + "st.d %4, %0, 8\n\t" + "addi.d %0, %0, 16\n\t" + "andi %3, %0, 15\n\t" + "sub.d %0, %0, %3\n\t" + "sub.d %1, %1, %3\n\t" + "addi.d $t0, $zero, 0xf\n\t" + "andn %2, %5, $t0\n\t" + "b 2f\n\t" + "\n1:\n\t" + "ld.d %3, %1, 0\n\t" + "ld.d %4, %1, 8\n\t" + "addi.d %1, %1, 16\n\t" + "st.d %3, %0, 0\n\t" + "st.d %4, %0, 8\n\t" + "addi.d %0, %0, 16\n\t" + "\n2:\n\t" + "bne %0, %2, 1b\n\t" + "ld.d %3, %6, -16\n\t" + "ld.d %4, %6, -8\n\t" + "st.d %3, %5, -16\n\t" + "st.d %4, %5, -8\n\t" + : "=&r" ( discard_dest ), + "=&r" ( discard_src ), + "=&r" ( discard_end ), + "=&r" ( discard_low ), + "=&r" ( discard_high ) + : "r" ( dest + len ), "r" ( src + len ), + "0" ( dest ), "1" ( src ) + : "memory", "t0" ); +} + +/** + * Zero memory region + * + * @v dest Destination region + * @v len Length + */ +void loong64_bzero ( void *dest, size_t len ) { + size_t discard_offset; + void *discard_dest; + void *discard_end; + + /* If length is too short, then just zero individual bytes. + */ + if ( len < 16 ) { + __asm__ __volatile__ ( "beqz %0, 2f\n\t" + "\n1:\n\t" + "addi.d %0, %0, -1\n\t" + "stx.b $zero, %1, %0\n\t" + "bnez %0, 1b\n\t" + "\n2:\n\t" + : "=&r" ( discard_offset ) + : "r" ( dest ), "0" ( len ) + : "memory" ); + return; + } + + /* To zero 16 bytes at a time: one initial + * potentially unaligned access, multiple aligned accesses, + * one final potentially unaligned access. + */ + + __asm__ __volatile__ ( "st.d $zero, %0, 0\n\t" + "st.d $zero, %0, 8\n\t" + "addi.d %0, %0, 16\n\t" + "addi.w $t0, $zero, 15\n\t" + "andn %0, %0, $t0\n\t" + "addi.w $t0, $zero, 15\n\t" + "andn %1, %2, $t0\n\t" + "b 2f\n\t" + "\n1:\n\t" + "st.d $zero, %0, 0\n\t" + "st.d $zero, %0, 8\n\t" + "addi.d %0, %0, 16\n\t" + "\n2:\n\t" + "bne %0, %1, 1b\n\t" + "st.d $zero, %2, -16\n\t" + "st.d $zero, %2, -8\n\t" + : "=&r" ( discard_dest ), + "=&r" ( discard_end ) + : "r" ( dest + len ), "0" ( dest ) + : "memory", "t0" ); +} + +/** + * Fill memory region + * + * @v dest Destination region + * @v len Length + * @v character Fill character + * + * The unusual parameter order is to allow for more efficient + * tail-calling to loong64_memset() when zeroing a region. + */ +void loong64_memset ( void *dest, size_t len, int character ) { + size_t discard_offset; + + /* Use optimised zeroing code if applicable */ + if ( character == 0 ) { + loong64_bzero ( dest, len ); + return; + } + + /* Fill one byte at a time. Calling memset() with a non-zero + * value is relatively rare and unlikely to be + * performance-critical. + */ + __asm__ __volatile__ ( "beqz %0, 2f\n\t" + "\n1:\n\t" + "addi.d %0, %0, -1\n\t" + "stx.b %2, %1, %0\n\t" + "bnez %0, 1b\n\t" + "\n2:\n\t" + : "=&r" ( discard_offset ) + : "r" ( dest ), "r" ( character ), "0" ( len ) + : "memory" ); +} + +/** + * Copy (possibly overlapping) memory region forwards + * + * @v dest Destination region + * @v src Source region + * @v len Length + */ +void loong64_memmove_forwards ( void *dest, const void *src, size_t len ) { + void *discard_dest; + const void *discard_src; + unsigned long discard_data; + + /* Assume memmove() is not performance-critical, and perform a + * bytewise copy for simplicity. + */ + __asm__ __volatile__ ( "b 2f\n\t" + "\n1:\n\t" + "ld.b %2, %1, 0\n\t" + "addi.d %1, %1, 1\n\t" + "st.b %2, %0, 0\n\t" + "addi.d %0, %0, 1\n\t" + "\n2:\n\t" + "bne %0, %3, 1b\n\t" + : "=&r" ( discard_dest ), + "=&r" ( discard_src ), + "=&r" ( discard_data ) + : "r" ( dest + len ), "0" ( dest ), "1" ( src ) + : "memory" ); +} + +/** + * Copy (possibly overlapping) memory region backwards + * + * @v dest Destination region + * @v src Source region + * @v len Length + */ +void loong64_memmove_backwards ( void *dest, const void *src, size_t len ) { + size_t discard_offset; + unsigned long discard_data; + + /* Assume memmove() is not performance-critical, and perform a + * bytewise copy for simplicity. + */ + __asm__ __volatile__ ( "beqz %0, 2f\n\t" + "\n1:\n\t" + "addi.d %0, %0, -1\n\t" + "ldx.b %1, %3, %0\n\t" + "stx.b %1, %2, %0\n\t" + "bnez %0, 1b\n\t" + "\n2:\n\t" + : "=&r" ( discard_offset ), + "=&r" ( discard_data ) + : "r" ( dest ), "r" ( src ), "0" ( len ) + : "memory" ); +} + +/** + * Copy (possibly overlapping) memory region + * + * @v dest Destination region + * @v src Source region + * @v len Length + */ +void loong64_memmove ( void *dest, const void *src, size_t len ) { + + if ( dest <= src ) { + loong64_memmove_forwards ( dest, src, len ); + } else { + loong64_memmove_backwards ( dest, src, len ); + } +} |