diff options
Diffstat (limited to 'src/c/gf-complete/src/neon/gf_w16_neon.c')
-rw-r--r-- | src/c/gf-complete/src/neon/gf_w16_neon.c | 356 |
1 files changed, 356 insertions, 0 deletions
diff --git a/src/c/gf-complete/src/neon/gf_w16_neon.c b/src/c/gf-complete/src/neon/gf_w16_neon.c new file mode 100644 index 0000000..95bfd80 --- /dev/null +++ b/src/c/gf-complete/src/neon/gf_w16_neon.c @@ -0,0 +1,356 @@ +/* + * GF-Complete: A Comprehensive Open Source Library for Galois Field Arithmetic + * James S. Plank, Ethan L. Miller, Kevin M. Greenan, + * Benjamin A. Arnold, John A. Burnum, Adam W. Disney, Allen C. McBride. + * + * Copyright (c) 2014: Janne Grunau <j@jannau.net> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * - Neither the name of the University of Tennessee nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * + * + * gf_w16_neon.c + * + * Neon routines for 16-bit Galois fields + * + */ + +#include "gf_int.h" +#include <stdio.h> +#include <stdlib.h> +#include "gf_w16.h" + +#ifdef ARCH_AARCH64 +static +inline +void +neon_w16_split_4_multiply_region(gf_t *gf, uint16_t *src, uint16_t *dst, + uint16_t *d_end, uint8_t *tbl, + gf_val_32_t val, int xor) +{ + unsigned i; + uint8_t *high = tbl + 4 * 16; + uint16x8_t va0, va1, r0, r1; + uint8x16_t loset, rl, rh; + uint8x16x2_t va; + + uint8x16_t tbl_h[4], tbl_l[4]; + for (i = 0; i < 4; i++) { + tbl_l[i] = vld1q_u8(tbl + i*16); + tbl_h[i] = vld1q_u8(high + i*16); + } + + loset = vdupq_n_u8(0xf); + + while (dst < d_end) { + va0 = vld1q_u16(src); + va1 = vld1q_u16(src + 8); + + va = vtrnq_u8(vreinterpretq_u8_u16(va0), vreinterpretq_u8_u16(va1)); + + rl = vqtbl1q_u8(tbl_l[0], vandq_u8(va.val[0], loset)); + rh = vqtbl1q_u8(tbl_h[0], vandq_u8(va.val[0], loset)); + rl = veorq_u8(rl, vqtbl1q_u8(tbl_l[2], vandq_u8(va.val[1], loset))); + rh = veorq_u8(rh, vqtbl1q_u8(tbl_h[2], vandq_u8(va.val[1], loset))); + + va.val[0] = vshrq_n_u8(va.val[0], 4); + va.val[1] = vshrq_n_u8(va.val[1], 4); + + rl = veorq_u8(rl, vqtbl1q_u8(tbl_l[1], va.val[0])); + rh = veorq_u8(rh, vqtbl1q_u8(tbl_h[1], va.val[0])); + rl = veorq_u8(rl, vqtbl1q_u8(tbl_l[3], va.val[1])); + rh = veorq_u8(rh, vqtbl1q_u8(tbl_h[3], va.val[1])); + + va = vtrnq_u8(rl, rh); + r0 = vreinterpretq_u16_u8(va.val[0]); + r1 = vreinterpretq_u16_u8(va.val[1]); + + if (xor) { + va0 = vld1q_u16(dst); + va1 = vld1q_u16(dst + 8); + r0 = veorq_u16(r0, va0); + r1 = veorq_u16(r1, va1); + } + vst1q_u16(dst, r0); + vst1q_u16(dst + 8, r1); + + src += 16; + dst += 16; + } +} + +static +inline +void +neon_w16_split_4_altmap_multiply_region(gf_t *gf, uint8_t *src, + uint8_t *dst, uint8_t *d_end, + uint8_t *tbl, gf_val_32_t val, + int xor) +{ + unsigned i; + uint8_t *high = tbl + 4 * 16; + uint8x16_t vh, vl, rh, rl; + uint8x16_t loset; + + uint8x16_t tbl_h[4], tbl_l[4]; + for (i = 0; i < 4; i++) { + tbl_l[i] = vld1q_u8(tbl + i*16); + tbl_h[i] = vld1q_u8(high + i*16); + } + + loset = vdupq_n_u8(0xf); + + while (dst < d_end) { + vh = vld1q_u8(src); + vl = vld1q_u8(src + 16); + + rl = vqtbl1q_u8(tbl_l[0], vandq_u8(vl, loset)); + rh = vqtbl1q_u8(tbl_h[0], vandq_u8(vl, loset)); + rl = veorq_u8(rl, vqtbl1q_u8(tbl_l[2], vandq_u8(vh, loset))); + rh = veorq_u8(rh, vqtbl1q_u8(tbl_h[2], vandq_u8(vh, loset))); + + vl = vshrq_n_u8(vl, 4); + vh = vshrq_n_u8(vh, 4); + + rl = veorq_u8(rl, vqtbl1q_u8(tbl_l[1], vl)); + rh = veorq_u8(rh, vqtbl1q_u8(tbl_h[1], vl)); + rl = veorq_u8(rl, vqtbl1q_u8(tbl_l[3], vh)); + rh = veorq_u8(rh, vqtbl1q_u8(tbl_h[3], vh)); + + if (xor) { + vh = vld1q_u8(dst); + vl = vld1q_u8(dst + 16); + rh = veorq_u8(rh, vh); + rl = veorq_u8(rl, vl); + } + vst1q_u8(dst, rh); + vst1q_u8(dst + 16, rl); + + src += 32; + dst += 32; + } +} + +#else /* ARCH_AARCH64 */ + +static +inline +void +neon_w16_split_4_multiply_region(gf_t *gf, uint16_t *src, uint16_t *dst, + uint16_t *d_end, uint8_t *tbl, + gf_val_32_t val, int xor) +{ + unsigned i; + uint8_t *high = tbl + 4 * 16; + uint16x8_t va, r; + uint8x8_t loset, vb, vc, rl, rh; + + uint8x8x2_t tbl_h[4], tbl_l[4]; + for (i = 0; i < 4; i++) { + tbl_l[i].val[0] = vld1_u8(tbl + i*16); + tbl_l[i].val[1] = vld1_u8(tbl + i*16 + 8); + tbl_h[i].val[0] = vld1_u8(high + i*16); + tbl_h[i].val[1] = vld1_u8(high + i*16 + 8); + } + + loset = vdup_n_u8(0xf); + + while (dst < d_end) { + va = vld1q_u16(src); + + vb = vmovn_u16(va); + vc = vshrn_n_u16(va, 8); + + rl = vtbl2_u8(tbl_l[0], vand_u8(vb, loset)); + rh = vtbl2_u8(tbl_h[0], vand_u8(vb, loset)); + vb = vshr_n_u8(vb, 4); + rl = veor_u8(rl, vtbl2_u8(tbl_l[2], vand_u8(vc, loset))); + rh = veor_u8(rh, vtbl2_u8(tbl_h[2], vand_u8(vc, loset))); + vc = vshr_n_u8(vc, 4); + rl = veor_u8(rl, vtbl2_u8(tbl_l[1], vb)); + rh = veor_u8(rh, vtbl2_u8(tbl_h[1], vb)); + rl = veor_u8(rl, vtbl2_u8(tbl_l[3], vc)); + rh = veor_u8(rh, vtbl2_u8(tbl_h[3], vc)); + + r = vmovl_u8(rl); + r = vorrq_u16(r, vshll_n_u8(rh, 8)); + + if (xor) { + va = vld1q_u16(dst); + r = veorq_u16(r, va); + } + vst1q_u16(dst, r); + + src += 8; + dst += 8; + } +} + +static +inline +void +neon_w16_split_4_altmap_multiply_region(gf_t *gf, uint8_t *src, + uint8_t *dst, uint8_t *d_end, + uint8_t *tbl, gf_val_32_t val, + int xor) +{ + unsigned i; + uint8_t *high = tbl + 4 * 16; + uint8x8_t vh0, vh1, vl0, vl1, r0, r1, r2, r3; + uint8x8_t loset; + + uint8x8x2_t tbl_h[4], tbl_l[4]; + for (i = 0; i < 4; i++) { + tbl_l[i].val[0] = vld1_u8(tbl + i*16); + tbl_l[i].val[1] = vld1_u8(tbl + i*16 + 8); + tbl_h[i].val[0] = vld1_u8(high + i*16); + tbl_h[i].val[1] = vld1_u8(high + i*16 + 8); + } + + loset = vdup_n_u8(0xf); + + while (dst < d_end) { + vh0 = vld1_u8(src); + vh1 = vld1_u8(src + 8); + vl0 = vld1_u8(src + 16); + vl1 = vld1_u8(src + 24); + + r0 = vtbl2_u8(tbl_l[0], vand_u8(vh0, loset)); + r1 = vtbl2_u8(tbl_h[0], vand_u8(vh1, loset)); + r2 = vtbl2_u8(tbl_l[2], vand_u8(vl0, loset)); + r3 = vtbl2_u8(tbl_h[2], vand_u8(vl1, loset)); + + vh0 = vshr_n_u8(vh0, 4); + vh1 = vshr_n_u8(vh1, 4); + vl0 = vshr_n_u8(vl0, 4); + vl1 = vshr_n_u8(vl1, 4); + + r0 = veor_u8(r0, vtbl2_u8(tbl_l[1], vh0)); + r1 = veor_u8(r1, vtbl2_u8(tbl_h[1], vh1)); + r2 = veor_u8(r2, vtbl2_u8(tbl_l[3], vl0)); + r3 = veor_u8(r3, vtbl2_u8(tbl_h[3], vl1)); + + if (xor) { + vh0 = vld1_u8(dst); + vh1 = vld1_u8(dst + 8); + vl0 = vld1_u8(dst + 16); + vl1 = vld1_u8(dst + 24); + r0 = veor_u8(r0, vh0); + r1 = veor_u8(r1, vh1); + r2 = veor_u8(r2, vl0); + r3 = veor_u8(r3, vl1); + } + vst1_u8(dst, r0); + vst1_u8(dst + 8, r1); + vst1_u8(dst + 16, r2); + vst1_u8(dst + 24, r3); + + src += 32; + dst += 32; + } +} +#endif /* ARCH_AARCH64 */ + +static +inline +void +neon_w16_split_4_16_lazy_multiply_region(gf_t *gf, void *src, void *dest, + gf_val_32_t val, int bytes, int xor, + int altmap) +{ + gf_region_data rd; + unsigned i, j; + uint64_t c, prod; + uint8_t tbl[2 * 4 * 16]; + uint8_t *high = tbl + 4 * 16; + + if (val == 0) { gf_multby_zero(dest, bytes, xor); return; } + if (val == 1) { gf_multby_one(src, dest, bytes, xor); return; } + + for (i = 0; i < 4; i++) { + for (j = 0; j < 16; j++) { + c = (j << (i*4)); + prod = gf->multiply.w32(gf, c, val); + tbl[i*16 + j] = prod & 0xff; + high[i*16 + j] = prod >> 8; + } + } + + gf_set_region_data(&rd, gf, src, dest, bytes, val, xor, 32); + gf_do_initial_region_alignment(&rd); + + if (altmap) { + uint8_t *s8 = rd.s_start; + uint8_t *d8 = rd.d_start; + uint8_t *end8 = rd.d_top; + if (xor) + neon_w16_split_4_altmap_multiply_region(gf, s8, d8, end8, tbl, val, 1); + else + neon_w16_split_4_altmap_multiply_region(gf, s8, d8, end8, tbl, val, 0); + } else { + uint16_t *s16 = rd.s_start; + uint16_t *d16 = rd.d_start; + uint16_t *end16 = rd.d_top; + if (xor) + neon_w16_split_4_multiply_region(gf, s16, d16, end16, tbl, val, 1); + else + neon_w16_split_4_multiply_region(gf, s16, d16, end16, tbl, val, 0); + } + + gf_do_final_region_alignment(&rd); +} + +static +void +gf_w16_split_4_16_lazy_multiply_region_neon(gf_t *gf, void *src, void *dest, + gf_val_32_t val, int bytes, int xor) +{ + neon_w16_split_4_16_lazy_multiply_region(gf, src, dest, val, bytes, xor, 0); +} + +static +void +gf_w16_split_4_16_lazy_altmap_multiply_region_neon(gf_t *gf, void *src, + void *dest, + gf_val_32_t val, int bytes, + int xor) +{ + neon_w16_split_4_16_lazy_multiply_region(gf, src, dest, val, bytes, xor, 1); +} + + +void gf_w16_neon_split_init(gf_t *gf) +{ + gf_internal_t *h = (gf_internal_t *) gf->scratch; + + if (h->region_type & GF_REGION_ALTMAP) + gf->multiply_region.w32 = gf_w16_split_4_16_lazy_altmap_multiply_region_neon; + else + gf->multiply_region.w32 = gf_w16_split_4_16_lazy_multiply_region_neon; +} |