diff options
Diffstat (limited to 'lib/accelerated/x86/macosx/aesni-x86.s')
| -rw-r--r-- | lib/accelerated/x86/macosx/aesni-x86.s | 1946 |
1 files changed, 1518 insertions, 428 deletions
diff --git a/lib/accelerated/x86/macosx/aesni-x86.s b/lib/accelerated/x86/macosx/aesni-x86.s index 09ca1cbc5c..ee50089146 100644 --- a/lib/accelerated/x86/macosx/aesni-x86.s +++ b/lib/accelerated/x86/macosx/aesni-x86.s @@ -1,4 +1,4 @@ -# Copyright (c) 2011-2013, Andy Polyakov <appro@openssl.org> +# Copyright (c) 2011-2016, Andy Polyakov <appro@openssl.org> # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -37,7 +37,6 @@ # # *** This file is auto-generated *** # -.file "devel/perlasm/aesni-x86.s" .text .globl _aesni_encrypt .align 4 @@ -59,7 +58,10 @@ L000enc1_loop_1: leal 16(%edx),%edx jnz L000enc1_loop_1 .byte 102,15,56,221,209 + pxor %xmm0,%xmm0 + pxor %xmm1,%xmm1 movups %xmm2,(%eax) + pxor %xmm2,%xmm2 ret .globl _aesni_decrypt .align 4 @@ -81,30 +83,84 @@ L001dec1_loop_2: leal 16(%edx),%edx jnz L001dec1_loop_2 .byte 102,15,56,223,209 + pxor %xmm0,%xmm0 + pxor %xmm1,%xmm1 movups %xmm2,(%eax) + pxor %xmm2,%xmm2 + ret +.align 4 +__aesni_encrypt2: + movups (%edx),%xmm0 + shll $4,%ecx + movups 16(%edx),%xmm1 + xorps %xmm0,%xmm2 + pxor %xmm0,%xmm3 + movups 32(%edx),%xmm0 + leal 32(%edx,%ecx,1),%edx + negl %ecx + addl $16,%ecx +L002enc2_loop: +.byte 102,15,56,220,209 +.byte 102,15,56,220,217 + movups (%edx,%ecx,1),%xmm1 + addl $32,%ecx +.byte 102,15,56,220,208 +.byte 102,15,56,220,216 + movups -16(%edx,%ecx,1),%xmm0 + jnz L002enc2_loop +.byte 102,15,56,220,209 +.byte 102,15,56,220,217 +.byte 102,15,56,221,208 +.byte 102,15,56,221,216 + ret +.align 4 +__aesni_decrypt2: + movups (%edx),%xmm0 + shll $4,%ecx + movups 16(%edx),%xmm1 + xorps %xmm0,%xmm2 + pxor %xmm0,%xmm3 + movups 32(%edx),%xmm0 + leal 32(%edx,%ecx,1),%edx + negl %ecx + addl $16,%ecx +L003dec2_loop: +.byte 102,15,56,222,209 +.byte 102,15,56,222,217 + movups (%edx,%ecx,1),%xmm1 + addl $32,%ecx +.byte 102,15,56,222,208 +.byte 102,15,56,222,216 + movups -16(%edx,%ecx,1),%xmm0 + jnz L003dec2_loop +.byte 102,15,56,222,209 +.byte 102,15,56,222,217 +.byte 102,15,56,223,208 +.byte 102,15,56,223,216 ret .align 4 __aesni_encrypt3: movups (%edx),%xmm0 - shrl $1,%ecx + shll $4,%ecx movups 16(%edx),%xmm1 - leal 32(%edx),%edx xorps %xmm0,%xmm2 pxor %xmm0,%xmm3 pxor %xmm0,%xmm4 - movups (%edx),%xmm0 -L002enc3_loop: + movups 32(%edx),%xmm0 + leal 32(%edx,%ecx,1),%edx + negl %ecx + addl $16,%ecx +L004enc3_loop: .byte 102,15,56,220,209 .byte 102,15,56,220,217 - decl %ecx .byte 102,15,56,220,225 - movups 16(%edx),%xmm1 + movups (%edx,%ecx,1),%xmm1 + addl $32,%ecx .byte 102,15,56,220,208 .byte 102,15,56,220,216 - leal 32(%edx),%edx .byte 102,15,56,220,224 - movups (%edx),%xmm0 - jnz L002enc3_loop + movups -16(%edx,%ecx,1),%xmm0 + jnz L004enc3_loop .byte 102,15,56,220,209 .byte 102,15,56,220,217 .byte 102,15,56,220,225 @@ -115,25 +171,26 @@ L002enc3_loop: .align 4 __aesni_decrypt3: movups (%edx),%xmm0 - shrl $1,%ecx + shll $4,%ecx movups 16(%edx),%xmm1 - leal 32(%edx),%edx xorps %xmm0,%xmm2 pxor %xmm0,%xmm3 pxor %xmm0,%xmm4 - movups (%edx),%xmm0 -L003dec3_loop: + movups 32(%edx),%xmm0 + leal 32(%edx,%ecx,1),%edx + negl %ecx + addl $16,%ecx +L005dec3_loop: .byte 102,15,56,222,209 .byte 102,15,56,222,217 - decl %ecx .byte 102,15,56,222,225 - movups 16(%edx),%xmm1 + movups (%edx,%ecx,1),%xmm1 + addl $32,%ecx .byte 102,15,56,222,208 .byte 102,15,56,222,216 - leal 32(%edx),%edx .byte 102,15,56,222,224 - movups (%edx),%xmm0 - jnz L003dec3_loop + movups -16(%edx,%ecx,1),%xmm0 + jnz L005dec3_loop .byte 102,15,56,222,209 .byte 102,15,56,222,217 .byte 102,15,56,222,225 @@ -145,27 +202,29 @@ L003dec3_loop: __aesni_encrypt4: movups (%edx),%xmm0 movups 16(%edx),%xmm1 - shrl $1,%ecx - leal 32(%edx),%edx + shll $4,%ecx xorps %xmm0,%xmm2 pxor %xmm0,%xmm3 pxor %xmm0,%xmm4 pxor %xmm0,%xmm5 - movups (%edx),%xmm0 -L004enc4_loop: + movups 32(%edx),%xmm0 + leal 32(%edx,%ecx,1),%edx + negl %ecx +.byte 15,31,64,0 + addl $16,%ecx +L006enc4_loop: .byte 102,15,56,220,209 .byte 102,15,56,220,217 - decl %ecx .byte 102,15,56,220,225 .byte 102,15,56,220,233 - movups 16(%edx),%xmm1 + movups (%edx,%ecx,1),%xmm1 + addl $32,%ecx .byte 102,15,56,220,208 .byte 102,15,56,220,216 - leal 32(%edx),%edx .byte 102,15,56,220,224 .byte 102,15,56,220,232 - movups (%edx),%xmm0 - jnz L004enc4_loop + movups -16(%edx,%ecx,1),%xmm0 + jnz L006enc4_loop .byte 102,15,56,220,209 .byte 102,15,56,220,217 .byte 102,15,56,220,225 @@ -179,27 +238,29 @@ L004enc4_loop: __aesni_decrypt4: movups (%edx),%xmm0 movups 16(%edx),%xmm1 - shrl $1,%ecx - leal 32(%edx),%edx + shll $4,%ecx xorps %xmm0,%xmm2 pxor %xmm0,%xmm3 pxor %xmm0,%xmm4 pxor %xmm0,%xmm5 - movups (%edx),%xmm0 -L005dec4_loop: + movups 32(%edx),%xmm0 + leal 32(%edx,%ecx,1),%edx + negl %ecx +.byte 15,31,64,0 + addl $16,%ecx +L007dec4_loop: .byte 102,15,56,222,209 .byte 102,15,56,222,217 - decl %ecx .byte 102,15,56,222,225 .byte 102,15,56,222,233 - movups 16(%edx),%xmm1 + movups (%edx,%ecx,1),%xmm1 + addl $32,%ecx .byte 102,15,56,222,208 .byte 102,15,56,222,216 - leal 32(%edx),%edx .byte 102,15,56,222,224 .byte 102,15,56,222,232 - movups (%edx),%xmm0 - jnz L005dec4_loop + movups -16(%edx,%ecx,1),%xmm0 + jnz L007dec4_loop .byte 102,15,56,222,209 .byte 102,15,56,222,217 .byte 102,15,56,222,225 @@ -212,45 +273,42 @@ L005dec4_loop: .align 4 __aesni_encrypt6: movups (%edx),%xmm0 - shrl $1,%ecx + shll $4,%ecx movups 16(%edx),%xmm1 - leal 32(%edx),%edx xorps %xmm0,%xmm2 pxor %xmm0,%xmm3 -.byte 102,15,56,220,209 pxor %xmm0,%xmm4 -.byte 102,15,56,220,217 +.byte 102,15,56,220,209 pxor %xmm0,%xmm5 - decl %ecx -.byte 102,15,56,220,225 pxor %xmm0,%xmm6 -.byte 102,15,56,220,233 +.byte 102,15,56,220,217 + leal 32(%edx,%ecx,1),%edx + negl %ecx +.byte 102,15,56,220,225 pxor %xmm0,%xmm7 -.byte 102,15,56,220,241 - movups (%edx),%xmm0 -.byte 102,15,56,220,249 - jmp L_aesni_encrypt6_enter + movups (%edx,%ecx,1),%xmm0 + addl $16,%ecx + jmp L008_aesni_encrypt6_inner .align 4,0x90 -L006enc6_loop: +L009enc6_loop: .byte 102,15,56,220,209 .byte 102,15,56,220,217 - decl %ecx .byte 102,15,56,220,225 +L008_aesni_encrypt6_inner: .byte 102,15,56,220,233 .byte 102,15,56,220,241 .byte 102,15,56,220,249 -.align 4,0x90 L_aesni_encrypt6_enter: - movups 16(%edx),%xmm1 + movups (%edx,%ecx,1),%xmm1 + addl $32,%ecx .byte 102,15,56,220,208 .byte 102,15,56,220,216 - leal 32(%edx),%edx .byte 102,15,56,220,224 .byte 102,15,56,220,232 .byte 102,15,56,220,240 .byte 102,15,56,220,248 - movups (%edx),%xmm0 - jnz L006enc6_loop + movups -16(%edx,%ecx,1),%xmm0 + jnz L009enc6_loop .byte 102,15,56,220,209 .byte 102,15,56,220,217 .byte 102,15,56,220,225 @@ -267,45 +325,42 @@ L_aesni_encrypt6_enter: .align 4 __aesni_decrypt6: movups (%edx),%xmm0 - shrl $1,%ecx + shll $4,%ecx movups 16(%edx),%xmm1 - leal 32(%edx),%edx xorps %xmm0,%xmm2 pxor %xmm0,%xmm3 -.byte 102,15,56,222,209 pxor %xmm0,%xmm4 -.byte 102,15,56,222,217 +.byte 102,15,56,222,209 pxor %xmm0,%xmm5 - decl %ecx -.byte 102,15,56,222,225 pxor %xmm0,%xmm6 -.byte 102,15,56,222,233 +.byte 102,15,56,222,217 + leal 32(%edx,%ecx,1),%edx + negl %ecx +.byte 102,15,56,222,225 pxor %xmm0,%xmm7 -.byte 102,15,56,222,241 - movups (%edx),%xmm0 -.byte 102,15,56,222,249 - jmp L_aesni_decrypt6_enter + movups (%edx,%ecx,1),%xmm0 + addl $16,%ecx + jmp L010_aesni_decrypt6_inner .align 4,0x90 -L007dec6_loop: +L011dec6_loop: .byte 102,15,56,222,209 .byte 102,15,56,222,217 - decl %ecx .byte 102,15,56,222,225 +L010_aesni_decrypt6_inner: .byte 102,15,56,222,233 .byte 102,15,56,222,241 .byte 102,15,56,222,249 -.align 4,0x90 L_aesni_decrypt6_enter: - movups 16(%edx),%xmm1 + movups (%edx,%ecx,1),%xmm1 + addl $32,%ecx .byte 102,15,56,222,208 .byte 102,15,56,222,216 - leal 32(%edx),%edx .byte 102,15,56,222,224 .byte 102,15,56,222,232 .byte 102,15,56,222,240 .byte 102,15,56,222,248 - movups (%edx),%xmm0 - jnz L007dec6_loop + movups -16(%edx,%ecx,1),%xmm0 + jnz L011dec6_loop .byte 102,15,56,222,209 .byte 102,15,56,222,217 .byte 102,15,56,222,225 @@ -333,14 +388,14 @@ L_aesni_ecb_encrypt_begin: movl 32(%esp),%edx movl 36(%esp),%ebx andl $-16,%eax - jz L008ecb_ret + jz L012ecb_ret movl 240(%edx),%ecx testl %ebx,%ebx - jz L009ecb_decrypt + jz L013ecb_decrypt movl %edx,%ebp movl %ecx,%ebx cmpl $96,%eax - jb L010ecb_enc_tail + jb L014ecb_enc_tail movdqu (%esi),%xmm2 movdqu 16(%esi),%xmm3 movdqu 32(%esi),%xmm4 @@ -349,9 +404,9 @@ L_aesni_ecb_encrypt_begin: movdqu 80(%esi),%xmm7 leal 96(%esi),%esi subl $96,%eax - jmp L011ecb_enc_loop6_enter + jmp L015ecb_enc_loop6_enter .align 4,0x90 -L012ecb_enc_loop6: +L016ecb_enc_loop6: movups %xmm2,(%edi) movdqu (%esi),%xmm2 movups %xmm3,16(%edi) @@ -366,12 +421,12 @@ L012ecb_enc_loop6: leal 96(%edi),%edi movdqu 80(%esi),%xmm7 leal 96(%esi),%esi -L011ecb_enc_loop6_enter: +L015ecb_enc_loop6_enter: call __aesni_encrypt6 movl %ebp,%edx movl %ebx,%ecx subl $96,%eax - jnc L012ecb_enc_loop6 + jnc L016ecb_enc_loop6 movups %xmm2,(%edi) movups %xmm3,16(%edi) movups %xmm4,32(%edi) @@ -380,18 +435,18 @@ L011ecb_enc_loop6_enter: movups %xmm7,80(%edi) leal 96(%edi),%edi addl $96,%eax - jz L008ecb_ret -L010ecb_enc_tail: + jz L012ecb_ret +L014ecb_enc_tail: movups (%esi),%xmm2 cmpl $32,%eax - jb L013ecb_enc_one + jb L017ecb_enc_one movups 16(%esi),%xmm3 - je L014ecb_enc_two + je L018ecb_enc_two movups 32(%esi),%xmm4 cmpl $64,%eax - jb L015ecb_enc_three + jb L019ecb_enc_three movups 48(%esi),%xmm5 - je L016ecb_enc_four + je L020ecb_enc_four movups 64(%esi),%xmm6 xorps %xmm7,%xmm7 call __aesni_encrypt6 @@ -400,50 +455,49 @@ L010ecb_enc_tail: movups %xmm4,32(%edi) movups %xmm5,48(%edi) movups %xmm6,64(%edi) - jmp L008ecb_ret + jmp L012ecb_ret .align 4,0x90 -L013ecb_enc_one: +L017ecb_enc_one: movups (%edx),%xmm0 movups 16(%edx),%xmm1 leal 32(%edx),%edx xorps %xmm0,%xmm2 -L017enc1_loop_3: +L021enc1_loop_3: .byte 102,15,56,220,209 decl %ecx movups (%edx),%xmm1 leal 16(%edx),%edx - jnz L017enc1_loop_3 + jnz L021enc1_loop_3 .byte 102,15,56,221,209 movups %xmm2,(%edi) - jmp L008ecb_ret + jmp L012ecb_ret .align 4,0x90 -L014ecb_enc_two: - xorps %xmm4,%xmm4 - call __aesni_encrypt3 +L018ecb_enc_two: + call __aesni_encrypt2 movups %xmm2,(%edi) movups %xmm3,16(%edi) - jmp L008ecb_ret + jmp L012ecb_ret .align 4,0x90 -L015ecb_enc_three: +L019ecb_enc_three: call __aesni_encrypt3 movups %xmm2,(%edi) movups %xmm3,16(%edi) movups %xmm4,32(%edi) - jmp L008ecb_ret + jmp L012ecb_ret .align 4,0x90 -L016ecb_enc_four: +L020ecb_enc_four: call __aesni_encrypt4 movups %xmm2,(%edi) movups %xmm3,16(%edi) movups %xmm4,32(%edi) movups %xmm5,48(%edi) - jmp L008ecb_ret + jmp L012ecb_ret .align 4,0x90 -L009ecb_decrypt: +L013ecb_decrypt: movl %edx,%ebp movl %ecx,%ebx cmpl $96,%eax - jb L018ecb_dec_tail + jb L022ecb_dec_tail movdqu (%esi),%xmm2 movdqu 16(%esi),%xmm3 movdqu 32(%esi),%xmm4 @@ -452,9 +506,9 @@ L009ecb_decrypt: movdqu 80(%esi),%xmm7 leal 96(%esi),%esi subl $96,%eax - jmp L019ecb_dec_loop6_enter + jmp L023ecb_dec_loop6_enter .align 4,0x90 -L020ecb_dec_loop6: +L024ecb_dec_loop6: movups %xmm2,(%edi) movdqu (%esi),%xmm2 movups %xmm3,16(%edi) @@ -469,12 +523,12 @@ L020ecb_dec_loop6: leal 96(%edi),%edi movdqu 80(%esi),%xmm7 leal 96(%esi),%esi -L019ecb_dec_loop6_enter: +L023ecb_dec_loop6_enter: call __aesni_decrypt6 movl %ebp,%edx movl %ebx,%ecx subl $96,%eax - jnc L020ecb_dec_loop6 + jnc L024ecb_dec_loop6 movups %xmm2,(%edi) movups %xmm3,16(%edi) movups %xmm4,32(%edi) @@ -483,18 +537,18 @@ L019ecb_dec_loop6_enter: movups %xmm7,80(%edi) leal 96(%edi),%edi addl $96,%eax - jz L008ecb_ret -L018ecb_dec_tail: + jz L012ecb_ret +L022ecb_dec_tail: movups (%esi),%xmm2 cmpl $32,%eax - jb L021ecb_dec_one + jb L025ecb_dec_one movups 16(%esi),%xmm3 - je L022ecb_dec_two + je L026ecb_dec_two movups 32(%esi),%xmm4 cmpl $64,%eax - jb L023ecb_dec_three + jb L027ecb_dec_three movups 48(%esi),%xmm5 - je L024ecb_dec_four + je L028ecb_dec_four movups 64(%esi),%xmm6 xorps %xmm7,%xmm7 call __aesni_decrypt6 @@ -503,44 +557,51 @@ L018ecb_dec_tail: movups %xmm4,32(%edi) movups %xmm5,48(%edi) movups %xmm6,64(%edi) - jmp L008ecb_ret + jmp L012ecb_ret .align 4,0x90 -L021ecb_dec_one: +L025ecb_dec_one: movups (%edx),%xmm0 movups 16(%edx),%xmm1 leal 32(%edx),%edx xorps %xmm0,%xmm2 -L025dec1_loop_4: +L029dec1_loop_4: .byte 102,15,56,222,209 decl %ecx movups (%edx),%xmm1 leal 16(%edx),%edx - jnz L025dec1_loop_4 + jnz L029dec1_loop_4 .byte 102,15,56,223,209 movups %xmm2,(%edi) - jmp L008ecb_ret + jmp L012ecb_ret .align 4,0x90 -L022ecb_dec_two: - xorps %xmm4,%xmm4 - call __aesni_decrypt3 +L026ecb_dec_two: + call __aesni_decrypt2 movups %xmm2,(%edi) movups %xmm3,16(%edi) - jmp L008ecb_ret + jmp L012ecb_ret .align 4,0x90 -L023ecb_dec_three: +L027ecb_dec_three: call __aesni_decrypt3 movups %xmm2,(%edi) movups %xmm3,16(%edi) movups %xmm4,32(%edi) - jmp L008ecb_ret + jmp L012ecb_ret .align 4,0x90 -L024ecb_dec_four: +L028ecb_dec_four: call __aesni_decrypt4 movups %xmm2,(%edi) movups %xmm3,16(%edi) movups %xmm4,32(%edi) movups %xmm5,48(%edi) -L008ecb_ret: +L012ecb_ret: + pxor %xmm0,%xmm0 + pxor %xmm1,%xmm1 + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 + pxor %xmm6,%xmm6 + pxor %xmm7,%xmm7 popl %edi popl %esi popl %ebx @@ -577,48 +638,56 @@ L_aesni_ccm64_encrypt_blocks_begin: movl %ebp,20(%esp) movl %ebp,24(%esp) movl %ebp,28(%esp) - shrl $1,%ecx + shll $4,%ecx + movl $16,%ebx leal (%edx),%ebp movdqa (%esp),%xmm5 movdqa %xmm7,%xmm2 - movl %ecx,%ebx + leal 32(%edx,%ecx,1),%edx + subl %ecx,%ebx .byte 102,15,56,0,253 -L026ccm64_enc_outer: +L030ccm64_enc_outer: movups (%ebp),%xmm0 movl %ebx,%ecx movups (%esi),%xmm6 xorps %xmm0,%xmm2 movups 16(%ebp),%xmm1 xorps %xmm6,%xmm0 - leal 32(%ebp),%edx xorps %xmm0,%xmm3 - movups (%edx),%xmm0 -L027ccm64_enc2_loop: + movups 32(%ebp),%xmm0 +L031ccm64_enc2_loop: .byte 102,15,56,220,209 - decl %ecx .byte 102,15,56,220,217 - movups 16(%edx),%xmm1 + movups (%edx,%ecx,1),%xmm1 + addl $32,%ecx .byte 102,15,56,220,208 - leal 32(%edx),%edx .byte 102,15,56,220,216 - movups (%edx),%xmm0 - jnz L027ccm64_enc2_loop + movups -16(%edx,%ecx,1),%xmm0 + jnz L031ccm64_enc2_loop .byte 102,15,56,220,209 .byte 102,15,56,220,217 paddq 16(%esp),%xmm7 + decl %eax .byte 102,15,56,221,208 .byte 102,15,56,221,216 - decl %eax leal 16(%esi),%esi xorps %xmm2,%xmm6 movdqa %xmm7,%xmm2 movups %xmm6,(%edi) - leal 16(%edi),%edi .byte 102,15,56,0,213 - jnz L026ccm64_enc_outer + leal 16(%edi),%edi + jnz L030ccm64_enc_outer movl 48(%esp),%esp movl 40(%esp),%edi movups %xmm3,(%edi) + pxor %xmm0,%xmm0 + pxor %xmm1,%xmm1 + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 + pxor %xmm6,%xmm6 + pxor %xmm7,%xmm7 popl %edi popl %esi popl %ebx @@ -664,71 +733,82 @@ L_aesni_ccm64_decrypt_blocks_begin: movups 16(%edx),%xmm1 leal 32(%edx),%edx xorps %xmm0,%xmm2 -L028enc1_loop_5: +L032enc1_loop_5: .byte 102,15,56,220,209 decl %ecx movups (%edx),%xmm1 leal 16(%edx),%edx - jnz L028enc1_loop_5 + jnz L032enc1_loop_5 .byte 102,15,56,221,209 + shll $4,%ebx + movl $16,%ecx movups (%esi),%xmm6 paddq 16(%esp),%xmm7 leal 16(%esi),%esi - jmp L029ccm64_dec_outer + subl %ebx,%ecx + leal 32(%ebp,%ebx,1),%edx + movl %ecx,%ebx + jmp L033ccm64_dec_outer .align 4,0x90 -L029ccm64_dec_outer: +L033ccm64_dec_outer: xorps %xmm2,%xmm6 movdqa %xmm7,%xmm2 - movl %ebx,%ecx movups %xmm6,(%edi) leal 16(%edi),%edi .byte 102,15,56,0,213 subl $1,%eax - jz L030ccm64_dec_break + jz L034ccm64_dec_break movups (%ebp),%xmm0 - shrl $1,%ecx + movl %ebx,%ecx movups 16(%ebp),%xmm1 xorps %xmm0,%xmm6 - leal 32(%ebp),%edx xorps %xmm0,%xmm2 xorps %xmm6,%xmm3 - movups (%edx),%xmm0 -L031ccm64_dec2_loop: + movups 32(%ebp),%xmm0 +L035ccm64_dec2_loop: .byte 102,15,56,220,209 - decl %ecx .byte 102,15,56,220,217 - movups 16(%edx),%xmm1 + movups (%edx,%ecx,1),%xmm1 + addl $32,%ecx .byte 102,15,56,220,208 - leal 32(%edx),%edx .byte 102,15,56,220,216 - movups (%edx),%xmm0 - jnz L031ccm64_dec2_loop + movups -16(%edx,%ecx,1),%xmm0 + jnz L035ccm64_dec2_loop movups (%esi),%xmm6 paddq 16(%esp),%xmm7 .byte 102,15,56,220,209 .byte 102,15,56,220,217 - leal 16(%esi),%esi .byte 102,15,56,221,208 .byte 102,15,56,221,216 - jmp L029ccm64_dec_outer + leal 16(%esi),%esi + jmp L033ccm64_dec_outer .align 4,0x90 -L030ccm64_dec_break: +L034ccm64_dec_break: + movl 240(%ebp),%ecx movl %ebp,%edx movups (%edx),%xmm0 movups 16(%edx),%xmm1 xorps %xmm0,%xmm6 leal 32(%edx),%edx xorps %xmm6,%xmm3 -L032enc1_loop_6: +L036enc1_loop_6: .byte 102,15,56,220,217 decl %ecx movups (%edx),%xmm1 leal 16(%edx),%edx - jnz L032enc1_loop_6 + jnz L036enc1_loop_6 .byte 102,15,56,221,217 movl 48(%esp),%esp movl 40(%esp),%edi movups %xmm3,(%edi) + pxor %xmm0,%xmm0 + pxor %xmm1,%xmm1 + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 + pxor %xmm6,%xmm6 + pxor %xmm7,%xmm7 popl %edi popl %esi popl %ebx @@ -752,7 +832,7 @@ L_aesni_ctr32_encrypt_blocks_begin: andl $-16,%esp movl %ebp,80(%esp) cmpl $1,%eax - je L033ctr32_one_shortcut + je L037ctr32_one_shortcut movdqu (%ebx),%xmm7 movl $202182159,(%esp) movl $134810123,4(%esp) @@ -768,63 +848,59 @@ L_aesni_ctr32_encrypt_blocks_begin: .byte 102,15,58,34,253,3 movl 240(%edx),%ecx bswap %ebx - pxor %xmm1,%xmm1 pxor %xmm0,%xmm0 + pxor %xmm1,%xmm1 movdqa (%esp),%xmm2 -.byte 102,15,58,34,203,0 +.byte 102,15,58,34,195,0 leal 3(%ebx),%ebp -.byte 102,15,58,34,197,0 +.byte 102,15,58,34,205,0 incl %ebx -.byte 102,15,58,34,203,1 +.byte 102,15,58,34,195,1 incl %ebp -.byte 102,15,58,34,197,1 +.byte 102,15,58,34,205,1 incl %ebx -.byte 102,15,58,34,203,2 +.byte 102,15,58,34,195,2 incl %ebp -.byte 102,15,58,34,197,2 - movdqa %xmm1,48(%esp) -.byte 102,15,56,0,202 - movdqa %xmm0,64(%esp) +.byte 102,15,58,34,205,2 + movdqa %xmm0,48(%esp) .byte 102,15,56,0,194 - pshufd $192,%xmm1,%xmm2 - pshufd $128,%xmm1,%xmm3 + movdqu (%edx),%xmm6 + movdqa %xmm1,64(%esp) +.byte 102,15,56,0,202 + pshufd $192,%xmm0,%xmm2 + pshufd $128,%xmm0,%xmm3 cmpl $6,%eax - jb L034ctr32_tail + jb L038ctr32_tail + pxor %xmm6,%xmm7 + shll $4,%ecx + movl $16,%ebx movdqa %xmm7,32(%esp) - shrl $1,%ecx movl %edx,%ebp - movl %ecx,%ebx + subl %ecx,%ebx + leal 32(%edx,%ecx,1),%edx subl $6,%eax - jmp L035ctr32_loop6 + jmp L039ctr32_loop6 .align 4,0x90 -L035ctr32_loop6: - pshufd $64,%xmm1,%xmm4 - movdqa 32(%esp),%xmm1 - pshufd $192,%xmm0,%xmm5 - por %xmm1,%xmm2 - pshufd $128,%xmm0,%xmm6 - por %xmm1,%xmm3 - pshufd $64,%xmm0,%xmm7 - por %xmm1,%xmm4 - por %xmm1,%xmm5 - por %xmm1,%xmm6 - por %xmm1,%xmm7 - movups (%ebp),%xmm0 - movups 16(%ebp),%xmm1 - leal 32(%ebp),%edx - decl %ecx +L039ctr32_loop6: + pshufd $64,%xmm0,%xmm4 + movdqa 32(%esp),%xmm0 + pshufd $192,%xmm1,%xmm5 pxor %xmm0,%xmm2 + pshufd $128,%xmm1,%xmm6 pxor %xmm0,%xmm3 -.byte 102,15,56,220,209 + pshufd $64,%xmm1,%xmm7 + movups 16(%ebp),%xmm1 pxor %xmm0,%xmm4 -.byte 102,15,56,220,217 pxor %xmm0,%xmm5 -.byte 102,15,56,220,225 +.byte 102,15,56,220,209 pxor %xmm0,%xmm6 -.byte 102,15,56,220,233 pxor %xmm0,%xmm7 +.byte 102,15,56,220,217 + movups 32(%ebp),%xmm0 + movl %ebx,%ecx +.byte 102,15,56,220,225 +.byte 102,15,56,220,233 .byte 102,15,56,220,241 - movups (%edx),%xmm0 .byte 102,15,56,220,249 call L_aesni_encrypt6_enter movups (%esi),%xmm1 @@ -835,51 +911,51 @@ L035ctr32_loop6: movups %xmm2,(%edi) movdqa 16(%esp),%xmm0 xorps %xmm1,%xmm4 - movdqa 48(%esp),%xmm1 + movdqa 64(%esp),%xmm1 movups %xmm3,16(%edi) movups %xmm4,32(%edi) paddd %xmm0,%xmm1 - paddd 64(%esp),%xmm0 + paddd 48(%esp),%xmm0 movdqa (%esp),%xmm2 movups 48(%esi),%xmm3 movups 64(%esi),%xmm4 xorps %xmm3,%xmm5 movups 80(%esi),%xmm3 leal 96(%esi),%esi - movdqa %xmm1,48(%esp) -.byte 102,15,56,0,202 + movdqa %xmm0,48(%esp) +.byte 102,15,56,0,194 xorps %xmm4,%xmm6 movups %xmm5,48(%edi) xorps %xmm3,%xmm7 - movdqa %xmm0,64(%esp) -.byte 102,15,56,0,194 + movdqa %xmm1,64(%esp) +.byte 102,15,56,0,202 movups %xmm6,64(%edi) - pshufd $192,%xmm1,%xmm2 + pshufd $192,%xmm0,%xmm2 movups %xmm7,80(%edi) leal 96(%edi),%edi - movl %ebx,%ecx - pshufd $128,%xmm1,%xmm3 + pshufd $128,%xmm0,%xmm3 subl $6,%eax - jnc L035ctr32_loop6 + jnc L039ctr32_loop6 addl $6,%eax - jz L036ctr32_ret + jz L040ctr32_ret + movdqu (%ebp),%xmm7 movl %ebp,%edx - leal 1(,%ecx,2),%ecx - movdqa 32(%esp),%xmm7 -L034ctr32_tail: + pxor 32(%esp),%xmm7 + movl 240(%ebp),%ecx +L038ctr32_tail: por %xmm7,%xmm2 cmpl $2,%eax - jb L037ctr32_one - pshufd $64,%xmm1,%xmm4 + jb L041ctr32_one + pshufd $64,%xmm0,%xmm4 por %xmm7,%xmm3 - je L038ctr32_two - pshufd $192,%xmm0,%xmm5 + je L042ctr32_two + pshufd $192,%xmm1,%xmm5 por %xmm7,%xmm4 cmpl $4,%eax - jb L039ctr32_three - pshufd $128,%xmm0,%xmm6 + jb L043ctr32_three + pshufd $128,%xmm1,%xmm6 por %xmm7,%xmm5 - je L040ctr32_four + je L044ctr32_four por %xmm7,%xmm6 call __aesni_encrypt6 movups (%esi),%xmm1 @@ -897,39 +973,39 @@ L034ctr32_tail: movups %xmm4,32(%edi) movups %xmm5,48(%edi) movups %xmm6,64(%edi) - jmp L036ctr32_ret + jmp L040ctr32_ret .align 4,0x90 -L033ctr32_one_shortcut: +L037ctr32_one_shortcut: movups (%ebx),%xmm2 movl 240(%edx),%ecx -L037ctr32_one: +L041ctr32_one: movups (%edx),%xmm0 movups 16(%edx),%xmm1 leal 32(%edx),%edx xorps %xmm0,%xmm2 -L041enc1_loop_7: +L045enc1_loop_7: .byte 102,15,56,220,209 decl %ecx movups (%edx),%xmm1 leal 16(%edx),%edx - jnz L041enc1_loop_7 + jnz L045enc1_loop_7 .byte 102,15,56,221,209 movups (%esi),%xmm6 xorps %xmm2,%xmm6 movups %xmm6,(%edi) - jmp L036ctr32_ret + jmp L040ctr32_ret .align 4,0x90 -L038ctr32_two: - call __aesni_encrypt3 +L042ctr32_two: + call __aesni_encrypt2 movups (%esi),%xmm5 movups 16(%esi),%xmm6 xorps %xmm5,%xmm2 xorps %xmm6,%xmm3 movups %xmm2,(%edi) movups %xmm3,16(%edi) - jmp L036ctr32_ret + jmp L040ctr32_ret .align 4,0x90 -L039ctr32_three: +L043ctr32_three: call __aesni_encrypt3 movups (%esi),%xmm5 movups 16(%esi),%xmm6 @@ -940,9 +1016,9 @@ L039ctr32_three: xorps %xmm7,%xmm4 movups %xmm3,16(%edi) movups %xmm4,32(%edi) - jmp L036ctr32_ret + jmp L040ctr32_ret .align 4,0x90 -L040ctr32_four: +L044ctr32_four: call __aesni_encrypt4 movups (%esi),%xmm6 movups 16(%esi),%xmm7 @@ -956,7 +1032,18 @@ L040ctr32_four: xorps %xmm0,%xmm5 movups %xmm4,32(%edi) movups %xmm5,48(%edi) -L036ctr32_ret: +L040ctr32_ret: + pxor %xmm0,%xmm0 + pxor %xmm1,%xmm1 + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + movdqa %xmm0,32(%esp) + pxor %xmm5,%xmm5 + movdqa %xmm0,48(%esp) + pxor %xmm6,%xmm6 + movdqa %xmm0,64(%esp) + pxor %xmm7,%xmm7 movl 80(%esp),%esp popl %edi popl %esi @@ -979,12 +1066,12 @@ L_aesni_xts_encrypt_begin: movups 16(%edx),%xmm1 leal 32(%edx),%edx xorps %xmm0,%xmm2 -L042enc1_loop_8: +L046enc1_loop_8: .byte 102,15,56,220,209 decl %ecx movups (%edx),%xmm1 leal 16(%edx),%edx - jnz L042enc1_loop_8 + jnz L046enc1_loop_8 .byte 102,15,56,221,209 movl 20(%esp),%esi movl 24(%esp),%edi @@ -1008,12 +1095,14 @@ L042enc1_loop_8: movl %edx,%ebp movl %ecx,%ebx subl $96,%eax - jc L043xts_enc_short - shrl $1,%ecx - movl %ecx,%ebx - jmp L044xts_enc_loop6 + jc L047xts_enc_short + shll $4,%ecx + movl $16,%ebx + subl %ecx,%ebx + leal 32(%edx,%ecx,1),%edx + jmp L048xts_enc_loop6 .align 4,0x90 -L044xts_enc_loop6: +L048xts_enc_loop6: pshufd $19,%xmm0,%xmm2 pxor %xmm0,%xmm0 movdqa %xmm1,(%esp) @@ -1049,6 +1138,7 @@ L044xts_enc_loop6: pand %xmm3,%xmm7 movups (%esi),%xmm2 pxor %xmm1,%xmm7 + movl %ebx,%ecx movdqu 16(%esi),%xmm3 xorps %xmm0,%xmm2 movdqu 32(%esi),%xmm4 @@ -1064,19 +1154,17 @@ L044xts_enc_loop6: movdqa %xmm7,80(%esp) pxor %xmm1,%xmm7 movups 16(%ebp),%xmm1 - leal 32(%ebp),%edx pxor 16(%esp),%xmm3 -.byte 102,15,56,220,209 pxor 32(%esp),%xmm4 -.byte 102,15,56,220,217 +.byte 102,15,56,220,209 pxor 48(%esp),%xmm5 - decl %ecx -.byte 102,15,56,220,225 pxor 64(%esp),%xmm6 -.byte 102,15,56,220,233 +.byte 102,15,56,220,217 pxor %xmm0,%xmm7 + movups 32(%ebp),%xmm0 +.byte 102,15,56,220,225 +.byte 102,15,56,220,233 .byte 102,15,56,220,241 - movups (%edx),%xmm0 .byte 102,15,56,220,249 call L_aesni_encrypt6_enter movdqa 80(%esp),%xmm1 @@ -1101,26 +1189,25 @@ L044xts_enc_loop6: paddq %xmm1,%xmm1 pand %xmm3,%xmm2 pcmpgtd %xmm1,%xmm0 - movl %ebx,%ecx pxor %xmm2,%xmm1 subl $96,%eax - jnc L044xts_enc_loop6 - leal 1(,%ecx,2),%ecx + jnc L048xts_enc_loop6 + movl 240(%ebp),%ecx movl %ebp,%edx movl %ecx,%ebx -L043xts_enc_short: +L047xts_enc_short: addl $96,%eax - jz L045xts_enc_done6x + jz L049xts_enc_done6x movdqa %xmm1,%xmm5 cmpl $32,%eax - jb L046xts_enc_one + jb L050xts_enc_one pshufd $19,%xmm0,%xmm2 pxor %xmm0,%xmm0 paddq %xmm1,%xmm1 pand %xmm3,%xmm2 pcmpgtd %xmm1,%xmm0 pxor %xmm2,%xmm1 - je L047xts_enc_two + je L051xts_enc_two pshufd $19,%xmm0,%xmm2 pxor %xmm0,%xmm0 movdqa %xmm1,%xmm6 @@ -1129,7 +1216,7 @@ L043xts_enc_short: pcmpgtd %xmm1,%xmm0 pxor %xmm2,%xmm1 cmpl $64,%eax - jb L048xts_enc_three + jb L052xts_enc_three pshufd $19,%xmm0,%xmm2 pxor %xmm0,%xmm0 movdqa %xmm1,%xmm7 @@ -1139,7 +1226,7 @@ L043xts_enc_short: pxor %xmm2,%xmm1 movdqa %xmm5,(%esp) movdqa %xmm6,16(%esp) - je L049xts_enc_four + je L053xts_enc_four movdqa %xmm7,32(%esp) pshufd $19,%xmm0,%xmm7 movdqa %xmm1,48(%esp) @@ -1171,9 +1258,9 @@ L043xts_enc_short: movups %xmm5,48(%edi) movups %xmm6,64(%edi) leal 80(%edi),%edi - jmp L050xts_enc_done + jmp L054xts_enc_done .align 4,0x90 -L046xts_enc_one: +L050xts_enc_one: movups (%esi),%xmm2 leal 16(%esi),%esi xorps %xmm5,%xmm2 @@ -1181,37 +1268,36 @@ L046xts_enc_one: movups 16(%edx),%xmm1 leal 32(%edx),%edx xorps %xmm0,%xmm2 -L051enc1_loop_9: +L055enc1_loop_9: .byte 102,15,56,220,209 decl %ecx movups (%edx),%xmm1 leal 16(%edx),%edx - jnz L051enc1_loop_9 + jnz L055enc1_loop_9 .byte 102,15,56,221,209 xorps %xmm5,%xmm2 movups %xmm2,(%edi) leal 16(%edi),%edi movdqa %xmm5,%xmm1 - jmp L050xts_enc_done + jmp L054xts_enc_done .align 4,0x90 -L047xts_enc_two: +L051xts_enc_two: movaps %xmm1,%xmm6 movups (%esi),%xmm2 movups 16(%esi),%xmm3 leal 32(%esi),%esi xorps %xmm5,%xmm2 xorps %xmm6,%xmm3 - xorps %xmm4,%xmm4 - call __aesni_encrypt3 + call __aesni_encrypt2 xorps %xmm5,%xmm2 xorps %xmm6,%xmm3 movups %xmm2,(%edi) movups %xmm3,16(%edi) leal 32(%edi),%edi movdqa %xmm6,%xmm1 - jmp L050xts_enc_done + jmp L054xts_enc_done .align 4,0x90 -L048xts_enc_three: +L052xts_enc_three: movaps %xmm1,%xmm7 movups (%esi),%xmm2 movups 16(%esi),%xmm3 @@ -1229,9 +1315,9 @@ L048xts_enc_three: movups %xmm4,32(%edi) leal 48(%edi),%edi movdqa %xmm7,%xmm1 - jmp L050xts_enc_done + jmp L054xts_enc_done .align 4,0x90 -L049xts_enc_four: +L053xts_enc_four: movaps %xmm1,%xmm6 movups (%esi),%xmm2 movups 16(%esi),%xmm3 @@ -1253,28 +1339,28 @@ L049xts_enc_four: movups %xmm5,48(%edi) leal 64(%edi),%edi movdqa %xmm6,%xmm1 - jmp L050xts_enc_done + jmp L054xts_enc_done .align 4,0x90 -L045xts_enc_done6x: +L049xts_enc_done6x: movl 112(%esp),%eax andl $15,%eax - jz L052xts_enc_ret + jz L056xts_enc_ret movdqa %xmm1,%xmm5 movl %eax,112(%esp) - jmp L053xts_enc_steal + jmp L057xts_enc_steal .align 4,0x90 -L050xts_enc_done: +L054xts_enc_done: movl 112(%esp),%eax pxor %xmm0,%xmm0 andl $15,%eax - jz L052xts_enc_ret + jz L056xts_enc_ret pcmpgtd %xmm1,%xmm0 movl %eax,112(%esp) pshufd $19,%xmm0,%xmm5 paddq %xmm1,%xmm1 pand 96(%esp),%xmm5 pxor %xmm1,%xmm5 -L053xts_enc_steal: +L057xts_enc_steal: movzbl (%esi),%ecx movzbl -16(%edi),%edx leal 1(%esi),%esi @@ -1282,7 +1368,7 @@ L053xts_enc_steal: movb %dl,(%edi) leal 1(%edi),%edi subl $1,%eax - jnz L053xts_enc_steal + jnz L057xts_enc_steal subl 112(%esp),%edi movl %ebp,%edx movl %ebx,%ecx @@ -1292,16 +1378,30 @@ L053xts_enc_steal: movups 16(%edx),%xmm1 leal 32(%edx),%edx xorps %xmm0,%xmm2 -L054enc1_loop_10: +L058enc1_loop_10: .byte 102,15,56,220,209 decl %ecx movups (%edx),%xmm1 leal 16(%edx),%edx - jnz L054enc1_loop_10 + jnz L058enc1_loop_10 .byte 102,15,56,221,209 xorps %xmm5,%xmm2 movups %xmm2,-16(%edi) -L052xts_enc_ret: +L056xts_enc_ret: + pxor %xmm0,%xmm0 + pxor %xmm1,%xmm1 + pxor %xmm2,%xmm2 + movdqa %xmm0,(%esp) + pxor %xmm3,%xmm3 + movdqa %xmm0,16(%esp) + pxor %xmm4,%xmm4 + movdqa %xmm0,32(%esp) + pxor %xmm5,%xmm5 + movdqa %xmm0,48(%esp) + pxor %xmm6,%xmm6 + movdqa %xmm0,64(%esp) + pxor %xmm7,%xmm7 + movdqa %xmm0,80(%esp) movl 116(%esp),%esp popl %edi popl %esi @@ -1324,12 +1424,12 @@ L_aesni_xts_decrypt_begin: movups 16(%edx),%xmm1 leal 32(%edx),%edx xorps %xmm0,%xmm2 -L055enc1_loop_11: +L059enc1_loop_11: .byte 102,15,56,220,209 decl %ecx movups (%edx),%xmm1 leal 16(%edx),%edx - jnz L055enc1_loop_11 + jnz L059enc1_loop_11 .byte 102,15,56,221,209 movl 20(%esp),%esi movl 24(%esp),%edi @@ -1358,12 +1458,14 @@ L055enc1_loop_11: pcmpgtd %xmm1,%xmm0 andl $-16,%eax subl $96,%eax - jc L056xts_dec_short - shrl $1,%ecx - movl %ecx,%ebx - jmp L057xts_dec_loop6 + jc L060xts_dec_short + shll $4,%ecx + movl $16,%ebx + subl %ecx,%ebx + leal 32(%edx,%ecx,1),%edx + jmp L061xts_dec_loop6 .align 4,0x90 -L057xts_dec_loop6: +L061xts_dec_loop6: pshufd $19,%xmm0,%xmm2 pxor %xmm0,%xmm0 movdqa %xmm1,(%esp) @@ -1399,6 +1501,7 @@ L057xts_dec_loop6: pand %xmm3,%xmm7 movups (%esi),%xmm2 pxor %xmm1,%xmm7 + movl %ebx,%ecx movdqu 16(%esi),%xmm3 xorps %xmm0,%xmm2 movdqu 32(%esi),%xmm4 @@ -1414,19 +1517,17 @@ L057xts_dec_loop6: movdqa %xmm7,80(%esp) pxor %xmm1,%xmm7 movups 16(%ebp),%xmm1 - leal 32(%ebp),%edx pxor 16(%esp),%xmm3 -.byte 102,15,56,222,209 pxor 32(%esp),%xmm4 -.byte 102,15,56,222,217 +.byte 102,15,56,222,209 pxor 48(%esp),%xmm5 - decl %ecx -.byte 102,15,56,222,225 pxor 64(%esp),%xmm6 -.byte 102,15,56,222,233 +.byte 102,15,56,222,217 pxor %xmm0,%xmm7 + movups 32(%ebp),%xmm0 +.byte 102,15,56,222,225 +.byte 102,15,56,222,233 .byte 102,15,56,222,241 - movups (%edx),%xmm0 .byte 102,15,56,222,249 call L_aesni_decrypt6_enter movdqa 80(%esp),%xmm1 @@ -1451,26 +1552,25 @@ L057xts_dec_loop6: paddq %xmm1,%xmm1 pand %xmm3,%xmm2 pcmpgtd %xmm1,%xmm0 - movl %ebx,%ecx pxor %xmm2,%xmm1 subl $96,%eax - jnc L057xts_dec_loop6 - leal 1(,%ecx,2),%ecx + jnc L061xts_dec_loop6 + movl 240(%ebp),%ecx movl %ebp,%edx movl %ecx,%ebx -L056xts_dec_short: +L060xts_dec_short: addl $96,%eax - jz L058xts_dec_done6x + jz L062xts_dec_done6x movdqa %xmm1,%xmm5 cmpl $32,%eax - jb L059xts_dec_one + jb L063xts_dec_one pshufd $19,%xmm0,%xmm2 pxor %xmm0,%xmm0 paddq %xmm1,%xmm1 pand %xmm3,%xmm2 pcmpgtd %xmm1,%xmm0 pxor %xmm2,%xmm1 - je L060xts_dec_two + je L064xts_dec_two pshufd $19,%xmm0,%xmm2 pxor %xmm0,%xmm0 movdqa %xmm1,%xmm6 @@ -1479,7 +1579,7 @@ L056xts_dec_short: pcmpgtd %xmm1,%xmm0 pxor %xmm2,%xmm1 cmpl $64,%eax - jb L061xts_dec_three + jb L065xts_dec_three pshufd $19,%xmm0,%xmm2 pxor %xmm0,%xmm0 movdqa %xmm1,%xmm7 @@ -1489,7 +1589,7 @@ L056xts_dec_short: pxor %xmm2,%xmm1 movdqa %xmm5,(%esp) movdqa %xmm6,16(%esp) - je L062xts_dec_four + je L066xts_dec_four movdqa %xmm7,32(%esp) pshufd $19,%xmm0,%xmm7 movdqa %xmm1,48(%esp) @@ -1521,9 +1621,9 @@ L056xts_dec_short: movups %xmm5,48(%edi) movups %xmm6,64(%edi) leal 80(%edi),%edi - jmp L063xts_dec_done + jmp L067xts_dec_done .align 4,0x90 -L059xts_dec_one: +L063xts_dec_one: movups (%esi),%xmm2 leal 16(%esi),%esi xorps %xmm5,%xmm2 @@ -1531,36 +1631,36 @@ L059xts_dec_one: movups 16(%edx),%xmm1 leal 32(%edx),%edx xorps %xmm0,%xmm2 -L064dec1_loop_12: +L068dec1_loop_12: .byte 102,15,56,222,209 decl %ecx movups (%edx),%xmm1 leal 16(%edx),%edx - jnz L064dec1_loop_12 + jnz L068dec1_loop_12 .byte 102,15,56,223,209 xorps %xmm5,%xmm2 movups %xmm2,(%edi) leal 16(%edi),%edi movdqa %xmm5,%xmm1 - jmp L063xts_dec_done + jmp L067xts_dec_done .align 4,0x90 -L060xts_dec_two: +L064xts_dec_two: movaps %xmm1,%xmm6 movups (%esi),%xmm2 movups 16(%esi),%xmm3 leal 32(%esi),%esi xorps %xmm5,%xmm2 xorps %xmm6,%xmm3 - call __aesni_decrypt3 + call __aesni_decrypt2 xorps %xmm5,%xmm2 xorps %xmm6,%xmm3 movups %xmm2,(%edi) movups %xmm3,16(%edi) leal 32(%edi),%edi movdqa %xmm6,%xmm1 - jmp L063xts_dec_done + jmp L067xts_dec_done .align 4,0x90 -L061xts_dec_three: +L065xts_dec_three: movaps %xmm1,%xmm7 movups (%esi),%xmm2 movups 16(%esi),%xmm3 @@ -1578,9 +1678,9 @@ L061xts_dec_three: movups %xmm4,32(%edi) leal 48(%edi),%edi movdqa %xmm7,%xmm1 - jmp L063xts_dec_done + jmp L067xts_dec_done .align 4,0x90 -L062xts_dec_four: +L066xts_dec_four: movaps %xmm1,%xmm6 movups (%esi),%xmm2 movups 16(%esi),%xmm3 @@ -1602,20 +1702,20 @@ L062xts_dec_four: movups %xmm5,48(%edi) leal 64(%edi),%edi movdqa %xmm6,%xmm1 - jmp L063xts_dec_done + jmp L067xts_dec_done .align 4,0x90 -L058xts_dec_done6x: +L062xts_dec_done6x: movl 112(%esp),%eax andl $15,%eax - jz L065xts_dec_ret + jz L069xts_dec_ret movl %eax,112(%esp) - jmp L066xts_dec_only_one_more + jmp L070xts_dec_only_one_more .align 4,0x90 -L063xts_dec_done: +L067xts_dec_done: movl 112(%esp),%eax pxor %xmm0,%xmm0 andl $15,%eax - jz L065xts_dec_ret + jz L069xts_dec_ret pcmpgtd %xmm1,%xmm0 movl %eax,112(%esp) pshufd $19,%xmm0,%xmm2 @@ -1625,7 +1725,7 @@ L063xts_dec_done: pand %xmm3,%xmm2 pcmpgtd %xmm1,%xmm0 pxor %xmm2,%xmm1 -L066xts_dec_only_one_more: +L070xts_dec_only_one_more: pshufd $19,%xmm0,%xmm5 movdqa %xmm1,%xmm6 paddq %xmm1,%xmm1 @@ -1639,16 +1739,16 @@ L066xts_dec_only_one_more: movups 16(%edx),%xmm1 leal 32(%edx),%edx xorps %xmm0,%xmm2 -L067dec1_loop_13: +L071dec1_loop_13: .byte 102,15,56,222,209 decl %ecx movups (%edx),%xmm1 leal 16(%edx),%edx - jnz L067dec1_loop_13 + jnz L071dec1_loop_13 .byte 102,15,56,223,209 xorps %xmm5,%xmm2 movups %xmm2,(%edi) -L068xts_dec_steal: +L072xts_dec_steal: movzbl 16(%esi),%ecx movzbl (%edi),%edx leal 1(%esi),%esi @@ -1656,7 +1756,7 @@ L068xts_dec_steal: movb %dl,16(%edi) leal 1(%edi),%edi subl $1,%eax - jnz L068xts_dec_steal + jnz L072xts_dec_steal subl 112(%esp),%edi movl %ebp,%edx movl %ebx,%ecx @@ -1666,22 +1766,822 @@ L068xts_dec_steal: movups 16(%edx),%xmm1 leal 32(%edx),%edx xorps %xmm0,%xmm2 -L069dec1_loop_14: +L073dec1_loop_14: .byte 102,15,56,222,209 decl %ecx movups (%edx),%xmm1 leal 16(%edx),%edx - jnz L069dec1_loop_14 + jnz L073dec1_loop_14 .byte 102,15,56,223,209 xorps %xmm6,%xmm2 movups %xmm2,(%edi) -L065xts_dec_ret: +L069xts_dec_ret: + pxor %xmm0,%xmm0 + pxor %xmm1,%xmm1 + pxor %xmm2,%xmm2 + movdqa %xmm0,(%esp) + pxor %xmm3,%xmm3 + movdqa %xmm0,16(%esp) + pxor %xmm4,%xmm4 + movdqa %xmm0,32(%esp) + pxor %xmm5,%xmm5 + movdqa %xmm0,48(%esp) + pxor %xmm6,%xmm6 + movdqa %xmm0,64(%esp) + pxor %xmm7,%xmm7 + movdqa %xmm0,80(%esp) movl 116(%esp),%esp popl %edi popl %esi popl %ebx popl %ebp ret +.globl _aesni_ocb_encrypt +.align 4 +_aesni_ocb_encrypt: +L_aesni_ocb_encrypt_begin: + pushl %ebp + pushl %ebx + pushl %esi + pushl %edi + movl 40(%esp),%ecx + movl 48(%esp),%ebx + movl 20(%esp),%esi + movl 24(%esp),%edi + movl 28(%esp),%eax + movl 32(%esp),%edx + movdqu (%ecx),%xmm0 + movl 36(%esp),%ebp + movdqu (%ebx),%xmm1 + movl 44(%esp),%ebx + movl %esp,%ecx + subl $132,%esp + andl $-16,%esp + subl %esi,%edi + shll $4,%eax + leal -96(%esi,%eax,1),%eax + movl %edi,120(%esp) + movl %eax,124(%esp) + movl %ecx,128(%esp) + movl 240(%edx),%ecx + testl $1,%ebp + jnz L074odd + bsfl %ebp,%eax + addl $1,%ebp + shll $4,%eax + movdqu (%ebx,%eax,1),%xmm7 + movl %edx,%eax + movdqu (%esi),%xmm2 + leal 16(%esi),%esi + pxor %xmm0,%xmm7 + pxor %xmm2,%xmm1 + pxor %xmm7,%xmm2 + movdqa %xmm1,%xmm6 + movups (%edx),%xmm0 + movups 16(%edx),%xmm1 + leal 32(%edx),%edx + xorps %xmm0,%xmm2 +L075enc1_loop_15: +.byte 102,15,56,220,209 + decl %ecx + movups (%edx),%xmm1 + leal 16(%edx),%edx + jnz L075enc1_loop_15 +.byte 102,15,56,221,209 + xorps %xmm7,%xmm2 + movdqa %xmm7,%xmm0 + movdqa %xmm6,%xmm1 + movups %xmm2,-16(%edi,%esi,1) + movl 240(%eax),%ecx + movl %eax,%edx + movl 124(%esp),%eax +L074odd: + shll $4,%ecx + movl $16,%edi + subl %ecx,%edi + movl %edx,112(%esp) + leal 32(%edx,%ecx,1),%edx + movl %edi,116(%esp) + cmpl %eax,%esi + ja L076short + jmp L077grandloop +.align 5,0x90 +L077grandloop: + leal 1(%ebp),%ecx + leal 3(%ebp),%eax + leal 5(%ebp),%edi + addl $6,%ebp + bsfl %ecx,%ecx + bsfl %eax,%eax + bsfl %edi,%edi + shll $4,%ecx + shll $4,%eax + shll $4,%edi + movdqu (%ebx),%xmm2 + movdqu (%ebx,%ecx,1),%xmm3 + movl 116(%esp),%ecx + movdqa %xmm2,%xmm4 + movdqu (%ebx,%eax,1),%xmm5 + movdqa %xmm2,%xmm6 + movdqu (%ebx,%edi,1),%xmm7 + pxor %xmm0,%xmm2 + pxor %xmm2,%xmm3 + movdqa %xmm2,(%esp) + pxor %xmm3,%xmm4 + movdqa %xmm3,16(%esp) + pxor %xmm4,%xmm5 + movdqa %xmm4,32(%esp) + pxor %xmm5,%xmm6 + movdqa %xmm5,48(%esp) + pxor %xmm6,%xmm7 + movdqa %xmm6,64(%esp) + movdqa %xmm7,80(%esp) + movups -48(%edx,%ecx,1),%xmm0 + movdqu (%esi),%xmm2 + movdqu 16(%esi),%xmm3 + movdqu 32(%esi),%xmm4 + movdqu 48(%esi),%xmm5 + movdqu 64(%esi),%xmm6 + movdqu 80(%esi),%xmm7 + leal 96(%esi),%esi + pxor %xmm2,%xmm1 + pxor %xmm0,%xmm2 + pxor %xmm3,%xmm1 + pxor %xmm0,%xmm3 + pxor %xmm4,%xmm1 + pxor %xmm0,%xmm4 + pxor %xmm5,%xmm1 + pxor %xmm0,%xmm5 + pxor %xmm6,%xmm1 + pxor %xmm0,%xmm6 + pxor %xmm7,%xmm1 + pxor %xmm0,%xmm7 + movdqa %xmm1,96(%esp) + movups -32(%edx,%ecx,1),%xmm1 + pxor (%esp),%xmm2 + pxor 16(%esp),%xmm3 + pxor 32(%esp),%xmm4 + pxor 48(%esp),%xmm5 + pxor 64(%esp),%xmm6 + pxor 80(%esp),%xmm7 + movups -16(%edx,%ecx,1),%xmm0 +.byte 102,15,56,220,209 +.byte 102,15,56,220,217 +.byte 102,15,56,220,225 +.byte 102,15,56,220,233 +.byte 102,15,56,220,241 +.byte 102,15,56,220,249 + movl 120(%esp),%edi + movl 124(%esp),%eax + call L_aesni_encrypt6_enter + movdqa 80(%esp),%xmm0 + pxor (%esp),%xmm2 + pxor 16(%esp),%xmm3 + pxor 32(%esp),%xmm4 + pxor 48(%esp),%xmm5 + pxor 64(%esp),%xmm6 + pxor %xmm0,%xmm7 + movdqa 96(%esp),%xmm1 + movdqu %xmm2,-96(%edi,%esi,1) + movdqu %xmm3,-80(%edi,%esi,1) + movdqu %xmm4,-64(%edi,%esi,1) + movdqu %xmm5,-48(%edi,%esi,1) + movdqu %xmm6,-32(%edi,%esi,1) + movdqu %xmm7,-16(%edi,%esi,1) + cmpl %eax,%esi + jb L077grandloop +L076short: + addl $96,%eax + subl %esi,%eax + jz L078done + cmpl $32,%eax + jb L079one + je L080two + cmpl $64,%eax + jb L081three + je L082four + leal 1(%ebp),%ecx + leal 3(%ebp),%eax + bsfl %ecx,%ecx + bsfl %eax,%eax + shll $4,%ecx + shll $4,%eax + movdqu (%ebx),%xmm2 + movdqu (%ebx,%ecx,1),%xmm3 + movl 116(%esp),%ecx + movdqa %xmm2,%xmm4 + movdqu (%ebx,%eax,1),%xmm5 + movdqa %xmm2,%xmm6 + pxor %xmm0,%xmm2 + pxor %xmm2,%xmm3 + movdqa %xmm2,(%esp) + pxor %xmm3,%xmm4 + movdqa %xmm3,16(%esp) + pxor %xmm4,%xmm5 + movdqa %xmm4,32(%esp) + pxor %xmm5,%xmm6 + movdqa %xmm5,48(%esp) + pxor %xmm6,%xmm7 + movdqa %xmm6,64(%esp) + movups -48(%edx,%ecx,1),%xmm0 + movdqu (%esi),%xmm2 + movdqu 16(%esi),%xmm3 + movdqu 32(%esi),%xmm4 + movdqu 48(%esi),%xmm5 + movdqu 64(%esi),%xmm6 + pxor %xmm7,%xmm7 + pxor %xmm2,%xmm1 + pxor %xmm0,%xmm2 + pxor %xmm3,%xmm1 + pxor %xmm0,%xmm3 + pxor %xmm4,%xmm1 + pxor %xmm0,%xmm4 + pxor %xmm5,%xmm1 + pxor %xmm0,%xmm5 + pxor %xmm6,%xmm1 + pxor %xmm0,%xmm6 + movdqa %xmm1,96(%esp) + movups -32(%edx,%ecx,1),%xmm1 + pxor (%esp),%xmm2 + pxor 16(%esp),%xmm3 + pxor 32(%esp),%xmm4 + pxor 48(%esp),%xmm5 + pxor 64(%esp),%xmm6 + movups -16(%edx,%ecx,1),%xmm0 +.byte 102,15,56,220,209 +.byte 102,15,56,220,217 +.byte 102,15,56,220,225 +.byte 102,15,56,220,233 +.byte 102,15,56,220,241 +.byte 102,15,56,220,249 + movl 120(%esp),%edi + call L_aesni_encrypt6_enter + movdqa 64(%esp),%xmm0 + pxor (%esp),%xmm2 + pxor 16(%esp),%xmm3 + pxor 32(%esp),%xmm4 + pxor 48(%esp),%xmm5 + pxor %xmm0,%xmm6 + movdqa 96(%esp),%xmm1 + movdqu %xmm2,(%edi,%esi,1) + movdqu %xmm3,16(%edi,%esi,1) + movdqu %xmm4,32(%edi,%esi,1) + movdqu %xmm5,48(%edi,%esi,1) + movdqu %xmm6,64(%edi,%esi,1) + jmp L078done +.align 4,0x90 +L079one: + movdqu (%ebx),%xmm7 + movl 112(%esp),%edx + movdqu (%esi),%xmm2 + movl 240(%edx),%ecx + pxor %xmm0,%xmm7 + pxor %xmm2,%xmm1 + pxor %xmm7,%xmm2 + movdqa %xmm1,%xmm6 + movl 120(%esp),%edi + movups (%edx),%xmm0 + movups 16(%edx),%xmm1 + leal 32(%edx),%edx + xorps %xmm0,%xmm2 +L083enc1_loop_16: +.byte 102,15,56,220,209 + decl %ecx + movups (%edx),%xmm1 + leal 16(%edx),%edx + jnz L083enc1_loop_16 +.byte 102,15,56,221,209 + xorps %xmm7,%xmm2 + movdqa %xmm7,%xmm0 + movdqa %xmm6,%xmm1 + movups %xmm2,(%edi,%esi,1) + jmp L078done +.align 4,0x90 +L080two: + leal 1(%ebp),%ecx + movl 112(%esp),%edx + bsfl %ecx,%ecx + shll $4,%ecx + movdqu (%ebx),%xmm6 + movdqu (%ebx,%ecx,1),%xmm7 + movdqu (%esi),%xmm2 + movdqu 16(%esi),%xmm3 + movl 240(%edx),%ecx + pxor %xmm0,%xmm6 + pxor %xmm6,%xmm7 + pxor %xmm2,%xmm1 + pxor %xmm6,%xmm2 + pxor %xmm3,%xmm1 + pxor %xmm7,%xmm3 + movdqa %xmm1,%xmm5 + movl 120(%esp),%edi + call __aesni_encrypt2 + xorps %xmm6,%xmm2 + xorps %xmm7,%xmm3 + movdqa %xmm7,%xmm0 + movdqa %xmm5,%xmm1 + movups %xmm2,(%edi,%esi,1) + movups %xmm3,16(%edi,%esi,1) + jmp L078done +.align 4,0x90 +L081three: + leal 1(%ebp),%ecx + movl 112(%esp),%edx + bsfl %ecx,%ecx + shll $4,%ecx + movdqu (%ebx),%xmm5 + movdqu (%ebx,%ecx,1),%xmm6 + movdqa %xmm5,%xmm7 + movdqu (%esi),%xmm2 + movdqu 16(%esi),%xmm3 + movdqu 32(%esi),%xmm4 + movl 240(%edx),%ecx + pxor %xmm0,%xmm5 + pxor %xmm5,%xmm6 + pxor %xmm6,%xmm7 + pxor %xmm2,%xmm1 + pxor %xmm5,%xmm2 + pxor %xmm3,%xmm1 + pxor %xmm6,%xmm3 + pxor %xmm4,%xmm1 + pxor %xmm7,%xmm4 + movdqa %xmm1,96(%esp) + movl 120(%esp),%edi + call __aesni_encrypt3 + xorps %xmm5,%xmm2 + xorps %xmm6,%xmm3 + xorps %xmm7,%xmm4 + movdqa %xmm7,%xmm0 + movdqa 96(%esp),%xmm1 + movups %xmm2,(%edi,%esi,1) + movups %xmm3,16(%edi,%esi,1) + movups %xmm4,32(%edi,%esi,1) + jmp L078done +.align 4,0x90 +L082four: + leal 1(%ebp),%ecx + leal 3(%ebp),%eax + bsfl %ecx,%ecx + bsfl %eax,%eax + movl 112(%esp),%edx + shll $4,%ecx + shll $4,%eax + movdqu (%ebx),%xmm4 + movdqu (%ebx,%ecx,1),%xmm5 + movdqa %xmm4,%xmm6 + movdqu (%ebx,%eax,1),%xmm7 + pxor %xmm0,%xmm4 + movdqu (%esi),%xmm2 + pxor %xmm4,%xmm5 + movdqu 16(%esi),%xmm3 + pxor %xmm5,%xmm6 + movdqa %xmm4,(%esp) + pxor %xmm6,%xmm7 + movdqa %xmm5,16(%esp) + movdqu 32(%esi),%xmm4 + movdqu 48(%esi),%xmm5 + movl 240(%edx),%ecx + pxor %xmm2,%xmm1 + pxor (%esp),%xmm2 + pxor %xmm3,%xmm1 + pxor 16(%esp),%xmm3 + pxor %xmm4,%xmm1 + pxor %xmm6,%xmm4 + pxor %xmm5,%xmm1 + pxor %xmm7,%xmm5 + movdqa %xmm1,96(%esp) + movl 120(%esp),%edi + call __aesni_encrypt4 + xorps (%esp),%xmm2 + xorps 16(%esp),%xmm3 + xorps %xmm6,%xmm4 + movups %xmm2,(%edi,%esi,1) + xorps %xmm7,%xmm5 + movups %xmm3,16(%edi,%esi,1) + movdqa %xmm7,%xmm0 + movups %xmm4,32(%edi,%esi,1) + movdqa 96(%esp),%xmm1 + movups %xmm5,48(%edi,%esi,1) +L078done: + movl 128(%esp),%edx + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + movdqa %xmm2,(%esp) + pxor %xmm4,%xmm4 + movdqa %xmm2,16(%esp) + pxor %xmm5,%xmm5 + movdqa %xmm2,32(%esp) + pxor %xmm6,%xmm6 + movdqa %xmm2,48(%esp) + pxor %xmm7,%xmm7 + movdqa %xmm2,64(%esp) + movdqa %xmm2,80(%esp) + movdqa %xmm2,96(%esp) + leal (%edx),%esp + movl 40(%esp),%ecx + movl 48(%esp),%ebx + movdqu %xmm0,(%ecx) + pxor %xmm0,%xmm0 + movdqu %xmm1,(%ebx) + pxor %xmm1,%xmm1 + popl %edi + popl %esi + popl %ebx + popl %ebp + ret +.globl _aesni_ocb_decrypt +.align 4 +_aesni_ocb_decrypt: +L_aesni_ocb_decrypt_begin: + pushl %ebp + pushl %ebx + pushl %esi + pushl %edi + movl 40(%esp),%ecx + movl 48(%esp),%ebx + movl 20(%esp),%esi + movl 24(%esp),%edi + movl 28(%esp),%eax + movl 32(%esp),%edx + movdqu (%ecx),%xmm0 + movl 36(%esp),%ebp + movdqu (%ebx),%xmm1 + movl 44(%esp),%ebx + movl %esp,%ecx + subl $132,%esp + andl $-16,%esp + subl %esi,%edi + shll $4,%eax + leal -96(%esi,%eax,1),%eax + movl %edi,120(%esp) + movl %eax,124(%esp) + movl %ecx,128(%esp) + movl 240(%edx),%ecx + testl $1,%ebp + jnz L084odd + bsfl %ebp,%eax + addl $1,%ebp + shll $4,%eax + movdqu (%ebx,%eax,1),%xmm7 + movl %edx,%eax + movdqu (%esi),%xmm2 + leal 16(%esi),%esi + pxor %xmm0,%xmm7 + pxor %xmm7,%xmm2 + movdqa %xmm1,%xmm6 + movups (%edx),%xmm0 + movups 16(%edx),%xmm1 + leal 32(%edx),%edx + xorps %xmm0,%xmm2 +L085dec1_loop_17: +.byte 102,15,56,222,209 + decl %ecx + movups (%edx),%xmm1 + leal 16(%edx),%edx + jnz L085dec1_loop_17 +.byte 102,15,56,223,209 + xorps %xmm7,%xmm2 + movaps %xmm6,%xmm1 + movdqa %xmm7,%xmm0 + xorps %xmm2,%xmm1 + movups %xmm2,-16(%edi,%esi,1) + movl 240(%eax),%ecx + movl %eax,%edx + movl 124(%esp),%eax +L084odd: + shll $4,%ecx + movl $16,%edi + subl %ecx,%edi + movl %edx,112(%esp) + leal 32(%edx,%ecx,1),%edx + movl %edi,116(%esp) + cmpl %eax,%esi + ja L086short + jmp L087grandloop +.align 5,0x90 +L087grandloop: + leal 1(%ebp),%ecx + leal 3(%ebp),%eax + leal 5(%ebp),%edi + addl $6,%ebp + bsfl %ecx,%ecx + bsfl %eax,%eax + bsfl %edi,%edi + shll $4,%ecx + shll $4,%eax + shll $4,%edi + movdqu (%ebx),%xmm2 + movdqu (%ebx,%ecx,1),%xmm3 + movl 116(%esp),%ecx + movdqa %xmm2,%xmm4 + movdqu (%ebx,%eax,1),%xmm5 + movdqa %xmm2,%xmm6 + movdqu (%ebx,%edi,1),%xmm7 + pxor %xmm0,%xmm2 + pxor %xmm2,%xmm3 + movdqa %xmm2,(%esp) + pxor %xmm3,%xmm4 + movdqa %xmm3,16(%esp) + pxor %xmm4,%xmm5 + movdqa %xmm4,32(%esp) + pxor %xmm5,%xmm6 + movdqa %xmm5,48(%esp) + pxor %xmm6,%xmm7 + movdqa %xmm6,64(%esp) + movdqa %xmm7,80(%esp) + movups -48(%edx,%ecx,1),%xmm0 + movdqu (%esi),%xmm2 + movdqu 16(%esi),%xmm3 + movdqu 32(%esi),%xmm4 + movdqu 48(%esi),%xmm5 + movdqu 64(%esi),%xmm6 + movdqu 80(%esi),%xmm7 + leal 96(%esi),%esi + movdqa %xmm1,96(%esp) + pxor %xmm0,%xmm2 + pxor %xmm0,%xmm3 + pxor %xmm0,%xmm4 + pxor %xmm0,%xmm5 + pxor %xmm0,%xmm6 + pxor %xmm0,%xmm7 + movups -32(%edx,%ecx,1),%xmm1 + pxor (%esp),%xmm2 + pxor 16(%esp),%xmm3 + pxor 32(%esp),%xmm4 + pxor 48(%esp),%xmm5 + pxor 64(%esp),%xmm6 + pxor 80(%esp),%xmm7 + movups -16(%edx,%ecx,1),%xmm0 +.byte 102,15,56,222,209 +.byte 102,15,56,222,217 +.byte 102,15,56,222,225 +.byte 102,15,56,222,233 +.byte 102,15,56,222,241 +.byte 102,15,56,222,249 + movl 120(%esp),%edi + movl 124(%esp),%eax + call L_aesni_decrypt6_enter + movdqa 80(%esp),%xmm0 + pxor (%esp),%xmm2 + movdqa 96(%esp),%xmm1 + pxor 16(%esp),%xmm3 + pxor 32(%esp),%xmm4 + pxor 48(%esp),%xmm5 + pxor 64(%esp),%xmm6 + pxor %xmm0,%xmm7 + pxor %xmm2,%xmm1 + movdqu %xmm2,-96(%edi,%esi,1) + pxor %xmm3,%xmm1 + movdqu %xmm3,-80(%edi,%esi,1) + pxor %xmm4,%xmm1 + movdqu %xmm4,-64(%edi,%esi,1) + pxor %xmm5,%xmm1 + movdqu %xmm5,-48(%edi,%esi,1) + pxor %xmm6,%xmm1 + movdqu %xmm6,-32(%edi,%esi,1) + pxor %xmm7,%xmm1 + movdqu %xmm7,-16(%edi,%esi,1) + cmpl %eax,%esi + jb L087grandloop +L086short: + addl $96,%eax + subl %esi,%eax + jz L088done + cmpl $32,%eax + jb L089one + je L090two + cmpl $64,%eax + jb L091three + je L092four + leal 1(%ebp),%ecx + leal 3(%ebp),%eax + bsfl %ecx,%ecx + bsfl %eax,%eax + shll $4,%ecx + shll $4,%eax + movdqu (%ebx),%xmm2 + movdqu (%ebx,%ecx,1),%xmm3 + movl 116(%esp),%ecx + movdqa %xmm2,%xmm4 + movdqu (%ebx,%eax,1),%xmm5 + movdqa %xmm2,%xmm6 + pxor %xmm0,%xmm2 + pxor %xmm2,%xmm3 + movdqa %xmm2,(%esp) + pxor %xmm3,%xmm4 + movdqa %xmm3,16(%esp) + pxor %xmm4,%xmm5 + movdqa %xmm4,32(%esp) + pxor %xmm5,%xmm6 + movdqa %xmm5,48(%esp) + pxor %xmm6,%xmm7 + movdqa %xmm6,64(%esp) + movups -48(%edx,%ecx,1),%xmm0 + movdqu (%esi),%xmm2 + movdqu 16(%esi),%xmm3 + movdqu 32(%esi),%xmm4 + movdqu 48(%esi),%xmm5 + movdqu 64(%esi),%xmm6 + pxor %xmm7,%xmm7 + movdqa %xmm1,96(%esp) + pxor %xmm0,%xmm2 + pxor %xmm0,%xmm3 + pxor %xmm0,%xmm4 + pxor %xmm0,%xmm5 + pxor %xmm0,%xmm6 + movups -32(%edx,%ecx,1),%xmm1 + pxor (%esp),%xmm2 + pxor 16(%esp),%xmm3 + pxor 32(%esp),%xmm4 + pxor 48(%esp),%xmm5 + pxor 64(%esp),%xmm6 + movups -16(%edx,%ecx,1),%xmm0 +.byte 102,15,56,222,209 +.byte 102,15,56,222,217 +.byte 102,15,56,222,225 +.byte 102,15,56,222,233 +.byte 102,15,56,222,241 +.byte 102,15,56,222,249 + movl 120(%esp),%edi + call L_aesni_decrypt6_enter + movdqa 64(%esp),%xmm0 + pxor (%esp),%xmm2 + movdqa 96(%esp),%xmm1 + pxor 16(%esp),%xmm3 + pxor 32(%esp),%xmm4 + pxor 48(%esp),%xmm5 + pxor %xmm0,%xmm6 + pxor %xmm2,%xmm1 + movdqu %xmm2,(%edi,%esi,1) + pxor %xmm3,%xmm1 + movdqu %xmm3,16(%edi,%esi,1) + pxor %xmm4,%xmm1 + movdqu %xmm4,32(%edi,%esi,1) + pxor %xmm5,%xmm1 + movdqu %xmm5,48(%edi,%esi,1) + pxor %xmm6,%xmm1 + movdqu %xmm6,64(%edi,%esi,1) + jmp L088done +.align 4,0x90 +L089one: + movdqu (%ebx),%xmm7 + movl 112(%esp),%edx + movdqu (%esi),%xmm2 + movl 240(%edx),%ecx + pxor %xmm0,%xmm7 + pxor %xmm7,%xmm2 + movdqa %xmm1,%xmm6 + movl 120(%esp),%edi + movups (%edx),%xmm0 + movups 16(%edx),%xmm1 + leal 32(%edx),%edx + xorps %xmm0,%xmm2 +L093dec1_loop_18: +.byte 102,15,56,222,209 + decl %ecx + movups (%edx),%xmm1 + leal 16(%edx),%edx + jnz L093dec1_loop_18 +.byte 102,15,56,223,209 + xorps %xmm7,%xmm2 + movaps %xmm6,%xmm1 + movdqa %xmm7,%xmm0 + xorps %xmm2,%xmm1 + movups %xmm2,(%edi,%esi,1) + jmp L088done +.align 4,0x90 +L090two: + leal 1(%ebp),%ecx + movl 112(%esp),%edx + bsfl %ecx,%ecx + shll $4,%ecx + movdqu (%ebx),%xmm6 + movdqu (%ebx,%ecx,1),%xmm7 + movdqu (%esi),%xmm2 + movdqu 16(%esi),%xmm3 + movl 240(%edx),%ecx + movdqa %xmm1,%xmm5 + pxor %xmm0,%xmm6 + pxor %xmm6,%xmm7 + pxor %xmm6,%xmm2 + pxor %xmm7,%xmm3 + movl 120(%esp),%edi + call __aesni_decrypt2 + xorps %xmm6,%xmm2 + xorps %xmm7,%xmm3 + movdqa %xmm7,%xmm0 + xorps %xmm2,%xmm5 + movups %xmm2,(%edi,%esi,1) + xorps %xmm3,%xmm5 + movups %xmm3,16(%edi,%esi,1) + movaps %xmm5,%xmm1 + jmp L088done +.align 4,0x90 +L091three: + leal 1(%ebp),%ecx + movl 112(%esp),%edx + bsfl %ecx,%ecx + shll $4,%ecx + movdqu (%ebx),%xmm5 + movdqu (%ebx,%ecx,1),%xmm6 + movdqa %xmm5,%xmm7 + movdqu (%esi),%xmm2 + movdqu 16(%esi),%xmm3 + movdqu 32(%esi),%xmm4 + movl 240(%edx),%ecx + movdqa %xmm1,96(%esp) + pxor %xmm0,%xmm5 + pxor %xmm5,%xmm6 + pxor %xmm6,%xmm7 + pxor %xmm5,%xmm2 + pxor %xmm6,%xmm3 + pxor %xmm7,%xmm4 + movl 120(%esp),%edi + call __aesni_decrypt3 + movdqa 96(%esp),%xmm1 + xorps %xmm5,%xmm2 + xorps %xmm6,%xmm3 + xorps %xmm7,%xmm4 + movups %xmm2,(%edi,%esi,1) + pxor %xmm2,%xmm1 + movdqa %xmm7,%xmm0 + movups %xmm3,16(%edi,%esi,1) + pxor %xmm3,%xmm1 + movups %xmm4,32(%edi,%esi,1) + pxor %xmm4,%xmm1 + jmp L088done +.align 4,0x90 +L092four: + leal 1(%ebp),%ecx + leal 3(%ebp),%eax + bsfl %ecx,%ecx + bsfl %eax,%eax + movl 112(%esp),%edx + shll $4,%ecx + shll $4,%eax + movdqu (%ebx),%xmm4 + movdqu (%ebx,%ecx,1),%xmm5 + movdqa %xmm4,%xmm6 + movdqu (%ebx,%eax,1),%xmm7 + pxor %xmm0,%xmm4 + movdqu (%esi),%xmm2 + pxor %xmm4,%xmm5 + movdqu 16(%esi),%xmm3 + pxor %xmm5,%xmm6 + movdqa %xmm4,(%esp) + pxor %xmm6,%xmm7 + movdqa %xmm5,16(%esp) + movdqu 32(%esi),%xmm4 + movdqu 48(%esi),%xmm5 + movl 240(%edx),%ecx + movdqa %xmm1,96(%esp) + pxor (%esp),%xmm2 + pxor 16(%esp),%xmm3 + pxor %xmm6,%xmm4 + pxor %xmm7,%xmm5 + movl 120(%esp),%edi + call __aesni_decrypt4 + movdqa 96(%esp),%xmm1 + xorps (%esp),%xmm2 + xorps 16(%esp),%xmm3 + xorps %xmm6,%xmm4 + movups %xmm2,(%edi,%esi,1) + pxor %xmm2,%xmm1 + xorps %xmm7,%xmm5 + movups %xmm3,16(%edi,%esi,1) + pxor %xmm3,%xmm1 + movdqa %xmm7,%xmm0 + movups %xmm4,32(%edi,%esi,1) + pxor %xmm4,%xmm1 + movups %xmm5,48(%edi,%esi,1) + pxor %xmm5,%xmm1 +L088done: + movl 128(%esp),%edx + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + movdqa %xmm2,(%esp) + pxor %xmm4,%xmm4 + movdqa %xmm2,16(%esp) + pxor %xmm5,%xmm5 + movdqa %xmm2,32(%esp) + pxor %xmm6,%xmm6 + movdqa %xmm2,48(%esp) + pxor %xmm7,%xmm7 + movdqa %xmm2,64(%esp) + movdqa %xmm2,80(%esp) + movdqa %xmm2,96(%esp) + leal (%edx),%esp + movl 40(%esp),%ecx + movl 48(%esp),%ebx + movdqu %xmm0,(%ecx) + pxor %xmm0,%xmm0 + movdqu %xmm1,(%ebx) + pxor %xmm1,%xmm1 + popl %edi + popl %esi + popl %ebx + popl %ebp + ret .globl _aesni_cbc_encrypt .align 4 _aesni_cbc_encrypt: @@ -1699,7 +2599,7 @@ L_aesni_cbc_encrypt_begin: movl 32(%esp),%edx movl 36(%esp),%ebp testl %eax,%eax - jz L070cbc_abort + jz L094cbc_abort cmpl $0,40(%esp) xchgl %esp,%ebx movups (%ebp),%xmm7 @@ -1707,14 +2607,14 @@ L_aesni_cbc_encrypt_begin: movl %edx,%ebp movl %ebx,16(%esp) movl %ecx,%ebx - je L071cbc_decrypt + je L095cbc_decrypt movaps %xmm7,%xmm2 cmpl $16,%eax - jb L072cbc_enc_tail + jb L096cbc_enc_tail subl $16,%eax - jmp L073cbc_enc_loop + jmp L097cbc_enc_loop .align 4,0x90 -L073cbc_enc_loop: +L097cbc_enc_loop: movups (%esi),%xmm7 leal 16(%esi),%esi movups (%edx),%xmm0 @@ -1722,24 +2622,25 @@ L073cbc_enc_loop: xorps %xmm0,%xmm7 leal 32(%edx),%edx xorps %xmm7,%xmm2 -L074enc1_loop_15: +L098enc1_loop_19: .byte 102,15,56,220,209 decl %ecx movups (%edx),%xmm1 leal 16(%edx),%edx - jnz L074enc1_loop_15 + jnz L098enc1_loop_19 .byte 102,15,56,221,209 movl %ebx,%ecx movl %ebp,%edx movups %xmm2,(%edi) leal 16(%edi),%edi subl $16,%eax - jnc L073cbc_enc_loop + jnc L097cbc_enc_loop addl $16,%eax - jnz L072cbc_enc_tail + jnz L096cbc_enc_tail movaps %xmm2,%xmm7 - jmp L075cbc_ret -L072cbc_enc_tail: + pxor %xmm2,%xmm2 + jmp L099cbc_ret +L096cbc_enc_tail: movl %eax,%ecx .long 2767451785 movl $16,%ecx @@ -1750,20 +2651,20 @@ L072cbc_enc_tail: movl %ebx,%ecx movl %edi,%esi movl %ebp,%edx - jmp L073cbc_enc_loop + jmp L097cbc_enc_loop .align 4,0x90 -L071cbc_decrypt: +L095cbc_decrypt: cmpl $80,%eax - jbe L076cbc_dec_tail + jbe L100cbc_dec_tail movaps %xmm7,(%esp) subl $80,%eax - jmp L077cbc_dec_loop6_enter + jmp L101cbc_dec_loop6_enter .align 4,0x90 -L078cbc_dec_loop6: +L102cbc_dec_loop6: movaps %xmm0,(%esp) movups %xmm7,(%edi) leal 16(%edi),%edi -L077cbc_dec_loop6_enter: +L101cbc_dec_loop6_enter: movdqu (%esi),%xmm2 movdqu 16(%esi),%xmm3 movdqu 32(%esi),%xmm4 @@ -1793,28 +2694,28 @@ L077cbc_dec_loop6_enter: movups %xmm6,64(%edi) leal 80(%edi),%edi subl $96,%eax - ja L078cbc_dec_loop6 + ja L102cbc_dec_loop6 movaps %xmm7,%xmm2 movaps %xmm0,%xmm7 addl $80,%eax - jle L079cbc_dec_tail_collected + jle L103cbc_dec_clear_tail_collected movups %xmm2,(%edi) leal 16(%edi),%edi -L076cbc_dec_tail: +L100cbc_dec_tail: movups (%esi),%xmm2 movaps %xmm2,%xmm6 cmpl $16,%eax - jbe L080cbc_dec_one + jbe L104cbc_dec_one movups 16(%esi),%xmm3 movaps %xmm3,%xmm5 cmpl $32,%eax - jbe L081cbc_dec_two + jbe L105cbc_dec_two movups 32(%esi),%xmm4 cmpl $48,%eax - jbe L082cbc_dec_three + jbe L106cbc_dec_three movups 48(%esi),%xmm5 cmpl $64,%eax - jbe L083cbc_dec_four + jbe L107cbc_dec_four movups 64(%esi),%xmm6 movaps %xmm7,(%esp) movups (%esi),%xmm2 @@ -1832,56 +2733,62 @@ L076cbc_dec_tail: xorps %xmm0,%xmm6 movups %xmm2,(%edi) movups %xmm3,16(%edi) + pxor %xmm3,%xmm3 movups %xmm4,32(%edi) + pxor %xmm4,%xmm4 movups %xmm5,48(%edi) + pxor %xmm5,%xmm5 leal 64(%edi),%edi movaps %xmm6,%xmm2 + pxor %xmm6,%xmm6 subl $80,%eax - jmp L079cbc_dec_tail_collected + jmp L108cbc_dec_tail_collected .align 4,0x90 -L080cbc_dec_one: +L104cbc_dec_one: movups (%edx),%xmm0 movups 16(%edx),%xmm1 leal 32(%edx),%edx xorps %xmm0,%xmm2 -L084dec1_loop_16: +L109dec1_loop_20: .byte 102,15,56,222,209 decl %ecx movups (%edx),%xmm1 leal 16(%edx),%edx - jnz L084dec1_loop_16 + jnz L109dec1_loop_20 .byte 102,15,56,223,209 xorps %xmm7,%xmm2 movaps %xmm6,%xmm7 subl $16,%eax - jmp L079cbc_dec_tail_collected + jmp L108cbc_dec_tail_collected .align 4,0x90 -L081cbc_dec_two: - xorps %xmm4,%xmm4 - call __aesni_decrypt3 +L105cbc_dec_two: + call __aesni_decrypt2 xorps %xmm7,%xmm2 xorps %xmm6,%xmm3 movups %xmm2,(%edi) movaps %xmm3,%xmm2 + pxor %xmm3,%xmm3 leal 16(%edi),%edi movaps %xmm5,%xmm7 subl $32,%eax - jmp L079cbc_dec_tail_collected + jmp L108cbc_dec_tail_collected .align 4,0x90 -L082cbc_dec_three: +L106cbc_dec_three: call __aesni_decrypt3 xorps %xmm7,%xmm2 xorps %xmm6,%xmm3 xorps %xmm5,%xmm4 movups %xmm2,(%edi) movaps %xmm4,%xmm2 + pxor %xmm4,%xmm4 movups %xmm3,16(%edi) + pxor %xmm3,%xmm3 leal 32(%edi),%edi movups 32(%esi),%xmm7 subl $48,%eax - jmp L079cbc_dec_tail_collected + jmp L108cbc_dec_tail_collected .align 4,0x90 -L083cbc_dec_four: +L107cbc_dec_four: call __aesni_decrypt4 movups 16(%esi),%xmm1 movups 32(%esi),%xmm0 @@ -1891,28 +2798,44 @@ L083cbc_dec_four: movups %xmm2,(%edi) xorps %xmm1,%xmm4 movups %xmm3,16(%edi) + pxor %xmm3,%xmm3 xorps %xmm0,%xmm5 movups %xmm4,32(%edi) + pxor %xmm4,%xmm4 leal 48(%edi),%edi movaps %xmm5,%xmm2 + pxor %xmm5,%xmm5 subl $64,%eax -L079cbc_dec_tail_collected: + jmp L108cbc_dec_tail_collected +.align 4,0x90 +L103cbc_dec_clear_tail_collected: + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 + pxor %xmm6,%xmm6 +L108cbc_dec_tail_collected: andl $15,%eax - jnz L085cbc_dec_tail_partial + jnz L110cbc_dec_tail_partial movups %xmm2,(%edi) - jmp L075cbc_ret + pxor %xmm0,%xmm0 + jmp L099cbc_ret .align 4,0x90 -L085cbc_dec_tail_partial: +L110cbc_dec_tail_partial: movaps %xmm2,(%esp) + pxor %xmm0,%xmm0 movl $16,%ecx movl %esp,%esi subl %eax,%ecx .long 2767451785 -L075cbc_ret: + movdqa %xmm2,(%esp) +L099cbc_ret: movl 16(%esp),%esp movl 36(%esp),%ebp + pxor %xmm2,%xmm2 + pxor %xmm1,%xmm1 movups %xmm7,(%ebp) -L070cbc_abort: + pxor %xmm7,%xmm7 +L094cbc_abort: popl %edi popl %esi popl %ebx @@ -1920,52 +2843,62 @@ L070cbc_abort: ret .align 4 __aesni_set_encrypt_key: + pushl %ebp + pushl %ebx testl %eax,%eax - jz L086bad_pointer + jz L111bad_pointer testl %edx,%edx - jz L086bad_pointer + jz L111bad_pointer + call L112pic +L112pic: + popl %ebx + leal Lkey_const-L112pic(%ebx),%ebx + movl L__gnutls_x86_cpuid_s$non_lazy_ptr-Lkey_const(%ebx),%ebp movups (%eax),%xmm0 xorps %xmm4,%xmm4 + movl 4(%ebp),%ebp leal 16(%edx),%edx + andl $268437504,%ebp cmpl $256,%ecx - je L08714rounds + je L11314rounds cmpl $192,%ecx - je L08812rounds + je L11412rounds cmpl $128,%ecx - jne L089bad_keybits + jne L115bad_keybits .align 4,0x90 -L09010rounds: +L11610rounds: + cmpl $268435456,%ebp + je L11710rounds_alt movl $9,%ecx movups %xmm0,-16(%edx) .byte 102,15,58,223,200,1 - call L091key_128_cold + call L118key_128_cold .byte 102,15,58,223,200,2 - call L092key_128 + call L119key_128 .byte 102,15,58,223,200,4 - call L092key_128 + call L119key_128 .byte 102,15,58,223,200,8 - call L092key_128 + call L119key_128 .byte 102,15,58,223,200,16 - call L092key_128 + call L119key_128 .byte 102,15,58,223,200,32 - call L092key_128 + call L119key_128 .byte 102,15,58,223,200,64 - call L092key_128 + call L119key_128 .byte 102,15,58,223,200,128 - call L092key_128 + call L119key_128 .byte 102,15,58,223,200,27 - call L092key_128 + call L119key_128 .byte 102,15,58,223,200,54 - call L092key_128 + call L119key_128 movups %xmm0,(%edx) movl %ecx,80(%edx) - xorl %eax,%eax - ret + jmp L120good_key .align 4,0x90 -L092key_128: +L119key_128: movups %xmm0,(%edx) leal 16(%edx),%edx -L091key_128_cold: +L118key_128_cold: shufps $16,%xmm0,%xmm4 xorps %xmm4,%xmm0 shufps $140,%xmm0,%xmm4 @@ -1974,38 +2907,91 @@ L091key_128_cold: xorps %xmm1,%xmm0 ret .align 4,0x90 -L08812rounds: +L11710rounds_alt: + movdqa (%ebx),%xmm5 + movl $8,%ecx + movdqa 32(%ebx),%xmm4 + movdqa %xmm0,%xmm2 + movdqu %xmm0,-16(%edx) +L121loop_key128: +.byte 102,15,56,0,197 +.byte 102,15,56,221,196 + pslld $1,%xmm4 + leal 16(%edx),%edx + movdqa %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm3,%xmm2 + pxor %xmm2,%xmm0 + movdqu %xmm0,-16(%edx) + movdqa %xmm0,%xmm2 + decl %ecx + jnz L121loop_key128 + movdqa 48(%ebx),%xmm4 +.byte 102,15,56,0,197 +.byte 102,15,56,221,196 + pslld $1,%xmm4 + movdqa %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm3,%xmm2 + pxor %xmm2,%xmm0 + movdqu %xmm0,(%edx) + movdqa %xmm0,%xmm2 +.byte 102,15,56,0,197 +.byte 102,15,56,221,196 + movdqa %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm2,%xmm3 + pslldq $4,%xmm2 + pxor %xmm3,%xmm2 + pxor %xmm2,%xmm0 + movdqu %xmm0,16(%edx) + movl $9,%ecx + movl %ecx,96(%edx) + jmp L120good_key +.align 4,0x90 +L11412rounds: movq 16(%eax),%xmm2 + cmpl $268435456,%ebp + je L12212rounds_alt movl $11,%ecx movups %xmm0,-16(%edx) .byte 102,15,58,223,202,1 - call L093key_192a_cold + call L123key_192a_cold .byte 102,15,58,223,202,2 - call L094key_192b + call L124key_192b .byte 102,15,58,223,202,4 - call L095key_192a + call L125key_192a .byte 102,15,58,223,202,8 - call L094key_192b + call L124key_192b .byte 102,15,58,223,202,16 - call L095key_192a + call L125key_192a .byte 102,15,58,223,202,32 - call L094key_192b + call L124key_192b .byte 102,15,58,223,202,64 - call L095key_192a + call L125key_192a .byte 102,15,58,223,202,128 - call L094key_192b + call L124key_192b movups %xmm0,(%edx) movl %ecx,48(%edx) - xorl %eax,%eax - ret + jmp L120good_key .align 4,0x90 -L095key_192a: +L125key_192a: movups %xmm0,(%edx) leal 16(%edx),%edx .align 4,0x90 -L093key_192a_cold: +L123key_192a_cold: movaps %xmm2,%xmm5 -L096key_192b_warm: +L126key_192b_warm: shufps $16,%xmm0,%xmm4 movdqa %xmm2,%xmm3 xorps %xmm4,%xmm0 @@ -2019,56 +3005,90 @@ L096key_192b_warm: pxor %xmm3,%xmm2 ret .align 4,0x90 -L094key_192b: +L124key_192b: movaps %xmm0,%xmm3 shufps $68,%xmm0,%xmm5 movups %xmm5,(%edx) shufps $78,%xmm2,%xmm3 movups %xmm3,16(%edx) leal 32(%edx),%edx - jmp L096key_192b_warm + jmp L126key_192b_warm +.align 4,0x90 +L12212rounds_alt: + movdqa 16(%ebx),%xmm5 + movdqa 32(%ebx),%xmm4 + movl $8,%ecx + movdqu %xmm0,-16(%edx) +L127loop_key192: + movq %xmm2,(%edx) + movdqa %xmm2,%xmm1 +.byte 102,15,56,0,213 +.byte 102,15,56,221,212 + pslld $1,%xmm4 + leal 24(%edx),%edx + movdqa %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm3,%xmm0 + pshufd $255,%xmm0,%xmm3 + pxor %xmm1,%xmm3 + pslldq $4,%xmm1 + pxor %xmm1,%xmm3 + pxor %xmm2,%xmm0 + pxor %xmm3,%xmm2 + movdqu %xmm0,-16(%edx) + decl %ecx + jnz L127loop_key192 + movl $11,%ecx + movl %ecx,32(%edx) + jmp L120good_key .align 4,0x90 -L08714rounds: +L11314rounds: movups 16(%eax),%xmm2 - movl $13,%ecx leal 16(%edx),%edx + cmpl $268435456,%ebp + je L12814rounds_alt + movl $13,%ecx movups %xmm0,-32(%edx) movups %xmm2,-16(%edx) .byte 102,15,58,223,202,1 - call L097key_256a_cold + call L129key_256a_cold .byte 102,15,58,223,200,1 - call L098key_256b + call L130key_256b .byte 102,15,58,223,202,2 - call L099key_256a + call L131key_256a .byte 102,15,58,223,200,2 - call L098key_256b + call L130key_256b .byte 102,15,58,223,202,4 - call L099key_256a + call L131key_256a .byte 102,15,58,223,200,4 - call L098key_256b + call L130key_256b .byte 102,15,58,223,202,8 - call L099key_256a + call L131key_256a .byte 102,15,58,223,200,8 - call L098key_256b + call L130key_256b .byte 102,15,58,223,202,16 - call L099key_256a + call L131key_256a .byte 102,15,58,223,200,16 - call L098key_256b + call L130key_256b .byte 102,15,58,223,202,32 - call L099key_256a + call L131key_256a .byte 102,15,58,223,200,32 - call L098key_256b + call L130key_256b .byte 102,15,58,223,202,64 - call L099key_256a + call L131key_256a movups %xmm0,(%edx) movl %ecx,16(%edx) xorl %eax,%eax - ret + jmp L120good_key .align 4,0x90 -L099key_256a: +L131key_256a: movups %xmm2,(%edx) leal 16(%edx),%edx -L097key_256a_cold: +L129key_256a_cold: shufps $16,%xmm0,%xmm4 xorps %xmm4,%xmm0 shufps $140,%xmm0,%xmm4 @@ -2077,7 +3097,7 @@ L097key_256a_cold: xorps %xmm1,%xmm0 ret .align 4,0x90 -L098key_256b: +L130key_256b: movups %xmm0,(%edx) leal 16(%edx),%edx shufps $16,%xmm2,%xmm4 @@ -2087,13 +3107,70 @@ L098key_256b: shufps $170,%xmm1,%xmm1 xorps %xmm1,%xmm2 ret +.align 4,0x90 +L12814rounds_alt: + movdqa (%ebx),%xmm5 + movdqa 32(%ebx),%xmm4 + movl $7,%ecx + movdqu %xmm0,-32(%edx) + movdqa %xmm2,%xmm1 + movdqu %xmm2,-16(%edx) +L132loop_key256: +.byte 102,15,56,0,213 +.byte 102,15,56,221,212 + movdqa %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm0,%xmm3 + pslldq $4,%xmm0 + pxor %xmm3,%xmm0 + pslld $1,%xmm4 + pxor %xmm2,%xmm0 + movdqu %xmm0,(%edx) + decl %ecx + jz L133done_key256 + pshufd $255,%xmm0,%xmm2 + pxor %xmm3,%xmm3 +.byte 102,15,56,221,211 + movdqa %xmm1,%xmm3 + pslldq $4,%xmm1 + pxor %xmm1,%xmm3 + pslldq $4,%xmm1 + pxor %xmm1,%xmm3 + pslldq $4,%xmm1 + pxor %xmm3,%xmm1 + pxor %xmm1,%xmm2 + movdqu %xmm2,16(%edx) + leal 32(%edx),%edx + movdqa %xmm2,%xmm1 + jmp L132loop_key256 +L133done_key256: + movl $13,%ecx + movl %ecx,16(%edx) +L120good_key: + pxor %xmm0,%xmm0 + pxor %xmm1,%xmm1 + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 + xorl %eax,%eax + popl %ebx + popl %ebp + ret .align 2,0x90 -L086bad_pointer: +L111bad_pointer: movl $-1,%eax + popl %ebx + popl %ebp ret .align 2,0x90 -L089bad_keybits: +L115bad_keybits: + pxor %xmm0,%xmm0 movl $-2,%eax + popl %ebx + popl %ebp ret .globl _aesni_set_encrypt_key .align 4 @@ -2115,7 +3192,7 @@ L_aesni_set_decrypt_key_begin: movl 12(%esp),%edx shll $4,%ecx testl %eax,%eax - jnz L100dec_key_ret + jnz L134dec_key_ret leal 16(%edx,%ecx,1),%eax movups (%edx),%xmm0 movups (%eax),%xmm1 @@ -2123,7 +3200,7 @@ L_aesni_set_decrypt_key_begin: movups %xmm1,(%edx) leal 16(%edx),%edx leal -16(%eax),%eax -L101dec_key_inverse: +L135dec_key_inverse: movups (%edx),%xmm0 movups (%eax),%xmm1 .byte 102,15,56,219,192 @@ -2133,15 +3210,28 @@ L101dec_key_inverse: movups %xmm0,16(%eax) movups %xmm1,-16(%edx) cmpl %edx,%eax - ja L101dec_key_inverse + ja L135dec_key_inverse movups (%edx),%xmm0 .byte 102,15,56,219,192 movups %xmm0,(%edx) + pxor %xmm0,%xmm0 + pxor %xmm1,%xmm1 xorl %eax,%eax -L100dec_key_ret: +L134dec_key_ret: ret +.align 6,0x90 +Lkey_const: +.long 202313229,202313229,202313229,202313229 +.long 67569157,67569157,67569157,67569157 +.long 1,1,1,1 +.long 27,27,27,27 .byte 65,69,83,32,102,111,114,32,73,110,116,101,108,32,65,69 .byte 83,45,78,73,44,32,67,82,89,80,84,79,71,65,77,83 .byte 32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115 .byte 115,108,46,111,114,103,62,0 +.section __IMPORT,__pointers,non_lazy_symbol_pointers +L__gnutls_x86_cpuid_s$non_lazy_ptr: +.indirect_symbol __gnutls_x86_cpuid_s +.long 0 +.comm __gnutls_x86_cpuid_s,16,2 |