Fix RDSEED hang on x86 (GH #872) (#873)

Calls to `MASM_RDSEED_GenerateBlock` would hang for an unknown reasons on Windows 10 and VS2017/VS2019 toolchains. Similar calls to `MASM_RDRAND_GenerateBlock` worked as expected. They were effectively the same code. The only differences were the function names and the opcodes (they were literally copy/paste).

Splitting `rdrand.asm` (with both `RDRAND` and `RDSEED`) into `rdrand.asm` (with `RDRAND`) and `rdseed.asm` (with `RDSEED`) resolved the issue. We don't know why.

1 files changed, 10 insertions, 198 deletions

diff --git a/rdrand.asm b/rdrand.asm
index a9434ba8..a1c3421a 100644
--- a/rdrand.asm
+++ b/rdrand.asm
@@ -1,9 +1,9 @@
 ;; rdrand.asm - written and placed in public domain by Jeffrey Walton and Uri Blumenthal.

 ;;              Copyright assigned to the Crypto++ project.

 

-;; This ASM file provides RDRAND and RDSEED to downlevel Microsoft tool chains.

-;; Everything "just works" under Visual Studio. Other platforms will have to

-;; run MASM/MASM-64 and then link to the object files.

+;; This ASM file provides RDRAND to downlevel Microsoft tool chains.

+;; Everything "just works" under Visual Studio. Other platforms will

+;; have to run MASM/MASM-64 and then link to the object files.

 

 ;; set ASFLAGS=/nologo /D_M_X86 /W3 /Cx /Zi /safeseh

 ;; set ASFLAGS64=/nologo /D_M_X64 /W3 /Cx /Zi

@@ -13,11 +13,10 @@
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

 

-TITLE    MASM_RDRAND_GenerateBlock and MASM_RDSEED_GenerateBlock

-SUBTITLE Microsoft specific ASM code to utilize RDRAND and RDSEED for down level Microsoft toolchains

+TITLE    MASM_RDRAND_GenerateBlock source file

+SUBTITLE Microsoft specific ASM code to utilize RDRAND for down level Microsoft toolchains

 

 PUBLIC MASM_RDRAND_GenerateBlock

-PUBLIC MASM_RDSEED_GenerateBlock

 

 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

@@ -38,7 +37,6 @@ IFDEF _M_X86    ;; Set via the command line
 

 ;; Fastcall calling conventions exports

 ALIAS <@MASM_RDRAND_GenerateBlock@8> = <MASM_RDRAND_GenerateBlock>

-ALIAS <@MASM_RDSEED_GenerateBlock@8> = <MASM_RDSEED_GenerateBlock>

 

 ENDIF

 

@@ -125,10 +123,10 @@ RDRAND_GenerateBlock_Return:
 

 MASM_RDRAND_GenerateBlock ENDP

 

-ENDIF    ;; _M_X86

+;; OPTION PROLOGUE:PrologueDef

+;; OPTION EPILOGUE:EpilogueDef

 

-OPTION PROLOGUE:PrologueDef

-OPTION EPILOGUE:EpilogueDef

+ENDIF    ;; _M_X86

 

 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

@@ -223,197 +221,11 @@ RDRAND_GenerateBlock_Return:
 

 MASM_RDRAND_GenerateBlock ENDP

 

-ENDIF    ;; _M_X64

-

-OPTION PROLOGUE:PrologueDef

-OPTION EPILOGUE:EpilogueDef

-

-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

-

-IFDEF _M_X86    ;; Set via the command line

-

-.CODE

-ALIGN   8

-OPTION PROLOGUE:NONE

-OPTION EPILOGUE:NONE

-

-;; No need for Load_Arguments due to fastcall

-;;   ECX (in): arg1, byte* buffer

-;;   EDX (in): arg2, size_t bsize

-

-MASM_RDSEED_GenerateBlock PROC   ;; arg1:DWORD, arg2:DWORD

-

-    MWSIZE EQU 04h    ;; machine word size

-    buffer EQU ecx

-    bsize  EQU edx

-

-            ;; Top of While loop

-RDSEED_GenerateBlock_Top:

-

-            ;; Check remaining size

-    cmp     bsize, 0

-    je      RDSEED_GenerateBlock_Return

-

-RDSEED_Call_EAX:

-            ;; RDSEED is not available prior to VS2012. Just emit

-            ;;   the byte codes using DB. This is `rdseed eax`.

-    DB      0Fh, 0C7h, 0F8h

-

-            ;; If CF=1, the number returned by RDSEED is valid.

-            ;; If CF=0, a random number was not available.

-

-            ;; Retry immediately

-    jnc     RDSEED_Call_EAX

-

-RDSEED_succeeded:

-

-    cmp     bsize, MWSIZE

-    jb      RDSEED_Partial_Machine_Word

-

-RDSEED_Full_Machine_Word:

-

-    mov     DWORD PTR [buffer], eax

-    add     buffer, MWSIZE        ;; No need for Intel Core 2 slow workarounds, like

-    sub     bsize, MWSIZE         ;;   `lea buffer,[buffer+MWSIZE]` for faster adds

-

-            ;; Continue

-    jmp     RDSEED_GenerateBlock_Top

-

-            ;; 1,2,3 bytes remain

-RDSEED_Partial_Machine_Word:

-

-            ;; Test bit 1 to see if size is at least 2

-    test    bsize, 2

-    jz      RDSEED_Bit_1_Not_Set

-

-    mov     WORD PTR [buffer], ax

-    shr     eax, 16

-    add     buffer, 2

-

-RDSEED_Bit_1_Not_Set:

-

-            ;; Test bit 0 to see if size is at least 1

-    test    bsize, 1

-    jz      RDSEED_Bit_0_Not_Set

-

-    mov     BYTE PTR [buffer], al

-

-RDSEED_Bit_0_Not_Set:

-

-            ;; We've hit all the bits

-

-RDSEED_GenerateBlock_Return:

-

-            ;; Clear artifacts

-    xor     eax, eax

-    ret

-

-MASM_RDSEED_GenerateBlock ENDP

-

-ENDIF    ;; _M_X86

-

-OPTION PROLOGUE:PrologueDef

-OPTION EPILOGUE:EpilogueDef

-

-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

-

-IFDEF _M_X64    ;; Set via the command line

-

-.CODE

-ALIGN   16

-OPTION PROLOGUE:NONE

-OPTION EPILOGUE:NONE

-

-;; No need for Load_Arguments due to fastcall

-;;   RCX (in): arg1, byte* buffer

-;;   RDX (in): arg2, size_t bsize

-

-MASM_RDSEED_GenerateBlock PROC   ;; arg1:QWORD, arg2:QWORD

-

-    MWSIZE EQU 08h    ;; machine word size

-    buffer EQU rcx

-    bsize  EQU rdx

-

-            ;; Top of While loop

-RDSEED_GenerateBlock_Top:

-

-            ;; Check remaining size

-    cmp     bsize, 0

-    je      RDSEED_GenerateBlock_Return

-

-RDSEED_Call_RAX:

-            ;; RDSEED is not available prior to VS2012. Just emit

-            ;;   the byte codes using DB. This is `rdseed rax`.

-    DB      048h, 0Fh, 0C7h, 0F8h

-

-            ;; If CF=1, the number returned by RDSEED is valid.

-            ;; If CF=0, a random number was not available.

-

-            ;; Retry immediately

-    jnc     RDSEED_Call_RAX

-

-RDSEED_succeeded:

-

-    cmp     bsize, MWSIZE

-    jb      RDSEED_Partial_Machine_Word

-

-RDSEED_Full_Machine_Word:

-

-    mov     QWORD PTR [buffer], rax

-    add     buffer, MWSIZE

-    sub     bsize, MWSIZE

-

-            ;; Continue

-    jmp     RDSEED_GenerateBlock_Top

-

-            ;; 1,2,3,4,5,6,7 bytes remain

-RDSEED_Partial_Machine_Word:

-

-            ;; Test bit 2 to see if size is at least 4

-    test    bsize, 4

-    jz      RDSEED_Bit_2_Not_Set

-

-    mov     DWORD PTR [buffer], eax

-    shr     rax, 32

-    add     buffer, 4

-

-RDSEED_Bit_2_Not_Set:

-

-            ;; Test bit 1 to see if size is at least 2

-    test    bsize, 2

-    jz      RDSEED_Bit_1_Not_Set

-

-    mov     WORD PTR [buffer], ax

-    shr     eax, 16

-    add     buffer, 2

-

-RDSEED_Bit_1_Not_Set:

-

-            ;; Test bit 0 to see if size is at least 1

-    test    bsize, 1

-    jz      RDSEED_Bit_0_Not_Set

-

-    mov     BYTE PTR [buffer], al

-

-RDSEED_Bit_0_Not_Set:

-

-            ;; We've hit all the bits

-

-RDSEED_GenerateBlock_Return:

-

-            ;; Clear artifacts

-    xor     rax, rax

-    ret

-

-MASM_RDSEED_GenerateBlock ENDP

+;; OPTION PROLOGUE:PrologueDef

+;; OPTION EPILOGUE:EpilogueDef

 

 ENDIF    ;; _M_X64

 

-OPTION PROLOGUE:PrologueDef

-OPTION EPILOGUE:EpilogueDef

-

 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;