Clear Asan finding in VMAC under 32-bit inline ASM (GH #860)

Second try. The first try cleared the Asan finding but broke at -O3. Eventually we will skin this cat.

1 files changed, 25 insertions, 13 deletions

diff --git a/vmac.cpp b/vmac.cpp
index 7bf07727..806904ec 100644
--- a/vmac.cpp
+++ b/vmac.cpp
@@ -178,11 +178,9 @@ unsigned int VMAC_Base::OptimalDataAlignment() const
 #if CRYPTOPP_MSC_VERSION

 # pragma warning(disable: 4731)	// frame pointer register 'ebp' modified by inline assembly code

 #endif

-void

-#ifdef __GNUC__

-__attribute__ ((noinline))		// Intel Compiler 9.1 workaround

-#endif

-VMAC_Base::VHASH_Update_SSE2(const word64 *data, size_t blocksRemainingInWord64, int tagPart)

+

+CRYPTOPP_NOINLINE

+void VMAC_Base::VHASH_Update_SSE2(const word64 *data, size_t blocksRemainingInWord64, int tagPart)

 {

 	const word64 *nhK = m_nhKey();

 	word64 *polyS = (word64*)(void*)m_polyState();

@@ -193,13 +191,27 @@ VMAC_Base::VHASH_Update_SSE2(const word64 *data, size_t blocksRemainingInWord64,
 	CRYPTOPP_UNUSED(L1KeyLength);

 	CRYPTOPP_UNUSED(blocksRemainingInWord64);

 

+	// This inline ASM is tricky, and down right difficult when PIC is

+	// in effect. The ASM uses all the general purpose registers. When

+	// PIC is in effect, GCC uses EBX as a base register. Saving EBX with

+	// 'mov %%ebx, %0' and restoring EBX with 'mov %0, %%ebx' causes GCC

+	// to generate 'mov -0x40(%ebx), %ebx' for the restore. That obviously

+	// won't work. We can push and pop EBX, but then we have to be careful

+	// because GCC references %1 (L1KeyLength) relative to ESP, which is

+	// also used in the function and no longer accurate. Attempting to

+	// sidestep the issues with clobber lists results in "error: ‘asm’

+	// operand has impossible constraints", though we were able to tell

+	// GCC that ESP is dirty. The problems with GCC are the reason for the

+	// pushes and pops rather than the original moves.

 #ifdef __GNUC__

-	word32 temp;

+	// word32 temp;

 	__asm__ __volatile__

 	(

-	AS2(	mov		%%ebx, %0)

-	// AS1(	push	%%ebx)

-	AS2(	mov		%1, %%ebx)

+	// AS2(	mov		%%ebx, %0)

+	// AS2(	mov		%1, %%ebx)  // L1KeyLength

+	AS1(	push	%%ebx)

+	AS1(	push	%0)         // L1KeyLength

+	AS1(	pop		%%ebx)

 	INTEL_NOPREFIX

 #else

 	#if defined(__INTEL_COMPILER)

@@ -419,12 +431,12 @@ VMAC_Base::VHASH_Update_SSE2(const word64 *data, size_t blocksRemainingInWord64,
 	AS1(	emms)

 #ifdef __GNUC__

 	ATT_PREFIX

-	// AS1(	pop		%%ebx)

-	AS2(	mov	%0, %%ebx)

-		: "=m" (temp)

+	AS1(	pop		%%ebx)

+	// AS2(	mov	%0, %%ebx)

+		: // "=m" (temp)

 		: "m" (L1KeyLength), "c" (blocksRemainingInWord64), "S" (data),

 		  "D" (nhK+tagPart*2), "d" (m_isFirstBlock), "a" (polyS+tagPart*4)

-		: "memory", "cc"

+		: "esp", "memory", "cc"

 	);

 #endif

 }