BR 2028910: fix decoding of VEX prefixes in 16- and 32-bit mode

We would incorrectly set a bunch of VEX-related state for C4 and C5
bytes, even though we had already rejected it as not a VEX prefix due
to the top two bits of the following byte not being 11.

1 files changed, 17 insertions, 16 deletions

diff --git a/disasm.c b/disasm.c
index c02a3adf..f89b11b4 100644
--- a/disasm.c
+++ b/disasm.c
@@ -1073,24 +1073,25 @@ int32_t disasm(uint8_t *data, char *output, int outbufsize, int segsize,
 	    if (segsize == 64 || (data[1] & 0xc0) == 0xc0) {
 		prefix.vex[0] = *data++;
 		prefix.vex[1] = *data++;
-		if (prefix.vex[0] == 0xc4)
+
+		prefix.rex = REX_V;
+
+		if (prefix.vex[0] == 0xc4) {
 		    prefix.vex[2] = *data++;
-	    }
-	    prefix.rex = REX_V;
-	    if (prefix.vex[0] == 0xc4) {
-		prefix.rex |= (~prefix.vex[1] >> 5) & 7; /* REX_RXB */
-		prefix.rex |= (prefix.vex[2] >> (7-3)) & REX_W;
-		prefix.vex_m = prefix.vex[1] & 0x1f;
-		prefix.vex_v = (~prefix.vex[2] >> 3) & 15;
-		prefix.vex_lp = prefix.vex[2] & 7;
-	    } else {
-		prefix.rex |= (~prefix.vex[1] >> (7-2)) & REX_R;
-		prefix.vex_m = 1;
-		prefix.vex_v = (~prefix.vex[1] >> 3) & 15;
-		prefix.vex_lp = prefix.vex[1] & 7;
-	    }
+		    prefix.rex |= (~prefix.vex[1] >> 5) & 7; /* REX_RXB */
+		    prefix.rex |= (prefix.vex[2] >> (7-3)) & REX_W;
+		    prefix.vex_m = prefix.vex[1] & 0x1f;
+		    prefix.vex_v = (~prefix.vex[2] >> 3) & 15;
+		    prefix.vex_lp = prefix.vex[2] & 7;
+		} else {
+		    prefix.rex |= (~prefix.vex[1] >> (7-2)) & REX_R;
+		    prefix.vex_m = 1;
+		    prefix.vex_v = (~prefix.vex[1] >> 3) & 15;
+		    prefix.vex_lp = prefix.vex[1] & 7;
+		}
 
-	    ix = itable_VEX[prefix.vex_m][prefix.vex_lp];
+		ix = itable_VEX[prefix.vex_m][prefix.vex_lp];
+	    }
 	    end_prefix = true;
 	    break;