BR 3392260: Handle instructions only separated by vector SIB size

There are two instructions (VGATHERQPS, VPGATHERQD) where the only
separation between two forms is the vector length given to the vector
SIB.  This means the *matcher* has to be able to distinguish
instructions by vector SIB length and the matcher only operates on the
operands and the instruction flags, not on the bytecode.

Export the vector index-ness into the operand flags and add to the
matcher.

This resolves BR 3392260.

Reported-by: Agner <agner@anger.org>
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>

4 files changed, 53 insertions, 25 deletions

diff --git a/insns.dat b/insns.dat
index caa0e643..9c39c6bd 100644
--- a/insns.dat
+++ b/insns.dat
@@ -3370,25 +3370,25 @@ VPSRLVQ		xmmreg,xmmreg*,xmmrm128		[rvm:	vex.nds.128.66.0f38.w1 45 /r]		FUTURE,AV
 VPSRLVD		ymmreg,ymmreg*,ymmrm256		[rvm:	vex.nds.256.66.0f38.w0 45 /r]		FUTURE,AVX2
 VPSRLVQ		ymmreg,ymmreg*,ymmrm256		[rvm:	vex.nds.256.66.0f38.w1 45 /r]		FUTURE,AVX2
 
-VGATHERDPD	xmmreg,mem64,xmmreg		[rmv:	vm32x vex.dds.128.66.0f38.w1 92 /r]	FUTURE,AVX2
-VGATHERQPD	xmmreg,mem64,xmmreg		[rmv:	vm64x vex.dds.128.66.0f38.w1 93 /r]	FUTURE,AVX2
-VGATHERDPD	ymmreg,mem64,ymmreg		[rmv:	vm32x vex.dds.256.66.0f38.w1 92 /r]	FUTURE,AVX2
-VGATHERQPD	ymmreg,mem64,ymmreg		[rmv:	vm64y vex.dds.256.66.0f38.w1 93 /r]	FUTURE,AVX2
-
-VGATHERDPS	xmmreg,mem32,xmmreg		[rmv:	vm32x vex.dds.128.66.0f38.w0 92 /r]	FUTURE,AVX2
-VGATHERQPS	xmmreg,mem32,xmmreg		[rmv:	vm64x vex.dds.128.66.0f38.w0 93 /r]	FUTURE,AVX2
-VGATHERDPS	ymmreg,mem32,ymmreg		[rmv:	vm32y vex.dds.256.66.0f38.w0 92 /r]	FUTURE,AVX2
-VGATHERQPS	xmmreg,mem32,xmmreg		[rmv:	vm64y vex.dds.256.66.0f38.w0 93 /r]	FUTURE,AVX2
-
-VPGATHERDD	xmmreg,mem32,xmmreg		[rmv:	vm32x vex.dds.128.66.0f38.w0 90 /r]	FUTURE,AVX2
-VPGATHERQD	xmmreg,mem32,xmmreg		[rmv:	vm64x vex.dds.128.66.0f38.w0 91 /r]	FUTURE,AVX2
-VPGATHERDD	ymmreg,mem32,ymmreg		[rmv:	vm32y vex.dds.256.66.0f38.w0 90 /r]	FUTURE,AVX2
-VPGATHERQD	xmmreg,mem32,xmmreg		[rmv:	vm64y vex.dds.256.66.0f38.w0 91 /r]	FUTURE,AVX2
-
-VPGATHERDQ	xmmreg,mem64,xmmreg		[rmv:	vm32x vex.dds.128.66.0f38.w1 90 /r]	FUTURE,AVX2
-VPGATHERQQ	xmmreg,mem64,xmmreg		[rmv:	vm64x vex.dds.128.66.0f38.w1 91 /r]	FUTURE,AVX2
-VPGATHERDQ	ymmreg,mem64,ymmreg		[rmv:	vm32x vex.dds.256.66.0f38.w1 90 /r]	FUTURE,AVX2
-VPGATHERQQ	ymmreg,mem64,ymmreg		[rmv:	vm64y vex.dds.256.66.0f38.w1 91 /r]	FUTURE,AVX2
+VGATHERDPD	xmmreg,xmem64,xmmreg		[rmv:	vm32x vex.dds.128.66.0f38.w1 92 /r]	FUTURE,AVX2
+VGATHERQPD	xmmreg,xmem64,xmmreg		[rmv:	vm64x vex.dds.128.66.0f38.w1 93 /r]	FUTURE,AVX2
+VGATHERDPD	ymmreg,xmem64,ymmreg		[rmv:	vm32x vex.dds.256.66.0f38.w1 92 /r]	FUTURE,AVX2
+VGATHERQPD	ymmreg,ymem64,ymmreg		[rmv:	vm64y vex.dds.256.66.0f38.w1 93 /r]	FUTURE,AVX2
+
+VGATHERDPS	xmmreg,xmem32,xmmreg		[rmv:	vm32x vex.dds.128.66.0f38.w0 92 /r]	FUTURE,AVX2
+VGATHERQPS	xmmreg,xmem32,xmmreg		[rmv:	vm64x vex.dds.128.66.0f38.w0 93 /r]	FUTURE,AVX2
+VGATHERDPS	ymmreg,ymem32,ymmreg		[rmv:	vm32y vex.dds.256.66.0f38.w0 92 /r]	FUTURE,AVX2
+VGATHERQPS	xmmreg,ymem32,xmmreg		[rmv:	vm64y vex.dds.256.66.0f38.w0 93 /r]	FUTURE,AVX2
+
+VPGATHERDD	xmmreg,xmem32,xmmreg		[rmv:	vm32x vex.dds.128.66.0f38.w0 90 /r]	FUTURE,AVX2
+VPGATHERQD	xmmreg,xmem32,xmmreg		[rmv:	vm64x vex.dds.128.66.0f38.w0 91 /r]	FUTURE,AVX2
+VPGATHERDD	ymmreg,ymem32,ymmreg		[rmv:	vm32y vex.dds.256.66.0f38.w0 90 /r]	FUTURE,AVX2
+VPGATHERQD	xmmreg,ymem32,xmmreg		[rmv:	vm64y vex.dds.256.66.0f38.w0 91 /r]	FUTURE,AVX2
+
+VPGATHERDQ	xmmreg,xmem64,xmmreg		[rmv:	vm32x vex.dds.128.66.0f38.w1 90 /r]	FUTURE,AVX2
+VPGATHERQQ	xmmreg,xmem64,xmmreg		[rmv:	vm64x vex.dds.128.66.0f38.w1 91 /r]	FUTURE,AVX2
+VPGATHERDQ	ymmreg,xmem64,ymmreg		[rmv:	vm32x vex.dds.256.66.0f38.w1 90 /r]	FUTURE,AVX2
+VPGATHERQQ	ymmreg,ymem64,ymmreg		[rmv:	vm64y vex.dds.256.66.0f38.w1 91 /r]	FUTURE,AVX2
 
 ;# Transactional Synchronization Extensions (TSX)
 XABORT		imm				[i:	c6 f8 ib]				FUTURE,RTM
diff --git a/opflags.h b/opflags.h
index 4022398b..41fce3d5 100644
--- a/opflags.h
+++ b/opflags.h
@@ -228,6 +228,8 @@ typedef uint64_t opflags_t;
 /* special types of EAs */
 #define MEM_OFFS                (GEN_SUBCLASS(1) | MEMORY)      /* simple [address] offset - absolute! */
 #define IP_REL                  (GEN_SUBCLASS(2) | MEMORY)      /* IP-relative offset */
+#define XMEM                    (GEN_SUBCLASS(3) | MEMORY)      /* 128-bit vector SIB */
+#define YMEM                    (GEN_SUBCLASS(4) | MEMORY)      /* 256-bit vector SIB */
 
 /* memory which matches any type of r/m operand */
 #define MEMORY_ANY              (MEMORY | RM_GPR | RM_MMX | RM_XMM | RM_YMM)
diff --git a/parser.c b/parser.c
index a71dd33d..afc422a9 100644
--- a/parser.c
+++ b/parser.c
@@ -1,6 +1,6 @@
 /* ----------------------------------------------------------------------- *
  *
- *   Copyright 1996-2009 The NASM Authors - All Rights Reserved
+ *   Copyright 1996-2013 The NASM Authors - All Rights Reserved
  *   See the file AUTHORS included with the NASM distribution for
  *   the specific copyright holders.
  *
@@ -736,19 +736,24 @@ is_expression:
             result->oprs[operand].hinttype = hints.type;
 
             if (e->type && e->type <= EXPR_REG_END) {   /* this bit's a register */
-                if (e->value == 1)      /* in fact it can be basereg */
-                    b = e->type;
-                else            /* no, it has to be indexreg */
+                bool is_gpr = is_class(REG_GPR,nasm_reg_flags[e->type]);
+                
+                if (is_gpr && e->value == 1)
+                    b = e->type;	/* It can be basereg */
+                else            	/* No, it has to be indexreg */
                     i = e->type, s = e->value;
                 e++;
             }
             if (e->type && e->type <= EXPR_REG_END) {   /* it's a 2nd register */
+                bool is_gpr = is_class(REG_GPR,nasm_reg_flags[e->type]);
+
                 if (b != -1)    /* If the first was the base, ... */
                     i = e->type, s = e->value;  /* second has to be indexreg */
 
-                else if (e->value != 1) {       /* If both want to be index */
+                else if (!is_gpr || e->value != 1) {
+                    /* If both want to be index */
                     nasm_error(ERR_NONFATAL,
-                          "beroset-p-592-invalid effective address");
+                               "invalid effective address: two index registers");
                     result->opcode = I_none;
                     return result;
                 } else
@@ -837,6 +842,16 @@ is_expression:
 
                 result->oprs[operand].type |= is_rel ? IP_REL : MEM_OFFS;
             }
+
+            if (i != -1) {
+                opflags_t iclass = nasm_reg_flags[i];
+
+                if (is_class(XMMREG,iclass))
+                    result->oprs[operand].type |= XMEM;
+                else if (is_class(YMMREG,iclass))
+                    result->oprs[operand].type |= YMEM;
+            }
+
             result->oprs[operand].basereg = b;
             result->oprs[operand].indexreg = i;
             result->oprs[operand].scale = s;
diff --git a/test/gather.asm b/test/gather.asm
new file mode 100644
index 00000000..03037008
--- /dev/null
+++ b/test/gather.asm
@@ -0,0 +1,11 @@
+	bits 64
+
+	VGATHERQPS xmm1, [xmm0 + rsi], xmm2   ; OK
+	VGATHERQPS xmm1, [ymm0 + rsi], xmm2 ; fail: error: invalid effective address
+	VGATHERDPD ymm1, [xmm0 + rsi], ymm2   ; OK
+	VGATHERDPD xmm1, [xmm0 + rsi], xmm2   ; OK
+	VGATHERQPD xmm1, [xmm0 + rsi], xmm2   ; OK
+	VGATHERQPD ymm1, [ymm0 + rsi], ymm2   ; OK
+	VPGATHERQD xmm1, [xmm0 + rsi], xmm2   ; OK
+	VPGATHERQD xmm1, [ymm0 + rsi], xmm2   ; fail: error: invalid effective address
+	VPGATHERDQ ymm1, [xmm0 + rsi], ymm2   ; OK