x86: restrict insn templates accepting negative 8-bit immediates

For shifts (but not ordinary rotates) and other cases where an immediate
describes e.g. a bit count or position, allowing negative operands is at
best confusing. An extreme example would be the two rotate-through-carry
insns, where a negative value would _not_ mean rotating the
corresponding number of bits in the other direction. To refuse such,
give meaning to the combination of Imm8 and Imm8S in templates (so far
these weren't used together anywhere). The issue was with
smallest_imm_type() blindly setting .imm8 for signed numbers determined
to fit in a byte.

VPROT{B,W,D,Q} is a little special: The rotate count there is a signed
quantity, so Imm8 is replaced by Imm8S. Adjust affected testcases
accordingly as well.

Another small adjustment to the testsuite is necessary: AAM and AAD were
never sensible to use with 0xffffff90 operands. This should have been an
error.

4 files changed, 51 insertions, 51 deletions

diff --git a/gas/testsuite/gas/i386/intel.s b/gas/testsuite/gas/i386/intel.s
index 7afc4901ffa..922f260154e 100644
--- a/gas/testsuite/gas/i386/intel.s
+++ b/gas/testsuite/gas/i386/intel.s
@@ -205,8 +205,8 @@ foo:
  rcl    dword ptr 0x90909090[eax]
  rcl    byte ptr 0x90909090[eax], cl
  rcl    dword ptr 0x90909090[eax], cl
- aam    0xffffff90
- aad    0xffffff90
+ aam    0x90
+ aad    0x90
  xlat   byte ptr ds:[ebx]
  fcom   dword ptr 0x90909090[eax]
  fst    dword ptr 0x90909090[eax]
diff --git a/gas/testsuite/gas/i386/opcode.s b/gas/testsuite/gas/i386/opcode.s
index e1227b7f2b4..8b87dd70694 100644
--- a/gas/testsuite/gas/i386/opcode.s
+++ b/gas/testsuite/gas/i386/opcode.s
@@ -202,8 +202,8 @@ foo:
  rcll   0x90909090(%eax)
  rclb   %cl,0x90909090(%eax)
  rcll   %cl,0x90909090(%eax)
- aam    $0xffffff90
- aad    $0xffffff90
+ aam    $0x90
+ aad    $0x90
  xlat   %ds:(%ebx)
  fcoms  0x90909090(%eax)
  fsts   0x90909090(%eax)
diff --git a/gas/testsuite/gas/i386/x86-64-xop.s b/gas/testsuite/gas/i386/x86-64-xop.s
index 98f3c46c6f3..f63bebed519 100644
--- a/gas/testsuite/gas/i386/x86-64-xop.s
+++ b/gas/testsuite/gas/i386/x86-64-xop.s
@@ -911,20 +911,20 @@ _start:
     VPROTB         %xmm15,%xmm15,%xmm15
 # Tests for op VPROTB imm8, xmm2, xmm1  (at&t syntax)
     VPROTB         $0x3,%xmm11,%xmm15
-    VPROTB         $0xFF,%xmm0,%xmm0
-    VPROTB         $0xFF,%xmm11,%xmm4
+    VPROTB         $-1,%xmm0,%xmm0
+    VPROTB         $-1,%xmm11,%xmm4
     VPROTB         $0x0,%xmm11,%xmm4
     VPROTB         $0x0,%xmm15,%xmm4
     VPROTB         $0x0,%xmm0,%xmm15
-    VPROTB         $0xFF,%xmm11,%xmm0
+    VPROTB         $-1,%xmm11,%xmm0
     VPROTB         $0x3,%xmm0,%xmm0
     VPROTB         $0x3,%xmm11,%xmm0
     VPROTB         $0x0,%xmm0,%xmm4
-    VPROTB         $0xFF,%xmm15,%xmm0
-    VPROTB         $0xFF,%xmm0,%xmm15
-    VPROTB         $0xFF,%xmm15,%xmm15
+    VPROTB         $-1,%xmm15,%xmm0
+    VPROTB         $-1,%xmm0,%xmm15
+    VPROTB         $-1,%xmm15,%xmm15
     VPROTB         $0x3,%xmm15,%xmm4
-    VPROTB         $0xFF,%xmm11,%xmm15
+    VPROTB         $-1,%xmm11,%xmm15
     VPROTB         $0x3,%xmm0,%xmm15
 # Tests for op VPROTD xmm3, xmm2/mem128, xmm1  (at&t syntax)
     VPROTD         %xmm2,%xmm0,%xmm15
@@ -964,17 +964,17 @@ _start:
     VPROTD         $0x0,%xmm15,%xmm15
     VPROTD         $0x0,(%rsi),%xmm15
     VPROTD         $0x0,%xmm0,%xmm11
-    VPROTD         $0xFF,%xmm15,%xmm0
+    VPROTD         $-1,%xmm15,%xmm0
     VPROTD         $0x3,%xmm0,%xmm0
     VPROTD         $0x3,%xmm15,%xmm0
     VPROTD         $0x0,%xmm11,%xmm11
     VPROTD         $0x0,%xmm0,%xmm15
     VPROTD         $0x3,(%rcx),%xmm0
-    VPROTD         $0xFF,(%rsi),%xmm0
+    VPROTD         $-1,(%rsi),%xmm0
     VPROTD         $0x0,(%rdi),%xmm15
-    VPROTD         $0xFF,%xmm15,%xmm15
-    VPROTD         $0xFF,%xmm11,%xmm11
-    VPROTD         $0xFF,(%rsi),%xmm11
+    VPROTD         $-1,%xmm15,%xmm15
+    VPROTD         $-1,%xmm11,%xmm11
+    VPROTD         $-1,(%rsi),%xmm11
     VPROTD         $0x3,(%rdi),%xmm15
     VPROTD         $0x3,%xmm15,%xmm11
 # Tests for op VPROTQ xmm3, xmm2/mem128, xmm1  (at&t syntax)
@@ -1015,17 +1015,17 @@ _start:
     VPROTQ         $0x0,%xmm15,%xmm15
     VPROTQ         $0x0,(%rsi),%xmm15
     VPROTQ         $0x0,%xmm0,%xmm11
-    VPROTQ         $0xFF,%xmm15,%xmm0
+    VPROTQ         $-1,%xmm15,%xmm0
     VPROTQ         $0x3,%xmm0,%xmm0
     VPROTQ         $0x3,%xmm15,%xmm0
     VPROTQ         $0x0,%xmm11,%xmm11
     VPROTQ         $0x0,%xmm0,%xmm15
     VPROTQ         $0x3,(%rcx),%xmm0
-    VPROTQ         $0xFF,(%rsi),%xmm0
+    VPROTQ         $-1,(%rsi),%xmm0
     VPROTQ         $0x0,(%rdi),%xmm15
-    VPROTQ         $0xFF,%xmm15,%xmm15
-    VPROTQ         $0xFF,%xmm11,%xmm11
-    VPROTQ         $0xFF,(%rsi),%xmm11
+    VPROTQ         $-1,%xmm15,%xmm15
+    VPROTQ         $-1,%xmm11,%xmm11
+    VPROTQ         $-1,(%rsi),%xmm11
     VPROTQ         $0x3,(%rdi),%xmm15
     VPROTQ         $0x3,%xmm15,%xmm11
 # Tests for op VPROTW xmm3, xmm2/mem128, xmm1  (at&t syntax)
@@ -1066,17 +1066,17 @@ _start:
     VPROTW         $0x0,%xmm15,%xmm15
     VPROTW         $0x0,(%rsi),%xmm15
     VPROTW         $0x0,%xmm0,%xmm11
-    VPROTW         $0xFF,%xmm15,%xmm0
+    VPROTW         $-1,%xmm15,%xmm0
     VPROTW         $0x3,%xmm0,%xmm0
     VPROTW         $0x3,%xmm15,%xmm0
     VPROTW         $0x0,%xmm11,%xmm11
     VPROTW         $0x0,%xmm0,%xmm15
     VPROTW         $0x3,(%rcx),%xmm0
-    VPROTW         $0xFF,(%rsi),%xmm0
+    VPROTW         $-1,(%rsi),%xmm0
     VPROTW         $0x0,(%rdi),%xmm15
-    VPROTW         $0xFF,%xmm15,%xmm15
-    VPROTW         $0xFF,%xmm11,%xmm11
-    VPROTW         $0xFF,(%rsi),%xmm11
+    VPROTW         $-1,%xmm15,%xmm15
+    VPROTW         $-1,%xmm11,%xmm11
+    VPROTW         $-1,(%rsi),%xmm11
     VPROTW         $0x3,(%rdi),%xmm15
     VPROTW         $0x3,%xmm15,%xmm11
 # Tests for op VPSHAB xmm3, xmm2/mem128, xmm1  (at&t syntax)
diff --git a/gas/testsuite/gas/i386/xop.s b/gas/testsuite/gas/i386/xop.s
index 5b7697a8e61..0a8d411e6ef 100644
--- a/gas/testsuite/gas/i386/xop.s
+++ b/gas/testsuite/gas/i386/xop.s
@@ -911,20 +911,20 @@ _start:
     VPROTB         %xmm1,%xmm7,%xmm3
 # Tests for op VPROTB imm8, xmm2, xmm1  (at&t syntax)
     VPROTB         $0x3,%xmm5,%xmm2
-    VPROTB         $0xFF,%xmm0,%xmm0
-    VPROTB         $0xFF,%xmm5,%xmm7
+    VPROTB         $-1,%xmm0,%xmm0
+    VPROTB         $-1,%xmm5,%xmm7
     VPROTB         $0x0,%xmm5,%xmm7
     VPROTB         $0x0,%xmm7,%xmm7
     VPROTB         $0x0,%xmm0,%xmm2
-    VPROTB         $0xFF,%xmm5,%xmm0
+    VPROTB         $-1,%xmm5,%xmm0
     VPROTB         $0x3,%xmm0,%xmm0
     VPROTB         $0x3,%xmm5,%xmm0
     VPROTB         $0x0,%xmm0,%xmm7
-    VPROTB         $0xFF,%xmm7,%xmm0
-    VPROTB         $0xFF,%xmm0,%xmm2
-    VPROTB         $0xFF,%xmm7,%xmm2
+    VPROTB         $-1,%xmm7,%xmm0
+    VPROTB         $-1,%xmm0,%xmm2
+    VPROTB         $-1,%xmm7,%xmm2
     VPROTB         $0x3,%xmm7,%xmm7
-    VPROTB         $0xFF,%xmm5,%xmm2
+    VPROTB         $-1,%xmm5,%xmm2
     VPROTB         $0x3,%xmm0,%xmm2
 # Tests for op VPROTD xmm3, xmm2/mem128, xmm1  (at&t syntax)
     VPROTD         %xmm7,%xmm0,%xmm3
@@ -964,18 +964,18 @@ _start:
     VPROTD         $0x0,%xmm7,%xmm7
     VPROTD         $0x0,(%ebx),%xmm7
     VPROTD         $0x0,%xmm0,%xmm5
-    VPROTD         $0xFF,%xmm5,%xmm0
+    VPROTD         $-1,%xmm5,%xmm0
     VPROTD         $0x3,%xmm0,%xmm0
     VPROTD         $0x3,%xmm7,%xmm0
     VPROTD         $0x0,%xmm5,%xmm5
     VPROTD         $0x0,%xmm0,%xmm7
     VPROTD         $0x3,(%eax),%xmm0
-    VPROTD         $0xFF,(%ebx),%xmm0
+    VPROTD         $-1,(%ebx),%xmm0
     VPROTD         $0x0,(%eax),%xmm7
-    VPROTD         $0xFF,%xmm7,%xmm7
-    VPROTD         $0xFF,%xmm5,%xmm5
-    VPROTD         $0xFF,(%ebx),%xmm5
-    VPROTD         $0xFF,%xmm7,%xmm0
+    VPROTD         $-1,%xmm7,%xmm7
+    VPROTD         $-1,%xmm5,%xmm5
+    VPROTD         $-1,(%ebx),%xmm5
+    VPROTD         $-1,%xmm7,%xmm0
     VPROTD         $0x3,(%eax),%xmm7
 # Tests for op VPROTQ xmm3, xmm2/mem128, xmm1  (at&t syntax)
     VPROTQ         %xmm7,%xmm0,%xmm3
@@ -1015,18 +1015,18 @@ _start:
     VPROTQ         $0x0,%xmm7,%xmm7
     VPROTQ         $0x0,(%ebx),%xmm7
     VPROTQ         $0x0,%xmm0,%xmm5
-    VPROTQ         $0xFF,%xmm5,%xmm0
+    VPROTQ         $-1,%xmm5,%xmm0
     VPROTQ         $0x3,%xmm0,%xmm0
     VPROTQ         $0x3,%xmm7,%xmm0
     VPROTQ         $0x0,%xmm5,%xmm5
     VPROTQ         $0x0,%xmm0,%xmm7
     VPROTQ         $0x3,(%eax),%xmm0
-    VPROTQ         $0xFF,(%ebx),%xmm0
+    VPROTQ         $-1,(%ebx),%xmm0
     VPROTQ         $0x0,(%eax),%xmm7
-    VPROTQ         $0xFF,%xmm7,%xmm7
-    VPROTQ         $0xFF,%xmm5,%xmm5
-    VPROTQ         $0xFF,(%ebx),%xmm5
-    VPROTQ         $0xFF,%xmm7,%xmm0
+    VPROTQ         $-1,%xmm7,%xmm7
+    VPROTQ         $-1,%xmm5,%xmm5
+    VPROTQ         $-1,(%ebx),%xmm5
+    VPROTQ         $-1,%xmm7,%xmm0
     VPROTQ         $0x3,(%eax),%xmm7
 # Tests for op VPROTW xmm3, xmm2/mem128, xmm1  (at&t syntax)
     VPROTW         %xmm7,%xmm0,%xmm3
@@ -1066,18 +1066,18 @@ _start:
     VPROTW         $0x0,%xmm7,%xmm7
     VPROTW         $0x0,(%ebx),%xmm7
     VPROTW         $0x0,%xmm0,%xmm5
-    VPROTW         $0xFF,%xmm5,%xmm0
+    VPROTW         $-1,%xmm5,%xmm0
     VPROTW         $0x3,%xmm0,%xmm0
     VPROTW         $0x3,%xmm7,%xmm0
     VPROTW         $0x0,%xmm5,%xmm5
     VPROTW         $0x0,%xmm0,%xmm7
     VPROTW         $0x3,(%eax),%xmm0
-    VPROTW         $0xFF,(%ebx),%xmm0
+    VPROTW         $-1,(%ebx),%xmm0
     VPROTW         $0x0,(%eax),%xmm7
-    VPROTW         $0xFF,%xmm7,%xmm7
-    VPROTW         $0xFF,%xmm5,%xmm5
-    VPROTW         $0xFF,(%ebx),%xmm5
-    VPROTW         $0xFF,%xmm7,%xmm0
+    VPROTW         $-1,%xmm7,%xmm7
+    VPROTW         $-1,%xmm5,%xmm5
+    VPROTW         $-1,(%ebx),%xmm5
+    VPROTW         $-1,%xmm7,%xmm0
     VPROTW         $0x3,(%eax),%xmm7
 # Tests for op VPSHAB xmm3, xmm2/mem128, xmm1  (at&t syntax)
     VPSHAB         %xmm7,%xmm0,%xmm3