Add extension bytecodes to support operands 4+

The bytecode format assumes max 4 operands pretty strictly, but we
already have one instruction with 5 operands, and it's likely to get
more.  Support them via extension prefixes (similar to REX prefixes).
For bytecodes which use argument bytes we encode the number directly,
however.

Signed-off-by: H. Peter Anvin <hpa@zytor.com>

3 files changed, 97 insertions, 43 deletions

diff --git a/assemble.c b/assemble.c
index 3f729918..3a94a04a 100644
--- a/assemble.c
+++ b/assemble.c
@@ -7,7 +7,10 @@
  *
  * the actual codes (C syntax, i.e. octal):
  * \0            - terminates the code. (Unless it's a literal of course.)
- * \1, \2, \3    - that many literal bytes follow in the code stream
+ * \1..\4	 - that many literal bytes follow in the code stream
+ * \5            - add 4 to the primary operand number (b, low octdigit)
+ * \6            - add 4 to the secondary operand number (a, middle octdigit)
+ * \7            - add 4 to both the primary and the secondary operand number
  * \10..\13      - a literal byte follows in the code stream, to be added
  *                 to the register value of operand 0..3
  * \14..\17      - a signed byte immediate operand, from operand 0..3
@@ -784,7 +787,9 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
     int64_t length = 0;
     uint8_t c;
     int rex_mask = ~0;
+    int op1, op2;
     struct operand *opx;
+    uint8_t opex = 0;
 
     ins->rex = 0;               /* Ensure REX is reset */
 
@@ -796,14 +801,25 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
 
     while (*codes) {
 	c = *codes++;
-	opx = &ins->oprs[c & 3];
+	op1 = (c & 3) + ((opex & 1) << 2);
+	op2 = ((c >> 3) & 3) + ((opex & 2) << 1);
+	opx = &ins->oprs[op1];
+	opex = 0;		/* For the next iteration */
+
         switch (c) {
         case 01:
         case 02:
         case 03:
+	case 04:
             codes += c, length += c;
             break;
 
+	case 05:
+	case 06:
+	case 07:
+	    opex = c;
+	    break;
+
 	case4(010):
 	    ins->rex |=
 		op_rexflags(opx, REX_B|REX_H|REX_P|REX_W);
@@ -1060,16 +1076,14 @@ static int64_t calcsize(int32_t segment, int64_t offset, int bits,
 
 		if (c <= 0177) {
 		    /* pick rfield from operand b */
-		    rflags = regflag(&ins->oprs[c & 7]);
-		    rfield = nasm_regvals[ins->oprs[c & 7].basereg];
+		    rflags = regflag(&ins->oprs[op1]);
+		    rfield = nasm_regvals[ins->oprs[op1].basereg];
 		} else {
 		    rflags = 0;
 		    rfield = c & 7;
 		}
-
-                if (!process_ea
-                    (&ins->oprs[(c >> 3) & 7], &ea_data, bits,
-		     ins->addr_size, rfield, rflags)) {
+                if (!process_ea(&ins->oprs[op2], &ea_data, bits,
+				ins->addr_size, rfield, rflags)) {
                     errfunc(ERR_NONFATAL, "invalid effective address");
                     return -1;
                 } else {
@@ -1170,25 +1184,38 @@ static void gencode(int32_t segment, int64_t offset, int bits,
     uint8_t bytes[4];
     int64_t size;
     int64_t data;
+    int op1, op2;
     struct operand *opx;
     const uint8_t *codes = temp->code;
+    uint8_t opex = 0;
 
     while (*codes) {
 	c = *codes++;
-	opx = &ins->oprs[c & 3];
+	op1 = (c & 3) + ((opex & 1) << 2);
+	op2 = ((c >> 3) & 3) + ((opex & 2) << 1);
+	opx = &ins->oprs[op1];
+	opex = 0;		/* For the next iteration */
+
         switch (c) {
         case 01:
         case 02:
         case 03:
+	case 04:
 	    EMIT_REX();
             out(offset, segment, codes, OUT_RAWDATA, c, NO_SEG, NO_SEG);
             codes += c;
             offset += c;
             break;
 
+	case 05:
+	case 06:
+	case 07:
+	    opex = c;
+	    break;
+
 	case4(010):
 	    EMIT_REX();
-            bytes[0] = *codes++ + ((regval(opx)) & 7);
+            bytes[0] = *codes++ + (regval(opx) & 7);
             out(offset, segment, bytes, OUT_RAWDATA, 1, NO_SEG, NO_SEG);
             offset += 1;
             break;
@@ -1199,7 +1226,7 @@ static void gencode(int32_t segment, int64_t offset, int bits,
 	       warn on explicit BYTE directives.  Also warn, obviously,
 	       if the optimizer isn't enabled. */
             if (((opx->type & BITS8) ||
-		 !(opx->type & temp->opd[c & 3] & BYTENESS)) &&
+		 !(opx->type & temp->opd[op1] & BYTENESS)) &&
 		(opx->offset < -128 || opx->offset > 127)) {
                 errfunc(ERR_WARNING | ERR_PASS2 | ERR_WARN_NOV,
 			"signed byte value exceeds bounds");
@@ -1749,8 +1776,8 @@ static void gencode(int32_t segment, int64_t offset, int bits,
 
                 if (c <= 0177) {
 		    /* pick rfield from operand b */
-		    rflags = regflag(&ins->oprs[c & 7]);
-                    rfield = nasm_regvals[ins->oprs[c & 7].basereg];
+		    rflags = regflag(&ins->oprs[op1]);
+                    rfield = nasm_regvals[ins->oprs[op1].basereg];
 		} else {
 		    /* rfield is constant */
 		    rflags = 0;
@@ -1758,7 +1785,7 @@ static void gencode(int32_t segment, int64_t offset, int bits,
 		}
 
                 if (!process_ea
-                    (&ins->oprs[(c >> 3) & 7], &ea_data, bits,
+                    (&ins->oprs[op2], &ea_data, bits,
 		     ins->addr_size, rfield, rflags)) {
                     errfunc(ERR_NONFATAL, "invalid effective address");
                 }
diff --git a/disasm.c b/disasm.c
index 4efcdc80..70bbfc1c 100644
--- a/disasm.c
+++ b/disasm.c
@@ -402,6 +402,7 @@ static int matches(const struct itemplate *t, uint8_t *data,
 	case 01:
 	case 02:
 	case 03:
+	case 04:
             while (c--)
                 if (*r++ != *data++)
                     return false;
diff --git a/insns.pl b/insns.pl
index b74017b2..ac03ae3b 100755
--- a/insns.pl
+++ b/insns.pl
@@ -324,7 +324,7 @@ sub count_bytecodes(@) {
 	    next;
 	}
 	$bytecode_count[$bc]++;
-	if ($bc >= 01 && $bc <= 03) {
+	if ($bc >= 01 && $bc <= 04) {
 	    $skip = $bc;
 	} elsif (($bc & ~03) == 010) {
 	    $skip = 1;
@@ -458,11 +458,11 @@ sub hexstr(@) {
 
 # Here we determine the range of possible starting bytes for a given
 # instruction. We need only consider the codes:
-# \1 \2 \3     mean literal bytes, of course
-# \4 \5 \6 \7  mean PUSH/POP of segment registers: special case
+# \[1234]      mean literal bytes, of course
 # \1[0123]     mean byte plus register value
 # \330         means byte plus condition code
 # \0 or \340   mean give up and return empty set
+# \34[4567]    mean PUSH/POP of segment registers: special case
 # \17[234]     skip is4 control byte
 # \26x \270    skip VEX control bytes
 sub startseq($) {
@@ -477,11 +477,11 @@ sub startseq($) {
 
   while ($c0 = shift(@codes)) {
       $c1 = $codes[0];
-      if ($c0 == 01 || $c0 == 02 || $c0 == 03) {
+      if ($c0 >= 01 && $c0 <= 04) {
 	  # Fixed byte string
 	  my $fbs = $prefix;
 	  while (1) {
-	      if ($c0 == 01 || $c0 == 02 || $c0 == 03) {
+	      if ($c0 >= 01 && $c0 <= 04) {
 		  while ($c0--) {
 		      $fbs .= sprintf("%02X", shift(@codes));
 		  }
@@ -565,6 +565,7 @@ sub byte_code_compile($) {
     my %oppos = ();
     my $i;
     my $op, $oq;
+    my $opex;
 
     unless ($str =~ /^(([^\s:]*)\:|)\s*(.*\S)\s*$/) {
 	die "$fname: $line: cannot parse: [$str]\n";
@@ -618,7 +619,8 @@ sub byte_code_compile($) {
 		push(@codes, 0360);
 	    }
 	} elsif ($op =~ /^[0-9a-f]{2}$/) {
-	    if (defined($litix) && $litix+$codes[$litix]+1 == scalar @codes) {
+	    if (defined($litix) && $litix+$codes[$litix]+1 == scalar @codes &&
+		$codes[$litix] < 4) {
 		$codes[$litix]++;
 		push(@codes, hex $op);
 	    } else {
@@ -630,13 +632,17 @@ sub byte_code_compile($) {
 	    if (!defined($oppos{'r'}) || !defined($oppos{'m'})) {
 		die "$fname: $line: $op requires r and m operands\n";
 	    }
-	    push(@codes, 0100 + ($oppos{'m'} << 3) + $oppos{'r'});
+	    $opex = (($oppos{'m'} & 4) ? 06 : 0) |
+		    (($oppos{'r'} & 4) ? 05 : 0);
+	    push(@codes, $opex) if ($opex);
+	    push(@codes, 0100 + (($oppos{'m'} & 3) << 3) + ($oppos{'r'} & 3));
 	    $prefix_ok = 0;
 	} elsif ($op =~ m:^/([0-7])$:) {
 	    if (!defined($oppos{'m'})) {
 		die "$fname: $line: $op requires m operand\n";
 	    }
-	    push(@codes, 0200 + ($oppos{'m'} << 3) + $1);
+	    push(@codes, 06) if ($oppos{'m'} & 4);
+	    push(@codes, 0200 + (($oppos{'m'} & 3) << 3) + $1);
 	    $prefix_ok = 0;
 	} elsif ($op =~ /^vex(|\..*)$/) {
 	    my ($m,$w,$l,$p) = (undef,2,undef,0);
@@ -685,7 +691,7 @@ sub byte_code_compile($) {
 	    if (defined($oppos{'v'}) && !$has_nds) {
 		die "$fname: $line: 'v' operand without vex.nds or vex.ndd\n";
 	    }
-	    push(@codes, defined($oppos{'v'}) ? 0260+$oppos{'v'} : 0270,
+	    push(@codes, defined($oppos{'v'}) ? 0260+($oppos{'v'} & 3) : 0270,
 		 $m, ($w << 3)+($l << 2)+$p);
 	    $prefix_ok = 0;
 	} elsif ($op =~ /^\/drex([01])$/) {
@@ -699,60 +705,79 @@ sub byte_code_compile($) {
 	    # this at (roughly) the position of the drex byte itself.
 	    # This allows us to match the AMD documentation and still
 	    # do the right thing.
-	    unshift(@codes, 0160+$oppos{'d'}+($oc0 ? 4 : 0));
+	    unshift(@codes, 0160+($oppos{'d'} & 3)+($oc0 ? 4 : 0));
+	    unshift(@codes, 05) if ($oppos{'d'} & 4);
 	} elsif ($op =~ /^(ib\,s|ib|ibx|ib\,w|iw|iwd|id|idx|iwdq|rel|rel8|rel16|rel32|iq|seg|ibw|ibd|ibd,s)$/) {
 	    if (!defined($oppos{'i'})) {
 		die "$fname: $line: $op without 'i' operand\n";
 	    }
 	    if ($op eq 'ib,s') { # Signed imm8
-		push(@codes, 014+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 014+($oppos{'i'} & 3));
 	    } elsif ($op eq 'ib') { # imm8
-		push(@codes, 020+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 020+($oppos{'i'} & 3));
 	    } elsif ($op eq 'ib,u') { # Unsigned imm8
-		push(@codes, 024+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 024+($oppos{'i'} & 3));
 	    } elsif ($op eq 'iw') { # imm16
-		push(@codes, 030+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 030+($oppos{'i'} & 3));
 	    } elsif ($op eq 'ibx') { # imm8 sign-extended to opsize
-		push(@codes, 0274+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 0274+($oppos{'i'} & 3));
 	    } elsif ($op eq 'iwd') { # imm16 or imm32, depending on opsize
-		push(@codes, 034+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 034+($oppos{'i'} & 3));
 	    } elsif ($op eq 'id') { # imm32
-		push(@codes, 040+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 040+($oppos{'i'} & 3));
 	    } elsif ($op eq 'idx') { # imm32 extended to 64 bits
-		push(@codes, 0254+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 0254+($oppos{'i'} & 3));
 	    } elsif ($op eq 'iwdq') { # imm16/32/64, depending on opsize
-		push(@codes, 044+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 044+($oppos{'i'} & 3));
 	    } elsif ($op eq 'rel8') {
-		push(@codes, 050+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 050+($oppos{'i'} & 3));
 	    } elsif ($op eq 'iq') {
-		push(@codes, 054+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 054+($oppos{'i'} & 3));
 	    } elsif ($op eq 'rel16') {
-		push(@codes, 060+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 060+($oppos{'i'} & 3));
 	    } elsif ($op eq 'rel') { # 16 or 32 bit relative operand
-		push(@codes, 064+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 064+($oppos{'i'} & 3));
 	    } elsif ($op eq 'rel32') {
-		push(@codes, 070+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 070+($oppos{'i'} & 3));
 	    } elsif ($op eq 'seg') {
-		push(@codes, 074+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 074+($oppos{'i'} & 3));
 	    } elsif ($op eq 'ibw') { # imm16 that can be bytified
 		if (!defined($s_pos)) {
 		    die "$fname: $line: $op without a +s byte\n";
 		}
 		$codes[$s_pos] += 0144;
-		push(@codes, 0140+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 0140+($oppos{'i'} & 3));
 	    } elsif ($op eq 'ibd') { # imm32 that can be bytified
 		if (!defined($s_pos)) {
 		    die "$fname: $line: $op without a +s byte\n";
 		}
 		$codes[$s_pos] += 0154;
-		push(@codes, 0150+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 0150+($oppos{'i'} & 3));
 	    } elsif ($op eq 'ibd,s') {
 		# imm32 that can be bytified, sign extended to 64 bits
 		if (!defined($s_pos)) {
 		    die "$fname: $line: $op without a +s byte\n";
 		}
 		$codes[$s_pos] += 0154;
-		push(@codes, 0250+$oppos{'i'});
+		push(@codes, 05) if ($oppos{'i'} & 4);
+		push(@codes, 0250+($oppos{'i'} & 3));
 	    }
 	    $prefix_ok = 0;
 	} elsif ($op eq '/is4') {
@@ -780,7 +805,8 @@ sub byte_code_compile($) {
 		die "$fname: $line: $op without 'i' operand\n";
 	    }
 	    $s_pos = scalar @codes;
-	    push(@codes, $oppos{'i'}, hex $1);
+	    push(@codes, 05) if ($oppos{'i'} & 4);
+	    push(@codes, $oppos{'i'} & 3, hex $1);
 	    $prefix_ok = 0;
 	} elsif ($op =~ /^([0-9a-f]{2})\+c$/) {
 	    push(@codes, 0330, hex $1);