Better warnings for out-of-range values

Issue better warnings for out-of-range values.  This is not yet
complete.

In particular, note we may have out-of-range for values that end up
being subject to optimization.  That is because the optimization takes
place on the *truncated* value, not the pre-truncated value.

Signed-off-by: H. Peter Anvin <hpa@zytor.com>

4 files changed, 42 insertions, 64 deletions

diff --git a/assemble.c b/assemble.c
index c7d14e83..d6abeac3 100644
--- a/assemble.c
+++ b/assemble.c
@@ -185,13 +185,14 @@ static const char *size_name(int size)
     }
 }
 
-static void warn_overflow(int size, int64_t data)
+static void warn_overflow(int size, const struct operand *o)
 {
-    if (size < 8) {
+    if (size < 8 && o->wrt == NO_SEG && o->segment == NO_SEG) {
 	int64_t lim = ((int64_t)1 << (size*8))-1;
+	int64_t data = o->offset;
 
 	if (data < ~lim || data > lim)
-	    errfunc(ERR_WARNING | ERR_WARN_NOV,
+	    errfunc(ERR_WARNING | ERR_PASS2 | ERR_WARN_NOV,
 		    "%s data exceeds bounds", size_name(size));
     }
 }
@@ -474,21 +475,21 @@ int64_t assemble(int32_t segment, int64_t offset, int bits, uint32_t cp,
                             break;
                         case R_CS:
                             if (bits == 64) {
-                                error(ERR_WARNING,
+                                error(ERR_WARNING | ERR_PASS2,
                                       "cs segment base generated, but will be ignored in 64-bit mode");
                             }
                             c = 0x2E;
                             break;
                         case R_DS:
                             if (bits == 64) {
-                                error(ERR_WARNING,
+                                error(ERR_WARNING | ERR_PASS2,
                                       "ds segment base generated, but will be ignored in 64-bit mode");
                             }
                             c = 0x3E;
                             break;
                         case R_ES:
                            if (bits == 64) {
-                                error(ERR_WARNING,
+                                error(ERR_WARNING | ERR_PASS2,
                                       "es segment base generated, but will be ignored in 64-bit mode");
                            }
                             c = 0x26;
@@ -501,7 +502,7 @@ int64_t assemble(int32_t segment, int64_t offset, int bits, uint32_t cp,
                             break;
                         case R_SS:
                             if (bits == 64) {
-                                error(ERR_WARNING,
+                                error(ERR_WARNING | ERR_PASS2,
                                       "ss segment base generated, but will be ignored in 64-bit mode");
                             }
                             c = 0x36;
@@ -744,8 +745,7 @@ int64_t insn_size(int32_t segment, int64_t offset, int bits, uint32_t cp,
 static bool possible_sbyte(operand *o)
 {
     return !(o->opflags & OPFLAG_FORWARD) &&
-	optimizing >= 0 && !(o->type & STRICT) &&
-	o->wrt == NO_SEG && o->segment == NO_SEG;
+	optimizing >= 0 && !(o->type & STRICT);
 }
 
 /* check that opn[op]  is a signed byte of size 16 or 32 */
@@ -771,23 +771,29 @@ static bool is_sbyte32(operand *o)
     return v >= -128 && v <= 127;
 }
 
-/* check that  opn[op]  is a signed byte of size 32; warn if this is not
+/* check that opn[op] is a signed byte of size 32; warn if this is not
    the original value when extended to 64 bits */
 static bool is_sbyte64(operand *o)
 {
     int64_t v64;
-    int32_t v32;
+    int32_t v;
 
-    /* dead in the water on forward reference or External */
-    if (!possible_sbyte(o))
-	return false;
+    if (!(o->wrt == NO_SEG && o->segment == NO_SEG))
+	return false;		/* Not a pure immediate */
 
     v64 = o->offset;
-    v32 = (int32_t)v64;
+    v = (int32_t)v64;
 
-    warn_overflow(32, v64);
+    if (v64 != v)
+	errfunc(ERR_WARNING | ERR_PASS2 | ERR_WARN_NOV,
+		"signed dword immediate exceeds bounds");
 
-    return v32 >= -128 && v32 <= 127;
+    /* dead in the water on forward reference or External */
+    if (!possible_sbyte(o))
+	return false;
+
+    v = o->offset;
+    return v >= -128 && v <= 127;
 }
 static int64_t calcsize(int32_t segment, int64_t offset, int bits,
                      insn * ins, const uint8_t *codes)
@@ -1265,7 +1271,7 @@ static void gencode(int32_t segment, int64_t offset, int bits,
             if (((opx->type & BITS8) ||
 		 !(opx->type & temp->opd[c & 3] & BYTENESS)) &&
 		(opx->offset < -128 || opx->offset > 127)) {
-                errfunc(ERR_WARNING | ERR_WARN_NOV,
+                errfunc(ERR_WARNING | ERR_PASS2 | ERR_WARN_NOV,
 			"signed byte value exceeds bounds");
 	    }
             if (opx->segment != NO_SEG) {
@@ -1285,7 +1291,7 @@ static void gencode(int32_t segment, int64_t offset, int bits,
         case 022:
 	case 023:
             if (opx->offset < -256 || opx->offset > 255) {
-                errfunc(ERR_WARNING | ERR_WARN_NOV,
+                errfunc(ERR_WARNING | ERR_PASS2 | ERR_WARN_NOV,
 			"byte value exceeds bounds");
             }
             if (opx->segment != NO_SEG) {
@@ -1305,7 +1311,7 @@ static void gencode(int32_t segment, int64_t offset, int bits,
         case 026:
 	case 027:
             if (opx->offset < 0 || opx->offset > 255)
-                errfunc(ERR_WARNING | ERR_WARN_NOV,
+                errfunc(ERR_WARNING | ERR_PASS2 | ERR_WARN_NOV,
 			"unsigned byte value exceeds bounds");
             if (opx->segment != NO_SEG) {
                 data = opx->offset;
@@ -1323,9 +1329,8 @@ static void gencode(int32_t segment, int64_t offset, int bits,
         case 031:
         case 032:
 	case 033:
+	    warn_overflow(2, opx);
             data = opx->offset;
-            if (opx->segment == NO_SEG && opx->wrt == NO_SEG)
-		warn_overflow(2, data);
             out(offset, segment, &data, OUT_ADDRESS, 2,
                 opx->segment, opx->wrt);
             offset += 2;
@@ -1339,9 +1344,8 @@ static void gencode(int32_t segment, int64_t offset, int bits,
                 size = (opx->type & BITS16) ? 2 : 4;
             else
                 size = (bits == 16) ? 2 : 4;
+	    warn_overflow(size, opx);
             data = opx->offset;
-            if (opx->segment == NO_SEG && opx->wrt == NO_SEG)
-		warn_overflow(size, data);
             out(offset, segment, &data, OUT_ADDRESS, size,
                 opx->segment, opx->wrt);
             offset += size;
@@ -1351,9 +1355,8 @@ static void gencode(int32_t segment, int64_t offset, int bits,
         case 041:
         case 042:
 	case 043:
+	    warn_overflow(4, opx);
             data = opx->offset;
-            if (opx->segment == NO_SEG && opx->wrt == NO_SEG)
-		warn_overflow(4, data);
             out(offset, segment, &data, OUT_ADDRESS, 4,
                 opx->segment, opx->wrt);
             offset += 4;
@@ -1365,9 +1368,7 @@ static void gencode(int32_t segment, int64_t offset, int bits,
 	case 047:
             data = opx->offset;
             size = ins->addr_size >> 3;
-	    if (opx->segment == NO_SEG &&
-		opx->wrt == NO_SEG)
-		warn_overflow(size, data);
+	    warn_overflow(size, opx);
             out(offset, segment, &data, OUT_ADDRESS, size,
                 opx->segment, opx->wrt);
             offset += size;
@@ -1472,15 +1473,13 @@ static void gencode(int32_t segment, int64_t offset, int bits,
         case 0142:
 	case 0143:
             data = opx->offset;
+	    warn_overflow(2, opx);
             if (is_sbyte16(opx)) {
                 bytes[0] = data;
                 out(offset, segment, bytes, OUT_RAWDATA, 1, NO_SEG,
                     NO_SEG);
                 offset++;
             } else {
-                if (opx->segment == NO_SEG &&
-                    opx->wrt == NO_SEG)
-		    warn_overflow(2, data);
                 out(offset, segment, &data, OUT_ADDRESS, 2,
                     opx->segment, opx->wrt);
                 offset += 2;
@@ -1504,6 +1503,7 @@ static void gencode(int32_t segment, int64_t offset, int bits,
         case 0152:
 	case 0153:
             data = opx->offset;
+	    warn_overflow(4, opx);
             if (is_sbyte32(opx)) {
                 bytes[0] = data;
                 out(offset, segment, bytes, OUT_RAWDATA, 1, NO_SEG,
@@ -1559,7 +1559,7 @@ static void gencode(int32_t segment, int64_t offset, int bits,
 			c & 7);
 	    } else {
 		if (opx->offset & ~15) {
-		    errfunc(ERR_WARNING | ERR_WARN_NOV,
+		    errfunc(ERR_WARNING | ERR_PASS2 | ERR_WARN_NOV,
 			    "four-bit argument exceeds bounds");
 		}
 		bytes[0] |= opx->offset & 15;
@@ -1590,8 +1590,8 @@ static void gencode(int32_t segment, int64_t offset, int bits,
         case 0252:
 	case 0253:
             data = opx->offset;
-	    /* is_sbyte32() is right here, we have already warned */
-            if (is_sbyte32(opx)) {
+	    warn_overflow(4, opx);
+            if (is_sbyte64(opx)) {
                 bytes[0] = data;
                 out(offset, segment, bytes, OUT_RAWDATA, 1, NO_SEG,
                     NO_SEG);
@@ -1839,7 +1839,7 @@ static void gencode(int32_t segment, int64_t offset, int bits,
                 case 2:
                 case 4:
                     data = ins->oprs[(c >> 3) & 7].offset;
-		    warn_overflow(ea_data.bytes, data);
+		    warn_overflow(ea_data.bytes, opx);
                     out(offset, segment, &data,
                         ea_data.rip ?  OUT_REL4ADR : OUT_ADDRESS,
 			ea_data.bytes,
diff --git a/insns.dat b/insns.dat
index 543634a8..e4a0d784 100644
--- a/insns.dat
+++ b/insns.dat
@@ -61,11 +61,8 @@ ADC		rm16,imm8			\320\1\x83\202\15				8086
 ADC		rm32,imm8			\321\1\x83\202\15				386
 ADC		rm64,imm8			\324\1\x83\202\15				X64
 ADC		reg_al,imm			\1\x14\21					8086,SM
-ADC		reg_ax,sbyte16			\320\1\x83\202\15				8086,SM,ND
 ADC		reg_ax,imm			\320\1\x15\31					8086,SM
-ADC		reg_eax,sbyte32			\321\1\x83\202\15				386,SM,ND
 ADC		reg_eax,imm			\321\1\x15\41					386,SM
-ADC		reg_rax,sbyte64			\324\1\x83\202\15				X64,SM,ND
 ADC		reg_rax,imm			\324\1\x15\41					X64,SM
 ADC		rm8,imm				\1\x80\202\21					8086,SM
 ADC		rm16,imm			\320\145\x81\202\141				8086,SM
@@ -94,11 +91,8 @@ ADD		rm16,imm8			\320\1\x83\200\15				8086
 ADD		rm32,imm8			\321\1\x83\200\15				386
 ADD		rm64,imm8			\324\1\x83\200\15				X64
 ADD		reg_al,imm			\1\x04\21					8086,SM
-ADD		reg_ax,sbyte16			\320\1\x83\200\15				8086,SM,ND
 ADD		reg_ax,imm			\320\1\x05\31					8086,SM
-ADD		reg_eax,sbyte32			\321\1\x83\200\15				386,SM,ND
 ADD		reg_eax,imm			\321\1\x05\41					386,SM
-ADD		reg_rax,sbyte64			\324\1\x83\200\15				X64,SM,ND
 ADD		reg_rax,imm			\324\1\x05\41					X64,SM
 ADD		rm8,imm				\1\x80\200\21					8086,SM
 ADD		rm16,imm			\320\145\x81\200\141				8086,SM
@@ -127,11 +121,8 @@ AND		rm16,imm8			\320\1\x83\204\15				8086
 AND		rm32,imm8			\321\1\x83\204\15				386
 AND		rm64,imm8			\324\1\x83\204\15				X64
 AND		reg_al,imm			\1\x24\21					8086,SM
-AND		reg_ax,sbyte16			\320\1\x83\204\15				8086,SM,ND
 AND		reg_ax,imm			\320\1\x25\31					8086,SM
-AND		reg_eax,sbyte32			\321\1\x83\204\15				386,SM,ND
 AND		reg_eax,imm			\321\1\x25\41					386,SM
-AND		reg_rax,sbyte64			\324\1\x83\204\15				X64,SM,ND
 AND		reg_rax,imm			\324\1\x25\41					X64,SM
 AND		rm8,imm				\1\x80\204\21					8086,SM
 AND		rm16,imm			\320\145\x81\204\141				8086,SM
@@ -255,11 +246,8 @@ CMP		rm16,imm8			\320\1\x83\207\15				8086
 CMP		rm32,imm8			\321\1\x83\207\15				386
 CMP		rm64,imm8			\324\1\x83\207\15				X64
 CMP		reg_al,imm			\1\x3C\21					8086,SM
-CMP		reg_ax,sbyte16			\320\1\x83\207\15				8086,SM,ND
 CMP		reg_ax,imm			\320\1\x3D\31					8086,SM
-CMP		reg_eax,sbyte32			\321\1\x83\207\15				386,SM,ND
 CMP		reg_eax,imm			\321\1\x3D\41					386,SM
-CMP		reg_rax,sbyte64			\324\1\x83\207\15				X64,SM,ND
 CMP		reg_rax,imm			\324\1\x3D\41					X64,SM
 CMP		rm8,imm				\1\x80\207\21					8086,SM
 CMP		rm16,imm			\320\145\x81\207\141				8086,SM
@@ -856,11 +844,8 @@ OR		rm16,imm8			\320\1\x83\201\15				8086
 OR		rm32,imm8			\321\1\x83\201\15				386
 OR		rm64,imm8			\324\1\x83\201\15				X64
 OR		reg_al,imm			\1\x0C\21					8086,SM
-OR		reg_ax,sbyte16			\320\1\x83\201\15				8086,SM,ND
 OR		reg_ax,imm			\320\1\x0D\31					8086,SM
-OR		reg_eax,sbyte32			\321\1\x83\201\15				386,SM,ND
 OR		reg_eax,imm			\321\1\x0D\41					386,SM
-OR		reg_rax,sbyte64			\324\1\x83\201\15				X64,SM,ND
 OR		reg_rax,imm			\324\1\x0D\41					X64,SM
 OR		rm8,imm				\1\x80\201\21					8086,SM
 OR		rm16,imm			\320\145\x81\201\141				8086,SM
@@ -1111,11 +1096,8 @@ SBB		rm16,imm8			\320\1\x83\203\15				8086
 SBB		rm32,imm8			\321\1\x83\203\15				386
 SBB		rm64,imm8			\324\1\x83\203\15				X64
 SBB		reg_al,imm			\1\x1C\21					8086,SM
-SBB		reg_ax,sbyte16			\320\1\x83\203\15				8086,SM,ND
 SBB		reg_ax,imm			\320\1\x1D\31					8086,SM
-SBB		reg_eax,sbyte32			\321\1\x83\203\15				386,SM,ND
 SBB		reg_eax,imm			\321\1\x1D\41					386,SM
-SBB		reg_rax,sbyte64			\324\1\x83\203\15				X64,SM,ND
 SBB		reg_rax,imm			\324\1\x1D\41					X64,SM
 SBB		rm8,imm				\1\x80\203\21					8086,SM
 SBB		rm16,imm			\320\145\x81\203\141				8086,SM
@@ -1227,11 +1209,8 @@ SUB		rm16,imm8			\320\1\x83\205\15				8086
 SUB		rm32,imm8			\321\1\x83\205\15				386
 SUB		rm64,imm8			\324\1\x83\205\15				X64
 SUB		reg_al,imm			\1\x2C\21					8086,SM
-SUB		reg_ax,sbyte16			\320\1\x83\205\15				8086,SM,ND
 SUB		reg_ax,imm			\320\1\x2D\31					8086,SM
-SUB		reg_eax,sbyte32			\321\1\x83\205\15				386,SM,ND
 SUB		reg_eax,imm			\321\1\x2D\41					386,SM
-SUB		reg_rax,sbyte64			\324\1\x83\205\15				X64,SM,ND
 SUB		reg_rax,imm			\324\1\x2D\41					X64,SM
 SUB		rm8,imm				\1\x80\205\21					8086,SM
 SUB		rm16,imm			\320\145\x81\205\141				8086,SM
@@ -1358,11 +1337,8 @@ XOR		rm16,imm8			\320\1\x83\206\15				8086
 XOR		rm32,imm8			\321\1\x83\206\15				386
 XOR		rm64,imm8			\324\1\x83\206\15				X64
 XOR		reg_al,imm			\1\x34\21					8086,SM
-XOR		reg_ax,sbyte16			\320\1\x83\206\15				8086,SM,ND
 XOR		reg_ax,imm			\320\1\x35\31					8086,SM
-XOR		reg_eax,sbyte32			\321\1\x83\206\15				386,SM,ND
 XOR		reg_eax,imm			\321\1\x35\41					386,SM
-XOR		reg_rax,sbyte64			\324\1\x83\206\15				X64,SM,ND
 XOR		reg_rax,imm			\324\1\x35\41					X64,SM
 XOR		rm8,imm				\1\x80\206\21					8086,SM
 XOR		rm16,imm			\320\145\x81\206\141				8086,SM
diff --git a/nasm.c b/nasm.c
index 7bdc96e2..76d1eee0 100644
--- a/nasm.c
+++ b/nasm.c
@@ -1888,7 +1888,8 @@ static bool is_suppressed_warning(int severity)
 	(((severity & ERR_WARN_MASK) != 0 &&
 	  !warning_on[(severity & ERR_WARN_MASK) >> ERR_WARN_SHR]) ||
 	 /* See if it's a pass-one only warning and we're not in pass one. */
-	 ((severity & ERR_PASS1) && pass0 != 1));
+	 ((severity & ERR_PASS1) && pass0 != 1) ||
+	 ((severity & ERR_PASS2) && pass0 != 2));
 }
 
 /**
diff --git a/nasmlib.h b/nasmlib.h
index 9cc285db..d001c4c2 100644
--- a/nasmlib.h
+++ b/nasmlib.h
@@ -71,14 +71,15 @@ extern efunc nasm_malloc_error;
 #define ERR_NOFILE	0x00000010      /* don't give source file name/line */
 #define ERR_USAGE	0x00000020      /* print a usage message */
 #define ERR_PASS1	0x00000040      /* only print this error on pass one */
-#define ERR_NO_SEVERITY	0x00000080      /* suppress printing severity */
+#define ERR_PASS2	0x00000080
+#define ERR_NO_SEVERITY	0x00000100      /* suppress printing severity */
 
 /*
  * These codes define specific types of suppressible warning.
  */
 
-#define ERR_WARN_MASK	0x0000FF00      /* the mask for this feature */
-#define ERR_WARN_SHR 	8               /* how far to shift right */
+#define ERR_WARN_MASK	0xFFFFF000      /* the mask for this feature */
+#define ERR_WARN_SHR 	12               /* how far to shift right */
 
 #define WARN(x) ((x) << ERR_WARN_SHR)