assemble: Only treat a displacement as signed if it is < asizesignrel

Only generate a signed relocation if the displacement size is less
than the address size.  This matters when involving address size
overrides.

It is technically impossible to do this one perfectly, because it is
never really knowable if the displacement offset is used as a base or
an index.

Signed-off-by: H. Peter Anvin <hpa@zytor.com>

2 files changed, 22 insertions, 17 deletions

diff --git a/assemble.c b/assemble.c
index 52a352fa..364e6c90 100644
--- a/assemble.c
+++ b/assemble.c
@@ -1849,13 +1849,7 @@ static void gencode(int32_t segment, int64_t offset, int bits,
                 offset += s;
                 s = 0;
 
-                switch (ea_data.bytes) {
-                case 0:
-                    break;
-                case 1:
-                case 2:
-                case 4:
-                case 8:
+                if (ea_data.bytes) {
                     /* use compressed displacement, if available */
                     data = ea_data.disp8 ? ea_data.disp8 : opy->offset;
                     s += ea_data.bytes;
@@ -1872,21 +1866,25 @@ static void gencode(int32_t segment, int64_t offset, int bits,
                                 insn_end - offset, opy->segment, opy->wrt);
                         }
                     } else {
-                        if (overflow_general(data, ins->addr_size >> 3) ||
+                        int asize = ins->addr_size >> 3;
+                        int atype = ea_data.bytes;
+
+                        if (overflow_general(data, asize) ||
                             signed_bits(data, ins->addr_size) !=
-                            signed_bits(data, ea_data.bytes * 8))
+                            signed_bits(data, ea_data.bytes << 3))
                             warn_overflow(ERR_PASS2, ea_data.bytes);
 
+                        if (asize > ea_data.bytes) {
+                            /*
+                             * If the address isn't the full width of
+                             * the address size, treat is as signed...
+                             */
+                            atype = -atype;
+                        }
+
                         out(offset, segment, &data, OUT_ADDRESS,
-                            -ea_data.bytes, opy->segment, opy->wrt);
+                            atype, opy->segment, opy->wrt);
                     }
-                    break;
-                default:
-                    /* Impossible! */
-                    errfunc(ERR_PANIC,
-                            "Invalid amount of bytes (%d) for offset?!",
-                            ea_data.bytes);
-                    break;
                 }
                 offset += s;
             }
diff --git a/test/relocs.asm b/test/relocs.asm
index a293f50f..cc560f28 100644
--- a/test/relocs.asm
+++ b/test/relocs.asm
@@ -5,6 +5,13 @@
 	mov rax,[foo]
 	mov rax,[qword foo]
 
+	mov eax,[a32 foo]
+	mov rax,[a32 foo]
+	mov rax,[a32 qword foo]
+
+	mov eax,foo
+	mov rax,dword foo
+	mov rax,qword foo
 	mov eax,foo
 	mov rax,dword foo
 	mov rax,qword foo