Add option to allow 64-bit code in any output format

Add the option --allow-64-bit to permit the generation of 64-bit code
even for a 16/32-bit output format.

Using NASM to do some boot strapping code and ran into trouble when
trying to emit a few 64-bit instructions in the OMF object file doing
the mode switching.  While I can see how the "error: obj output format
does not support 64-bit code" message can be a useful reality check
for application programmers, it prevents low-level programmers from
doing what they want.  It if was just a harmless warning, it wouldn't
be so bad, but it turns BITS 64 into BITS 16. The main trick to mixing
64-bit code into OMF and other 32-bit output formats is to avoid
64-bit sized fixups, which normally isn't too hard.

[hpa: shortened the option name to --allow-64-bit, minor code cleanups]

Signed-off-by: Knut St. Osmundsen <bird-nasm@anduin.net>
Signed-off-by: H. Peter Anvin <hpa@zytor.com>

2 files changed, 38 insertions, 15 deletions

diff --git a/doc/nasmdoc.src b/doc/nasmdoc.src
index e4682480..ec0cfcca 100644
--- a/doc/nasmdoc.src
+++ b/doc/nasmdoc.src
@@ -1,6 +1,6 @@
 \# --------------------------------------------------------------------------
 \#
-\#   Copyright 1996-2014 The NASM Authors - All Rights Reserved
+\#   Copyright 1996-2016 The NASM Authors - All Rights Reserved
 \#   See the file AUTHORS included with the NASM distribution for
 \#   the specific copyright holders.
 \#
@@ -1006,8 +1006,18 @@ is indicated by an asterisk. For example:
 The \c{--prefix} and \c{--postfix} options prepend or append
 (respectively) the given argument to all \c{global} or
 \c{extern} variables. E.g. \c{--prefix _} will prepend the
-underscore to all global and external variables, as C sometimes
-(but not always) likes it.
+underscore to all global and external variables, as C requires it in
+some, but not all, system calling conventions.
+
+
+\S{opt-allow64bit} The \i\c{--allow-64-bit} Option.
+
+The \c{--allow-64-bit} option allows using 64-bit instructions in a
+32-bit or 16-bit output format.  This is useful in a few situations,
+such as when writing code switching from 32-bit to 64-bit mode and
+linking into a 32-bit module.  However, it is important to be aware of
+the restriction the output format poses on you in terms of
+relocations.  Use with caution!
 
 
 \S{nasmenv} The \i\c{NASMENV} \i{Environment} Variable
diff --git a/nasm.c b/nasm.c
index 8557b4db..17ee5538 100644
--- a/nasm.c
+++ b/nasm.c
@@ -1,6 +1,6 @@
 /* ----------------------------------------------------------------------- *
  *
- *   Copyright 1996-2013 The NASM Authors - All Rights Reserved
+ *   Copyright 1996-2016 The NASM Authors - All Rights Reserved
  *   See the file AUTHORS included with the NASM distribution for
  *   the specific copyright holders.
  *
@@ -88,6 +88,7 @@ static int using_debug_info, opt_verbose_info;
 bool tasm_compatible_mode = false;
 int pass0, passn;
 int maxbits = 0;
+static bool allow_64_bit = false;
 int globalrel = 0;
 int globalbnd = 0;
 
@@ -614,11 +615,15 @@ struct textargs {
     int value;
 };
 
-#define OPT_PREFIX 0
-#define OPT_POSTFIX 1
+enum text_options {
+    OPT_PREFIX,
+    OPT_POSTFIX,
+    OPT_ALLOW_64_BIT
+};
 struct textargs textopts[] = {
     {"prefix", OPT_PREFIX},
     {"postfix", OPT_POSTFIX},
+    {"allow-64-bit", OPT_ALLOW_64_BIT},
     {NULL, 0}
 };
 
@@ -804,7 +809,10 @@ static bool process_arg(char *p, char *q)
                  "    -h          show invocation summary and exit\n\n"
                  "--prefix,--postfix\n"
                  "  this options prepend or append the given argument to all\n"
-                 "  extern and global variables\n\n"
+                 "  extern and global variables\n"
+                 "--allow-64-bit\n"
+                 "  do not restrict 64-bit code to 64-bit capable output\n"
+                 "  formats (use with care, no complaining)\n\n"
                  "Warnings:\n");
             for (i = 0; i <= ERR_WARN_MAX; i++)
                 printf("    %-23s %s (default %s)\n",
@@ -959,18 +967,23 @@ set_warning:
                             advance = 1, param = q;
                         }
 
-                        if (s == OPT_PREFIX) {
-                            strncpy(lprefix, param, PREFIX_MAX - 1);
-                            lprefix[PREFIX_MAX - 1] = 0;
+                        switch (s) {
+                        case OPT_PREFIX:
+                            strlcpy(lprefix, param, PREFIX_MAX);
                             break;
-                        }
-                        if (s == OPT_POSTFIX) {
-                            strncpy(lpostfix, param, POSTFIX_MAX - 1);
-                            lpostfix[POSTFIX_MAX - 1] = 0;
+                        case OPT_POSTFIX:
+                            strlcpy(lpostfix, param, POSTFIX_MAX);
+                            break;
+                        default:
+                            nasm_error(ERR_PANIC | ERR_NOFILE,
+                                       "internal error");
                             break;
                         }
                         break;
                     }
+                case OPT_ALLOW_64_BIT:
+                    allow_64_bit = true;
+                    break;
                 default:
                     {
                         nasm_error(ERR_NONFATAL | ERR_NOFILE | ERR_USAGE,
@@ -2088,7 +2101,7 @@ static int get_bits(char *value)
                          "cannot specify 64-bit segment on processor below an x86-64");
             i = 16;
         }
-        if (i != maxbits) {
+        if (i != maxbits && !allow_64_bit) {
             nasm_error(ERR_NONFATAL,
                          "%s output format does not support 64-bit code",
                          ofmt->shortname);