errors: be more robust in handling unexpected fatal errors

Introduce a new error level, ERR_CRITICAL, beyond which we will
minimize the amount of code that will be executed before we die; in
particular don't execute any memory allocations, and if we somehow end
up recursing, abort() immediately.

Basically, "less than panic, more than fatal."

At this point this level is used by nasm_alloc_failed().

Signed-off-by: H. Peter Anvin <hpa@zytor.com>

7 files changed, 136 insertions, 94 deletions

diff --git a/asm/error.c b/asm/error.c
index 95d7bef4..52f812b8 100644
--- a/asm/error.c
+++ b/asm/error.c
@@ -41,33 +41,38 @@
 #include "nasmlib.h"
 #include "error.h"
 
-/* Global error handling function */
-vefunc nasm_verror;
+/*
+ * Global error handling function. If we call this before it is
+ * initialized, it is a fatal error!
+ */
+vefunc nasm_verror = (vefunc)nasm_verror_critical;
 
 /* Common function body */
-#define nasm_do_error(s)			\
-	va_list ap;				\
-        va_start(ap, fmt);                      \
-	nasm_verror((s), fmt, ap);		\
-	va_end(ap);
+#define nasm_do_error(_sev,_flags)				\
+	va_list ap;						\
+        va_start(ap, fmt);					\
+	if ((_sev) >= ERR_CRITICAL)				\
+		nasm_verror_critical((_sev)|(_flags), fmt, ap);	\
+	else							\
+		nasm_verror((_sev)|(_flags), fmt, ap);		\
+	va_end(ap);						\
+	if ((_sev) >= ERR_FATAL)                                \
+		abort();
+
 
 void nasm_error(errflags severity, const char *fmt, ...)
 {
-	nasm_do_error(severity);
+	nasm_do_error(severity & ERR_MASK, severity & ~ERR_MASK);
 }
 
 #define nasm_err_helpers(_type, _name, _sev)				\
 _type nasm_ ## _name ## f (errflags flags, const char *fmt, ...)	\
 {									\
-	nasm_do_error((_sev)|flags);					\
-	if (_sev >= ERR_FATAL)						\
-		abort();						\
+	nasm_do_error(_sev, flags);					\
 }									\
 _type nasm_ ## _name (const char *fmt, ...)				\
 {									\
-	nasm_do_error(_sev);						\
-	if (_sev >= ERR_FATAL)						\
-		abort();						\
+	nasm_do_error(_sev, 0);						\
 }
 
 nasm_err_helpers(void,       listmsg,  ERR_LISTMSG)
@@ -75,6 +80,7 @@ nasm_err_helpers(void,       debug,    ERR_DEBUG)
 nasm_err_helpers(void,       info,     ERR_INFO)
 nasm_err_helpers(void,       nonfatal, ERR_NONFATAL)
 nasm_err_helpers(fatal_func, fatal,    ERR_FATAL)
+nasm_err_helpers(fatal_func, critical, ERR_CRITICAL)
 nasm_err_helpers(fatal_func, panic,    ERR_PANIC)
 
 /*
@@ -82,9 +88,9 @@ nasm_err_helpers(fatal_func, panic,    ERR_PANIC)
  * This means nasm_warn() is the equivalent of the -f variants of the
  * other ones.
  */
-void nasm_warn(errflags severity, const char *fmt, ...)
+void nasm_warn(errflags flags, const char *fmt, ...)
 {
-	nasm_do_error(ERR_WARNING|severity);
+	nasm_do_error(ERR_WARNING, flags);
 }
 
 fatal_func nasm_panic_from_macro(const char *file, int line)
diff --git a/asm/nasm.c b/asm/nasm.c
index ac005edb..b7a32cb5 100644
--- a/asm/nasm.c
+++ b/asm/nasm.c
@@ -472,14 +472,15 @@ static void timestamp(void)
 
 int main(int argc, char **argv)
 {
+    /* Do these as early as possible */
+    error_file = stderr;
+    nasm_set_verror(nasm_verror_asm);
     _progname = argv[0];
     if (!_progname || !_progname[0])
         _progname = "nasm";
 
     timestamp();
 
-    error_file = stderr;
-
     iflag_set_default_cpu(&cpu);
     iflag_set_default_cpu(&cmd_cpu);
 
@@ -491,7 +492,6 @@ int main(int argc, char **argv)
     _passn = 0;
 
     want_usage = terminate_after_phase = false;
-    nasm_set_verror(nasm_verror_asm);
 
     nasm_ctype_init();
     src_init();
@@ -1802,6 +1802,80 @@ static errflags true_error_type(errflags severity)
     return type;
 }
 
+/*
+ * The various error type prefixes
+ */
+static const char * const error_pfx_table[ERR_MASK+1] = {
+    ";;; ", "debug: ", "info: ", "warning: ",
+        "error: ", "fatal: ", "critical: ", "panic: "
+};
+static const char no_file_name[] = "nasm"; /* What to print if no file name */
+
+/*
+ * For fatal/critical/panic errors, kill this process.
+ */
+static fatal_func die_hard(errflags true_type, errflags severity)
+{
+    fflush(NULL);
+
+    if (true_type == ERR_PANIC && abort_on_panic)
+        abort();
+
+    if (ofile) {
+        fclose(ofile);
+        if (!keep_all)
+            remove(outname);
+        ofile = NULL;
+    }
+
+    if (severity & ERR_USAGE)
+        usage();
+
+    /* Terminate immediately */
+    exit(true_type - ERR_FATAL + 1);
+}
+
+/*
+ * error reporting for critical and panic errors: minimize
+ * the amount of system dependencies for getting a message out,
+ * and in particular try to avoid memory allocations.
+ */
+fatal_func nasm_verror_critical(errflags severity, const char *fmt, va_list args)
+{
+    const char *currentfile = no_file_name;
+    int32_t lineno = 0;
+    errflags true_type = severity & ERR_MASK;
+    static bool been_here = false;
+
+    if (unlikely(been_here))
+        abort();                /* Recursive error... just die */
+
+    been_here = true;
+
+    if (!(severity & ERR_NOFILE)) {
+        src_get(&lineno, &currentfile);
+        if (!currentfile) {
+            currentfile =
+                inname && inname[0] ? inname :
+                outname && outname[0] ? outname :
+                no_file_name;
+                lineno = 0;
+        }
+    }
+
+    fputs(error_pfx_table[severity], error_file);
+    fputs(currentfile, error_file);
+    if (lineno) {
+        fprintf(error_file, "%s%"PRId32"%s",
+                errfmt->beforeline, lineno, errfmt->afterline);
+    }
+    fputs(errfmt->beforemsg, error_file);
+    vfprintf(error_file, fmt, args);
+    fputc('\n', error_file);
+
+    die_hard(true_type, severity);
+}
+
 /**
  * common error reporting
  * This is the common back end of the error reporting schemes currently
@@ -1821,10 +1895,9 @@ static void nasm_verror_asm(errflags severity, const char *fmt, va_list args)
     errflags true_type = true_error_type(severity);
     const char *currentfile = NULL;
     int32_t lineno = 0;
-    static const char * const pfx_table[ERR_MASK+1] = {
-        ";;; ", "debug: ", "info: ", "warning: ",
-        "error: ", "", "fatal: ", "panic: "
-    };
+
+    if (true_type >= ERR_CRITICAL)
+        nasm_verror_critical(severity, fmt, args);
 
     if (is_suppressed(severity))
         return;
@@ -1832,18 +1905,18 @@ static void nasm_verror_asm(errflags severity, const char *fmt, va_list args)
     if (!(severity & ERR_NOFILE)) {
         src_get(&lineno, &currentfile);
         if (!currentfile) {
-            currentfile = currentfile ? currentfile :
+            currentfile =
                 inname && inname[0] ? inname :
                 outname && outname[0] ? outname :
                 NULL;
-            lineno = 0;
+                lineno = 0;
         }
     }
 
     if (severity & ERR_NO_SEVERITY)
         pfx = "";
     else
-        pfx = pfx_table[true_type];
+        pfx = error_pfx_table[true_type];
 
     vsnprintf(msg, sizeof msg, fmt, args);
     *warnsuf = 0;
@@ -1864,7 +1937,7 @@ static void nasm_verror_asm(errflags severity, const char *fmt, va_list args)
     }
 
     if (!skip_this_pass(severity)) {
-        const char *file = currentfile ? currentfile : "nasm";
+        const char *file = currentfile ? currentfile : no_file_name;
         const char *here = "";
 
         if (severity & ERR_HERE)
@@ -1922,52 +1995,15 @@ static void nasm_verror_asm(errflags severity, const char *fmt, va_list args)
     if (skip_this_pass(severity))
         return;
 
-    if (severity & ERR_USAGE)
-        want_usage = true;
-
     /* error_list_macros can for obvious reasons not work with ERR_HERE */
     if (!(severity & ERR_HERE))
         if (preproc)
             preproc->error_list_macros(severity);
 
-    switch (true_type) {
-    case ERR_LISTMSG:
-    case ERR_DEBUG:
-    case ERR_INFO:
-    case ERR_WARNING:
-        /* no further action, by definition */
-        break;
-    case ERR_NONFATAL:
+    if (true_type >= ERR_FATAL)
+        die_hard(true_type, severity);
+    else if (true_type >= ERR_NONFATAL)
         terminate_after_phase = true;
-        break;
-    case ERR_FATAL:
-        if (ofile) {
-            fclose(ofile);
-            if (!keep_all)
-                remove(outname);
-            ofile = NULL;
-        }
-        if (want_usage)
-            usage();
-        exit(1);                /* instantly die */
-        break;                  /* placate silly compilers */
-    case ERR_PANIC:
-        fflush(NULL);
-
-        if (abort_on_panic)
-            abort();            /* halt, catch fire, dump core/stop debugger */
-
-        if (ofile) {
-            fclose(ofile);
-            if (!keep_all)
-                remove(outname);
-            ofile = NULL;
-        }
-        exit(3);
-        break;
-    default:
-        break;                  /* ??? */
-    }
 }
 
 static void usage(void)
diff --git a/disasm/ndisasm.c b/disasm/ndisasm.c
index 6b4ca49d..3269c367 100644
--- a/disasm/ndisasm.c
+++ b/disasm/ndisasm.c
@@ -65,12 +65,18 @@ static const char *help =
 static void output_ins(uint64_t, uint8_t *, int, char *);
 static void skip(uint32_t dist, FILE * fp);
 
-static void ndisasm_verror(errflags severity, const char *fmt, va_list va)
+fatal_func nasm_verror_critical(errflags severity, const char *fmt, va_list val)
 {
-    vfprintf(stderr, fmt, va);
+    vfprintf(stderr, fmt, val);
+    exit((severity & ERR_MASK) - ERR_FATAL + 2);
+}
 
-    if (severity & ERR_FATAL)
-	exit(1);
+static void ndisasm_verror(errflags severity, const char *fmt, va_list val)
+{
+    if ((severity & ERR_MASK) >= ERR_FATAL)
+        nasm_verror_critical(severity, fmt, val);
+    else
+        vfprintf(stderr, fmt, val);
 }
 
 int main(int argc, char **argv)
diff --git a/include/error.h b/include/error.h
index fde99e01..07bd5c7d 100644
--- a/include/error.h
+++ b/include/error.h
@@ -65,6 +65,8 @@ void printf_func(1, 2) nasm_nonfatal(const char *fmt, ...);
 void printf_func(2, 3) nasm_nonfatalf(errflags flags, const char *fmt, ...);
 fatal_func printf_func(1, 2) nasm_fatal(const char *fmt, ...);
 fatal_func printf_func(2, 3) nasm_fatalf(errflags flags, const char *fmt, ...);
+fatal_func printf_func(1, 2) nasm_critical(const char *fmt, ...);
+fatal_func printf_func(2, 3) nasm_criticalf(errflags flags, const char *fmt, ...);
 fatal_func printf_func(1, 2) nasm_panic(const char *fmt, ...);
 fatal_func printf_func(2, 3) nasm_panicf(errflags flags, const char *fmt, ...);
 fatal_func nasm_panic_from_macro(const char *file, int line);
@@ -72,6 +74,7 @@ fatal_func nasm_panic_from_macro(const char *file, int line);
 
 typedef void (*vefunc) (errflags severity, const char *fmt, va_list ap);
 extern vefunc nasm_verror;
+fatal_func nasm_verror_critical(errflags severity, const char *fmt, va_list ap);
 
 static inline vefunc nasm_set_verror(vefunc ve)
 {
@@ -89,7 +92,8 @@ static inline vefunc nasm_set_verror(vefunc ve)
 #define ERR_INFO		0x00000002	/* information for the list file */
 #define ERR_WARNING		0x00000003	/* warn only: no further action */
 #define ERR_NONFATAL		0x00000004	/* terminate assembly after phase */
-#define ERR_FATAL		0x00000006	/* instantly fatal: exit with error */
+#define ERR_FATAL		0x00000005	/* instantly fatal: exit with error */
+#define ERR_CRITICAL		0x00000006      /* fatal, but minimize code before exit */
 #define ERR_PANIC		0x00000007	/* internal error: panic instantly
 						 * and dump core for reference */
 #define ERR_MASK		0x00000007	/* mask off the above codes */
@@ -110,7 +114,7 @@ static inline vefunc nasm_set_verror(vefunc ve)
  * severity code.
  */
 #define WARN_SHR		12              /* how far to shift right */
-#define WARN_IDX(x)     	(((errflags)(x)) >> WARN_SHR)
+#define WARN_IDX(x)		(((errflags)(x)) >> WARN_SHR)
 #define WARN_MASK		((~(errflags)0) << WARN_SHR)
 
 /* This is a bitmask */
diff --git a/nasmlib/alloc.c b/nasmlib/alloc.c
index b0d623a2..e25e0e0a 100644
--- a/nasmlib/alloc.c
+++ b/nasmlib/alloc.c
@@ -42,25 +42,9 @@
 
 size_t _nasm_last_string_size;
 
-no_return nasm_alloc_failed(void)
+fatal_func nasm_alloc_failed(void)
 {
-    /* If nasm_fatal() gets us back here, then croak hard */
-    static bool already_here = false;
-    FILE *errfile;
-
-    if (likely(!already_here)) {
-        already_here = true;
-        nasm_fatal("out of memory!");
-    }
-
-    errfile = error_file;
-    if (!errfile)
-        error_file = stderr;
-
-    fprintf(error_file, "nasm: out of memory!\n");
-    fflush(error_file);
-    fflush(NULL);
-    abort();
+    nasm_critical("out of memory!");
 }
 
 void *nasm_malloc(size_t size)
diff --git a/nasmlib/alloc.h b/nasmlib/alloc.h
index 1b896585..9b8ad192 100644
--- a/nasmlib/alloc.h
+++ b/nasmlib/alloc.h
@@ -36,7 +36,7 @@
 
 #include "compiler.h"
 
-no_return nasm_alloc_failed(void);
+fatal_func nasm_alloc_failed(void);
 
 static inline void *validate_ptr(void *p)
 {
diff --git a/rdoff/rdoff.c b/rdoff/rdoff.c
index 3fd2a0e9..bdba82ae 100644
--- a/rdoff/rdoff.c
+++ b/rdoff/rdoff.c
@@ -228,12 +228,18 @@ int rdf_errno = 0;
 /* ========================================================================
  * Hook for nasm_error() to work
  * ======================================================================== */
-static void rdoff_verror(errflags severity, const char *fmt, va_list val)
+fatal_func nasm_verror_critical(errflags severity, const char *fmt, va_list val)
 {
     vfprintf(stderr, fmt, val);
+    exit((severity & ERR_MASK) - ERR_FATAL + 2);
+}
 
+static void rdoff_verror(errflags severity, const char *fmt, va_list val)
+{
     if ((severity & ERR_MASK) >= ERR_FATAL)
-        exit(1);
+        nasm_verror_critical(severity, fmt, val);
+    else
+        vfprintf(stderr, fmt, val);
 }
 
 void rdoff_init(void)