malloc: handle potential infinite loop in nasm_alloc_failed()

It is possible on memory exhaustion that nasm_fatal() might cause
another allocation error, thus calling nasm_alloc_failed() again. If
we find us in nasm_alloc_failed() for a second time, try to get a
message out and then call abort().

Signed-off-by: H. Peter Anvin (Intel) <hpa@zytor.com>

7 files changed, 28 insertions, 10 deletions

diff --git a/Makefile.in b/Makefile.in
index 949d0131..32ef3d91 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -96,7 +96,7 @@ LIBOBJ = stdlib/snprintf.$(O) stdlib/vsnprintf.$(O) stdlib/strlcpy.$(O) \
 	stdlib/strnlen.$(O) stdlib/strrchrnul.$(O) \
 	\
 	nasmlib/ver.$(O) \
-	nasmlib/crc64.$(O) nasmlib/malloc.$(O) \
+	nasmlib/crc64.$(O) nasmlib/malloc.$(O) nasmlib/errfile.$(O) \
 	nasmlib/md5c.$(O) nasmlib/string.$(O) \
 	nasmlib/file.$(O) nasmlib/mmap.$(O) nasmlib/ilog2.$(O) \
 	nasmlib/realpath.$(O) nasmlib/path.$(O) \
diff --git a/Mkfiles/msvc.mak b/Mkfiles/msvc.mak
index 9aec80fc..c7bd9109 100644
--- a/Mkfiles/msvc.mak
+++ b/Mkfiles/msvc.mak
@@ -68,7 +68,7 @@ LIBOBJ = stdlib\snprintf.$(O) stdlib\vsnprintf.$(O) stdlib\strlcpy.$(O) \
 	stdlib\strnlen.$(O) stdlib\strrchrnul.$(O) \
 	\
 	nasmlib\ver.$(O) \
-	nasmlib\crc64.$(O) nasmlib\malloc.$(O) \
+	nasmlib\crc64.$(O) nasmlib\malloc.$(O) nasmlib\errfile.$(O) \
 	nasmlib\md5c.$(O) nasmlib\string.$(O) \
 	nasmlib\file.$(O) nasmlib\mmap.$(O) nasmlib\ilog2.$(O) \
 	nasmlib\realpath.$(O) nasmlib\path.$(O) \
diff --git a/Mkfiles/openwcom.mak b/Mkfiles/openwcom.mak
index 084014bb..11582ecf 100644
--- a/Mkfiles/openwcom.mak
+++ b/Mkfiles/openwcom.mak
@@ -57,7 +57,7 @@ LIBOBJ = stdlib\snprintf.$(O) stdlib\vsnprintf.$(O) stdlib\strlcpy.$(O) &
 	stdlib\strnlen.$(O) stdlib\strrchrnul.$(O) &
 	&
 	nasmlib\ver.$(O) &
-	nasmlib\crc64.$(O) nasmlib\malloc.$(O) &
+	nasmlib\crc64.$(O) nasmlib\malloc.$(O) nasmlib\errfile.$(O) &
 	nasmlib\md5c.$(O) nasmlib\string.$(O) &
 	nasmlib\file.$(O) nasmlib\mmap.$(O) nasmlib\ilog2.$(O) &
 	nasmlib\realpath.$(O) nasmlib\path.$(O) &
diff --git a/asm/nasm.c b/asm/nasm.c
index ddc3404e..1825aa35 100644
--- a/asm/nasm.c
+++ b/asm/nasm.c
@@ -121,7 +121,7 @@ const struct ofmt *ofmt = &OF_DEFAULT;
 const struct ofmt_alias *ofmt_alias = NULL;
 const struct dfmt *dfmt;
 
-static FILE *error_file;        /* Where to write error messages */
+FILE *error_file;               /* Where to write error messages */
 
 FILE *ofile = NULL;
 struct optimization optimizing =
@@ -455,6 +455,8 @@ int main(int argc, char **argv)
 
     timestamp();
 
+    error_file = stderr;
+
     iflag_set_default_cpu(&cpu);
     iflag_set_default_cpu(&cmd_cpu);
 
@@ -462,8 +464,6 @@ int main(int argc, char **argv)
     want_usage = terminate_after_phase = false;
     nasm_set_verror(nasm_verror_asm);
 
-    error_file = stderr;
-
     tolower_init();
     src_init();
 
diff --git a/include/error.h b/include/error.h
index 5a676e67..477a26d7 100644
--- a/include/error.h
+++ b/include/error.h
@@ -41,6 +41,11 @@
 #include "compiler.h"
 
 /*
+ * File pointer for error messages
+ */
+extern FILE *error_file;        /* Error file descriptor */
+
+/*
  * An error reporting function should look like this.
  */
 void printf_func(2, 3) nasm_error(int severity, const char *fmt, ...);
diff --git a/nasmlib/malloc.c b/nasmlib/malloc.c
index ccbc0c75..dbb7384a 100644
--- a/nasmlib/malloc.c
+++ b/nasmlib/malloc.c
@@ -44,7 +44,23 @@
 
 static no_return nasm_alloc_failed(void)
 {
-    nasm_fatal(0, "out of memory");
+    /* If nasm_fatal() gets us back here, then croak hard */
+    static bool already_here = false;
+    FILE *errfile;
+
+    if (likely(!already_here)) {
+        already_here = true;
+        nasm_fatal(0, "out of memory!");
+    }
+
+    errfile = error_file;
+    if (!errfile)
+        error_file = stderr;
+
+    fprintf(error_file, "nasm: out of memory!\n");
+    fflush(error_file);
+    fflush(NULL);
+    abort();
 }
 
 static inline void *validate_ptr(void *p)
diff --git a/rdoff/ldrdf.c b/rdoff/ldrdf.c
index dd80d70e..49729b4f 100644
--- a/rdoff/ldrdf.c
+++ b/rdoff/ldrdf.c
@@ -126,9 +126,6 @@ char *generic_rec_file = NULL;
 /* module name to be added at the beginning of output file */
 char *modname_specified = NULL;
 
-/* error file */
-static FILE *error_file;
-
 /* the header of the output file, built up stage by stage */
 rdf_headerbuf *newheader = NULL;