disam: explicitly change stdin to binary mode

The binary mode has no difference from text mode in
POSIX-compliant operating systems. The two modes are
distinguishable from each other on Windows, and perhaps
on other systems as well.

The binary stream has scalability and other advantages.
Windows treats the standard input stream as text mode by
default. So the code changes it to binary mode.

Also, add a helper function, nasm_set_binary_mode(),
that is OS-agnostic, in the library.

Reported-by: Didier Stevens <didier.stevens@gmail.com>
Suggested-by: Didier Stevens <didier.stevens@gmail.com>
Link: https://bugzilla.nasm.us/show_bug.cgi?id=3392649
Signed-off-by: Chang S. Bae <chang.seok.bae@intel.com>

4 files changed, 32 insertions, 1 deletions

diff --git a/disasm/ndisasm.c b/disasm/ndisasm.c
index f3c23b00..01e0c557 100644
--- a/disasm/ndisasm.c
+++ b/disasm/ndisasm.c
@@ -280,8 +280,10 @@ int main(int argc, char **argv)
                     pname, filename, strerror(errno));
             return 1;
         }
-    } else
+    } else {
+        nasm_set_binary_mode(stdin);
         fp = stdin;
+    }
 
     if (initskip > 0)
         skip(initskip, fp);
diff --git a/include/nasmlib.h b/include/nasmlib.h
index 940f1cb7..c4b4ac4c 100644
--- a/include/nasmlib.h
+++ b/include/nasmlib.h
@@ -365,6 +365,8 @@ enum file_flags {
 FILE *nasm_open_read(const char *filename, enum file_flags flags);
 FILE *nasm_open_write(const char *filename, enum file_flags flags);
 
+void nasm_set_binary_mode(FILE *f);
+
 /* Probe for existence of a file */
 bool nasm_file_exists(const char *filename);
 
diff --git a/nasmlib/file.c b/nasmlib/file.c
index a8cd3057..62b854de 100644
--- a/nasmlib/file.c
+++ b/nasmlib/file.c
@@ -148,6 +148,11 @@ os_filename os_mangle_filename(const char *filename)
 
 #endif
 
+void nasm_set_binary_mode(FILE *f)
+{
+	os_set_binary_mode(f);
+}
+
 FILE *nasm_open_read(const char *filename, enum file_flags flags)
 {
     FILE *f = NULL;
diff --git a/nasmlib/file.h b/nasmlib/file.h
index 4f0420ec..fc8f893d 100644
--- a/nasmlib/file.h
+++ b/nasmlib/file.h
@@ -103,6 +103,24 @@ typedef struct _stati64 os_struct_stat;
 # define os_stat  _wstati64
 # define os_fstat _fstati64
 
+/*
+ * On Win32/64, freopen() and _wfreopen() fails when the mode string
+ * is with the letter 'b' that represents to set binary mode. On
+ * POSIX operating systems, the 'b' is ignored, without failure.
+ */
+
+#include <io.h>
+#include <fcntl.h>
+
+static inline void os_set_binary_mode(FILE *f) {
+    int ret = _setmode(_fileno(f), _O_BINARY);
+
+    if (ret == -1) {
+        nasm_fatalf(ERR_NOFILE, "unable to open file: %s",
+                    strerror(errno));
+    }
+}
+
 #else  /* not _WIN32 */
 
 typedef const char *os_filename;
@@ -117,6 +135,10 @@ static inline void os_free_filename(os_filename filename)
     (void)filename;             /* Nothing to do */
 }
 
+static inline void os_set_binary_mode(FILE *f) {
+    (void)f;
+}
+
 # define os_fopen  fopen
 
 #if defined(HAVE_FACCESSAT) && defined(AT_EACCESS)