disam: explicitly change stdin to binary mode

The binary mode has no difference from text mode in
POSIX-compliant operating systems. The two modes are
distinguishable from each other on Windows, and perhaps
on other systems as well.

The binary stream has scalability and other advantages.
Windows treats the standard input stream as text mode by
default. So the code changes it to binary mode.

Also, add a helper function, nasm_set_binary_mode(),
that is OS-agnostic, in the library.

Reported-by: Didier Stevens <didier.stevens@gmail.com>
Suggested-by: Didier Stevens <didier.stevens@gmail.com>
Link: https://bugzilla.nasm.us/show_bug.cgi?id=3392649
Signed-off-by: Chang S. Bae <chang.seok.bae@intel.com>

disasm/ndisasm.c

@@ -280,8 +280,10 @@ int main(int argc, char **argv)
                     pname, filename, strerror(errno));
             return 1;
         }
-    } else
+    } else {
+        nasm_set_binary_mode(stdin);
         fp = stdin;
+    }
 
     if (initskip > 0)
         skip(initskip, fp);

include/nasmlib.h

@@ -365,6 +365,8 @@ enum file_flags {
 FILE *nasm_open_read(const char *filename, enum file_flags flags);
 FILE *nasm_open_write(const char *filename, enum file_flags flags);
 
+void nasm_set_binary_mode(FILE *f);
+
 /* Probe for existence of a file */
 bool nasm_file_exists(const char *filename);
 

nasmlib/file.c

@@ -148,6 +148,11 @@ os_filename os_mangle_filename(const char *filename)
 
 #endif
 
+void nasm_set_binary_mode(FILE *f)
+{
+	os_set_binary_mode(f);
+}
+
 FILE *nasm_open_read(const char *filename, enum file_flags flags)
 {
     FILE *f = NULL;

nasmlib/file.h

@@ -103,6 +103,24 @@ typedef struct _stati64 os_struct_stat;
 # define os_stat  _wstati64
 # define os_fstat _fstati64
 
+/*
+ * On Win32/64, freopen() and _wfreopen() fails when the mode string
+ * is with the letter 'b' that represents to set binary mode. On
+ * POSIX operating systems, the 'b' is ignored, without failure.
+ */
+
+#include <io.h>
+#include <fcntl.h>
+
+static inline void os_set_binary_mode(FILE *f) {
+    int ret = _setmode(_fileno(f), _O_BINARY);
+
+    if (ret == -1) {
+        nasm_fatalf(ERR_NOFILE, "unable to open file: %s",
+                    strerror(errno));
+    }
+}
+
 #else  /* not _WIN32 */
 
 typedef const char *os_filename;
@@ -117,6 +135,10 @@ static inline void os_free_filename(os_filename filename)
     (void)filename;             /* Nothing to do */
 }
 
+static inline void os_set_binary_mode(FILE *f) {
+    (void)f;
+}
+
 # define os_fopen  fopen
 
 #if defined(HAVE_FACCESSAT) && defined(AT_EACCESS)