BUG# 9148: Denial of service

This is a second patch needing another review.  The first patch didn't solve
the entire problem.  open and fopen on Windows will still open
files like "com1.sym" when they shouldn't.  This patch
checks that the file exists before trying to open it.

2 files changed, 21 insertions, 3 deletions

diff --git a/mysys/my_fopen.c b/mysys/my_fopen.c
index e918b7b0de2..208e7e80fd8 100644
--- a/mysys/my_fopen.c
+++ b/mysys/my_fopen.c
@@ -33,9 +33,21 @@ FILE *my_fopen(const char *FileName, int Flags, myf MyFlags)
   DBUG_ENTER("my_fopen");
   DBUG_PRINT("my",("Name: '%s'  Flags: %d  MyFlags: %d",
 		   FileName, Flags, MyFlags));
-
-  make_ftype(type,Flags);
-  if ((fd = fopen(FileName, type)) != 0)
+  /* 
+   * if we are not creating, then we need to use my_access to make sure  
+   * the file exists since Windows doesn't handle files like "com1.sym" very  well 
+  */
+#ifdef __WIN__
+  if (! (Flags & O_CREAT) && my_access(FileName, F_OK))
+	  fd=0;
+  else
+#endif
+  {
+  	make_ftype(type,Flags);
+  	fd = fopen(FileName, type);
+  }
+  
+  if (fd != 0)
   {
     /*
       The test works if MY_NFILE < 128. The problem is that fileno() is char
diff --git a/mysys/my_open.c b/mysys/my_open.c
index ca5c0d8683f..1f3bb95b5a2 100644
--- a/mysys/my_open.c
+++ b/mysys/my_open.c
@@ -46,6 +46,12 @@ File my_open(const char *FileName, int Flags, myf MyFlags)
   DBUG_PRINT("my",("Name: '%s'  Flags: %d  MyFlags: %d",
 		   FileName, Flags, MyFlags));
 #if defined(MSDOS) || defined(__WIN__) || defined(__EMX__) || defined(OS2)
+  /* if we are not creating, then we need to use my_access to make 
+   * sure  the file exists since Windows doesn't handle files like 
+   * "com1.sym" very  well 
+  */  
+  if (! (Flags & O_CREAT) && my_access(FileName, F_OK))    
+	  return -1;
   if (Flags & O_SHARE)
     fd = sopen((my_string) FileName, (Flags & ~O_SHARE) | O_BINARY, SH_DENYNO,
 	       MY_S_IREAD | MY_S_IWRITE);