Fix BZ #17916 - fopen unbounded stack usage for ccs= modes

2 files changed, 39 insertions, 2 deletions

diff --git a/libio/fileops.c b/libio/fileops.c
index 297b4784cc..2427320325 100644
--- a/libio/fileops.c
+++ b/libio/fileops.c
@@ -353,7 +353,15 @@ _IO_new_file_fopen (_IO_FILE *fp, const char *filename, const char *mode,
 	  struct gconv_fcts fcts;
 	  struct _IO_codecvt *cc;
 	  char *endp = __strchrnul (cs + 5, ',');
-	  char ccs[endp - (cs + 5) + 3];
+	  char *ccs = malloc (endp - (cs + 5) + 3);
+
+	  if (ccs == NULL)
+	    {
+	      int malloc_err = errno;  /* Whatever malloc failed with.  */
+	      (void) _IO_file_close_it (fp);
+	      __set_errno (malloc_err);
+	      return NULL;
+	    }
 
 	  *((char *) __mempcpy (ccs, cs + 5, endp - (cs + 5))) = '\0';
 	  strip (ccs, ccs);
@@ -365,10 +373,13 @@ _IO_new_file_fopen (_IO_FILE *fp, const char *filename, const char *mode,
 		 This means we cannot proceed since the user explicitly asked
 		 for these.  */
 	      (void) _IO_file_close_it (fp);
+	      free (ccs);
 	      __set_errno (EINVAL);
 	      return NULL;
 	    }
 
+	  free (ccs);
+
 	  assert (fcts.towc_nsteps == 1);
 	  assert (fcts.tomb_nsteps == 1);
 
diff --git a/libio/tst-fopenloc.c b/libio/tst-fopenloc.c
index 1336023b5a..48c2d3b41c 100644
--- a/libio/tst-fopenloc.c
+++ b/libio/tst-fopenloc.c
@@ -24,10 +24,36 @@
 #include <stdlib.h>
 #include <string.h>
 #include <wchar.h>
+#include <sys/resource.h>
 
 
 static const char inputfile[] = "../iconvdata/testdata/ISO-8859-1";
 
+static
+int do_bz17916 (void)
+{
+  /* BZ #17916 -- check invalid large ccs= case.  */
+  struct rlimit rl;
+  getrlimit (RLIMIT_STACK, &rl);
+  rl.rlim_cur = 1024 * 1024;
+  setrlimit (RLIMIT_STACK, &rl);
+
+  const size_t sz = 2 * 1024 * 1024;
+  char *ccs = malloc (sz);
+  strcpy (ccs, "r,ccs=");
+  memset (ccs + 6, 'A', sz - 6 - 1);
+  ccs[sz - 1] = '\0';
+
+  FILE *fp = fopen (inputfile, ccs);
+  if (fp != NULL)
+    {
+      printf ("unxpected success\n");
+      return 1;
+    }
+  free (ccs);
+
+  return 0;
+}
 
 static int
 do_test (void)
@@ -57,7 +83,7 @@ do_test (void)
 
   fclose (fp);
 
-  return 0;
+  return do_bz17916 ();
 }
 
 #define TEST_FUNCTION do_test ()