Fix BZ #18820 -- fmemopen may leak memory on failure.

4 files changed, 58 insertions, 4 deletions

diff --git a/libio/Makefile b/libio/Makefile
index 7b3bcf90fe..604f4196b3 100644
--- a/libio/Makefile
+++ b/libio/Makefile
@@ -148,8 +148,10 @@ CFLAGS-tst_putwc.c = -DOBJPFX=\"$(objpfx)\"
 
 tst_wprintf2-ARGS = "Some Text"
 
+test-fmemopen-ENV = MALLOC_TRACE=$(objpfx)test-fmemopen.mtrace
 tst-fopenloc-ENV = MALLOC_TRACE=$(objpfx)tst-fopenloc.mtrace
 
+generated += test-fmemopen.mtrace test-fmemopen.check
 generated += tst-fopenloc.mtrace tst-fopenloc.check
 
 aux	:= fileops genops stdfiles stdio strops
@@ -164,7 +166,7 @@ shared-only-routines = oldiofopen oldiofdopen oldiofclose oldfileops	\
 		       oldiofsetpos64
 
 ifeq ($(run-built-tests),yes)
-tests-special += $(objpfx)test-freopen.out
+tests-special += $(objpfx)test-freopen.out $(objpfx)test-fmemopen-mem.out
 ifeq (yes,$(build-shared))
 # Run tst-fopenloc-cmp.out and tst-openloc-mem.out only if shared
 # library is enabled since they depend on tst-fopenloc.out.
@@ -184,6 +186,10 @@ $(objpfx)tst-fopenloc-cmp.out: ../iconvdata/testdata/ISO-8859-1..UTF8 \
 	cmp $^ > $@; \
 	$(evaluate-test)
 
+$(objpfx)test-fmemopen-mem.out: $(objpfx)test-fmemopen.out
+	$(common-objpfx)malloc/mtrace $(objpfx)test-fmemopen.mtrace > $@; \
+	$(evaluate-test)
+
 $(objpfx)tst-fopenloc-mem.out: $(objpfx)tst-fopenloc.out
 	$(common-objpfx)malloc/mtrace $(objpfx)tst-fopenloc.mtrace > $@; \
 	$(evaluate-test)
diff --git a/libio/fmemopen.c b/libio/fmemopen.c
index 3ab3e8dac4..66e2d833ee 100644
--- a/libio/fmemopen.c
+++ b/libio/fmemopen.c
@@ -149,6 +149,7 @@ __fmemopen (void *buf, size_t len, const char *mode)
 {
   cookie_io_functions_t iof;
   fmemopen_cookie_t *c;
+  FILE *result;
 
   c = (fmemopen_cookie_t *) calloc (sizeof (fmemopen_cookie_t), 1);
   if (c == NULL)
@@ -209,7 +210,16 @@ __fmemopen (void *buf, size_t len, const char *mode)
   iof.seek = fmemopen_seek;
   iof.close = fmemopen_close;
 
-  return _IO_fopencookie (c, mode, iof);
+  result = _IO_fopencookie (c, mode, iof);
+  if (__glibc_unlikely (result == NULL))
+    {
+      if (c->mybuffer)
+	free (c->buffer);
+
+      free (c);
+    }
+
+  return result;
 }
 libc_hidden_def (__fmemopen)
 versioned_symbol (libc, __fmemopen, fmemopen, GLIBC_2_22);
diff --git a/libio/oldfmemopen.c b/libio/oldfmemopen.c
index 8e35672327..88ef8fa7f0 100644
--- a/libio/oldfmemopen.c
+++ b/libio/oldfmemopen.c
@@ -204,6 +204,7 @@ __old_fmemopen (void *buf, size_t len, const char *mode)
 {
   cookie_io_functions_t iof;
   fmemopen_cookie_t *c;
+  FILE *result;
 
   if (__glibc_unlikely (len == 0))
     {
@@ -259,7 +260,16 @@ __old_fmemopen (void *buf, size_t len, const char *mode)
   iof.seek = fmemopen_seek;
   iof.close = fmemopen_close;
 
-  return _IO_fopencookie (c, mode, iof);
+  result = _IO_fopencookie (c, mode, iof);
+  if (__glibc_unlikely (result == NULL))
+    {
+      if (c->mybuffer)
+	free (c->buffer);
+
+      free (c);
+    }
+
+  return result;
 }
 compat_symbol (libc, __old_fmemopen, fmemopen, GLIBC_2_2);
 #endif
diff --git a/libio/test-fmemopen.c b/libio/test-fmemopen.c
index 63ca89f300..e8e757f351 100644
--- a/libio/test-fmemopen.c
+++ b/libio/test-fmemopen.c
@@ -22,6 +22,32 @@ static char buffer[] = "foobar";
 #include <stdio.h>
 #include <string.h>
 #include <errno.h>
+#include <mcheck.h>
+
+static int
+do_bz18820 (void)
+{
+  char ch;
+  FILE *stream;
+
+  stream = fmemopen (&ch, 1, "?");
+  if (stream)
+    {
+      printf ("fmemopen: expected NULL, got %p\n", stream);
+      fclose (stream);
+      return 1;
+    }
+
+  stream = fmemopen (NULL, 42, "?");
+  if (stream)
+    {
+      printf ("fmemopen: expected NULL, got %p\n", stream);
+      fclose (stream);
+      return 2;
+    }
+
+  return 0;
+}
 
 static int
 do_test (void)
@@ -30,6 +56,8 @@ do_test (void)
   FILE *stream;
   int ret = 0;
 
+  mtrace ();
+
   stream = fmemopen (buffer, strlen (buffer), "r+");
 
   while ((ch = fgetc (stream)) != EOF)
@@ -44,7 +72,7 @@ do_test (void)
 
   fclose (stream);
 
-  return ret;
+  return ret + do_bz18820 ();
 }
 
 #define TEST_FUNCTION do_test ()