reduce max size of dinamically allocated arrays to minimize the nasty

behaviour some versions of IRIX exhibit of committing suicide on big
mallocs instead of just returning a friendly null pointer

2 files changed, 35 insertions, 7 deletions

diff --git a/tests/libtest/lib518.c b/tests/libtest/lib518.c
index 4fc06dfb2..d91625f98 100644
--- a/tests/libtest/lib518.c
+++ b/tests/libtest/lib518.c
@@ -202,7 +202,7 @@ static int rlimit(int keep_open)
      (rl.rlim_cur <= num_open.rlim_cur)) {
     sprintf(strbuff2, fmt, rl.rlim_cur);
     sprintf(strbuff1, fmt, num_open.rlim_cur);
-    sprintf(strbuff, "fd needed (%s) > system limit (%s)",
+    sprintf(strbuff, "fds needed %s > system limit %s",
             strbuff1, strbuff2);
     store_errmsg(strbuff, 0);
     fprintf(stderr, "%s\n", msgbuff);
@@ -218,11 +218,19 @@ static int rlimit(int keep_open)
    * that it becomes available to the test.
    */
 
-  nitems = INT_MAX / sizeof(*memchunk);
+  for (nitems = i = 1; nitems <= i; i *= 2)
+    nitems = i;
+  if (nitems > 0x7fff)
+    nitems = 0x40000;
   do {
+    num_open.rlim_max = sizeof(*memchunk) * (size_t)nitems;
+    sprintf(strbuff, fmt, num_open.rlim_max);
+    fprintf(stderr, "allocating memchunk %s byte array\n", strbuff);
     memchunk = malloc(sizeof(*memchunk) * (size_t)nitems);
-    if (!memchunk)
+    if (!memchunk) {
+      fprintf(stderr, "memchunk, malloc() failed\n");
       nitems /= 2;
+    }
   } while (nitems && !memchunk);
   if (!memchunk) {
     store_errmsg("memchunk, malloc() failed", our_errno());
@@ -232,6 +240,8 @@ static int rlimit(int keep_open)
 
   /* initialize it to fight lazy allocation */
 
+  fprintf(stderr, "initializing memchunk array\n");
+
   for (i = 0; i < nitems; i++)
     memchunk[i] = -1;
 
@@ -266,6 +276,8 @@ static int rlimit(int keep_open)
 
   /* initialize it to fight lazy allocation */
 
+  fprintf(stderr, "initializing fd array\n");
+
   for (num_open.rlim_cur = 0;
        num_open.rlim_cur < num_open.rlim_max;
        num_open.rlim_cur++)
@@ -302,7 +314,7 @@ static int rlimit(int keep_open)
       num_open.rlim_max = NUM_NEEDED;
       sprintf(strbuff2, fmt, num_open.rlim_max);
       sprintf(strbuff1, fmt, num_open.rlim_cur);
-      sprintf(strbuff, "fd needed (%s) > system limit (%s)",
+      sprintf(strbuff, "fds needed %s > system limit %s",
               strbuff2, strbuff1);
       store_errmsg(strbuff, 0);
       fprintf(stderr, "%s\n", msgbuff);
diff --git a/tests/libtest/lib537.c b/tests/libtest/lib537.c
index c9c4dd4f1..08bf1209c 100644
--- a/tests/libtest/lib537.c
+++ b/tests/libtest/lib537.c
@@ -185,11 +185,19 @@ static int rlimit(int keep_open)
    * that it becomes available to the test.
    */
 
-  nitems = INT_MAX / sizeof(*memchunk);
+  for (nitems = i = 1; nitems <= i; i *= 2)
+    nitems = i;
+  if (nitems > 0x7fff)
+    nitems = 0x40000;
   do {
+    num_open.rlim_max = sizeof(*memchunk) * (size_t)nitems;
+    sprintf(strbuff, fmt, num_open.rlim_max);
+    fprintf(stderr, "allocating memchunk %s byte array\n", strbuff);
     memchunk = malloc(sizeof(*memchunk) * (size_t)nitems);
-    if (!memchunk)
+    if (!memchunk) {
+      fprintf(stderr, "memchunk, malloc() failed\n");
       nitems /= 2;
+    }
   } while (nitems && !memchunk);
   if (!memchunk) {
     store_errmsg("memchunk, malloc() failed", our_errno());
@@ -199,6 +207,8 @@ static int rlimit(int keep_open)
 
   /* initialize it to fight lazy allocation */
 
+  fprintf(stderr, "initializing memchunk array\n");
+
   for (i = 0; i < nitems; i++)
     memchunk[i] = -1;
 
@@ -214,7 +224,11 @@ static int rlimit(int keep_open)
   }
   else {
     /* a huge number of file descriptors */
-    num_open.rlim_max = INT_MAX / sizeof(*fd);
+    for (nitems = i = 1; nitems <= i; i *= 2)
+      nitems = i;
+    if (nitems > 0x7fff)
+      nitems = 0x40000;
+    num_open.rlim_max = nitems;
   }
 
   /* verify that we won't overflow size_t in malloc() */
@@ -249,6 +263,8 @@ static int rlimit(int keep_open)
 
   /* initialize it to fight lazy allocation */
 
+  fprintf(stderr, "initializing fd array\n");
+
   for (num_open.rlim_cur = 0;
        num_open.rlim_cur < num_open.rlim_max;
        num_open.rlim_cur++)