Introduce and use `caml_ext_table_add_noexc` (#12202)

Same functionality as `caml_ext_table_add`, but returns -1 when running
out of memory instead of raising an exception.

Use the new function in `caml_read_directory`, which must not raise exceptions
since it is called from blocking sections.  Instead, `caml_read_directory`
returns an error code if `caml_ext_table_add_noexc` runs out of memory.

Also: harden `caml_ext_table_add{,_noexc}` against integer overflow
when computing the new table size for resizing.

Fixes: #10403

5 files changed, 39 insertions, 7 deletions

diff --git a/Changes b/Changes
index 2d2c90be0f..cb25863e2a 100644
--- a/Changes
+++ b/Changes
@@ -5,6 +5,11 @@ Working version
 
 ### Runtime system:
 
+- #10403, #12202: introduce `caml_ext_table_add_noexc` that does not
+  raise `Out_of_memory` exceptions and use it inside the blocking sections
+  of `caml_read_directory`.  Also, check for overflows in ext table sizes.
+  (Xavier Leroy, report by Arseniy Alekseyev, review by Gabriel Scherer)
+
 ### Code generation and optimizations:
 
 ### Standard library:
diff --git a/runtime/caml/misc.h b/runtime/caml/misc.h
index a9790d8a77..ed458e833f 100644
--- a/runtime/caml/misc.h
+++ b/runtime/caml/misc.h
@@ -457,6 +457,7 @@ struct ext_table {
 
 extern void caml_ext_table_init(struct ext_table * tbl, int init_capa);
 extern int caml_ext_table_add(struct ext_table * tbl, void * data);
+extern int caml_ext_table_add_noexc(struct ext_table * tbl, void * data);
 extern void caml_ext_table_remove(struct ext_table * tbl, void * data);
 extern void caml_ext_table_free(struct ext_table * tbl, int free_entries);
 extern void caml_ext_table_clear(struct ext_table * tbl, int free_entries);
diff --git a/runtime/misc.c b/runtime/misc.c
index 43fc5d600a..fd584cb719 100644
--- a/runtime/misc.c
+++ b/runtime/misc.c
@@ -30,6 +30,7 @@ __declspec(noreturn) void __cdecl abort(void);
 #include <stdarg.h>
 #include <stdlib.h>
 #include "caml/config.h"
+#include "caml/fail.h"
 #include "caml/misc.h"
 #include "caml/memory.h"
 #include "caml/osdeps.h"
@@ -136,20 +137,35 @@ CAMLexport void caml_fatal_error_arg2 (const char *fmt1, const char *arg1,
   exit(2);
 }
 
+#ifdef ARCH_SIXTYFOUR
+#define MAX_EXT_TABLE_CAPACITY INT_MAX
+#else
+#define MAX_EXT_TABLE_CAPACITY ((asize_t) (-1) / sizeof(void *))
+#endif
+
 void caml_ext_table_init(struct ext_table * tbl, int init_capa)
 {
+  CAMLassert (init_capa <= MAX_EXT_TABLE_CAPACITY);
   tbl->size = 0;
   tbl->capacity = init_capa;
-  tbl->contents = caml_stat_alloc(sizeof(void *) * init_capa);
+  tbl->contents = caml_stat_alloc(sizeof(void *) * (asize_t) init_capa);
 }
 
-int caml_ext_table_add(struct ext_table * tbl, caml_stat_block data)
+int caml_ext_table_add_noexc(struct ext_table * tbl, caml_stat_block data)
 {
   int res;
   if (tbl->size >= tbl->capacity) {
-    tbl->capacity *= 2;
-    tbl->contents =
-      caml_stat_resize(tbl->contents, sizeof(void *) * tbl->capacity);
+    if (tbl->capacity == MAX_EXT_TABLE_CAPACITY) return -1; /* overflow */
+    int new_capacity =
+      tbl->capacity <= MAX_EXT_TABLE_CAPACITY / 2
+      ? tbl->capacity * 2
+      : MAX_EXT_TABLE_CAPACITY;
+    void ** new_contents =
+      caml_stat_resize_noexc(tbl->contents,
+                             sizeof(void *) * (asize_t) new_capacity);
+    if (new_contents == NULL) return -1;
+    tbl->capacity = new_capacity;
+    tbl->contents = new_contents;
   }
   res = tbl->size;
   tbl->contents[res] = data;
@@ -157,6 +173,13 @@ int caml_ext_table_add(struct ext_table * tbl, caml_stat_block data)
   return res;
 }
 
+int caml_ext_table_add(struct ext_table * tbl, caml_stat_block data)
+{
+  int res = caml_ext_table_add_noexc(tbl, data);
+  if (res == -1) caml_raise_out_of_memory();
+  return res;
+}
+
 void caml_ext_table_remove(struct ext_table * tbl, caml_stat_block data)
 {
   int i;
diff --git a/runtime/unix.c b/runtime/unix.c
index 0877ec37a0..a71980497c 100644
--- a/runtime/unix.c
+++ b/runtime/unix.c
@@ -354,7 +354,8 @@ CAMLexport int caml_read_directory(char * dirname, struct ext_table * contents)
     e = readdir(d);
     if (e == NULL) break;
     if (strcmp(e->d_name, ".") == 0 || strcmp(e->d_name, "..") == 0) continue;
-    caml_ext_table_add(contents, caml_stat_strdup(e->d_name));
+    int rc = caml_ext_table_add_noexc(contents, caml_stat_strdup(e->d_name));
+    if (rc == -1) { closedir(d); errno = ENOMEM; return -1; }
   }
   closedir(d);
   return 0;
diff --git a/runtime/win32.c b/runtime/win32.c
index 70c90f7a11..599805145f 100644
--- a/runtime/win32.c
+++ b/runtime/win32.c
@@ -445,7 +445,9 @@ CAMLexport int caml_read_directory(wchar_t * dirname,
   }
   do {
     if (wcscmp(fileinfo.name, L".") != 0 && wcscmp(fileinfo.name, L"..") != 0) {
-      caml_ext_table_add(contents, caml_stat_strdup_of_utf16(fileinfo.name));
+      res = caml_ext_table_add_noexc(contents,
+                                     caml_stat_strdup_of_utf16(fileinfo.name));
+      if (res == -1) { _findclose(h); errno = ENOMEM; return -1; }
     }
   } while (_wfindnext(h, &fileinfo) == 0);
   _findclose(h);