patch 8.1.0633: crash when out of memory while opening a terminal windowv8.1.0633

Problem:    Crash when out of memory while opening a terminal window.
Solution:   Handle out-of-memory more gracefully.

3 files changed, 43 insertions, 13 deletions

diff --git a/src/libvterm/src/state.c b/src/libvterm/src/state.c
index d977ecac6..8b02093bb 100644
--- a/src/libvterm/src/state.c
+++ b/src/libvterm/src/state.c
@@ -53,6 +53,8 @@ static VTermState *vterm_state_new(VTerm *vt)
 {
   VTermState *state = vterm_allocator_malloc(vt, sizeof(VTermState));
 
+  if (state == NULL)
+    return NULL;
   state->vt = vt;
 
   state->rows = vt->rows;
@@ -1693,6 +1695,10 @@ static const VTermParserCallbacks parser_callbacks = {
   on_resize /* resize */
 };
 
+/*
+ * Return the existing state or create a new one.
+ * Returns NULL when out of memory.
+ */
 VTermState *vterm_obtain_state(VTerm *vt)
 {
   VTermState *state;
@@ -1700,6 +1706,8 @@ VTermState *vterm_obtain_state(VTerm *vt)
     return vt->state;
 
   state = vterm_state_new(vt);
+  if (state == NULL)
+    return NULL;
   vt->state = state;
 
   state->combine_chars_size = 16;
diff --git a/src/libvterm/src/termscreen.c b/src/libvterm/src/termscreen.c
index cfae088ba..0cd31cedf 100644
--- a/src/libvterm/src/termscreen.c
+++ b/src/libvterm/src/termscreen.c
@@ -1,5 +1,6 @@
 #include "vterm_internal.h"
 
+/* vim: set sw=2 : */
 #include <stdio.h>
 #include <string.h>
 
@@ -95,8 +96,7 @@ static ScreenCell *realloc_buffer(VTermScreen *screen, ScreenCell *buffer, int n
     }
   }
 
-  if(buffer)
-    vterm_allocator_free(screen->vt, buffer);
+  vterm_allocator_free(screen->vt, buffer);
 
   return new_buffer;
 }
@@ -518,8 +518,7 @@ static int resize(int new_rows, int new_cols, VTermPos *delta, void *user)
   screen->rows = new_rows;
   screen->cols = new_cols;
 
-  if(screen->sb_buffer)
-    vterm_allocator_free(screen->vt, screen->sb_buffer);
+  vterm_allocator_free(screen->vt, screen->sb_buffer);
 
   screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * new_cols);
 
@@ -619,16 +618,21 @@ static VTermStateCallbacks state_cbs = {
   &setlineinfo /* setlineinfo */
 };
 
+/*
+ * Allocate a new screen and return it.
+ * Return NULL when out of memory.
+ */
 static VTermScreen *screen_new(VTerm *vt)
 {
   VTermState *state = vterm_obtain_state(vt);
   VTermScreen *screen;
   int rows, cols;
 
-  if(!state)
+  if (state == NULL)
     return NULL;
-
   screen = vterm_allocator_malloc(vt, sizeof(VTermScreen));
+  if (screen == NULL)
+    return NULL;
 
   vterm_get_size(vt, &rows, &cols);
 
@@ -646,10 +650,13 @@ static VTermScreen *screen_new(VTerm *vt)
   screen->cbdata    = NULL;
 
   screen->buffers[0] = realloc_buffer(screen, NULL, rows, cols);
-
   screen->buffer = screen->buffers[0];
-
   screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * cols);
+  if (screen->buffer == NULL || screen->sb_buffer == NULL)
+  {
+    vterm_screen_free(screen);
+    return NULL;
+  }
 
   vterm_state_set_callbacks(screen->state, &state_cbs, screen);
 
@@ -659,11 +666,8 @@ static VTermScreen *screen_new(VTerm *vt)
 INTERNAL void vterm_screen_free(VTermScreen *screen)
 {
   vterm_allocator_free(screen->vt, screen->buffers[0]);
-  if(screen->buffers[1])
-    vterm_allocator_free(screen->vt, screen->buffers[1]);
-
+  vterm_allocator_free(screen->vt, screen->buffers[1]);
   vterm_allocator_free(screen->vt, screen->sb_buffer);
-
   vterm_allocator_free(screen->vt, screen);
 }
 
diff --git a/src/libvterm/src/vterm.c b/src/libvterm/src/vterm.c
index 9025da444..5e4722ce3 100644
--- a/src/libvterm/src/vterm.c
+++ b/src/libvterm/src/vterm.c
@@ -1,5 +1,6 @@
 #define DEFINE_INLINES
 
+/* vim: set sw=2 : */
 #include "vterm_internal.h"
 
 #include <stdio.h>
@@ -41,6 +42,8 @@ VTerm *vterm_new_with_allocator(int rows, int cols, VTermAllocatorFunctions *fun
   /* Need to bootstrap using the allocator function directly */
   VTerm *vt = (*funcs->malloc)(sizeof(VTerm), allocdata);
 
+  if (vt == NULL)
+    return NULL;
   vt->allocator = funcs;
   vt->allocdata = allocdata;
 
@@ -55,10 +58,21 @@ VTerm *vterm_new_with_allocator(int rows, int cols, VTermAllocatorFunctions *fun
   vt->parser.strbuffer_len = 500; /* should be able to hold an OSC string */
   vt->parser.strbuffer_cur = 0;
   vt->parser.strbuffer = vterm_allocator_malloc(vt, vt->parser.strbuffer_len);
+  if (vt->parser.strbuffer == NULL)
+  {
+    vterm_allocator_free(vt, vt);
+    return NULL;
+  }
 
   vt->outbuffer_len = 200;
   vt->outbuffer_cur = 0;
   vt->outbuffer = vterm_allocator_malloc(vt, vt->outbuffer_len);
+  if (vt->outbuffer == NULL)
+  {
+    vterm_allocator_free(vt, vt->parser.strbuffer);
+    vterm_allocator_free(vt, vt);
+    return NULL;
+  }
 
   return vt;
 }
@@ -82,9 +96,13 @@ INTERNAL void *vterm_allocator_malloc(VTerm *vt, size_t size)
   return (*vt->allocator->malloc)(size, vt->allocdata);
 }
 
+/*
+ * Free "ptr" unless it is NULL.
+ */
 INTERNAL void vterm_allocator_free(VTerm *vt, void *ptr)
 {
-  (*vt->allocator->free)(ptr, vt->allocdata);
+  if (ptr)
+    (*vt->allocator->free)(ptr, vt->allocdata);
 }
 
 void vterm_get_size(const VTerm *vt, int *rowsp, int *colsp)