Exit process immediately when the max heap size is exceeded

Executing receive-related instructions would cause the process to
enter a suspended state forever. Avoid that by scheduling out
immediately after any GC if the process is exiting.

6 files changed, 136 insertions, 1 deletions

diff --git a/erts/emulator/beam/emu/bs_instrs.tab b/erts/emulator/beam/emu/bs_instrs.tab
index bfb364c26b..751adfcc43 100644
--- a/erts/emulator/beam/emu/bs_instrs.tab
+++ b/erts/emulator/beam/emu/bs_instrs.tab
@@ -132,6 +132,7 @@ TEST_BIN_VHEAP(VNh, Nh, Live) {
         ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
         PROCESS_MAIN_CHK_LOCKS(c_p);
         SWAPIN;
+        $MAYBE_EXIT_AFTER_GC();
     }
     HEAP_SPACE_VERIFIED(need);
 }
diff --git a/erts/emulator/beam/emu/instrs.tab b/erts/emulator/beam/emu/instrs.tab
index d86439e570..b677e99002 100644
--- a/erts/emulator/beam/emu/instrs.tab
+++ b/erts/emulator/beam/emu/instrs.tab
@@ -1108,6 +1108,7 @@ catch_end(Y) {
                 ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
                 PROCESS_MAIN_CHK_LOCKS(c_p);
                 SWAPIN;
+                $MAYBE_EXIT_AFTER_GC();
             }
             r(0) = TUPLE2(HTOP, am_EXIT, x(2));
             HTOP += 3;
diff --git a/erts/emulator/beam/emu/macros.tab b/erts/emulator/beam/emu/macros.tab
index f7aa49634a..e7816003c7 100644
--- a/erts/emulator/beam/emu/macros.tab
+++ b/erts/emulator/beam/emu/macros.tab
@@ -74,6 +74,12 @@ GC_SWAPOUT() {
     c_p->i = I;
 }
 
+MAYBE_EXIT_AFTER_GC() {
+    if (ERTS_PSFLG_EXITING & erts_atomic32_read_nob(&c_p->state)) {
+        goto context_switch3;
+    }
+}
+
 GC_TEST(Ns, Nh, Live) {
     Uint need = $Nh + $Ns;
     if (ERTS_UNLIKELY((E - HTOP) < (need + S_RESERVED))) {
@@ -83,6 +89,7 @@ GC_TEST(Ns, Nh, Live) {
        ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
        PROCESS_MAIN_CHK_LOCKS(c_p);
        SWAPIN;
+       $MAYBE_EXIT_AFTER_GC();
     }
     HEAP_SPACE_VERIFIED($Nh);
 }
@@ -98,6 +105,7 @@ GC_TEST_PRESERVE(NeedHeap, Live, PreserveTerm) {
        PROCESS_MAIN_CHK_LOCKS(c_p);
        $PreserveTerm = reg[$Live];
        SWAPIN;
+       $MAYBE_EXIT_AFTER_GC();
     }
     HEAP_SPACE_VERIFIED($NeedHeap);
 }
diff --git a/erts/emulator/beam/erl_gc.c b/erts/emulator/beam/erl_gc.c
index 679d7f3a21..cbcf262d4a 100644
--- a/erts/emulator/beam/erl_gc.c
+++ b/erts/emulator/beam/erl_gc.c
@@ -454,7 +454,11 @@ erts_gc_after_bif_call_lhf(Process* p, ErlHeapFragment *live_hf_end,
 
 	val[0] = result;
 	cost = garbage_collect(p, live_hf_end, 0, val, 1, p->fcalls, 0);
-	result = val[0];
+        if (ERTS_PROC_IS_EXITING(p)) {
+            result = THE_NON_VALUE;
+        } else {
+            result = val[0];
+        }
     }
     BUMP_REDS(p, cost);
 
diff --git a/erts/emulator/beam/jit/x86/beam_asm_global.cpp b/erts/emulator/beam/jit/x86/beam_asm_global.cpp
index a65b254bea..a9a1da7621 100644
--- a/erts/emulator/beam/jit/x86/beam_asm_global.cpp
+++ b/erts/emulator/beam/jit/x86/beam_asm_global.cpp
@@ -120,6 +120,8 @@ void BeamGlobalAssembler::emit_handle_error() {
 /* ARG3 = (HTOP + bytes needed) !!
  * ARG4 = Live registers */
 void BeamGlobalAssembler::emit_garbage_collect() {
+    Label exiting = a.newLabel();
+
     /* Convert ARG3 to words needed and move it to the correct argument slot */
     a.sub(ARG3, HTOP);
     a.shr(ARG3, imm(3));
@@ -140,7 +142,19 @@ void BeamGlobalAssembler::emit_garbage_collect() {
 
     emit_leave_runtime<Update::eStack | Update::eHeap>();
 
+#ifdef WIN32
+    a.mov(ARG1d, x86::dword_ptr(c_p, offsetof(Process, state.value)));
+#else
+    a.mov(ARG1d, x86::dword_ptr(c_p, offsetof(Process, state.counter)));
+#endif
+    a.test(ARG1d, imm(ERTS_PSFLG_EXITING));
+    a.short_().jne(exiting);
+
     a.ret();
+
+    a.bind(exiting);
+    emit_discard_cp();
+    a.jmp(labels[do_schedule]);
 }
 
 /* Handles trapping to exports from C code, setting registers up in the same
diff --git a/erts/emulator/test/process_SUITE.erl b/erts/emulator/test/process_SUITE.erl
index f5abc2cdbd..4f682f72d5 100644
--- a/erts/emulator/test/process_SUITE.erl
+++ b/erts/emulator/test/process_SUITE.erl
@@ -51,6 +51,7 @@
 	 process_info_messages/1, process_flag_badarg/1,
          process_flag_fullsweep_after/1, process_flag_heap_size/1,
 	 spawn_opt_heap_size/1, spawn_opt_max_heap_size/1,
+         more_spawn_opt_max_heap_size/1,
 	 processes_large_tab/1, processes_default_tab/1, processes_small_tab/1,
 	 processes_this_tab/1, processes_apply_trap/1,
 	 processes_last_call_trap/1, processes_gc_trap/1,
@@ -112,6 +113,7 @@ all() ->
      process_flag_badarg,
      process_flag_fullsweep_after, process_flag_heap_size,
      spawn_opt_heap_size, spawn_opt_max_heap_size,
+     more_spawn_opt_max_heap_size,
      spawn_huge_arglist,
      spawn_request_bif,
      spawn_request_monitor_demonitor,
@@ -2395,6 +2397,111 @@ flush() ->
             ok
     end.
 
+%% Make sure that when maximum allowed heap size is exceeded, the
+%% process will actually terminate.
+%%
+%% Despite the timetrap and limit of number of iterations, bugs
+%% provoked by the test case can cause the runtime system to hang in
+%% this test case.
+more_spawn_opt_max_heap_size(_Config) ->
+    ct:timetrap({minutes,1}),
+    Funs = [fun build_and_bif/0,
+            fun build_bin_and_bif/0,
+            fun build_and_recv_timeout/0,
+            fun build_and_recv_msg/0,
+            fun bif_and_recv_timeout/0,
+            fun bif_and_recv_msg/0
+           ],
+    _ = [begin
+             {Pid,Ref} = spawn_opt(F, [{max_heap_size,
+                                        #{size => 233, kill => true,
+                                          error_logger => false}},
+                                       monitor]),
+             io:format("~p ~p\n", [Pid,F]),
+             receive
+                 {'DOWN',Ref,process,Pid,Reason} ->
+                     killed = Reason
+             end
+         end || F <- Funs],
+    ok.
+
+%% This number should be greater than the default heap size.
+-define(MANY_ITERATIONS, 10_000).
+
+build_and_bif() ->
+    build_and_bif(?MANY_ITERATIONS, []).
+
+build_and_bif(0, Acc0) ->
+    Acc0;
+build_and_bif(N, Acc0) ->
+    Acc = [0|Acc0],
+    _ = erlang:crc32(Acc),
+    build_and_bif(N-1, Acc).
+
+build_bin_and_bif() ->
+    build_bin_and_bif(?MANY_ITERATIONS, <<>>).
+
+build_bin_and_bif(0, Acc0) ->
+    Acc0;
+build_bin_and_bif(N, Acc0) ->
+    Acc = <<0, Acc0/binary>>,
+    _ = erlang:crc32(Acc),
+    build_bin_and_bif(N-1, Acc).
+
+build_and_recv_timeout() ->
+    build_and_recv_timeout(?MANY_ITERATIONS, []).
+
+build_and_recv_timeout(0, Acc0) ->
+    Acc0;
+build_and_recv_timeout(N, Acc0) ->
+    Acc = [0|Acc0],
+    receive
+    after 1 ->
+            ok
+    end,
+    build_and_recv_timeout(N-1, Acc).
+
+build_and_recv_msg() ->
+    build_and_recv_msg(?MANY_ITERATIONS, []).
+
+build_and_recv_msg(0, Acc0) ->
+    Acc0;
+build_and_recv_msg(N, Acc0) ->
+    Acc = [0|Acc0],
+    receive
+        _ ->
+            ok
+    after 0 ->
+            ok
+    end,
+    build_and_recv_msg(N-1, Acc).
+
+bif_and_recv_timeout() ->
+    Bin = <<0:?MANY_ITERATIONS/unit:8>>,
+    bif_and_recv_timeout(Bin).
+
+bif_and_recv_timeout(Bin) ->
+    List = binary_to_list(Bin),
+    receive
+    after 1 ->
+            ok
+    end,
+    List.
+
+bif_and_recv_msg() ->
+    Bin = <<0:?MANY_ITERATIONS/unit:8>>,
+    bif_and_recv_msg(Bin).
+
+bif_and_recv_msg(Bin) ->
+    List = binary_to_list(Bin),
+    receive
+        _ ->
+            ok
+    after 0 ->
+            ok
+    end,
+    List.
+
 %% error_logger report handler proxy
 init(Pid) ->
     {ok, Pid}.