Since a crash in handle_event is so disasterous, and since gen_event handlers are not very supervisable, let's wrap everything in a try/catch. Prinitng to stdout is not great but we just failed to log properly - better to give some indication than none.bug26236

3 files changed, 31 insertions, 6 deletions

diff --git a/src/rabbit_error_logger.erl b/src/rabbit_error_logger.erl
index 993f56f9..353da0a7 100644
--- a/src/rabbit_error_logger.erl
+++ b/src/rabbit_error_logger.erl
@@ -27,6 +27,8 @@
 -export([init/1, terminate/2, code_change/3, handle_call/2, handle_event/2,
          handle_info/2]).
 
+-import(rabbit_error_logger_file_h, [safe_handle_event/3]).
+
 %%----------------------------------------------------------------------------
 
 -ifdef(use_specs).
@@ -65,10 +67,13 @@ code_change(_OldVsn, State, _Extra) ->
 handle_call(_Request, State) ->
     {ok, not_understood, State}.
 
-handle_event({Kind, _Gleader, {_Pid, Format, Data}}, State) ->
+handle_event(Event, State) ->
+    safe_handle_event(fun handle_event0/2, Event, State).
+
+handle_event0({Kind, _Gleader, {_Pid, Format, Data}}, State) ->
     ok = publish(Kind, Format, Data, State),
     {ok, State};
-handle_event(_Event, State) ->
+handle_event0(_Event, State) ->
     {ok, State}.
 
 handle_info(_Info, State) ->
diff --git a/src/rabbit_error_logger_file_h.erl b/src/rabbit_error_logger_file_h.erl
index 16ab6d3a..be842739 100644
--- a/src/rabbit_error_logger_file_h.erl
+++ b/src/rabbit_error_logger_file_h.erl
@@ -22,6 +22,8 @@
 -export([init/1, handle_event/2, handle_call/2, handle_info/2, terminate/2,
          code_change/3]).
 
+-export([safe_handle_event/3]).
+
 %% rabbit_error_logger_file_h is a wrapper around the error_logger_file_h
 %% module because the original's init/1 does not match properly
 %% with the result of closing the old handler when swapping handlers.
@@ -77,8 +79,21 @@ init_file(File, PrevHandler) ->
         Error   -> Error
     end.
 
+handle_event(Event, State) ->
+    safe_handle_event(fun handle_event0/2, Event, State).
+
+safe_handle_event(HandleEvent, Event, State) ->
+    try
+        HandleEvent(Event, State)
+    catch
+        _:Error ->
+            io:format("Event crashed log handler:~n~P~n~P~n",
+                      [Event, 30, Error, 30]),
+            {ok, State}
+    end.
+
 %% filter out "application: foo; exited: stopped; type: temporary"
-handle_event({info_report, _, {_, std_info, _}}, State) ->
+handle_event0({info_report, _, {_, std_info, _}}, State) ->
     {ok, State};
 %% When a node restarts quickly it is possible the rest of the cluster
 %% will not have had the chance to remove its queues from
@@ -88,7 +103,7 @@ handle_event({info_report, _, {_, std_info, _}}, State) ->
 %% logs an event for every one of those messages; in extremis this can
 %% bring the server to its knees just logging "Discarding..."
 %% again and again. So just log the first one, then go silent.
-handle_event(Event = {error, _, {emulator, _, ["Discarding message" ++ _]}},
+handle_event0(Event = {error, _, {emulator, _, ["Discarding message" ++ _]}},
              State) ->
     case get(discarding_message_seen) of
         true      -> {ok, State};
@@ -96,10 +111,10 @@ handle_event(Event = {error, _, {emulator, _, ["Discarding message" ++ _]}},
                      error_logger_file_h:handle_event(t(Event), State)
     end;
 %% Clear this state if we log anything else (but not a progress report).
-handle_event(Event = {info_msg, _, _}, State) ->
+handle_event0(Event = {info_msg, _, _}, State) ->
     erase(discarding_message_seen),
     error_logger_file_h:handle_event(t(Event), State);
-handle_event(Event, State) ->
+handle_event0(Event, State) ->
     error_logger_file_h:handle_event(t(Event), State).
 
 handle_info(Info, State) ->
diff --git a/src/rabbit_sasl_report_file_h.erl b/src/rabbit_sasl_report_file_h.erl
index 4881210d..2dd16702 100644
--- a/src/rabbit_sasl_report_file_h.erl
+++ b/src/rabbit_sasl_report_file_h.erl
@@ -22,6 +22,8 @@
 -export([init/1, handle_event/2, handle_call/2, handle_info/2, terminate/2,
          code_change/3]).
 
+-import(rabbit_error_logger_file_h, [safe_handle_event/3]).
+
 %% rabbit_sasl_report_file_h is a wrapper around the sasl_report_file_h
 %% module because the original's init/1 does not match properly
 %% with the result of closing the old handler when swapping handlers.
@@ -67,6 +69,9 @@ init_file({File, Type}) ->
     end.
 
 handle_event(Event, State) ->
+    safe_handle_event(fun handle_event0/2, Event, State).
+
+handle_event0(Event, State) ->
     sasl_report_file_h:handle_event(
       truncate:log_event(Event, ?LOG_TRUNC), State).