Merge branch 'tpollard/loop_exception' into 'master'traveltissues/mr5

scheduler.py: Handle exceptions that are caught under the event loop

Closes #1245

See merge request BuildStream/buildstream!1788

2 files changed, 25 insertions, 2 deletions

diff --git a/src/buildstream/_frontend/app.py b/src/buildstream/_frontend/app.py
index 09610851f..b3687911b 100644
--- a/src/buildstream/_frontend/app.py
+++ b/src/buildstream/_frontend/app.py
@@ -507,10 +507,13 @@ class App:
 
     # Exception handler
     #
-    def _global_exception_handler(self, etype, value, tb):
+    def _global_exception_handler(self, etype, value, tb, exc=True):
 
         # Print the regular BUG message
-        formatted = "".join(traceback.format_exception(etype, value, tb))
+        formatted = None
+        if exc:
+            # Format the exception & traceback by default
+            formatted = "".join(traceback.format_exception(etype, value, tb))
         self._message(MessageType.BUG, str(value), detail=formatted)
 
         # If the scheduler has started, try to terminate all jobs gracefully,
diff --git a/src/buildstream/_scheduler/scheduler.py b/src/buildstream/_scheduler/scheduler.py
index ca7e9279c..0bcbb7182 100644
--- a/src/buildstream/_scheduler/scheduler.py
+++ b/src/buildstream/_scheduler/scheduler.py
@@ -25,6 +25,7 @@ import asyncio
 from itertools import chain
 import signal
 import datetime
+import sys
 
 # Local imports
 from .resources import Resources
@@ -188,6 +189,9 @@ class Scheduler:
         # Add timeouts
         self.loop.call_later(1, self._tick)
 
+        # Add exception handler
+        self.loop.set_exception_handler(self._handle_exception)
+
         # Handle unix signals while running
         self._connect_signals()
 
@@ -596,6 +600,22 @@ class Scheduler:
             # as we don't want to pickle exceptions between processes
             raise ValueError("Unrecognised notification type received")
 
+    def _handle_exception(self, loop, context: dict) -> None:
+        e = context.get("exception")
+        exc = bool(e)
+        if e is None:
+            # https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.call_exception_handler
+            # If no optional Exception generate a generic exception with message value.
+            # exc will be False, instructing the global handler to skip formatting the
+            # assumed exception & related traceback.
+            e = Exception(str(context.get("message")) + " asyncio exception handler called, but no Exception() given")
+
+        # Call the sys global exception handler directly, as to avoid the default
+        # async handler raising an unhandled exception here. App will treat this
+        # as a 'BUG', format it appropriately & exit. mypy needs to ignore parameter
+        # types here as we're overriding sys globally in App._global_exception_handler()
+        sys.excepthook(type(e), e, e.__traceback__, exc)  # type: ignore
+
     def __getstate__(self):
         # The only use-cases for pickling in BuildStream at the time of writing
         # are enabling the 'spawn' method of starting child processes, and