summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTom Pollard <tom.pollard@codethink.co.uk>2020-01-02 15:10:31 +0000
committerTom Pollard <tom.pollard@codethink.co.uk>2020-01-10 16:48:52 +0000
commitaed5c340f66a7c3505b8364e6a1948522d549398 (patch)
treed0cb5da51e913fd882b3b1ed611040d438f05241
parent928d8e6b77daee3eb259d96681f4bce9dc512afd (diff)
downloadbuildstream-tpollard/loop_exception.tar.gz
scheduler.py: Handle exceptions that are caught under the event looptpollard/loop_exception
The default exception handler of the async event loop bypasses our custom global exception handler. Ensure that isn't the case, as these exceptions are BUGS & should cause bst to exit.
-rw-r--r--src/buildstream/_frontend/app.py7
-rw-r--r--src/buildstream/_scheduler/scheduler.py20
2 files changed, 25 insertions, 2 deletions
diff --git a/src/buildstream/_frontend/app.py b/src/buildstream/_frontend/app.py
index 09610851f..b3687911b 100644
--- a/src/buildstream/_frontend/app.py
+++ b/src/buildstream/_frontend/app.py
@@ -507,10 +507,13 @@ class App:
# Exception handler
#
- def _global_exception_handler(self, etype, value, tb):
+ def _global_exception_handler(self, etype, value, tb, exc=True):
# Print the regular BUG message
- formatted = "".join(traceback.format_exception(etype, value, tb))
+ formatted = None
+ if exc:
+ # Format the exception & traceback by default
+ formatted = "".join(traceback.format_exception(etype, value, tb))
self._message(MessageType.BUG, str(value), detail=formatted)
# If the scheduler has started, try to terminate all jobs gracefully,
diff --git a/src/buildstream/_scheduler/scheduler.py b/src/buildstream/_scheduler/scheduler.py
index ca7e9279c..0bcbb7182 100644
--- a/src/buildstream/_scheduler/scheduler.py
+++ b/src/buildstream/_scheduler/scheduler.py
@@ -25,6 +25,7 @@ import asyncio
from itertools import chain
import signal
import datetime
+import sys
# Local imports
from .resources import Resources
@@ -188,6 +189,9 @@ class Scheduler:
# Add timeouts
self.loop.call_later(1, self._tick)
+ # Add exception handler
+ self.loop.set_exception_handler(self._handle_exception)
+
# Handle unix signals while running
self._connect_signals()
@@ -596,6 +600,22 @@ class Scheduler:
# as we don't want to pickle exceptions between processes
raise ValueError("Unrecognised notification type received")
+ def _handle_exception(self, loop, context: dict) -> None:
+ e = context.get("exception")
+ exc = bool(e)
+ if e is None:
+ # https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.call_exception_handler
+ # If no optional Exception generate a generic exception with message value.
+ # exc will be False, instructing the global handler to skip formatting the
+ # assumed exception & related traceback.
+ e = Exception(str(context.get("message")) + " asyncio exception handler called, but no Exception() given")
+
+ # Call the sys global exception handler directly, as to avoid the default
+ # async handler raising an unhandled exception here. App will treat this
+ # as a 'BUG', format it appropriately & exit. mypy needs to ignore parameter
+ # types here as we're overriding sys globally in App._global_exception_handler()
+ sys.excepthook(type(e), e, e.__traceback__, exc) # type: ignore
+
def __getstate__(self):
# The only use-cases for pickling in BuildStream at the time of writing
# are enabling the 'spawn' method of starting child processes, and