cascache.py: Send message in case of unclean termination of buildbox-casdbschubert/ensure-buildbox-alive

This adds messages in the various mis-termination of Buildbox-casd, to
notify users that something might have gone wrong there.

It also adds a few tests to validate the various behaviors.

2 files changed, 90 insertions, 2 deletions

diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index 974524fcf..0f79c8693 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -39,6 +39,7 @@ from .._protos.build.buildgrid import local_cas_pb2, local_cas_pb2_grpc
 
 from .. import utils
 from .._exceptions import CASCacheError
+from .._message import Message, MessageType
 
 from .casremote import _CASBatchRead, _CASBatchUpdate
 
@@ -902,11 +903,23 @@ class CASCache():
     #   messenger (buildstream._messenger.Messenger): Messenger to forward information to the frontend
     #
     def _terminate_casd_process(self, messenger=None):
+        return_code = self._casd_process.poll()
+
+        if return_code is not None:
+            # buildbox-casd is already dead
+            self._casd_process = None
+
+            if messenger:
+                messenger.message(
+                    Message(MessageType.BUG, "Buildbox-casd died during the run. Exit code: {}".format(return_code))
+                )
+            return
+
         self._casd_process.terminate()
 
         try:
             # Don't print anything if buildbox-casd terminates quickly
-            self._casd_process.wait(timeout=0.5)
+            return_code = self._casd_process.wait(timeout=0.5)
         except subprocess.TimeoutExpired:
             if messenger:
                 cm = messenger.timed_activity("Terminating buildbox-casd")
@@ -914,11 +927,23 @@ class CASCache():
                 cm = contextlib.suppress()
             with cm:
                 try:
-                    self._casd_process.wait(timeout=15)
+                    return_code = self._casd_process.wait(timeout=15)
                 except subprocess.TimeoutExpired:
                     self._casd_process.kill()
                     self._casd_process.wait(timeout=15)
 
+                    if messenger:
+                        messenger.message(
+                            Message(MessageType.WARN, "Buildbox-casd didn't exit in time and has been killed")
+                        )
+                    self._casd_process = None
+                    return
+
+        if return_code != 0 and messenger:
+            messenger.message(
+                Message(MessageType.BUG, "Buildbox-casd didn't exit cleanly. Exit code: {}".format(return_code))
+            )
+
         self._casd_process = None
 
     # get_cache_usage():
diff --git a/tests/internals/cascache.py b/tests/internals/cascache.py
new file mode 100644
index 000000000..8eb5cc29f
--- /dev/null
+++ b/tests/internals/cascache.py
@@ -0,0 +1,63 @@
+import os
+import time
+from unittest.mock import MagicMock
+
+from buildstream._cas.cascache import CASCache
+from buildstream._message import MessageType
+from buildstream._messenger import Messenger
+
+
+def test_report_when_cascache_dies_before_asked_to(tmp_path, monkeypatch):
+    dummy_buildbox_casd = tmp_path.joinpath("buildbox-casd")
+    dummy_buildbox_casd.write_text("#!/bin/bash\nexit 0")
+    dummy_buildbox_casd.chmod(0o777)
+    monkeypatch.setenv("PATH", str(tmp_path), prepend=os.pathsep)
+
+    messenger = MagicMock(spec_set=Messenger)
+    cache = CASCache(str(tmp_path.joinpath("casd")), casd=True)
+    time.sleep(1)
+    cache.release_resources(messenger)
+
+    assert messenger.message.call_count == 1
+
+    message = messenger.message.call_args[0][0]
+    assert message.message_type == MessageType.BUG
+    assert "0" in message.message
+    assert "died" in message.message
+
+
+def test_report_when_cascache_exist_not_cleanly(tmp_path, monkeypatch):
+    dummy_buildbox_casd = tmp_path.joinpath("buildbox-casd")
+    dummy_buildbox_casd.write_text("#!/bin/bash\nwhile :\ndo\nsleep 60\ndone")
+    dummy_buildbox_casd.chmod(0o777)
+    monkeypatch.setenv("PATH", str(tmp_path), prepend=os.pathsep)
+
+    messenger = MagicMock(spec_set=Messenger)
+    cache = CASCache(str(tmp_path.joinpath("casd")), casd=True)
+    time.sleep(1)
+    cache.release_resources(messenger)
+
+    assert messenger.message.call_count == 1
+
+    message = messenger.message.call_args[0][0]
+    assert message.message_type == MessageType.BUG
+    assert "-15" in message.message
+    assert "cleanly" in message.message
+
+
+def test_report_when_cascache_is_forcefully_killed(tmp_path, monkeypatch):
+    dummy_buildbox_casd = tmp_path.joinpath("buildbox-casd")
+    dummy_buildbox_casd.write_text("#!/bin/bash\ntrap 'echo hello' SIGTERM\nwhile :\ndo\nsleep 60\ndone")
+    dummy_buildbox_casd.chmod(0o777)
+    monkeypatch.setenv("PATH", str(tmp_path), prepend=os.pathsep)
+
+    messenger = MagicMock(spec_set=Messenger)
+    cache = CASCache(str(tmp_path.joinpath("casd")), casd=True)
+    time.sleep(1)
+    cache.release_resources(messenger)
+
+    assert messenger.message.call_count == 1
+
+    message = messenger.message.call_args[0][0]
+    assert message.message_type == MessageType.WARN
+    assert "killed" in message.message