Merge branch 'bschubert/ensure-buildbox-alive' into 'master'

Report when Buildbox-casd is not alive at the end of a run

See merge request BuildStream/buildstream!1605

2 files changed, 116 insertions, 17 deletions

diff --git a/src/buildstream/_cas/cascache.py b/src/buildstream/_cas/cascache.py
index 97ee2edd7..6315c1f93 100644
--- a/src/buildstream/_cas/cascache.py
+++ b/src/buildstream/_cas/cascache.py
@@ -39,6 +39,7 @@ from .._protos.build.buildgrid import local_cas_pb2, local_cas_pb2_grpc
 
 from .. import utils
 from .._exceptions import CASCacheError
+from .._message import Message, MessageType
 
 from .casremote import _CASBatchRead, _CASBatchUpdate
 
@@ -183,23 +184,7 @@ class CASCache():
 
         if self._casd_process:
             self.close_channel()
-            self._casd_process.terminate()
-            try:
-                # Don't print anything if buildbox-casd terminates quickly
-                self._casd_process.wait(timeout=0.5)
-            except subprocess.TimeoutExpired:
-                if messenger:
-                    cm = messenger.timed_activity("Terminating buildbox-casd")
-                else:
-                    cm = contextlib.suppress()
-                with cm:
-                    try:
-                        self._casd_process.wait(timeout=15)
-                    except subprocess.TimeoutExpired:
-                        self._casd_process.kill()
-                        self._casd_process.wait(timeout=15)
-            self._casd_process = None
-
+            self._terminate_casd_process(messenger)
             shutil.rmtree(self._casd_socket_tempdir)
 
     # contains():
@@ -945,6 +930,57 @@ class CASCache():
         # Upload any blobs missing on the server
         self.send_blobs(remote, missing_blobs)
 
+    # _terminate_casd_process()
+    #
+    # Terminate the buildbox casd process
+    #
+    # Args:
+    #   messenger (buildstream._messenger.Messenger): Messenger to forward information to the frontend
+    #
+    def _terminate_casd_process(self, messenger=None):
+        return_code = self._casd_process.poll()
+
+        if return_code is not None:
+            # buildbox-casd is already dead
+            self._casd_process = None
+
+            if messenger:
+                messenger.message(
+                    Message(MessageType.BUG, "Buildbox-casd died during the run. Exit code: {}".format(return_code))
+                )
+            return
+
+        self._casd_process.terminate()
+
+        try:
+            # Don't print anything if buildbox-casd terminates quickly
+            return_code = self._casd_process.wait(timeout=0.5)
+        except subprocess.TimeoutExpired:
+            if messenger:
+                cm = messenger.timed_activity("Terminating buildbox-casd")
+            else:
+                cm = contextlib.suppress()
+            with cm:
+                try:
+                    return_code = self._casd_process.wait(timeout=15)
+                except subprocess.TimeoutExpired:
+                    self._casd_process.kill()
+                    self._casd_process.wait(timeout=15)
+
+                    if messenger:
+                        messenger.message(
+                            Message(MessageType.WARN, "Buildbox-casd didn't exit in time and has been killed")
+                        )
+                    self._casd_process = None
+                    return
+
+        if return_code != 0 and messenger:
+            messenger.message(
+                Message(MessageType.BUG, "Buildbox-casd didn't exit cleanly. Exit code: {}".format(return_code))
+            )
+
+        self._casd_process = None
+
     # get_cache_usage():
     #
     # Fetches the current usage of the CAS local cache.
diff --git a/tests/internals/cascache.py b/tests/internals/cascache.py
new file mode 100644
index 000000000..8eb5cc29f
--- /dev/null
+++ b/tests/internals/cascache.py
@@ -0,0 +1,63 @@
+import os
+import time
+from unittest.mock import MagicMock
+
+from buildstream._cas.cascache import CASCache
+from buildstream._message import MessageType
+from buildstream._messenger import Messenger
+
+
+def test_report_when_cascache_dies_before_asked_to(tmp_path, monkeypatch):
+    dummy_buildbox_casd = tmp_path.joinpath("buildbox-casd")
+    dummy_buildbox_casd.write_text("#!/bin/bash\nexit 0")
+    dummy_buildbox_casd.chmod(0o777)
+    monkeypatch.setenv("PATH", str(tmp_path), prepend=os.pathsep)
+
+    messenger = MagicMock(spec_set=Messenger)
+    cache = CASCache(str(tmp_path.joinpath("casd")), casd=True)
+    time.sleep(1)
+    cache.release_resources(messenger)
+
+    assert messenger.message.call_count == 1
+
+    message = messenger.message.call_args[0][0]
+    assert message.message_type == MessageType.BUG
+    assert "0" in message.message
+    assert "died" in message.message
+
+
+def test_report_when_cascache_exist_not_cleanly(tmp_path, monkeypatch):
+    dummy_buildbox_casd = tmp_path.joinpath("buildbox-casd")
+    dummy_buildbox_casd.write_text("#!/bin/bash\nwhile :\ndo\nsleep 60\ndone")
+    dummy_buildbox_casd.chmod(0o777)
+    monkeypatch.setenv("PATH", str(tmp_path), prepend=os.pathsep)
+
+    messenger = MagicMock(spec_set=Messenger)
+    cache = CASCache(str(tmp_path.joinpath("casd")), casd=True)
+    time.sleep(1)
+    cache.release_resources(messenger)
+
+    assert messenger.message.call_count == 1
+
+    message = messenger.message.call_args[0][0]
+    assert message.message_type == MessageType.BUG
+    assert "-15" in message.message
+    assert "cleanly" in message.message
+
+
+def test_report_when_cascache_is_forcefully_killed(tmp_path, monkeypatch):
+    dummy_buildbox_casd = tmp_path.joinpath("buildbox-casd")
+    dummy_buildbox_casd.write_text("#!/bin/bash\ntrap 'echo hello' SIGTERM\nwhile :\ndo\nsleep 60\ndone")
+    dummy_buildbox_casd.chmod(0o777)
+    monkeypatch.setenv("PATH", str(tmp_path), prepend=os.pathsep)
+
+    messenger = MagicMock(spec_set=Messenger)
+    cache = CASCache(str(tmp_path.joinpath("casd")), casd=True)
+    time.sleep(1)
+    cache.release_resources(messenger)
+
+    assert messenger.message.call_count == 1
+
+    message = messenger.message.call_args[0][0]
+    assert message.message_type == MessageType.WARN
+    assert "killed" in message.message