casdprocessmanager.py: Check if buildbox-casd process is alive while waiting for connectionctolentino/casd-timeout

1 files changed, 12 insertions, 2 deletions

diff --git a/src/buildstream/_cas/casdprocessmanager.py b/src/buildstream/_cas/casdprocessmanager.py
index 89488696b..5ba192d1e 100644
--- a/src/buildstream/_cas/casdprocessmanager.py
+++ b/src/buildstream/_cas/casdprocessmanager.py
@@ -25,6 +25,7 @@ import stat
 import subprocess
 import tempfile
 import time
+import psutil
 
 import grpc
 
@@ -223,11 +224,11 @@ class CASDProcessManager:
     # established until it is needed.
     #
     def create_channel(self):
-        return CASDChannel(self._socket_path, self._connection_string, self._start_time)
+        return CASDChannel(self._socket_path, self._connection_string, self._start_time, self.process.pid)
 
 
 class CASDChannel:
-    def __init__(self, socket_path, connection_string, start_time):
+    def __init__(self, socket_path, connection_string, start_time, casd_pid):
         self._socket_path = socket_path
         self._connection_string = connection_string
         self._start_time = start_time
@@ -235,6 +236,7 @@ class CASDChannel:
         self._bytestream = None
         self._casd_cas = None
         self._local_cas = None
+        self._casd_pid = casd_pid
 
     def _establish_connection(self):
         assert self._casd_channel is None
@@ -245,6 +247,14 @@ class CASDChannel:
             if time.time() > self._start_time + _CASD_TIMEOUT:
                 raise CASCacheError("Timed out waiting for buildbox-casd to become ready")
 
+            # check that process is still alive
+            try:
+                proc = psutil.Process(self._casd_pid)
+                if not proc.is_running():
+                    raise CASCacheError(f"buildbox-casd process died before connection could be established")
+            except psutil.NoSuchProcess:
+                raise CASCacheError("buildbox-casd process died before connection could be established")
+
             time.sleep(0.01)
 
         self._casd_channel = grpc.insecure_channel(self._connection_string)