job.py: Don't use 'terminate_wait', as it uses waitpid()

Using `join()` on the subprocess calls `waitpid()` under the hood
which breaks our child watcher.

Instead, schedule a task for 20 seconds later that will effectively
kill the tasks.

Note that the task will only be called if we still have active jobs.
Otherwise, it will just be skipped and we won't wait as long.

2 files changed, 6 insertions, 28 deletions

diff --git a/src/buildstream/_scheduler/jobs/job.py b/src/buildstream/_scheduler/jobs/job.py
index 4e6199e16..00de9053c 100644
--- a/src/buildstream/_scheduler/jobs/job.py
+++ b/src/buildstream/_scheduler/jobs/job.py
@@ -249,22 +249,6 @@ class Job():
     def get_terminated(self):
         return self._terminated
 
-    # terminate_wait()
-    #
-    # Wait for terminated jobs to complete
-    #
-    # Args:
-    #    timeout (float): Seconds to wait
-    #
-    # Returns:
-    #    (bool): True if the process terminated cleanly, otherwise False
-    #
-    def terminate_wait(self, timeout):
-
-        # Join the child process after sending SIGTERM
-        self._process.join(timeout)
-        return self._process.exitcode is not None
-
     # kill()
     #
     # Forcefully kill the process, and any children it might have.
diff --git a/src/buildstream/_scheduler/scheduler.py b/src/buildstream/_scheduler/scheduler.py
index 7ef5c5fe3..ba81a48bf 100644
--- a/src/buildstream/_scheduler/scheduler.py
+++ b/src/buildstream/_scheduler/scheduler.py
@@ -526,21 +526,15 @@ class Scheduler():
         self.loop.remove_signal_handler(signal.SIGTERM)
 
     def _terminate_jobs_real(self):
-        # 20 seconds is a long time, it can take a while and sometimes
-        # we still fail, need to look deeper into this again.
-        wait_start = datetime.datetime.now()
-        wait_limit = 20.0
+        def kill_jobs():
+            for job_ in self._active_jobs:
+                job_.kill()
 
-        # First tell all jobs to terminate
-        for job in self._active_jobs:
-            job.terminate()
+        # Schedule all jobs to be killed if they have not exited in 20 sec
+        self.loop.call_later(20, kill_jobs)
 
-        # Now wait for them to really terminate
         for job in self._active_jobs:
-            elapsed = datetime.datetime.now() - wait_start
-            timeout = max(wait_limit - elapsed.total_seconds(), 0.0)
-            if not job.terminate_wait(timeout):
-                job.kill()
+            job.terminate()
 
     # Regular timeout for driving status in the UI
     def _tick(self):