Add support for Python 3.8valentindavid/bst-1/python3.8

`ChildWatcher.add_child_handler` needs to be called from a main loop
and is dispatched through that main loop.

To be able to properly dispatched the ChildWatcher handlers,
`Scheduler._terminate_jobs_real` had to be made loop friendly by
yielding between termination and kill so that it knows which process
have been terminated.

Since `Scheduler._terminate_jobs_real` now yields instead of blocking,
we need to avoid calling it multiple times from the main loop.

6 files changed, 33 insertions, 26 deletions

diff --git a/buildstream/_scheduler/jobs/job.py b/buildstream/_scheduler/jobs/job.py
index b8b4a2c76..77ae4a7d1 100644
--- a/buildstream/_scheduler/jobs/job.py
+++ b/buildstream/_scheduler/jobs/job.py
@@ -159,7 +159,9 @@ class Job():
         # the callback is called immediately.
         #
         self._watcher = asyncio.get_child_watcher()
-        self._watcher.add_child_handler(self._process.pid, self._parent_child_completed)
+        self._scheduler.loop.call_soon(self._watcher.add_child_handler,
+                                       self._process.pid,
+                                       self._parent_child_completed)
 
     # terminate()
     #
diff --git a/buildstream/_scheduler/scheduler.py b/buildstream/_scheduler/scheduler.py
index 68c115c1b..727078260 100644
--- a/buildstream/_scheduler/scheduler.py
+++ b/buildstream/_scheduler/scheduler.py
@@ -183,6 +183,9 @@ class Scheduler():
     #
     def terminate_jobs(self):
 
+        if self.terminated:
+            return
+
         # Set this right away, the frontend will check this
         # attribute to decide whether or not to print status info
         # etc and the following code block will trigger some callbacks.
@@ -515,22 +518,22 @@ class Scheduler():
         self.loop.remove_signal_handler(signal.SIGTSTP)
         self.loop.remove_signal_handler(signal.SIGTERM)
 
-    def _terminate_jobs_real(self):
-        # 20 seconds is a long time, it can take a while and sometimes
-        # we still fail, need to look deeper into this again.
-        wait_start = datetime.datetime.now()
-        wait_limit = 20.0
+    def _kill_job(self, job):
+        if job in self._active_jobs:
+            job.kill()
 
+    def _kill_jobs(self):
+        for job in self._active_jobs:
+            self.loop.call_soon(self._kill_job, job)
+
+    def _terminate_jobs_real(self):
         # First tell all jobs to terminate
         for job in self._active_jobs:
             job.terminate()
 
-        # Now wait for them to really terminate
-        for job in self._active_jobs:
-            elapsed = datetime.datetime.now() - wait_start
-            timeout = max(wait_limit - elapsed.total_seconds(), 0.0)
-            if not job.terminate_wait(timeout):
-                job.kill()
+        # 20 seconds is a long time, it can take a while and sometimes
+        # we still fail, need to look deeper into this again.
+        self.loop.call_later(20.0, self._kill_jobs)
 
     # Regular timeout for driving status in the UI
     def _tick(self):
diff --git a/requirements/cov-requirements.in b/requirements/cov-requirements.in
index 455b91ba6..1911f3506 100644
--- a/requirements/cov-requirements.in
+++ b/requirements/cov-requirements.in
@@ -1,2 +1,3 @@
-coverage == 4.4.0
+coverage == 4.4.0 ; python_version < '3.8'
+coverage == 4.5.4 ; python_version >= '3.8'
 pytest-cov >= 2.5.0
diff --git a/requirements/cov-requirements.txt b/requirements/cov-requirements.txt
index 46c70432d..51bc2bd28 100644
--- a/requirements/cov-requirements.txt
+++ b/requirements/cov-requirements.txt
@@ -1,4 +1,5 @@
-coverage==4.4
+coverage==4.4.0; python_version < '3.8'
+coverage==4.5.4; python_version >= '3.8'
 pytest-cov==2.7.1
 ## The following requirements were added by pip freeze:
 atomicwrites==1.3.0
@@ -6,7 +7,7 @@ attrs==19.1.0
 importlib-metadata==0.20
 more-itertools==7.2.0
 packaging==19.1
-pluggy==0.12.0
+pluggy==0.13.1
 py==1.8.0
 pyparsing==2.4.2
 pytest==5.1.2
diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt
index e6f327284..dfc991e9b 100644
--- a/requirements/dev-requirements.txt
+++ b/requirements/dev-requirements.txt
@@ -19,7 +19,7 @@ lazy-object-proxy==1.4.2
 mccabe==0.6.1
 more-itertools==7.2.0
 packaging==19.1
-pluggy==0.12.0
+pluggy==0.13.1
 py==1.8.0
 pyparsing==2.4.2
 pytest-cache==1.0
diff --git a/tox.ini b/tox.ini
index 42b85a108..903ae9af8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -2,7 +2,7 @@
 # Tox global configuration
 #
 [tox]
-envlist = py35,py36,py37
+envlist = py35,py36,py37,py38
 skip_missing_interpreters = true
 
 #
@@ -13,16 +13,16 @@ skip_missing_interpreters = true
 [testenv]
 commands =
     # Running with coverage reporting enabled
-    py{35,36,37}-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc {posargs}
-    py{35,36,37}-!nocover: mkdir -p .coverage-reports
-    py{35,36,37}-!nocover: mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname}
+    py{35,36,37,38}-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc {posargs}
+    py{35,36,37,38}-!nocover: mkdir -p .coverage-reports
+    py{35,36,37,38}-!nocover: mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname}
 
     # Running with coverage reporting disabled
-    py{35,36,37}-nocover: pytest --basetemp {envtmpdir} {posargs}
+    py{35,36,37,38}-nocover: pytest --basetemp {envtmpdir} {posargs}
 deps =
-    py{35,36,37}: -rrequirements/requirements.txt
-    py{35,36,37}: -rrequirements/dev-requirements.txt
-    py{35,36,37}: -rrequirements/plugin-requirements.txt
+    py{35,36,37,38}: -rrequirements/requirements.txt
+    py{35,36,37,38}: -rrequirements/dev-requirements.txt
+    py{35,36,37,38}: -rrequirements/plugin-requirements.txt
 
     # Only require coverage and pytest-cov when using it
     !nocover: -rrequirements/cov-requirements.txt
@@ -35,9 +35,9 @@ passenv =
 # These keys are not inherited by any other sections
 #
 setenv =
-    py{35,36,37}: COVERAGE_FILE = {envtmpdir}/.coverage
+    py{35,36,37,38}: COVERAGE_FILE = {envtmpdir}/.coverage
 whitelist_externals =
-    py{35,36,37}:
+    py{35,36,37,38}:
         mv
         mkdir