diff options
| author | Tom Pollard <tom.pollard@codethink.co.uk> | 2019-11-12 16:55:12 +0000 |
|---|---|---|
| committer | Tom Pollard <tom.pollard@codethink.co.uk> | 2019-12-11 10:01:01 +0000 |
| commit | ac2433491154c458e8d0c7cc68ccd6cded508153 (patch) | |
| tree | 22623b8808f84b82bca193cfd63c100bba9d261d | |
| parent | 5c8a716a04fcb0c36ec5b6ac0669e2a1b2c7ac6d (diff) | |
| download | buildstream-tpollard/buildsubprocess.tar.gz | |
Apply AsyncioSafeProcess to Stream's multiprocesstpollard/buildsubprocess
Note this stops explictly using the get_context object from
multiprocessing which allows for fork to be used in a process
where spawn is the default. This obviously breaks the linux CI
targets for FORCE SPAWN.
| -rw-r--r-- | src/buildstream/_multiprocessing.py (renamed from src/buildstream/_scheduler/_multiprocessing.py) | 0 | ||||
| -rw-r--r-- | src/buildstream/_scheduler/jobs/job.py | 3 | ||||
| -rw-r--r-- | src/buildstream/_stream.py | 6 |
3 files changed, 4 insertions, 5 deletions
diff --git a/src/buildstream/_scheduler/_multiprocessing.py b/src/buildstream/_multiprocessing.py index 4864e140c..4864e140c 100644 --- a/src/buildstream/_scheduler/_multiprocessing.py +++ b/src/buildstream/_multiprocessing.py diff --git a/src/buildstream/_scheduler/jobs/job.py b/src/buildstream/_scheduler/jobs/job.py index 460f63dbf..9d135c61b 100644 --- a/src/buildstream/_scheduler/jobs/job.py +++ b/src/buildstream/_scheduler/jobs/job.py @@ -33,8 +33,7 @@ import traceback from ..._exceptions import ImplError, BstError, SkipJob from ..._message import Message, MessageType, unconditional_messages from ...types import FastEnum -from ... import _signals, utils -from .. import _multiprocessing +from ... import _signals, utils, _multiprocessing from .jobpickler import pickle_child_job, do_pickled_child_job diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index ebabcac5e..86eea4905 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -65,7 +65,7 @@ from ._profile import Topics, PROFILER from ._state import State from .types import _KeyStrength, _SchedulerErrorAction from .plugin import Plugin -from . import utils, _yaml, _site, _signals +from . import utils, _yaml, _site, _signals, _multiprocessing from . import Scope @@ -152,7 +152,7 @@ class Stream: def run_in_subprocess(self, func, *args, **kwargs): assert not self._subprocess - mp_context = mp.get_context(method="fork") + # mp_context = _multiprocessing.get_context(method='fork') process_name = "stream-{}".format(func.__name__) self._notify_front_queue = mp.Queue() @@ -165,7 +165,7 @@ class Stream: args.insert(0, self._notify_front_queue) args.insert(0, func) - self._subprocess = mp_context.Process( + self._subprocess = _multiprocessing.AsyncioSafeProcess( target=Stream._subprocess_main, args=args, kwargs=kwargs, name=process_name ) |