_scheduler: don't pass whole queue to child jobaevri/smallerjobs2

Stop passing the scheduler's job queue's across to child jobs, via the
'action_cb' parameter. Instead pass a module-level function, which will
pickle nicely.

This isn't much of a problem while we are in the 'fork' multiprocessing
model. As we move towards supporting the 'spawn' model for win32, then
we need to consider what we will be pickling and unpickling, to cross
the process boundary.

1 files changed, 7 insertions, 4 deletions

diff --git a/src/buildstream/_scheduler/queues/pullqueue.py b/src/buildstream/_scheduler/queues/pullqueue.py
index 245293342..dfb00aa21 100644
--- a/src/buildstream/_scheduler/queues/pullqueue.py
+++ b/src/buildstream/_scheduler/queues/pullqueue.py
@@ -33,10 +33,8 @@ class PullQueue(Queue):
     complete_name = "Pulled"
     resources = [ResourceType.DOWNLOAD, ResourceType.CACHE]
 
-    def process(self, element):
-        # returns whether an artifact was downloaded or not
-        if not element._pull():
-            raise SkipJob(self.action_name)
+    def get_process_func(self):
+        return PullQueue._raise_skip_if_not_pulled
 
     def status(self, element):
         if not element._can_query_cache():
@@ -65,3 +63,8 @@ class PullQueue(Queue):
         # immediately ready to query the artifact cache so that it
         # may be pulled.
         element._set_can_query_cache_callback(self._enqueue_element)
+
+    @staticmethod
+    def _raise_skip_if_not_pulled(element):
+        if not element._pull():
+            raise SkipJob(PullQueue.action_name)