jobs.py: Reduce FD leaks from queues and process objectsdanielsilverstone-ct/maybe-reduce-fd-leaks

The garbage collector can take too long to get around to cleaning
up the Queue and Process instances in completed Job instances.  As
such, FDs tend to leak and in very large projects this can result
in running out of FDs before a build, fetch, track, or other process
can complete.  This patch reduces the chance of that by only creating
the queue when it's needed, and forcing the queue and process instances
to be deleted when the parent is finished with them.

Signed-off-by: Daniel Silverstone <daniel.silverstone@codethink.co.uk>

1 files changed, 6 insertions, 1 deletions

diff --git a/buildstream/_scheduler/jobs/job.py b/buildstream/_scheduler/jobs/job.py
index 165c7c83f..c55219b58 100644
--- a/buildstream/_scheduler/jobs/job.py
+++ b/buildstream/_scheduler/jobs/job.py
@@ -109,7 +109,7 @@ class Job():
         # Private members
         #
         self._scheduler = scheduler            # The scheduler
-        self._queue = multiprocessing.Queue()  # A message passing queue
+        self._queue = None                     # A message passing queue
         self._process = None                   # The Process object
         self._watcher = None                   # Child process watcher
         self._listening = False                # Whether the parent is currently listening
@@ -130,6 +130,8 @@ class Job():
     #
     def spawn(self):
 
+        self._queue = multiprocessing.Queue()
+
         self._tries += 1
         self._parent_start_listening()
 
@@ -552,6 +554,9 @@ class Job():
         self.parent_complete(returncode == RC_OK, self._result)
         self._scheduler.job_completed(self, returncode == RC_OK)
 
+        # Force the deletion of the queue and process objects to try and clean up FDs
+        self._queue = self._process = None
+
     # _parent_process_envelope()
     #
     # Processes a message Envelope deserialized form the message queue.