diff options
| author | Daniel Silverstone <daniel.silverstone@codethink.co.uk> | 2019-02-19 09:46:40 +0000 |
|---|---|---|
| committer | Daniel Silverstone <daniel.silverstone@codethink.co.uk> | 2019-02-26 14:29:18 +0000 |
| commit | 58c51dc7cef80b0b215d33536cd124e9eda2e269 (patch) | |
| tree | bb85895cc879ce554518ab4ebc851f6b2e597204 | |
| parent | f52f75ee52e8d80120eb7af031ea9af30bf3bda9 (diff) | |
| download | buildstream-danielsilverstone-ct/gc-play.tar.gz | |
_stream.py, _project.py: Manage GC during pipeline loaddanielsilverstone-ct/gc-play
Since during pipeline load we are generating a lot of objects
which need to live for the duration of the BuildStream instance
there is benefit to disabling GC for the duration of the load since
that drastically reduces wasted CPU cycles iterating data which
will remain around always. In order to limit the increase in
the peak memory consumption though, we do an explicit gc.collect()
after loading the YAML in, since without that, we use 60% more
memory at peak, and with it, only 20%.
Signed-off-by: Daniel Silverstone <daniel.silverstone@codethink.co.UK.
| -rw-r--r-- | buildstream/_project.py | 4 | ||||
| -rw-r--r-- | buildstream/_stream.py | 1 |
2 files changed, 5 insertions, 0 deletions
diff --git a/buildstream/_project.py b/buildstream/_project.py index 92ac8cab5..6df54f70f 100644 --- a/buildstream/_project.py +++ b/buildstream/_project.py @@ -18,6 +18,7 @@ # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> # Tiago Gomes <tiago.gomes@codethink.co.uk> +import gc import os from collections import OrderedDict from collections.abc import Mapping @@ -352,6 +353,9 @@ class Project(): ticker=None, fetch_subprojects=fetch_subprojects) + # Loading elements generates a lot of garbage, clear it now + gc.collect() + with self._context.timed_activity("Resolving elements"): elements = [ Element._new_from_meta(meta) diff --git a/buildstream/_stream.py b/buildstream/_stream.py index caaa48908..b0fce3817 100644 --- a/buildstream/_stream.py +++ b/buildstream/_stream.py @@ -941,6 +941,7 @@ class Stream(): # (list of Element): The primary element selection # (list of Element): The tracking element selection # + @utils._with_gc_disabled def _load(self, targets, track_targets, *, selection=PipelineSelection.NONE, track_selection=PipelineSelection.NONE, |