Merge branch 'bschubert/cleanup-local-state' into 'master'

Cleanup MetaElement local state

See merge request BuildStream/buildstream!1147

4 files changed, 47 insertions, 5 deletions

diff --git a/buildstream/_exceptions.py b/buildstream/_exceptions.py
index 0797e7207..8728f6e69 100644
--- a/buildstream/_exceptions.py
+++ b/buildstream/_exceptions.py
@@ -19,6 +19,7 @@
 #        Tiago Gomes <tiago.gomes@codethink.co.uk>
 
 from enum import Enum
+import os
 
 # Disable pylint warnings for whole file here:
 # pylint: disable=global-statement
@@ -50,6 +51,9 @@ def get_last_exception():
 # Used by regression tests
 #
 def get_last_task_error():
+    if 'BST_TEST_SUITE' not in os.environ:
+        raise BstError("Getting the last task error is only supported when running tests")
+
     global _last_task_error_domain
     global _last_task_error_reason
 
@@ -67,11 +71,12 @@ def get_last_task_error():
 # tests about how things failed in a machine readable way
 #
 def set_last_task_error(domain, reason):
-    global _last_task_error_domain
-    global _last_task_error_reason
+    if 'BST_TEST_SUITE' in os.environ:
+        global _last_task_error_domain
+        global _last_task_error_reason
 
-    _last_task_error_domain = domain
-    _last_task_error_reason = reason
+        _last_task_error_domain = domain
+        _last_task_error_reason = reason
 
 
 class ErrorDomain(Enum):
@@ -126,7 +131,8 @@ class BstError(Exception):
         self.reason = reason
 
         # Hold on to the last raised exception for testing purposes
-        _last_exception = self
+        if 'BST_TEST_SUITE' in os.environ:
+            _last_exception = self
 
 
 # PluginError
diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py
index d0cfa8522..17d7c63fb 100644
--- a/buildstream/_loader/loader.py
+++ b/buildstream/_loader/loader.py
@@ -152,8 +152,27 @@ class Loader():
             #
             ret.append(loader._collect_element(element))
 
+        self._clean_caches()
+
         return ret
 
+    # clean_caches()
+    #
+    # Clean internal loader caches, recursively
+    #
+    # When loading the elements, the loaders use caches in order to not load the
+    # same element twice. These are kept after loading and prevent garbage
+    # collection. Cleaning them explicitely is required.
+    #
+    def _clean_caches(self):
+        for loader in self._loaders.values():
+            # value may be None with nested junctions without overrides
+            if loader is not None:
+                loader._clean_caches()
+
+        self._meta_elements = {}
+        self._elements = {}
+
     ###########################################
     #            Private Methods              #
     ###########################################
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 028bdcc9f..92ac8cab5 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -358,6 +358,8 @@ class Project():
                 for meta in meta_elements
             ]
 
+        Element._clear_meta_elements_cache()
+
         # Now warn about any redundant source references which may have
         # been discovered in the resolve() phase.
         redundant_refs = Element._get_redundant_source_refs()
diff --git a/buildstream/element.py b/buildstream/element.py
index a3bf59dc6..5ecc25ab2 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -966,6 +966,21 @@ class Element(Plugin):
 
         return element
 
+    # _clear_meta_elements_cache()
+    #
+    # Clear the internal meta elements cache.
+    #
+    # When loading elements from meta, we cache already instantiated elements
+    # in order to not have to load the same elements twice.
+    # This clears the cache.
+    #
+    # It should be called whenever we are done loading all elements in order
+    # to save memory.
+    #
+    @classmethod
+    def _clear_meta_elements_cache(cls):
+        cls.__instantiated_elements = {}
+
     # _get_redundant_source_refs()
     #
     # Fetches a list of (Source, ref) tuples of all the Sources