Add ElementSourcesCache

Sources have been cached in CAS individually, except for sources that
transform other sources, which have been cached combined with all
previous sources of the element. This caching structure may be confusing
as sources are specified in the element as a list and this is not a good
fit for #1274 where we want to support caching individual sources in a
Remote Asset server with a BuildStream-independent URI (especially the
`directory` configuration would be problematic).

This replaces the combined caching of 'previous' sources with an
element-level source cache, which caches all sources of an element
staged together. Sources that don't depend on previous sources are still
cached individually.

This also makes it possible to add a list of all element sources to the
source proto used by the element-level source cache.

1 files changed, 12 insertions, 0 deletions

diff --git a/src/buildstream/_context.py b/src/buildstream/_context.py
index 8b559153e..0c2d1a150 100644
--- a/src/buildstream/_context.py
+++ b/src/buildstream/_context.py
@@ -28,6 +28,7 @@ from ._messenger import Messenger
 from ._profile import Topics, PROFILER
 from ._platform import Platform
 from ._artifactcache import ArtifactCache
+from ._elementsourcescache import ElementSourcesCache
 from ._sourcecache import SourceCache
 from ._cas import CASCache, CASLogLevel
 from .types import _CacheBuildTrees, _PipelineSelection, _SchedulerErrorAction
@@ -171,6 +172,7 @@ class Context:
         # Private variables
         self._platform = None
         self._artifactcache = None
+        self._elementsourcescache = None
         self._sourcecache = None
         self._projects = []
         self._project_overrides = Node.from_dict({})
@@ -193,6 +195,9 @@ class Context:
         if self._artifactcache:
             self._artifactcache.release_resources()
 
+        if self._elementsourcescache:
+            self._elementsourcescache.release_resources()
+
         if self._sourcecache:
             self._sourcecache.release_resources()
 
@@ -421,6 +426,13 @@ class Context:
         return self._artifactcache
 
     @property
+    def elementsourcescache(self):
+        if not self._elementsourcescache:
+            self._elementsourcescache = ElementSourcesCache(self)
+
+        return self._elementsourcescache
+
+    @property
     def sourcecache(self):
         if not self._sourcecache:
             self._sourcecache = SourceCache(self)