_artifact.py: Store sandbox config and environment in artifactstristan/artifact-extensions

This commit adds sandbox config and environment variables to the
artifact, so that they can be used later to shell into a downloaded
artifact or to run commands such as `bst artifact checkout --integrate`
which require running sandboxes on downloaded artifacts.

Summary:

  * _artifact.py:

    - Store new data in the new proto digest, similar
      to how we store public data.

    - Added new accessors to extract these from a loaded artifact.

    - Bump the proto version number for compatibility

  * _versions.py: Bump the global artifact version number

  * element.py: Call Artifact.cache() with new parameters

  * tests/cachekey: Updated cache key test

32 files changed, 97 insertions, 34 deletions

diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py
index c110e57f0..0b78a60d6 100644
--- a/src/buildstream/_artifact.py
+++ b/src/buildstream/_artifact.py
@@ -29,13 +29,15 @@ artifact composite interaction away from Element class
 """
 
 import os
+from typing import Dict
 
 from ._protos.buildstream.v2.artifact_pb2 import Artifact as ArtifactProto
 from . import _yaml
 from . import utils
+from .node import Node
 from .types import _Scope
 from .storage._casbaseddirectory import CasBasedDirectory
-
+from .sandbox._config import SandboxConfig
 
 # An Artifact class to abstract artifact operations
 # from the Element class
@@ -48,7 +50,7 @@ from .storage._casbaseddirectory import CasBasedDirectory
 #
 class Artifact:
 
-    version = 0
+    version = 1
 
     def __init__(self, element, context, *, strong_key=None, weak_key=None):
         self._element = element
@@ -137,11 +139,13 @@ class Artifact:
     #    sourcesvdir (Directory): Virtual Directoy object for the staged sources
     #    buildresult (tuple): bool, short desc and detailed desc of result
     #    publicdata (dict): dict of public data to commit to artifact metadata
+    #    environment (dict): dict of the element's environment variables
+    #    sandboxconfig (SandboxConfig): The element's SandboxConfig
     #
     # Returns:
     #    (int): The size of the newly cached artifact
     #
-    def cache(self, sandbox_build_dir, collectvdir, sourcesvdir, buildresult, publicdata):
+    def cache(self, sandbox_build_dir, collectvdir, sourcesvdir, buildresult, publicdata, environment, sandboxconfig):
 
         context = self._context
         element = self._element
@@ -180,6 +184,17 @@ class Artifact:
             artifact.public_data.CopyFrom(public_data_digest)
             size += public_data_digest.size_bytes
 
+        # Store sandbox data, including SandboxConfig and environment variables
+        with utils._tempnamedfile_name(dir=self._tmpdir) as tmpname:
+            sandbox_config = sandboxconfig.to_dict()
+            sandbox_dict = {"environment": environment, "sandbox-config": sandbox_config}
+            sandboxdata = Node.from_dict(sandbox_dict)
+
+            _yaml.roundtrip_dump(sandboxdata, tmpname)
+            sandbox_data_digest = self._cas.add_object(path=tmpname, link_directly=True)
+            artifact.sandbox_data.CopyFrom(sandbox_data_digest)
+            size += sandbox_data_digest.size_bytes
+
         # store build dependencies
         for e in element._dependencies(_Scope.BUILD):
             new_build = artifact.build_deps.add()
@@ -282,6 +297,46 @@ class Artifact:
 
         return data
 
+    # load_sandbox_config():
+    #
+    # Loads the sandbox configuration from the cached artifact
+    #
+    # Returns:
+    #    The stored SandboxConfig object
+    #
+    def load_sandbox_config(self) -> SandboxConfig:
+
+        # Load the sandbox data from the artifact
+        artifact = self._get_proto()
+        meta_file = self._cas.objpath(artifact.sandbox_data)
+        data = _yaml.load(meta_file, shortname="sandbox.yaml")
+
+        # Extract the sandbox data
+        config = data.get_mapping("sandbox-config")
+
+        # Return a SandboxConfig
+        return SandboxConfig.new_from_node(config)
+
+    # load_environment():
+    #
+    # Loads the environment variables from the cached artifact
+    #
+    # Returns:
+    #    The environment variables
+    #
+    def load_environment(self) -> Dict[str, str]:
+
+        # Load the sandbox data from the artifact
+        artifact = self._get_proto()
+        meta_file = self._cas.objpath(artifact.sandbox_data)
+        data = _yaml.load(meta_file, shortname="sandbox.yaml")
+
+        # Extract the environment
+        config = data.get_mapping("environment")
+
+        # Return the environment
+        return config.strip_node_info()
+
     # load_build_result():
     #
     # Load the build result from the cached artifact
diff --git a/src/buildstream/_versions.py b/src/buildstream/_versions.py
index f97560b4d..37a7a0acd 100644
--- a/src/buildstream/_versions.py
+++ b/src/buildstream/_versions.py
@@ -24,4 +24,4 @@
 # or if buildstream was changed in a way which can cause
 # the same cache key to produce something that is no longer
 # the same.
-BST_CORE_ARTIFACT_VERSION = 9
+BST_CORE_ARTIFACT_VERSION = 10
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 4a03e1c7d..1cc3c9b6f 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1849,7 +1849,15 @@ class Element(Plugin):
         assert self.__artifact._cache_key is not None
 
         with self.timed_activity("Caching artifact"):
-            artifact_size = self.__artifact.cache(sandbox_build_dir, collectvdir, sourcesvdir, buildresult, publicdata)
+            artifact_size = self.__artifact.cache(
+                sandbox_build_dir,
+                collectvdir,
+                sourcesvdir,
+                buildresult,
+                publicdata,
+                self.__environment,
+                self.__sandbox_config,
+            )
 
         if collect is not None and collectvdir is None:
             raise ElementError(
diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected
index 369036a16..421373946 100644
--- a/tests/cachekey/project/elements/build1.expected
+++ b/tests/cachekey/project/elements/build1.expected
@@ -1 +1 @@
-e47d9b32ba894f1d454b66d8d30d3d20226b54c327e8b6a3893c9c55ff02378c
\ No newline at end of file
+5413d1e87db53d8c3120af005a86d221ac79d98a9253ddb49ab7b3d56133e698
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected
index 6e9dfd075..542c322e4 100644
--- a/tests/cachekey/project/elements/build2.expected
+++ b/tests/cachekey/project/elements/build2.expected
@@ -1 +1 @@
-ffe535e1ac311448584d9f0b17f51bb85c8cc9bd3c081a411f9c3ea464f5fcd7
\ No newline at end of file
+613857cdd1b6cb5e0f8fb98b1e32539803a787d03cbcc522c1fcf5b8db4fb260
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/build3.expected b/tests/cachekey/project/elements/build3.expected
index 7b3aabe9f..d914e5a36 100644
--- a/tests/cachekey/project/elements/build3.expected
+++ b/tests/cachekey/project/elements/build3.expected
@@ -1 +1 @@
-c8bae677ec6af58543e4b779a295c9f6bee0c1eface38d9be62a66c247a68ba5
\ No newline at end of file
+e024f294c661f8df3a7b4463b51d17340c242ea90a352c5056199e2dc5fbd061
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected
index fd0ca75fd..4609f8b40 100644
--- a/tests/cachekey/project/elements/compose1.expected
+++ b/tests/cachekey/project/elements/compose1.expected
@@ -1 +1 @@
-3a033cd6b00a0b7e113f2f78f67e73923bdadad33e7d8825fabb0e75f0386417
\ No newline at end of file
+ca9e0e964d2277f8a8bc502962955d350932f9cf496c7eca7052719285352012
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected
index 5ab72fe76..c5fda499a 100644
--- a/tests/cachekey/project/elements/compose2.expected
+++ b/tests/cachekey/project/elements/compose2.expected
@@ -1 +1 @@
-ec2cf079f662a497af5687d2458489fe3ad83810a2db1815ea489fb30f416fc9
\ No newline at end of file
+f293cd2989b3255d4bae77fcbf000163f299faf6303f2eba59b10a4f2a5fe060
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected
index 5a6620a09..19a59afb0 100644
--- a/tests/cachekey/project/elements/compose3.expected
+++ b/tests/cachekey/project/elements/compose3.expected
@@ -1 +1 @@
-d24fd1c4beca8b40aad24e8bd4a6106e1278e193b10872e1ed33b0829eb4831a
\ No newline at end of file
+2254279497818a92d7380ab00b30408914bb8d2b4adbef02ca125753ab1acadc
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected
index 2c4ab4c48..b7e49623c 100644
--- a/tests/cachekey/project/elements/compose4.expected
+++ b/tests/cachekey/project/elements/compose4.expected
@@ -1 +1 @@
-37dd09028bd0a95e64b83df32a1446932f81a72c4c9a6a2cac5c0165ac61e29b
\ No newline at end of file
+1d842d5830bd62a931ee7478a1bf14ec6345a97cea8d94d78e239d04b00e97fd
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected
index 425c55e96..c02a097f8 100644
--- a/tests/cachekey/project/elements/compose5.expected
+++ b/tests/cachekey/project/elements/compose5.expected
@@ -1 +1 @@
-4f2eea4571545a5ce896b13320aa7e6c55ca874237fcb7ebeea017f8456e1b5c
\ No newline at end of file
+ba6db166dd116166426b11bf42f8319daa2152b430eb43d3df917fd710d676d7
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected
index d9e854ed6..7aa0afb42 100644
--- a/tests/cachekey/project/elements/import1.expected
+++ b/tests/cachekey/project/elements/import1.expected
@@ -1 +1 @@
-f27187b2e1b43a5d4d1855fc46fabe256d561d49d6aff2186a52d4d75315fe19
\ No newline at end of file
+99dd881c8b3435c8e724a3fa71f62fdd8e7b0a122c30f66e3a1b253b139b71b9
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected
index 263684b4e..f7448dded 100644
--- a/tests/cachekey/project/elements/import2.expected
+++ b/tests/cachekey/project/elements/import2.expected
@@ -1 +1 @@
-815467a6401a32162cfbcff05f11ff366f9f2d11338bc89d9cf41715da7db89c
\ No newline at end of file
+969b77ee0c906eb3e7f027a4da7a2d68bf673cba38df681d5551882ea74c364c
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected
index 24d6a031e..0b3e7feca 100644
--- a/tests/cachekey/project/elements/import3.expected
+++ b/tests/cachekey/project/elements/import3.expected
@@ -1 +1 @@
-3fef8b7e07efe9b9599e32ce72b1f0f8ab31accd61788c94c57951770ab184c2
\ No newline at end of file
+bb438a86df411aefbe479d0495d84601014e230a14dfb4807bdb242a6fa20084
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected
index 5d9799f1a..b27de64c3 100644
--- a/tests/cachekey/project/elements/script1.expected
+++ b/tests/cachekey/project/elements/script1.expected
@@ -1 +1 @@
-1220b2849910f60f864c2c325cda301809dcb9b730281862448508fbeae5fb91
\ No newline at end of file
+40b501ce478821f39823a6c02124b9b93bfe2b0a286bb80bbe61531357c02b24
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/bzr1.expected b/tests/cachekey/project/sources/bzr1.expected
index 4e66d6c12..69c417957 100644
--- a/tests/cachekey/project/sources/bzr1.expected
+++ b/tests/cachekey/project/sources/bzr1.expected
@@ -1 +1 @@
-ce45db2c82a50eec39416dc857c7a676121a5276b396ea6c7917b123a932ea34
\ No newline at end of file
+2c6d7d9bbecc6b4ee47be936491c77bcbd9a31685fe2cb27cbb456ab01d84df0
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git1.expected b/tests/cachekey/project/sources/git1.expected
index 7c6974590..81cb3dd6c 100644
--- a/tests/cachekey/project/sources/git1.expected
+++ b/tests/cachekey/project/sources/git1.expected
@@ -1 +1 @@
-f02ac8cc516819ee72f56bdc3c1a6e4bb3a154a506a361a8d34a63b37dfa8dc7
\ No newline at end of file
+ec9ffc62b2b41080bbfcaa58be70862235fd79bd2b86b76ded83dadb31b9067c
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git2.expected b/tests/cachekey/project/sources/git2.expected
index 4fa7db829..b80bc1c8a 100644
--- a/tests/cachekey/project/sources/git2.expected
+++ b/tests/cachekey/project/sources/git2.expected
@@ -1 +1 @@
-59964ee437e38e37dbbb78362118937e90defd89bdf358667394a35f6ed2ba46
\ No newline at end of file
+7836185b731178dd71a5516ea1c2e3ea415e51c9ccaf8b1a09eb4e328d44a5ef
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git3.expected b/tests/cachekey/project/sources/git3.expected
index 91cb0f882..02470bfc3 100644
--- a/tests/cachekey/project/sources/git3.expected
+++ b/tests/cachekey/project/sources/git3.expected
@@ -1 +1 @@
-10dd7b3883cfe8fd8f7f29dd258098080227ee73c53ff825542d62d204f1e6c3
\ No newline at end of file
+bbd785fc38a9102f3eca0127f91a5106a204e813c272e687cebf4e27db2d0e55
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected
index 961785aa5..301a04d91 100644
--- a/tests/cachekey/project/sources/local1.expected
+++ b/tests/cachekey/project/sources/local1.expected
@@ -1 +1 @@
-13acaa66115bd1d374b236b0ddd2cef804df75b3aa28878f4200efa7c758b14c
\ No newline at end of file
+a5348ed332c979cf36d32c47305af704c4e68492994c0f05506de988fb1bac32
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected
index 762e7bc48..0c46a4177 100644
--- a/tests/cachekey/project/sources/local2.expected
+++ b/tests/cachekey/project/sources/local2.expected
@@ -1 +1 @@
-214238ba50ead82896e27d95e833c3df990f9e3ab728fdb0e44a3b07f986ab3c
\ No newline at end of file
+c23efecb2fbfe5918e109a626c0b25395b575966700a5cf400b367ce47d6af3c
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch1.expected b/tests/cachekey/project/sources/patch1.expected
index 4593895c5..7dee82352 100644
--- a/tests/cachekey/project/sources/patch1.expected
+++ b/tests/cachekey/project/sources/patch1.expected
@@ -1 +1 @@
-3b607c5657b600ab0e2c8b5ea09ed26fc65b3899b67d107676cf687b2cde374d
\ No newline at end of file
+0d02000882610679920ae10fb373aae949a05f78bb38067dd0bdd31f702c4ddc
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch2.expected b/tests/cachekey/project/sources/patch2.expected
index 221c9c779..7e685dad9 100644
--- a/tests/cachekey/project/sources/patch2.expected
+++ b/tests/cachekey/project/sources/patch2.expected
@@ -1 +1 @@
-1911adbe62cbd7d9f0ded708530274c7f591798bd0a0b5f7e8358f7fcda3066a
\ No newline at end of file
+df5efec58942441502a76434535bf968deefc1239d0e7115f5493156bf4842e0
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch3.expected b/tests/cachekey/project/sources/patch3.expected
index 1ad6d9fce..93c8b13b4 100644
--- a/tests/cachekey/project/sources/patch3.expected
+++ b/tests/cachekey/project/sources/patch3.expected
@@ -1 +1 @@
-175ff52608a55a6af98466822e22f401c46b2659116ccfeee6dc78e09249da78
\ No newline at end of file
+f8a51d2fa8ca36f953841e02b64a95ddfda4b415aa74e4d84c20e217328ca3a6
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected
index 59f11863b..0b872f797 100644
--- a/tests/cachekey/project/sources/pip1.expected
+++ b/tests/cachekey/project/sources/pip1.expected
@@ -1 +1 @@
-20ac777cc56dfeafcf4543e0d9ed31108b32075c8a3e9d25d3deec16fbf7f246
\ No newline at end of file
+14a8d44049e574eeca422afab39aa8bf9514b7e8b1753c6bcd26317c0ca2e549
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote1.expected b/tests/cachekey/project/sources/remote1.expected
index 9970589f3..8a087f0cb 100644
--- a/tests/cachekey/project/sources/remote1.expected
+++ b/tests/cachekey/project/sources/remote1.expected
@@ -1 +1 @@
-4d13955e7eeb0f0d77b0a3a72e77acd8636bdc8284712975593742d0667f88f7
\ No newline at end of file
+fd13350d98b1aa5824cb6d3a2e562922e7df5d545993e016f2eba2ef253347d5
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote2.expected b/tests/cachekey/project/sources/remote2.expected
index 139e4bf74..92ef794f4 100644
--- a/tests/cachekey/project/sources/remote2.expected
+++ b/tests/cachekey/project/sources/remote2.expected
@@ -1 +1 @@
-5f904fd43ca49f268af3141afe73116f1260800918a738afeaf51a94f99fffb2
\ No newline at end of file
+00021ba6cab57cac9cc18ad2ca68e390eba0c6e7e95e9caa64717d794989d74b
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected
index 89a492067..66897ca35 100644
--- a/tests/cachekey/project/sources/tar1.expected
+++ b/tests/cachekey/project/sources/tar1.expected
@@ -1 +1 @@
-f516c30a7a493acd74e49c4ac91cf697dfd32e8821d4230becac100cdac7df64
\ No newline at end of file
+fa75a47c4b50ce5c87517614253a584d94a632209fa3e28b0473a0e17f355c6d
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected
index e9928d767..85a4ee0b9 100644
--- a/tests/cachekey/project/sources/tar2.expected
+++ b/tests/cachekey/project/sources/tar2.expected
@@ -1 +1 @@
-eba29361b50cf14128dbf34362635bc2170474c8bb13c916638b2531174bf04a
\ No newline at end of file
+ca8e454d9be8c210c32d5b102a8694ed5b5098ef6f0a6daed8845aa0c471d310
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip1.expected b/tests/cachekey/project/sources/zip1.expected
index 41292f2a2..92296c0c0 100644
--- a/tests/cachekey/project/sources/zip1.expected
+++ b/tests/cachekey/project/sources/zip1.expected
@@ -1 +1 @@
-a759e15073fdeed9224e41af4c094464e268bb8ced95dedf6e6dc35ec524d003
\ No newline at end of file
+903d568d7c49092a3175c80fdccc385046ab6080bed8021798775fa4a6209186
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip2.expected b/tests/cachekey/project/sources/zip2.expected
index e723f4852..c6f5ab73d 100644
--- a/tests/cachekey/project/sources/zip2.expected
+++ b/tests/cachekey/project/sources/zip2.expected
@@ -1 +1 @@
-d9c5a347340a387c4cecf24c29ad4b3c524773e8224683380221a62ec820abd9
\ No newline at end of file
+7251cee8748e932cd55fa5c8c06aa6c6b35b1a8c555823c7c7eedbeb490e762a
\ No newline at end of file
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index 9ea9991e9..495638e2d 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-cb3b1bbf3d8b7be1a0ee7305c1056e95225f84b2c2b5189cfee69a66dcbd0786
\ No newline at end of file
+af4ceb04c152f244e2f03a83d5103ceaed3d8c50097ba64c743db01aa4d32872
\ No newline at end of file