element.py: always check sources for caching

Check that sources are cached even if they are workspaced and do not
reset workspace cache data

partially reverts !1470
closes #1088

element.py: remove workspaces in cache key calc

Using the workspace source plugin, workspaces should now be handled like
sources for the purpose of calculating element cache keys.

partially reverts !1470
works towards #1073

Since the source keys are now calculated using the unique keys of the
workspace source, this change will break external tracking for open
workspaces. In future attempting to track an open workspace might raise
a SourceError. The test is rewritten to close the workspace before
tracking.

30 files changed, 52 insertions, 46 deletions

diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 711723ebe..0ad2da17f 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1279,10 +1279,6 @@ class Element(Plugin):
             # Tracking may still be pending
             return
 
-        if self._get_workspace() and self.__assemble_scheduled:
-            self.__reset_cache_data()
-            return
-
         self.__update_cache_keys()
         self.__update_artifact_state()
 
@@ -2236,7 +2232,6 @@ class Element(Plugin):
             }
 
             project = self._get_project()
-            workspace = self._get_workspace()
 
             self.__cache_key_dict = {
                 'core-artifact-version': BST_CORE_ARTIFACT_VERSION,
@@ -2252,15 +2247,9 @@ class Element(Plugin):
                 return {'key': _source._get_unique_key(True),
                         'name': _source._get_source_name()}
 
-            def __get_workspace_entry(workspace):
-                return {'key': workspace.get_key()}
-
-            if workspace is None:
-                self.__cache_key_dict['sources'] = \
-                    [__get_source_entry(s) for s in self.__sources]
-            else:
-                self.__cache_key_dict['sources'] = \
-                    [__get_workspace_entry(workspace)]
+            self._source_cached()
+            self.__cache_key_dict['sources'] = \
+                [__get_source_entry(s) for s in self.__sources]
 
             self.__cache_key_dict['fatal-warnings'] = sorted(project._fatal_warnings)
 
@@ -2271,7 +2260,7 @@ class Element(Plugin):
 
     # Check if sources are cached, generating the source key if it hasn't been
     def _source_cached(self):
-        if self.__sources and not self._get_workspace():
+        if self.__sources:
             sourcecache = self._get_context().sourcecache
 
             # Go through sources we'll cache generating keys
diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected
index 275abd2d0..d335bb846 100644
--- a/tests/cachekey/project/elements/build1.expected
+++ b/tests/cachekey/project/elements/build1.expected
@@ -1 +1 @@
-ba3cc6aeaef5d4c559acbd3a92e7a2512a72674c3e98aaf44256eb1c951a1a29
\ No newline at end of file
+f5e3487eb2802ee1a7ebcbca6c3c185af2709aba98f4de2004bb5e5ab587884c
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected
index ded4d5ba8..d8183a9ed 100644
--- a/tests/cachekey/project/elements/build2.expected
+++ b/tests/cachekey/project/elements/build2.expected
@@ -1 +1 @@
-707ca16d9d0efd925cdf7e5f2586c7ca6446ad739a0ecce22d3cb967dc557edb
\ No newline at end of file
+98bd367885521bd42a60f926a93cb0d9d5795829fbf433703d331cad10cd3875
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected
index e62634f37..cb99535cc 100644
--- a/tests/cachekey/project/elements/compose1.expected
+++ b/tests/cachekey/project/elements/compose1.expected
@@ -1 +1 @@
-67fe128c47c16737fbb0c675a866a07bea3921715953b2c191ac282b1100b747
\ No newline at end of file
+9c17f7ae6220dd70a099a2a20444a102c26eec76c09861791a0e9f6d67cc660f
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected
index 9c1169326..25e199334 100644
--- a/tests/cachekey/project/elements/compose2.expected
+++ b/tests/cachekey/project/elements/compose2.expected
@@ -1 +1 @@
-436d4c69b1d120315c7b073f564ae07568fd7d3ead0be3df9e90a702793ad107
\ No newline at end of file
+94022d64108335210fec8604ea60c99471586d69c9cb3deef95f29d09e2cb3b7
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected
index 74347beab..62053720e 100644
--- a/tests/cachekey/project/elements/compose3.expected
+++ b/tests/cachekey/project/elements/compose3.expected
@@ -1 +1 @@
-cfb87e685ad0569f92eb2ac7d0f121dd62e3eb4e0a82f559ccf402887461d7aa
\ No newline at end of file
+ee78f0fb0b3247259af41c5bbd4180feeed209da644a2ed373b8e2721da89ca5
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected
index 4a809312d..b353dd4af 100644
--- a/tests/cachekey/project/elements/compose4.expected
+++ b/tests/cachekey/project/elements/compose4.expected
@@ -1 +1 @@
-dc962993b42725a3fbfcb0e38557a663b610e5033e6e2a31dcb8a2e87a5b9117
\ No newline at end of file
+627befddfd44d93b93e59dc8d10906f9cb3616325bdd64c607858e29e7c7622f
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected
index 355a9fefc..be875921d 100644
--- a/tests/cachekey/project/elements/compose5.expected
+++ b/tests/cachekey/project/elements/compose5.expected
@@ -1 +1 @@
-ea2ca85b60271c877f45067de8c7786bbb5317447e127fbf011cc888e47c264e
\ No newline at end of file
+8acb2a21f97dc3622a983449e8e64d0e6b1d1ae7ad072dc164f9daea5241e4eb
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected
index 5cb6a1973..dfb798e77 100644
--- a/tests/cachekey/project/elements/import1.expected
+++ b/tests/cachekey/project/elements/import1.expected
@@ -1 +1 @@
-402b946498422abb82133fa5998b97f93b08d66ee1f332154413886687d8e7e9
\ No newline at end of file
+8920a31ae292e892d62692702bcd43bbb08af53187f1766dcba07c9eea9ea1f0
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected
index 5815a2075..6d37ae6fe 100644
--- a/tests/cachekey/project/elements/import2.expected
+++ b/tests/cachekey/project/elements/import2.expected
@@ -1 +1 @@
-030b97f6f854c270126aa3cc7ac5d541e77e42f984b61aef63b2bce183982310
\ No newline at end of file
+98f292215ab793c2607aa18d8a61ef026d4c99e39f70be5aaaf466659ba2f19c
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected
index ecd6f8f08..1d20a9c44 100644
--- a/tests/cachekey/project/elements/import3.expected
+++ b/tests/cachekey/project/elements/import3.expected
@@ -1 +1 @@
-6c0b0c42d438312a775084341f99547149b6a269ef4a24122b6f75b343802877
\ No newline at end of file
+f57a93e7711b973a6a541272540d21b47579b7b7ef389a336558244592da2d50
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected
index 1b43ee996..bf7cf9209 100644
--- a/tests/cachekey/project/elements/script1.expected
+++ b/tests/cachekey/project/elements/script1.expected
@@ -1 +1 @@
-fb83946f47528e7df4d322ca1848616905d6e259e6db3cea26f906093ce519e0
\ No newline at end of file
+372af727f8f7c8ca235a140bd6c3f9baa8ca9da765b61ff7070f61a9b6a44c41
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/bzr1.expected b/tests/cachekey/project/sources/bzr1.expected
index a53c63adb..9ef16836f 100644
--- a/tests/cachekey/project/sources/bzr1.expected
+++ b/tests/cachekey/project/sources/bzr1.expected
@@ -1 +1 @@
-673bb938cc3fabe0be55e98c6b8b80853168becc86b4fa102fc0c538879bf83a
\ No newline at end of file
+88fee9ea1330cc7a7bc34ba8bac23a4c2dcf21126eae023042fb0eb48a4123f8
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git1.expected b/tests/cachekey/project/sources/git1.expected
index 9fa018ccc..bcbcb78ac 100644
--- a/tests/cachekey/project/sources/git1.expected
+++ b/tests/cachekey/project/sources/git1.expected
@@ -1 +1 @@
-53a367133fb8f3ca86ba772801ea62681414271da9582800dd56a62e9c6d7e5d
\ No newline at end of file
+2811ce3bbe1dc77cda9da12563487183da660c06783b220d9b1bfc4b151f9869
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git2.expected b/tests/cachekey/project/sources/git2.expected
index d9cad4dcd..aa2b09e64 100644
--- a/tests/cachekey/project/sources/git2.expected
+++ b/tests/cachekey/project/sources/git2.expected
@@ -1 +1 @@
-86e0cf4f3154fa006899acf64317930ebf08ca6d01113abfa35ccceed2961fcd
\ No newline at end of file
+84e90059df19d521073b69b512a40cf3e149d2ac8ee6d9ebd075145fd516d603
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git3.expected b/tests/cachekey/project/sources/git3.expected
index df76f3108..42da1512f 100644
--- a/tests/cachekey/project/sources/git3.expected
+++ b/tests/cachekey/project/sources/git3.expected
@@ -1 +1 @@
-ad5dff8a422c9de7c3d02773aeed7b425d43501ee5c2d5d13064b6f1e1ed9dec
\ No newline at end of file
+45b5d40a80309002754478344228bc0347151b282173823c7919e1512fdea502
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected
index 5cb6a1973..dfb798e77 100644
--- a/tests/cachekey/project/sources/local1.expected
+++ b/tests/cachekey/project/sources/local1.expected
@@ -1 +1 @@
-402b946498422abb82133fa5998b97f93b08d66ee1f332154413886687d8e7e9
\ No newline at end of file
+8920a31ae292e892d62692702bcd43bbb08af53187f1766dcba07c9eea9ea1f0
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected
index e098d9b72..4df7d6598 100644
--- a/tests/cachekey/project/sources/local2.expected
+++ b/tests/cachekey/project/sources/local2.expected
@@ -1 +1 @@
-2eff179c8bab2441c2a6f115f5c313ece4a131c067b15e7fa23b58480341550d
\ No newline at end of file
+787bf27af094bce38eb749b20b75885a427f69cea66dd614912685d08e38745c
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch1.expected b/tests/cachekey/project/sources/patch1.expected
index c51922bf4..23c82305c 100644
--- a/tests/cachekey/project/sources/patch1.expected
+++ b/tests/cachekey/project/sources/patch1.expected
@@ -1 +1 @@
-4d2d8e8e92a20255a38d167abd93b5e6843f7b2738cdfe11ce64bc662fcaa886
\ No newline at end of file
+a821f7acacd978ba0fec07b8df3b06b5a275e20506a7b6518265214bb47bc4c5
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch2.expected b/tests/cachekey/project/sources/patch2.expected
index c9abff542..382bfb550 100644
--- a/tests/cachekey/project/sources/patch2.expected
+++ b/tests/cachekey/project/sources/patch2.expected
@@ -1 +1 @@
-21a53c232671f21cd717a4c94274e2decdba2c916dde56e030f944fe92ae785e
\ No newline at end of file
+81a1bf0b99d5888d9366fe3a69a508fa184bb884a16dd6dcc6b9c3276ec708d2
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch3.expected b/tests/cachekey/project/sources/patch3.expected
index 03c01c4ef..f3af6ae8c 100644
--- a/tests/cachekey/project/sources/patch3.expected
+++ b/tests/cachekey/project/sources/patch3.expected
@@ -1 +1 @@
-5ec3023b14bb2a44c94e205d4edc0e366d187357d6661bbc699f73e014b0630b
\ No newline at end of file
+f4c515a5ea11dd25215001c6bd3bf56f7cd431a6ea16f6d19273dc60c40d5439
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected
index fc24de1b0..e950952c8 100644
--- a/tests/cachekey/project/sources/pip1.expected
+++ b/tests/cachekey/project/sources/pip1.expected
@@ -1 +1 @@
-60d6200ba331e3cff4b3255cb218569e387c571ee57761f6b3883b1283a937a2
\ No newline at end of file
+355a16ca00ac83589f97c5544102748930b61e8ca7210f888358cec28ecebebb
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote1.expected b/tests/cachekey/project/sources/remote1.expected
index 8d0c82516..c385682e8 100644
--- a/tests/cachekey/project/sources/remote1.expected
+++ b/tests/cachekey/project/sources/remote1.expected
@@ -1 +1 @@
-46f09da7ea078bf0d630ec2e14a668f8144df5175ee1c19c9af367873047b482
\ No newline at end of file
+2edc017887e7b64a83b768d3014e37dbec723dff8694a0e74a71ebb3b51c43cb
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote2.expected b/tests/cachekey/project/sources/remote2.expected
index 8e50da27a..743a4321e 100644
--- a/tests/cachekey/project/sources/remote2.expected
+++ b/tests/cachekey/project/sources/remote2.expected
@@ -1 +1 @@
-aaa2d0c22b40d2f9b87d40ff24c37769240edba4902c50fa948e8ab6c9848f6f
\ No newline at end of file
+f94a23cd2cdc858c90f47485f546c25f478ff44b8519d21e231cd1cf50395be2
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected
index 5805323c3..e9ed6c005 100644
--- a/tests/cachekey/project/sources/tar1.expected
+++ b/tests/cachekey/project/sources/tar1.expected
@@ -1 +1 @@
-6b550e20ab7b8a11912ca14171e39c76badf7fa161a01c83d817c789b84e45c3
\ No newline at end of file
+5fadb949d4db843c325a07e75a5a9688a0a45a87f05e204f1703ab92eba3d30c
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected
index bd4c41be3..bd13d194d 100644
--- a/tests/cachekey/project/sources/tar2.expected
+++ b/tests/cachekey/project/sources/tar2.expected
@@ -1 +1 @@
-f890b611cc83036b9c52dddf4eb2a02ccac5a73ae3ddcb34586406d7deba5a11
\ No newline at end of file
+4d1c56b1064d158c257c60aad45fdbf9c7fde404fea87d31c0736da629e66d77
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip1.expected b/tests/cachekey/project/sources/zip1.expected
index 892073dc2..6358b912e 100644
--- a/tests/cachekey/project/sources/zip1.expected
+++ b/tests/cachekey/project/sources/zip1.expected
@@ -1 +1 @@
-5393d513abcc88bd1cdbf03cff65d470285a906a43cf2e192ce0770fbceb933d
\ No newline at end of file
+7950c66d1ed51d1ee0a0980897a1addae574d14c8443c6c725800a64f607566d
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip2.expected b/tests/cachekey/project/sources/zip2.expected
index 3ca0b3542..e677e163c 100644
--- a/tests/cachekey/project/sources/zip2.expected
+++ b/tests/cachekey/project/sources/zip2.expected
@@ -1 +1 @@
-a03196c4878e0a585c54c0e75cabe069068d5e37b49f07ca95f5aeb6e3b1cf5b
\ No newline at end of file
+2bc0bb553d0e3de63eb93daf95f5fc202b340c85c3fd256fb2de88ec894dd63f
\ No newline at end of file
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index 276b5286b..0f142186f 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-aeb288aa36ad3121822a5d5858a851670292828dedfa1840fd986bbbd59235a8
\ No newline at end of file
+27bbe79819a8fc3be20a33f606b0176caa7cb7d6845f37d053dd42800d6d0ef0
\ No newline at end of file
diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py
index a21538325..a49762cae 100644
--- a/tests/frontend/workspace.py
+++ b/tests/frontend/workspace.py
@@ -1142,6 +1142,9 @@ def test_external_push_pull(cli, datafiles, tmpdir_factory, guess_element):
         result.assert_success()
 
 
+# Attempting to track in an open workspace is not a sensible thing and it's not compatible with workspaces as plugin
+# sources: The new ref (if it differed from the old) would have been ignored regardless.
+# The user should be expected to simply close the workspace before tracking.
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"])
 def test_external_track(cli, datafiles, tmpdir_factory, guess_element):
@@ -1151,17 +1154,31 @@ def test_external_track(cli, datafiles, tmpdir_factory, guess_element):
     arg_elm = [element_name] if not guess_element else []
 
     # Delete the ref from the source so that we can detect if the
-    # element has been tracked
+    # element has been tracked after closing the workspace
     element_contents = _yaml.load(element_file)
+    ref1 = element_contents.get_sequence('sources').mapping_at(0).get_str('ref')
     del element_contents.get_sequence('sources').mapping_at(0)['ref']
     _yaml.roundtrip_dump(element_contents, element_file)
 
     result = cli.run(project=project, args=['-C', workspace, 'source', 'track', *arg_elm])
     result.assert_success()
 
-    # Element is tracked now
+    # Element is not tracked now
     element_contents = _yaml.load(element_file)
-    assert 'ref' in element_contents.get_sequence('sources').mapping_at(0)
+    assert 'ref' not in element_contents.get_sequence('sources').mapping_at(0)
+
+    # close the workspace
+    result = cli.run(project=project, args=['-C', workspace, 'workspace', 'close', *arg_elm])
+    result.assert_success()
+
+    # and retrack the element
+    result = cli.run(project=project, args=['source', 'track', element_name])
+    result.assert_success()
+
+    element_contents = _yaml.load(element_file)
+    ref2 = element_contents.get_sequence('sources').mapping_at(0).get_str('ref')
+    # these values should be equivalent
+    assert ref1 == ref2
 
 
 @pytest.mark.datafiles(DATA_DIR)