local.py: use extended API, set BST_NO_PRESTAGE_KEYtraveltissues/1161

tests:
* local sources are cached
* use git sources instead of local when removing objects
* update expected cache keys

19 files changed, 81 insertions, 53 deletions

diff --git a/src/buildstream/plugins/sources/local.py b/src/buildstream/plugins/sources/local.py
index e28098c38..98f74c16c 100644
--- a/src/buildstream/plugins/sources/local.py
+++ b/src/buildstream/plugins/sources/local.py
@@ -46,6 +46,7 @@ class LocalSource(Source):
     # pylint: disable=attribute-defined-outside-init
 
     BST_STAGE_VIRTUAL_DIRECTORY = True
+    BST_NO_PRESTAGE_KEY = True
 
     def __init__(self, context, project, meta):
         super().__init__(context, project, meta)
@@ -61,20 +62,6 @@ class LocalSource(Source):
     def preflight(self):
         pass
 
-    def get_unique_key(self):
-        if self.__unique_key is None:
-            # Get a list of tuples of the the project relative paths and fullpaths
-            if os.path.isdir(self.fullpath):
-                filelist = utils.list_relative_paths(self.fullpath)
-                filelist = [(relpath, os.path.join(self.fullpath, relpath)) for relpath in filelist]
-            else:
-                filelist = [(self.path, self.fullpath)]
-
-            # Return a list of (relative filename, sha256 digest) tuples, a sorted list
-            # has already been returned by list_relative_paths()
-            self.__unique_key = [(relpath, unique_key(fullpath)) for relpath, fullpath in filelist]
-        return self.__unique_key
-
     def get_consistency(self):
         return Consistency.CACHED
 
@@ -110,20 +97,6 @@ class LocalSource(Source):
         return self.fullpath
 
 
-# Create a unique key for a file
-def unique_key(filename):
-
-    # Return some hard coded things for files which
-    # have no content to calculate a key for
-    if os.path.islink(filename):
-        # For a symbolic link, use the link target as its unique identifier
-        return os.readlink(filename)
-    elif os.path.isdir(filename):
-        return "0"
-
-    return utils.sha256sum(filename)
-
-
 # Plugin entry point
 def setup():
     return LocalSource
diff --git a/tests/cachekey/cachekey.py b/tests/cachekey/cachekey.py
index acfe97947..265c8a27b 100644
--- a/tests/cachekey/cachekey.py
+++ b/tests/cachekey/cachekey.py
@@ -167,6 +167,9 @@ def test_cache_key(datafiles, cli):
                                 'usr', 'bin', 'goodbye')
     os.unlink(goodbye_link)
     os.symlink('hello', goodbye_link)
+    # pytest-datafiles does not copy mode bits
+    # https://github.com/omarkohl/pytest-datafiles/issues/11
+    os.chmod(goodbye_link,  0o755)
 
     result = cli.run(project=project, silent=True, args=[
         'show',
diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected
index 7df19dbad..e3653d845 100644
--- a/tests/cachekey/project/elements/build1.expected
+++ b/tests/cachekey/project/elements/build1.expected
@@ -1 +1 @@
-ba3cc6aeaef5d4c559acbd3a92e7a2512a72674c3e98aaf44256eb1c951a1a29
+87391ccead1223d376cbc5261a42a163762f895d5c035430084b9e419b004174
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected
index b783de411..e5f7e165c 100644
--- a/tests/cachekey/project/elements/build2.expected
+++ b/tests/cachekey/project/elements/build2.expected
@@ -1 +1 @@
-707ca16d9d0efd925cdf7e5f2586c7ca6446ad739a0ecce22d3cb967dc557edb
+55eff1fd81182b3a441d5e587d99ceff025d866bbe614260b3aa8b8356eb79aa
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected
index e945445a5..3ceee5957 100644
--- a/tests/cachekey/project/elements/compose1.expected
+++ b/tests/cachekey/project/elements/compose1.expected
@@ -1 +1 @@
-67fe128c47c16737fbb0c675a866a07bea3921715953b2c191ac282b1100b747
+a5bcbca1752d8378cbd1a85ea93f4ecf7a74476fa9cd81f899c2cdc2a5f4fb43
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected
index a355b565e..c906f88f5 100644
--- a/tests/cachekey/project/elements/compose2.expected
+++ b/tests/cachekey/project/elements/compose2.expected
@@ -1 +1 @@
-436d4c69b1d120315c7b073f564ae07568fd7d3ead0be3df9e90a702793ad107
+07f80eaf2c6de2e019b64ecccae1dd45696427d6a6d05680e429785fa341e318
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected
index 4af11a238..b795a5bd3 100644
--- a/tests/cachekey/project/elements/compose3.expected
+++ b/tests/cachekey/project/elements/compose3.expected
@@ -1 +1 @@
-cfb87e685ad0569f92eb2ac7d0f121dd62e3eb4e0a82f559ccf402887461d7aa
+1fed855571867d0b1bd7f84128ad21aab60550a7bd6b32c679b093c1d3f8fdb8
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected
index 4ac15ba81..c9986deed 100644
--- a/tests/cachekey/project/elements/compose4.expected
+++ b/tests/cachekey/project/elements/compose4.expected
@@ -1 +1 @@
-dc962993b42725a3fbfcb0e38557a663b610e5033e6e2a31dcb8a2e87a5b9117
+8f1a61a35a40e968acfaf298403f8546f80d63597954a974a261d8aea1a715e7
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected
index 758fc130b..214600317 100644
--- a/tests/cachekey/project/elements/compose5.expected
+++ b/tests/cachekey/project/elements/compose5.expected
@@ -1 +1 @@
-ea2ca85b60271c877f45067de8c7786bbb5317447e127fbf011cc888e47c264e
+96622c2c119879119c54412437eba4d8b4e906c5d27bae1e7bffc1da3a2a6b5e
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected
index ec3b14a74..78f4972fa 100644
--- a/tests/cachekey/project/elements/import1.expected
+++ b/tests/cachekey/project/elements/import1.expected
@@ -1 +1 @@
-402b946498422abb82133fa5998b97f93b08d66ee1f332154413886687d8e7e9
+7596dd8b5e930d95b0d93f8aef1fb41b4ee0ec952dccb5cd1a4364cd6d63dd41
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected
index 007848e30..52b074d6c 100644
--- a/tests/cachekey/project/elements/import2.expected
+++ b/tests/cachekey/project/elements/import2.expected
@@ -1 +1 @@
-030b97f6f854c270126aa3cc7ac5d541e77e42f984b61aef63b2bce183982310
+1895fc7749e6839c26b881f1957e8778e7a459ef1ffaca8cf409d45324330362
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected
index e114a9420..9a0a4ae66 100644
--- a/tests/cachekey/project/elements/import3.expected
+++ b/tests/cachekey/project/elements/import3.expected
@@ -1 +1 @@
-6c0b0c42d438312a775084341f99547149b6a269ef4a24122b6f75b343802877
+a51f965f654e70ef4630d2f6cbaeb8c9f27bc190d3c7827e0e95feb4fc83b944
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected
index 39840727a..51e5b97e1 100644
--- a/tests/cachekey/project/elements/script1.expected
+++ b/tests/cachekey/project/elements/script1.expected
@@ -1 +1 @@
-fb83946f47528e7df4d322ca1848616905d6e259e6db3cea26f906093ce519e0
+00788f98d62eb5fafac1fc1bd52530fcb8f03091edf205b308c61d2b621bebe3
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected
index ec3b14a74..78f4972fa 100644
--- a/tests/cachekey/project/sources/local1.expected
+++ b/tests/cachekey/project/sources/local1.expected
@@ -1 +1 @@
-402b946498422abb82133fa5998b97f93b08d66ee1f332154413886687d8e7e9
+7596dd8b5e930d95b0d93f8aef1fb41b4ee0ec952dccb5cd1a4364cd6d63dd41
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected
index 4ad7a888f..e32afb042 100644
--- a/tests/cachekey/project/sources/local2.expected
+++ b/tests/cachekey/project/sources/local2.expected
@@ -1 +1 @@
-2eff179c8bab2441c2a6f115f5c313ece4a131c067b15e7fa23b58480341550d
+427439145ad4a9c3b839f6db8fc81d693d0163da5f3e48f36e6f1e39fc5d816b
\ No newline at end of file
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index b26a9e84e..17d7376dc 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-aeb288aa36ad3121822a5d5858a851670292828dedfa1840fd986bbbd59235a8
+9fba84d9e7fbbb03121e4bc24fb58dd9a101c88abdb0437c82402e342d46b88f
\ No newline at end of file
diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py
index 5079800ab..a24446d61 100644
--- a/tests/frontend/buildcheckout.py
+++ b/tests/frontend/buildcheckout.py
@@ -10,6 +10,7 @@ import re
 import pytest
 
 from buildstream.testing import cli  # pylint: disable=unused-import
+from buildstream.testing import create_repo
 from buildstream.testing._utils.site import IS_WINDOWS
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
@@ -1101,9 +1102,26 @@ def test_build_junction_transitive_short_notation_with_junction(cli, tmpdir, dat
 @pytest.mark.datafiles(DATA_DIR)
 def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir):
     project = str(datafiles)
-    build_elt = 'import-bin.bst'
     checkout_dir = os.path.join(str(tmpdir), 'checkout')
 
+    repo = create_repo('git', str(tmpdir))
+    repo.create(os.path.join(str(datafiles), "files"))
+    element_dir = os.path.join(str(tmpdir), 'elements')
+    project = str(tmpdir)
+    project_config = {
+        "name": "partial-artifact-checkout-fetch",
+        "element-path": "elements",
+    }
+    project_file = os.path.join(str(tmpdir), "project.conf")
+    _yaml.roundtrip_dump(project_config, project_file)
+    input_config = {
+        "kind": "import",
+        "sources": [repo.source_config()],
+    }
+    input_name = 'input.bst'
+    input_file = os.path.join(element_dir, input_name)
+    _yaml.roundtrip_dump(input_config, input_file)
+
     with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
 
         cli.configure({'artifacts': {
@@ -1111,7 +1129,9 @@ def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir):
             'push': True
         }})
 
-        result = cli.run(project=project, args=['build', build_elt])
+        result = cli.run(project=project, args=['source', 'track', input_name])
+        result.assert_success()
+        result = cli.run(project=project, args=['build', input_name])
         result.assert_success()
 
         # A push artifact cache means we have to pull to push to them, so
@@ -1122,18 +1142,18 @@ def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir):
 
         # Verify that the build-only dependency is not (complete) in the local cache
         result = cli.run(project=project, args=[
-            'artifact', 'checkout', build_elt,
+            'artifact', 'checkout', input_name,
             '--directory', checkout_dir])
         result.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt')
 
         # Verify that the pull method fetches relevant artifacts in order to stage
         result = cli.run(project=project, args=[
-            'artifact', 'checkout', '--pull', build_elt,
+            'artifact', 'checkout', '--pull', input_name,
             '--directory', checkout_dir])
         result.assert_success()
 
         # should have pulled whatever was deleted previous
-        assert 'import-bin.bst' in result.get_pulled_elements()
+        assert input_name in result.get_pulled_elements()
 
 
 @pytest.mark.datafiles(DATA_DIR)
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index f978258f1..2dee10ac0 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -5,8 +5,9 @@ import os
 import shutil
 import stat
 import pytest
-from buildstream import utils
+from buildstream import utils, _yaml
 from buildstream.testing import cli  # pylint: disable=unused-import
+from buildstream.testing import create_repo
 from tests.testutils import create_artifact_share, generate_junction, assert_shared, assert_not_shared
 
 
@@ -389,6 +390,33 @@ def test_pull_missing_blob(cli, tmpdir, datafiles):
 @pytest.mark.datafiles(DATA_DIR)
 def test_pull_missing_local_blob(cli, tmpdir, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)
+    repo = create_repo('git', str(tmpdir))
+    repo.create(os.path.join(str(datafiles), "files"))
+    element_dir = os.path.join(str(tmpdir), 'elements')
+    project = str(tmpdir)
+    project_config = {
+        "name": "pull-missing-local-blob",
+        "element-path": "elements",
+    }
+    project_file = os.path.join(str(tmpdir), "project.conf")
+    _yaml.roundtrip_dump(project_config, project_file)
+    input_config = {
+        "kind": "import",
+        "sources": [repo.source_config()],
+    }
+    input_name = 'input.bst'
+    input_file = os.path.join(element_dir, input_name)
+    _yaml.roundtrip_dump(input_config, input_file)
+
+    depends_name = 'depends.bst'
+    depends_config = {
+        "kind": "stack",
+        "depends": [
+            {"filename": input_name, "type": "build"}
+        ]
+    }
+    depends_file = os.path.join(element_dir, depends_name)
+    _yaml.roundtrip_dump(depends_config, depends_file)
 
     with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
 
@@ -396,9 +424,12 @@ def test_pull_missing_local_blob(cli, tmpdir, datafiles):
         cli.configure({
             'artifacts': {'url': share.repo, 'push': True}
         })
-        result = cli.run(project=project, args=['build', 'import-bin.bst'])
+
+        result = cli.run(project=project, args=['source', 'track', input_name])
+        result.assert_success()
+        result = cli.run(project=project, args=['build', input_name])
         result.assert_success()
-        assert cli.get_element_state(project, 'import-bin.bst') == 'cached'
+        assert cli.get_element_state(project, input_name) == 'cached'
 
         # Delete a file blob from the local cache.
         # This is a placeholder to test partial CAS handling until we support
@@ -409,11 +440,11 @@ def test_pull_missing_local_blob(cli, tmpdir, datafiles):
         os.unlink(objpath)
 
         # Now try bst build
-        result = cli.run(project=project, args=['build', 'target.bst'])
+        result = cli.run(project=project, args=['build', depends_name])
         result.assert_success()
 
         # Assert that the import-bin artifact was pulled (completing the partial artifact)
-        assert result.get_pulled_elements() == ['import-bin.bst']
+        assert result.get_pulled_elements() == [input_name]
 
 
 @pytest.mark.datafiles(DATA_DIR)
diff --git a/tests/sourcecache/staging.py b/tests/sourcecache/staging.py
index 186a4bd9f..d7254b1ae 100644
--- a/tests/sourcecache/staging.py
+++ b/tests/sourcecache/staging.py
@@ -138,7 +138,8 @@ def test_staged_source_build(tmpdir, datafiles, cli):
         element = project.load_elements(["import-dev.bst"])[0]
 
         # check consistency of the source
-        assert not element._source_cached()
+        # local sources set BST_NO_PRESTAGE_KEY so this is cached
+        assert element._source_cached()
 
     res = cli.run(project=project_dir, args=['build', 'target.bst'])
     res.assert_success()