summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/buildstream/plugins/sources/local.py29
-rw-r--r--tests/cachekey/cachekey.py3
-rw-r--r--tests/cachekey/project/elements/build1.expected2
-rw-r--r--tests/cachekey/project/elements/build2.expected2
-rw-r--r--tests/cachekey/project/elements/compose1.expected2
-rw-r--r--tests/cachekey/project/elements/compose2.expected2
-rw-r--r--tests/cachekey/project/elements/compose3.expected2
-rw-r--r--tests/cachekey/project/elements/compose4.expected2
-rw-r--r--tests/cachekey/project/elements/compose5.expected2
-rw-r--r--tests/cachekey/project/elements/import1.expected2
-rw-r--r--tests/cachekey/project/elements/import2.expected2
-rw-r--r--tests/cachekey/project/elements/import3.expected2
-rw-r--r--tests/cachekey/project/elements/script1.expected2
-rw-r--r--tests/cachekey/project/sources/local1.expected2
-rw-r--r--tests/cachekey/project/sources/local2.expected2
-rw-r--r--tests/cachekey/project/target.expected2
-rw-r--r--tests/frontend/buildcheckout.py30
-rw-r--r--tests/frontend/pull.py41
-rw-r--r--tests/sourcecache/staging.py3
19 files changed, 81 insertions, 53 deletions
diff --git a/src/buildstream/plugins/sources/local.py b/src/buildstream/plugins/sources/local.py
index e28098c38..98f74c16c 100644
--- a/src/buildstream/plugins/sources/local.py
+++ b/src/buildstream/plugins/sources/local.py
@@ -46,6 +46,7 @@ class LocalSource(Source):
# pylint: disable=attribute-defined-outside-init
BST_STAGE_VIRTUAL_DIRECTORY = True
+ BST_NO_PRESTAGE_KEY = True
def __init__(self, context, project, meta):
super().__init__(context, project, meta)
@@ -61,20 +62,6 @@ class LocalSource(Source):
def preflight(self):
pass
- def get_unique_key(self):
- if self.__unique_key is None:
- # Get a list of tuples of the the project relative paths and fullpaths
- if os.path.isdir(self.fullpath):
- filelist = utils.list_relative_paths(self.fullpath)
- filelist = [(relpath, os.path.join(self.fullpath, relpath)) for relpath in filelist]
- else:
- filelist = [(self.path, self.fullpath)]
-
- # Return a list of (relative filename, sha256 digest) tuples, a sorted list
- # has already been returned by list_relative_paths()
- self.__unique_key = [(relpath, unique_key(fullpath)) for relpath, fullpath in filelist]
- return self.__unique_key
-
def get_consistency(self):
return Consistency.CACHED
@@ -110,20 +97,6 @@ class LocalSource(Source):
return self.fullpath
-# Create a unique key for a file
-def unique_key(filename):
-
- # Return some hard coded things for files which
- # have no content to calculate a key for
- if os.path.islink(filename):
- # For a symbolic link, use the link target as its unique identifier
- return os.readlink(filename)
- elif os.path.isdir(filename):
- return "0"
-
- return utils.sha256sum(filename)
-
-
# Plugin entry point
def setup():
return LocalSource
diff --git a/tests/cachekey/cachekey.py b/tests/cachekey/cachekey.py
index acfe97947..fa93f5746 100644
--- a/tests/cachekey/cachekey.py
+++ b/tests/cachekey/cachekey.py
@@ -167,6 +167,9 @@ def test_cache_key(datafiles, cli):
'usr', 'bin', 'goodbye')
os.unlink(goodbye_link)
os.symlink('hello', goodbye_link)
+ # pytest-datafiles does not copy mode bits
+ # https://github.com/omarkohl/pytest-datafiles/issues/11
+ os.chmod(goodbye_link, 0o755)
result = cli.run(project=project, silent=True, args=[
'show',
diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected
index 7df19dbad..e3653d845 100644
--- a/tests/cachekey/project/elements/build1.expected
+++ b/tests/cachekey/project/elements/build1.expected
@@ -1 +1 @@
-ba3cc6aeaef5d4c559acbd3a92e7a2512a72674c3e98aaf44256eb1c951a1a29
+87391ccead1223d376cbc5261a42a163762f895d5c035430084b9e419b004174 \ No newline at end of file
diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected
index b783de411..e5f7e165c 100644
--- a/tests/cachekey/project/elements/build2.expected
+++ b/tests/cachekey/project/elements/build2.expected
@@ -1 +1 @@
-707ca16d9d0efd925cdf7e5f2586c7ca6446ad739a0ecce22d3cb967dc557edb
+55eff1fd81182b3a441d5e587d99ceff025d866bbe614260b3aa8b8356eb79aa \ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected
index e945445a5..3ceee5957 100644
--- a/tests/cachekey/project/elements/compose1.expected
+++ b/tests/cachekey/project/elements/compose1.expected
@@ -1 +1 @@
-67fe128c47c16737fbb0c675a866a07bea3921715953b2c191ac282b1100b747
+a5bcbca1752d8378cbd1a85ea93f4ecf7a74476fa9cd81f899c2cdc2a5f4fb43 \ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected
index a355b565e..c906f88f5 100644
--- a/tests/cachekey/project/elements/compose2.expected
+++ b/tests/cachekey/project/elements/compose2.expected
@@ -1 +1 @@
-436d4c69b1d120315c7b073f564ae07568fd7d3ead0be3df9e90a702793ad107
+07f80eaf2c6de2e019b64ecccae1dd45696427d6a6d05680e429785fa341e318 \ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected
index 4af11a238..b795a5bd3 100644
--- a/tests/cachekey/project/elements/compose3.expected
+++ b/tests/cachekey/project/elements/compose3.expected
@@ -1 +1 @@
-cfb87e685ad0569f92eb2ac7d0f121dd62e3eb4e0a82f559ccf402887461d7aa
+1fed855571867d0b1bd7f84128ad21aab60550a7bd6b32c679b093c1d3f8fdb8 \ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected
index 4ac15ba81..c9986deed 100644
--- a/tests/cachekey/project/elements/compose4.expected
+++ b/tests/cachekey/project/elements/compose4.expected
@@ -1 +1 @@
-dc962993b42725a3fbfcb0e38557a663b610e5033e6e2a31dcb8a2e87a5b9117
+8f1a61a35a40e968acfaf298403f8546f80d63597954a974a261d8aea1a715e7 \ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected
index 758fc130b..214600317 100644
--- a/tests/cachekey/project/elements/compose5.expected
+++ b/tests/cachekey/project/elements/compose5.expected
@@ -1 +1 @@
-ea2ca85b60271c877f45067de8c7786bbb5317447e127fbf011cc888e47c264e
+96622c2c119879119c54412437eba4d8b4e906c5d27bae1e7bffc1da3a2a6b5e \ No newline at end of file
diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected
index ec3b14a74..78f4972fa 100644
--- a/tests/cachekey/project/elements/import1.expected
+++ b/tests/cachekey/project/elements/import1.expected
@@ -1 +1 @@
-402b946498422abb82133fa5998b97f93b08d66ee1f332154413886687d8e7e9
+7596dd8b5e930d95b0d93f8aef1fb41b4ee0ec952dccb5cd1a4364cd6d63dd41 \ No newline at end of file
diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected
index 007848e30..52b074d6c 100644
--- a/tests/cachekey/project/elements/import2.expected
+++ b/tests/cachekey/project/elements/import2.expected
@@ -1 +1 @@
-030b97f6f854c270126aa3cc7ac5d541e77e42f984b61aef63b2bce183982310
+1895fc7749e6839c26b881f1957e8778e7a459ef1ffaca8cf409d45324330362 \ No newline at end of file
diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected
index e114a9420..9a0a4ae66 100644
--- a/tests/cachekey/project/elements/import3.expected
+++ b/tests/cachekey/project/elements/import3.expected
@@ -1 +1 @@
-6c0b0c42d438312a775084341f99547149b6a269ef4a24122b6f75b343802877
+a51f965f654e70ef4630d2f6cbaeb8c9f27bc190d3c7827e0e95feb4fc83b944 \ No newline at end of file
diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected
index 39840727a..51e5b97e1 100644
--- a/tests/cachekey/project/elements/script1.expected
+++ b/tests/cachekey/project/elements/script1.expected
@@ -1 +1 @@
-fb83946f47528e7df4d322ca1848616905d6e259e6db3cea26f906093ce519e0
+00788f98d62eb5fafac1fc1bd52530fcb8f03091edf205b308c61d2b621bebe3 \ No newline at end of file
diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected
index ec3b14a74..78f4972fa 100644
--- a/tests/cachekey/project/sources/local1.expected
+++ b/tests/cachekey/project/sources/local1.expected
@@ -1 +1 @@
-402b946498422abb82133fa5998b97f93b08d66ee1f332154413886687d8e7e9
+7596dd8b5e930d95b0d93f8aef1fb41b4ee0ec952dccb5cd1a4364cd6d63dd41 \ No newline at end of file
diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected
index 4ad7a888f..e32afb042 100644
--- a/tests/cachekey/project/sources/local2.expected
+++ b/tests/cachekey/project/sources/local2.expected
@@ -1 +1 @@
-2eff179c8bab2441c2a6f115f5c313ece4a131c067b15e7fa23b58480341550d
+427439145ad4a9c3b839f6db8fc81d693d0163da5f3e48f36e6f1e39fc5d816b \ No newline at end of file
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index b26a9e84e..17d7376dc 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-aeb288aa36ad3121822a5d5858a851670292828dedfa1840fd986bbbd59235a8
+9fba84d9e7fbbb03121e4bc24fb58dd9a101c88abdb0437c82402e342d46b88f \ No newline at end of file
diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py
index 5079800ab..a24446d61 100644
--- a/tests/frontend/buildcheckout.py
+++ b/tests/frontend/buildcheckout.py
@@ -10,6 +10,7 @@ import re
import pytest
from buildstream.testing import cli # pylint: disable=unused-import
+from buildstream.testing import create_repo
from buildstream.testing._utils.site import IS_WINDOWS
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain, LoadErrorReason
@@ -1101,9 +1102,26 @@ def test_build_junction_transitive_short_notation_with_junction(cli, tmpdir, dat
@pytest.mark.datafiles(DATA_DIR)
def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir):
project = str(datafiles)
- build_elt = 'import-bin.bst'
checkout_dir = os.path.join(str(tmpdir), 'checkout')
+ repo = create_repo('git', str(tmpdir))
+ repo.create(os.path.join(str(datafiles), "files"))
+ element_dir = os.path.join(str(tmpdir), 'elements')
+ project = str(tmpdir)
+ project_config = {
+ "name": "partial-artifact-checkout-fetch",
+ "element-path": "elements",
+ }
+ project_file = os.path.join(str(tmpdir), "project.conf")
+ _yaml.roundtrip_dump(project_config, project_file)
+ input_config = {
+ "kind": "import",
+ "sources": [repo.source_config()],
+ }
+ input_name = 'input.bst'
+ input_file = os.path.join(element_dir, input_name)
+ _yaml.roundtrip_dump(input_config, input_file)
+
with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
cli.configure({'artifacts': {
@@ -1111,7 +1129,9 @@ def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir):
'push': True
}})
- result = cli.run(project=project, args=['build', build_elt])
+ result = cli.run(project=project, args=['source', 'track', input_name])
+ result.assert_success()
+ result = cli.run(project=project, args=['build', input_name])
result.assert_success()
# A push artifact cache means we have to pull to push to them, so
@@ -1122,18 +1142,18 @@ def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir):
# Verify that the build-only dependency is not (complete) in the local cache
result = cli.run(project=project, args=[
- 'artifact', 'checkout', build_elt,
+ 'artifact', 'checkout', input_name,
'--directory', checkout_dir])
result.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt')
# Verify that the pull method fetches relevant artifacts in order to stage
result = cli.run(project=project, args=[
- 'artifact', 'checkout', '--pull', build_elt,
+ 'artifact', 'checkout', '--pull', input_name,
'--directory', checkout_dir])
result.assert_success()
# should have pulled whatever was deleted previous
- assert 'import-bin.bst' in result.get_pulled_elements()
+ assert input_name in result.get_pulled_elements()
@pytest.mark.datafiles(DATA_DIR)
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index f978258f1..2dee10ac0 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -5,8 +5,9 @@ import os
import shutil
import stat
import pytest
-from buildstream import utils
+from buildstream import utils, _yaml
from buildstream.testing import cli # pylint: disable=unused-import
+from buildstream.testing import create_repo
from tests.testutils import create_artifact_share, generate_junction, assert_shared, assert_not_shared
@@ -389,6 +390,33 @@ def test_pull_missing_blob(cli, tmpdir, datafiles):
@pytest.mark.datafiles(DATA_DIR)
def test_pull_missing_local_blob(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
+ repo = create_repo('git', str(tmpdir))
+ repo.create(os.path.join(str(datafiles), "files"))
+ element_dir = os.path.join(str(tmpdir), 'elements')
+ project = str(tmpdir)
+ project_config = {
+ "name": "pull-missing-local-blob",
+ "element-path": "elements",
+ }
+ project_file = os.path.join(str(tmpdir), "project.conf")
+ _yaml.roundtrip_dump(project_config, project_file)
+ input_config = {
+ "kind": "import",
+ "sources": [repo.source_config()],
+ }
+ input_name = 'input.bst'
+ input_file = os.path.join(element_dir, input_name)
+ _yaml.roundtrip_dump(input_config, input_file)
+
+ depends_name = 'depends.bst'
+ depends_config = {
+ "kind": "stack",
+ "depends": [
+ {"filename": input_name, "type": "build"}
+ ]
+ }
+ depends_file = os.path.join(element_dir, depends_name)
+ _yaml.roundtrip_dump(depends_config, depends_file)
with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
@@ -396,9 +424,12 @@ def test_pull_missing_local_blob(cli, tmpdir, datafiles):
cli.configure({
'artifacts': {'url': share.repo, 'push': True}
})
- result = cli.run(project=project, args=['build', 'import-bin.bst'])
+
+ result = cli.run(project=project, args=['source', 'track', input_name])
+ result.assert_success()
+ result = cli.run(project=project, args=['build', input_name])
result.assert_success()
- assert cli.get_element_state(project, 'import-bin.bst') == 'cached'
+ assert cli.get_element_state(project, input_name) == 'cached'
# Delete a file blob from the local cache.
# This is a placeholder to test partial CAS handling until we support
@@ -409,11 +440,11 @@ def test_pull_missing_local_blob(cli, tmpdir, datafiles):
os.unlink(objpath)
# Now try bst build
- result = cli.run(project=project, args=['build', 'target.bst'])
+ result = cli.run(project=project, args=['build', depends_name])
result.assert_success()
# Assert that the import-bin artifact was pulled (completing the partial artifact)
- assert result.get_pulled_elements() == ['import-bin.bst']
+ assert result.get_pulled_elements() == [input_name]
@pytest.mark.datafiles(DATA_DIR)
diff --git a/tests/sourcecache/staging.py b/tests/sourcecache/staging.py
index 186a4bd9f..d7254b1ae 100644
--- a/tests/sourcecache/staging.py
+++ b/tests/sourcecache/staging.py
@@ -138,7 +138,8 @@ def test_staged_source_build(tmpdir, datafiles, cli):
element = project.load_elements(["import-dev.bst"])[0]
# check consistency of the source
- assert not element._source_cached()
+ # local sources set BST_NO_PRESTAGE_KEY so this is cached
+ assert element._source_cached()
res = cli.run(project=project_dir, args=['build', 'target.bst'])
res.assert_success()