diff options
19 files changed, 81 insertions, 53 deletions
diff --git a/src/buildstream/plugins/sources/local.py b/src/buildstream/plugins/sources/local.py index e28098c38..98f74c16c 100644 --- a/src/buildstream/plugins/sources/local.py +++ b/src/buildstream/plugins/sources/local.py @@ -46,6 +46,7 @@ class LocalSource(Source): # pylint: disable=attribute-defined-outside-init BST_STAGE_VIRTUAL_DIRECTORY = True + BST_NO_PRESTAGE_KEY = True def __init__(self, context, project, meta): super().__init__(context, project, meta) @@ -61,20 +62,6 @@ class LocalSource(Source): def preflight(self): pass - def get_unique_key(self): - if self.__unique_key is None: - # Get a list of tuples of the the project relative paths and fullpaths - if os.path.isdir(self.fullpath): - filelist = utils.list_relative_paths(self.fullpath) - filelist = [(relpath, os.path.join(self.fullpath, relpath)) for relpath in filelist] - else: - filelist = [(self.path, self.fullpath)] - - # Return a list of (relative filename, sha256 digest) tuples, a sorted list - # has already been returned by list_relative_paths() - self.__unique_key = [(relpath, unique_key(fullpath)) for relpath, fullpath in filelist] - return self.__unique_key - def get_consistency(self): return Consistency.CACHED @@ -110,20 +97,6 @@ class LocalSource(Source): return self.fullpath -# Create a unique key for a file -def unique_key(filename): - - # Return some hard coded things for files which - # have no content to calculate a key for - if os.path.islink(filename): - # For a symbolic link, use the link target as its unique identifier - return os.readlink(filename) - elif os.path.isdir(filename): - return "0" - - return utils.sha256sum(filename) - - # Plugin entry point def setup(): return LocalSource diff --git a/tests/cachekey/cachekey.py b/tests/cachekey/cachekey.py index acfe97947..fa93f5746 100644 --- a/tests/cachekey/cachekey.py +++ b/tests/cachekey/cachekey.py @@ -167,6 +167,9 @@ def test_cache_key(datafiles, cli): 'usr', 'bin', 'goodbye') os.unlink(goodbye_link) os.symlink('hello', goodbye_link) + # pytest-datafiles does not copy mode bits + # https://github.com/omarkohl/pytest-datafiles/issues/11 + os.chmod(goodbye_link, 0o755) result = cli.run(project=project, silent=True, args=[ 'show', diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected index 7df19dbad..e3653d845 100644 --- a/tests/cachekey/project/elements/build1.expected +++ b/tests/cachekey/project/elements/build1.expected @@ -1 +1 @@ -ba3cc6aeaef5d4c559acbd3a92e7a2512a72674c3e98aaf44256eb1c951a1a29 +87391ccead1223d376cbc5261a42a163762f895d5c035430084b9e419b004174
\ No newline at end of file diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected index b783de411..e5f7e165c 100644 --- a/tests/cachekey/project/elements/build2.expected +++ b/tests/cachekey/project/elements/build2.expected @@ -1 +1 @@ -707ca16d9d0efd925cdf7e5f2586c7ca6446ad739a0ecce22d3cb967dc557edb +55eff1fd81182b3a441d5e587d99ceff025d866bbe614260b3aa8b8356eb79aa
\ No newline at end of file diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected index e945445a5..3ceee5957 100644 --- a/tests/cachekey/project/elements/compose1.expected +++ b/tests/cachekey/project/elements/compose1.expected @@ -1 +1 @@ -67fe128c47c16737fbb0c675a866a07bea3921715953b2c191ac282b1100b747 +a5bcbca1752d8378cbd1a85ea93f4ecf7a74476fa9cd81f899c2cdc2a5f4fb43
\ No newline at end of file diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected index a355b565e..c906f88f5 100644 --- a/tests/cachekey/project/elements/compose2.expected +++ b/tests/cachekey/project/elements/compose2.expected @@ -1 +1 @@ -436d4c69b1d120315c7b073f564ae07568fd7d3ead0be3df9e90a702793ad107 +07f80eaf2c6de2e019b64ecccae1dd45696427d6a6d05680e429785fa341e318
\ No newline at end of file diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected index 4af11a238..b795a5bd3 100644 --- a/tests/cachekey/project/elements/compose3.expected +++ b/tests/cachekey/project/elements/compose3.expected @@ -1 +1 @@ -cfb87e685ad0569f92eb2ac7d0f121dd62e3eb4e0a82f559ccf402887461d7aa +1fed855571867d0b1bd7f84128ad21aab60550a7bd6b32c679b093c1d3f8fdb8
\ No newline at end of file diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected index 4ac15ba81..c9986deed 100644 --- a/tests/cachekey/project/elements/compose4.expected +++ b/tests/cachekey/project/elements/compose4.expected @@ -1 +1 @@ -dc962993b42725a3fbfcb0e38557a663b610e5033e6e2a31dcb8a2e87a5b9117 +8f1a61a35a40e968acfaf298403f8546f80d63597954a974a261d8aea1a715e7
\ No newline at end of file diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected index 758fc130b..214600317 100644 --- a/tests/cachekey/project/elements/compose5.expected +++ b/tests/cachekey/project/elements/compose5.expected @@ -1 +1 @@ -ea2ca85b60271c877f45067de8c7786bbb5317447e127fbf011cc888e47c264e +96622c2c119879119c54412437eba4d8b4e906c5d27bae1e7bffc1da3a2a6b5e
\ No newline at end of file diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected index ec3b14a74..78f4972fa 100644 --- a/tests/cachekey/project/elements/import1.expected +++ b/tests/cachekey/project/elements/import1.expected @@ -1 +1 @@ -402b946498422abb82133fa5998b97f93b08d66ee1f332154413886687d8e7e9 +7596dd8b5e930d95b0d93f8aef1fb41b4ee0ec952dccb5cd1a4364cd6d63dd41
\ No newline at end of file diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected index 007848e30..52b074d6c 100644 --- a/tests/cachekey/project/elements/import2.expected +++ b/tests/cachekey/project/elements/import2.expected @@ -1 +1 @@ -030b97f6f854c270126aa3cc7ac5d541e77e42f984b61aef63b2bce183982310 +1895fc7749e6839c26b881f1957e8778e7a459ef1ffaca8cf409d45324330362
\ No newline at end of file diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected index e114a9420..9a0a4ae66 100644 --- a/tests/cachekey/project/elements/import3.expected +++ b/tests/cachekey/project/elements/import3.expected @@ -1 +1 @@ -6c0b0c42d438312a775084341f99547149b6a269ef4a24122b6f75b343802877 +a51f965f654e70ef4630d2f6cbaeb8c9f27bc190d3c7827e0e95feb4fc83b944
\ No newline at end of file diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected index 39840727a..51e5b97e1 100644 --- a/tests/cachekey/project/elements/script1.expected +++ b/tests/cachekey/project/elements/script1.expected @@ -1 +1 @@ -fb83946f47528e7df4d322ca1848616905d6e259e6db3cea26f906093ce519e0 +00788f98d62eb5fafac1fc1bd52530fcb8f03091edf205b308c61d2b621bebe3
\ No newline at end of file diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected index ec3b14a74..78f4972fa 100644 --- a/tests/cachekey/project/sources/local1.expected +++ b/tests/cachekey/project/sources/local1.expected @@ -1 +1 @@ -402b946498422abb82133fa5998b97f93b08d66ee1f332154413886687d8e7e9 +7596dd8b5e930d95b0d93f8aef1fb41b4ee0ec952dccb5cd1a4364cd6d63dd41
\ No newline at end of file diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected index 4ad7a888f..e32afb042 100644 --- a/tests/cachekey/project/sources/local2.expected +++ b/tests/cachekey/project/sources/local2.expected @@ -1 +1 @@ -2eff179c8bab2441c2a6f115f5c313ece4a131c067b15e7fa23b58480341550d +427439145ad4a9c3b839f6db8fc81d693d0163da5f3e48f36e6f1e39fc5d816b
\ No newline at end of file diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected index b26a9e84e..17d7376dc 100644 --- a/tests/cachekey/project/target.expected +++ b/tests/cachekey/project/target.expected @@ -1 +1 @@ -aeb288aa36ad3121822a5d5858a851670292828dedfa1840fd986bbbd59235a8 +9fba84d9e7fbbb03121e4bc24fb58dd9a101c88abdb0437c82402e342d46b88f
\ No newline at end of file diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py index 5079800ab..a24446d61 100644 --- a/tests/frontend/buildcheckout.py +++ b/tests/frontend/buildcheckout.py @@ -10,6 +10,7 @@ import re import pytest from buildstream.testing import cli # pylint: disable=unused-import +from buildstream.testing import create_repo from buildstream.testing._utils.site import IS_WINDOWS from buildstream import _yaml from buildstream._exceptions import ErrorDomain, LoadErrorReason @@ -1101,9 +1102,26 @@ def test_build_junction_transitive_short_notation_with_junction(cli, tmpdir, dat @pytest.mark.datafiles(DATA_DIR) def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir): project = str(datafiles) - build_elt = 'import-bin.bst' checkout_dir = os.path.join(str(tmpdir), 'checkout') + repo = create_repo('git', str(tmpdir)) + repo.create(os.path.join(str(datafiles), "files")) + element_dir = os.path.join(str(tmpdir), 'elements') + project = str(tmpdir) + project_config = { + "name": "partial-artifact-checkout-fetch", + "element-path": "elements", + } + project_file = os.path.join(str(tmpdir), "project.conf") + _yaml.roundtrip_dump(project_config, project_file) + input_config = { + "kind": "import", + "sources": [repo.source_config()], + } + input_name = 'input.bst' + input_file = os.path.join(element_dir, input_name) + _yaml.roundtrip_dump(input_config, input_file) + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: cli.configure({'artifacts': { @@ -1111,7 +1129,9 @@ def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir): 'push': True }}) - result = cli.run(project=project, args=['build', build_elt]) + result = cli.run(project=project, args=['source', 'track', input_name]) + result.assert_success() + result = cli.run(project=project, args=['build', input_name]) result.assert_success() # A push artifact cache means we have to pull to push to them, so @@ -1122,18 +1142,18 @@ def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir): # Verify that the build-only dependency is not (complete) in the local cache result = cli.run(project=project, args=[ - 'artifact', 'checkout', build_elt, + 'artifact', 'checkout', input_name, '--directory', checkout_dir]) result.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt') # Verify that the pull method fetches relevant artifacts in order to stage result = cli.run(project=project, args=[ - 'artifact', 'checkout', '--pull', build_elt, + 'artifact', 'checkout', '--pull', input_name, '--directory', checkout_dir]) result.assert_success() # should have pulled whatever was deleted previous - assert 'import-bin.bst' in result.get_pulled_elements() + assert input_name in result.get_pulled_elements() @pytest.mark.datafiles(DATA_DIR) diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py index f978258f1..2dee10ac0 100644 --- a/tests/frontend/pull.py +++ b/tests/frontend/pull.py @@ -5,8 +5,9 @@ import os import shutil import stat import pytest -from buildstream import utils +from buildstream import utils, _yaml from buildstream.testing import cli # pylint: disable=unused-import +from buildstream.testing import create_repo from tests.testutils import create_artifact_share, generate_junction, assert_shared, assert_not_shared @@ -389,6 +390,33 @@ def test_pull_missing_blob(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_pull_missing_local_blob(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) + repo = create_repo('git', str(tmpdir)) + repo.create(os.path.join(str(datafiles), "files")) + element_dir = os.path.join(str(tmpdir), 'elements') + project = str(tmpdir) + project_config = { + "name": "pull-missing-local-blob", + "element-path": "elements", + } + project_file = os.path.join(str(tmpdir), "project.conf") + _yaml.roundtrip_dump(project_config, project_file) + input_config = { + "kind": "import", + "sources": [repo.source_config()], + } + input_name = 'input.bst' + input_file = os.path.join(element_dir, input_name) + _yaml.roundtrip_dump(input_config, input_file) + + depends_name = 'depends.bst' + depends_config = { + "kind": "stack", + "depends": [ + {"filename": input_name, "type": "build"} + ] + } + depends_file = os.path.join(element_dir, depends_name) + _yaml.roundtrip_dump(depends_config, depends_file) with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: @@ -396,9 +424,12 @@ def test_pull_missing_local_blob(cli, tmpdir, datafiles): cli.configure({ 'artifacts': {'url': share.repo, 'push': True} }) - result = cli.run(project=project, args=['build', 'import-bin.bst']) + + result = cli.run(project=project, args=['source', 'track', input_name]) + result.assert_success() + result = cli.run(project=project, args=['build', input_name]) result.assert_success() - assert cli.get_element_state(project, 'import-bin.bst') == 'cached' + assert cli.get_element_state(project, input_name) == 'cached' # Delete a file blob from the local cache. # This is a placeholder to test partial CAS handling until we support @@ -409,11 +440,11 @@ def test_pull_missing_local_blob(cli, tmpdir, datafiles): os.unlink(objpath) # Now try bst build - result = cli.run(project=project, args=['build', 'target.bst']) + result = cli.run(project=project, args=['build', depends_name]) result.assert_success() # Assert that the import-bin artifact was pulled (completing the partial artifact) - assert result.get_pulled_elements() == ['import-bin.bst'] + assert result.get_pulled_elements() == [input_name] @pytest.mark.datafiles(DATA_DIR) diff --git a/tests/sourcecache/staging.py b/tests/sourcecache/staging.py index 186a4bd9f..d7254b1ae 100644 --- a/tests/sourcecache/staging.py +++ b/tests/sourcecache/staging.py @@ -138,7 +138,8 @@ def test_staged_source_build(tmpdir, datafiles, cli): element = project.load_elements(["import-dev.bst"])[0] # check consistency of the source - assert not element._source_cached() + # local sources set BST_NO_PRESTAGE_KEY so this is cached + assert element._source_cached() res = cli.run(project=project_dir, args=['build', 'target.bst']) res.assert_success() |