diff options
author | Tiago Gomes <tiago.gomes@codethink.co.uk> | 2018-07-31 12:39:17 +0100 |
---|---|---|
committer | Tiago Gomes <tiago.gomes@codethink.co.uk> | 2018-08-02 12:33:10 +0100 |
commit | ca390d0b8cb916a527d75d0aff09f4f63f4d698e (patch) | |
tree | 420584b735f9f165b64d72c2c3ef3cf9c6e8a24f | |
parent | 70e3bec83484663e264428ae2b62c21b5dc08235 (diff) | |
download | buildstream-ca390d0b8cb916a527d75d0aff09f4f63f4d698e.tar.gz |
local plugin: validate project paths
-rw-r--r-- | buildstream/plugins/sources/local.py | 12 | ||||
-rw-r--r-- | tests/artifactcache/expiry.py | 67 | ||||
-rw-r--r-- | tests/frontend/push.py | 22 | ||||
-rw-r--r-- | tests/sources/local.py | 6 | ||||
-rw-r--r-- | tests/testutils/element_generators.py | 11 |
5 files changed, 51 insertions, 67 deletions
diff --git a/buildstream/plugins/sources/local.py b/buildstream/plugins/sources/local.py index e3b019f1a..058553424 100644 --- a/buildstream/plugins/sources/local.py +++ b/buildstream/plugins/sources/local.py @@ -1,5 +1,5 @@ # -# Copyright (C) 2016 Codethink Limited +# Copyright (C) 2018 Codethink Limited # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -16,6 +16,7 @@ # # Authors: # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> +# Tiago Gomes <tiago.gomes@codethink.co.uk> """ local - stage local files and directories @@ -36,7 +37,7 @@ local - stage local files and directories """ import os -from buildstream import Source, SourceError, Consistency +from buildstream import Source, Consistency from buildstream import utils @@ -51,14 +52,11 @@ class LocalSource(Source): def configure(self, node): self.node_validate(node, ['path'] + Source.COMMON_CONFIG_KEYS) - - self.path = self.node_get_member(node, str, 'path') + self.path = self.node_get_project_path(node, 'path') self.fullpath = os.path.join(self.get_project_directory(), self.path) def preflight(self): - # Check if the configured file or directory really exists - if not os.path.exists(self.fullpath): - raise SourceError("Specified path '{}' does not exist".format(self.path)) + pass def get_unique_key(self): if self.__unique_key is None: diff --git a/tests/artifactcache/expiry.py b/tests/artifactcache/expiry.py index 4c741054b..62c066605 100644 --- a/tests/artifactcache/expiry.py +++ b/tests/artifactcache/expiry.py @@ -5,7 +5,7 @@ import pytest from buildstream import _yaml from buildstream._exceptions import ErrorDomain, LoadErrorReason -from tests.testutils import cli +from tests.testutils import cli, create_element_size DATA_DIR = os.path.join( @@ -14,32 +14,12 @@ DATA_DIR = os.path.join( ) -def create_element(name, path, dependencies, size): - os.makedirs(path, exist_ok=True) - - # Create a file to be included in this element's artifact - with open(os.path.join(path, name + '_data'), 'wb+') as f: - f.write(os.urandom(size)) - - element = { - 'kind': 'import', - 'sources': [ - { - 'kind': 'local', - 'path': os.path.join(path, name + '_data') - } - ], - 'depends': dependencies - } - _yaml.dump(element, os.path.join(path, name)) - - # Ensure that the cache successfully removes an old artifact if we do # not have enough space left. @pytest.mark.datafiles(DATA_DIR) def test_artifact_expires(cli, datafiles, tmpdir): project = os.path.join(datafiles.dirname, datafiles.basename) - element_path = os.path.join(project, 'elements') + element_path = 'elements' cache_location = os.path.join(project, 'cache', 'artifacts', 'ostree') checkout = os.path.join(project, 'checkout') @@ -52,7 +32,7 @@ def test_artifact_expires(cli, datafiles, tmpdir): # Create an element that uses almost the entire cache (an empty # ostree cache starts at about ~10KiB, so we need a bit of a # buffer) - create_element('target.bst', element_path, [], 6000000) + create_element_size('target.bst', project, element_path, [], 6000000) res = cli.run(project=project, args=['build', 'target.bst']) res.assert_success() @@ -61,7 +41,7 @@ def test_artifact_expires(cli, datafiles, tmpdir): # Our cache should now be almost full. Let's create another # artifact and see if we can cause buildstream to delete the old # one. - create_element('target2.bst', element_path, [], 6000000) + create_element_size('target2.bst', project, element_path, [], 6000000) res = cli.run(project=project, args=['build', 'target2.bst']) res.assert_success() @@ -82,7 +62,7 @@ def test_artifact_expires(cli, datafiles, tmpdir): @pytest.mark.datafiles(DATA_DIR) def test_artifact_too_large(cli, datafiles, tmpdir, size): project = os.path.join(datafiles.dirname, datafiles.basename) - element_path = os.path.join(project, 'elements') + element_path = 'elements' cli.configure({ 'cache': { @@ -91,7 +71,7 @@ def test_artifact_too_large(cli, datafiles, tmpdir, size): }) # Create an element whose artifact is too large - create_element('target.bst', element_path, [], size) + create_element_size('target.bst', project, element_path, [], size) res = cli.run(project=project, args=['build', 'target.bst']) res.assert_main_error(ErrorDomain.STREAM, None) @@ -99,7 +79,7 @@ def test_artifact_too_large(cli, datafiles, tmpdir, size): @pytest.mark.datafiles(DATA_DIR) def test_expiry_order(cli, datafiles, tmpdir): project = os.path.join(datafiles.dirname, datafiles.basename) - element_path = os.path.join(project, 'elements') + element_path = 'elements' cache_location = os.path.join(project, 'cache', 'artifacts', 'ostree') checkout = os.path.join(project, 'workspace') @@ -110,21 +90,21 @@ def test_expiry_order(cli, datafiles, tmpdir): }) # Create an artifact - create_element('dep.bst', element_path, [], 2000000) + create_element_size('dep.bst', project, element_path, [], 2000000) res = cli.run(project=project, args=['build', 'dep.bst']) res.assert_success() # Create another artifact - create_element('unrelated.bst', element_path, [], 2000000) + create_element_size('unrelated.bst', project, element_path, [], 2000000) res = cli.run(project=project, args=['build', 'unrelated.bst']) res.assert_success() # And build something else - create_element('target.bst', element_path, [], 2000000) + create_element_size('target.bst', project, element_path, [], 2000000) res = cli.run(project=project, args=['build', 'target.bst']) res.assert_success() - create_element('target2.bst', element_path, [], 2000000) + create_element_size('target2.bst', project, element_path, [], 2000000) res = cli.run(project=project, args=['build', 'target2.bst']) res.assert_success() @@ -133,7 +113,7 @@ def test_expiry_order(cli, datafiles, tmpdir): res.assert_success() # Finally, build something that will cause the cache to overflow - create_element('expire.bst', element_path, [], 2000000) + create_element_size('expire.bst', project, element_path, [], 2000000) res = cli.run(project=project, args=['build', 'expire.bst']) res.assert_success() @@ -153,7 +133,7 @@ def test_expiry_order(cli, datafiles, tmpdir): @pytest.mark.datafiles(DATA_DIR) def test_keep_dependencies(cli, datafiles, tmpdir): project = os.path.join(datafiles.dirname, datafiles.basename) - element_path = os.path.join(project, 'elements') + element_path = 'elements' cache_location = os.path.join(project, 'cache', 'artifacts', 'ostree') cli.configure({ @@ -163,12 +143,12 @@ def test_keep_dependencies(cli, datafiles, tmpdir): }) # Create a pretty big dependency - create_element('dependency.bst', element_path, [], 5000000) + create_element_size('dependency.bst', project, element_path, [], 5000000) res = cli.run(project=project, args=['build', 'dependency.bst']) res.assert_success() # Now create some other unrelated artifact - create_element('unrelated.bst', element_path, [], 4000000) + create_element_size('unrelated.bst', project, element_path, [], 4000000) res = cli.run(project=project, args=['build', 'unrelated.bst']) res.assert_success() @@ -184,7 +164,8 @@ def test_keep_dependencies(cli, datafiles, tmpdir): # duplicating artifacts (bad!) we need to make this equal in size # or smaller than half the size of its dependencies. # - create_element('target.bst', element_path, ['dependency.bst'], 2000000) + create_element_size('target.bst', project, + element_path, ['dependency.bst'], 2000000) res = cli.run(project=project, args=['build', 'target.bst']) res.assert_success() @@ -197,7 +178,7 @@ def test_keep_dependencies(cli, datafiles, tmpdir): @pytest.mark.datafiles(DATA_DIR) def test_never_delete_dependencies(cli, datafiles, tmpdir): project = os.path.join(datafiles.dirname, datafiles.basename) - element_path = os.path.join(project, 'elements') + element_path = 'elements' cli.configure({ 'cache': { @@ -206,10 +187,14 @@ def test_never_delete_dependencies(cli, datafiles, tmpdir): }) # Create a build tree - create_element('dependency.bst', element_path, [], 8000000) - create_element('related.bst', element_path, ['dependency.bst'], 8000000) - create_element('target.bst', element_path, ['related.bst'], 8000000) - create_element('target2.bst', element_path, ['target.bst'], 8000000) + create_element_size('dependency.bst', project, + element_path, [], 8000000) + create_element_size('related.bst', project, + element_path, ['dependency.bst'], 8000000) + create_element_size('target.bst', project, + element_path, ['related.bst'], 8000000) + create_element_size('target2.bst', project, + element_path, ['target.bst'], 8000000) # We try to build this pipeline, but it's too big for the # cache. Since all elements are required, the build should fail. diff --git a/tests/frontend/push.py b/tests/frontend/push.py index 471991ef7..be324ca53 100644 --- a/tests/frontend/push.py +++ b/tests/frontend/push.py @@ -202,7 +202,7 @@ def test_push_after_pull(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_artifact_expires(cli, datafiles, tmpdir): project = os.path.join(datafiles.dirname, datafiles.basename) - element_path = os.path.join(project, 'elements') + element_path = 'elements' # Create an artifact share (remote artifact cache) in the tmpdir/artifactshare # Mock a file system with 12 MB free disk space @@ -215,12 +215,12 @@ def test_artifact_expires(cli, datafiles, tmpdir): }) # Create and build an element of 5 MB - create_element_size('element1.bst', element_path, [], int(5e6)) # [] => no deps + create_element_size('element1.bst', project, element_path, [], int(5e6)) result = cli.run(project=project, args=['build', 'element1.bst']) result.assert_success() # Create and build an element of 5 MB - create_element_size('element2.bst', element_path, [], int(5e6)) # [] => no deps + create_element_size('element2.bst', project, element_path, [], int(5e6)) result = cli.run(project=project, args=['build', 'element2.bst']) result.assert_success() @@ -231,7 +231,7 @@ def test_artifact_expires(cli, datafiles, tmpdir): assert_shared(cli, share, project, 'element2.bst') # Create and build another element of 5 MB (This will exceed the free disk space available) - create_element_size('element3.bst', element_path, [], int(5e6)) + create_element_size('element3.bst', project, element_path, [], int(5e6)) result = cli.run(project=project, args=['build', 'element3.bst']) result.assert_success() @@ -250,7 +250,7 @@ def test_artifact_expires(cli, datafiles, tmpdir): @pytest.mark.datafiles(DATA_DIR) def test_artifact_too_large(cli, datafiles, tmpdir): project = os.path.join(datafiles.dirname, datafiles.basename) - element_path = os.path.join(project, 'elements') + element_path = 'elements' # Create an artifact share (remote cache) in tmpdir/artifactshare # Mock a file system with 5 MB total space @@ -263,12 +263,12 @@ def test_artifact_too_large(cli, datafiles, tmpdir): }) # Create and push a 3MB element - create_element_size('small_element.bst', element_path, [], int(3e6)) + create_element_size('small_element.bst', project, element_path, [], int(3e6)) result = cli.run(project=project, args=['build', 'small_element.bst']) result.assert_success() # Create and try to push a 6MB element. - create_element_size('large_element.bst', element_path, [], int(6e6)) + create_element_size('large_element.bst', project, element_path, [], int(6e6)) result = cli.run(project=project, args=['build', 'large_element.bst']) result.assert_success() @@ -285,7 +285,7 @@ def test_artifact_too_large(cli, datafiles, tmpdir): @pytest.mark.datafiles(DATA_DIR) def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir): project = os.path.join(datafiles.dirname, datafiles.basename) - element_path = os.path.join(project, 'elements') + element_path = 'elements' # Create an artifact share (remote cache) in tmpdir/artifactshare # Mock a file system with 12 MB free disk space @@ -298,11 +298,11 @@ def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir): }) # Create and build 2 elements, each of 5 MB. - create_element_size('element1.bst', element_path, [], int(5e6)) + create_element_size('element1.bst', project, element_path, [], int(5e6)) result = cli.run(project=project, args=['build', 'element1.bst']) result.assert_success() - create_element_size('element2.bst', element_path, [], int(5e6)) + create_element_size('element2.bst', project, element_path, [], int(5e6)) result = cli.run(project=project, args=['build', 'element2.bst']) result.assert_success() @@ -327,7 +327,7 @@ def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir): assert cli.get_element_state(project, 'element1.bst') == 'cached' # Create and build the element3 (of 5 MB) - create_element_size('element3.bst', element_path, [], int(5e6)) + create_element_size('element3.bst', project, element_path, [], int(5e6)) result = cli.run(project=project, args=['build', 'element3.bst']) result.assert_success() diff --git a/tests/sources/local.py b/tests/sources/local.py index 9dfb5f972..8d2307e27 100644 --- a/tests/sources/local.py +++ b/tests/sources/local.py @@ -1,7 +1,7 @@ import os import pytest -from buildstream._exceptions import ErrorDomain +from buildstream._exceptions import ErrorDomain, LoadErrorReason from tests.testutils import cli DATA_DIR = os.path.join( @@ -15,13 +15,13 @@ def test_missing_file(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) # Removing the local file causes preflight to fail - localfile = os.path.join(datafiles.dirname, datafiles.basename, 'file.txt') + localfile = os.path.join(project, 'file.txt') os.remove(localfile) result = cli.run(project=project, args=[ 'show', 'target.bst' ]) - result.assert_main_error(ErrorDomain.SOURCE, None) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE) @pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic')) diff --git a/tests/testutils/element_generators.py b/tests/testutils/element_generators.py index 3f6090da8..49f235c61 100644 --- a/tests/testutils/element_generators.py +++ b/tests/testutils/element_generators.py @@ -18,11 +18,12 @@ from buildstream import _yaml # Returns: # Nothing (creates a .bst file of specified size) # -def create_element_size(name, path, dependencies, size): - os.makedirs(path, exist_ok=True) +def create_element_size(name, project_dir, elements_path, dependencies, size): + full_elements_path = os.path.join(project_dir, elements_path) + os.makedirs(full_elements_path, exist_ok=True) # Create a file to be included in this element's artifact - with open(os.path.join(path, name + '_data'), 'wb+') as f: + with open(os.path.join(project_dir, name + '_data'), 'wb+') as f: f.write(os.urandom(size)) # Simplest case: We want this file (of specified size) to just @@ -32,9 +33,9 @@ def create_element_size(name, path, dependencies, size): 'sources': [ { 'kind': 'local', - 'path': os.path.join(path, name + '_data') + 'path': name + '_data' } ], 'depends': dependencies } - _yaml.dump(element, os.path.join(path, name)) + _yaml.dump(element, os.path.join(project_dir, elements_path, name)) |