local plugin: validate project paths

The autotools example had to be copied over inside of the junction
example, as referring to a path outside of the project directory is
now disallowed.

12 files changed, 106 insertions, 106 deletions

diff --git a/buildstream/plugins/sources/local.py b/buildstream/plugins/sources/local.py
index e3b019f1a..058553424 100644
--- a/buildstream/plugins/sources/local.py
+++ b/buildstream/plugins/sources/local.py
@@ -1,5 +1,5 @@
 #
-#  Copyright (C) 2016 Codethink Limited
+#  Copyright (C) 2018 Codethink Limited
 #
 #  This program is free software; you can redistribute it and/or
 #  modify it under the terms of the GNU Lesser General Public
@@ -16,6 +16,7 @@
 #
 #  Authors:
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+#        Tiago Gomes <tiago.gomes@codethink.co.uk>
 
 """
 local - stage local files and directories
@@ -36,7 +37,7 @@ local - stage local files and directories
 """
 
 import os
-from buildstream import Source, SourceError, Consistency
+from buildstream import Source, Consistency
 from buildstream import utils
 
 
@@ -51,14 +52,11 @@ class LocalSource(Source):
 
     def configure(self, node):
         self.node_validate(node, ['path'] + Source.COMMON_CONFIG_KEYS)
-
-        self.path = self.node_get_member(node, str, 'path')
+        self.path = self.node_get_project_path(node, 'path')
         self.fullpath = os.path.join(self.get_project_directory(), self.path)
 
     def preflight(self):
-        # Check if the configured file or directory really exists
-        if not os.path.exists(self.fullpath):
-            raise SourceError("Specified path '{}' does not exist".format(self.path))
+        pass
 
     def get_unique_key(self):
         if self.__unique_key is None:
diff --git a/doc/examples/junctions/autotools/elements/base.bst b/doc/examples/junctions/autotools/elements/base.bst
new file mode 100644
index 000000000..1b85a9e8c
--- /dev/null
+++ b/doc/examples/junctions/autotools/elements/base.bst
@@ -0,0 +1,5 @@
+kind: stack
+description: Base stack
+
+depends:
+- base/alpine.bst
diff --git a/doc/examples/junctions/autotools/elements/base/alpine.bst b/doc/examples/junctions/autotools/elements/base/alpine.bst
new file mode 100644
index 000000000..cf85df5bf
--- /dev/null
+++ b/doc/examples/junctions/autotools/elements/base/alpine.bst
@@ -0,0 +1,13 @@
+kind: import
+description: |
+
+    Alpine Linux base runtime
+
+sources:
+- kind: tar
+
+  # This is a post doctored, trimmed down system image
+  # of the Alpine linux distribution.
+  #
+  url: alpine:integration-tests-base.v1.x86_64.tar.xz
+  ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639
diff --git a/doc/examples/junctions/autotools/elements/hello.bst b/doc/examples/junctions/autotools/elements/hello.bst
new file mode 100644
index 000000000..510f5b975
--- /dev/null
+++ b/doc/examples/junctions/autotools/elements/hello.bst
@@ -0,0 +1,21 @@
+kind: autotools
+description: |
+
+    Hello world example from automake
+
+variables:
+
+  # The hello world example lives in the doc/amhello folder.
+  #
+  # Set the %{command-subdir} variable to that location
+  # and just have the autotools element run it's commands there.
+  #
+  command-subdir: doc/amhello
+
+sources:
+- kind: tar
+  url: gnu:automake-1.16.tar.gz
+  ref: 80da43bb5665596ee389e6d8b64b4f122ea4b92a685b1dbd813cd1f0e0c2d83f
+
+depends:
+- base.bst
diff --git a/doc/examples/junctions/autotools/project.conf b/doc/examples/junctions/autotools/project.conf
new file mode 100644
index 000000000..7ee58b589
--- /dev/null
+++ b/doc/examples/junctions/autotools/project.conf
@@ -0,0 +1,13 @@
+# Unique project name
+name: autotools
+
+# Required BuildStream format version
+format-version: 9
+
+# Subdirectory where elements are stored
+element-path: elements
+
+# Define some aliases for the tarballs we download
+aliases:
+  alpine: https://gnome7.codethink.co.uk/tarballs/
+  gnu: https://ftp.gnu.org/gnu/automake/
diff --git a/doc/examples/junctions/elements/hello-junction.bst b/doc/examples/junctions/elements/hello-junction.bst
index dda865ecf..6d01e36a1 100644
--- a/doc/examples/junctions/elements/hello-junction.bst
+++ b/doc/examples/junctions/elements/hello-junction.bst
@@ -1,8 +1,4 @@
 kind: junction
-
-# Specify the source of the BuildStream project
-# We are going to use the autotools examples distributed with BuildStream in the
-# doc/examples/autotools directory
 sources:
 - kind: local
-  path: ../autotools
+  path: autotools
diff --git a/doc/source/advanced-features/junction-elements.rst b/doc/source/advanced-features/junction-elements.rst
index 929ac1217..81fc01a05 100644
--- a/doc/source/advanced-features/junction-elements.rst
+++ b/doc/source/advanced-features/junction-elements.rst
@@ -21,8 +21,8 @@ Below is a simple example of bst file for a junction element:
 .. literalinclude:: ../../examples/junctions/elements/hello-junction.bst
     :language: yaml
 
-This element imports the autotools example project found in the BuildStream
-doc/examples/autotools subdirectory.
+This element imports the autotools example subproject found in the
+BuildStream doc/examples/junctions/autotools subdirectory.
 
 .. note::
 
diff --git a/tests/artifactcache/expiry.py b/tests/artifactcache/expiry.py
index 4c741054b..62c066605 100644
--- a/tests/artifactcache/expiry.py
+++ b/tests/artifactcache/expiry.py
@@ -5,7 +5,7 @@ import pytest
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
 
-from tests.testutils import cli
+from tests.testutils import cli, create_element_size
 
 
 DATA_DIR = os.path.join(
@@ -14,32 +14,12 @@ DATA_DIR = os.path.join(
 )
 
 
-def create_element(name, path, dependencies, size):
-    os.makedirs(path, exist_ok=True)
-
-    # Create a file to be included in this element's artifact
-    with open(os.path.join(path, name + '_data'), 'wb+') as f:
-        f.write(os.urandom(size))
-
-    element = {
-        'kind': 'import',
-        'sources': [
-            {
-                'kind': 'local',
-                'path': os.path.join(path, name + '_data')
-            }
-        ],
-        'depends': dependencies
-    }
-    _yaml.dump(element, os.path.join(path, name))
-
-
 # Ensure that the cache successfully removes an old artifact if we do
 # not have enough space left.
 @pytest.mark.datafiles(DATA_DIR)
 def test_artifact_expires(cli, datafiles, tmpdir):
     project = os.path.join(datafiles.dirname, datafiles.basename)
-    element_path = os.path.join(project, 'elements')
+    element_path = 'elements'
     cache_location = os.path.join(project, 'cache', 'artifacts', 'ostree')
     checkout = os.path.join(project, 'checkout')
 
@@ -52,7 +32,7 @@ def test_artifact_expires(cli, datafiles, tmpdir):
     # Create an element that uses almost the entire cache (an empty
     # ostree cache starts at about ~10KiB, so we need a bit of a
     # buffer)
-    create_element('target.bst', element_path, [], 6000000)
+    create_element_size('target.bst', project, element_path, [], 6000000)
     res = cli.run(project=project, args=['build', 'target.bst'])
     res.assert_success()
 
@@ -61,7 +41,7 @@ def test_artifact_expires(cli, datafiles, tmpdir):
     # Our cache should now be almost full. Let's create another
     # artifact and see if we can cause buildstream to delete the old
     # one.
-    create_element('target2.bst', element_path, [], 6000000)
+    create_element_size('target2.bst', project, element_path, [], 6000000)
     res = cli.run(project=project, args=['build', 'target2.bst'])
     res.assert_success()
 
@@ -82,7 +62,7 @@ def test_artifact_expires(cli, datafiles, tmpdir):
 @pytest.mark.datafiles(DATA_DIR)
 def test_artifact_too_large(cli, datafiles, tmpdir, size):
     project = os.path.join(datafiles.dirname, datafiles.basename)
-    element_path = os.path.join(project, 'elements')
+    element_path = 'elements'
 
     cli.configure({
         'cache': {
@@ -91,7 +71,7 @@ def test_artifact_too_large(cli, datafiles, tmpdir, size):
     })
 
     # Create an element whose artifact is too large
-    create_element('target.bst', element_path, [], size)
+    create_element_size('target.bst', project, element_path, [], size)
     res = cli.run(project=project, args=['build', 'target.bst'])
     res.assert_main_error(ErrorDomain.STREAM, None)
 
@@ -99,7 +79,7 @@ def test_artifact_too_large(cli, datafiles, tmpdir, size):
 @pytest.mark.datafiles(DATA_DIR)
 def test_expiry_order(cli, datafiles, tmpdir):
     project = os.path.join(datafiles.dirname, datafiles.basename)
-    element_path = os.path.join(project, 'elements')
+    element_path = 'elements'
     cache_location = os.path.join(project, 'cache', 'artifacts', 'ostree')
     checkout = os.path.join(project, 'workspace')
 
@@ -110,21 +90,21 @@ def test_expiry_order(cli, datafiles, tmpdir):
     })
 
     # Create an artifact
-    create_element('dep.bst', element_path, [], 2000000)
+    create_element_size('dep.bst', project, element_path, [], 2000000)
     res = cli.run(project=project, args=['build', 'dep.bst'])
     res.assert_success()
 
     # Create another artifact
-    create_element('unrelated.bst', element_path, [], 2000000)
+    create_element_size('unrelated.bst', project, element_path, [], 2000000)
     res = cli.run(project=project, args=['build', 'unrelated.bst'])
     res.assert_success()
 
     # And build something else
-    create_element('target.bst', element_path, [], 2000000)
+    create_element_size('target.bst', project, element_path, [], 2000000)
     res = cli.run(project=project, args=['build', 'target.bst'])
     res.assert_success()
 
-    create_element('target2.bst', element_path, [], 2000000)
+    create_element_size('target2.bst', project, element_path, [], 2000000)
     res = cli.run(project=project, args=['build', 'target2.bst'])
     res.assert_success()
 
@@ -133,7 +113,7 @@ def test_expiry_order(cli, datafiles, tmpdir):
     res.assert_success()
 
     # Finally, build something that will cause the cache to overflow
-    create_element('expire.bst', element_path, [], 2000000)
+    create_element_size('expire.bst', project, element_path, [], 2000000)
     res = cli.run(project=project, args=['build', 'expire.bst'])
     res.assert_success()
 
@@ -153,7 +133,7 @@ def test_expiry_order(cli, datafiles, tmpdir):
 @pytest.mark.datafiles(DATA_DIR)
 def test_keep_dependencies(cli, datafiles, tmpdir):
     project = os.path.join(datafiles.dirname, datafiles.basename)
-    element_path = os.path.join(project, 'elements')
+    element_path = 'elements'
     cache_location = os.path.join(project, 'cache', 'artifacts', 'ostree')
 
     cli.configure({
@@ -163,12 +143,12 @@ def test_keep_dependencies(cli, datafiles, tmpdir):
     })
 
     # Create a pretty big dependency
-    create_element('dependency.bst', element_path, [], 5000000)
+    create_element_size('dependency.bst', project, element_path, [], 5000000)
     res = cli.run(project=project, args=['build', 'dependency.bst'])
     res.assert_success()
 
     # Now create some other unrelated artifact
-    create_element('unrelated.bst', element_path, [], 4000000)
+    create_element_size('unrelated.bst', project, element_path, [], 4000000)
     res = cli.run(project=project, args=['build', 'unrelated.bst'])
     res.assert_success()
 
@@ -184,7 +164,8 @@ def test_keep_dependencies(cli, datafiles, tmpdir):
     # duplicating artifacts (bad!) we need to make this equal in size
     # or smaller than half the size of its dependencies.
     #
-    create_element('target.bst', element_path, ['dependency.bst'], 2000000)
+    create_element_size('target.bst', project,
+                        element_path, ['dependency.bst'], 2000000)
     res = cli.run(project=project, args=['build', 'target.bst'])
     res.assert_success()
 
@@ -197,7 +178,7 @@ def test_keep_dependencies(cli, datafiles, tmpdir):
 @pytest.mark.datafiles(DATA_DIR)
 def test_never_delete_dependencies(cli, datafiles, tmpdir):
     project = os.path.join(datafiles.dirname, datafiles.basename)
-    element_path = os.path.join(project, 'elements')
+    element_path = 'elements'
 
     cli.configure({
         'cache': {
@@ -206,10 +187,14 @@ def test_never_delete_dependencies(cli, datafiles, tmpdir):
     })
 
     # Create a build tree
-    create_element('dependency.bst', element_path, [], 8000000)
-    create_element('related.bst', element_path, ['dependency.bst'], 8000000)
-    create_element('target.bst', element_path, ['related.bst'], 8000000)
-    create_element('target2.bst', element_path, ['target.bst'], 8000000)
+    create_element_size('dependency.bst', project,
+                        element_path, [], 8000000)
+    create_element_size('related.bst', project,
+                        element_path, ['dependency.bst'], 8000000)
+    create_element_size('target.bst', project,
+                        element_path, ['related.bst'], 8000000)
+    create_element_size('target2.bst', project,
+                        element_path, ['target.bst'], 8000000)
 
     # We try to build this pipeline, but it's too big for the
     # cache. Since all elements are required, the build should fail.
diff --git a/tests/examples/junctions.py b/tests/examples/junctions.py
index 49e2ebbff..d2a653884 100644
--- a/tests/examples/junctions.py
+++ b/tests/examples/junctions.py
@@ -11,42 +11,12 @@ DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)), '..', '..', 'doc', 'examples', 'junctions'
 )
 
-JUNCTION_IMPORT_PATH = os.path.join(
-    os.path.dirname(os.path.realpath(__file__)), '..', '..', 'doc', 'examples', 'autotools'
-)
-
-
-def ammend_juntion_path_paths(tmpdir):
-    # The junction element in the examples/junctions project uses a local source type.
-    # It's "path:" must specify a relative path from the project's root directory.
-    # For the hello-junction element to function during these tests, the copy of the junctions
-    # project made in the buildstream/tmp/directory, "path:" must be ammended to be the relative
-    # path to the autotools example from the temporary test directory.
-    junction_element = os.path.join(tmpdir, "elements", "hello-junction.bst")
-    junction_element_bst = ""
-    junction_relative_path = os.path.relpath(JUNCTION_IMPORT_PATH, tmpdir)
-    with open(junction_element, 'r') as f:
-        junction_element_bst = f.read()
-    ammended_element_bst = junction_element_bst.replace("../autotools", junction_relative_path)
-    with open(junction_element, 'w') as f:
-        f.write(ammended_element_bst)
-
-
-# Check that the autotools project is where the junctions example expects and
-# contains the hello.bst element.
-@pytest.mark.datafiles(DATA_DIR)
-def test_autotools_example_is_present(datafiles):
-    autotools_path = JUNCTION_IMPORT_PATH
-    assert os.path.exists(autotools_path)
-    assert os.path.exists(os.path.join(autotools_path, "elements", "hello.bst"))
-
 
 # Test that the project builds successfully
 @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
 @pytest.mark.datafiles(DATA_DIR)
 def test_build(cli, tmpdir, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)
-    ammend_juntion_path_paths(str(tmpdir))
 
     result = cli.run(project=project, args=['build', 'callHello.bst'])
     result.assert_success()
@@ -57,7 +27,6 @@ def test_build(cli, tmpdir, datafiles):
 @pytest.mark.datafiles(DATA_DIR)
 def test_shell_call_hello(cli, tmpdir, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)
-    ammend_juntion_path_paths(str(tmpdir))
 
     result = cli.run(project=project, args=['build', 'callHello.bst'])
     result.assert_success()
@@ -73,7 +42,6 @@ def test_shell_call_hello(cli, tmpdir, datafiles):
 def test_open_cross_junction_workspace(cli, tmpdir, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)
     workspace_dir = os.path.join(str(tmpdir), "workspace_hello_junction")
-    ammend_juntion_path_paths(str(tmpdir))
 
     result = cli.run(project=project,
                      args=['workspace', 'open', 'hello-junction.bst:hello.bst', workspace_dir])
diff --git a/tests/frontend/push.py b/tests/frontend/push.py
index 471991ef7..be324ca53 100644
--- a/tests/frontend/push.py
+++ b/tests/frontend/push.py
@@ -202,7 +202,7 @@ def test_push_after_pull(cli, tmpdir, datafiles):
 @pytest.mark.datafiles(DATA_DIR)
 def test_artifact_expires(cli, datafiles, tmpdir):
     project = os.path.join(datafiles.dirname, datafiles.basename)
-    element_path = os.path.join(project, 'elements')
+    element_path = 'elements'
 
     # Create an artifact share (remote artifact cache) in the tmpdir/artifactshare
     # Mock a file system with 12 MB free disk space
@@ -215,12 +215,12 @@ def test_artifact_expires(cli, datafiles, tmpdir):
         })
 
         # Create and build an element of 5 MB
-        create_element_size('element1.bst', element_path, [], int(5e6))  # [] => no deps
+        create_element_size('element1.bst', project, element_path, [], int(5e6))
         result = cli.run(project=project, args=['build', 'element1.bst'])
         result.assert_success()
 
         # Create and build an element of 5 MB
-        create_element_size('element2.bst', element_path, [], int(5e6))  # [] => no deps
+        create_element_size('element2.bst', project, element_path, [], int(5e6))
         result = cli.run(project=project, args=['build', 'element2.bst'])
         result.assert_success()
 
@@ -231,7 +231,7 @@ def test_artifact_expires(cli, datafiles, tmpdir):
         assert_shared(cli, share, project, 'element2.bst')
 
         # Create and build another element of 5 MB (This will exceed the free disk space available)
-        create_element_size('element3.bst', element_path, [], int(5e6))
+        create_element_size('element3.bst', project, element_path, [], int(5e6))
         result = cli.run(project=project, args=['build', 'element3.bst'])
         result.assert_success()
 
@@ -250,7 +250,7 @@ def test_artifact_expires(cli, datafiles, tmpdir):
 @pytest.mark.datafiles(DATA_DIR)
 def test_artifact_too_large(cli, datafiles, tmpdir):
     project = os.path.join(datafiles.dirname, datafiles.basename)
-    element_path = os.path.join(project, 'elements')
+    element_path = 'elements'
 
     # Create an artifact share (remote cache) in tmpdir/artifactshare
     # Mock a file system with 5 MB total space
@@ -263,12 +263,12 @@ def test_artifact_too_large(cli, datafiles, tmpdir):
         })
 
         # Create and push a 3MB element
-        create_element_size('small_element.bst', element_path, [], int(3e6))
+        create_element_size('small_element.bst', project, element_path, [], int(3e6))
         result = cli.run(project=project, args=['build', 'small_element.bst'])
         result.assert_success()
 
         # Create and try to push a 6MB element.
-        create_element_size('large_element.bst', element_path, [], int(6e6))
+        create_element_size('large_element.bst', project, element_path, [], int(6e6))
         result = cli.run(project=project, args=['build', 'large_element.bst'])
         result.assert_success()
 
@@ -285,7 +285,7 @@ def test_artifact_too_large(cli, datafiles, tmpdir):
 @pytest.mark.datafiles(DATA_DIR)
 def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
     project = os.path.join(datafiles.dirname, datafiles.basename)
-    element_path = os.path.join(project, 'elements')
+    element_path = 'elements'
 
     # Create an artifact share (remote cache) in tmpdir/artifactshare
     # Mock a file system with 12 MB free disk space
@@ -298,11 +298,11 @@ def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
         })
 
         # Create and build 2 elements, each of 5 MB.
-        create_element_size('element1.bst', element_path, [], int(5e6))
+        create_element_size('element1.bst', project, element_path, [], int(5e6))
         result = cli.run(project=project, args=['build', 'element1.bst'])
         result.assert_success()
 
-        create_element_size('element2.bst', element_path, [], int(5e6))
+        create_element_size('element2.bst', project, element_path, [], int(5e6))
         result = cli.run(project=project, args=['build', 'element2.bst'])
         result.assert_success()
 
@@ -327,7 +327,7 @@ def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
         assert cli.get_element_state(project, 'element1.bst') == 'cached'
 
         # Create and build the element3 (of 5 MB)
-        create_element_size('element3.bst', element_path, [], int(5e6))
+        create_element_size('element3.bst', project, element_path, [], int(5e6))
         result = cli.run(project=project, args=['build', 'element3.bst'])
         result.assert_success()
 
diff --git a/tests/sources/local.py b/tests/sources/local.py
index 9dfb5f972..8d2307e27 100644
--- a/tests/sources/local.py
+++ b/tests/sources/local.py
@@ -1,7 +1,7 @@
 import os
 import pytest
 
-from buildstream._exceptions import ErrorDomain
+from buildstream._exceptions import ErrorDomain, LoadErrorReason
 from tests.testutils import cli
 
 DATA_DIR = os.path.join(
@@ -15,13 +15,13 @@ def test_missing_file(cli, tmpdir, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)
 
     # Removing the local file causes preflight to fail
-    localfile = os.path.join(datafiles.dirname, datafiles.basename, 'file.txt')
+    localfile = os.path.join(project, 'file.txt')
     os.remove(localfile)
 
     result = cli.run(project=project, args=[
         'show', 'target.bst'
     ])
-    result.assert_main_error(ErrorDomain.SOURCE, None)
+    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.MISSING_FILE)
 
 
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
diff --git a/tests/testutils/element_generators.py b/tests/testutils/element_generators.py
index 3f6090da8..49f235c61 100644
--- a/tests/testutils/element_generators.py
+++ b/tests/testutils/element_generators.py
@@ -18,11 +18,12 @@ from buildstream import _yaml
 # Returns:
 #  Nothing (creates a .bst file of specified size)
 #
-def create_element_size(name, path, dependencies, size):
-    os.makedirs(path, exist_ok=True)
+def create_element_size(name, project_dir, elements_path, dependencies, size):
+    full_elements_path = os.path.join(project_dir, elements_path)
+    os.makedirs(full_elements_path, exist_ok=True)
 
     # Create a file to be included in this element's artifact
-    with open(os.path.join(path, name + '_data'), 'wb+') as f:
+    with open(os.path.join(project_dir, name + '_data'), 'wb+') as f:
         f.write(os.urandom(size))
 
     # Simplest case: We want this file (of specified size) to just
@@ -32,9 +33,9 @@ def create_element_size(name, path, dependencies, size):
         'sources': [
             {
                 'kind': 'local',
-                'path': os.path.join(path, name + '_data')
+                'path': name + '_data'
             }
         ],
         'depends': dependencies
     }
-    _yaml.dump(element, os.path.join(path, name))
+    _yaml.dump(element, os.path.join(project_dir, elements_path, name))