Merge branch 'tristan/organizing-tests' into 'master'

General refactor in tests directory

See merge request BuildStream/buildstream!1062

247 files changed, 500 insertions, 739 deletions

diff --git a/tests/context/__init__.py b/tests/elements/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/context/__init__.py
+++ b/tests/elements/__init__.py
diff --git a/tests/plugins/filter.py b/tests/elements/filter.py
index 31b23c124..31b23c124 100644
--- a/tests/plugins/filter.py
+++ b/tests/elements/filter.py
diff --git a/tests/plugins/filter/basic/element_plugins/dynamic.py b/tests/elements/filter/basic/element_plugins/dynamic.py
index 1208a4a4d..1208a4a4d 100644
--- a/tests/plugins/filter/basic/element_plugins/dynamic.py
+++ b/tests/elements/filter/basic/element_plugins/dynamic.py
diff --git a/tests/plugins/filter/basic/elements/deps-permitted.bst b/tests/elements/filter/basic/elements/deps-permitted.bst
index 00883b1f8..00883b1f8 100644
--- a/tests/plugins/filter/basic/elements/deps-permitted.bst
+++ b/tests/elements/filter/basic/elements/deps-permitted.bst
diff --git a/tests/plugins/filter/basic/elements/forbidden-also-rdep.bst b/tests/elements/filter/basic/elements/forbidden-also-rdep.bst
index aa996a649..aa996a649 100644
--- a/tests/plugins/filter/basic/elements/forbidden-also-rdep.bst
+++ b/tests/elements/filter/basic/elements/forbidden-also-rdep.bst
diff --git a/tests/plugins/filter/basic/elements/forbidden-multi-bdep.bst b/tests/elements/filter/basic/elements/forbidden-multi-bdep.bst
index 2d5d0a2ca..2d5d0a2ca 100644
--- a/tests/plugins/filter/basic/elements/forbidden-multi-bdep.bst
+++ b/tests/elements/filter/basic/elements/forbidden-multi-bdep.bst
diff --git a/tests/plugins/filter/basic/elements/forbidden-no-bdep.bst b/tests/elements/filter/basic/elements/forbidden-no-bdep.bst
index a7d405458..a7d405458 100644
--- a/tests/plugins/filter/basic/elements/forbidden-no-bdep.bst
+++ b/tests/elements/filter/basic/elements/forbidden-no-bdep.bst
diff --git a/tests/plugins/filter/basic/elements/forbidden-source.bst b/tests/elements/filter/basic/elements/forbidden-source.bst
index d9eb07921..d9eb07921 100644
--- a/tests/plugins/filter/basic/elements/forbidden-source.bst
+++ b/tests/elements/filter/basic/elements/forbidden-source.bst
diff --git a/tests/plugins/filter/basic/elements/input-dynamic.bst b/tests/elements/filter/basic/elements/input-dynamic.bst
index e39cefe74..e39cefe74 100644
--- a/tests/plugins/filter/basic/elements/input-dynamic.bst
+++ b/tests/elements/filter/basic/elements/input-dynamic.bst
diff --git a/tests/plugins/filter/basic/elements/input.bst b/tests/elements/filter/basic/elements/input.bst
index fb3f5d194..fb3f5d194 100644
--- a/tests/plugins/filter/basic/elements/input.bst
+++ b/tests/elements/filter/basic/elements/input.bst
diff --git a/tests/plugins/filter/basic/elements/output-dynamic-include.bst b/tests/elements/filter/basic/elements/output-dynamic-include.bst
index ea45c96ef..ea45c96ef 100644
--- a/tests/plugins/filter/basic/elements/output-dynamic-include.bst
+++ b/tests/elements/filter/basic/elements/output-dynamic-include.bst
diff --git a/tests/plugins/filter/basic/elements/output-exclude.bst b/tests/elements/filter/basic/elements/output-exclude.bst
index c1ec17a45..c1ec17a45 100644
--- a/tests/plugins/filter/basic/elements/output-exclude.bst
+++ b/tests/elements/filter/basic/elements/output-exclude.bst
diff --git a/tests/plugins/filter/basic/elements/output-include.bst b/tests/elements/filter/basic/elements/output-include.bst
index c05cbdb83..c05cbdb83 100644
--- a/tests/plugins/filter/basic/elements/output-include.bst
+++ b/tests/elements/filter/basic/elements/output-include.bst
diff --git a/tests/plugins/filter/basic/elements/output-orphans.bst b/tests/elements/filter/basic/elements/output-orphans.bst
index e682fca7a..e682fca7a 100644
--- a/tests/plugins/filter/basic/elements/output-orphans.bst
+++ b/tests/elements/filter/basic/elements/output-orphans.bst
diff --git a/tests/plugins/filter/basic/files/bar b/tests/elements/filter/basic/files/bar
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/filter/basic/files/bar
+++ b/tests/elements/filter/basic/files/bar
diff --git a/tests/plugins/filter/basic/files/baz b/tests/elements/filter/basic/files/baz
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/filter/basic/files/baz
+++ b/tests/elements/filter/basic/files/baz
diff --git a/tests/loader/junctions/inconsistent-names/files/foo b/tests/elements/filter/basic/files/foo
index e69de29bb..e69de29bb 100644
--- a/tests/loader/junctions/inconsistent-names/files/foo
+++ b/tests/elements/filter/basic/files/foo
diff --git a/tests/plugins/filter/basic/project.conf b/tests/elements/filter/basic/project.conf
index 418ed02c6..418ed02c6 100644
--- a/tests/plugins/filter/basic/project.conf
+++ b/tests/elements/filter/basic/project.conf
diff --git a/tests/format/dependencies.py b/tests/format/dependencies.py
new file mode 100644
index 000000000..a6b417a66
--- /dev/null
+++ b/tests/format/dependencies.py
@@ -0,0 +1,224 @@
+import os
+import pytest
+
+from buildstream._exceptions import ErrorDomain, LoadErrorReason
+from tests.testutils import cli
+
+DATA_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+#
+# Exercising some different ways of loading the dependencies
+#
+@pytest.mark.datafiles(DATA_DIR)
+def test_two_files(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies1')
+
+    elements = cli.get_pipeline(project, ['target.bst'])
+    assert elements == ['firstdep.bst', 'target.bst']
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_shared_dependency(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies1')
+
+    elements = cli.get_pipeline(project, ['shareddeptarget.bst'])
+    assert elements == ['firstdep.bst', 'shareddep.bst', 'shareddeptarget.bst']
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_dependency_dict(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies1')
+    elements = cli.get_pipeline(project, ['target-depdict.bst'])
+    assert elements == ['firstdep.bst', 'target-depdict.bst']
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_invalid_dependency_declaration(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies1')
+    result = cli.run(project=project, args=['show', 'invaliddep.bst'])
+    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_invalid_dependency_type(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies1')
+    result = cli.run(project=project, args=['show', 'invaliddeptype.bst'])
+    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_circular_dependency(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies1')
+    result = cli.run(project=project, args=['show', 'circulartarget.bst'])
+    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.CIRCULAR_DEPENDENCY)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_build_dependency(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies1')
+
+    elements = cli.get_pipeline(project, ['builddep.bst'], scope='run')
+    assert elements == ['builddep.bst']
+
+    elements = cli.get_pipeline(project, ['builddep.bst'], scope='build')
+    assert elements == ['firstdep.bst']
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_runtime_dependency(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies1')
+    elements = cli.get_pipeline(project, ['runtimedep.bst'], scope='build')
+
+    # FIXME: The empty line should probably never happen here when there are no results.
+    assert elements == ['']
+    elements = cli.get_pipeline(project, ['runtimedep.bst'], scope='run')
+    assert elements == ['firstdep.bst', 'runtimedep.bst']
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_all_dependency(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies1')
+
+    elements = cli.get_pipeline(project, ['alldep.bst'], scope='build')
+    assert elements == ['firstdep.bst']
+
+    elements = cli.get_pipeline(project, ['alldep.bst'], scope='run')
+    assert elements == ['firstdep.bst', 'alldep.bst']
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_list_build_dependency(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies1')
+
+    # Check that the pipeline includes the build dependency
+    deps = cli.get_pipeline(project, ['builddep-list.bst'], scope="build")
+    assert "firstdep.bst" in deps
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_list_runtime_dependency(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies1')
+
+    # Check that the pipeline includes the runtime dependency
+    deps = cli.get_pipeline(project, ['runtimedep-list.bst'], scope="run")
+    assert "firstdep.bst" in deps
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_list_dependencies_combined(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies1')
+
+    # Check that runtime deps get combined
+    rundeps = cli.get_pipeline(project, ['list-combine.bst'], scope="run")
+    assert "firstdep.bst" not in rundeps
+    assert "seconddep.bst" in rundeps
+    assert "thirddep.bst" in rundeps
+
+    # Check that build deps get combined
+    builddeps = cli.get_pipeline(project, ['list-combine.bst'], scope="build")
+    assert "firstdep.bst" in builddeps
+    assert "seconddep.bst" not in builddeps
+    assert "thirddep.bst" in builddeps
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_list_overlap(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies1')
+
+    # Check that dependencies get merged
+    rundeps = cli.get_pipeline(project, ['list-overlap.bst'], scope="run")
+    assert "firstdep.bst" in rundeps
+    builddeps = cli.get_pipeline(project, ['list-overlap.bst'], scope="build")
+    assert "firstdep.bst" in builddeps
+
+
+#
+# Testing the order of elements reported when iterating through
+# Element.dependencies() with various scopes.
+#
+@pytest.mark.datafiles(DATA_DIR)
+def test_scope_all(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies2')
+    elements = ['target.bst']
+
+    element_list = cli.get_pipeline(project, elements, scope='all')
+
+    assert(len(element_list) == 7)
+
+    assert(element_list[0] == "build-build.bst")
+    assert(element_list[1] == "run-build.bst")
+    assert(element_list[2] == "build.bst")
+    assert(element_list[3] == "dep-one.bst")
+    assert(element_list[4] == "run.bst")
+    assert(element_list[5] == "dep-two.bst")
+    assert(element_list[6] == "target.bst")
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_scope_run(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies2')
+    elements = ['target.bst']
+
+    element_list = cli.get_pipeline(project, elements, scope='run')
+
+    assert(len(element_list) == 4)
+
+    assert(element_list[0] == "dep-one.bst")
+    assert(element_list[1] == "run.bst")
+    assert(element_list[2] == "dep-two.bst")
+    assert(element_list[3] == "target.bst")
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_scope_build(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies2')
+    elements = ['target.bst']
+
+    element_list = cli.get_pipeline(project, elements, scope='build')
+
+    assert(len(element_list) == 3)
+
+    assert(element_list[0] == "dep-one.bst")
+    assert(element_list[1] == "run.bst")
+    assert(element_list[2] == "dep-two.bst")
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_scope_build_of_child(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies2')
+    elements = ['target.bst']
+
+    element_list = cli.get_pipeline(project, elements, scope='build')
+
+    # First pass, lets check dep-two
+    element = element_list[2]
+
+    # Pass two, let's look at these
+    element_list = cli.get_pipeline(project, [element], scope='build')
+
+    assert(len(element_list) == 2)
+
+    assert(element_list[0] == "run-build.bst")
+    assert(element_list[1] == "build.bst")
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_no_recurse(cli, datafiles):
+    project = os.path.join(str(datafiles), 'dependencies2')
+    elements = ['target.bst']
+
+    # We abuse the 'plan' scope here to ensure that we call
+    # element.dependencies() with recurse=False - currently, no `bst
+    # show` option does this directly.
+    element_list = cli.get_pipeline(project, elements, scope='plan')
+
+    assert(len(element_list) == 7)
+
+    assert(element_list[0] == 'build-build.bst')
+    assert(element_list[1] in ['build.bst', 'run-build.bst'])
+    assert(element_list[2] in ['build.bst', 'run-build.bst'])
+    assert(element_list[3] in ['dep-one.bst', 'run.bst', 'dep-two.bst'])
+    assert(element_list[4] in ['dep-one.bst', 'run.bst', 'dep-two.bst'])
+    assert(element_list[5] in ['dep-one.bst', 'run.bst', 'dep-two.bst'])
+    assert(element_list[6] == 'target.bst')
diff --git a/tests/loader/dependencies/elements/alldep.bst b/tests/format/dependencies1/elements/alldep.bst
index c61c7ea15..7df79c92d 100644
--- a/tests/loader/dependencies/elements/alldep.bst
+++ b/tests/format/dependencies1/elements/alldep.bst
@@ -1,5 +1,5 @@
-kind: pony
+kind: manual
 description: This element has a dependency with type 'all'
 depends:
-  - filename: elements/firstdep.bst
+  - filename: firstdep.bst
     type: all
diff --git a/tests/loader/dependencies/elements/builddep-list.bst b/tests/format/dependencies1/elements/builddep-list.bst
index 925de3afd..a0cbcaf23 100644
--- a/tests/loader/dependencies/elements/builddep-list.bst
+++ b/tests/format/dependencies1/elements/builddep-list.bst
@@ -1,4 +1,4 @@
 kind: stack
 description: This element has a build-only dependency specified via build-depends
 build-depends:
-  - elements/firstdep.bst
+  - firstdep.bst
diff --git a/tests/loader/dependencies/elements/builddep.bst b/tests/format/dependencies1/elements/builddep.bst
index 6d17f8498..ab8171e2c 100644
--- a/tests/loader/dependencies/elements/builddep.bst
+++ b/tests/format/dependencies1/elements/builddep.bst
@@ -1,5 +1,5 @@
-kind: pony
+kind: manual
 description: This element has a build-only dependency
 depends:
-  - filename: elements/firstdep.bst
+  - filename: firstdep.bst
     type: build
diff --git a/tests/loader/dependencies/elements/circular-firstdep.bst b/tests/format/dependencies1/elements/circular-firstdep.bst
index 92f6993a2..604212042 100644
--- a/tests/loader/dependencies/elements/circular-firstdep.bst
+++ b/tests/format/dependencies1/elements/circular-firstdep.bst
@@ -1,4 +1,4 @@
-kind: pony
+kind: manual
 description: Depend on another dep which depends on the target
 depends:
-- elements/circular-seconddep.bst
+- circular-seconddep.bst
diff --git a/tests/loader/dependencies/elements/circular-seconddep.bst b/tests/format/dependencies1/elements/circular-seconddep.bst
index 12a67ba75..1b944c912 100644
--- a/tests/loader/dependencies/elements/circular-seconddep.bst
+++ b/tests/format/dependencies1/elements/circular-seconddep.bst
@@ -1,4 +1,4 @@
-kind: pony
+kind: manual
 description: Depend on the target, creating a circular dependency
 depends:
-- elements/circulartarget.bst
+- circulartarget.bst
diff --git a/tests/loader/dependencies/elements/circulartarget.bst b/tests/format/dependencies1/elements/circulartarget.bst
index 14db24682..eac50469c 100644
--- a/tests/loader/dependencies/elements/circulartarget.bst
+++ b/tests/format/dependencies1/elements/circulartarget.bst
@@ -1,4 +1,4 @@
-kind: pony
+kind: stack
 description: This is a main target which introduces a circular dependency
 depends:
-- elements/circular-firstdep.bst
+- circular-firstdep.bst
diff --git a/tests/loader/dependencies/elements/firstdep.bst b/tests/format/dependencies1/elements/firstdep.bst
index 5c9c1c1e6..5c9c1c1e6 100644
--- a/tests/loader/dependencies/elements/firstdep.bst
+++ b/tests/format/dependencies1/elements/firstdep.bst
diff --git a/tests/loader/dependencies/elements/invaliddep.bst b/tests/format/dependencies1/elements/invaliddep.bst
index 65c43c60a..65c43c60a 100644
--- a/tests/loader/dependencies/elements/invaliddep.bst
+++ b/tests/format/dependencies1/elements/invaliddep.bst
diff --git a/tests/loader/dependencies/elements/invaliddeptype.bst b/tests/format/dependencies1/elements/invaliddeptype.bst
index 50ed798f0..4f4dac973 100644
--- a/tests/loader/dependencies/elements/invaliddeptype.bst
+++ b/tests/format/dependencies1/elements/invaliddeptype.bst
@@ -1,5 +1,5 @@
-kind: pony
+kind: manual
 description: This is an invalid dependency type
 depends:
-  - filename: elements/firstdep.bst
+  - filename: firstdep.bst
     type: should be build or runtime
diff --git a/tests/loader/dependencies/elements/list-combine.bst b/tests/format/dependencies1/elements/list-combine.bst
index 2010d7008..ed3452206 100644
--- a/tests/loader/dependencies/elements/list-combine.bst
+++ b/tests/format/dependencies1/elements/list-combine.bst
@@ -1,8 +1,8 @@
 kind: stack
 description: This element depends on three elements in different ways
 build-depends:
-- elements/firstdep.bst
+- firstdep.bst
 runtime-depends:
-- elements/seconddep.bst
+- seconddep.bst
 depends:
-- elements/thirddep.bst
+- thirddep.bst
diff --git a/tests/loader/dependencies/elements/list-overlap.bst b/tests/format/dependencies1/elements/list-overlap.bst
index 1e98a20a5..0ee94f11c 100644
--- a/tests/loader/dependencies/elements/list-overlap.bst
+++ b/tests/format/dependencies1/elements/list-overlap.bst
@@ -1,7 +1,7 @@
 kind: stack
 description: This element depends on two elements in different ways
 build-depends:
-- elements/firstdep.bst
+- firstdep.bst
 depends:
-- filename: elements/firstdep.bst
+- filename: firstdep.bst
   type: runtime
diff --git a/tests/loader/dependencies/elements/runtimedep-list.bst b/tests/format/dependencies1/elements/runtimedep-list.bst
index 790fa4d54..1207a492d 100644
--- a/tests/loader/dependencies/elements/runtimedep-list.bst
+++ b/tests/format/dependencies1/elements/runtimedep-list.bst
@@ -1,4 +1,4 @@
 kind: stack
 description: This element has a runtime-only dependency
 runtime-depends:
-  - elements/firstdep.bst
+  - firstdep.bst
diff --git a/tests/loader/dependencies/elements/runtimedep.bst b/tests/format/dependencies1/elements/runtimedep.bst
index a08b59ebf..6a5247117 100644
--- a/tests/loader/dependencies/elements/runtimedep.bst
+++ b/tests/format/dependencies1/elements/runtimedep.bst
@@ -1,5 +1,5 @@
-kind: pony
+kind: manual
 description: This element has a runtime-only dependency
 depends:
-  - filename: elements/firstdep.bst
+  - filename: firstdep.bst
     type: runtime
diff --git a/tests/loader/dependencies/elements/seconddep.bst b/tests/format/dependencies1/elements/seconddep.bst
index 93ded4359..93ded4359 100644
--- a/tests/loader/dependencies/elements/seconddep.bst
+++ b/tests/format/dependencies1/elements/seconddep.bst
diff --git a/tests/loader/dependencies/elements/shareddep.bst b/tests/format/dependencies1/elements/shareddep.bst
index cb98171f6..ac123e253 100644
--- a/tests/loader/dependencies/elements/shareddep.bst
+++ b/tests/format/dependencies1/elements/shareddep.bst
@@ -1,4 +1,4 @@
-kind: shareddep
+kind: stack
 description: This is the first dependency
 depends:
-- elements/firstdep.bst
+- firstdep.bst
diff --git a/tests/format/dependencies1/elements/shareddeptarget.bst b/tests/format/dependencies1/elements/shareddeptarget.bst
new file mode 100644
index 000000000..b6efb1306
--- /dev/null
+++ b/tests/format/dependencies1/elements/shareddeptarget.bst
@@ -0,0 +1,5 @@
+kind: stack
+description: This is the main target
+depends:
+- firstdep.bst
+- shareddep.bst
diff --git a/tests/loader/dependencies/elements/target-depdict.bst b/tests/format/dependencies1/elements/target-depdict.bst
index 22bc87910..6a150753c 100644
--- a/tests/loader/dependencies/elements/target-depdict.bst
+++ b/tests/format/dependencies1/elements/target-depdict.bst
@@ -1,4 +1,4 @@
-kind: pony
+kind: manual
 description: This is the main target
 depends:
-- filename: elements/firstdep.bst
+- filename: firstdep.bst
diff --git a/tests/loader/dependencies/elements/target.bst b/tests/format/dependencies1/elements/target.bst
index 05c767956..b856b0f93 100644
--- a/tests/loader/dependencies/elements/target.bst
+++ b/tests/format/dependencies1/elements/target.bst
@@ -1,4 +1,4 @@
-kind: pony
+kind: stack
 description: This is the main target
 depends:
-- elements/firstdep.bst
+- firstdep.bst
diff --git a/tests/loader/dependencies/elements/thirddep.bst b/tests/format/dependencies1/elements/thirddep.bst
index 39b58e57a..39b58e57a 100644
--- a/tests/loader/dependencies/elements/thirddep.bst
+++ b/tests/format/dependencies1/elements/thirddep.bst
diff --git a/tests/format/dependencies1/project.conf b/tests/format/dependencies1/project.conf
new file mode 100644
index 000000000..fcd60b6df
--- /dev/null
+++ b/tests/format/dependencies1/project.conf
@@ -0,0 +1,3 @@
+# Basic project
+name: test
+element-path: elements
diff --git a/tests/pipeline/load/iterate/build-build.bst b/tests/format/dependencies2/build-build.bst
index cd1f73968..cd1f73968 100644
--- a/tests/pipeline/load/iterate/build-build.bst
+++ b/tests/format/dependencies2/build-build.bst
diff --git a/tests/pipeline/load/iterate/build.bst b/tests/format/dependencies2/build.bst
index 9f3ad36eb..9f3ad36eb 100644
--- a/tests/pipeline/load/iterate/build.bst
+++ b/tests/format/dependencies2/build.bst
diff --git a/tests/pipeline/load/iterate/dep-one.bst b/tests/format/dependencies2/dep-one.bst
index a4cb637ec..a4cb637ec 100644
--- a/tests/pipeline/load/iterate/dep-one.bst
+++ b/tests/format/dependencies2/dep-one.bst
diff --git a/tests/pipeline/load/iterate/dep-two.bst b/tests/format/dependencies2/dep-two.bst
index 19ab1b325..19ab1b325 100644
--- a/tests/pipeline/load/iterate/dep-two.bst
+++ b/tests/format/dependencies2/dep-two.bst
diff --git a/tests/pipeline/load/exceptions/project.conf b/tests/format/dependencies2/project.conf
index 2027cc27a..2027cc27a 100644
--- a/tests/pipeline/load/exceptions/project.conf
+++ b/tests/format/dependencies2/project.conf
diff --git a/tests/pipeline/load/iterate/run-build.bst b/tests/format/dependencies2/run-build.bst
index cd1f73968..cd1f73968 100644
--- a/tests/pipeline/load/iterate/run-build.bst
+++ b/tests/format/dependencies2/run-build.bst
diff --git a/tests/pipeline/load/iterate/run.bst b/tests/format/dependencies2/run.bst
index cd1f73968..cd1f73968 100644
--- a/tests/pipeline/load/iterate/run.bst
+++ b/tests/format/dependencies2/run.bst
diff --git a/tests/pipeline/load/iterate/target.bst b/tests/format/dependencies2/target.bst
index 8a63c5fea..8a63c5fea 100644
--- a/tests/pipeline/load/iterate/target.bst
+++ b/tests/format/dependencies2/target.bst
diff --git a/tests/loader/junctions.py b/tests/format/junctions.py
index d97c9f702..d97c9f702 100644
--- a/tests/loader/junctions.py
+++ b/tests/format/junctions.py
diff --git a/tests/loader/junctions/bar/app.bst b/tests/format/junctions/bar/app.bst
index a1a7a0ed9..a1a7a0ed9 100644
--- a/tests/loader/junctions/bar/app.bst
+++ b/tests/format/junctions/bar/app.bst
diff --git a/tests/loader/junctions/bar/bar.txt b/tests/format/junctions/bar/bar.txt
index 5716ca598..5716ca598 100644
--- a/tests/loader/junctions/bar/bar.txt
+++ b/tests/format/junctions/bar/bar.txt
diff --git a/tests/loader/junctions/bar/base.bst b/tests/format/junctions/bar/base.bst
index 10ce559a9..10ce559a9 100644
--- a/tests/loader/junctions/bar/base.bst
+++ b/tests/format/junctions/bar/base.bst
diff --git a/tests/loader/junctions/bar/project.conf b/tests/format/junctions/bar/project.conf
index f8ed6621d..f8ed6621d 100644
--- a/tests/loader/junctions/bar/project.conf
+++ b/tests/format/junctions/bar/project.conf
diff --git a/tests/loader/junctions/bar/target.bst b/tests/format/junctions/bar/target.bst
index 70b78a3fc..70b78a3fc 100644
--- a/tests/loader/junctions/bar/target.bst
+++ b/tests/format/junctions/bar/target.bst
diff --git a/tests/loader/junctions/base/base.txt b/tests/format/junctions/base/base.txt
index a496efee8..a496efee8 100644
--- a/tests/loader/junctions/base/base.txt
+++ b/tests/format/junctions/base/base.txt
diff --git a/tests/loader/junctions/base/project.conf b/tests/format/junctions/base/project.conf
index 951ea1a34..951ea1a34 100644
--- a/tests/loader/junctions/base/project.conf
+++ b/tests/format/junctions/base/project.conf
diff --git a/tests/loader/junctions/base/target.bst b/tests/format/junctions/base/target.bst
index 2b61c518b..2b61c518b 100644
--- a/tests/loader/junctions/base/target.bst
+++ b/tests/format/junctions/base/target.bst
diff --git a/tests/loader/junctions/conflict/bar.bst b/tests/format/junctions/conflict/bar.bst
index 62eee825a..62eee825a 100644
--- a/tests/loader/junctions/conflict/bar.bst
+++ b/tests/format/junctions/conflict/bar.bst
diff --git a/tests/loader/junctions/conflict/foo.bst b/tests/format/junctions/conflict/foo.bst
index 1feb4010b..1feb4010b 100644
--- a/tests/loader/junctions/conflict/foo.bst
+++ b/tests/format/junctions/conflict/foo.bst
diff --git a/tests/loader/junctions/conflict/project.conf b/tests/format/junctions/conflict/project.conf
index 395bb8423..395bb8423 100644
--- a/tests/loader/junctions/conflict/project.conf
+++ b/tests/format/junctions/conflict/project.conf
diff --git a/tests/loader/junctions/conflict/target.bst b/tests/format/junctions/conflict/target.bst
index 5e280e542..5e280e542 100644
--- a/tests/loader/junctions/conflict/target.bst
+++ b/tests/format/junctions/conflict/target.bst
diff --git a/tests/loader/junctions/foo/app.bst b/tests/format/junctions/foo/app.bst
index e658628b0..e658628b0 100644
--- a/tests/loader/junctions/foo/app.bst
+++ b/tests/format/junctions/foo/app.bst
diff --git a/tests/loader/junctions/foo/base.bst b/tests/format/junctions/foo/base.bst
index 10ce559a9..10ce559a9 100644
--- a/tests/loader/junctions/foo/base.bst
+++ b/tests/format/junctions/foo/base.bst
diff --git a/tests/loader/junctions/foo/foo.txt b/tests/format/junctions/foo/foo.txt
index 257cc5642..257cc5642 100644
--- a/tests/loader/junctions/foo/foo.txt
+++ b/tests/format/junctions/foo/foo.txt
diff --git a/tests/loader/junctions/foo/project.conf b/tests/format/junctions/foo/project.conf
index 5a240e3ed..5a240e3ed 100644
--- a/tests/loader/junctions/foo/project.conf
+++ b/tests/format/junctions/foo/project.conf
diff --git a/tests/loader/junctions/foo/target.bst b/tests/format/junctions/foo/target.bst
index 70b78a3fc..70b78a3fc 100644
--- a/tests/loader/junctions/foo/target.bst
+++ b/tests/format/junctions/foo/target.bst
diff --git a/tests/loader/junctions/inconsistent-names/elements/junction-A.bst b/tests/format/junctions/inconsistent-names/elements/junction-A.bst
index 74079f990..74079f990 100644
--- a/tests/loader/junctions/inconsistent-names/elements/junction-A.bst
+++ b/tests/format/junctions/inconsistent-names/elements/junction-A.bst
diff --git a/tests/loader/junctions/inconsistent-names/elements/junction-B-diff-name.bst b/tests/format/junctions/inconsistent-names/elements/junction-B-diff-name.bst
index 3b33406e5..3b33406e5 100644
--- a/tests/loader/junctions/inconsistent-names/elements/junction-B-diff-name.bst
+++ b/tests/format/junctions/inconsistent-names/elements/junction-B-diff-name.bst
diff --git a/tests/loader/junctions/inconsistent-names/elements/target.bst b/tests/format/junctions/inconsistent-names/elements/target.bst
index 7eba141de..7eba141de 100644
--- a/tests/loader/junctions/inconsistent-names/elements/target.bst
+++ b/tests/format/junctions/inconsistent-names/elements/target.bst
diff --git a/tests/plugins/filter/basic/files/foo b/tests/format/junctions/inconsistent-names/files/foo
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/filter/basic/files/foo
+++ b/tests/format/junctions/inconsistent-names/files/foo
diff --git a/tests/loader/junctions/inconsistent-names/junctionA/elements/app.bst b/tests/format/junctions/inconsistent-names/junctionA/elements/app.bst
index 473aaee0b..473aaee0b 100644
--- a/tests/loader/junctions/inconsistent-names/junctionA/elements/app.bst
+++ b/tests/format/junctions/inconsistent-names/junctionA/elements/app.bst
diff --git a/tests/loader/junctions/inconsistent-names/junctionA/elements/junction-B.bst b/tests/format/junctions/inconsistent-names/junctionA/elements/junction-B.bst
index bc66d7851..bc66d7851 100644
--- a/tests/loader/junctions/inconsistent-names/junctionA/elements/junction-B.bst
+++ b/tests/format/junctions/inconsistent-names/junctionA/elements/junction-B.bst
diff --git a/tests/loader/junctions/inconsistent-names/junctionA/elements/lib.bst b/tests/format/junctions/inconsistent-names/junctionA/elements/lib.bst
index 684a64315..684a64315 100644
--- a/tests/loader/junctions/inconsistent-names/junctionA/elements/lib.bst
+++ b/tests/format/junctions/inconsistent-names/junctionA/elements/lib.bst
diff --git a/tests/loader/junctions/inconsistent-names/junctionA/files/app b/tests/format/junctions/inconsistent-names/junctionA/files/app
index e69de29bb..e69de29bb 100644
--- a/tests/loader/junctions/inconsistent-names/junctionA/files/app
+++ b/tests/format/junctions/inconsistent-names/junctionA/files/app
diff --git a/tests/loader/junctions/inconsistent-names/junctionA/files/lib b/tests/format/junctions/inconsistent-names/junctionA/files/lib
index e69de29bb..e69de29bb 100644
--- a/tests/loader/junctions/inconsistent-names/junctionA/files/lib
+++ b/tests/format/junctions/inconsistent-names/junctionA/files/lib
diff --git a/tests/loader/junctions/inconsistent-names/junctionA/junctionB/base/baseimg b/tests/format/junctions/inconsistent-names/junctionA/junctionB/base/baseimg
index e69de29bb..e69de29bb 100644
--- a/tests/loader/junctions/inconsistent-names/junctionA/junctionB/base/baseimg
+++ b/tests/format/junctions/inconsistent-names/junctionA/junctionB/base/baseimg
diff --git a/tests/loader/junctions/inconsistent-names/junctionA/junctionB/elements/base.bst b/tests/format/junctions/inconsistent-names/junctionA/junctionB/elements/base.bst
index ecdc57c79..ecdc57c79 100644
--- a/tests/loader/junctions/inconsistent-names/junctionA/junctionB/elements/base.bst
+++ b/tests/format/junctions/inconsistent-names/junctionA/junctionB/elements/base.bst
diff --git a/tests/loader/junctions/inconsistent-names/junctionA/junctionB/elements/lib2.bst b/tests/format/junctions/inconsistent-names/junctionA/junctionB/elements/lib2.bst
index 5a7c17b99..5a7c17b99 100644
--- a/tests/loader/junctions/inconsistent-names/junctionA/junctionB/elements/lib2.bst
+++ b/tests/format/junctions/inconsistent-names/junctionA/junctionB/elements/lib2.bst
diff --git a/tests/loader/junctions/inconsistent-names/junctionA/junctionB/files/lib2 b/tests/format/junctions/inconsistent-names/junctionA/junctionB/files/lib2
index e69de29bb..e69de29bb 100644
--- a/tests/loader/junctions/inconsistent-names/junctionA/junctionB/files/lib2
+++ b/tests/format/junctions/inconsistent-names/junctionA/junctionB/files/lib2
diff --git a/tests/loader/junctions/inconsistent-names/junctionA/junctionB/project.conf b/tests/format/junctions/inconsistent-names/junctionA/junctionB/project.conf
index 2e96170f8..2e96170f8 100644
--- a/tests/loader/junctions/inconsistent-names/junctionA/junctionB/project.conf
+++ b/tests/format/junctions/inconsistent-names/junctionA/junctionB/project.conf
diff --git a/tests/loader/junctions/inconsistent-names/junctionA/project.conf b/tests/format/junctions/inconsistent-names/junctionA/project.conf
index 166c4b78c..166c4b78c 100644
--- a/tests/loader/junctions/inconsistent-names/junctionA/project.conf
+++ b/tests/format/junctions/inconsistent-names/junctionA/project.conf
diff --git a/tests/loader/junctions/inconsistent-names/project.conf b/tests/format/junctions/inconsistent-names/project.conf
index 064bbc588..064bbc588 100644
--- a/tests/loader/junctions/inconsistent-names/project.conf
+++ b/tests/format/junctions/inconsistent-names/project.conf
diff --git a/tests/loader/junctions/invalid/app.bst b/tests/format/junctions/invalid/app.bst
index 9906149a6..9906149a6 100644
--- a/tests/loader/junctions/invalid/app.bst
+++ b/tests/format/junctions/invalid/app.bst
diff --git a/tests/loader/junctions/invalid/base-with-deps.bst b/tests/format/junctions/invalid/base-with-deps.bst
index a30eb3a52..a30eb3a52 100644
--- a/tests/loader/junctions/invalid/base-with-deps.bst
+++ b/tests/format/junctions/invalid/base-with-deps.bst
diff --git a/tests/loader/junctions/invalid/base.bst b/tests/format/junctions/invalid/base.bst
index 10ce559a9..10ce559a9 100644
--- a/tests/loader/junctions/invalid/base.bst
+++ b/tests/format/junctions/invalid/base.bst
diff --git a/tests/loader/junctions/invalid/foo.txt b/tests/format/junctions/invalid/foo.txt
index 257cc5642..257cc5642 100644
--- a/tests/loader/junctions/invalid/foo.txt
+++ b/tests/format/junctions/invalid/foo.txt
diff --git a/tests/loader/junctions/invalid/junction-dep.bst b/tests/format/junctions/invalid/junction-dep.bst
index 20fd30011..20fd30011 100644
--- a/tests/loader/junctions/invalid/junction-dep.bst
+++ b/tests/format/junctions/invalid/junction-dep.bst
diff --git a/tests/loader/junctions/invalid/junction-with-deps.bst b/tests/format/junctions/invalid/junction-with-deps.bst
index d2ca337dd..d2ca337dd 100644
--- a/tests/loader/junctions/invalid/junction-with-deps.bst
+++ b/tests/format/junctions/invalid/junction-with-deps.bst
diff --git a/tests/loader/junctions/invalid/missing-element.bst b/tests/format/junctions/invalid/missing-element.bst
index 4c29221af..4c29221af 100644
--- a/tests/loader/junctions/invalid/missing-element.bst
+++ b/tests/format/junctions/invalid/missing-element.bst
diff --git a/tests/loader/junctions/invalid/missing.bst b/tests/format/junctions/invalid/missing.bst
index 672e967fe..672e967fe 100644
--- a/tests/loader/junctions/invalid/missing.bst
+++ b/tests/format/junctions/invalid/missing.bst
diff --git a/tests/loader/junctions/invalid/project.conf b/tests/format/junctions/invalid/project.conf
index 38070bf71..38070bf71 100644
--- a/tests/loader/junctions/invalid/project.conf
+++ b/tests/format/junctions/invalid/project.conf
diff --git a/tests/loader/junctions/nested/foo.bst b/tests/format/junctions/nested/foo.bst
index 1feb4010b..1feb4010b 100644
--- a/tests/loader/junctions/nested/foo.bst
+++ b/tests/format/junctions/nested/foo.bst
diff --git a/tests/loader/junctions/nested/project.conf b/tests/format/junctions/nested/project.conf
index 551f51c6a..551f51c6a 100644
--- a/tests/loader/junctions/nested/project.conf
+++ b/tests/format/junctions/nested/project.conf
diff --git a/tests/loader/junctions/nested/target.bst b/tests/format/junctions/nested/target.bst
index db59499a7..db59499a7 100644
--- a/tests/loader/junctions/nested/target.bst
+++ b/tests/format/junctions/nested/target.bst
diff --git a/tests/loader/junctions/options-base/horsy.txt b/tests/format/junctions/options-base/horsy.txt
index 063dad656..063dad656 100644
--- a/tests/loader/junctions/options-base/horsy.txt
+++ b/tests/format/junctions/options-base/horsy.txt
diff --git a/tests/loader/junctions/options-base/pony.txt b/tests/format/junctions/options-base/pony.txt
index f62144808..f62144808 100644
--- a/tests/loader/junctions/options-base/pony.txt
+++ b/tests/format/junctions/options-base/pony.txt
diff --git a/tests/loader/junctions/options-base/project.conf b/tests/format/junctions/options-base/project.conf
index a98ebd5ed..a98ebd5ed 100644
--- a/tests/loader/junctions/options-base/project.conf
+++ b/tests/format/junctions/options-base/project.conf
diff --git a/tests/loader/junctions/options-base/target.bst b/tests/format/junctions/options-base/target.bst
index cda17af98..cda17af98 100644
--- a/tests/loader/junctions/options-base/target.bst
+++ b/tests/format/junctions/options-base/target.bst
diff --git a/tests/loader/junctions/options-default/base.bst b/tests/format/junctions/options-default/base.bst
index 5d42e5c31..5d42e5c31 100644
--- a/tests/loader/junctions/options-default/base.bst
+++ b/tests/format/junctions/options-default/base.bst
diff --git a/tests/loader/junctions/options-default/project.conf b/tests/format/junctions/options-default/project.conf
index b9e58a582..b9e58a582 100644
--- a/tests/loader/junctions/options-default/project.conf
+++ b/tests/format/junctions/options-default/project.conf
diff --git a/tests/loader/junctions/options-default/target.bst b/tests/format/junctions/options-default/target.bst
index 8395c0c77..8395c0c77 100644
--- a/tests/loader/junctions/options-default/target.bst
+++ b/tests/format/junctions/options-default/target.bst
diff --git a/tests/loader/junctions/options-inherit/base.bst b/tests/format/junctions/options-inherit/base.bst
index 8623d0d01..8623d0d01 100644
--- a/tests/loader/junctions/options-inherit/base.bst
+++ b/tests/format/junctions/options-inherit/base.bst
diff --git a/tests/loader/junctions/options-inherit/project.conf b/tests/format/junctions/options-inherit/project.conf
index 1e0ea4464..1e0ea4464 100644
--- a/tests/loader/junctions/options-inherit/project.conf
+++ b/tests/format/junctions/options-inherit/project.conf
diff --git a/tests/loader/junctions/options-inherit/target.bst b/tests/format/junctions/options-inherit/target.bst
index 8395c0c77..8395c0c77 100644
--- a/tests/loader/junctions/options-inherit/target.bst
+++ b/tests/format/junctions/options-inherit/target.bst
diff --git a/tests/loader/junctions/options/base.bst b/tests/format/junctions/options/base.bst
index a100e5e1e..a100e5e1e 100644
--- a/tests/loader/junctions/options/base.bst
+++ b/tests/format/junctions/options/base.bst
diff --git a/tests/loader/junctions/options/project.conf b/tests/format/junctions/options/project.conf
index de53ae176..de53ae176 100644
--- a/tests/loader/junctions/options/project.conf
+++ b/tests/format/junctions/options/project.conf
diff --git a/tests/loader/junctions/options/target.bst b/tests/format/junctions/options/target.bst
index 8395c0c77..8395c0c77 100644
--- a/tests/loader/junctions/options/target.bst
+++ b/tests/format/junctions/options/target.bst
diff --git a/tests/loader/junctions/toplevel/bar.bst b/tests/format/junctions/toplevel/bar.bst
index 62eee825a..62eee825a 100644
--- a/tests/loader/junctions/toplevel/bar.bst
+++ b/tests/format/junctions/toplevel/bar.bst
diff --git a/tests/loader/junctions/toplevel/base.bst b/tests/format/junctions/toplevel/base.bst
index 10ce559a9..10ce559a9 100644
--- a/tests/loader/junctions/toplevel/base.bst
+++ b/tests/format/junctions/toplevel/base.bst
diff --git a/tests/loader/junctions/toplevel/foo.bst b/tests/format/junctions/toplevel/foo.bst
index 1feb4010b..1feb4010b 100644
--- a/tests/loader/junctions/toplevel/foo.bst
+++ b/tests/format/junctions/toplevel/foo.bst
diff --git a/tests/loader/junctions/toplevel/project.conf b/tests/format/junctions/toplevel/project.conf
index 496c82128..496c82128 100644
--- a/tests/loader/junctions/toplevel/project.conf
+++ b/tests/format/junctions/toplevel/project.conf
diff --git a/tests/loader/junctions/toplevel/target.bst b/tests/format/junctions/toplevel/target.bst
index 5e280e542..5e280e542 100644
--- a/tests/loader/junctions/toplevel/target.bst
+++ b/tests/format/junctions/toplevel/target.bst
diff --git a/tests/format/project.py b/tests/format/project.py
index 02e8afb37..0433df99e 100644
--- a/tests/format/project.py
+++ b/tests/format/project.py
@@ -140,7 +140,7 @@ def test_local_plugin_not_directory(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-def test_project_plugin_load_allowed(cli, datafiles):
+def test_plugin_load_allowed(cli, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename, 'plugin-allowed')
     result = cli.run(project=project, silent=True, args=[
         'show', 'element.bst'])
@@ -148,7 +148,7 @@ def test_project_plugin_load_allowed(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-def test_project_plugin_load_forbidden(cli, datafiles):
+def test_plugin_load_forbidden(cli, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename, 'plugin-forbidden')
     result = cli.run(project=project, silent=True, args=[
         'show', 'element.bst'])
@@ -156,7 +156,46 @@ def test_project_plugin_load_forbidden(cli, datafiles):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-def test_project_conf_duplicate_plugins(cli, datafiles):
+@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+def test_plugin_no_load_ref(cli, datafiles, ref_storage):
+    project = os.path.join(datafiles.dirname, datafiles.basename, 'plugin-no-load-ref')
+
+    # Generate project with access to the noloadref plugin and project.refs enabled
+    #
+    config = {
+        'name': 'test',
+        'ref-storage': ref_storage,
+        'plugins': [
+            {
+                'origin': 'local',
+                'path': 'plugins',
+                'sources': {
+                    'noloadref': 0
+                }
+            }
+        ]
+    }
+    _yaml.dump(config, os.path.join(project, 'project.conf'))
+
+    result = cli.run(project=project, silent=True, args=['show', 'noloadref.bst'])
+
+    # There is no error if project.refs is not in use, otherwise we
+    # assert our graceful failure
+    if ref_storage == 'inline':
+        result.assert_success()
+    else:
+        result.assert_main_error(ErrorDomain.SOURCE, 'unsupported-load-ref')
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_plugin_preflight_error(cli, datafiles, tmpdir):
+    project = os.path.join(datafiles.dirname, datafiles.basename, 'plugin-preflight-error')
+    result = cli.run(project=project, args=['source', 'fetch', 'error.bst'])
+    result.assert_main_error(ErrorDomain.SOURCE, "the-preflight-error")
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_duplicate_plugins(cli, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename, 'duplicate-plugins')
     result = cli.run(project=project, silent=True, args=[
         'show', 'element.bst'])
diff --git a/tests/pipeline/load/noloadref/noloadref.bst b/tests/format/project/plugin-no-load-ref/noloadref.bst
index 23a52ef8f..23a52ef8f 100644
--- a/tests/pipeline/load/noloadref/noloadref.bst
+++ b/tests/format/project/plugin-no-load-ref/noloadref.bst
diff --git a/tests/pipeline/load/noloadref/plugins/noloadref.py b/tests/format/project/plugin-no-load-ref/plugins/noloadref.py
index 0cc457f07..0cc457f07 100644
--- a/tests/pipeline/load/noloadref/plugins/noloadref.py
+++ b/tests/format/project/plugin-no-load-ref/plugins/noloadref.py
diff --git a/tests/pipeline/load/noloadref/project.refs b/tests/format/project/plugin-no-load-ref/project.refs
index b06a4577d..b06a4577d 100644
--- a/tests/pipeline/load/noloadref/project.refs
+++ b/tests/format/project/plugin-no-load-ref/project.refs
diff --git a/tests/pipeline/preflight-error/error.bst b/tests/format/project/plugin-preflight-error/error.bst
index a9772770c..a9772770c 100644
--- a/tests/pipeline/preflight-error/error.bst
+++ b/tests/format/project/plugin-preflight-error/error.bst
diff --git a/tests/pipeline/preflight-error/errorplugin/__init__.py b/tests/format/project/plugin-preflight-error/errorplugin/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/pipeline/preflight-error/errorplugin/__init__.py
+++ b/tests/format/project/plugin-preflight-error/errorplugin/__init__.py
diff --git a/tests/pipeline/preflight-error/errorplugin/preflighterror.py b/tests/format/project/plugin-preflight-error/errorplugin/preflighterror.py
index 0eee463ad..0eee463ad 100644
--- a/tests/pipeline/preflight-error/errorplugin/preflighterror.py
+++ b/tests/format/project/plugin-preflight-error/errorplugin/preflighterror.py
diff --git a/tests/pipeline/preflight-error/project.conf b/tests/format/project/plugin-preflight-error/project.conf
index 20a234413..20a234413 100644
--- a/tests/pipeline/preflight-error/project.conf
+++ b/tests/format/project/plugin-preflight-error/project.conf
diff --git a/tests/format/variables.py b/tests/format/variables.py
index 46e22d0e6..be7faefc5 100644
--- a/tests/format/variables.py
+++ b/tests/format/variables.py
@@ -12,6 +12,14 @@ DATA_DIR = os.path.join(
     "variables"
 )
 
+# List of BuildStream protected variables
+PROTECTED_VARIABLES = [('project-name'), ('element-name'), ('max-jobs')]
+
+
+def print_warning(msg):
+    RED, END = "\033[91m", "\033[0m"
+    print(("\n{}{}{}").format(RED, msg, END), file=sys.stderr)
+
 
 ###############################################################
 #  Test proper loading of some default commands from plugins  #
@@ -87,6 +95,84 @@ def test_cyclic_variables(cli, datafiles):
     result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.RECURSIVE_VARIABLE)
 
 
-def print_warning(msg):
-    RED, END = "\033[91m", "\033[0m"
-    print(("\n{}{}{}").format(RED, msg, END), file=sys.stderr)
+@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'protected-vars'))
+def test_use_of_protected_var_project_conf(cli, tmpdir, datafiles, protected_var):
+    project = str(datafiles)
+    conf = {
+        'name': 'test',
+        'variables': {
+            protected_var: 'some-value'
+        }
+    }
+    _yaml.dump(conf, os.path.join(project, 'project.conf'))
+
+    element = {
+        'kind': 'import',
+        'sources': [
+            {
+                'kind': 'local',
+                'path': 'foo.txt'
+            }
+        ],
+    }
+    _yaml.dump(element, os.path.join(project, 'target.bst'))
+
+    result = cli.run(project=project, args=['build', 'target.bst'])
+    result.assert_main_error(ErrorDomain.LOAD,
+                             LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
+
+
+@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'protected-vars'))
+def test_use_of_protected_var_element_overrides(cli, tmpdir, datafiles, protected_var):
+    project = str(datafiles)
+    conf = {
+        'name': 'test',
+        'elements': {
+            'manual': {
+                'variables': {
+                    protected_var: 'some-value'
+                }
+            }
+        }
+    }
+    _yaml.dump(conf, os.path.join(project, 'project.conf'))
+
+    element = {
+        'kind': 'manual',
+        'sources': [
+            {
+                'kind': 'local',
+                'path': 'foo.txt'
+            }
+        ],
+    }
+    _yaml.dump(element, os.path.join(project, 'target.bst'))
+
+    result = cli.run(project=project, args=['build', 'target.bst'])
+    result.assert_main_error(ErrorDomain.LOAD,
+                             LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
+
+
+@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'protected-vars'))
+def test_use_of_protected_var_in_element(cli, tmpdir, datafiles, protected_var):
+    project = str(datafiles)
+    element = {
+        'kind': 'import',
+        'sources': [
+            {
+                'kind': 'local',
+                'path': 'foo.txt'
+            }
+        ],
+        'variables': {
+            protected_var: 'some-value'
+        }
+    }
+    _yaml.dump(element, os.path.join(project, 'target.bst'))
+
+    result = cli.run(project=project, args=['build', 'target.bst'])
+    result.assert_main_error(ErrorDomain.LOAD,
+                             LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
diff --git a/tests/loader/variables/simple/foo.txt b/tests/format/variables/protected-vars/foo.txt
index 257cc5642..257cc5642 100644
--- a/tests/loader/variables/simple/foo.txt
+++ b/tests/format/variables/protected-vars/foo.txt
diff --git a/tests/loader/variables/simple/project.conf b/tests/format/variables/protected-vars/project.conf
index 5a240e3ed..5a240e3ed 100644
--- a/tests/loader/variables/simple/project.conf
+++ b/tests/format/variables/protected-vars/project.conf
diff --git a/tests/completions/completions.py b/tests/frontend/completions.py
index 6c75b31e2..0e4c6ad7e 100644
--- a/tests/completions/completions.py
+++ b/tests/frontend/completions.py
@@ -3,7 +3,10 @@ import pytest
 from tests.testutils import cli
 
 # Project directory
-DATA_DIR = os.path.dirname(os.path.realpath(__file__))
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    'completions'
+)
 
 MAIN_COMMANDS = [
     'artifact ',
diff --git a/tests/completions/no-element-path/compose-all.bst b/tests/frontend/completions/no-element-path/compose-all.bst
index ba47081b3..ba47081b3 100644
--- a/tests/completions/no-element-path/compose-all.bst
+++ b/tests/frontend/completions/no-element-path/compose-all.bst
diff --git a/tests/completions/no-element-path/compose-exclude-dev.bst b/tests/frontend/completions/no-element-path/compose-exclude-dev.bst
index 75c14378c..75c14378c 100644
--- a/tests/completions/no-element-path/compose-exclude-dev.bst
+++ b/tests/frontend/completions/no-element-path/compose-exclude-dev.bst
diff --git a/tests/completions/no-element-path/compose-include-bin.bst b/tests/frontend/completions/no-element-path/compose-include-bin.bst
index 9571203c6..9571203c6 100644
--- a/tests/completions/no-element-path/compose-include-bin.bst
+++ b/tests/frontend/completions/no-element-path/compose-include-bin.bst
diff --git a/tests/completions/no-element-path/files/bin-files/usr/bin/hello b/tests/frontend/completions/no-element-path/files/bin-files/usr/bin/hello
index f534a4083..f534a4083 100755
--- a/tests/completions/no-element-path/files/bin-files/usr/bin/hello
+++ b/tests/frontend/completions/no-element-path/files/bin-files/usr/bin/hello
diff --git a/tests/completions/no-element-path/files/dev-files/usr/include/pony.h b/tests/frontend/completions/no-element-path/files/dev-files/usr/include/pony.h
index 40bd0c2e7..40bd0c2e7 100644
--- a/tests/completions/no-element-path/files/dev-files/usr/include/pony.h
+++ b/tests/frontend/completions/no-element-path/files/dev-files/usr/include/pony.h
diff --git a/tests/completions/no-element-path/import-bin.bst b/tests/frontend/completions/no-element-path/import-bin.bst
index a847c0c23..a847c0c23 100644
--- a/tests/completions/no-element-path/import-bin.bst
+++ b/tests/frontend/completions/no-element-path/import-bin.bst
diff --git a/tests/completions/no-element-path/import-dev.bst b/tests/frontend/completions/no-element-path/import-dev.bst
index 152a54667..152a54667 100644
--- a/tests/completions/no-element-path/import-dev.bst
+++ b/tests/frontend/completions/no-element-path/import-dev.bst
diff --git a/tests/completions/no-element-path/project.conf b/tests/frontend/completions/no-element-path/project.conf
index e7e35e716..e7e35e716 100644
--- a/tests/completions/no-element-path/project.conf
+++ b/tests/frontend/completions/no-element-path/project.conf
diff --git a/tests/completions/no-element-path/target.bst b/tests/frontend/completions/no-element-path/target.bst
index b9432fafa..b9432fafa 100644
--- a/tests/completions/no-element-path/target.bst
+++ b/tests/frontend/completions/no-element-path/target.bst
diff --git a/tests/completions/project/elements/compose-all.bst b/tests/frontend/completions/project/elements/compose-all.bst
index ba47081b3..ba47081b3 100644
--- a/tests/completions/project/elements/compose-all.bst
+++ b/tests/frontend/completions/project/elements/compose-all.bst
diff --git a/tests/completions/project/elements/compose-exclude-dev.bst b/tests/frontend/completions/project/elements/compose-exclude-dev.bst
index 75c14378c..75c14378c 100644
--- a/tests/completions/project/elements/compose-exclude-dev.bst
+++ b/tests/frontend/completions/project/elements/compose-exclude-dev.bst
diff --git a/tests/completions/project/elements/compose-include-bin.bst b/tests/frontend/completions/project/elements/compose-include-bin.bst
index 9571203c6..9571203c6 100644
--- a/tests/completions/project/elements/compose-include-bin.bst
+++ b/tests/frontend/completions/project/elements/compose-include-bin.bst
diff --git a/tests/completions/project/elements/import-bin.bst b/tests/frontend/completions/project/elements/import-bin.bst
index a847c0c23..a847c0c23 100644
--- a/tests/completions/project/elements/import-bin.bst
+++ b/tests/frontend/completions/project/elements/import-bin.bst
diff --git a/tests/completions/project/elements/import-dev.bst b/tests/frontend/completions/project/elements/import-dev.bst
index 152a54667..152a54667 100644
--- a/tests/completions/project/elements/import-dev.bst
+++ b/tests/frontend/completions/project/elements/import-dev.bst
diff --git a/tests/completions/project/elements/target.bst b/tests/frontend/completions/project/elements/target.bst
index b9432fafa..b9432fafa 100644
--- a/tests/completions/project/elements/target.bst
+++ b/tests/frontend/completions/project/elements/target.bst
diff --git a/tests/completions/project/files/bin-files/usr/bin/hello b/tests/frontend/completions/project/files/bin-files/usr/bin/hello
index f534a4083..f534a4083 100755
--- a/tests/completions/project/files/bin-files/usr/bin/hello
+++ b/tests/frontend/completions/project/files/bin-files/usr/bin/hello
diff --git a/tests/completions/project/files/dev-files/usr/include/pony.h b/tests/frontend/completions/project/files/dev-files/usr/include/pony.h
index 40bd0c2e7..40bd0c2e7 100644
--- a/tests/completions/project/files/dev-files/usr/include/pony.h
+++ b/tests/frontend/completions/project/files/dev-files/usr/include/pony.h
diff --git a/tests/completions/project/project.conf b/tests/frontend/completions/project/project.conf
index 854e38693..854e38693 100644
--- a/tests/completions/project/project.conf
+++ b/tests/frontend/completions/project/project.conf
diff --git a/tests/completions/sub-folders/base/unwanted.bst b/tests/frontend/completions/sub-folders/base/unwanted.bst
index bd510513d..bd510513d 100644
--- a/tests/completions/sub-folders/base/unwanted.bst
+++ b/tests/frontend/completions/sub-folders/base/unwanted.bst
diff --git a/tests/completions/sub-folders/elements/base.bst b/tests/frontend/completions/sub-folders/elements/base.bst
index 4e97e8ac0..4e97e8ac0 100644
--- a/tests/completions/sub-folders/elements/base.bst
+++ b/tests/frontend/completions/sub-folders/elements/base.bst
diff --git a/tests/completions/sub-folders/elements/base/wanted.bst b/tests/frontend/completions/sub-folders/elements/base/wanted.bst
index 57656210d..57656210d 100644
--- a/tests/completions/sub-folders/elements/base/wanted.bst
+++ b/tests/frontend/completions/sub-folders/elements/base/wanted.bst
diff --git a/tests/completions/sub-folders/elements/hello.bst b/tests/frontend/completions/sub-folders/elements/hello.bst
index 93fcaa04b..93fcaa04b 100644
--- a/tests/completions/sub-folders/elements/hello.bst
+++ b/tests/frontend/completions/sub-folders/elements/hello.bst
diff --git a/tests/completions/sub-folders/project.conf b/tests/frontend/completions/sub-folders/project.conf
index 854e38693..854e38693 100644
--- a/tests/completions/sub-folders/project.conf
+++ b/tests/frontend/completions/sub-folders/project.conf
diff --git a/tests/pipeline/load/exceptions/build.bst b/tests/frontend/exceptions/build.bst
index d0c5e2138..d0c5e2138 100644
--- a/tests/pipeline/load/exceptions/build.bst
+++ b/tests/frontend/exceptions/build.bst
diff --git a/tests/pipeline/load/exceptions/first-level-1.bst b/tests/frontend/exceptions/first-level-1.bst
index 3c870b7bc..3c870b7bc 100644
--- a/tests/pipeline/load/exceptions/first-level-1.bst
+++ b/tests/frontend/exceptions/first-level-1.bst
diff --git a/tests/pipeline/load/exceptions/first-level-2.bst b/tests/frontend/exceptions/first-level-2.bst
index fd7ac1704..fd7ac1704 100644
--- a/tests/pipeline/load/exceptions/first-level-2.bst
+++ b/tests/frontend/exceptions/first-level-2.bst
diff --git a/tests/pipeline/load/exceptions/fourth-level-1.bst b/tests/frontend/exceptions/fourth-level-1.bst
index 531378acc..531378acc 100644
--- a/tests/pipeline/load/exceptions/fourth-level-1.bst
+++ b/tests/frontend/exceptions/fourth-level-1.bst
diff --git a/tests/pipeline/load/exceptions/fourth-level-2.bst b/tests/frontend/exceptions/fourth-level-2.bst
index e0d34d70e..e0d34d70e 100644
--- a/tests/pipeline/load/exceptions/fourth-level-2.bst
+++ b/tests/frontend/exceptions/fourth-level-2.bst
diff --git a/tests/pipeline/load/exceptions/fourth-level-3.bst b/tests/frontend/exceptions/fourth-level-3.bst
index 6812e2fa1..6812e2fa1 100644
--- a/tests/pipeline/load/exceptions/fourth-level-3.bst
+++ b/tests/frontend/exceptions/fourth-level-3.bst
diff --git a/tests/pipeline/load/iterate/project.conf b/tests/frontend/exceptions/project.conf
index 2027cc27a..2027cc27a 100644
--- a/tests/pipeline/load/iterate/project.conf
+++ b/tests/frontend/exceptions/project.conf
diff --git a/tests/pipeline/load/exceptions/second-level-1.bst b/tests/frontend/exceptions/second-level-1.bst
index 3183e670b..3183e670b 100644
--- a/tests/pipeline/load/exceptions/second-level-1.bst
+++ b/tests/frontend/exceptions/second-level-1.bst
diff --git a/tests/pipeline/load/exceptions/third-level-1.bst b/tests/frontend/exceptions/third-level-1.bst
index 960419d68..960419d68 100644
--- a/tests/pipeline/load/exceptions/third-level-1.bst
+++ b/tests/frontend/exceptions/third-level-1.bst
diff --git a/tests/pipeline/load/exceptions/third-level-2.bst b/tests/frontend/exceptions/third-level-2.bst
index 067e6f95a..067e6f95a 100644
--- a/tests/pipeline/load/exceptions/third-level-2.bst
+++ b/tests/frontend/exceptions/third-level-2.bst
diff --git a/tests/pipeline/load/exceptions/third-level-3.bst b/tests/frontend/exceptions/third-level-3.bst
index 56fbe5586..56fbe5586 100644
--- a/tests/pipeline/load/exceptions/third-level-3.bst
+++ b/tests/frontend/exceptions/third-level-3.bst
diff --git a/tests/pipeline/load/exceptions/unrelated-1.bst b/tests/frontend/exceptions/unrelated-1.bst
index 10de373ca..10de373ca 100644
--- a/tests/pipeline/load/exceptions/unrelated-1.bst
+++ b/tests/frontend/exceptions/unrelated-1.bst
diff --git a/tests/pipeline/load/exceptions/unrelated-2.bst b/tests/frontend/exceptions/unrelated-2.bst
index f90b755a5..f90b755a5 100644
--- a/tests/pipeline/load/exceptions/unrelated-2.bst
+++ b/tests/frontend/exceptions/unrelated-2.bst
diff --git a/tests/frontend/show.py b/tests/frontend/show.py
index 57f04e46d..9c32dd664 100644
--- a/tests/frontend/show.py
+++ b/tests/frontend/show.py
@@ -12,11 +12,10 @@ from . import configure_project
 # Project directory
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
-    "project",
 )
 
 
-@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
 @pytest.mark.parametrize("target,format,expected", [
     ('import-bin.bst', '%{name}', 'import-bin.bst'),
     ('import-bin.bst', '%{state}', 'buildable'),
@@ -49,14 +48,14 @@ def test_show_invalid_element_path(cli, datafiles):
     result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
 
 
-@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
 @pytest.mark.parametrize("target,except_,expected", [
     ('target.bst', 'import-bin.bst', ['import-dev.bst', 'compose-all.bst', 'target.bst']),
     ('target.bst', 'import-dev.bst', ['import-bin.bst', 'compose-all.bst', 'target.bst']),
     ('target.bst', 'compose-all.bst', ['import-bin.bst', 'target.bst']),
     ('compose-all.bst', 'import-bin.bst', ['import-dev.bst', 'compose-all.bst'])
 ])
-def test_show_except(cli, datafiles, target, except_, expected):
+def test_show_except_simple(cli, datafiles, target, except_, expected):
     project = os.path.join(datafiles.dirname, datafiles.basename)
     result = cli.run(project=project, silent=True, args=[
         'show',
@@ -73,10 +72,88 @@ def test_show_except(cli, datafiles, target, except_, expected):
                              .format(expected, results))
 
 
+# This test checks various constructions of a pipeline
+# with one or more targets and 0 or more exception elements,
+# each data set provides the targets, exceptions and expected
+# result list.
+#
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'exceptions'))
+@pytest.mark.parametrize("targets,exceptions,expected", [
+
+    # Test without exceptions, lets just see the whole list here
+    (['build.bst'], None, [
+        'fourth-level-1.bst',
+        'third-level-1.bst',
+        'fourth-level-2.bst',
+        'third-level-2.bst',
+        'fourth-level-3.bst',
+        'third-level-3.bst',
+        'second-level-1.bst',
+        'first-level-1.bst',
+        'first-level-2.bst',
+        'build.bst',
+    ]),
+
+    # Test one target and excepting a part of the pipeline, this
+    # removes forth-level-1 and third-level-1
+    (['build.bst'], ['third-level-1.bst'], [
+        'fourth-level-2.bst',
+        'third-level-2.bst',
+        'fourth-level-3.bst',
+        'third-level-3.bst',
+        'second-level-1.bst',
+        'first-level-1.bst',
+        'first-level-2.bst',
+        'build.bst',
+    ]),
+
+    # Test one target and excepting a part of the pipeline, check that
+    # excepted dependencies remain in the pipeline if depended on from
+    # outside of the except element
+    (['build.bst'], ['second-level-1.bst'], [
+        'fourth-level-2.bst',
+        'third-level-2.bst',  # first-level-2 depends on this, so not excepted
+        'first-level-1.bst',
+        'first-level-2.bst',
+        'build.bst',
+    ]),
+
+    # The same as the above test, but excluding the toplevel build.bst,
+    # instead only select the two toplevel dependencies as targets
+    (['first-level-1.bst', 'first-level-2.bst'], ['second-level-1.bst'], [
+        'fourth-level-2.bst',
+        'third-level-2.bst',  # first-level-2 depends on this, so not excepted
+        'first-level-1.bst',
+        'first-level-2.bst',
+    ]),
+
+    # Test one target and excepting an element outisde the pipeline
+    (['build.bst'], ['unrelated-1.bst'], [
+        'fourth-level-2.bst',
+        'third-level-2.bst',  # first-level-2 depends on this, so not excepted
+        'first-level-1.bst',
+        'first-level-2.bst',
+        'build.bst',
+    ]),
+
+    # Test one target and excepting two elements
+    (['build.bst'], ['unrelated-1.bst', 'unrelated-2.bst'], [
+        'first-level-1.bst',
+        'build.bst',
+    ]),
+])
+def test_show_except(cli, datafiles, targets, exceptions, expected):
+    basedir = os.path.join(datafiles.dirname, datafiles.basename)
+    results = cli.get_pipeline(basedir, targets, except_=exceptions, scope='all')
+    if results != expected:
+        raise AssertionError("Expected elements:\n{}\nInstead received elements:\n{}"
+                             .format(expected, results))
+
+
 ###############################################################
 #                   Testing multiple targets                  #
 ###############################################################
-@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
 def test_parallel_order(cli, tmpdir, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)
     elements = ['multiple_targets/order/0.bst',
@@ -105,7 +182,7 @@ def test_parallel_order(cli, tmpdir, datafiles):
     assert names in orderings, "We got: {}".format(", ".join(names))
 
 
-@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
 def test_target_is_dependency(cli, tmpdir, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)
     elements = ['multiple_targets/dependency/zebry.bst',
@@ -123,7 +200,7 @@ def test_target_is_dependency(cli, tmpdir, datafiles):
     assert names == ['pony.bst', 'horsey.bst', 'zebry.bst']
 
 
-@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
 @pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
 @pytest.mark.parametrize("element_name", ['junction-dep.bst', 'junction.bst:import-etc.bst'])
 def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage, element_name):
@@ -175,7 +252,7 @@ def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage, element_name):
     result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_FETCH_NEEDED)
 
 
-@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
 @pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
 @pytest.mark.parametrize("element_name", ['junction-dep.bst', 'junction.bst:import-etc.bst'])
 def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage, element_name):
@@ -211,7 +288,7 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage, element_name
     result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT)
 
 
-@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project'))
 @pytest.mark.parametrize("element_name", ['junction-dep.bst', 'junction.bst:import-etc.bst'])
 def test_fetched_junction(cli, tmpdir, datafiles, element_name):
     project = os.path.join(datafiles.dirname, datafiles.basename)
diff --git a/tests/plugins/__init__.py b/tests/internals/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/__init__.py
+++ b/tests/internals/__init__.py
diff --git a/tests/context/context.py b/tests/internals/context.py
index 35428105b..5a4b37ac5 100644
--- a/tests/context/context.py
+++ b/tests/internals/context.py
@@ -6,7 +6,7 @@ from buildstream._exceptions import LoadError, LoadErrorReason
 
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
-    'data',
+    'context',
 )
 
 
diff --git a/tests/context/data/malformed.yaml b/tests/internals/context/malformed.yaml
index 88d1960b4..88d1960b4 100644
--- a/tests/context/data/malformed.yaml
+++ b/tests/internals/context/malformed.yaml
diff --git a/tests/context/data/notdict.yaml b/tests/internals/context/notdict.yaml
index e15deca98..e15deca98 100644
--- a/tests/context/data/notdict.yaml
+++ b/tests/internals/context/notdict.yaml
diff --git a/tests/context/data/userconf.yaml b/tests/internals/context/userconf.yaml
index e8e72d5ce..e8e72d5ce 100644
--- a/tests/context/data/userconf.yaml
+++ b/tests/internals/context/userconf.yaml
diff --git a/tests/loader/basics.py b/tests/internals/loader.py
index d7fc28f83..87a5c3b35 100644
--- a/tests/loader/basics.py
+++ b/tests/internals/loader.py
@@ -2,15 +2,28 @@ import os
 import pytest
 
 from buildstream._exceptions import LoadError, LoadErrorReason
+from buildstream._context import Context
+from buildstream._project import Project
 from buildstream._loader import Loader, MetaElement
-from . import make_loader
 
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
-    'basics',
+    'loader',
 )
 
 
+def dummy_handler(message, context):
+    pass
+
+
+def make_loader(basedir):
+    context = Context()
+    context.load(config=os.devnull)
+    context.set_message_handler(dummy_handler)
+    project = Project(basedir, context)
+    return project.loader
+
+
 ##############################################################
 #  Basics: Test behavior loading the simplest of projects    #
 ##############################################################
diff --git a/tests/loader/basics/onefile/elements/badfile.bst b/tests/internals/loader/onefile/elements/badfile.bst
index df68dbf2a..df68dbf2a 100644
--- a/tests/loader/basics/onefile/elements/badfile.bst
+++ b/tests/internals/loader/onefile/elements/badfile.bst
diff --git a/tests/loader/basics/onefile/elements/badreference.bst b/tests/internals/loader/onefile/elements/badreference.bst
index 9d320b2bf..9d320b2bf 100644
--- a/tests/loader/basics/onefile/elements/badreference.bst
+++ b/tests/internals/loader/onefile/elements/badreference.bst
diff --git a/tests/loader/basics/onefile/elements/invalidkey.bst b/tests/internals/loader/onefile/elements/invalidkey.bst
index 5674ab7bd..5674ab7bd 100644
--- a/tests/loader/basics/onefile/elements/invalidkey.bst
+++ b/tests/internals/loader/onefile/elements/invalidkey.bst
diff --git a/tests/loader/basics/onefile/elements/invalidsourcekey.bst b/tests/internals/loader/onefile/elements/invalidsourcekey.bst
index 5677af347..5677af347 100644
--- a/tests/loader/basics/onefile/elements/invalidsourcekey.bst
+++ b/tests/internals/loader/onefile/elements/invalidsourcekey.bst
diff --git a/tests/loader/basics/onefile/elements/onefile.bst b/tests/internals/loader/onefile/elements/onefile.bst
index ae4950585..ae4950585 100644
--- a/tests/loader/basics/onefile/elements/onefile.bst
+++ b/tests/internals/loader/onefile/elements/onefile.bst
diff --git a/tests/loader/basics/onefile/project.conf b/tests/internals/loader/onefile/project.conf
index afa0f5475..afa0f5475 100644
--- a/tests/loader/basics/onefile/project.conf
+++ b/tests/internals/loader/onefile/project.conf
diff --git a/tests/plugins/basics.py b/tests/internals/pluginfactory.py
index 4c9cbafe4..28ab039bd 100644
--- a/tests/plugins/basics.py
+++ b/tests/internals/pluginfactory.py
@@ -8,7 +8,7 @@ from buildstream._exceptions import PluginError
 
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
-    'basics',
+    'pluginfactory',
 )
 
 
diff --git a/tests/plugins/basics/anotherelement/__init__.py b/tests/internals/pluginfactory/anotherelement/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/basics/anotherelement/__init__.py
+++ b/tests/internals/pluginfactory/anotherelement/__init__.py
diff --git a/tests/plugins/basics/anotherelement/foo.py b/tests/internals/pluginfactory/anotherelement/foo.py
index 2e067a94f..2e067a94f 100644
--- a/tests/plugins/basics/anotherelement/foo.py
+++ b/tests/internals/pluginfactory/anotherelement/foo.py
diff --git a/tests/plugins/basics/anothersource/__init__.py b/tests/internals/pluginfactory/anothersource/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/basics/anothersource/__init__.py
+++ b/tests/internals/pluginfactory/anothersource/__init__.py
diff --git a/tests/plugins/basics/anothersource/foo.py b/tests/internals/pluginfactory/anothersource/foo.py
index 4675b965f..4675b965f 100644
--- a/tests/plugins/basics/anothersource/foo.py
+++ b/tests/internals/pluginfactory/anothersource/foo.py
diff --git a/tests/plugins/basics/badsetup/__init__.py b/tests/internals/pluginfactory/badsetup/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/basics/badsetup/__init__.py
+++ b/tests/internals/pluginfactory/badsetup/__init__.py
diff --git a/tests/plugins/basics/badsetup/foo.py b/tests/internals/pluginfactory/badsetup/foo.py
index 145f2577b..145f2577b 100644
--- a/tests/plugins/basics/badsetup/foo.py
+++ b/tests/internals/pluginfactory/badsetup/foo.py
diff --git a/tests/plugins/basics/badversionelement/__init__.py b/tests/internals/pluginfactory/badversionelement/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/basics/badversionelement/__init__.py
+++ b/tests/internals/pluginfactory/badversionelement/__init__.py
diff --git a/tests/plugins/basics/badversionelement/foo.py b/tests/internals/pluginfactory/badversionelement/foo.py
index 2a8b12abe..2a8b12abe 100644
--- a/tests/plugins/basics/badversionelement/foo.py
+++ b/tests/internals/pluginfactory/badversionelement/foo.py
diff --git a/tests/plugins/basics/badversionsource/__init__.py b/tests/internals/pluginfactory/badversionsource/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/basics/badversionsource/__init__.py
+++ b/tests/internals/pluginfactory/badversionsource/__init__.py
diff --git a/tests/plugins/basics/badversionsource/foo.py b/tests/internals/pluginfactory/badversionsource/foo.py
index 23333a9d8..23333a9d8 100644
--- a/tests/plugins/basics/badversionsource/foo.py
+++ b/tests/internals/pluginfactory/badversionsource/foo.py
diff --git a/tests/plugins/basics/customelement/__init__.py b/tests/internals/pluginfactory/customelement/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/basics/customelement/__init__.py
+++ b/tests/internals/pluginfactory/customelement/__init__.py
diff --git a/tests/plugins/basics/customelement/foo.py b/tests/internals/pluginfactory/customelement/foo.py
index 260de8b27..260de8b27 100644
--- a/tests/plugins/basics/customelement/foo.py
+++ b/tests/internals/pluginfactory/customelement/foo.py
diff --git a/tests/plugins/basics/customsource/__init__.py b/tests/internals/pluginfactory/customsource/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/basics/customsource/__init__.py
+++ b/tests/internals/pluginfactory/customsource/__init__.py
diff --git a/tests/plugins/basics/customsource/foo.py b/tests/internals/pluginfactory/customsource/foo.py
index de78a00ce..de78a00ce 100644
--- a/tests/plugins/basics/customsource/foo.py
+++ b/tests/internals/pluginfactory/customsource/foo.py
diff --git a/tests/plugins/basics/nosetup/__init__.py b/tests/internals/pluginfactory/nosetup/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/basics/nosetup/__init__.py
+++ b/tests/internals/pluginfactory/nosetup/__init__.py
diff --git a/tests/plugins/basics/nosetup/foo.py b/tests/internals/pluginfactory/nosetup/foo.py
index 0b5a4fa7e..0b5a4fa7e 100644
--- a/tests/plugins/basics/nosetup/foo.py
+++ b/tests/internals/pluginfactory/nosetup/foo.py
diff --git a/tests/plugins/basics/notatype/__init__.py b/tests/internals/pluginfactory/notatype/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/basics/notatype/__init__.py
+++ b/tests/internals/pluginfactory/notatype/__init__.py
diff --git a/tests/plugins/basics/notatype/foo.py b/tests/internals/pluginfactory/notatype/foo.py
index 311a4fb32..311a4fb32 100644
--- a/tests/plugins/basics/notatype/foo.py
+++ b/tests/internals/pluginfactory/notatype/foo.py
diff --git a/tests/plugins/basics/wrongtype/__init__.py b/tests/internals/pluginfactory/wrongtype/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/basics/wrongtype/__init__.py
+++ b/tests/internals/pluginfactory/wrongtype/__init__.py
diff --git a/tests/plugins/basics/wrongtype/foo.py b/tests/internals/pluginfactory/wrongtype/foo.py
index 3fe9a1a62..3fe9a1a62 100644
--- a/tests/plugins/basics/wrongtype/foo.py
+++ b/tests/internals/pluginfactory/wrongtype/foo.py
diff --git a/tests/plugins/pipeline.py b/tests/internals/pluginloading.py
index 6aee104e2..574e59f12 100644
--- a/tests/plugins/pipeline.py
+++ b/tests/internals/pluginloading.py
@@ -8,7 +8,7 @@ from buildstream._pipeline import Pipeline
 
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
-    'pipeline',
+    'pluginloading',
 )
 
 
diff --git a/tests/plugins/pipeline/badversionelement/customelements/__init__.py b/tests/internals/pluginloading/badversionelement/customelements/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/pipeline/badversionelement/customelements/__init__.py
+++ b/tests/internals/pluginloading/badversionelement/customelements/__init__.py
diff --git a/tests/plugins/pipeline/badversionelement/customelements/foo.py b/tests/internals/pluginloading/badversionelement/customelements/foo.py
index 75536e87f..75536e87f 100644
--- a/tests/plugins/pipeline/badversionelement/customelements/foo.py
+++ b/tests/internals/pluginloading/badversionelement/customelements/foo.py
diff --git a/tests/plugins/pipeline/badversionelement/elements/simple.bst b/tests/internals/pluginloading/badversionelement/elements/simple.bst
index f949dc5b5..f949dc5b5 100644
--- a/tests/plugins/pipeline/badversionelement/elements/simple.bst
+++ b/tests/internals/pluginloading/badversionelement/elements/simple.bst
diff --git a/tests/plugins/pipeline/badversionelement/project.conf b/tests/internals/pluginloading/badversionelement/project.conf
index 4f32752c3..4f32752c3 100644
--- a/tests/plugins/pipeline/badversionelement/project.conf
+++ b/tests/internals/pluginloading/badversionelement/project.conf
diff --git a/tests/plugins/pipeline/badversionsource/customsources/__init__.py b/tests/internals/pluginloading/badversionsource/customsources/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/pipeline/badversionsource/customsources/__init__.py
+++ b/tests/internals/pluginloading/badversionsource/customsources/__init__.py
diff --git a/tests/plugins/pipeline/badversionsource/customsources/foo.py b/tests/internals/pluginloading/badversionsource/customsources/foo.py
index f50855fd1..f50855fd1 100644
--- a/tests/plugins/pipeline/badversionsource/customsources/foo.py
+++ b/tests/internals/pluginloading/badversionsource/customsources/foo.py
diff --git a/tests/plugins/pipeline/badversionsource/elements/simple.bst b/tests/internals/pluginloading/badversionsource/elements/simple.bst
index 7e0cc43b7..7e0cc43b7 100644
--- a/tests/plugins/pipeline/badversionsource/elements/simple.bst
+++ b/tests/internals/pluginloading/badversionsource/elements/simple.bst
diff --git a/tests/plugins/pipeline/badversionsource/project.conf b/tests/internals/pluginloading/badversionsource/project.conf
index 58cf9c577..58cf9c577 100644
--- a/tests/plugins/pipeline/badversionsource/project.conf
+++ b/tests/internals/pluginloading/badversionsource/project.conf
diff --git a/tests/plugins/pipeline/customelement/elements/simple.bst b/tests/internals/pluginloading/customelement/elements/simple.bst
index fc48e3ba9..fc48e3ba9 100644
--- a/tests/plugins/pipeline/customelement/elements/simple.bst
+++ b/tests/internals/pluginloading/customelement/elements/simple.bst
diff --git a/tests/plugins/pipeline/customelement/pluginelements/__init__.py b/tests/internals/pluginloading/customelement/pluginelements/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/pipeline/customelement/pluginelements/__init__.py
+++ b/tests/internals/pluginloading/customelement/pluginelements/__init__.py
diff --git a/tests/plugins/pipeline/customelement/pluginelements/foo.py b/tests/internals/pluginloading/customelement/pluginelements/foo.py
index 823306ebc..823306ebc 100644
--- a/tests/plugins/pipeline/customelement/pluginelements/foo.py
+++ b/tests/internals/pluginloading/customelement/pluginelements/foo.py
diff --git a/tests/plugins/pipeline/customelement/project.conf b/tests/internals/pluginloading/customelement/project.conf
index bdc909985..bdc909985 100644
--- a/tests/plugins/pipeline/customelement/project.conf
+++ b/tests/internals/pluginloading/customelement/project.conf
diff --git a/tests/plugins/pipeline/customsource/elements/simple.bst b/tests/internals/pluginloading/customsource/elements/simple.bst
index 7e0cc43b7..7e0cc43b7 100644
--- a/tests/plugins/pipeline/customsource/elements/simple.bst
+++ b/tests/internals/pluginloading/customsource/elements/simple.bst
diff --git a/tests/plugins/pipeline/customsource/pluginsources/__init__.py b/tests/internals/pluginloading/customsource/pluginsources/__init__.py
index e69de29bb..e69de29bb 100644
--- a/tests/plugins/pipeline/customsource/pluginsources/__init__.py
+++ b/tests/internals/pluginloading/customsource/pluginsources/__init__.py
diff --git a/tests/plugins/pipeline/customsource/pluginsources/foo.py b/tests/internals/pluginloading/customsource/pluginsources/foo.py
index d2b0d9c6d..d2b0d9c6d 100644
--- a/tests/plugins/pipeline/customsource/pluginsources/foo.py
+++ b/tests/internals/pluginloading/customsource/pluginsources/foo.py
diff --git a/tests/plugins/pipeline/customsource/project.conf b/tests/internals/pluginloading/customsource/project.conf
index 8205d185a..8205d185a 100644
--- a/tests/plugins/pipeline/customsource/project.conf
+++ b/tests/internals/pluginloading/customsource/project.conf
diff --git a/tests/sandboxes/storage-tests.py b/tests/internals/storage.py
index 38716776a..c09ff4380 100644
--- a/tests/sandboxes/storage-tests.py
+++ b/tests/internals/storage.py
@@ -9,7 +9,7 @@ from buildstream.storage._filebaseddirectory import FileBasedDirectory
 
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
-    "storage-test"
+    "storage"
 )
 
 
diff --git a/tests/sandboxes/storage-test/original/bin/bash b/tests/internals/storage/original/bin/bash
index a221b564c..a221b564c 100644
--- a/tests/sandboxes/storage-test/original/bin/bash
+++ b/tests/internals/storage/original/bin/bash
diff --git a/tests/sandboxes/storage-test/original/bin/hello b/tests/internals/storage/original/bin/hello
index 5524e9677..5524e9677 100644
--- a/tests/sandboxes/storage-test/original/bin/hello
+++ b/tests/internals/storage/original/bin/hello
diff --git a/tests/sandboxes/storage-test/overlay/bin/bash b/tests/internals/storage/overlay/bin/bash
index b639d94ec..b639d94ec 100644
--- a/tests/sandboxes/storage-test/overlay/bin/bash
+++ b/tests/internals/storage/overlay/bin/bash
diff --git a/tests/storage/virtual_directory_import.py b/tests/internals/storage_vdir_import.py
index 0bb47e3cd..0bb47e3cd 100644
--- a/tests/storage/virtual_directory_import.py
+++ b/tests/internals/storage_vdir_import.py
diff --git a/tests/utils/misc.py b/tests/internals/utils.py
index a34d3cda5..a34d3cda5 100644
--- a/tests/utils/misc.py
+++ b/tests/internals/utils.py
diff --git a/tests/utils/movedirectory.py b/tests/internals/utils_move_atomic.py
index 6edbec611..6edbec611 100755..100644
--- a/tests/utils/movedirectory.py
+++ b/tests/internals/utils_move_atomic.py
diff --git a/tests/utils/savefile.py b/tests/internals/utils_save_atomic.py
index 0731f7bea..0731f7bea 100644
--- a/tests/utils/savefile.py
+++ b/tests/internals/utils_save_atomic.py
diff --git a/tests/yaml/yaml.py b/tests/internals/yaml.py
index 26515caee..bc513deb4 100644
--- a/tests/yaml/yaml.py
+++ b/tests/internals/yaml.py
@@ -10,7 +10,7 @@ from buildstream._yamlcache import YamlCache
 
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
-    'data',
+    'yaml',
 )
 
 
diff --git a/tests/yaml/data/basics.yaml b/tests/internals/yaml/basics.yaml
index e57d2163a..e57d2163a 100644
--- a/tests/yaml/data/basics.yaml
+++ b/tests/internals/yaml/basics.yaml
diff --git a/tests/yaml/data/composite.yaml b/tests/internals/yaml/composite.yaml
index d98456fd5..d98456fd5 100644
--- a/tests/yaml/data/composite.yaml
+++ b/tests/internals/yaml/composite.yaml
diff --git a/tests/yaml/data/convert_value_to_str.yaml b/tests/internals/yaml/convert_value_to_str.yaml
index 4a59e3287..4a59e3287 100644
--- a/tests/yaml/data/convert_value_to_str.yaml
+++ b/tests/internals/yaml/convert_value_to_str.yaml
diff --git a/tests/yaml/data/implicitoverwrite.yaml b/tests/internals/yaml/implicitoverwrite.yaml
index 957d2c084..957d2c084 100644
--- a/tests/yaml/data/implicitoverwrite.yaml
+++ b/tests/internals/yaml/implicitoverwrite.yaml
diff --git a/tests/yaml/data/invalid.yaml b/tests/internals/yaml/invalid.yaml
index bc7f1ae22..bc7f1ae22 100644
--- a/tests/yaml/data/invalid.yaml
+++ b/tests/internals/yaml/invalid.yaml
diff --git a/tests/yaml/data/listappend.yaml b/tests/internals/yaml/listappend.yaml
index 169308c54..169308c54 100644
--- a/tests/yaml/data/listappend.yaml
+++ b/tests/internals/yaml/listappend.yaml
diff --git a/tests/yaml/data/listappendprepend.yaml b/tests/internals/yaml/listappendprepend.yaml
index 7e4b59a9c..7e4b59a9c 100644
--- a/tests/yaml/data/listappendprepend.yaml
+++ b/tests/internals/yaml/listappendprepend.yaml
diff --git a/tests/yaml/data/listoverwrite.yaml b/tests/internals/yaml/listoverwrite.yaml
index eb9d7b1d9..eb9d7b1d9 100644
--- a/tests/yaml/data/listoverwrite.yaml
+++ b/tests/internals/yaml/listoverwrite.yaml
diff --git a/tests/yaml/data/listoverwriteempty.yaml b/tests/internals/yaml/listoverwriteempty.yaml
index f97a1eafe..f97a1eafe 100644
--- a/tests/yaml/data/listoverwriteempty.yaml
+++ b/tests/internals/yaml/listoverwriteempty.yaml
diff --git a/tests/yaml/data/listprepend.yaml b/tests/internals/yaml/listprepend.yaml
index a19a9d5d7..a19a9d5d7 100644
--- a/tests/yaml/data/listprepend.yaml
+++ b/tests/internals/yaml/listprepend.yaml
diff --git a/tests/yaml/data/secondappend.yaml b/tests/internals/yaml/secondappend.yaml
index 20cfc1ad6..20cfc1ad6 100644
--- a/tests/yaml/data/secondappend.yaml
+++ b/tests/internals/yaml/secondappend.yaml
diff --git a/tests/yaml/data/secondprepend.yaml b/tests/internals/yaml/secondprepend.yaml
index 5be8b9e72..5be8b9e72 100644
--- a/tests/yaml/data/secondprepend.yaml
+++ b/tests/internals/yaml/secondprepend.yaml
diff --git a/tests/loader/__init__.py b/tests/loader/__init__.py
deleted file mode 100644
index 812888181..000000000
--- a/tests/loader/__init__.py
+++ /dev/null
@@ -1,22 +0,0 @@
-import os
-from buildstream._context import Context
-from buildstream._project import Project
-from buildstream._loader import Loader
-
-#
-# This is used by the loader test modules, these should
-# be removed in favor of testing the functionality via
-# the CLI like in the frontend tests anyway.
-#
-
-
-def dummy_handler(message, context):
-    pass
-
-
-def make_loader(basedir):
-    context = Context()
-    context.load(config=os.devnull)
-    context.set_message_handler(dummy_handler)
-    project = Project(basedir, context)
-    return project.loader
diff --git a/tests/loader/dependencies.py b/tests/loader/dependencies.py
deleted file mode 100644
index 98374f6d0..000000000
--- a/tests/loader/dependencies.py
+++ /dev/null
@@ -1,236 +0,0 @@
-import os
-import pytest
-
-from buildstream._exceptions import LoadError, LoadErrorReason
-from buildstream._loader import Loader, MetaElement
-from tests.testutils import cli
-from . import make_loader
-
-DATA_DIR = os.path.join(
-    os.path.dirname(os.path.realpath(__file__)),
-    'dependencies',
-)
-
-
-##############################################################
-#  Basics: Test behavior loading projects with dependencies  #
-##############################################################
-@pytest.mark.datafiles(DATA_DIR)
-def test_two_files(datafiles):
-
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = make_loader(basedir)
-    element = loader.load(['elements/target.bst'])[0]
-
-    assert(isinstance(element, MetaElement))
-    assert(element.kind == 'pony')
-
-    assert(len(element.dependencies) == 1)
-    firstdep = element.dependencies[0]
-    assert(isinstance(firstdep, MetaElement))
-    assert(firstdep.kind == 'manual')
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_shared_dependency(datafiles):
-
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = make_loader(basedir)
-    element = loader.load(['elements/shareddeptarget.bst'])[0]
-
-    # Toplevel is 'pony' with 2 dependencies
-    #
-    assert(isinstance(element, MetaElement))
-    assert(element.kind == 'pony')
-    assert(len(element.dependencies) == 2)
-
-    # The first specified dependency is 'thefirstdep'
-    #
-    firstdep = element.dependencies[0]
-    assert(isinstance(firstdep, MetaElement))
-    assert(firstdep.kind == 'manual')
-    assert(len(firstdep.dependencies) == 0)
-
-    # The second specified dependency is 'shareddep'
-    #
-    shareddep = element.dependencies[1]
-    assert(isinstance(shareddep, MetaElement))
-    assert(shareddep.kind == 'shareddep')
-    assert(len(shareddep.dependencies) == 1)
-
-    # The element which shareddep depends on is
-    # the same element in memory as firstdep
-    #
-    shareddepdep = shareddep.dependencies[0]
-    assert(isinstance(shareddepdep, MetaElement))
-
-    # Assert they are in fact the same LoadElement
-    #
-    # Note we must use 'is' to test that both variables
-    # refer to the same object in memory, not a regular
-    # equality test with '==' which is one of those operator
-    # overridable thingies.
-    #
-    assert(shareddepdep is firstdep)
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_dependency_dict(datafiles):
-
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = make_loader(basedir)
-    element = loader.load(['elements/target-depdict.bst'])[0]
-
-    assert(isinstance(element, MetaElement))
-    assert(element.kind == 'pony')
-
-    assert(len(element.dependencies) == 1)
-    firstdep = element.dependencies[0]
-    assert(isinstance(firstdep, MetaElement))
-    assert(firstdep.kind == 'manual')
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_invalid_dependency_declaration(datafiles):
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = make_loader(basedir)
-
-    with pytest.raises(LoadError) as exc:
-        element = loader.load(['elements/invaliddep.bst'])[0]
-
-    assert (exc.value.reason == LoadErrorReason.INVALID_DATA)
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_circular_dependency(datafiles):
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = make_loader(basedir)
-
-    with pytest.raises(LoadError) as exc:
-        element = loader.load(['elements/circulartarget.bst'])[0]
-
-    assert (exc.value.reason == LoadErrorReason.CIRCULAR_DEPENDENCY)
-    assert ("seconddep" in exc.value.args[0])
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_invalid_dependency_type(datafiles):
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = make_loader(basedir)
-
-    with pytest.raises(LoadError) as exc:
-        element = loader.load(['elements/invaliddeptype.bst'])[0]
-
-    assert (exc.value.reason == LoadErrorReason.INVALID_DATA)
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_build_dependency(datafiles):
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = make_loader(basedir)
-    element = loader.load(['elements/builddep.bst'])[0]
-
-    assert(isinstance(element, MetaElement))
-    assert(element.kind == 'pony')
-
-    assert(len(element.build_dependencies) == 1)
-    firstdep = element.build_dependencies[0]
-    assert(isinstance(firstdep, MetaElement))
-
-    assert(len(element.dependencies) == 0)
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_runtime_dependency(datafiles):
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = make_loader(basedir)
-    element = loader.load(['elements/runtimedep.bst'])[0]
-
-    assert(isinstance(element, MetaElement))
-    assert(element.kind == 'pony')
-
-    assert(len(element.dependencies) == 1)
-    firstdep = element.dependencies[0]
-    assert(isinstance(firstdep, MetaElement))
-
-    assert(len(element.build_dependencies) == 0)
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_build_runtime_dependency(datafiles):
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = make_loader(basedir)
-    element = loader.load(['elements/target.bst'])[0]
-
-    assert(isinstance(element, MetaElement))
-    assert(element.kind == 'pony')
-
-    assert(len(element.dependencies) == 1)
-    assert(len(element.build_dependencies) == 1)
-    firstdep = element.dependencies[0]
-    assert(isinstance(firstdep, MetaElement))
-    firstbuilddep = element.build_dependencies[0]
-    assert(firstdep == firstbuilddep)
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_all_dependency(datafiles):
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = make_loader(basedir)
-    element = loader.load(['elements/alldep.bst'])[0]
-
-    assert(isinstance(element, MetaElement))
-    assert(element.kind == 'pony')
-
-    assert(len(element.dependencies) == 1)
-    assert(len(element.build_dependencies) == 1)
-    firstdep = element.dependencies[0]
-    assert(isinstance(firstdep, MetaElement))
-    firstbuilddep = element.build_dependencies[0]
-    assert(firstdep == firstbuilddep)
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_list_build_dependency(cli, datafiles):
-    project = str(datafiles)
-
-    # Check that the pipeline includes the build dependency
-    deps = cli.get_pipeline(project, ['elements/builddep-list.bst'], scope="build")
-    assert "elements/firstdep.bst" in deps
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_list_runtime_dependency(cli, datafiles):
-    project = str(datafiles)
-
-    # Check that the pipeline includes the runtime dependency
-    deps = cli.get_pipeline(project, ['elements/runtimedep-list.bst'], scope="run")
-    assert "elements/firstdep.bst" in deps
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_list_dependencies_combined(cli, datafiles):
-    project = str(datafiles)
-
-    # Check that runtime deps get combined
-    rundeps = cli.get_pipeline(project, ['elements/list-combine.bst'], scope="run")
-    assert "elements/firstdep.bst" not in rundeps
-    assert "elements/seconddep.bst" in rundeps
-    assert "elements/thirddep.bst" in rundeps
-
-    # Check that build deps get combined
-    builddeps = cli.get_pipeline(project, ['elements/list-combine.bst'], scope="build")
-    assert "elements/firstdep.bst" in builddeps
-    assert "elements/seconddep.bst" not in builddeps
-    assert "elements/thirddep.bst" in builddeps
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_list_overlap(cli, datafiles):
-    project = str(datafiles)
-
-    # Check that dependencies get merged
-    rundeps = cli.get_pipeline(project, ['elements/list-overlap.bst'], scope="run")
-    assert "elements/firstdep.bst" in rundeps
-    builddeps = cli.get_pipeline(project, ['elements/list-overlap.bst'], scope="build")
-    assert "elements/firstdep.bst" in builddeps
diff --git a/tests/loader/dependencies/elements/shareddeptarget.bst b/tests/loader/dependencies/elements/shareddeptarget.bst
deleted file mode 100644
index 03982d69f..000000000
--- a/tests/loader/dependencies/elements/shareddeptarget.bst
+++ /dev/null
@@ -1,5 +0,0 @@
-kind: pony
-description: This is the main target
-depends:
-- elements/firstdep.bst
-- elements/shareddep.bst
diff --git a/tests/loader/dependencies/project.conf b/tests/loader/dependencies/project.conf
deleted file mode 100644
index afa0f5475..000000000
--- a/tests/loader/dependencies/project.conf
+++ /dev/null
@@ -1,2 +0,0 @@
-# Basic project
-name: foo
diff --git a/tests/loader/variables.py b/tests/loader/variables.py
deleted file mode 100644
index 9871d63c6..000000000
--- a/tests/loader/variables.py
+++ /dev/null
@@ -1,99 +0,0 @@
-import os
-import pytest
-
-from buildstream import _yaml
-from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils import cli
-
-DATA_DIR = os.path.join(
-    os.path.dirname(os.path.realpath(__file__)),
-    'variables',
-)
-
-PROTECTED_VARIABLES = [('project-name'), ('element-name'), ('max-jobs')]
-
-
-@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
-@pytest.mark.datafiles(DATA_DIR)
-def test_use_of_protected_var_project_conf(cli, tmpdir, datafiles, protected_var):
-    project = os.path.join(str(datafiles), 'simple')
-
-    conf = {
-        'name': 'test',
-        'variables': {
-            protected_var: 'some-value'
-        }
-    }
-    _yaml.dump(conf, os.path.join(project, 'project.conf'))
-
-    element = {
-        'kind': 'import',
-        'sources': [
-            {
-                'kind': 'local',
-                'path': 'foo.txt'
-            }
-        ],
-    }
-    _yaml.dump(element, os.path.join(project, 'target.bst'))
-
-    result = cli.run(project=project, args=['build', 'target.bst'])
-    result.assert_main_error(ErrorDomain.LOAD,
-                             LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
-
-
-@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
-@pytest.mark.datafiles(DATA_DIR)
-def test_use_of_protected_var_element_overrides(cli, tmpdir, datafiles, protected_var):
-    project = os.path.join(str(datafiles), 'simple')
-
-    conf = {
-        'name': 'test',
-        'elements': {
-            'manual': {
-                'variables': {
-                    protected_var: 'some-value'
-                }
-            }
-        }
-    }
-    _yaml.dump(conf, os.path.join(project, 'project.conf'))
-
-    element = {
-        'kind': 'manual',
-        'sources': [
-            {
-                'kind': 'local',
-                'path': 'foo.txt'
-            }
-        ],
-    }
-    _yaml.dump(element, os.path.join(project, 'target.bst'))
-
-    result = cli.run(project=project, args=['build', 'target.bst'])
-    result.assert_main_error(ErrorDomain.LOAD,
-                             LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
-
-
-@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
-@pytest.mark.datafiles(DATA_DIR)
-def test_use_of_protected_var_in_element(cli, tmpdir, datafiles, protected_var):
-    project = os.path.join(str(datafiles), 'simple')
-
-    element = {
-        'kind': 'import',
-        'sources': [
-            {
-                'kind': 'local',
-                'path': 'foo.txt'
-            }
-        ],
-        'variables': {
-            protected_var: 'some-value'
-        }
-    }
-    _yaml.dump(element, os.path.join(project, 'target.bst'))
-
-    result = cli.run(project=project, args=['build', 'target.bst'])
-    result.assert_main_error(ErrorDomain.LOAD,
-                             LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
diff --git a/tests/pipeline/load.py b/tests/pipeline/load.py
deleted file mode 100644
index 29ceb6dde..000000000
--- a/tests/pipeline/load.py
+++ /dev/null
@@ -1,219 +0,0 @@
-import os
-import pytest
-from buildstream._exceptions import ErrorDomain
-from buildstream import _yaml
-from tests.testutils.runcli import cli
-
-DATA_DIR = os.path.join(
-    os.path.dirname(os.path.realpath(__file__)),
-    'load',
-)
-
-
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'simple'))
-def test_load_simple(cli, datafiles):
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    result = cli.get_element_config(basedir, 'simple.bst')
-
-    assert(result['configure-commands'][0] == 'pony')
-
-
-###############################################################
-#        Testing Element.dependencies() iteration             #
-###############################################################
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'iterate'))
-def test_iterate_scope_all(cli, datafiles):
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    elements = ['target.bst']
-
-    element_list = cli.get_pipeline(basedir, elements, scope='all')
-
-    assert(len(element_list) == 7)
-
-    assert(element_list[0] == "build-build.bst")
-    assert(element_list[1] == "run-build.bst")
-    assert(element_list[2] == "build.bst")
-    assert(element_list[3] == "dep-one.bst")
-    assert(element_list[4] == "run.bst")
-    assert(element_list[5] == "dep-two.bst")
-    assert(element_list[6] == "target.bst")
-
-
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'iterate'))
-def test_iterate_scope_run(cli, datafiles):
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    elements = ['target.bst']
-
-    element_list = cli.get_pipeline(basedir, elements, scope='run')
-
-    assert(len(element_list) == 4)
-
-    assert(element_list[0] == "dep-one.bst")
-    assert(element_list[1] == "run.bst")
-    assert(element_list[2] == "dep-two.bst")
-    assert(element_list[3] == "target.bst")
-
-
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'iterate'))
-def test_iterate_scope_build(cli, datafiles):
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    elements = ['target.bst']
-
-    element_list = cli.get_pipeline(basedir, elements, scope='build')
-
-    assert(len(element_list) == 3)
-
-    assert(element_list[0] == "dep-one.bst")
-    assert(element_list[1] == "run.bst")
-    assert(element_list[2] == "dep-two.bst")
-
-
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'iterate'))
-def test_iterate_scope_build_of_child(cli, datafiles):
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    elements = ['target.bst']
-
-    element_list = cli.get_pipeline(basedir, elements, scope='build')
-
-    # First pass, lets check dep-two
-    element = element_list[2]
-
-    # Pass two, let's look at these
-    element_list = cli.get_pipeline(basedir, [element], scope='build')
-
-    assert(len(element_list) == 2)
-
-    assert(element_list[0] == "run-build.bst")
-    assert(element_list[1] == "build.bst")
-
-
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'iterate'))
-def test_iterate_no_recurse(cli, datafiles):
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    elements = ['target.bst']
-
-    # We abuse the 'plan' scope here to ensure that we call
-    # element.dependencies() with recurse=False - currently, no `bst
-    # show` option does this directly.
-    element_list = cli.get_pipeline(basedir, elements, scope='plan')
-
-    assert(len(element_list) == 7)
-
-    assert(element_list[0] == 'build-build.bst')
-    assert(element_list[1] in ['build.bst', 'run-build.bst'])
-    assert(element_list[2] in ['build.bst', 'run-build.bst'])
-    assert(element_list[3] in ['dep-one.bst', 'run.bst', 'dep-two.bst'])
-    assert(element_list[4] in ['dep-one.bst', 'run.bst', 'dep-two.bst'])
-    assert(element_list[5] in ['dep-one.bst', 'run.bst', 'dep-two.bst'])
-    assert(element_list[6] == 'target.bst')
-
-
-# This test checks various constructions of a pipeline
-# with one or more targets and 0 or more exception elements,
-# each data set provides the targets, exceptions and expected
-# result list.
-#
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'exceptions'))
-@pytest.mark.parametrize("elements,exceptions,results", [
-
-    # Test without exceptions, lets just see the whole list here
-    (['build.bst'], None, [
-        'fourth-level-1.bst',
-        'third-level-1.bst',
-        'fourth-level-2.bst',
-        'third-level-2.bst',
-        'fourth-level-3.bst',
-        'third-level-3.bst',
-        'second-level-1.bst',
-        'first-level-1.bst',
-        'first-level-2.bst',
-        'build.bst',
-    ]),
-
-    # Test one target and excepting a part of the pipeline, this
-    # removes forth-level-1 and third-level-1
-    (['build.bst'], ['third-level-1.bst'], [
-        'fourth-level-2.bst',
-        'third-level-2.bst',
-        'fourth-level-3.bst',
-        'third-level-3.bst',
-        'second-level-1.bst',
-        'first-level-1.bst',
-        'first-level-2.bst',
-        'build.bst',
-    ]),
-
-    # Test one target and excepting a part of the pipeline, check that
-    # excepted dependencies remain in the pipeline if depended on from
-    # outside of the except element
-    (['build.bst'], ['second-level-1.bst'], [
-        'fourth-level-2.bst',
-        'third-level-2.bst',  # first-level-2 depends on this, so not excepted
-        'first-level-1.bst',
-        'first-level-2.bst',
-        'build.bst',
-    ]),
-
-    # The same as the above test, but excluding the toplevel build.bst,
-    # instead only select the two toplevel dependencies as targets
-    (['first-level-1.bst', 'first-level-2.bst'], ['second-level-1.bst'], [
-        'fourth-level-2.bst',
-        'third-level-2.bst',  # first-level-2 depends on this, so not excepted
-        'first-level-1.bst',
-        'first-level-2.bst',
-    ]),
-
-    # Test one target and excepting an element outisde the pipeline
-    (['build.bst'], ['unrelated-1.bst'], [
-        'fourth-level-2.bst',
-        'third-level-2.bst',  # first-level-2 depends on this, so not excepted
-        'first-level-1.bst',
-        'first-level-2.bst',
-        'build.bst',
-    ]),
-
-    # Test one target and excepting two elements
-    (['build.bst'], ['unrelated-1.bst', 'unrelated-2.bst'], [
-        'first-level-1.bst',
-        'build.bst',
-    ]),
-])
-def test_except_elements(cli, datafiles, elements, exceptions, results):
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-
-    # Except second-level-2 and check that the correct dependencies
-    # are removed.
-    element_list = cli.get_pipeline(basedir, elements, except_=exceptions, scope='all')
-    assert element_list == results
-
-
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'noloadref'))
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
-def test_unsupported_load_ref(cli, datafiles, ref_storage):
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-
-    # Generate project with access to the noloadref plugin and project.refs enabled
-    #
-    config = {
-        'name': 'test',
-        'ref-storage': ref_storage,
-        'plugins': [
-            {
-                'origin': 'local',
-                'path': 'plugins',
-                'sources': {
-                    'noloadref': 0
-                }
-            }
-        ]
-    }
-    _yaml.dump(config, os.path.join(basedir, 'project.conf'))
-
-    result = cli.run(project=basedir, silent=True, args=['show', 'noloadref.bst'])
-
-    # There is no error if project.refs is not in use, otherwise we
-    # assert our graceful failure
-    if ref_storage == 'inline':
-        result.assert_success()
-    else:
-        result.assert_main_error(ErrorDomain.SOURCE, 'unsupported-load-ref')
diff --git a/tests/pipeline/load/simple/project.conf b/tests/pipeline/load/simple/project.conf
deleted file mode 100644
index 2027cc27a..000000000
--- a/tests/pipeline/load/simple/project.conf
+++ /dev/null
@@ -1,3 +0,0 @@
-# Basic project configuration that doesnt override anything
-#
-name: pony
diff --git a/tests/pipeline/load/simple/simple.bst b/tests/pipeline/load/simple/simple.bst
deleted file mode 100644
index cd1f73968..000000000
--- a/tests/pipeline/load/simple/simple.bst
+++ /dev/null
@@ -1,5 +0,0 @@
-kind: autotools
-description: Some kinda autotools element
-config:
-  configure-commands:
-  - pony
diff --git a/tests/pipeline/preflight.py b/tests/pipeline/preflight.py
deleted file mode 100644
index 938f93191..000000000
--- a/tests/pipeline/preflight.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import os
-import pytest
-
-from buildstream._exceptions import ErrorDomain
-from tests.testutils.runcli import cli
-
-DATA_DIR = os.path.join(
-    os.path.dirname(os.path.realpath(__file__)),
-    'preflight-error',
-)
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_load_simple(cli, datafiles, tmpdir):
-    basedir = os.path.join(datafiles.dirname, datafiles.basename)
-
-    # Lets try to fetch it...
-    result = cli.run(project=basedir, args=['source', 'fetch', 'error.bst'])
-    result.assert_main_error(ErrorDomain.SOURCE, "the-preflight-error")
diff --git a/tests/plugins/third_party.py b/tests/plugins/third_party.py
deleted file mode 100644
index b7791a2e4..000000000
--- a/tests/plugins/third_party.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import os
-import pytest
-
-from pluginbase import PluginBase
-from buildstream._elementfactory import ElementFactory
-from buildstream._sourcefactory import SourceFactory
-
-from tests.testutils.setuptools import entry_fixture
-
-DATA_DIR = os.path.join(
-    os.path.dirname(os.path.realpath(__file__)),
-    'third_party'
-)
-
-
-# Simple fixture to create a PluginBase object that
-# we use for loading plugins.
-@pytest.fixture()
-def plugin_fixture():
-    return {
-        'base': PluginBase(package='buildstream.plugins')
-    }
-
-
-##################################################################
-#                              Tests                             #
-##################################################################
-# Test that external element plugin loading works.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'third_party_element'))
-def test_custom_pip_element(plugin_fixture, entry_fixture, datafiles):
-    origin_data = [{
-        'origin': 'local',
-        'path': str(datafiles),
-        'plugins': {'foop': 0}
-    }]
-    factory = ElementFactory(plugin_fixture['base'],
-                             plugin_origins=origin_data)
-    assert(isinstance(factory, ElementFactory))
-
-    entry_fixture(datafiles, 'buildstream.plugins', 'third_party_element:foop')
-
-    foo_type, _ = factory.lookup('foop')
-    assert(foo_type.__name__ == 'FooElement')
-
-
-# Test that external source plugin loading works.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'third_party_source'))
-def test_custom_pip_source(plugin_fixture, entry_fixture, datafiles):
-    origin_data = [{
-        'origin': 'local',
-        'path': str(datafiles),
-        'plugins': {'foop': 0}
-    }]
-    factory = SourceFactory(plugin_fixture['base'],
-                            plugin_origins=origin_data)
-    assert(isinstance(factory, SourceFactory))
-
-    entry_fixture(datafiles, 'buildstream.plugins', 'third_party_source:foop')
-
-    foo_type, _ = factory.lookup('foop')
-    assert(foo_type.__name__ == 'FooSource')
diff --git a/tests/plugins/third_party/third_party_element/__init__.py b/tests/plugins/third_party/third_party_element/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/tests/plugins/third_party/third_party_element/__init__.py
+++ /dev/null
diff --git a/tests/plugins/third_party/third_party_element/foop.py b/tests/plugins/third_party/third_party_element/foop.py
deleted file mode 100644
index 260de8b27..000000000
--- a/tests/plugins/third_party/third_party_element/foop.py
+++ /dev/null
@@ -1,9 +0,0 @@
-from buildstream import Element
-
-
-class FooElement(Element):
-    pass
-
-
-def setup():
-    return FooElement
diff --git a/tests/plugins/third_party/third_party_source/__init__.py b/tests/plugins/third_party/third_party_source/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/tests/plugins/third_party/third_party_source/__init__.py
+++ /dev/null
diff --git a/tests/plugins/third_party/third_party_source/foop.py b/tests/plugins/third_party/third_party_source/foop.py
deleted file mode 100644
index de78a00ce..000000000
--- a/tests/plugins/third_party/third_party_source/foop.py
+++ /dev/null
@@ -1,9 +0,0 @@
-from buildstream import Source
-
-
-class FooSource(Source):
-    pass
-
-
-def setup():
-    return FooSource
diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/tests/utils/__init__.py
+++ /dev/null
diff --git a/tests/yaml/__init__.py b/tests/yaml/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--- a/tests/yaml/__init__.py
+++ /dev/null