Merge branch 'bschubert/register-sources-on-test' into 'master'

testing/sources: Automatically register plugin sources

See merge request BuildStream/buildstream!1537

11 files changed, 235 insertions, 171 deletions

diff --git a/src/buildstream/testing/__init__.py b/src/buildstream/testing/__init__.py
index 0b1c1fd73..5a034cf30 100644
--- a/src/buildstream/testing/__init__.py
+++ b/src/buildstream/testing/__init__.py
@@ -21,7 +21,6 @@ This package contains various utilities which make it easier to test plugins.
 
 import os
 from collections import OrderedDict
-from . import _sourcetests
 from .repo import Repo
 from .runcli import cli, cli_integration, cli_remote_execution
 from .integration import integration_cache
@@ -38,6 +37,7 @@ except ImportError:
     raise ImportError(msg)
 
 
+# Of the form plugin_name -> (repo_class, plugin_package)
 ALL_REPO_KINDS = OrderedDict()
 
 
@@ -58,10 +58,10 @@ def create_repo(kind, directory, subdir='repo'):
     except KeyError as e:
         raise AssertionError("Unsupported repo kind {}".format(kind)) from e
 
-    return constructor(directory, subdir=subdir)
+    return constructor[0](directory, subdir=subdir)
 
 
-def register_repo_kind(kind, cls):
+def register_repo_kind(kind, cls, plugin_package):
     """Register a new repo kind.
 
     Registering a repo kind will allow the use of the `create_repo`
@@ -75,9 +75,10 @@ def register_repo_kind(kind, cls):
     Args:
        kind (str): The kind of repo to create (a source plugin basename)
        cls (cls) : A class derived from Repo.
+       plugin_package (str): The name of the python package containing the plugin
 
     """
-    ALL_REPO_KINDS[kind] = cls
+    ALL_REPO_KINDS[kind] = (cls, plugin_package)
 
 
 def sourcetests_collection_hook(session):
@@ -110,12 +111,13 @@ def sourcetests_collection_hook(session):
         # spot and is less likely to result in bug not being found.
         return True
 
-    SOURCE_TESTS_PATH = os.path.dirname(_sourcetests.__file__)
+    from . import _sourcetests
+    source_test_path = os.path.dirname(_sourcetests.__file__)
     # Add the location of the source tests to the session's
     # python_files config. Without this, pytest may filter out these
     # tests during collection.
-    session.config.addinivalue_line("python_files", os.path.join(SOURCE_TESTS_PATH, "*.py"))
+    session.config.addinivalue_line("python_files", os.path.join(source_test_path, "*.py"))
     # If test invocation has specified specic tests, don't
     # automatically collect templated tests.
     if should_collect_tests(session.config):
-        session.config.args.append(SOURCE_TESTS_PATH)
+        session.config.args.append(source_test_path)
diff --git a/src/buildstream/testing/_sourcetests/build_checkout.py b/src/buildstream/testing/_sourcetests/build_checkout.py
index e2842e0e0..4d4bcf0e2 100644
--- a/src/buildstream/testing/_sourcetests/build_checkout.py
+++ b/src/buildstream/testing/_sourcetests/build_checkout.py
@@ -22,18 +22,15 @@
 import os
 import pytest
 
-from buildstream.testing import create_repo, ALL_REPO_KINDS
+from buildstream.testing import create_repo
 from buildstream.testing import cli  # pylint: disable=unused-import
 from buildstream import _yaml
+from .utils import kind  # pylint: disable=unused-import
 
 # Project directory
 TOP_DIR = os.path.dirname(os.path.realpath(__file__))
 DATA_DIR = os.path.join(TOP_DIR, 'project')
 
-fetch_build_checkout_combos = \
-    [("strict", kind) for kind in ALL_REPO_KINDS] + \
-    [("non-strict", kind) for kind in ALL_REPO_KINDS]
-
 
 def strict_args(args, strict):
     if strict != "strict":
@@ -42,7 +39,7 @@ def strict_args(args, strict):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("strict,kind", fetch_build_checkout_combos)
+@pytest.mark.parametrize("strict", ["strict", "non-strict"])
 def test_fetch_build_checkout(cli, tmpdir, datafiles, strict, kind):
     checkout = os.path.join(cli.directory, 'checkout')
     project = str(datafiles)
diff --git a/src/buildstream/testing/_sourcetests/fetch.py b/src/buildstream/testing/_sourcetests/fetch.py
index d9b0876c6..897752297 100644
--- a/src/buildstream/testing/_sourcetests/fetch.py
+++ b/src/buildstream/testing/_sourcetests/fetch.py
@@ -23,9 +23,12 @@ import os
 import pytest
 
 from buildstream import _yaml
-from .._utils import generate_junction, configure_project
-from .. import create_repo, ALL_REPO_KINDS
+from .._utils import generate_junction
+from .. import create_repo
 from .. import cli  # pylint: disable=unused-import
+from .utils import update_project_configuration
+from .utils import kind  # pylint: disable=unused-import
+
 
 # Project directory
 TOP_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -33,7 +36,6 @@ DATA_DIR = os.path.join(TOP_DIR, 'project')
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
 def test_fetch(cli, tmpdir, datafiles, kind):
     project = str(datafiles)
     bin_files_path = os.path.join(project, 'files', 'bin-files')
@@ -69,8 +71,7 @@ def test_fetch(cli, tmpdir, datafiles, kind):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs'])
 def test_fetch_cross_junction(cli, tmpdir, datafiles, ref_storage, kind):
     project = str(datafiles)
     subproject_path = os.path.join(project, 'files', 'sub-project')
@@ -90,7 +91,7 @@ def test_fetch_cross_junction(cli, tmpdir, datafiles, ref_storage, kind):
     }
     _yaml.roundtrip_dump(element, import_etc_path)
 
-    configure_project(project, {
+    update_project_configuration(project, {
         'ref-storage': ref_storage
     })
 
diff --git a/src/buildstream/testing/_sourcetests/mirror.py b/src/buildstream/testing/_sourcetests/mirror.py
index f532049dd..b6316045d 100644
--- a/src/buildstream/testing/_sourcetests/mirror.py
+++ b/src/buildstream/testing/_sourcetests/mirror.py
@@ -25,23 +25,42 @@ import pytest
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain
 from .._utils import generate_junction
-from .. import create_repo, ALL_REPO_KINDS
+from .. import create_repo
 from .. import cli  # pylint: disable=unused-import
+from .utils import kind  # pylint: disable=unused-import
 
 # Project directory
 TOP_DIR = os.path.dirname(os.path.realpath(__file__))
 DATA_DIR = os.path.join(TOP_DIR, 'project')
 
 
+def _set_project_mirrors_and_aliases(project_path, mirrors, aliases):
+    project_conf_path = os.path.join(project_path, 'project.conf')
+    project_conf = _yaml.roundtrip_load(project_conf_path)
+
+    project_conf['mirrors'] = mirrors
+    project_conf['aliases'].update(aliases)
+
+    _yaml.roundtrip_dump(project_conf, project_conf_path)
+
+
+def _set_project_includes_and_aliases(project_path, includes, aliases):
+    project_conf_path = os.path.join(project_path, 'project.conf')
+    project_conf = _yaml.roundtrip_load(project_conf_path)
+
+    project_conf['aliases'].update(aliases)
+    project_conf['(@)'] = includes
+
+    _yaml.roundtrip_dump(project_conf, project_conf_path)
+
+
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
 def test_mirror_fetch(cli, tmpdir, datafiles, kind):
-    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
-    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
+    project_dir = str(datafiles)
+    bin_files_path = os.path.join(project_dir, 'files', 'bin-files', 'usr')
+    dev_files_path = os.path.join(project_dir, 'files', 'dev-files', 'usr')
     upstream_repodir = os.path.join(str(tmpdir), 'upstream')
     mirror_repodir = os.path.join(str(tmpdir), 'mirror')
-    project_dir = os.path.join(str(tmpdir), 'project')
-    os.makedirs(project_dir)
     element_dir = os.path.join(project_dir, 'elements')
 
     # Create repo objects of the upstream and mirror
@@ -65,26 +84,20 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind):
     element['sources'][0]['url'] = aliased_repo
     full_mirror = mirror_repo.source_config()['url']
     mirror_map, _ = os.path.split(full_mirror)
-    os.makedirs(element_dir)
     _yaml.roundtrip_dump(element, element_path)
 
-    project = {
-        'name': 'test',
-        'element-path': 'elements',
-        'aliases': {
-            alias: upstream_map + "/"
-        },
-        'mirrors': [
+    _set_project_mirrors_and_aliases(
+        project_dir,
+        [
             {
                 'name': 'middle-earth',
                 'aliases': {
-                    alias: [mirror_map + "/"],
+                    alias: [mirror_map + '/'],
                 },
             },
-        ]
-    }
-    project_file = os.path.join(project_dir, 'project.conf')
-    _yaml.roundtrip_dump(project, project_file)
+        ],
+        {alias: upstream_map + '/'},
+    )
 
     # No obvious ways of checking that the mirror has been fetched
     # But at least we can be sure it succeeds
@@ -93,13 +106,11 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
 def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind):
-    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
-    upstream_repodir = os.path.join(str(tmpdir), 'upstream')
+    project_dir = str(datafiles)
+    dev_files_path = os.path.join(project_dir, 'files', 'dev-files', 'usr')
+    upstream_repodir = os.path.join(project_dir, 'upstream')
     mirror_repodir = os.path.join(str(tmpdir), 'mirror')
-    project_dir = os.path.join(str(tmpdir), 'project')
-    os.makedirs(project_dir)
     element_dir = os.path.join(project_dir, 'elements')
 
     # Create repo objects of the upstream and mirror
@@ -123,39 +134,31 @@ def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind):
     element['sources'][0]['url'] = aliased_repo
     full_mirror = mirror_repo.source_config()['url']
     mirror_map, _ = os.path.split(full_mirror)
-    os.makedirs(element_dir)
     _yaml.roundtrip_dump(element, element_path)
 
-    project = {
-        'name': 'test',
-        'element-path': 'elements',
-        'aliases': {
-            alias: 'http://www.example.com/'
-        },
-        'mirrors': [
+    _set_project_mirrors_and_aliases(
+        project_dir,
+        [
             {
                 'name': 'middle-earth',
                 'aliases': {
-                    alias: [mirror_map + "/"],
+                    alias: [mirror_map + "/"]
                 },
             },
-        ]
-    }
-    project_file = os.path.join(project_dir, 'project.conf')
-    _yaml.roundtrip_dump(project, project_file)
+        ],
+        {alias: 'http://www.example.com'},
+    )
 
     result = cli.run(project=project_dir, args=['source', 'fetch', element_name])
     result.assert_success()
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
 def test_mirror_from_includes(cli, tmpdir, datafiles, kind):
-    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
+    project_dir = str(datafiles)
+    bin_files_path = os.path.join(project_dir, 'files', 'bin-files', 'usr')
     upstream_repodir = os.path.join(str(tmpdir), 'upstream')
     mirror_repodir = os.path.join(str(tmpdir), 'mirror')
-    project_dir = os.path.join(str(tmpdir), 'project')
-    os.makedirs(project_dir)
     element_dir = os.path.join(project_dir, 'elements')
 
     # Create repo objects of the upstream and mirror
@@ -178,7 +181,6 @@ def test_mirror_from_includes(cli, tmpdir, datafiles, kind):
     element['sources'][0]['url'] = aliased_repo
     full_mirror = mirror_repo.source_config()['url']
     mirror_map, _ = os.path.split(full_mirror)
-    os.makedirs(element_dir)
     _yaml.roundtrip_dump(element, element_path)
 
     config_project_dir = str(tmpdir.join('config'))
@@ -202,18 +204,11 @@ def test_mirror_from_includes(cli, tmpdir, datafiles, kind):
                       config_project_dir,
                       os.path.join(element_dir, 'config.bst'))
 
-    project = {
-        'name': 'test',
-        'element-path': 'elements',
-        'aliases': {
-            alias: upstream_map + "/"
-        },
-        '(@)': [
-            'config.bst:mirrors.yml'
-        ]
-    }
-    project_file = os.path.join(project_dir, 'project.conf')
-    _yaml.roundtrip_dump(project, project_file)
+    _set_project_includes_and_aliases(
+        project_dir,
+        ['config.bst:mirrors.yml'],
+        {alias: upstream_map + '/'},
+    )
 
     # Now make the upstream unavailable.
     os.rename(upstream_repo.repo, '{}.bak'.format(upstream_repo.repo))
@@ -222,13 +217,11 @@ def test_mirror_from_includes(cli, tmpdir, datafiles, kind):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
 def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind):
-    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
+    project_dir = str(datafiles)
+    bin_files_path = os.path.join(project_dir, 'files', 'bin-files', 'usr')
     upstream_repodir = os.path.join(str(tmpdir), 'upstream')
     mirror_repodir = os.path.join(str(tmpdir), 'mirror')
-    project_dir = os.path.join(str(tmpdir), 'project')
-    os.makedirs(project_dir)
     element_dir = os.path.join(project_dir, 'elements')
 
     # Create repo objects of the upstream and mirror
@@ -251,7 +244,6 @@ def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind):
     element['sources'][0]['url'] = aliased_repo
     full_mirror = mirror_repo.source_config()['url']
     mirror_map, _ = os.path.split(full_mirror)
-    os.makedirs(element_dir)
     _yaml.roundtrip_dump(element, element_path)
 
     config_project_dir = str(tmpdir.join('config'))
@@ -275,18 +267,11 @@ def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind):
                       config_project_dir,
                       os.path.join(element_dir, 'config.bst'))
 
-    project = {
-        'name': 'test',
-        'element-path': 'elements',
-        'aliases': {
-            alias: upstream_map + "/"
-        },
-        '(@)': [
-            'config.bst:mirrors.yml'
-        ]
-    }
-    project_file = os.path.join(project_dir, 'project.conf')
-    _yaml.roundtrip_dump(project, project_file)
+    _set_project_includes_and_aliases(
+        project_dir,
+        ['config.bst:mirrors.yml'],
+        {alias: upstream_map + '/'}
+    )
 
     # Now make the upstream unavailable.
     os.rename(upstream_repo.repo, '{}.bak'.format(upstream_repo.repo))
@@ -299,14 +284,12 @@ def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
 def test_mirror_track_upstream_present(cli, tmpdir, datafiles, kind):
-    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
-    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
+    project_dir = str(datafiles)
+    bin_files_path = os.path.join(project_dir, 'files', 'bin-files', 'usr')
+    dev_files_path = os.path.join(project_dir, 'files', 'dev-files', 'usr')
     upstream_repodir = os.path.join(str(tmpdir), 'upstream')
     mirror_repodir = os.path.join(str(tmpdir), 'mirror')
-    project_dir = os.path.join(str(tmpdir), 'project')
-    os.makedirs(project_dir)
     element_dir = os.path.join(project_dir, 'elements')
 
     # Create repo objects of the upstream and mirror
@@ -331,26 +314,20 @@ def test_mirror_track_upstream_present(cli, tmpdir, datafiles, kind):
     element['sources'][0]['url'] = aliased_repo
     full_mirror = mirror_repo.source_config()['url']
     mirror_map, _ = os.path.split(full_mirror)
-    os.makedirs(element_dir)
     _yaml.roundtrip_dump(element, element_path)
 
-    project = {
-        'name': 'test',
-        'element-path': 'elements',
-        'aliases': {
-            alias: upstream_map + "/"
-        },
-        'mirrors': [
+    _set_project_mirrors_and_aliases(
+        project_dir,
+        [
             {
                 'name': 'middle-earth',
                 'aliases': {
-                    alias: [mirror_map + "/"],
+                    alias: [mirror_map + '/'],
                 },
             },
-        ]
-    }
-    project_file = os.path.join(project_dir, 'project.conf')
-    _yaml.roundtrip_dump(project, project_file)
+        ],
+        {alias: upstream_map + '/'},
+    )
 
     result = cli.run(project=project_dir, args=['source', 'track', element_name])
     result.assert_success()
@@ -363,14 +340,12 @@ def test_mirror_track_upstream_present(cli, tmpdir, datafiles, kind):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
 def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind):
-    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
-    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
+    project_dir = str(datafiles)
+    bin_files_path = os.path.join(project_dir, 'files', 'bin-files', 'usr')
+    dev_files_path = os.path.join(project_dir, 'files', 'dev-files', 'usr')
     upstream_repodir = os.path.join(str(tmpdir), 'upstream')
     mirror_repodir = os.path.join(str(tmpdir), 'mirror')
-    project_dir = os.path.join(str(tmpdir), 'project')
-    os.makedirs(project_dir)
     element_dir = os.path.join(project_dir, 'elements')
 
     # Create repo objects of the upstream and mirror
@@ -396,26 +371,20 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind):
     element['sources'][0]['url'] = aliased_repo
     full_mirror = mirror_repo.source_config()['url']
     mirror_map, _ = os.path.split(full_mirror)
-    os.makedirs(element_dir)
     _yaml.roundtrip_dump(element, element_path)
 
-    project = {
-        'name': 'test',
-        'element-path': 'elements',
-        'aliases': {
-            alias: 'http://www.example.com/'
-        },
-        'mirrors': [
+    _set_project_mirrors_and_aliases(
+        project_dir,
+        [
             {
                 'name': 'middle-earth',
                 'aliases': {
-                    alias: [mirror_map + "/"],
+                    alias: [mirror_map + '/'],
                 },
             },
-        ]
-    }
-    project_file = os.path.join(project_dir, 'project.conf')
-    _yaml.roundtrip_dump(project, project_file)
+        ],
+        {alias: 'http://www.example.com'},
+    )
 
     result = cli.run(project=project_dir, args=['source', 'track', element_name])
     result.assert_success()
diff --git a/src/buildstream/testing/_sourcetests/source_determinism.py b/src/buildstream/testing/_sourcetests/source_determinism.py
index d46d38b33..fc0e4618c 100644
--- a/src/buildstream/testing/_sourcetests/source_determinism.py
+++ b/src/buildstream/testing/_sourcetests/source_determinism.py
@@ -24,8 +24,9 @@ import pytest
 
 from buildstream import _yaml
 from .._utils.site import HAVE_SANDBOX
-from .. import create_repo, ALL_REPO_KINDS
+from .. import create_repo
 from .. import cli  # pylint: disable=unused-import
+from .utils import kind  # pylint: disable=unused-import
 
 # Project directory
 TOP_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -48,7 +49,6 @@ def create_test_directory(*path, mode=0o644):
 
 @pytest.mark.integration
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("kind", [*ALL_REPO_KINDS])
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 @pytest.mark.skipif(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox, Must Fix')
 def test_deterministic_source_umask(cli, tmpdir, datafiles, kind):
diff --git a/src/buildstream/testing/_sourcetests/track.py b/src/buildstream/testing/_sourcetests/track.py
index 01a39951f..48856b351 100644
--- a/src/buildstream/testing/_sourcetests/track.py
+++ b/src/buildstream/testing/_sourcetests/track.py
@@ -24,9 +24,12 @@ import pytest
 
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain
-from .._utils import generate_junction, configure_project
-from .. import create_repo, ALL_REPO_KINDS
+from .._utils import generate_junction
+from .. import create_repo
 from .. import cli  # pylint: disable=unused-import
+from .utils import update_project_configuration
+from .utils import kind  # pylint: disable=unused-import
+
 
 # Project directory
 TOP_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -47,15 +50,14 @@ def generate_element(repo, element_path, dep_name=None):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs'])
 def test_track(cli, tmpdir, datafiles, ref_storage, kind):
     project = str(datafiles)
     dev_files_path = os.path.join(project, 'files', 'dev-files')
     element_path = os.path.join(project, 'elements')
     element_name = 'track-test-{}.bst'.format(kind)
 
-    configure_project(project, {
+    update_project_configuration(project, {
         'ref-storage': ref_storage
     })
 
@@ -107,8 +109,7 @@ def test_track(cli, tmpdir, datafiles, ref_storage, kind):
 #    please refactor that aspect into another test.
 #
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("amount", [(1), (10)])
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+@pytest.mark.parametrize("amount", [1, 10])
 def test_track_recurse(cli, tmpdir, datafiles, kind, amount):
     project = str(datafiles)
     dev_files_path = os.path.join(project, 'files', 'dev-files')
@@ -172,7 +173,6 @@ def test_track_recurse(cli, tmpdir, datafiles, kind, amount):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
 def test_track_recurse_except(cli, tmpdir, datafiles, kind):
     project = str(datafiles)
     dev_files_path = os.path.join(project, 'files', 'dev-files')
@@ -218,8 +218,7 @@ def test_track_recurse_except(cli, tmpdir, datafiles, kind):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs'])
 def test_cross_junction(cli, tmpdir, datafiles, ref_storage, kind):
     project = str(datafiles)
     subproject_path = os.path.join(project, 'files', 'sub-project')
@@ -228,7 +227,7 @@ def test_cross_junction(cli, tmpdir, datafiles, ref_storage, kind):
     repo_element_path = os.path.join(subproject_path, 'elements',
                                      'import-etc-repo.bst')
 
-    configure_project(project, {
+    update_project_configuration(project, {
         'ref-storage': ref_storage
     })
 
@@ -261,15 +260,14 @@ def test_cross_junction(cli, tmpdir, datafiles, ref_storage, kind):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs'])
 def test_track_include(cli, tmpdir, datafiles, ref_storage, kind):
     project = str(datafiles)
     dev_files_path = os.path.join(project, 'files', 'dev-files')
     element_path = os.path.join(project, 'elements')
     element_name = 'track-test-{}.bst'.format(kind)
 
-    configure_project(project, {
+    update_project_configuration(project, {
         'ref-storage': ref_storage
     })
 
@@ -331,8 +329,7 @@ def test_track_include(cli, tmpdir, datafiles, ref_storage, kind):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs'])
 def test_track_include_junction(cli, tmpdir, datafiles, ref_storage, kind):
     project = str(datafiles)
     dev_files_path = os.path.join(project, 'files', 'dev-files')
@@ -342,7 +339,7 @@ def test_track_include_junction(cli, tmpdir, datafiles, ref_storage, kind):
     sub_element_path = os.path.join(subproject_path, 'elements')
     junction_path = os.path.join(element_path, 'junction.bst')
 
-    configure_project(project, {
+    update_project_configuration(project, {
         'ref-storage': ref_storage
     })
 
@@ -400,15 +397,14 @@ def test_track_include_junction(cli, tmpdir, datafiles, ref_storage, kind):
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+@pytest.mark.parametrize("ref_storage", ['inline', 'project.refs'])
 def test_track_junction_included(cli, tmpdir, datafiles, ref_storage, kind):
     project = str(datafiles)
     element_path = os.path.join(project, 'elements')
     subproject_path = os.path.join(project, 'files', 'sub-project')
     junction_path = os.path.join(element_path, 'junction.bst')
 
-    configure_project(project, {
+    update_project_configuration(project, {
         'ref-storage': ref_storage,
         '(@)': ['junction.bst:test.yml']
     })
diff --git a/src/buildstream/testing/_sourcetests/track_cross_junction.py b/src/buildstream/testing/_sourcetests/track_cross_junction.py
index 31443bdf9..550f57faf 100644
--- a/src/buildstream/testing/_sourcetests/track_cross_junction.py
+++ b/src/buildstream/testing/_sourcetests/track_cross_junction.py
@@ -20,12 +20,15 @@
 # pylint: disable=redefined-outer-name
 
 import os
+
 import pytest
 
 from buildstream import _yaml
 from .._utils import generate_junction
 from .. import create_repo, ALL_REPO_KINDS
 from .. import cli  # pylint: disable=unused-import
+from .utils import add_plugins_conf
+
 
 # Project directory
 TOP_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -62,7 +65,7 @@ def generate_import_element(tmpdir, kind, project, name):
     return element_name
 
 
-def generate_project(tmpdir, name, config=None):
+def generate_project(tmpdir, name, kind, config=None):
     if config is None:
         config = {}
 
@@ -76,6 +79,7 @@ def generate_project(tmpdir, name, config=None):
     }
     project_conf.update(config)
     _yaml.roundtrip_dump(project_conf, os.path.join(subproject_path, 'project.conf'))
+    add_plugins_conf(subproject_path, kind)
 
     return project_name, subproject_path
 
@@ -101,14 +105,14 @@ def generate_cross_element(project, subproject_name, import_name):
                                  }])
 
 
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+@pytest.mark.parametrize("kind", ALL_REPO_KINDS.keys())
 def test_cross_junction_multiple_projects(cli, tmpdir, kind):
     tmpdir = tmpdir.join(kind)
 
     # Generate 3 projects: main, a, b
-    _, project = generate_project(tmpdir, 'main', {'ref-storage': 'project.refs'})
-    project_a, project_a_path = generate_project(tmpdir, 'a')
-    project_b, project_b_path = generate_project(tmpdir, 'b')
+    _, project = generate_project(tmpdir, 'main', kind, {'ref-storage': 'project.refs'})
+    project_a, project_a_path = generate_project(tmpdir, 'a', kind)
+    project_b, project_b_path = generate_project(tmpdir, 'b', kind)
 
     # Generate an element with a trackable source for each project
     element_a = generate_import_element(tmpdir, kind, project_a_path, 'a')
@@ -152,12 +156,12 @@ def test_cross_junction_multiple_projects(cli, tmpdir, kind):
     assert set(result.get_tracked_elements()) == set(expected)
 
 
-@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+@pytest.mark.parametrize("kind", ALL_REPO_KINDS.keys())
 def test_track_exceptions(cli, tmpdir, kind):
     tmpdir = tmpdir.join(kind)
 
-    _, project = generate_project(tmpdir, 'main', {'ref-storage': 'project.refs'})
-    project_a, project_a_path = generate_project(tmpdir, 'a')
+    _, project = generate_project(tmpdir, 'main', kind, {'ref-storage': 'project.refs'})
+    project_a, project_a_path = generate_project(tmpdir, 'a', kind)
 
     element_a = generate_import_element(tmpdir, kind, project_a_path, 'a')
     element_b = generate_import_element(tmpdir, kind, project_a_path, 'b')
diff --git a/src/buildstream/testing/_sourcetests/utils.py b/src/buildstream/testing/_sourcetests/utils.py
new file mode 100644
index 000000000..a0e65b4f4
--- /dev/null
+++ b/src/buildstream/testing/_sourcetests/utils.py
@@ -0,0 +1,104 @@
+#
+#  Copyright (C) 2018 Codethink Limited
+#  Copyright (C) 2019 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#   Authors:
+#       Benjamin Schubert (bschubert15@bloomberg.net)
+#
+
+import os
+
+# To make use of these test utilities it is necessary to have pytest
+# available. However, we don't want to have a hard dependency on
+# pytest.
+try:
+    import pytest
+except ImportError:
+    module_name = globals()['__name__']
+    msg = "Could not import pytest:\n" \
+          "To use the {} module, you must have pytest installed.".format(module_name)
+    raise ImportError(msg)
+
+from buildstream import _yaml
+from .. import ALL_REPO_KINDS
+
+
+# kind()
+#
+# Pytest fixture to get all the registered source plugins.
+#
+# This assumes the usage of the standard `project` and will automatically
+# register the plugin in the project configuration, in addition to its junction
+# configuration
+#
+# Yields:
+#   the plugin kind name
+#
+@pytest.fixture(params=ALL_REPO_KINDS.keys())
+def kind(request, datafiles):
+    # Register plugins both on the toplevel project and on its junctions
+    for project_dir in [str(datafiles), os.path.join(str(datafiles), "files", "sub-project")]:
+        add_plugins_conf(project_dir, request.param)
+
+    yield request.param
+
+
+# add_plugins_conf()
+#
+# Add the given plugin to the configuration of the given project.
+#
+# Args:
+#   project (str): path to the project on which to register the plugin
+#   plugin_kind (str): name of the plugin kind to register
+#
+def add_plugins_conf(project, plugin_kind):
+    _scaffolder, plugin_package = ALL_REPO_KINDS[plugin_kind]
+
+    project_conf_file = os.path.join(project, "project.conf")
+    project_conf = _yaml.roundtrip_load(project_conf_file)
+
+    if plugin_package is not None:
+        project_conf["plugins"] = [
+            {
+                "origin": "pip",
+                "package-name": plugin_package,
+                "sources": {
+                    plugin_kind: 0,
+                },
+            },
+        ]
+
+    _yaml.roundtrip_dump(project_conf, project_conf_file)
+
+
+# update_project_configuration()
+#
+# Update the project configuration with the given updated configuration.
+#
+# Note: This does a simple `dict.update()` call, which will not merge recursively
+#       but will replace every defined key.
+#
+# Args:
+#   project_path (str): the path to the root of the project
+#   updated_configuration (dict): configuration to merge into the existing one
+#
+def update_project_configuration(project_path, updated_configuration):
+    project_conf_path = os.path.join(project_path, 'project.conf')
+    project_conf = _yaml.roundtrip_load(project_conf_path)
+
+    project_conf.update(updated_configuration)
+
+    _yaml.roundtrip_dump(project_conf, project_conf_path)
diff --git a/src/buildstream/testing/_sourcetests/workspace.py b/src/buildstream/testing/_sourcetests/workspace.py
index 5ceab5108..dd7977e76 100644
--- a/src/buildstream/testing/_sourcetests/workspace.py
+++ b/src/buildstream/testing/_sourcetests/workspace.py
@@ -24,8 +24,9 @@ import shutil
 import pytest
 
 from buildstream import _yaml
-from .. import create_repo, ALL_REPO_KINDS
+from .. import create_repo
 from .. import cli  # pylint: disable=unused-import
+from .utils import kind  # pylint: disable=unused-import
 
 # Project directory
 TOP_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -155,6 +156,5 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir
 
 
 @pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.parametrize("kind", ALL_REPO_KINDS)
 def test_open(cli, tmpdir, datafiles, kind):
     open_workspace(cli, tmpdir, datafiles, kind, False)
diff --git a/src/buildstream/testing/_utils/__init__.py b/src/buildstream/testing/_utils/__init__.py
index 575226e22..5938e6a5e 100644
--- a/src/buildstream/testing/_utils/__init__.py
+++ b/src/buildstream/testing/_utils/__init__.py
@@ -1,10 +1 @@
-import os
-
-from buildstream import _yaml
 from .junction import generate_junction
-
-
-def configure_project(path, config):
-    config['name'] = 'test'
-    config['element-path'] = 'elements'
-    _yaml.roundtrip_dump(config, os.path.join(path, 'project.conf'))
diff --git a/tests/conftest.py b/tests/conftest.py
index 77f8666a6..d6b0b02e0 100755
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -104,10 +104,10 @@ def remote_services(request):
 #################################################
 # Setup for templated source tests              #
 #################################################
-register_repo_kind('git', Git)
-register_repo_kind('bzr', Bzr)
-register_repo_kind('tar', Tar)
-register_repo_kind('zip', Zip)
+register_repo_kind('git', Git, None)
+register_repo_kind('bzr', Bzr, None)
+register_repo_kind('tar', Tar, None)
+register_repo_kind('zip', Zip, None)
 
 
 # This hook enables pytest to collect the templated source tests from