Merge branch 'bschubert/standardized-tests' into 'master'

introduce cross-repo standardized source tests

See merge request BuildStream/buildstream!1757

27 files changed, 62 insertions, 351 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 002146f9b..a5133a86a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -29,7 +29,7 @@ variables:
   INTEGRATION_CACHE: "${CI_PROJECT_DIR}/cache/integration-cache"
   PYTEST_ARGS: "--color=yes --integration -n 2"
   TEST_COMMAND: "tox -- ${PYTEST_ARGS}"
-  EXTERNAL_TESTS_COMMAND: "tox -e py35-external,py36-external,py37-external,py38-external -- ${PYTEST_ARGS}"
+  PLUGINS_TESTS_COMMAND: "tox -e py35-plugins,py36-plugins,py37-plugins -- ${PYTEST_ARGS}"
   COVERAGE_PREFIX: "${CI_JOB_NAME}."
 
 
@@ -55,7 +55,7 @@ variables:
 
   # Run the tests as a simple user to test for permission issues
   - su buildstream -c "${TEST_COMMAND}"
-  - su buildstream -c "${EXTERNAL_TESTS_COMMAND}"
+  - su buildstream -c "${PLUGINS_TESTS_COMMAND}"
 
   after_script:
   except:
@@ -92,16 +92,23 @@ tests-python-3.8-buster:
   image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-python:3.8-buster-${DOCKER_IMAGE_VERSION}
   <<: *tests
   variables:
+    # This particular testsuite image has both Python 3.7 and Python 3.8 so we
+    # need to explicitly force the 3.8 environment.
+    # Once Python 3.8 is available in distros, we should switch to such an
+
     # Our testsuite has issues with coverage on Python 3.8 so disable coverage
     # in the meantime. For more details, see
     # https://gitlab.com/BuildStream/buildstream/issues/1173.
     TEST_COMMAND: "tox -e py38-nocover -- ${PYTEST_ARGS}"
+    PLUGINS_TESTS_COMMAND: "tox -e py38-plugins-nocover -- ${PYTEST_ARGS}"
 
-    # This particular testsuite image has both Python 3.7 and Python 3.8 so we
-    # need to explicitly force the 3.8 environment.
-    # Once Python 3.8 is available in distros, we should switch to such an
-    # image, and remove the following override.
-    EXTERNAL_TESTS_COMMAND: "tox -e py38-external -- ${PYTEST_ARGS}"
+# Test the master version of some external plugins
+tests-plugins-master:
+  <<: *tests
+  allow_failure: true
+
+  variables:
+    BST_PLUGINS_EXPERIMENTAL_VERSION: master
 
 overnight-fedora-30-aarch64:
   image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:aarch64-30-${DOCKER_IMAGE_VERSION}
@@ -152,7 +159,7 @@ tests-userchroot:
 
     # Run the tests as a simple user to test for permission issues
     - su buildstream -c "umask 002 && ${TEST_COMMAND}"
-    - su buildstream -c "umask 002 && ${EXTERNAL_TESTS_COMMAND}"
+    - su buildstream -c "umask 002 && ${PLUGINS_TESTS_COMMAND}"
 
 tests-fedora-missing-deps:
   # Ensure that tests behave nicely while missing bwrap and ostree
@@ -170,6 +177,8 @@ tests-fedora-missing-deps:
     - chown -R buildstream:buildstream .
 
     - ${TEST_COMMAND}
+    - ${PLUGINS_TESTS_COMMAND}
+
 
 tests-fedora-update-deps:
   # Check if the tests pass after updating requirements to their latest
@@ -186,6 +195,7 @@ tests-fedora-update-deps:
     - cat requirements/*.txt
 
     - su buildstream -c "${TEST_COMMAND}"
+    - su buildstream -c "${PLUGINS_TESTS_COMMAND}"
 
 tests-remote-execution:
   allow_failure: true
diff --git a/NEWS b/NEWS
index abd53d903..602b14b51 100644
--- a/NEWS
+++ b/NEWS
@@ -9,6 +9,17 @@ CLI
     of `buildable` for the state of junction elements, as they can't
     be built.
 
+API
+---
+
+  o External plugins can now register a `buildstream.tests.source_plugins` entrypoint.
+    The entry point can have an arbitrary name, but its value should point to a module
+    containing a `register_sources()` method.
+    This method should call `register_repo_kind` for all sources you want to have
+    tested in BuildStream.
+    Plugins authors that do this and believe BuildStream should be testing that
+    part of their plugins should open an issue on BuildStream.
+
 
 ==================
 buildstream 1.91.3
diff --git a/tests/conftest.py b/tests/conftest.py
index cfca4ad06..8d33fa024 100755
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -21,6 +21,8 @@
 #
 import os
 import multiprocessing
+
+import pkg_resources
 import pytest
 
 from buildstream.testing import register_repo_kind, sourcetests_collection_hook
@@ -46,7 +48,7 @@ from tests.testutils.repo.zip import Zip
 #################################################
 def pytest_addoption(parser):
     parser.addoption("--integration", action="store_true", default=False, help="Run integration tests")
-
+    parser.addoption("--plugins", action="store_true", default=False, help="Run only plugins tests")
     parser.addoption("--remote-execution", action="store_true", default=False, help="Run remote-execution tests only")
 
 
@@ -66,6 +68,11 @@ def pytest_runtest_setup(item):
         if item.get_closest_marker("remoteexecution"):
             pytest.skip("skipping remote-execution test")
 
+    # With --plugins only run plugins tests
+    if item.config.getvalue("plugins"):
+        if not item.get_closest_marker("generic_source_test"):
+            pytest.skip("Skipping not generic source test")
+
 
 #################################################
 #           remote_services fixture             #
@@ -112,6 +119,12 @@ register_repo_kind("zip", Zip, None)
 # This hook enables pytest to collect the templated source tests from
 # buildstream.testing
 def pytest_sessionstart(session):
+    if session.config.getvalue("plugins"):
+        # Enable all plugins that implement the 'buildstream.tests.source_plugins' hook
+        for entrypoint in pkg_resources.iter_entry_points("buildstream.tests.source_plugins"):
+            module = entrypoint.load()
+            module.register_sources()
+
     sourcetests_collection_hook(session)
 
 
diff --git a/tests/external_plugins.py b/tests/external_plugins.py
deleted file mode 100644
index 9a5e04501..000000000
--- a/tests/external_plugins.py
+++ /dev/null
@@ -1,84 +0,0 @@
-import glob
-import os
-import subprocess
-import sys
-
-import pytest
-
-
-# An ExternalPluginRepo represents a git repository containing a plugin
-# with tests that we wish to use as part of our test suite.
-#
-# Args:
-#    name (str): The name of the repository. This is used for impormational purposes
-#    url (str): The location from which therepository can be cloned
-#    ref (str): A known git ref that we wish to test against
-#    test_match_patterns (list[str]): A list of shell style globs which may be
-#          used to specify a subset of test files from the repository to run.
-#          These must be specified relative to the root of the repository.
-class ExternalPluginRepo:
-    def __init__(self, name, url, ref, test_match_patterns=None):
-        self.name = name
-        self.url = url
-        self.ref = ref
-
-        if test_match_patterns is None:
-            test_match_patterns = ["tests"]
-
-        self._test_match_patterns = test_match_patterns
-        self._clone_location = None
-
-    def clone(self, location):
-        self._clone_location = os.path.join(location, self.name)
-        subprocess.run(
-            ["git", "clone", "--single-branch", "--branch", self.ref, "--depth", "1", self.url, self._clone_location,],
-            check=True,
-        )
-        return self._clone_location
-
-    def install(self):
-        subprocess.run(["pip3", "install", self._clone_location], check=True)
-
-    def test(self, args):
-        test_files = self._match_test_patterns()
-        return pytest.main(args + test_files)
-
-    def _match_test_patterns(self):
-        match_list = []
-        for pattern in self._test_match_patterns:
-            abs_pattern = os.path.join(self._clone_location, pattern)
-            print("matching pattern: ", abs_pattern)
-            matches = glob.glob(abs_pattern)
-            match_list.extend(matches)
-
-        if not match_list:
-            raise ValueError("No matches found for patterns {}".format(self._test_match_patterns))
-        return match_list
-
-
-def run_repo_tests(repo, tempdir, args):
-    print("Cloning repo {} to {}...".format(repo.name, tempdir))
-    repo.clone(tempdir)
-
-    print("Installing {}...".format(repo.name))
-    repo.install()
-
-    print("Testing {}...".format(repo.name))
-    return repo.test(args)
-
-
-if __name__ == "__main__":
-    # Args:
-    #    tmpdir: The directory in which this script will clone external
-    #            repositories and use pytest's tmpdir.
-    #    pytest_args: any remaining arguments to this script will be passed
-    #                 directly to it's pytest invocations
-    _, tmpdir, *pytest_args = sys.argv
-
-    ALL_EXTERNAL_PLUGINS = []
-
-    exit_code = 0
-    for plugin in ALL_EXTERNAL_PLUGINS:
-        exit_code = run_repo_tests(plugin, tmpdir, pytest_args)
-        if exit_code != 0:
-            sys.exit(exit_code)
diff --git a/tests/format/variables.py b/tests/format/variables.py
index 35b105f66..1d8b5aff9 100644
--- a/tests/format/variables.py
+++ b/tests/format/variables.py
@@ -27,25 +27,7 @@ def print_warning(msg):
 #  Test proper loading of some default commands from plugins  #
 ###############################################################
 @pytest.mark.parametrize(
-    "target,varname,expected",
-    [
-        ("autotools.bst", "make-install", 'make -j1 DESTDIR="/buildstream-install" install'),
-        (
-            "cmake.bst",
-            "cmake",
-            'cmake -B_builddir -H"." -G"Unix Makefiles" '
-            + '-DCMAKE_INSTALL_PREFIX:PATH="/usr" \\\n'
-            + '-DCMAKE_INSTALL_LIBDIR:PATH="lib"',
-        ),
-        (
-            "distutils.bst",
-            "python-install",
-            'python3 ./setup.py install --prefix "/usr" \\\n' + '--root "/buildstream-install"',
-        ),
-        ("makemaker.bst", "configure", "perl Makefile.PL PREFIX=/buildstream-install/usr"),
-        ("modulebuild.bst", "configure", 'perl Build.PL --prefix "/buildstream-install/usr"'),
-        ("qmake.bst", "make-install", 'make -j1 INSTALL_ROOT="/buildstream-install" install'),
-    ],
+    "target,varname,expected", [("autotools.bst", "make-install", 'make -j1 DESTDIR="/buildstream-install" install')],
 )
 @pytest.mark.datafiles(os.path.join(DATA_DIR, "defaults"))
 def test_defaults(cli, datafiles, target, varname, expected):
@@ -60,25 +42,7 @@ def test_defaults(cli, datafiles, target, varname, expected):
 #  Test overriding of variables to produce different commands  #
 ################################################################
 @pytest.mark.parametrize(
-    "target,varname,expected",
-    [
-        ("autotools.bst", "make-install", 'make -j1 DESTDIR="/custom/install/root" install'),
-        (
-            "cmake.bst",
-            "cmake",
-            'cmake -B_builddir -H"." -G"Ninja" '
-            + '-DCMAKE_INSTALL_PREFIX:PATH="/opt" \\\n'
-            + '-DCMAKE_INSTALL_LIBDIR:PATH="lib"',
-        ),
-        (
-            "distutils.bst",
-            "python-install",
-            'python3 ./setup.py install --prefix "/opt" \\\n' + '--root "/custom/install/root"',
-        ),
-        ("makemaker.bst", "configure", "perl Makefile.PL PREFIX=/custom/install/root/opt"),
-        ("modulebuild.bst", "configure", 'perl Build.PL --prefix "/custom/install/root/opt"'),
-        ("qmake.bst", "make-install", 'make -j1 INSTALL_ROOT="/custom/install/root" install'),
-    ],
+    "target,varname,expected", [("autotools.bst", "make-install", 'make -j1 DESTDIR="/custom/install/root" install')],
 )
 @pytest.mark.datafiles(os.path.join(DATA_DIR, "overrides"))
 def test_overrides(cli, datafiles, target, varname, expected):
diff --git a/tests/format/variables/defaults/cmake.bst b/tests/format/variables/defaults/cmake.bst
deleted file mode 100644
index 4e7db662c..000000000
--- a/tests/format/variables/defaults/cmake.bst
+++ /dev/null
@@ -1,2 +0,0 @@
-kind: cmake
-description: Some kinda cmake element
diff --git a/tests/format/variables/defaults/distutils.bst b/tests/format/variables/defaults/distutils.bst
deleted file mode 100644
index 2354bea39..000000000
--- a/tests/format/variables/defaults/distutils.bst
+++ /dev/null
@@ -1,2 +0,0 @@
-kind: distutils
-description: Some kinda distutils element
diff --git a/tests/format/variables/defaults/makemaker.bst b/tests/format/variables/defaults/makemaker.bst
deleted file mode 100644
index 2205cece1..000000000
--- a/tests/format/variables/defaults/makemaker.bst
+++ /dev/null
@@ -1,2 +0,0 @@
-kind: makemaker
-description: Some kinda makemaker element
diff --git a/tests/format/variables/defaults/modulebuild.bst b/tests/format/variables/defaults/modulebuild.bst
deleted file mode 100644
index 5b5e9b91a..000000000
--- a/tests/format/variables/defaults/modulebuild.bst
+++ /dev/null
@@ -1,2 +0,0 @@
-kind: modulebuild
-description: Some kinda modulebuild element
diff --git a/tests/format/variables/defaults/project.conf b/tests/format/variables/defaults/project.conf
index 029f07c3a..2027cc27a 100644
--- a/tests/format/variables/defaults/project.conf
+++ b/tests/format/variables/defaults/project.conf
@@ -1,13 +1,3 @@
 # Basic project configuration that doesnt override anything
 #
 name: pony
-
-plugins:
-- origin: pip
-  package-name: bst-plugins-experimental
-  elements:
-    cmake: 0
-    distutils: 0
-    makemaker: 0
-    modulebuild: 0
-    qmake: 0
diff --git a/tests/format/variables/defaults/qmake.bst b/tests/format/variables/defaults/qmake.bst
deleted file mode 100644
index 53209760d..000000000
--- a/tests/format/variables/defaults/qmake.bst
+++ /dev/null
@@ -1,2 +0,0 @@
-kind: qmake
-description: Some kinda qmake element
diff --git a/tests/format/variables/overrides/cmake.bst b/tests/format/variables/overrides/cmake.bst
deleted file mode 100644
index ceea2bb12..000000000
--- a/tests/format/variables/overrides/cmake.bst
+++ /dev/null
@@ -1,7 +0,0 @@
-kind: cmake
-description: Some kinda cmake element
-
-variables:
-  generator: Ninja
-  install-root: /custom/install/root
-  prefix: /opt
diff --git a/tests/format/variables/overrides/distutils.bst b/tests/format/variables/overrides/distutils.bst
deleted file mode 100644
index 624f775a5..000000000
--- a/tests/format/variables/overrides/distutils.bst
+++ /dev/null
@@ -1,6 +0,0 @@
-kind: distutils
-description: Some kinda distutils element
-
-variables:
-  install-root: /custom/install/root
-  prefix: /opt
diff --git a/tests/format/variables/overrides/makemaker.bst b/tests/format/variables/overrides/makemaker.bst
deleted file mode 100644
index faa5f365f..000000000
--- a/tests/format/variables/overrides/makemaker.bst
+++ /dev/null
@@ -1,6 +0,0 @@
-kind: makemaker
-description: Some kinda makemaker element
-
-variables:
-  install-root: /custom/install/root
-  prefix: /opt
diff --git a/tests/format/variables/overrides/modulebuild.bst b/tests/format/variables/overrides/modulebuild.bst
deleted file mode 100644
index db50057e2..000000000
--- a/tests/format/variables/overrides/modulebuild.bst
+++ /dev/null
@@ -1,6 +0,0 @@
-kind: modulebuild
-description: Some kinda modulebuild element
-
-variables:
-  install-root: /custom/install/root
-  prefix: /opt
diff --git a/tests/format/variables/overrides/project.conf b/tests/format/variables/overrides/project.conf
index e4854a646..2027cc27a 100644
--- a/tests/format/variables/overrides/project.conf
+++ b/tests/format/variables/overrides/project.conf
@@ -1,12 +1,3 @@
 # Basic project configuration that doesnt override anything
 #
 name: pony
-plugins:
-- origin: pip
-  package-name: bst-plugins-experimental
-  elements:
-    cmake: 0
-    distutils: 0
-    makemaker: 0
-    modulebuild: 0
-    qmake: 0
diff --git a/tests/format/variables/overrides/qmake.bst b/tests/format/variables/overrides/qmake.bst
deleted file mode 100644
index 5dff2c944..000000000
--- a/tests/format/variables/overrides/qmake.bst
+++ /dev/null
@@ -1,6 +0,0 @@
-kind: qmake
-description: Some kinda qmake element
-
-variables:
-  install-root: /custom/install/root
-  prefix: /opt
diff --git a/tests/integration/cmake.py b/tests/integration/cmake.py
deleted file mode 100644
index db0f07531..000000000
--- a/tests/integration/cmake.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# Pylint doesn't play well with fixtures and dependency injection from pytest
-# pylint: disable=redefined-outer-name
-
-import os
-import pytest
-
-from buildstream.testing import cli_integration as cli  # pylint: disable=unused-import
-from buildstream.testing.integration import assert_contains
-from buildstream.testing._utils.site import HAVE_SANDBOX
-
-
-pytestmark = pytest.mark.integration
-
-
-DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
-
-
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_cmake_build(cli, datafiles):
-    project = str(datafiles)
-    checkout = os.path.join(cli.directory, "checkout")
-    element_name = "cmake/cmakehello.bst"
-
-    result = cli.run(project=project, args=["build", element_name])
-    assert result.exit_code == 0
-
-    result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
-    assert result.exit_code == 0
-
-    assert_contains(checkout, ["/usr", "/usr/bin", "/usr/bin/hello"])
-
-
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_cmake_confroot_build(cli, datafiles):
-    project = str(datafiles)
-    checkout = os.path.join(cli.directory, "checkout")
-    element_name = "cmake/cmakeconfroothello.bst"
-
-    result = cli.run(project=project, args=["build", element_name])
-    assert result.exit_code == 0
-
-    result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
-    assert result.exit_code == 0
-
-    assert_contains(checkout, ["/usr", "/usr/bin", "/usr/bin/hello"])
-
-
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_cmake_run(cli, datafiles):
-    project = str(datafiles)
-    element_name = "cmake/cmakehello.bst"
-
-    result = cli.run(project=project, args=["build", element_name])
-    assert result.exit_code == 0
-
-    result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"])
-    assert result.exit_code == 0
-
-    assert (
-        result.output
-        == """Hello World!
-This is hello.
-"""
-    )
diff --git a/tests/integration/make.py b/tests/integration/make.py
deleted file mode 100644
index 109777622..000000000
--- a/tests/integration/make.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# Pylint doesn't play well with fixtures and dependency injection from pytest
-# pylint: disable=redefined-outer-name
-
-import os
-import pytest
-
-from buildstream.testing import cli_integration as cli  # pylint: disable=unused-import
-from buildstream.testing.integration import assert_contains
-from buildstream.testing._utils.site import HAVE_SANDBOX
-
-
-pytestmark = pytest.mark.integration
-
-
-DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
-
-
-# Test that a make build 'works' - we use the make sample
-# makehello project for this.
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_make_build(cli, datafiles):
-    project = str(datafiles)
-    checkout = os.path.join(cli.directory, "checkout")
-    element_name = "make/makehello.bst"
-
-    result = cli.run(project=project, args=["build", element_name])
-    assert result.exit_code == 0
-
-    result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
-    assert result.exit_code == 0
-
-    assert_contains(checkout, ["/usr", "/usr/bin", "/usr/bin/hello"])
-
-
-# Test running an executable built with make
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_make_run(cli, datafiles):
-    project = str(datafiles)
-    element_name = "make/makehello.bst"
-
-    result = cli.run(project=project, args=["build", element_name])
-    assert result.exit_code == 0
-
-    result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"])
-    assert result.exit_code == 0
-    assert result.output == "Hello, world\n"
diff --git a/tests/integration/project/elements/cmake/cmakeconfroothello.bst b/tests/integration/project/elements/cmake/cmakeconfroothello.bst
deleted file mode 100644
index cd33dee99..000000000
--- a/tests/integration/project/elements/cmake/cmakeconfroothello.bst
+++ /dev/null
@@ -1,15 +0,0 @@
-kind: cmake
-description: Cmake test
-
-depends:
-  - base.bst
-
-sources:
-  - kind: tar
-    directory: Source
-    url: project_dir:/files/cmakehello.tar.gz
-    ref: 508266f40dbc5875293bd24c4e50a9eb6b88cbacab742033f7b92f8c087b64e5
-
-variables:
-  conf-root: "%{build-root}/Source"
-  command-subdir: build
diff --git a/tests/integration/project/elements/cmake/cmakehello.bst b/tests/integration/project/elements/cmake/cmakehello.bst
deleted file mode 100644
index c5fe496ad..000000000
--- a/tests/integration/project/elements/cmake/cmakehello.bst
+++ /dev/null
@@ -1,10 +0,0 @@
-kind: cmake
-description: Cmake test
-
-depends:
-  - base.bst
-
-sources:
-  - kind: tar
-    url: project_dir:/files/cmakehello.tar.gz
-    ref: 508266f40dbc5875293bd24c4e50a9eb6b88cbacab742033f7b92f8c087b64e5
diff --git a/tests/integration/project/elements/make/makehello.bst b/tests/integration/project/elements/make/makehello.bst
deleted file mode 100644
index 4b5c5ac3b..000000000
--- a/tests/integration/project/elements/make/makehello.bst
+++ /dev/null
@@ -1,10 +0,0 @@
-kind: make
-description: make test
-
-depends:
-- base.bst
-
-sources:
-- kind: tar
-  url: project_dir:/files/makehello.tar.gz
-  ref: fd342a36503a0a0dd37b81ddb4d2b78bd398d912d813339e0de44a6b6c393b8e
diff --git a/tests/integration/project/files/cmakehello.tar.gz b/tests/integration/project/files/cmakehello.tar.gz
deleted file mode 100644
index 54d950575..000000000
--- a/tests/integration/project/files/cmakehello.tar.gz
+++ /dev/null
diff --git a/tests/integration/project/files/makehello.tar.gz b/tests/integration/project/files/makehello.tar.gz
deleted file mode 100644
index d0edcb29c..000000000
--- a/tests/integration/project/files/makehello.tar.gz
+++ /dev/null
diff --git a/tests/sources/ostree/template/project.conf b/tests/sources/ostree/template/project.conf
deleted file mode 100644
index afa0f5475..000000000
--- a/tests/sources/ostree/template/project.conf
+++ /dev/null
@@ -1,2 +0,0 @@
-# Basic project
-name: foo
diff --git a/tests/sources/ostree/template/repofiles/file b/tests/sources/ostree/template/repofiles/file
deleted file mode 100644
index e69de29bb..000000000
--- a/tests/sources/ostree/template/repofiles/file
+++ /dev/null
diff --git a/tox.ini b/tox.ini
index 85697085d..ea45c1acc 100644
--- a/tox.ini
+++ b/tox.ini
@@ -6,6 +6,10 @@ envlist = py{35,36,37,38}
 skip_missing_interpreters = true
 isolated_build = true
 
+# Configuration variables to share accross environments
+[config]
+BST_PLUGINS_EXPERIMENTAL_VERSION = 0.13.0
+
 #
 # Defaults for all environments
 #
@@ -18,19 +22,23 @@ usedevelop =
 
 commands =
     # Running with coverage reporting enabled
-    py{35,36,37,38}-!external-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc {posargs}
-    py{35,36,37,38}-!nocover: mkdir -p .coverage-reports
-    external-!nocover: {envpython} {toxinidir}/tests/external_plugins.py {envtmpdir} --cov=buildstream --cov-append {posargs}
-    py{35,36,37,38}-!external-!nocover: mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname}
-    py{35,36,37,38}-external-!nocover: - mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname}
+    py{35,36,37,38}-!plugins-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc {posargs}
     # Running with coverage reporting disabled
-    py{35,36,37,38}-!external-nocover: pytest --basetemp {envtmpdir} {posargs}
-    external-nocover: {envpython} {toxinidir}/tests/external_plugins.py {envtmpdir} {posargs}
+    py{35,36,37,38}-!plugins-nocover: pytest --basetemp {envtmpdir} {posargs}
+    # Running external plugins tests with coverage reporting enabled
+    py{35,36,37,38}-plugins-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc --plugins {posargs}
+    # Running external plugins tests with coverage disabled
+    py{35,36,37,38}-plugins-nocover: pytest --basetemp {envtmpdir} --plugins {posargs}
+commands_post:
+    py{35,36,37,38}-!nocover: mkdir -p .coverage-reports
+    py{35,36,37,38}-!nocover: mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname}
 deps =
     py{35,36,37,38}: -rrequirements/requirements.txt
     py{35,36,37,38}: -rrequirements/dev-requirements.txt
     py{35,36,37,38}: -rrequirements/plugin-requirements.txt
-    git+https://gitlab.com/BuildStream/bst-plugins-experimental.git@5b004e5850ab0e987c00c681b2c768f6ae02586b
+
+    # Install external plugins for plugin tests
+    py{35,36,37,38}-plugins: git+https://gitlab.com/buildstream/bst-plugins-experimental.git@{env:BST_PLUGINS_EXPERIMENTAL_VERSION:{[config]BST_PLUGINS_EXPERIMENTAL_VERSION}}#egg=bst_plugins_experimental[ostree]
 
     # Only require coverage and pytest-cov when using it
     !nocover: -rrequirements/cov-requirements.txt
@@ -54,6 +62,7 @@ passenv =
     REMOTE_EXECUTION_SERVICE
     SOURCE_CACHE_SERVICE
     SSL_CERT_FILE
+    BST_PLUGINS_EXPERIMENTAL_VERSION
 #
 # These keys are not inherited by any other sections
 #