Allow testing of external plugins via toxphil/external-plugin-testing

2 files changed, 122 insertions, 2 deletions

diff --git a/tests/external_plugins.py b/tests/external_plugins.py
new file mode 100644
index 000000000..84a1abdce
--- /dev/null
+++ b/tests/external_plugins.py
@@ -0,0 +1,93 @@
+import glob
+import os
+import pytest
+import shutil
+import subprocess
+import sys
+import time
+
+
+# An ExternalPluginRepo represents a git repository containing a plugin
+# with tests that we wish to use as part of our test suite.
+#
+# Args:
+#    name (str): The name of the repository. This is used for impormational purposes
+#    url (str): The location from which therepository can be cloned
+#    ref (str): A known git ref that we wish to test against
+#    test_match_patterns (list[str]): A list of shell style globs which may be
+#          used to specify a subset of test files from the repository to run.
+#          These must be specified relative to the root of the repository.
+class ExternalPluginRepo():
+    def __init__(self, name, url, ref, test_match_patterns=["tests"]):
+        self.name = name
+        self.url = url
+        self.ref = ref
+
+        self._test_match_patterns = test_match_patterns
+        self._clone_location = None
+
+    def clone(self, location):
+        self._clone_location = os.path.join(location, self.name)
+        subprocess.run(['git', 'clone',
+                        '--single-branch',
+                        '--branch', self.ref,
+                        '--depth', '1',
+                        self.url,
+                        self._clone_location,
+                        ])
+        return self._clone_location
+
+    def install(self):
+        subprocess.run(['pip3', 'install', self._clone_location])
+
+    def test(self, pytest_args):
+        test_files = self._match_test_patterns()
+        return pytest.main(pytest_args + test_files)
+
+    def _match_test_patterns(self):
+        match_list = []
+        for pattern in self._test_match_patterns:
+            abs_pattern = os.path.join(self._clone_location, pattern)
+            print("matching pattern: ", abs_pattern)
+            matches = glob.glob(abs_pattern)
+            match_list.extend(matches)
+
+        if not match_list:
+            raise ValueError("No matches found for patterns {}".format(self._test_match_patterns))
+        return match_list
+
+
+def run_repo_tests(repo, tmpdir, pytest_args):
+    print("Cloning repo {} to {}...".format(repo.name, tmpdir))
+    repo.clone(tmpdir)
+
+    print("Installing {}...".format(repo.name))
+    repo.install()
+
+    print("Testing {}...".format(repo.name))
+    return repo.test(pytest_args)
+
+
+if __name__ == "__main__":
+    from sys import argv
+
+    # Args:
+    #    tmpdir: The directory in which this script will clone external
+    #            repositories and use pytest's tmpdir.
+    #    pytest_args: any remaining arguments to this script will be passed
+    #                 directly to it's pytest invocations
+    _, tmpdir, *pytest_args = argv
+
+    ALL_EXTERNAL_PLUGINS = [
+        ExternalPluginRepo(
+            name='bst-plugins-template',
+            url='https://gitlab.com/BuildStream/bst-plugins-template.git',
+            ref='master'
+        ),
+    ]
+
+    exit_code = 0
+    for plugin in ALL_EXTERNAL_PLUGINS:
+        exit_code = run_repo_tests(plugin, tmpdir, pytest_args)
+        if exit_code != 0:
+            sys.exit(exit_code)
diff --git a/tox.ini b/tox.ini
index 5057c7c99..3734bd6a6 100644
--- a/tox.ini
+++ b/tox.ini
@@ -8,17 +8,20 @@ skip_missing_interpreters = true
 #
 # Defaults for all environments
 #
-# Anything specified here is iherited by the sections
+# Anything specified here is inherited by the sections
 #
 [testenv]
 commands =
     # Running with coverage reporting enabled
     py{35,36,37}-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc {posargs}
+    py{35,36,37}-!nocover: pip3 install -e {toxinidir}
+    py{35,36,37}-!nocover: {envpython} {toxinidir}/tests/external_plugins.py {envtmpdir} --cov=buildstrean --cov-append  {posargs}
     py{35,36,37}-!nocover: mkdir -p .coverage-reports
     py{35,36,37}-!nocover: mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname}
-
     # Running with coverage reporting disabled
     py{35,36,37}-nocover: pytest --basetemp {envtmpdir} {posargs}
+    py{35,36,37}-nocover: pip3 install -e {toxinidir}
+    py{35,36,37}-nocover: {envpython} {toxinidir}/tests/external_plugins.py {envtmpdir} {posargs}
 deps =
     py{35,36,37}: -rrequirements/requirements.txt
     py{35,36,37}: -rrequirements/dev-requirements.txt
@@ -49,6 +52,29 @@ whitelist_externals =
         mkdir
 
 #
+# Run only the tests that we use form external plugins
+#
+[testenv:external]
+commands =
+    {envpython} {toxinidir}/tests/external_plugins.py {envtmpdir} {posargs}
+deps =
+    -rrequirements/requirements.txt
+    -rrequirements/dev-requirements.txt
+    -rrequirements/plugin-requirements.txt
+
+
+#
+# Run only the tests which are defined by the BuildStream project
+#
+[testenv:internal]
+commands =
+    pytest --basetemp {envtmpdir} {posargs}
+deps =
+    -rrequirements/requirements.txt
+    -rrequirements/dev-requirements.txt
+    -rrequirements/plugin-requirements.txt
+
+#
 # Coverage reporting
 #
 [testenv:coverage]
@@ -74,6 +100,7 @@ deps =
     -rrequirements/dev-requirements.txt
     -rrequirements/plugin-requirements.txt
 
+
 #
 # Building documentation
 #