tox.ini: Add a external plugins environment test and run it in CIbschubert/standardized-tests

This runs two versions of the plugins:

- The latest stable is not allowed failures and is run on every platform
- The master version is allowed failure, and only runs on a single
architecture

This also adds a new entrypoint to register source tests to run against
BuildStream.

4 files changed, 57 insertions, 9 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 0c5bf1e5d..a5133a86a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -29,6 +29,7 @@ variables:
   INTEGRATION_CACHE: "${CI_PROJECT_DIR}/cache/integration-cache"
   PYTEST_ARGS: "--color=yes --integration -n 2"
   TEST_COMMAND: "tox -- ${PYTEST_ARGS}"
+  PLUGINS_TESTS_COMMAND: "tox -e py35-plugins,py36-plugins,py37-plugins -- ${PYTEST_ARGS}"
   COVERAGE_PREFIX: "${CI_JOB_NAME}."
 
 
@@ -54,6 +55,7 @@ variables:
 
   # Run the tests as a simple user to test for permission issues
   - su buildstream -c "${TEST_COMMAND}"
+  - su buildstream -c "${PLUGINS_TESTS_COMMAND}"
 
   after_script:
   except:
@@ -90,16 +92,23 @@ tests-python-3.8-buster:
   image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-python:3.8-buster-${DOCKER_IMAGE_VERSION}
   <<: *tests
   variables:
+    # This particular testsuite image has both Python 3.7 and Python 3.8 so we
+    # need to explicitly force the 3.8 environment.
+    # Once Python 3.8 is available in distros, we should switch to such an
+
     # Our testsuite has issues with coverage on Python 3.8 so disable coverage
     # in the meantime. For more details, see
     # https://gitlab.com/BuildStream/buildstream/issues/1173.
     TEST_COMMAND: "tox -e py38-nocover -- ${PYTEST_ARGS}"
+    PLUGINS_TESTS_COMMAND: "tox -e py38-plugins-nocover -- ${PYTEST_ARGS}"
 
-    # This particular testsuite image has both Python 3.7 and Python 3.8 so we
-    # need to explicitly force the 3.8 environment.
-    # Once Python 3.8 is available in distros, we should switch to such an
-    # image, and remove the following override.
-    EXTERNAL_TESTS_COMMAND: "tox -e py38-external -- ${PYTEST_ARGS}"
+# Test the master version of some external plugins
+tests-plugins-master:
+  <<: *tests
+  allow_failure: true
+
+  variables:
+    BST_PLUGINS_EXPERIMENTAL_VERSION: master
 
 overnight-fedora-30-aarch64:
   image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:aarch64-30-${DOCKER_IMAGE_VERSION}
@@ -150,7 +159,7 @@ tests-userchroot:
 
     # Run the tests as a simple user to test for permission issues
     - su buildstream -c "umask 002 && ${TEST_COMMAND}"
-    - su buildstream -c "umask 002 && ${EXTERNAL_TESTS_COMMAND}"
+    - su buildstream -c "umask 002 && ${PLUGINS_TESTS_COMMAND}"
 
 tests-fedora-missing-deps:
   # Ensure that tests behave nicely while missing bwrap and ostree
@@ -168,6 +177,8 @@ tests-fedora-missing-deps:
     - chown -R buildstream:buildstream .
 
     - ${TEST_COMMAND}
+    - ${PLUGINS_TESTS_COMMAND}
+
 
 tests-fedora-update-deps:
   # Check if the tests pass after updating requirements to their latest
@@ -184,6 +195,7 @@ tests-fedora-update-deps:
     - cat requirements/*.txt
 
     - su buildstream -c "${TEST_COMMAND}"
+    - su buildstream -c "${PLUGINS_TESTS_COMMAND}"
 
 tests-remote-execution:
   allow_failure: true
diff --git a/NEWS b/NEWS
index abd53d903..602b14b51 100644
--- a/NEWS
+++ b/NEWS
@@ -9,6 +9,17 @@ CLI
     of `buildable` for the state of junction elements, as they can't
     be built.
 
+API
+---
+
+  o External plugins can now register a `buildstream.tests.source_plugins` entrypoint.
+    The entry point can have an arbitrary name, but its value should point to a module
+    containing a `register_sources()` method.
+    This method should call `register_repo_kind` for all sources you want to have
+    tested in BuildStream.
+    Plugins authors that do this and believe BuildStream should be testing that
+    part of their plugins should open an issue on BuildStream.
+
 
 ==================
 buildstream 1.91.3
diff --git a/tests/conftest.py b/tests/conftest.py
index cfca4ad06..8d33fa024 100755
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -21,6 +21,8 @@
 #
 import os
 import multiprocessing
+
+import pkg_resources
 import pytest
 
 from buildstream.testing import register_repo_kind, sourcetests_collection_hook
@@ -46,7 +48,7 @@ from tests.testutils.repo.zip import Zip
 #################################################
 def pytest_addoption(parser):
     parser.addoption("--integration", action="store_true", default=False, help="Run integration tests")
-
+    parser.addoption("--plugins", action="store_true", default=False, help="Run only plugins tests")
     parser.addoption("--remote-execution", action="store_true", default=False, help="Run remote-execution tests only")
 
 
@@ -66,6 +68,11 @@ def pytest_runtest_setup(item):
         if item.get_closest_marker("remoteexecution"):
             pytest.skip("skipping remote-execution test")
 
+    # With --plugins only run plugins tests
+    if item.config.getvalue("plugins"):
+        if not item.get_closest_marker("generic_source_test"):
+            pytest.skip("Skipping not generic source test")
+
 
 #################################################
 #           remote_services fixture             #
@@ -112,6 +119,12 @@ register_repo_kind("zip", Zip, None)
 # This hook enables pytest to collect the templated source tests from
 # buildstream.testing
 def pytest_sessionstart(session):
+    if session.config.getvalue("plugins"):
+        # Enable all plugins that implement the 'buildstream.tests.source_plugins' hook
+        for entrypoint in pkg_resources.iter_entry_points("buildstream.tests.source_plugins"):
+            module = entrypoint.load()
+            module.register_sources()
+
     sourcetests_collection_hook(session)
 
 
diff --git a/tox.ini b/tox.ini
index ab7830fff..ea45c1acc 100644
--- a/tox.ini
+++ b/tox.ini
@@ -6,6 +6,10 @@ envlist = py{35,36,37,38}
 skip_missing_interpreters = true
 isolated_build = true
 
+# Configuration variables to share accross environments
+[config]
+BST_PLUGINS_EXPERIMENTAL_VERSION = 0.13.0
+
 #
 # Defaults for all environments
 #
@@ -18,9 +22,13 @@ usedevelop =
 
 commands =
     # Running with coverage reporting enabled
-    py{35,36,37,38}-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc {posargs}
+    py{35,36,37,38}-!plugins-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc {posargs}
     # Running with coverage reporting disabled
-    py{35,36,37,38}-nocover: pytest --basetemp {envtmpdir} {posargs}
+    py{35,36,37,38}-!plugins-nocover: pytest --basetemp {envtmpdir} {posargs}
+    # Running external plugins tests with coverage reporting enabled
+    py{35,36,37,38}-plugins-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc --plugins {posargs}
+    # Running external plugins tests with coverage disabled
+    py{35,36,37,38}-plugins-nocover: pytest --basetemp {envtmpdir} --plugins {posargs}
 commands_post:
     py{35,36,37,38}-!nocover: mkdir -p .coverage-reports
     py{35,36,37,38}-!nocover: mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname}
@@ -29,6 +37,9 @@ deps =
     py{35,36,37,38}: -rrequirements/dev-requirements.txt
     py{35,36,37,38}: -rrequirements/plugin-requirements.txt
 
+    # Install external plugins for plugin tests
+    py{35,36,37,38}-plugins: git+https://gitlab.com/buildstream/bst-plugins-experimental.git@{env:BST_PLUGINS_EXPERIMENTAL_VERSION:{[config]BST_PLUGINS_EXPERIMENTAL_VERSION}}#egg=bst_plugins_experimental[ostree]
+
     # Only require coverage and pytest-cov when using it
     !nocover: -rrequirements/cov-requirements.txt
 
@@ -51,6 +62,7 @@ passenv =
     REMOTE_EXECUTION_SERVICE
     SOURCE_CACHE_SERVICE
     SSL_CERT_FILE
+    BST_PLUGINS_EXPERIMENTAL_VERSION
 #
 # These keys are not inherited by any other sections
 #