Expose basic api for testing external plugins.phil/plugin-testing-api

We want external plugins to be able to make use of the core testing utils.
This commit exposes the basic utilities which are currently in use in
bst-external plugins. If necessary, more utilities could be exposed in the
future.

Moves the following files from tests/testutils/ to
buildstream/plugintestingutils/:
o runcli.py
o integration.py

As part of this, this commit makes the following changes to runcli.py
and integration.py:
o runcli.py: Fix linting errors
o runcli.py: Add user facing documentation
o Integration.py: Add user facing documentation

101 files changed, 274 insertions, 159 deletions

diff --git a/buildstream/plugintestutils/__init__.py b/buildstream/plugintestutils/__init__.py
new file mode 100644
index 000000000..c7238a29c
--- /dev/null
+++ b/buildstream/plugintestutils/__init__.py
@@ -0,0 +1,30 @@
+#
+#  Copyright (C) 2019 Codethink Limited
+#  Copyright (C) 2019 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+
+from .runcli import cli, cli_integration
+
+# To make use of these test utilities it is necessary to have pytest
+# available. However, we don't want to have a hard dependency on
+# pytest.
+try:
+    import pytest
+except ImportError:
+    module_name = globals()['__name__']
+    msg = "Could not import pytest:\n" \
+          "To use the {} module, you must have pytest installed.".format(module_name)
+    raise ImportError(msg)
diff --git a/buildstream/plugintestutils/integration.py b/buildstream/plugintestutils/integration.py
new file mode 100644
index 000000000..e29f480ea
--- /dev/null
+++ b/buildstream/plugintestutils/integration.py
@@ -0,0 +1,51 @@
+#
+#  Copyright (C) 2017 Codethink Limited
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+"""
+Integration - tools for inspecting the output of plugin integration tests
+=========================================================================
+
+This module contains utilities for inspecting the artifacts produced during
+integration tests.
+"""
+
+import os
+
+
+# Return a list of files relative to the given directory
+def walk_dir(root):
+    for dirname, dirnames, filenames in os.walk(root):
+        # ensure consistent traversal order, needed for consistent
+        # handling of symlinks.
+        dirnames.sort()
+        filenames.sort()
+
+        # print path to all subdirectories first.
+        for subdirname in dirnames:
+            yield os.path.join(dirname, subdirname)[len(root):]
+
+        # print path to all filenames.
+        for filename in filenames:
+            yield os.path.join(dirname, filename)[len(root):]
+
+
+# Ensure that a directory contains the given filenames.
+def assert_contains(directory, expected):
+    missing = set(expected)
+    missing.difference_update(walk_dir(directory))
+    if missing:
+        raise AssertionError("Missing {} expected elements from list: {}"
+                             .format(len(missing), missing))
diff --git a/tests/testutils/runcli.py b/buildstream/plugintestutils/runcli.py
index b051dec21..fb7c23c6c 100644
--- a/tests/testutils/runcli.py
+++ b/buildstream/plugintestutils/runcli.py
@@ -1,3 +1,34 @@
+#
+#  Copyright (C) 2017 Codethink Limited
+#  Copyright (C) 2018 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+"""
+runcli - Test fixtures used for running BuildStream commands
+============================================================
+
+:function:'cli' Use result = cli.run([arg1, arg2]) to run buildstream commands
+
+:function:'cli_integration' A variant of the main fixture that keeps persistent
+                            artifact and source caches. It also does not use
+                            the click test runner to avoid deadlock issues when
+                            running `bst shell`, but unfortunately cannot produce
+                            nice stacktraces.
+
+"""
+
+
 import os
 import re
 import sys
@@ -5,7 +36,6 @@ import shutil
 import tempfile
 import itertools
 import traceback
-import subprocess
 from contextlib import contextmanager, ExitStack
 from ruamel import yaml
 import pytest
@@ -333,7 +363,7 @@ class Cli():
 
         return result
 
-    def invoke(self, cli, args=None, color=False, binary_capture=False, **extra):
+    def invoke(self, cli_object, args=None, color=False, binary_capture=False, **extra):
         exc_info = None
         exception = None
         exit_code = 0
@@ -348,7 +378,7 @@ class Cli():
             capture.start_capturing()
 
             try:
-                cli.main(args=args or (), prog_name=cli.name, **extra)
+                cli_object.main(args=args or (), prog_name=cli_object.name, **extra)
             except SystemExit as e:
                 if e.code != 0:
                     exception = e
@@ -361,7 +391,7 @@ class Cli():
                     sys.stdout.write(str(exit_code))
                     sys.stdout.write('\n')
                     exit_code = 1
-            except Exception as e:
+            except Exception as e:  # pylint: disable=broad-except
                 exception = e
                 exit_code = -1
                 exc_info = sys.exc_info()
diff --git a/tests/artifactcache/cache_size.py b/tests/artifactcache/cache_size.py
index 63ab9ad07..88f8eaddf 100644
--- a/tests/artifactcache/cache_size.py
+++ b/tests/artifactcache/cache_size.py
@@ -5,8 +5,9 @@ from unittest import mock
 from buildstream import _yaml
 from buildstream._artifactcache import CACHE_SIZE_FILE
 from buildstream._exceptions import ErrorDomain
+from buildstream.plugintestutils import cli
 
-from tests.testutils import cli, create_element_size
+from tests.testutils import create_element_size
 
 # XXX: Currently lacking:
 #      * A way to check whether it's faster to read cache size on
diff --git a/tests/artifactcache/config.py b/tests/artifactcache/config.py
index fecb3dd2e..fda309725 100644
--- a/tests/artifactcache/config.py
+++ b/tests/artifactcache/config.py
@@ -10,7 +10,7 @@ from buildstream.utils import _deduplicate
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
 
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 
 DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/artifactcache/expiry.py b/tests/artifactcache/expiry.py
index 2cc59e03c..d92e68f0b 100644
--- a/tests/artifactcache/expiry.py
+++ b/tests/artifactcache/expiry.py
@@ -25,8 +25,9 @@ import pytest
 
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
+from buildstream.plugintestutils import cli
 
-from tests.testutils import cli, create_element_size, update_element_size, wait_for_cache_granularity
+from tests.testutils import create_element_size, update_element_size, wait_for_cache_granularity
 
 
 DATA_DIR = os.path.join(
diff --git a/tests/artifactcache/junctions.py b/tests/artifactcache/junctions.py
index c6d6921c8..d5de16282 100644
--- a/tests/artifactcache/junctions.py
+++ b/tests/artifactcache/junctions.py
@@ -1,9 +1,11 @@
 import os
 import shutil
 import pytest
-from tests.testutils import cli, create_artifact_share
 
 from buildstream import _yaml
+from buildstream.plugintestutils import cli
+
+from tests.testutils import create_artifact_share
 
 
 DATA_DIR = os.path.join(
diff --git a/tests/artifactcache/pull.py b/tests/artifactcache/pull.py
index 4c332bf36..edd5a93ba 100644
--- a/tests/artifactcache/pull.py
+++ b/tests/artifactcache/pull.py
@@ -9,8 +9,9 @@ from buildstream import _yaml, _signals, utils
 from buildstream._context import Context
 from buildstream._project import Project
 from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
+from buildstream.plugintestutils import cli
 
-from tests.testutils import cli, create_artifact_share
+from tests.testutils import create_artifact_share
 
 
 # Project directory
diff --git a/tests/artifactcache/push.py b/tests/artifactcache/push.py
index 116fa7865..ed2a140e7 100644
--- a/tests/artifactcache/push.py
+++ b/tests/artifactcache/push.py
@@ -10,8 +10,8 @@ from buildstream._context import Context
 from buildstream._project import Project
 from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
 from buildstream.storage._casbaseddirectory import CasBasedDirectory
-
-from tests.testutils import cli, create_artifact_share
+from buildstream.plugintestutils import cli
+from tests.testutils import create_artifact_share
 
 
 # Project directory
diff --git a/tests/cachekey/cachekey.py b/tests/cachekey/cachekey.py
index 761ff0c76..4a6896888 100644
--- a/tests/cachekey/cachekey.py
+++ b/tests/cachekey/cachekey.py
@@ -35,7 +35,7 @@
 # run over to the corresponding .expected source files and commit
 # the result.
 #
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 from tests.testutils.site import HAVE_BZR, HAVE_GIT, HAVE_OSTREE, IS_LINUX, MACHINE_ARCH
 from buildstream.plugin import CoreWarnings
 from buildstream import _yaml
diff --git a/tests/cachekey/update.py b/tests/cachekey/update.py
index d574d07b3..3e0924925 100755
--- a/tests/cachekey/update.py
+++ b/tests/cachekey/update.py
@@ -13,7 +13,7 @@
 #
 import os
 import tempfile
-from tests.testutils.runcli import Cli
+from buildstream.plugintestutils.runcli import Cli
 
 # This weird try / except is needed, because this will be imported differently
 # when pytest runner imports them vs when you run the updater directly from
diff --git a/tests/elements/filter.py b/tests/elements/filter.py
index d40a8bdd1..c0ef2568a 100644
--- a/tests/elements/filter.py
+++ b/tests/elements/filter.py
@@ -1,7 +1,8 @@
 import os
 import pytest
 import shutil
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS
+from tests.testutils import create_repo, ALL_REPO_KINDS
+from buildstream.plugintestutils import cli
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
 
diff --git a/tests/examples/autotools.py b/tests/examples/autotools.py
index 30f50768b..96827ff4c 100644
--- a/tests/examples/autotools.py
+++ b/tests/examples/autotools.py
@@ -1,8 +1,8 @@
 import os
 import pytest
 
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
 from tests.testutils.site import HAVE_BWRAP, IS_LINUX, MACHINE_ARCH
 
 pytestmark = pytest.mark.integration
diff --git a/tests/examples/developing.py b/tests/examples/developing.py
index 166fcf374..3ef78fd36 100644
--- a/tests/examples/developing.py
+++ b/tests/examples/developing.py
@@ -2,8 +2,8 @@ import os
 import pytest
 
 import tests.testutils.patch as patch
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
 from tests.testutils.site import HAVE_BWRAP, IS_LINUX, MACHINE_ARCH
 
 pytestmark = pytest.mark.integration
diff --git a/tests/examples/first-project.py b/tests/examples/first-project.py
index 821d2c190..35839770c 100644
--- a/tests/examples/first-project.py
+++ b/tests/examples/first-project.py
@@ -1,8 +1,8 @@
 import os
 import pytest
 
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
 from tests.testutils.site import IS_LINUX
 
 
diff --git a/tests/examples/flatpak-autotools.py b/tests/examples/flatpak-autotools.py
index 4153a9563..454b8d0bb 100644
--- a/tests/examples/flatpak-autotools.py
+++ b/tests/examples/flatpak-autotools.py
@@ -1,8 +1,8 @@
 import os
 import pytest
 
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
 from tests.testutils.site import HAVE_OSTREE, IS_LINUX, MACHINE_ARCH
 
 
diff --git a/tests/examples/integration-commands.py b/tests/examples/integration-commands.py
index 71e383008..abc64d951 100644
--- a/tests/examples/integration-commands.py
+++ b/tests/examples/integration-commands.py
@@ -1,8 +1,8 @@
 import os
 import pytest
 
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
 from tests.testutils.site import HAVE_BWRAP, IS_LINUX, MACHINE_ARCH
 
 
diff --git a/tests/examples/junctions.py b/tests/examples/junctions.py
index 753fa2dc0..3992b1520 100644
--- a/tests/examples/junctions.py
+++ b/tests/examples/junctions.py
@@ -1,8 +1,8 @@
 import os
 import pytest
 
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
 from tests.testutils.site import IS_LINUX, HAVE_BWRAP, MACHINE_ARCH
 
 pytestmark = pytest.mark.integration
diff --git a/tests/examples/running-commands.py b/tests/examples/running-commands.py
index 61e23fd9c..6290204a9 100644
--- a/tests/examples/running-commands.py
+++ b/tests/examples/running-commands.py
@@ -1,8 +1,8 @@
 import os
 import pytest
 
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
 from tests.testutils.site import IS_LINUX, HAVE_BWRAP, MACHINE_ARCH
 
 
diff --git a/tests/format/assertion.py b/tests/format/assertion.py
index 42f69b759..b787e07fc 100644
--- a/tests/format/assertion.py
+++ b/tests/format/assertion.py
@@ -1,7 +1,7 @@
 import os
 import pytest
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 # Project directory
 DATA_DIR = os.path.join(
diff --git a/tests/format/dependencies.py b/tests/format/dependencies.py
index a6b417a66..5513077ee 100644
--- a/tests/format/dependencies.py
+++ b/tests/format/dependencies.py
@@ -2,7 +2,7 @@ import os
 import pytest
 
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
 
 DATA_DIR = os.path.dirname(os.path.realpath(__file__))
 
diff --git a/tests/format/include.py b/tests/format/include.py
index 36e723ed0..1db37083e 100644
--- a/tests/format/include.py
+++ b/tests/format/include.py
@@ -2,7 +2,9 @@ import os
 import pytest
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils import cli, generate_junction, create_repo
+from buildstream.plugintestutils import cli
+
+from tests.testutils import generate_junction, create_repo
 
 
 # Project directory
diff --git a/tests/format/junctions.py b/tests/format/junctions.py
index 90608d0fc..7f27b5982 100644
--- a/tests/format/junctions.py
+++ b/tests/format/junctions.py
@@ -4,7 +4,8 @@ import shutil
 
 from buildstream import _yaml, ElementError
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils import cli, create_repo
+from buildstream.plugintestutils import cli
+from tests.testutils import create_repo
 from tests.testutils.site import HAVE_GIT
 
 
diff --git a/tests/format/listdirectiveerrors.py b/tests/format/listdirectiveerrors.py
index 4a2de86fc..8c397252c 100644
--- a/tests/format/listdirectiveerrors.py
+++ b/tests/format/listdirectiveerrors.py
@@ -1,7 +1,7 @@
 import os
 import pytest
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 # Project directory
 DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/optionarch.py b/tests/format/optionarch.py
index 09f9c07c9..a1237002d 100644
--- a/tests/format/optionarch.py
+++ b/tests/format/optionarch.py
@@ -3,7 +3,7 @@ import pytest
 from contextlib import contextmanager
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 # Project directory
 DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/optionbool.py b/tests/format/optionbool.py
index f02f0eee3..598496cd3 100644
--- a/tests/format/optionbool.py
+++ b/tests/format/optionbool.py
@@ -2,7 +2,7 @@ import os
 import pytest
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 # Project directory
 DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/optioneltmask.py b/tests/format/optioneltmask.py
index b49eee88a..7cbbd88f4 100644
--- a/tests/format/optioneltmask.py
+++ b/tests/format/optioneltmask.py
@@ -2,7 +2,7 @@ import os
 import pytest
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 # Project directory
 DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/optionenum.py b/tests/format/optionenum.py
index 6990ad915..936126dfc 100644
--- a/tests/format/optionenum.py
+++ b/tests/format/optionenum.py
@@ -2,7 +2,7 @@ import os
 import pytest
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 # Project directory
 DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/optionexports.py b/tests/format/optionexports.py
index 26eadc677..5e65e9721 100644
--- a/tests/format/optionexports.py
+++ b/tests/format/optionexports.py
@@ -1,7 +1,7 @@
 import os
 import pytest
 from buildstream import _yaml
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 # Project directory
 DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/optionflags.py b/tests/format/optionflags.py
index 3585f5ea5..341bb8c80 100644
--- a/tests/format/optionflags.py
+++ b/tests/format/optionflags.py
@@ -2,7 +2,7 @@ import os
 import pytest
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 # Project directory
 DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/optionos.py b/tests/format/optionos.py
index b05d7e49e..ba6ab2a48 100644
--- a/tests/format/optionos.py
+++ b/tests/format/optionos.py
@@ -4,7 +4,7 @@ from contextlib import contextmanager
 
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 DATA_DIR = os.path.dirname(os.path.realpath(__file__))
 
diff --git a/tests/format/optionoverrides.py b/tests/format/optionoverrides.py
index e5c37b3a5..256562edf 100644
--- a/tests/format/optionoverrides.py
+++ b/tests/format/optionoverrides.py
@@ -1,7 +1,7 @@
 import os
 import pytest
 from buildstream import _yaml
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 # Project directory
 DATA_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/format/options.py b/tests/format/options.py
index 2076106cb..3a1b12877 100644
--- a/tests/format/options.py
+++ b/tests/format/options.py
@@ -2,7 +2,7 @@ import os
 import pytest
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 # Project directory
 DATA_DIR = os.path.join(
diff --git a/tests/format/project.py b/tests/format/project.py
index 0433df99e..db6604409 100644
--- a/tests/format/project.py
+++ b/tests/format/project.py
@@ -2,7 +2,9 @@ import os
 import pytest
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils import cli, filetypegenerator
+from buildstream.plugintestutils import cli
+
+from tests.testutils import filetypegenerator
 
 
 # Project directory
diff --git a/tests/format/projectoverrides.py b/tests/format/projectoverrides.py
index d63b0a23d..cf9a1d35e 100644
--- a/tests/format/projectoverrides.py
+++ b/tests/format/projectoverrides.py
@@ -2,7 +2,7 @@
 import os
 import pytest
 from buildstream import _yaml
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 # Project directory
 DATA_DIR = os.path.join(
diff --git a/tests/format/variables.py b/tests/format/variables.py
index be7faefc5..5a0d21206 100644
--- a/tests/format/variables.py
+++ b/tests/format/variables.py
@@ -3,7 +3,7 @@ import pytest
 import sys
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 
 # Project directory
diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py
index b35b14820..80d710f6f 100644
--- a/tests/frontend/buildcheckout.py
+++ b/tests/frontend/buildcheckout.py
@@ -3,9 +3,10 @@ import tarfile
 import hashlib
 import pytest
 import subprocess
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS, generate_junction
 from tests.testutils.site import IS_WINDOWS
+from tests.testutils import create_repo, ALL_REPO_KINDS, generate_junction
 
+from buildstream.plugintestutils import cli
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
 
diff --git a/tests/frontend/buildtrack.py b/tests/frontend/buildtrack.py
index 720ab7efc..9c56fb4a0 100644
--- a/tests/frontend/buildtrack.py
+++ b/tests/frontend/buildtrack.py
@@ -4,9 +4,11 @@ import shutil
 import itertools
 
 import pytest
-from tests.testutils import cli, create_repo
+
+from tests.testutils import create_repo
 
 from buildstream import _yaml
+from buildstream.plugintestutils import cli
 from buildstream._exceptions import ErrorDomain
 
 from . import configure_project
diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py
index 93b908f60..cb151c284 100644
--- a/tests/frontend/completions.py
+++ b/tests/frontend/completions.py
@@ -1,6 +1,6 @@
 import os
 import pytest
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
 
 # Project directory
 DATA_DIR = os.path.join(
diff --git a/tests/frontend/compose_splits.py b/tests/frontend/compose_splits.py
index 97558b653..3d2db44fb 100644
--- a/tests/frontend/compose_splits.py
+++ b/tests/frontend/compose_splits.py
@@ -1,6 +1,6 @@
 import os
 import pytest
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 # Project directory
 DATA_DIR = os.path.join(
diff --git a/tests/frontend/configurable_warnings.py b/tests/frontend/configurable_warnings.py
index e8a7b6ac8..042874529 100644
--- a/tests/frontend/configurable_warnings.py
+++ b/tests/frontend/configurable_warnings.py
@@ -4,7 +4,7 @@ import os
 from buildstream.plugin import CoreWarnings
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
 from buildstream import _yaml
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 TOP_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
diff --git a/tests/frontend/cross_junction_workspace.py b/tests/frontend/cross_junction_workspace.py
index a10eb7421..837a09f8a 100644
--- a/tests/frontend/cross_junction_workspace.py
+++ b/tests/frontend/cross_junction_workspace.py
@@ -1,7 +1,9 @@
 import os
-from tests.testutils import cli, create_repo
+from buildstream.plugintestutils import cli
 from buildstream import _yaml
 
+from tests.testutils import create_repo
+
 
 def prepare_junction_project(cli, tmpdir):
     main_project = tmpdir.join("main")
diff --git a/tests/frontend/fetch.py b/tests/frontend/fetch.py
index 24d9a36a6..9edfad94b 100644
--- a/tests/frontend/fetch.py
+++ b/tests/frontend/fetch.py
@@ -1,7 +1,9 @@
 import os
 import pytest
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS, generate_junction
 
+from tests.testutils import create_repo, ALL_REPO_KINDS, generate_junction
+
+from buildstream.plugintestutils import cli
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
 
diff --git a/tests/frontend/help.py b/tests/frontend/help.py
index 331eb43c0..540a3a60f 100644
--- a/tests/frontend/help.py
+++ b/tests/frontend/help.py
@@ -1,5 +1,5 @@
 import pytest
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 
 def assert_help(cli_output):
diff --git a/tests/frontend/init.py b/tests/frontend/init.py
index f6e7dd68a..b323a01c8 100644
--- a/tests/frontend/init.py
+++ b/tests/frontend/init.py
@@ -1,6 +1,6 @@
 import os
 import pytest
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
 
 from buildstream import _yaml
 from buildstream._frontend.app import App
diff --git a/tests/frontend/logging.py b/tests/frontend/logging.py
index ddaadfa26..17d925cd1 100644
--- a/tests/frontend/logging.py
+++ b/tests/frontend/logging.py
@@ -1,10 +1,11 @@
 import os
 import pytest
 import re
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS
+from tests.testutils import create_repo, ALL_REPO_KINDS
 
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain
+from buildstream.plugintestutils import cli
 
 # Project directory
 DATA_DIR = os.path.join(
diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py
index 3cadd15f3..c0c2e7059 100644
--- a/tests/frontend/mirror.py
+++ b/tests/frontend/mirror.py
@@ -1,10 +1,11 @@
 import os
 import pytest
 
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS, generate_junction
+from tests.testutils import create_repo, ALL_REPO_KINDS, generate_junction
 
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain
+from buildstream.plugintestutils import cli
 
 # Project directory
 TOP_DIR = os.path.dirname(os.path.realpath(__file__))
diff --git a/tests/frontend/order.py b/tests/frontend/order.py
index 97ebc2bf6..422fce260 100644
--- a/tests/frontend/order.py
+++ b/tests/frontend/order.py
@@ -1,8 +1,8 @@
 import os
 
 import pytest
-from tests.testutils import cli, create_repo
-
+from tests.testutils import create_repo
+from buildstream.plugintestutils import cli
 from buildstream import _yaml
 
 # Project directory
diff --git a/tests/frontend/overlaps.py b/tests/frontend/overlaps.py
index 27be8deac..22365e442 100644
--- a/tests/frontend/overlaps.py
+++ b/tests/frontend/overlaps.py
@@ -1,6 +1,6 @@
 import os
 import pytest
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
 from buildstream.plugin import CoreWarnings
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index a2c710fb8..9579d9f27 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -1,7 +1,8 @@
 import os
 import shutil
 import pytest
-from tests.testutils import cli, create_artifact_share, generate_junction
+from buildstream.plugintestutils import cli
+from tests.testutils import create_artifact_share, generate_junction
 
 
 # Project directory
diff --git a/tests/frontend/push.py b/tests/frontend/push.py
index 729df9ae9..7a4cb3c0d 100644
--- a/tests/frontend/push.py
+++ b/tests/frontend/push.py
@@ -24,8 +24,8 @@ import os
 import pytest
 
 from buildstream._exceptions import ErrorDomain
-from tests.testutils import cli, create_artifact_share, create_element_size
-from tests.testutils import generate_junction, wait_for_cache_granularity
+from buildstream.plugintestutils import cli
+from tests.testutils import create_artifact_share, create_element_size, generate_junction, wait_for_cache_granularity
 from . import configure_project
 
 
diff --git a/tests/frontend/rebuild.py b/tests/frontend/rebuild.py
index d93aac0dc..6a46e855e 100644
--- a/tests/frontend/rebuild.py
+++ b/tests/frontend/rebuild.py
@@ -1,6 +1,6 @@
 import os
 import pytest
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
 
 # Project directory
 DATA_DIR = os.path.join(
diff --git a/tests/frontend/show.py b/tests/frontend/show.py
index 88f38dd6a..ad3ae3591 100644
--- a/tests/frontend/show.py
+++ b/tests/frontend/show.py
@@ -3,7 +3,8 @@ import sys
 import shutil
 import itertools
 import pytest
-from tests.testutils import cli, generate_junction
+from tests.testutils import generate_junction
+from buildstream.plugintestutils import cli
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
 
diff --git a/tests/frontend/source_checkout.py b/tests/frontend/source_checkout.py
index d7ff86d70..b41d32491 100644
--- a/tests/frontend/source_checkout.py
+++ b/tests/frontend/source_checkout.py
@@ -3,7 +3,7 @@ import pytest
 import tarfile
 from pathlib import Path
 
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
 
 from buildstream import utils, _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
diff --git a/tests/frontend/track.py b/tests/frontend/track.py
index a4ace92b6..486a4123c 100644
--- a/tests/frontend/track.py
+++ b/tests/frontend/track.py
@@ -1,8 +1,9 @@
 import stat
 import os
 import pytest
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS, generate_junction
+from tests.testutils import create_repo, ALL_REPO_KINDS, generate_junction
 
+from buildstream.plugintestutils import cli
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
 from buildstream import _yaml
 
diff --git a/tests/frontend/track_cross_junction.py b/tests/frontend/track_cross_junction.py
index 4bbf2db18..7d9d468ae 100644
--- a/tests/frontend/track_cross_junction.py
+++ b/tests/frontend/track_cross_junction.py
@@ -1,6 +1,8 @@
 import os
 import pytest
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS, generate_junction
+from tests.testutils import create_repo, ALL_REPO_KINDS, generate_junction
+
+from buildstream.plugintestutils import cli
 from buildstream import _yaml
 
 
diff --git a/tests/frontend/version.py b/tests/frontend/version.py
index 997eb2340..1d36a33b5 100644
--- a/tests/frontend/version.py
+++ b/tests/frontend/version.py
@@ -1,4 +1,4 @@
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 # For utils.get_bst_version()
 from buildstream import utils
diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py
index 6d4270e11..f6d12e8bf 100644
--- a/tests/frontend/workspace.py
+++ b/tests/frontend/workspace.py
@@ -30,9 +30,10 @@ import pytest
 import shutil
 import subprocess
 from ruamel.yaml.comments import CommentedSet
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS, wait_for_cache_granularity
+from tests.testutils import create_repo, ALL_REPO_KINDS, wait_for_cache_granularity
 from tests.testutils import create_artifact_share, create_element_size
 
+from buildstream.plugintestutils import cli
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason
 from buildstream._workspaces import BST_WORKSPACE_FORMAT_VERSION
diff --git a/tests/frontend/yamlcache.py b/tests/frontend/yamlcache.py
index 20388593e..99b5d71c3 100644
--- a/tests/frontend/yamlcache.py
+++ b/tests/frontend/yamlcache.py
@@ -4,7 +4,8 @@ import hashlib
 import tempfile
 from ruamel import yaml
 
-from tests.testutils import cli, generate_junction, create_element_size, create_repo
+from tests.testutils import generate_junction, create_element_size, create_repo
+from buildstream.plugintestutils import cli
 from buildstream import _yaml
 from buildstream._yamlcache import YamlCache
 from buildstream._project import Project
diff --git a/tests/integration/artifact.py b/tests/integration/artifact.py
index 2e12e712c..459241209 100644
--- a/tests/integration/artifact.py
+++ b/tests/integration/artifact.py
@@ -21,7 +21,7 @@
 import os
 import pytest
 
-from tests.testutils import cli_integration as cli
+from buildstream.plugintestutils import cli_integration as cli
 
 
 pytestmark = pytest.mark.integration
diff --git a/tests/integration/autotools.py b/tests/integration/autotools.py
index b1d33f497..3ed0533cf 100644
--- a/tests/integration/autotools.py
+++ b/tests/integration/autotools.py
@@ -1,8 +1,8 @@
 import os
 import pytest
 
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
 from tests.testutils.site import HAVE_SANDBOX
 
 
diff --git a/tests/integration/build-tree.py b/tests/integration/build-tree.py
index bacbf1cc7..98bb5b1e8 100644
--- a/tests/integration/build-tree.py
+++ b/tests/integration/build-tree.py
@@ -2,8 +2,9 @@ import os
 import pytest
 import shutil
 
-from tests.testutils import cli, cli_integration, create_artifact_share
+from tests.testutils import create_artifact_share
 from tests.testutils.site import HAVE_SANDBOX
+from buildstream.plugintestutils import cli, cli_integration
 from buildstream._exceptions import ErrorDomain
 
 
diff --git a/tests/integration/build-uid.py b/tests/integration/build-uid.py
index 06779ee20..88b887b5e 100644
--- a/tests/integration/build-uid.py
+++ b/tests/integration/build-uid.py
@@ -3,8 +3,8 @@ import pytest
 
 from buildstream import _yaml
 
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
 from tests.testutils.site import HAVE_BWRAP, IS_LINUX, HAVE_SANDBOX
 
 
diff --git a/tests/integration/cachedfail.py b/tests/integration/cachedfail.py
index 08dbf81e1..5335ff5ed 100644
--- a/tests/integration/cachedfail.py
+++ b/tests/integration/cachedfail.py
@@ -3,10 +3,10 @@ import pytest
 
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain
+from buildstream.plugintestutils import cli_integration as cli
 
 from conftest import clean_platform_cache
-
-from tests.testutils import cli_integration as cli, create_artifact_share
+from tests.testutils import create_artifact_share
 from tests.testutils.site import HAVE_BWRAP, IS_LINUX, HAVE_SANDBOX
 
 
diff --git a/tests/integration/cmake.py b/tests/integration/cmake.py
index 7d03e291d..51e215655 100644
--- a/tests/integration/cmake.py
+++ b/tests/integration/cmake.py
@@ -1,8 +1,8 @@
 import os
 import pytest
 
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
 from tests.testutils.site import HAVE_SANDBOX
 
 
diff --git a/tests/integration/compose-symlinks.py b/tests/integration/compose-symlinks.py
index 2599d8bcd..c6027bf2b 100644
--- a/tests/integration/compose-symlinks.py
+++ b/tests/integration/compose-symlinks.py
@@ -5,8 +5,8 @@ import pytest
 
 from buildstream import _yaml
 
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import walk_dir
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import walk_dir
 
 
 pytestmark = pytest.mark.integration
diff --git a/tests/integration/compose.py b/tests/integration/compose.py
index 077d5eea0..386e08438 100644
--- a/tests/integration/compose.py
+++ b/tests/integration/compose.py
@@ -5,8 +5,8 @@ import pytest
 
 from buildstream import _yaml
 
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import walk_dir
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import walk_dir
 from tests.testutils.site import HAVE_SANDBOX
 
 
diff --git a/tests/integration/import.py b/tests/integration/import.py
index 842640515..deef66bc1 100644
--- a/tests/integration/import.py
+++ b/tests/integration/import.py
@@ -3,8 +3,8 @@ import pytest
 
 from buildstream import _yaml
 
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import walk_dir
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import walk_dir
 
 
 pytestmark = pytest.mark.integration
diff --git a/tests/integration/make.py b/tests/integration/make.py
index 12c027b61..a76fe9a41 100644
--- a/tests/integration/make.py
+++ b/tests/integration/make.py
@@ -1,8 +1,8 @@
 import os
 import pytest
 
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
 from tests.testutils.site import HAVE_SANDBOX
 
 
diff --git a/tests/integration/manual.py b/tests/integration/manual.py
index b9f09e25c..97cbec537 100644
--- a/tests/integration/manual.py
+++ b/tests/integration/manual.py
@@ -3,7 +3,7 @@ import pytest
 
 from buildstream import _yaml
 
-from tests.testutils import cli_integration as cli
+from buildstream.plugintestutils import cli_integration as cli
 from tests.testutils.site import HAVE_SANDBOX
 
 
diff --git a/tests/integration/messages.py b/tests/integration/messages.py
index 03c922472..7f4194e0f 100644
--- a/tests/integration/messages.py
+++ b/tests/integration/messages.py
@@ -22,8 +22,7 @@ import pytest
 
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain
-
-from tests.testutils import cli_integration as cli
+from buildstream.plugintestutils import cli_integration as cli
 from tests.testutils.site import HAVE_SANDBOX
 
 
diff --git a/tests/integration/pip_element.py b/tests/integration/pip_element.py
index 807123620..48ca13bf5 100644
--- a/tests/integration/pip_element.py
+++ b/tests/integration/pip_element.py
@@ -4,8 +4,8 @@ import pytest
 
 from buildstream import _yaml
 
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
 from tests.testutils.site import HAVE_SANDBOX
 
 
diff --git a/tests/integration/pip_source.py b/tests/integration/pip_source.py
index d43b380aa..993b3a182 100644
--- a/tests/integration/pip_source.py
+++ b/tests/integration/pip_source.py
@@ -3,9 +3,9 @@ import pytest
 
 from buildstream import _yaml
 
-from tests.testutils import cli_integration as cli
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
 from tests.testutils.python_repo import setup_pypi_repo
-from tests.testutils.integration import assert_contains
 from tests.testutils.site import HAVE_SANDBOX
 
 
diff --git a/tests/integration/pullbuildtrees.py b/tests/integration/pullbuildtrees.py
index ca6c40f94..24fac7e00 100644
--- a/tests/integration/pullbuildtrees.py
+++ b/tests/integration/pullbuildtrees.py
@@ -2,9 +2,12 @@ import os
 import shutil
 import pytest
 
-from tests.testutils import cli, cli_integration as cli2, create_artifact_share
-from tests.testutils.integration import assert_contains
+from tests.testutils import create_artifact_share
 from tests.testutils.site import HAVE_SANDBOX
+
+from buildstream.plugintestutils.integration import assert_contains
+from buildstream.plugintestutils import cli, cli_integration as cli2
+from buildstream.plugintestutils.integration import assert_contains
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
 
 
diff --git a/tests/integration/sandbox-bwrap.py b/tests/integration/sandbox-bwrap.py
index 66e9f5b57..2c939e9be 100644
--- a/tests/integration/sandbox-bwrap.py
+++ b/tests/integration/sandbox-bwrap.py
@@ -3,8 +3,8 @@ import pytest
 
 from buildstream._exceptions import ErrorDomain
 
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
 from tests.testutils.site import HAVE_BWRAP, HAVE_BWRAP_JSON_STATUS
 
 
diff --git a/tests/integration/script.py b/tests/integration/script.py
index 446a63f3d..0297a3f0a 100644
--- a/tests/integration/script.py
+++ b/tests/integration/script.py
@@ -2,8 +2,7 @@ import os
 import pytest
 
 from buildstream import _yaml
-
-from tests.testutils import cli_integration as cli
+from buildstream.plugintestutils import cli_integration as cli
 from tests.testutils.site import HAVE_SANDBOX
 
 
diff --git a/tests/integration/shell.py b/tests/integration/shell.py
index 9b35db883..d1a551f53 100644
--- a/tests/integration/shell.py
+++ b/tests/integration/shell.py
@@ -3,8 +3,7 @@ import pytest
 
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain
-
-from tests.testutils import cli_integration as cli
+from buildstream.plugintestutils import cli_integration as cli
 from tests.testutils.site import HAVE_SANDBOX
 
 
diff --git a/tests/integration/sockets.py b/tests/integration/sockets.py
index 30cf622cf..fc2fb2b92 100644
--- a/tests/integration/sockets.py
+++ b/tests/integration/sockets.py
@@ -2,9 +2,8 @@ import os
 import pytest
 
 from buildstream import _yaml
-
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
 from tests.testutils.site import HAVE_SANDBOX
 
 
diff --git a/tests/integration/source-determinism.py b/tests/integration/source-determinism.py
index 6be6a4d4a..f7b2bf2ac 100644
--- a/tests/integration/source-determinism.py
+++ b/tests/integration/source-determinism.py
@@ -2,8 +2,8 @@ import os
 import pytest
 
 from buildstream import _yaml, utils
+from buildstream.plugintestutils import cli_integration as cli
 from tests.testutils import create_repo, ALL_REPO_KINDS
-from tests.testutils import cli_integration as cli
 from tests.testutils.site import HAVE_SANDBOX
 
 
diff --git a/tests/integration/stack.py b/tests/integration/stack.py
index 1b8d38198..d5208d7be 100644
--- a/tests/integration/stack.py
+++ b/tests/integration/stack.py
@@ -2,8 +2,7 @@ import os
 import pytest
 
 from buildstream import _yaml
-
-from tests.testutils import cli_integration as cli
+from buildstream.plugintestutils import cli_integration as cli
 from tests.testutils.site import HAVE_SANDBOX
 
 
diff --git a/tests/integration/symlinks.py b/tests/integration/symlinks.py
index c45c81177..22ff527f8 100644
--- a/tests/integration/symlinks.py
+++ b/tests/integration/symlinks.py
@@ -4,8 +4,8 @@ import pytest
 
 from buildstream import _yaml
 
-from tests.testutils import cli_integration as cli
-from tests.testutils.integration import assert_contains
+from buildstream.plugintestutils import cli_integration as cli
+from buildstream.plugintestutils.integration import assert_contains
 from tests.testutils.site import HAVE_SANDBOX
 
 
diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py
index ee1d575bf..f1d8d6fcd 100644
--- a/tests/integration/workspace.py
+++ b/tests/integration/workspace.py
@@ -2,9 +2,9 @@ import os
 import pytest
 
 from buildstream import _yaml
-from tests.testutils import cli_integration as cli
+from buildstream.plugintestutils import cli_integration as cli
 from tests.testutils.site import HAVE_SANDBOX
-from tests.testutils.integration import walk_dir
+from buildstream.plugintestutils.integration import walk_dir
 
 
 pytestmark = pytest.mark.integration
diff --git a/tests/internals/storage_vdir_import.py b/tests/internals/storage_vdir_import.py
index 0bb47e3cd..1d61a6e5f 100644
--- a/tests/internals/storage_vdir_import.py
+++ b/tests/internals/storage_vdir_import.py
@@ -3,7 +3,7 @@ import os
 import pytest
 import random
 import tempfile
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
 
 from buildstream.storage._casbaseddirectory import CasBasedDirectory
 from buildstream.storage._filebaseddirectory import FileBasedDirectory
diff --git a/tests/sandboxes/missing-command.py b/tests/sandboxes/missing-command.py
index 8f210bcec..ddf1c487f 100644
--- a/tests/sandboxes/missing-command.py
+++ b/tests/sandboxes/missing-command.py
@@ -3,7 +3,7 @@ import pytest
 
 from buildstream._exceptions import ErrorDomain
 
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
 
 
 DATA_DIR = os.path.join(
diff --git a/tests/sandboxes/missing_dependencies.py b/tests/sandboxes/missing_dependencies.py
index d77674c64..19515a187 100644
--- a/tests/sandboxes/missing_dependencies.py
+++ b/tests/sandboxes/missing_dependencies.py
@@ -1,6 +1,6 @@
 import os
 import pytest
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
 from tests.testutils.site import IS_LINUX
 
 from buildstream import _yaml
diff --git a/tests/sandboxes/remote-exec-config.py b/tests/sandboxes/remote-exec-config.py
index 9cf0a8f08..9864f2591 100644
--- a/tests/sandboxes/remote-exec-config.py
+++ b/tests/sandboxes/remote-exec-config.py
@@ -6,7 +6,7 @@ import os
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
 
-from tests.testutils.runcli import cli
+from buildstream.plugintestutils.runcli import cli
 
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
diff --git a/tests/sources/bzr.py b/tests/sources/bzr.py
index a56005f5e..bf7d1e844 100644
--- a/tests/sources/bzr.py
+++ b/tests/sources/bzr.py
@@ -4,7 +4,8 @@ import pytest
 from buildstream._pipeline import PipelineError
 from buildstream import _yaml
 
-from tests.testutils import cli, create_repo
+from buildstream.plugintestutils import cli
+from tests.testutils import create_repo
 from tests.testutils.site import HAVE_BZR
 
 DATA_DIR = os.path.join(
diff --git a/tests/sources/deb.py b/tests/sources/deb.py
index b40358eaa..480bea6f4 100644
--- a/tests/sources/deb.py
+++ b/tests/sources/deb.py
@@ -8,7 +8,7 @@ import shutil
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
 from tempfile import TemporaryFile
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
 from tests.testutils.site import HAVE_ARPY
 from . import list_dir_contents
 
diff --git a/tests/sources/git.py b/tests/sources/git.py
index 795908329..f3d2e2809 100644
--- a/tests/sources/git.py
+++ b/tests/sources/git.py
@@ -28,8 +28,10 @@ import shutil
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
 from buildstream.plugin import CoreWarnings
+from buildstream.plugintestutils import cli
 
-from tests.testutils import cli, create_repo
+from tests.testutils.site import HAVE_GIT, HAVE_OLD_GIT
+from tests.testutils import create_repo
 from tests.testutils.site import HAVE_GIT, HAVE_OLD_GIT
 
 DATA_DIR = os.path.join(
diff --git a/tests/sources/local.py b/tests/sources/local.py
index 4a0851d72..f7c1f4bd2 100644
--- a/tests/sources/local.py
+++ b/tests/sources/local.py
@@ -2,7 +2,8 @@ import os
 import pytest
 
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils import cli, filetypegenerator
+from buildstream.plugintestutils import cli
+from tests.testutils import filetypegenerator
 
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
diff --git a/tests/sources/no_fetch_cached.py b/tests/sources/no_fetch_cached.py
index d2880f628..9ef838cf4 100644
--- a/tests/sources/no_fetch_cached.py
+++ b/tests/sources/no_fetch_cached.py
@@ -3,7 +3,8 @@ import pytest
 
 from buildstream import _yaml
 
-from tests.testutils import cli, create_repo
+from buildstream.plugintestutils import cli
+from tests.testutils import create_repo
 from tests.testutils.site import HAVE_GIT
 
 DATA_DIR = os.path.join(
diff --git a/tests/sources/ostree.py b/tests/sources/ostree.py
index e059a882f..7ddea77f6 100644
--- a/tests/sources/ostree.py
+++ b/tests/sources/ostree.py
@@ -22,8 +22,9 @@ import pytest
 
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
+from buildstream.plugintestutils import cli
 
-from tests.testutils import cli, create_repo
+from tests.testutils import create_repo
 
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
diff --git a/tests/sources/patch.py b/tests/sources/patch.py
index 51ae69021..2a82bf184 100644
--- a/tests/sources/patch.py
+++ b/tests/sources/patch.py
@@ -2,7 +2,8 @@ import os
 import pytest
 
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
-from tests.testutils import cli, filetypegenerator
+from buildstream.plugintestutils import cli
+from tests.testutils import filetypegenerator
 
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
diff --git a/tests/sources/pip.py b/tests/sources/pip.py
index 6e1a347d9..3114a824e 100644
--- a/tests/sources/pip.py
+++ b/tests/sources/pip.py
@@ -4,7 +4,7 @@ import pytest
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
 from buildstream.plugins.sources.pip import _match_package_name
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
 
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
diff --git a/tests/sources/previous_source_access.py b/tests/sources/previous_source_access.py
index 26640d180..9d831781a 100644
--- a/tests/sources/previous_source_access.py
+++ b/tests/sources/previous_source_access.py
@@ -1,7 +1,7 @@
 import os
 import pytest
 
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
 
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
diff --git a/tests/sources/remote.py b/tests/sources/remote.py
index 1d7727302..30b65b4cc 100644
--- a/tests/sources/remote.py
+++ b/tests/sources/remote.py
@@ -4,7 +4,7 @@ import pytest
 
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
 from tests.testutils.file_server import create_file_server
 
 DATA_DIR = os.path.join(
diff --git a/tests/sources/tar.py b/tests/sources/tar.py
index 406d6702b..959ff890c 100644
--- a/tests/sources/tar.py
+++ b/tests/sources/tar.py
@@ -8,7 +8,7 @@ from shutil import copyfile, rmtree
 
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
 from tests.testutils.file_server import create_file_server
 from tests.testutils.site import HAVE_LZIP
 from . import list_dir_contents
diff --git a/tests/sources/zip.py b/tests/sources/zip.py
index 009862e1c..dd72dd579 100644
--- a/tests/sources/zip.py
+++ b/tests/sources/zip.py
@@ -4,7 +4,7 @@ import zipfile
 
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
-from tests.testutils import cli
+from buildstream.plugintestutils import cli
 from tests.testutils.file_server import create_file_server
 from . import list_dir_contents
 
diff --git a/tests/testutils/__init__.py b/tests/testutils/__init__.py
index eb7211ea8..173cc7c3e 100644
--- a/tests/testutils/__init__.py
+++ b/tests/testutils/__init__.py
@@ -23,7 +23,6 @@
 #           William Salmon <will.salmon@codethink.co.uk>
 #
 
-from .runcli import cli, cli_integration
 from .repo import create_repo, ALL_REPO_KINDS
 from .artifactshare import create_artifact_share
 from .element_generators import create_element_size, update_element_size
diff --git a/tests/testutils/integration.py b/tests/testutils/integration.py
deleted file mode 100644
index b2cf9fba4..000000000
--- a/tests/testutils/integration.py
+++ /dev/null
@@ -1,29 +0,0 @@
-import os
-
-from buildstream import _yaml
-
-
-# Return a list of files relative to the given directory
-def walk_dir(root):
-    for dirname, dirnames, filenames in os.walk(root):
-        # ensure consistent traversal order, needed for consistent
-        # handling of symlinks.
-        dirnames.sort()
-        filenames.sort()
-
-        # print path to all subdirectories first.
-        for subdirname in dirnames:
-            yield os.path.join(dirname, subdirname)[len(root):]
-
-        # print path to all filenames.
-        for filename in filenames:
-            yield os.path.join(dirname, filename)[len(root):]
-
-
-# Ensure that a directory contains the given filenames.
-def assert_contains(directory, expected):
-    missing = set(expected)
-    missing.difference_update(walk_dir(directory))
-    if len(missing) > 0:
-        raise AssertionError("Missing {} expected elements from list: {}"
-                             .format(len(missing), missing))
diff --git a/tox.ini b/tox.ini
index 6683df796..9183e023f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -70,6 +70,7 @@ deps =
     sphinx
     sphinx-click
     sphinx_rtd_theme >= 0.4.2
+    pytest
     -rrequirements/requirements.txt
     -rrequirements/plugin-requirements.txt
 passenv =