runcli.py: Add a CLI test fixture for remote-execution

https://gitlab.com/BuildStream/buildstream/issues/629

3 files changed, 135 insertions, 2 deletions

diff --git a/buildstream/plugintestutils/__init__.py b/buildstream/plugintestutils/__init__.py
index c7238a29c..9ec18df19 100644
--- a/buildstream/plugintestutils/__init__.py
+++ b/buildstream/plugintestutils/__init__.py
@@ -16,7 +16,7 @@
 #  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 
 
-from .runcli import cli, cli_integration
+from .runcli import cli, cli_integration, cli_remote_execution
 
 # To make use of these test utilities it is necessary to have pytest
 # available. However, we don't want to have a hard dependency on
diff --git a/buildstream/plugintestutils/runcli.py b/buildstream/plugintestutils/runcli.py
index 71d4b4039..2320189bd 100644
--- a/buildstream/plugintestutils/runcli.py
+++ b/buildstream/plugintestutils/runcli.py
@@ -557,6 +557,65 @@ class CliIntegration(Cli):
         return super().run(*args, **kwargs)
 
 
+class CliRemote(CliIntegration):
+
+    # ensure_services():
+    #
+    # Make sure that required services are configured and that
+    # non-required ones are not.
+    #
+    # Args:
+    #    actions (bool): Whether to use the 'action-cache' service
+    #    artifacts (bool): Whether to use the 'artifact-cache' service
+    #    execution (bool): Whether to use the 'execution' service
+    #    sources (bool): Whether to use the 'source-cache' service
+    #    storage (bool): Whether to use the 'storage' service
+    #
+    # Returns a list of configured services (by names).
+    #
+    def ensure_services(self, actions=True, execution=True, storage=True,
+                        artifacts=False, sources=False):
+        # Build a list of configured services by name:
+        configured_services = []
+        if not self.config:
+            return configured_services
+
+        if 'remote-execution' in self.config:
+            rexec_config = self.config['remote-execution']
+
+            if 'action-cache-service' in rexec_config:
+                if actions:
+                    configured_services.append('action-cache')
+                else:
+                    rexec_config.pop('action-cache-service')
+
+            if 'execution-service' in rexec_config:
+                if execution:
+                    configured_services.append('execution')
+                else:
+                    rexec_config.pop('execution-service')
+
+            if 'storage-service' in rexec_config:
+                if storage:
+                    configured_services.append('storage')
+                else:
+                    rexec_config.pop('storage-service')
+
+        if 'artifacts' in self.config:
+            if artifacts:
+                configured_services.append('artifact-cache')
+            else:
+                self.config.pop('artifacts')
+
+        if 'source-caches' in self.config:
+            if sources:
+                configured_services.append('source-cache')
+            else:
+                self.config.pop('source-caches')
+
+        return configured_services
+
+
 # Main fixture
 #
 # Use result = cli.run([arg1, arg2]) to run buildstream commands
@@ -604,6 +663,47 @@ def cli_integration(tmpdir, integration_cache):
         pass
 
 
+# A variant of the main fixture that is configured for remote-execution.
+#
+# It also does not use the click test runner to avoid deadlock issues
+# when running `bst shell`, but unfortunately cannot produce nice
+# stacktraces.
+@pytest.fixture()
+def cli_remote_execution(tmpdir, remote_services):
+    directory = os.path.join(str(tmpdir), 'cache')
+    os.makedirs(directory)
+
+    fixture = CliRemote(directory)
+
+    if remote_services.artifact_service:
+        fixture.configure({'artifacts': [{
+            'url': remote_services.artifact_service,
+        }]})
+
+    remote_execution = {}
+    if remote_services.action_service:
+        remote_execution['action-cache-service'] = {
+            'url': remote_services.action_service,
+        }
+    if remote_services.exec_service:
+        remote_execution['execution-service'] = {
+            'url': remote_services.exec_service,
+        }
+    if remote_services.storage_service:
+        remote_execution['storage-service'] = {
+            'url': remote_services.storage_service,
+        }
+    if remote_execution:
+        fixture.configure({'remote-execution': remote_execution})
+
+    if remote_services.source_service:
+        fixture.configure({'source-caches': [{
+            'url': remote_services.source_service,
+        }]})
+
+    return fixture
+
+
 @contextmanager
 def chdir(directory):
     old_dir = os.getcwd()
diff --git a/tests/conftest.py b/tests/conftest.py
index 32af72df9..30577870f 100755
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -84,7 +84,6 @@ class IntegrationCache():
 
 @pytest.fixture(scope='session')
 def integration_cache(request):
-
     # Set the cache dir to the INTEGRATION_CACHE variable, or the
     # default if that is not set.
     if 'INTEGRATION_CACHE' in os.environ:
@@ -109,6 +108,40 @@ def integration_cache(request):
 
 
 #################################################
+#           remote_services fixture             #
+#################################################
+#
+# This is returned by the `remote_services` fixture
+#
+class RemoteServices():
+
+    def __init__(self, **kwargs):
+        self.action_service = kwargs.get('action_service')
+        self.artifact_service = kwargs.get('artifact_service')
+        self.exec_service = kwargs.get('exec_service')
+        self.source_service = kwargs.get('source_service')
+        self.storage_service = kwargs.get('storage_service')
+
+
+@pytest.fixture(scope='session')
+def remote_services(request):
+    kwargs = {}
+    # Look for remote services configuration in environment.
+    if 'ARTIFACT_CACHE_SERVICE' in os.environ:
+        kwargs['artifact_service'] = os.environ.get('ARTIFACT_CACHE_SERVICE')
+
+    if 'REMOTE_EXECUTION_SERVICE' in os.environ:
+        kwargs['action_service'] = os.environ.get('REMOTE_EXECUTION_SERVICE')
+        kwargs['exec_service'] = os.environ.get('REMOTE_EXECUTION_SERVICE')
+        kwargs['storage_service'] = os.environ.get('REMOTE_EXECUTION_SERVICE')
+
+    if 'SOURCE_CACHE_SERVICE' in os.environ:
+        kwargs['source_service'] = os.environ.get('SOURCE_CACHE_SERVICE')
+
+    return RemoteServices(**kwargs)
+
+
+#################################################
 #         Automatically reset the platform      #
 #################################################
 #