Merge branch 'mablanch/629-remote-execution-test' into 'master'

Add remote execution tests to the CI pipeline

See merge request BuildStream/buildstream!1239

12 files changed, 371 insertions, 7 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 210de8df4..2d3a24700 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,6 +8,7 @@ cache:
 stages:
   - test
   - post
+  - publish
 
 variables:
   PYTEST_ADDOPTS: "--color=yes"
@@ -138,6 +139,29 @@ tests-fedora-update-deps:
 
     - su buildstream -c "${TEST_COMMAND}"
 
+tests-remote-execution:
+  allow_failure: true
+  image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:29-master-47052095
+  <<: *tests
+  before_script:
+    - dnf install -y docker docker-compose
+    - docker-compose --file ${COMPOSE_MANIFEST} up --detach
+  after_script:
+    - docker-compose --file ${COMPOSE_MANIFEST} stop
+    - docker-compose --file ${COMPOSE_MANIFEST} logs
+    - docker-compose --file ${COMPOSE_MANIFEST} down
+  services:
+    - docker:stable-dind
+  variables:
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_DRIVER: overlay2
+    COMPOSE_MANIFEST: .gitlab-ci/buildgrid-compose.yml
+    ARTIFACT_CACHE_SERVICE: http://docker:50052
+    REMOTE_EXECUTION_SERVICE: http://docker:50051
+    SOURCE_CACHE_SERVICE: http://docker:50052
+    PYTEST_ARGS: "--color=yes --remote-execution"
+
+
 # Lint separately from testing
 lint:
   stage: test
@@ -308,6 +332,7 @@ coverage:
   - tests-fedora-29
   - tests-fedora-missing-deps
   - tests-fedora-update-deps
+  - tests-remote-execution
   - tests-ubuntu-18.04
   - tests-unix
   except:
@@ -320,17 +345,21 @@ coverage:
 # Deploy, only for merges which land on master branch.
 #
 pages:
-  stage: post
+  stage: publish
   dependencies:
+  - coverage
   - docs
   variables:
     ACME_DIR: public/.well-known/acme-challenge
+    COVERAGE_DIR: public/coverage
   script:
   - mkdir -p ${ACME_DIR}
     # Required to finish the creation of the Let's Encrypt certificate,
     # which allows using https://docs.buildstream.build/ for accessing
     # the documentation.
   - echo ${ACME_CHALLENGE} > ${ACME_DIR}/$(echo ${ACME_CHALLENGE} | cut -c1-43)
+  - mkdir -p ${COVERAGE_DIR}
+  - cp -a ./coverage-report/ ${COVERAGE_DIR}
   artifacts:
     paths:
     - public/
diff --git a/.gitlab-ci/buildgrid-compose.yml b/.gitlab-ci/buildgrid-compose.yml
new file mode 100644
index 000000000..09165818b
--- /dev/null
+++ b/.gitlab-ci/buildgrid-compose.yml
@@ -0,0 +1,72 @@
+##
+# BuildGrid Compose manifest for BuildStream.
+#
+# Spins-up a unnamed and unauthenticated grid:
+#  - Controller + CAS + AC at http://localhost:50051
+#  - Ref. + CAS at: http://localhost:50052
+#
+# BuildStream configuration snippet:
+#
+#    artifacts:
+#      url: http://localhost:50052
+#      push: true
+#    remote-execution:
+#      execution-service:
+#        url: http://localhost:50051
+#      action-cache-service:
+#        url: http://localhost:50051
+#      storage-service:
+#        url: http://localhost:50051
+#
+# Basic usage:
+#  - docker-compose -f buildgrid-compose.yml up
+#  - docker-compose -f buildgrid-compose.yml down
+#
+version: "3.2"
+
+services:
+  controller:
+    image: registry.gitlab.com/buildgrid/buildgrid.hub.docker.com/buildgrid:nightly
+    command: [
+      "bgd", "server", "start", "-vvv",
+      "/etc/buildgrid/default.conf"]
+    ports:
+      - 50051:50051
+    networks:
+      - grid
+
+  bot:
+    image: registry.gitlab.com/buildgrid/buildgrid.hub.docker.com/buildgrid:nightly
+    command: [
+      "bgd", "bot", "--parent=", "-vvv",
+      "--remote=http://controller:50051",
+      "--remote-cas=http://controller:50051",
+      "buildbox",
+      "--local-cas", "/var/lib/buildgrid/cache",
+      "--fuse-dir", "/mnt"]
+    privileged: true
+    volumes:
+      - type: volume
+        source: cache
+        target: /var/lib/buildgrid/cache
+    depends_on:
+      - controller
+    networks:
+      - grid
+
+  storage:
+    image: registry.gitlab.com/buildgrid/buildgrid.hub.docker.com/buildgrid:nightly
+    command: [
+      "bgd", "server", "start", "-vvv",
+      "/etc/buildgrid/artifacts.conf"]
+    ports:
+      - 50052:50052
+    networks:
+      - grid
+
+networks:
+  grid:
+    driver: bridge
+
+volumes:
+  cache:
diff --git a/buildstream/plugintestutils/__init__.py b/buildstream/plugintestutils/__init__.py
index c7238a29c..9ec18df19 100644
--- a/buildstream/plugintestutils/__init__.py
+++ b/buildstream/plugintestutils/__init__.py
@@ -16,7 +16,7 @@
 #  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 
 
-from .runcli import cli, cli_integration
+from .runcli import cli, cli_integration, cli_remote_execution
 
 # To make use of these test utilities it is necessary to have pytest
 # available. However, we don't want to have a hard dependency on
diff --git a/buildstream/plugintestutils/runcli.py b/buildstream/plugintestutils/runcli.py
index 71d4b4039..2320189bd 100644
--- a/buildstream/plugintestutils/runcli.py
+++ b/buildstream/plugintestutils/runcli.py
@@ -557,6 +557,65 @@ class CliIntegration(Cli):
         return super().run(*args, **kwargs)
 
 
+class CliRemote(CliIntegration):
+
+    # ensure_services():
+    #
+    # Make sure that required services are configured and that
+    # non-required ones are not.
+    #
+    # Args:
+    #    actions (bool): Whether to use the 'action-cache' service
+    #    artifacts (bool): Whether to use the 'artifact-cache' service
+    #    execution (bool): Whether to use the 'execution' service
+    #    sources (bool): Whether to use the 'source-cache' service
+    #    storage (bool): Whether to use the 'storage' service
+    #
+    # Returns a list of configured services (by names).
+    #
+    def ensure_services(self, actions=True, execution=True, storage=True,
+                        artifacts=False, sources=False):
+        # Build a list of configured services by name:
+        configured_services = []
+        if not self.config:
+            return configured_services
+
+        if 'remote-execution' in self.config:
+            rexec_config = self.config['remote-execution']
+
+            if 'action-cache-service' in rexec_config:
+                if actions:
+                    configured_services.append('action-cache')
+                else:
+                    rexec_config.pop('action-cache-service')
+
+            if 'execution-service' in rexec_config:
+                if execution:
+                    configured_services.append('execution')
+                else:
+                    rexec_config.pop('execution-service')
+
+            if 'storage-service' in rexec_config:
+                if storage:
+                    configured_services.append('storage')
+                else:
+                    rexec_config.pop('storage-service')
+
+        if 'artifacts' in self.config:
+            if artifacts:
+                configured_services.append('artifact-cache')
+            else:
+                self.config.pop('artifacts')
+
+        if 'source-caches' in self.config:
+            if sources:
+                configured_services.append('source-cache')
+            else:
+                self.config.pop('source-caches')
+
+        return configured_services
+
+
 # Main fixture
 #
 # Use result = cli.run([arg1, arg2]) to run buildstream commands
@@ -604,6 +663,47 @@ def cli_integration(tmpdir, integration_cache):
         pass
 
 
+# A variant of the main fixture that is configured for remote-execution.
+#
+# It also does not use the click test runner to avoid deadlock issues
+# when running `bst shell`, but unfortunately cannot produce nice
+# stacktraces.
+@pytest.fixture()
+def cli_remote_execution(tmpdir, remote_services):
+    directory = os.path.join(str(tmpdir), 'cache')
+    os.makedirs(directory)
+
+    fixture = CliRemote(directory)
+
+    if remote_services.artifact_service:
+        fixture.configure({'artifacts': [{
+            'url': remote_services.artifact_service,
+        }]})
+
+    remote_execution = {}
+    if remote_services.action_service:
+        remote_execution['action-cache-service'] = {
+            'url': remote_services.action_service,
+        }
+    if remote_services.exec_service:
+        remote_execution['execution-service'] = {
+            'url': remote_services.exec_service,
+        }
+    if remote_services.storage_service:
+        remote_execution['storage-service'] = {
+            'url': remote_services.storage_service,
+        }
+    if remote_execution:
+        fixture.configure({'remote-execution': remote_execution})
+
+    if remote_services.source_service:
+        fixture.configure({'source-caches': [{
+            'url': remote_services.source_service,
+        }]})
+
+    return fixture
+
+
 @contextmanager
 def chdir(directory):
     old_dir = os.getcwd()
diff --git a/tests/conftest.py b/tests/conftest.py
index 7f8da3633..30577870f 100755
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -38,10 +38,25 @@ def pytest_addoption(parser):
     parser.addoption('--integration', action='store_true', default=False,
                      help='Run integration tests')
 
+    parser.addoption('--remote-execution', action='store_true', default=False,
+                     help='Run remote-execution tests only')
+
 
 def pytest_runtest_setup(item):
-    if item.get_closest_marker('integration') and not item.config.getvalue('integration'):
-        pytest.skip('skipping integration test')
+    # Without --integration: skip tests not marked with 'integration'
+    if not item.config.getvalue('integration'):
+        if item.get_closest_marker('integration'):
+            pytest.skip('skipping integration test')
+
+    # With --remote-execution: only run tests marked with 'remoteexecution'
+    if item.config.getvalue('remote_execution'):
+        if not item.get_closest_marker('remoteexecution'):
+            pytest.skip('skipping non remote-execution test')
+
+    # Without --remote-execution: skip tests marked with 'remoteexecution'
+    else:
+        if item.get_closest_marker('remoteexecution'):
+            pytest.skip('skipping remote-execution test')
 
 
 #################################################
@@ -69,7 +84,6 @@ class IntegrationCache():
 
 @pytest.fixture(scope='session')
 def integration_cache(request):
-
     # Set the cache dir to the INTEGRATION_CACHE variable, or the
     # default if that is not set.
     if 'INTEGRATION_CACHE' in os.environ:
@@ -94,6 +108,40 @@ def integration_cache(request):
 
 
 #################################################
+#           remote_services fixture             #
+#################################################
+#
+# This is returned by the `remote_services` fixture
+#
+class RemoteServices():
+
+    def __init__(self, **kwargs):
+        self.action_service = kwargs.get('action_service')
+        self.artifact_service = kwargs.get('artifact_service')
+        self.exec_service = kwargs.get('exec_service')
+        self.source_service = kwargs.get('source_service')
+        self.storage_service = kwargs.get('storage_service')
+
+
+@pytest.fixture(scope='session')
+def remote_services(request):
+    kwargs = {}
+    # Look for remote services configuration in environment.
+    if 'ARTIFACT_CACHE_SERVICE' in os.environ:
+        kwargs['artifact_service'] = os.environ.get('ARTIFACT_CACHE_SERVICE')
+
+    if 'REMOTE_EXECUTION_SERVICE' in os.environ:
+        kwargs['action_service'] = os.environ.get('REMOTE_EXECUTION_SERVICE')
+        kwargs['exec_service'] = os.environ.get('REMOTE_EXECUTION_SERVICE')
+        kwargs['storage_service'] = os.environ.get('REMOTE_EXECUTION_SERVICE')
+
+    if 'SOURCE_CACHE_SERVICE' in os.environ:
+        kwargs['source_service'] = os.environ.get('SOURCE_CACHE_SERVICE')
+
+    return RemoteServices(**kwargs)
+
+
+#################################################
 #         Automatically reset the platform      #
 #################################################
 #
diff --git a/tests/remoteexecution/project/elements/autotools/amhello.bst b/tests/remoteexecution/project/elements/autotools/amhello.bst
new file mode 100644
index 000000000..ee3a029d8
--- /dev/null
+++ b/tests/remoteexecution/project/elements/autotools/amhello.bst
@@ -0,0 +1,10 @@
+kind: autotools
+description: Autotools test
+
+depends:
+- base.bst
+
+sources:
+- kind: tar
+  url: project_dir:/files/amhello.tar.gz
+  ref: 9ba123fa4e660929e9a0aa99f0c487b7eee59c5e7594f3284d015640b90f5590
diff --git a/tests/remoteexecution/project/elements/base.bst b/tests/remoteexecution/project/elements/base.bst
new file mode 100644
index 000000000..428afa736
--- /dev/null
+++ b/tests/remoteexecution/project/elements/base.bst
@@ -0,0 +1,5 @@
+# elements/base.bst
+
+kind: stack
+depends:
+  - base/base-alpine.bst
diff --git a/tests/remoteexecution/project/elements/base/base-alpine.bst b/tests/remoteexecution/project/elements/base/base-alpine.bst
new file mode 100644
index 000000000..c5833095d
--- /dev/null
+++ b/tests/remoteexecution/project/elements/base/base-alpine.bst
@@ -0,0 +1,17 @@
+kind: import
+
+description: |
+  Alpine Linux base for tests
+
+  Generated using the `tests/integration-tests/base/generate-base.sh` script.
+
+sources:
+  - kind: tar
+    base-dir: ''
+    (?):
+    - arch == "x86-64":
+        ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639
+        url: "alpine:integration-tests-base.v1.x86_64.tar.xz"
+    - arch == "aarch64":
+        ref: 431fb5362032ede6f172e70a3258354a8fd71fcbdeb1edebc0e20968c792329a
+        url: "alpine:integration-tests-base.v1.aarch64.tar.xz"
diff --git a/tests/remoteexecution/project/files/amhello.tar.gz b/tests/remoteexecution/project/files/amhello.tar.gz
new file mode 100644
index 000000000..afe189908
--- /dev/null
+++ b/tests/remoteexecution/project/files/amhello.tar.gz
diff --git a/tests/remoteexecution/project/project.conf b/tests/remoteexecution/project/project.conf
new file mode 100644
index 000000000..ddfe47b6d
--- /dev/null
+++ b/tests/remoteexecution/project/project.conf
@@ -0,0 +1,23 @@
+# Project config for frontend build test
+name: test
+element-path: elements
+aliases:
+  alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/
+  project_dir: file://{project_dir}
+options:
+  linux:
+    type: bool
+    description: Whether to expect a linux platform
+    default: True
+  arch:
+    type: arch
+    description: Current architecture
+    values:
+      - x86-64
+      - aarch64
+split-rules:
+  test:
+    - |
+      /tests
+    - |
+      /tests/*
diff --git a/tests/remoteexecution/simple.py b/tests/remoteexecution/simple.py
new file mode 100644
index 000000000..152a40e31
--- /dev/null
+++ b/tests/remoteexecution/simple.py
@@ -0,0 +1,56 @@
+import os
+import pytest
+
+from buildstream.plugintestutils import cli_remote_execution as cli
+from buildstream.plugintestutils.integration import assert_contains
+
+
+pytestmark = pytest.mark.remoteexecution
+
+
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    "project"
+)
+
+
+# Test building an executable with remote-execution:
+@pytest.mark.datafiles(DATA_DIR)
+def test_remote_autotools_build(cli, datafiles):
+    project = str(datafiles)
+    checkout = os.path.join(cli.directory, 'checkout')
+    element_name = 'autotools/amhello.bst'
+
+    services = cli.ensure_services()
+    assert set(services) == set(['action-cache', 'execution', 'storage'])
+
+    result = cli.run(project=project, args=['build', element_name])
+    result.assert_success()
+
+    result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkout])
+    result.assert_success()
+
+    assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin',
+                               '/usr/share',
+                               '/usr/bin/hello', '/usr/share/doc',
+                               '/usr/share/doc/amhello',
+                               '/usr/share/doc/amhello/README'])
+
+
+# Test running an executable built with remote-execution:
+@pytest.mark.datafiles(DATA_DIR)
+def test_remote_autotools_run(cli, datafiles):
+    project = str(datafiles)
+    element_name = 'autotools/amhello.bst'
+
+    services = cli.ensure_services()
+    assert set(services) == set(['action-cache', 'execution', 'storage'])
+
+    services = cli.ensure_services()
+
+    result = cli.run(project=project, args=['build', element_name])
+    result.assert_success()
+
+    result = cli.run(project=project, args=['shell', element_name, '/usr/bin/hello'])
+    result.assert_success()
+    assert result.output == 'Hello World!\nThis is amhello 1.0.\n'
diff --git a/tox.ini b/tox.ini
index 21c94e6fa..29ea10ffb 100644
--- a/tox.ini
+++ b/tox.ini
@@ -28,6 +28,7 @@ deps =
     # Only require coverage and pytest-cov when using it
     !nocover: -rrequirements/cov-requirements.txt
 passenv =
+    ARTIFACT_CACHE_SERVICE
     BST_FORCE_BACKEND
     GI_TYPELIB_PATH
     INTEGRATION_CACHE
@@ -37,6 +38,8 @@ passenv =
     HTTPS_PROXY
     no_proxy
     NO_PROXY
+    REMOTE_EXECUTION_SERVICE
+    SOURCE_CACHE_SERVICE
     SSL_CERT_FILE
 
 #
@@ -58,8 +61,9 @@ whitelist_externals =
 #
 [testenv:coverage]
 commands =
-    - coverage combine --rcfile={toxinidir}/.coveragerc {toxinidir}/.coverage-reports/
-    coverage report --rcfile={toxinidir}/.coveragerc -m
+    coverage combine --rcfile={toxinidir}/.coveragerc {toxinidir}/.coverage-reports/
+    coverage html --rcfile={toxinidir}/.coveragerc --directory={toxinidir}/.coverage-reports/
+    coverage report --rcfile={toxinidir}/.coveragerc --show-missing
 deps =
     -rrequirements/requirements.txt
     -rrequirements/dev-requirements.txt