Merge branch 'qinusty/bb-asset-hub-ci' into 'master'

Add artifact cache testing with buildbarn Asset API implementation

See merge request BuildStream/buildstream!2037

7 files changed, 152 insertions, 6 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b1e673a64..537114a25 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -158,6 +158,8 @@ tests-fedora-update-deps:
 
     - su buildstream -c "${TEST_COMMAND}"
 
+# This template ensures that the server stack defined in COMPOSE_MANIFEST is spun up for script execution
+# and cleaned up after the script is completed.
 .compose-test-boilerplate: &remote-test
   allow_failure: true
   image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION}
@@ -179,10 +181,10 @@ tests-fedora-update-deps:
 
 tests-remote-execution:
   <<: *tests
-  <<: *remote-test
+  <<: *remote-test # Spin up server stack
   variables:
     <<: *docker-variables
-    COMPOSE_MANIFEST: .gitlab-ci/buildgrid-remote-execution.yml
+    COMPOSE_MANIFEST: .gitlab-ci/buildgrid-remote-execution.yml # < *remote-test
     ARTIFACT_CACHE_SERVICE: http://docker:50052
     REMOTE_EXECUTION_SERVICE: http://docker:50051
     SOURCE_CACHE_SERVICE: http://docker:50052
@@ -190,13 +192,23 @@ tests-remote-execution:
 
 tests-remote-cache:
   <<: *tests
-  <<: *remote-test
+  <<: *remote-test # Spin up/down server stack
   variables:
     <<: *docker-variables
-    COMPOSE_MANIFEST: .gitlab-ci/cache-server.yml
+    COMPOSE_MANIFEST: .gitlab-ci/cache-server.yml # < *remote-test
     ARTIFACT_CACHE_SERVICE: http://docker:50052
     PYTEST_ARGS: "--color=yes --remote-cache"
 
+tests-bb-remote-cache:
+  <<: *tests
+  <<: *remote-test # Spin up/down server stack
+  variables:
+    <<: *docker-variables
+    COMPOSE_MANIFEST: .gitlab-ci/buildbarn-remote-cache.yml # < *remote-test
+    ARTIFACT_INDEX_SERVICE: http://docker:7981
+    ARTIFACT_STORAGE_SERVICE: http://docker:7982
+    PYTEST_ARGS: "--color=yes --remote-cache"
+
 tests-no-usedevelop:
   # Ensure that tests also pass without `--develop` flag.
   image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:32-${DOCKER_IMAGE_VERSION}
@@ -356,7 +368,7 @@ overnight-tests:
     # Artifacts
     #
     artifacts:
-    - url: https://cache-test.buildstream.build:11002
+    - url: https://bb-cache.buildstream.build:11002
       client-cert: $OVERNIGHT_CACHE_PUSH_CERT
       client-key: $OVERNIGHT_CACHE_PUSH_KEY
       push: true
diff --git a/.gitlab-ci/buildbarn-remote-cache.yml b/.gitlab-ci/buildbarn-remote-cache.yml
new file mode 100644
index 000000000..0122fa804
--- /dev/null
+++ b/.gitlab-ci/buildbarn-remote-cache.yml
@@ -0,0 +1,59 @@
+##
+# Buildbarn Compose manifest for BuildStream.
+#
+# Spins-up a unnamed and unauthenticated cache server:
+#  - STORAGE at http://localhost:7982
+#  - INDEX at: http://localhost:7981
+#
+# BuildStream configuration snippet:
+#
+# artifacts:
+#   - url: https://localhost:7981
+#     type: index
+#     push: true
+#   - url: https://localhost:7982
+#     type: storage
+#     push: true
+#
+# Basic usage:
+#  - docker-compose -f buildbarn-remote-cache.yml up
+#  - docker-compose -f buildbarn-remote-cache.yml down
+
+version: '3.4'
+
+services:
+  bb-asset:
+    image: qinusty/bb-asset-hub:20200814T141139Z-5e72dd1
+    command: /config/asset.jsonnet
+    restart: unless-stopped
+    expose:
+    - "7981"
+    ports:
+    - "7981:7981"
+    volumes:
+    - type: volume
+      source: assets
+      target: /storage
+    - type: bind
+      source: ./config/
+      target: /config
+
+  bb-storage:
+    image: buildbarn/bb-storage:20200810T194216Z-94a85b4
+    command: /config/storage.jsonnet
+    restart: unless-stopped
+    expose:
+    - "7982"
+    ports:
+    - "7982:7982"
+    volumes:
+    - type: volume
+      source: cas
+      target: /cas
+    - type: bind
+      source: ./config/
+      target: /config
+
+volumes:
+  assets:
+  cas:
diff --git a/.gitlab-ci/config/asset.jsonnet b/.gitlab-ci/config/asset.jsonnet
new file mode 100644
index 000000000..aad06a22e
--- /dev/null
+++ b/.gitlab-ci/config/asset.jsonnet
@@ -0,0 +1,32 @@
+{
+  fetcher: {
+    caching: {
+      fetcher: {
+        // We should never be fetching anything which is not already returned by the caching fetcher.
+        'error': {
+          code: 5,
+          message: "Asset Not Found",
+        }
+      }
+    }
+  },
+
+  assetStore: {
+    circular: {
+      directory: '/storage',
+      offsetFileSizeBytes: 1024 * 1024,
+      offsetCacheSize: 1000,
+      dataFileSizeBytes: 100 * 1024 * 1024,
+      dataAllocationChunkSizeBytes: 1048576,
+      instances: [''],
+    },
+  },
+  httpListenAddress: ':1111',
+  grpcServers: [{
+    listenAddresses: [':7981'],
+    authenticationPolicy: { allow: {} },
+  }],
+  allowUpdatesForInstances: [''],
+  maximumMessageSizeBytes: 16 * 1024 * 1024,
+}
+
diff --git a/.gitlab-ci/config/storage.jsonnet b/.gitlab-ci/config/storage.jsonnet
new file mode 100644
index 000000000..11fbdbee7
--- /dev/null
+++ b/.gitlab-ci/config/storage.jsonnet
@@ -0,0 +1,26 @@
+{
+  blobstore: {
+    contentAddressableStorage: {
+      circular: {
+        directory: '/cas',
+        offsetFileSizeBytes: 16 * 1024 * 1024,
+        offsetCacheSize: 10000,
+        dataFileSizeBytes: 10 * 1024 * 1024 * 1024,
+        dataAllocationChunkSizeBytes: 16 * 1024 * 1024,
+      },
+    },
+    actionCache: {
+      'error': {
+          code: 12, # UNIMPLEMENTED
+          message: "AC requests are not supported for this endpoint.",
+        }
+    },
+  },
+  httpListenAddress: ':6981',
+  grpcServers: [{
+    listenAddresses: [':7982'],
+    authenticationPolicy: { allow: {} },
+  }],
+  allowAcUpdatesForInstanceNamePrefixes: [''],
+  maximumMessageSizeBytes: 16 * 1024 * 1024,
+}
diff --git a/src/buildstream/testing/runcli.py b/src/buildstream/testing/runcli.py
index af15b56f1..de4327cd8 100644
--- a/src/buildstream/testing/runcli.py
+++ b/src/buildstream/testing/runcli.py
@@ -775,8 +775,15 @@ def cli_remote_execution(tmpdir, remote_services):
 
     fixture = CliRemote(directory)
 
+    artifacts = []
     if remote_services.artifact_service:
-        fixture.configure({"artifacts": [{"url": remote_services.artifact_service, "push": True,}]})
+        artifacts.append({"url": remote_services.artifact_service, "push": True})
+    if remote_services.artifact_index_service:
+        artifacts.append({"url": remote_services.artifact_index_service, "push": True, "type": "index"})
+    if remote_services.artifact_storage_service:
+        artifacts.append({"url": remote_services.artifact_storage_service, "push": True, "type": "storage"})
+    if artifacts:
+        fixture.configure({"artifacts": artifacts})
 
     remote_execution = {}
     if remote_services.action_service:
diff --git a/tests/conftest.py b/tests/conftest.py
index 476113105..d79ad40b0 100755
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -101,6 +101,8 @@ class RemoteServices:
         self.exec_service = kwargs.get("exec_service")
         self.source_service = kwargs.get("source_service")
         self.storage_service = kwargs.get("storage_service")
+        self.artifact_index_service = kwargs.get("artifact_index_service")
+        self.artifact_storage_service = kwargs.get("artifact_storage_service")
 
 
 @pytest.fixture(scope="session")
@@ -110,6 +112,12 @@ def remote_services(request):
     if "ARTIFACT_CACHE_SERVICE" in os.environ:
         kwargs["artifact_service"] = os.environ.get("ARTIFACT_CACHE_SERVICE")
 
+    if "ARTIFACT_INDEX_SERVICE" in os.environ:
+        kwargs["artifact_index_service"] = os.environ.get("ARTIFACT_INDEX_SERVICE")
+
+    if "ARTIFACT_STORAGE_SERVICE" in os.environ:
+        kwargs["artifact_storage_service"] = os.environ.get("ARTIFACT_STORAGE_SERVICE")
+
     if "REMOTE_EXECUTION_SERVICE" in os.environ:
         kwargs["action_service"] = os.environ.get("REMOTE_EXECUTION_SERVICE")
         kwargs["exec_service"] = os.environ.get("REMOTE_EXECUTION_SERVICE")
diff --git a/tox.ini b/tox.ini
index 481101ab1..f347e06e3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -49,6 +49,8 @@ deps =
     randomized: pytest-random-order
 passenv =
     ARTIFACT_CACHE_SERVICE
+    ARTIFACT_INDEX_SERVICE
+    ARTIFACT_STORAGE_SERVICE
     BST_CAS_STAGING_ROOT
     GI_TYPELIB_PATH
     INTEGRATION_CACHE