Merge integration tests into general tests

34 files changed, 109 insertions, 69 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 53b0be433..1e44fc7b2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,8 +1,9 @@
 image: buildstream/buildstream-fedora:master-42-571406d
 
 cache:
+  key: "$CI_JOB_NAME-"
   paths:
-    - cache/buildstream/sources/
+    - cache/
 
 stages:
   - prepare
@@ -14,9 +15,9 @@ before_script:
   - mount
   - df -h
 
-  # Store cache in the project directory
-  - if [ -d "$(pwd)/cache" ]; then chmod -R a+rw "$(pwd)/cache"; fi
-  - export XDG_CACHE_HOME="$(pwd)/cache"
+  - mkdir -p "$(pwd)/cache"
+  - chmod -R a+rw "$(pwd)/cache"
+
   - adduser -m buildstream
   - chown -R buildstream:buildstream .
 
@@ -66,10 +67,11 @@ source_dist:
 
 # Run premerge commits
 #
-pytest_linux:
+linux-tests:
   stage: test
   variables:
     PYTEST_ADDOPTS: "--color=yes"
+    INTEGRATION_CACHE: "/builds/BuildStream/cache/"
   script:
 
   # Unpack and get into dist/buildstream
@@ -79,46 +81,24 @@ pytest_linux:
 
   # Run the tests from the source distribution, We run as a simple
   # user to test for permission issues
-  - su buildstream -c 'python3 setup.py test --index-url invalid://uri'
+  - su buildstream -c 'python3 setup.py test --index-url invalid://uri --addopts --integration'
 
   # Go back to the toplevel and collect our reports
   - cd ../..
-  - mkdir -p coverage-pytest-linux/
-  - cp dist/buildstream/.coverage.* coverage-pytest-linux/coverage.pytest-linux
-  artifacts:
-    paths:
-    - coverage-pytest-linux/
-
-# Run integration tests
-#
-integration_linux:
-  stage: test
-
-  script:
-    - cd dist && ./install.sh && cd ..
-    - cd integration-tests
-
-    # We run as a simple user to test for permission issues
-    - su buildstream -c './run-test.sh --arg --colors --cov ../.coveragerc --sources ${XDG_CACHE_HOME}/buildstream/sources test'
-
-    - cd ..
-    - mkdir -p coverage-linux/
-    - cp integration-tests/.coverage coverage-linux/coverage.linux
-    - cp -a integration-tests/tmp/logs logs-linux
-
+  - mkdir -p coverage-linux/
+  - cp dist/buildstream/.coverage.* coverage-linux/coverage.linux
   artifacts:
     paths:
     - coverage-linux/
-    - logs-linux/
-
   dependencies:
   - source_dist
 
-pytest_unix:
+unix-tests:
   stage: test
   variables:
     BST_FORCE_BACKEND: "unix"
     PYTEST_ADDOPTS: "--color=yes"
+    INTEGRATION_CACHE: "/builds/BuildStream/cache/"
   script:
     # We remove the Bubblewrap and OSTree packages here so that we catch any
     # codepaths that try to use them. Removing OSTree causes fuse-libs to
@@ -130,32 +110,12 @@ pytest_unix:
     - cd dist && ./unpack.sh && cd buildstream
 
     # Since the unix platform is required to run as root, no user change required
-    - python3 setup.py test --index-url invalid://uri
+    - python3 setup.py test --index-url invalid://uri --addopts --integration
 
     # Go back to the toplevel and collect our reports
     - cd ../..
-    - mkdir -p coverage-pytest-unix
-    - cp dist/buildstream/.coverage.* coverage-pytest-unix/coverage.pytest-unix
-  artifacts:
-    paths:
-      - coverage-pytest-unix/
-
-integration_unix:
-  stage: test
-  variables:
-    BST_FORCE_BACKEND: "unix"
-  script:
-    - cd dist && ./install.sh && cd ..
-    - cd integration-tests
-
-    # Since the unix platform is required to run as root, no user change required
-    - ./run-test.sh --arg --colors --cov ../.coveragerc --sources ${XDG_CACHE_HOME}/buildstream/sources test
-
-    - cd ..
     - mkdir -p coverage-unix/
-    - cp integration-tests/.coverage coverage-unix/coverage.unix
-    - cp -a integration-tests/tmp/logs logs-unix
-
+    - cp dist/buildstream/.coverage.* coverage-unix/coverage.unix
   artifacts:
     paths:
     - coverage-unix/
@@ -197,22 +157,19 @@ docs:
 coverage:
   stage: post
   script:
+    - cd dist && ./unpack.sh && cd buildstream
     - pip3 install --no-index .
     - mkdir report
     - cd report
-    - cp ../coverage-linux/coverage.linux .coverage
-    - cp ../coverage-unix/coverage.unix .
-    - coverage combine --rcfile=../.coveragerc -a ../coverage-unix/coverage.unix
-    - cp ../coverage-pytest-linux/coverage.pytest-linux .
-    - coverage combine --rcfile=../.coveragerc -a coverage.pytest-linux
-    - cp ../coverage-pytest-unix/coverage.pytest-unix .
-    - coverage combine --rcfile=../.coveragerc -a coverage.pytest-unix
+    - cp ../../../coverage-linux/coverage.linux .
+    - cp ../../../coverage-unix/coverage.unix .
+    - coverage combine --rcfile=../.coveragerc -a coverage.linux
+    - coverage combine --rcfile=../.coveragerc -a coverage.unix
     - coverage report --rcfile=../.coveragerc -m
   dependencies:
-  - pytest_linux
-  - integration_linux
-  - pytest_unix
-  - integration_unix
+  - linux-tests
+  - unix-tests
+  - source_dist
 
 # Deploy, only for merges which land on master branch.
 #
diff --git a/conftest.py b/conftest.py
new file mode 100755
index 000000000..22eaae23b
--- /dev/null
+++ b/conftest.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+#
+#  Copyright (C) 2018 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Maat <tristan.maat@codethink.co.uk>
+
+import os
+import pytest
+import shutil
+import tempfile
+
+
+def pytest_addoption(parser):
+    parser.addoption('--integration', action='store_true', default=False,
+                     help='Run integration tests')
+
+
+def pytest_runtest_setup(item):
+    if item.get_marker('integration') and not item.config.getvalue('integration'):
+        pytest.skip('skipping integration test')
+
+
+@pytest.fixture(scope='session')
+def integration_cache(request):
+
+    # Set the tempdir to the INTEGRATION_CACHE variable, or the
+    # default if that is not set.
+    cache_dir = os.environ.get('INTEGRATION_CACHE', tempfile.gettempdir())
+
+    # We use a separate tempdir to cache sources and artifacts to
+    # increase test speed
+    cache = os.path.join(cache_dir, 'integration-cache')
+    yield cache
+
+    # Clean up the artifacts after each test run - we only want to
+    # cache sources
+    try:
+        shutil.rmtree(os.path.join(cache, 'artifacts'))
+    except FileNotFoundError:
+        pass
diff --git a/integration-tests/elements/project/files/usrmerge.tar.xz b/integration-tests/elements/project/files/usrmerge.tar.xz
deleted file mode 100644
index fa6131af8..000000000
--- a/integration-tests/elements/project/files/usrmerge.tar.xz
+++ /dev/null
diff --git a/setup.cfg b/setup.cfg
index 1cd27506e..146f15347 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -3,7 +3,7 @@ test=pytest
 
 [tool:pytest]
 addopts = --verbose --basetemp ./tmp --pep8 --cov=buildstream --cov-config .coveragerc
-norecursedirs = integration-tests*
+norecursedirs = integration-cache tmp __pycache__
 python_files = tests/*/*.py
 pep8maxlinelength = 119
 pep8ignore =
diff --git a/integration-tests/elements/autotools.py b/tests/integration/autotools.py
index d9ee6a831..a955df1c2 100644
--- a/integration-tests/elements/autotools.py
+++ b/tests/integration/autotools.py
@@ -5,6 +5,9 @@ from tests.testutils import cli_integration as cli
 from tests.testutils.integration import format_files, assert_contains
 
 
+pytestmark = pytest.mark.integration
+
+
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
     "project"
@@ -13,6 +16,7 @@ DATA_DIR = os.path.join(
 
 # Test that an autotools build 'works' - we use the autotools sample
 # amhello project for this.
+@pytest.mark.integration
 @pytest.mark.datafiles(DATA_DIR)
 def test_autotools_build(cli, tmpdir, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)
diff --git a/integration-tests/elements/cmake.py b/tests/integration/cmake.py
index 17b8f8f8a..5a9eacf97 100644
--- a/integration-tests/elements/cmake.py
+++ b/tests/integration/cmake.py
@@ -5,6 +5,9 @@ from tests.testutils import cli_integration as cli
 from tests.testutils.integration import format_files, assert_contains
 
 
+pytestmark = pytest.mark.integration
+
+
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
     "project"
diff --git a/integration-tests/elements/compose.py b/tests/integration/compose.py
index b7d96f979..079c3844f 100644
--- a/integration-tests/elements/compose.py
+++ b/tests/integration/compose.py
@@ -9,6 +9,9 @@ from tests.testutils import cli_integration as cli
 from tests.testutils.integration import format_files, walk_dir
 
 
+pytestmark = pytest.mark.integration
+
+
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
     "project"
diff --git a/integration-tests/elements/import.py b/tests/integration/import.py
index 3a79daef2..ea08ff0c2 100644
--- a/integration-tests/elements/import.py
+++ b/tests/integration/import.py
@@ -7,6 +7,9 @@ from tests.testutils import cli_integration as cli
 from tests.testutils.integration import format_files, walk_dir
 
 
+pytestmark = pytest.mark.integration
+
+
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
     "project"
diff --git a/integration-tests/elements/manual.py b/tests/integration/manual.py
index 76b5a010a..914c90dd4 100644
--- a/integration-tests/elements/manual.py
+++ b/tests/integration/manual.py
@@ -7,6 +7,9 @@ from tests.testutils import cli_integration as cli
 from tests.testutils.integration import format_files, walk_dir
 
 
+pytestmark = pytest.mark.integration
+
+
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
     "project"
diff --git a/integration-tests/elements/pip.py b/tests/integration/pip.py
index c1fd656ec..6471bd242 100644
--- a/integration-tests/elements/pip.py
+++ b/tests/integration/pip.py
@@ -8,6 +8,9 @@ from tests.testutils import cli_integration as cli
 from tests.testutils.integration import assert_contains
 
 
+pytestmark = pytest.mark.integration
+
+
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
     "project"
diff --git a/integration-tests/elements/project/elements/autotools/amhello.bst b/tests/integration/project/elements/autotools/amhello.bst
index ee3a029d8..ee3a029d8 100644
--- a/integration-tests/elements/project/elements/autotools/amhello.bst
+++ b/tests/integration/project/elements/autotools/amhello.bst
diff --git a/integration-tests/elements/project/elements/base.bst b/tests/integration/project/elements/base.bst
index 6f2c56fa9..6f2c56fa9 100644
--- a/integration-tests/elements/project/elements/base.bst
+++ b/tests/integration/project/elements/base.bst
diff --git a/integration-tests/elements/project/elements/base/base-sdk.bst b/tests/integration/project/elements/base/base-sdk.bst
index 6b7418e99..6b7418e99 100644
--- a/integration-tests/elements/project/elements/base/base-sdk.bst
+++ b/tests/integration/project/elements/base/base-sdk.bst
diff --git a/integration-tests/elements/project/elements/base/usermerge.bst b/tests/integration/project/elements/base/usermerge.bst
index d21702c12..fb9cffba9 100644
--- a/integration-tests/elements/project/elements/base/usermerge.bst
+++ b/tests/integration/project/elements/base/usermerge.bst
@@ -3,4 +3,4 @@ description: Some symlinks for the flatpak runtime environment
 sources:
   - kind: tar
     url: project_dir:/files/usrmerge.tar.xz
-    ref: 14dd149ad8b177b0d8e42f1a4e522cb99a00ed666e2d70777e031a1ee6348265
+    ref: 96e4458fafb85c10fe6271ad8839754b6eabeceac8523f8f602dea6470db5961
diff --git a/integration-tests/elements/project/elements/cmake/step7.bst b/tests/integration/project/elements/cmake/step7.bst
index 625bb8583..625bb8583 100644
--- a/integration-tests/elements/project/elements/cmake/step7.bst
+++ b/tests/integration/project/elements/cmake/step7.bst
diff --git a/integration-tests/elements/project/elements/compose/amhello.bst b/tests/integration/project/elements/compose/amhello.bst
index bec7e312a..bec7e312a 100644
--- a/integration-tests/elements/project/elements/compose/amhello.bst
+++ b/tests/integration/project/elements/compose/amhello.bst
diff --git a/integration-tests/elements/project/elements/compose/test.bst b/tests/integration/project/elements/compose/test.bst
index 870dc7daa..870dc7daa 100644
--- a/integration-tests/elements/project/elements/compose/test.bst
+++ b/tests/integration/project/elements/compose/test.bst
diff --git a/integration-tests/elements/project/elements/script/script-layout.bst b/tests/integration/project/elements/script/script-layout.bst
index 11ca353e3..11ca353e3 100644
--- a/integration-tests/elements/project/elements/script/script-layout.bst
+++ b/tests/integration/project/elements/script/script-layout.bst
diff --git a/integration-tests/elements/project/elements/script/script.bst b/tests/integration/project/elements/script/script.bst
index ffca23ab7..ffca23ab7 100644
--- a/integration-tests/elements/project/elements/script/script.bst
+++ b/tests/integration/project/elements/script/script.bst
diff --git a/integration-tests/elements/project/elements/stack/another-hi.bst b/tests/integration/project/elements/stack/another-hi.bst
index eb98dea36..eb98dea36 100644
--- a/integration-tests/elements/project/elements/stack/another-hi.bst
+++ b/tests/integration/project/elements/stack/another-hi.bst
diff --git a/integration-tests/elements/project/elements/stack/hi.bst b/tests/integration/project/elements/stack/hi.bst
index adbf81537..adbf81537 100644
--- a/integration-tests/elements/project/elements/stack/hi.bst
+++ b/tests/integration/project/elements/stack/hi.bst
diff --git a/integration-tests/elements/project/elements/stack/stack.bst b/tests/integration/project/elements/stack/stack.bst
index bbfc31ad4..bbfc31ad4 100644
--- a/integration-tests/elements/project/elements/stack/stack.bst
+++ b/tests/integration/project/elements/stack/stack.bst
diff --git a/integration-tests/elements/project/files/amhello.tar.gz b/tests/integration/project/files/amhello.tar.gz
index afe189908..afe189908 100644
--- a/integration-tests/elements/project/files/amhello.tar.gz
+++ b/tests/integration/project/files/amhello.tar.gz
diff --git a/integration-tests/elements/project/files/hello.tar.xz b/tests/integration/project/files/hello.tar.xz
index 72ec9b399..72ec9b399 100644
--- a/integration-tests/elements/project/files/hello.tar.xz
+++ b/tests/integration/project/files/hello.tar.xz
diff --git a/integration-tests/elements/project/files/import-source/subdir/test.txt b/tests/integration/project/files/import-source/subdir/test.txt
index d73906c87..d73906c87 100644
--- a/integration-tests/elements/project/files/import-source/subdir/test.txt
+++ b/tests/integration/project/files/import-source/subdir/test.txt
diff --git a/integration-tests/elements/project/files/import-source/test.txt b/tests/integration/project/files/import-source/test.txt
index 0527e6bd2..0527e6bd2 100644
--- a/integration-tests/elements/project/files/import-source/test.txt
+++ b/tests/integration/project/files/import-source/test.txt
diff --git a/integration-tests/elements/project/files/step7.tar.gz b/tests/integration/project/files/step7.tar.gz
index f9643afce..f9643afce 100644
--- a/integration-tests/elements/project/files/step7.tar.gz
+++ b/tests/integration/project/files/step7.tar.gz
diff --git a/tests/integration/project/files/usrmerge.tar.xz b/tests/integration/project/files/usrmerge.tar.xz
new file mode 100644
index 000000000..e54fbbb43
--- /dev/null
+++ b/tests/integration/project/files/usrmerge.tar.xz
diff --git a/integration-tests/elements/project/keys/gnome-sdk.gpg b/tests/integration/project/keys/gnome-sdk.gpg
index 8434b686c..8434b686c 100644
--- a/integration-tests/elements/project/keys/gnome-sdk.gpg
+++ b/tests/integration/project/keys/gnome-sdk.gpg
diff --git a/integration-tests/elements/project/project.conf b/tests/integration/project/project.conf
index a677129fb..a677129fb 100644
--- a/integration-tests/elements/project/project.conf
+++ b/tests/integration/project/project.conf
diff --git a/integration-tests/elements/script.py b/tests/integration/script.py
index f720d1ebb..d7448c4d5 100644
--- a/integration-tests/elements/script.py
+++ b/tests/integration/script.py
@@ -7,6 +7,9 @@ from tests.testutils import cli_integration as cli
 from tests.testutils.integration import format_files, walk_dir
 
 
+pytestmark = pytest.mark.integration
+
+
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
     "project"
diff --git a/integration-tests/elements/shell.py b/tests/integration/shell.py
index 7606e3bd2..1e14f736c 100644
--- a/integration-tests/elements/shell.py
+++ b/tests/integration/shell.py
@@ -7,6 +7,9 @@ from buildstream import _yaml
 from tests.testutils import cli_integration as cli
 
 
+pytestmark = pytest.mark.integration
+
+
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
     "project"
diff --git a/integration-tests/elements/stack.py b/tests/integration/stack.py
index 9650578de..a049b4892 100644
--- a/integration-tests/elements/stack.py
+++ b/tests/integration/stack.py
@@ -7,6 +7,9 @@ from tests.testutils import cli_integration as cli
 from tests.testutils.integration import format_files, walk_dir
 
 
+pytestmark = pytest.mark.integration
+
+
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
     "project"
diff --git a/tests/testutils/runcli.py b/tests/testutils/runcli.py
index bc33a4a2a..3479364f7 100644
--- a/tests/testutils/runcli.py
+++ b/tests/testutils/runcli.py
@@ -402,7 +402,7 @@ def cli(tmpdir):
 # when running `bst shell`, but unfortunately cannot produce nice
 # stacktraces.
 @pytest.fixture()
-def cli_integration(tmpdir):
+def cli_integration(tmpdir, integration_cache):
     directory = os.path.join(str(tmpdir), 'cache')
     os.makedirs(directory)
 
@@ -414,7 +414,8 @@ def cli_integration(tmpdir):
     # We want to cache sources for integration tests more permanently,
     # to avoid downloading the huge base-sdk repeatedly
     fixture.configure({
-        'sourcedir': os.path.join(os.getcwd(), 'integration-cache', 'sources')
+        'sourcedir': os.path.join(integration_cache, 'sources'),
+        'artifactdir': os.path.join(integration_cache, 'artifacts')
     })
 
     return fixture