Merge branch 'tristan/detox-tests' into 'master'

Allow using detox, and fix/refactor collection of coverage reports

Closes #844

See merge request BuildStream/buildstream!1051

5 files changed, 76 insertions, 19 deletions

diff --git a/.coveragerc b/.coveragerc
index d81aec1a2..af22a1282 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -9,6 +9,8 @@ omit =
   # Omit generated code
   */buildstream/_protos/*
   */.eggs/*
+  # Omit .tox directory
+  */.tox/*
 
 [report]
 show_missing = True
diff --git a/.gitignore b/.gitignore
index 3fd1addbc..00cb4e2c4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,11 +13,12 @@ tests/**/*.pyc
 integration-cache/
 tmp
 .coverage
+.coverage-reports/
 .coverage.*
 .cache
 .pytest_cache/
 *.bst/
-.tox
+.tox/
 
 # Pycache, in case buildstream is ran directly from within the source
 # tree
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9706037a4..78e725da8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -13,6 +13,7 @@ variables:
   PYTEST_ADDOPTS: "--color=yes"
   INTEGRATION_CACHE: "${CI_PROJECT_DIR}/cache/integration-cache"
   TEST_COMMAND: "tox -- --color=yes --integration"
+  COVERAGE_PREFIX: "${CI_JOB_NAME}."
 
 
 #####################################################
@@ -24,9 +25,6 @@ variables:
 .tests-template: &tests
   stage: test
 
-  variables:
-    COVERAGE_DIR: coverage-linux
-
   before_script:
   # Diagnostics
   - mount
@@ -40,14 +38,11 @@ variables:
   - su buildstream -c "${TEST_COMMAND}"
 
   after_script:
-  # Collect our reports
-  - mkdir -p ${COVERAGE_DIR}
-  - cp .coverage ${COVERAGE_DIR}/coverage."${CI_JOB_NAME}"
   except:
   - schedules
   artifacts:
     paths:
-    - ${COVERAGE_DIR}
+    - .coverage-reports
 
 tests-debian-9:
   image: buildstream/testsuite-debian:9-5da27168-32c47d1c
@@ -83,7 +78,6 @@ tests-unix:
   <<: *tests
   variables:
     BST_FORCE_BACKEND: "unix"
-    COVERAGE_DIR: coverage-unix
 
   script:
 
@@ -239,22 +233,22 @@ coverage:
   stage: post
   coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
   script:
-    - pip3 install -r requirements/requirements.txt -r requirements/dev-requirements.txt
-    - pip3 install --no-index .
-    - mkdir report
-    - cd report
-    - cp ../coverage-unix/coverage.* .
-    - cp ../coverage-linux/coverage.* .
-    - ls coverage.*
-    - coverage combine --rcfile=../.coveragerc -a coverage.*
-    - coverage report --rcfile=../.coveragerc -m
+    - cp -a .coverage-reports/ ./coverage-sources
+    - tox -e coverage
+    - cp -a .coverage-reports/ ./coverage-report
   dependencies:
   - tests-debian-9
   - tests-fedora-27
   - tests-fedora-28
+  - tests-fedora-missing-deps
+  - tests-ubuntu-18.04
   - tests-unix
   except:
   - schedules
+  artifacts:
+    paths:
+    - coverage-sources/
+    - coverage-report/
 
 # Deploy, only for merges which land on master branch.
 #
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
index 1e672adf1..595fc5389 100644
--- a/CONTRIBUTING.rst
+++ b/CONTRIBUTING.rst
@@ -1498,6 +1498,13 @@ option when running tox::
 
   tox -e py37
 
+If you would like to test and lint at the same time, or if you do have multiple
+python versions installed and would like to test against multiple versions, then
+we recommend using `detox <https://github.com/tox-dev/detox>`_, just run it with
+the same arguments you would give `tox`::
+
+  detox -e lint,py36,py37
+
 Linting is performed separately from testing. In order to run the linting step which
 consists of running the ``pycodestyle`` and ``pylint`` tools, run the following::
 
@@ -1574,6 +1581,19 @@ can run ``tox`` with ``-r`` or  ``--recreate`` option.
      ./setup.py test --addopts 'tests/frontend/buildtrack.py::test_build_track'
 
 
+Observing coverage
+~~~~~~~~~~~~~~~~~~
+Once you have run the tests using `tox` (or `detox`), some coverage reports will
+have been left behind.
+
+To view the coverage report of the last test run, simply run::
+
+  tox -e coverage
+
+This will collate any reports from separate python environments that may be
+under test before displaying the combined coverage.
+
+
 Adding tests
 ~~~~~~~~~~~~
 Tests are found in the tests subdirectory, inside of which
diff --git a/tox.ini b/tox.ini
index 3b6496d3f..abc13d03a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,9 +1,20 @@
+#
+# Tox global configuration
+#
 [tox]
 envlist = py35,py36,py37
 skip_missing_interpreters = true
 
+#
+# Defaults for all environments
+#
+# Anything specified here is iherited by the sections
+#
 [testenv]
-commands = pytest {posargs}
+commands =
+    pytest --basetemp {envtmpdir} {posargs}
+    mkdir -p .coverage-reports
+    mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname}
 deps =
     -rrequirements/requirements.txt
     -rrequirements/dev-requirements.txt
@@ -13,6 +24,32 @@ passenv =
     GI_TYPELIB_PATH
     INTEGRATION_CACHE
 
+#
+# These keys are not inherited by any other sections
+#
+setenv =
+    py{35,36,37}: COVERAGE_FILE = {envtmpdir}/.coverage
+whitelist_externals =
+    py{35,36,37}:
+        mv
+        mkdir
+
+#
+# Coverage reporting
+#
+[testenv:coverage]
+commands =
+    - coverage combine --rcfile={toxinidir}/.coveragerc {toxinidir}/.coverage-reports/
+    coverage report --rcfile={toxinidir}/.coveragerc -m
+deps =
+    -rrequirements/requirements.txt
+    -rrequirements/dev-requirements.txt
+setenv =
+    COVERAGE_FILE = {toxinidir}/.coverage-reports/.coverage
+
+#
+# Running linters
+#
 [testenv:lint]
 commands =
     pycodestyle
@@ -22,6 +59,9 @@ deps =
     -rrequirements/dev-requirements.txt
     -rrequirements/plugin-requirements.txt
 
+#
+# Building documentation
+#
 [testenv:docs]
 commands =
     make -C doc