From f29a099580cd311517701da4bd125fa83c193f66 Mon Sep 17 00:00:00 2001 From: Tristan Van Berkom Date: Tue, 8 Jan 2019 14:22:58 -0500 Subject: Support running test environments in parallel with `detox` This patch namespaces the test temp directory and the output coverage report file with the name of the environment under test, such that separately run tests do not access the same files. When running tests without tox, directly through setup.py, then the tmp directory will still be `./tmp`. * .gitignore: Added new .coverage-reports/ directory * .gitlab-ci.yml: Rely on tox to combine and report coverage, only tell tox about the COVERAGE_PREFIX so that results can be namespaced by CI job name. This change also publishes the sources and final combined `.coverage` file in an output gitlab artifact for inspection, and lists some missing dependencies to the `coverage` job. * tox.ini: Add comments and refactor main [testenv] section so that other environments dont inherit too much unrelated cruft. Generate the coverate reports in the respective {envtmpdir} so that all per-process coverage files are prefixed with a full path, ensuring that concurrent runs don't mix reports and addressing concerns raised in #844. Also implemented new `tox -e coverage` environment to combine any found coverage and print a report. * .coveragerc: Omit .tox/ directory from coverage stats Fixes issue #844 --- .coveragerc | 2 ++ .gitignore | 3 ++- .gitlab-ci.yml | 28 +++++++++++----------------- tox.ini | 42 +++++++++++++++++++++++++++++++++++++++++- 4 files changed, 56 insertions(+), 19 deletions(-) diff --git a/.coveragerc b/.coveragerc index d81aec1a2..af22a1282 100644 --- a/.coveragerc +++ b/.coveragerc @@ -9,6 +9,8 @@ omit = # Omit generated code */buildstream/_protos/* */.eggs/* + # Omit .tox directory + */.tox/* [report] show_missing = True diff --git a/.gitignore b/.gitignore index 3fd1addbc..00cb4e2c4 100644 --- a/.gitignore +++ b/.gitignore @@ -13,11 +13,12 @@ tests/**/*.pyc integration-cache/ tmp .coverage +.coverage-reports/ .coverage.* .cache .pytest_cache/ *.bst/ -.tox +.tox/ # Pycache, in case buildstream is ran directly from within the source # tree diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9706037a4..78e725da8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,7 @@ variables: PYTEST_ADDOPTS: "--color=yes" INTEGRATION_CACHE: "${CI_PROJECT_DIR}/cache/integration-cache" TEST_COMMAND: "tox -- --color=yes --integration" + COVERAGE_PREFIX: "${CI_JOB_NAME}." ##################################################### @@ -24,9 +25,6 @@ variables: .tests-template: &tests stage: test - variables: - COVERAGE_DIR: coverage-linux - before_script: # Diagnostics - mount @@ -40,14 +38,11 @@ variables: - su buildstream -c "${TEST_COMMAND}" after_script: - # Collect our reports - - mkdir -p ${COVERAGE_DIR} - - cp .coverage ${COVERAGE_DIR}/coverage."${CI_JOB_NAME}" except: - schedules artifacts: paths: - - ${COVERAGE_DIR} + - .coverage-reports tests-debian-9: image: buildstream/testsuite-debian:9-5da27168-32c47d1c @@ -83,7 +78,6 @@ tests-unix: <<: *tests variables: BST_FORCE_BACKEND: "unix" - COVERAGE_DIR: coverage-unix script: @@ -239,22 +233,22 @@ coverage: stage: post coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/' script: - - pip3 install -r requirements/requirements.txt -r requirements/dev-requirements.txt - - pip3 install --no-index . - - mkdir report - - cd report - - cp ../coverage-unix/coverage.* . - - cp ../coverage-linux/coverage.* . - - ls coverage.* - - coverage combine --rcfile=../.coveragerc -a coverage.* - - coverage report --rcfile=../.coveragerc -m + - cp -a .coverage-reports/ ./coverage-sources + - tox -e coverage + - cp -a .coverage-reports/ ./coverage-report dependencies: - tests-debian-9 - tests-fedora-27 - tests-fedora-28 + - tests-fedora-missing-deps + - tests-ubuntu-18.04 - tests-unix except: - schedules + artifacts: + paths: + - coverage-sources/ + - coverage-report/ # Deploy, only for merges which land on master branch. # diff --git a/tox.ini b/tox.ini index 3b6496d3f..abc13d03a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,20 @@ +# +# Tox global configuration +# [tox] envlist = py35,py36,py37 skip_missing_interpreters = true +# +# Defaults for all environments +# +# Anything specified here is iherited by the sections +# [testenv] -commands = pytest {posargs} +commands = + pytest --basetemp {envtmpdir} {posargs} + mkdir -p .coverage-reports + mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname} deps = -rrequirements/requirements.txt -rrequirements/dev-requirements.txt @@ -13,6 +24,32 @@ passenv = GI_TYPELIB_PATH INTEGRATION_CACHE +# +# These keys are not inherited by any other sections +# +setenv = + py{35,36,37}: COVERAGE_FILE = {envtmpdir}/.coverage +whitelist_externals = + py{35,36,37}: + mv + mkdir + +# +# Coverage reporting +# +[testenv:coverage] +commands = + - coverage combine --rcfile={toxinidir}/.coveragerc {toxinidir}/.coverage-reports/ + coverage report --rcfile={toxinidir}/.coveragerc -m +deps = + -rrequirements/requirements.txt + -rrequirements/dev-requirements.txt +setenv = + COVERAGE_FILE = {toxinidir}/.coverage-reports/.coverage + +# +# Running linters +# [testenv:lint] commands = pycodestyle @@ -22,6 +59,9 @@ deps = -rrequirements/dev-requirements.txt -rrequirements/plugin-requirements.txt +# +# Building documentation +# [testenv:docs] commands = make -C doc -- cgit v1.2.1 From d364ad0227aa82681d92a1000fddad10ee45ea10 Mon Sep 17 00:00:00 2001 From: Tristan Van Berkom Date: Tue, 8 Jan 2019 21:44:18 -0500 Subject: CONTRIBUTING.rst: Updated to mention new `coverage` tox environment. Also point out that it is possible to run test environments in parallel using the `detox` tool. --- CONTRIBUTING.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 1e672adf1..595fc5389 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1498,6 +1498,13 @@ option when running tox:: tox -e py37 +If you would like to test and lint at the same time, or if you do have multiple +python versions installed and would like to test against multiple versions, then +we recommend using `detox `_, just run it with +the same arguments you would give `tox`:: + + detox -e lint,py36,py37 + Linting is performed separately from testing. In order to run the linting step which consists of running the ``pycodestyle`` and ``pylint`` tools, run the following:: @@ -1574,6 +1581,19 @@ can run ``tox`` with ``-r`` or ``--recreate`` option. ./setup.py test --addopts 'tests/frontend/buildtrack.py::test_build_track' +Observing coverage +~~~~~~~~~~~~~~~~~~ +Once you have run the tests using `tox` (or `detox`), some coverage reports will +have been left behind. + +To view the coverage report of the last test run, simply run:: + + tox -e coverage + +This will collate any reports from separate python environments that may be +under test before displaying the combined coverage. + + Adding tests ~~~~~~~~~~~~ Tests are found in the tests subdirectory, inside of which -- cgit v1.2.1