diff options
author | Benjamin Schubert <bschubert15@bloomberg.net> | 2019-12-04 10:10:15 +0000 |
---|---|---|
committer | Benjamin Schubert <bschubert15@bloomberg.net> | 2019-12-10 13:50:43 +0000 |
commit | 6c5c271f254e492b470d9ec553e1866007eca4b2 (patch) | |
tree | 2df37fb8e949a157abf7d75aaca4f841d631426a | |
parent | 397a8fe5e8d32a8ec6a9ef252cd7a55007ad4f66 (diff) | |
download | buildstream-6c5c271f254e492b470d9ec553e1866007eca4b2.tar.gz |
tox.ini: Add ability to run tests in a randomized order
Also register them to run nightly.
Running tests in a randomized order is a good way of checking whether
our tests depend on each others or not.
Since we order them globally, tests will take longer to run, but that
will give us higher confidence that they are correct.
-rw-r--r-- | .gitlab-ci.yml | 20 | ||||
-rw-r--r-- | tox.ini | 4 |
2 files changed, 24 insertions, 0 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 05932c443..8cf593f0a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -366,6 +366,26 @@ overnight-tests-no-cache: tags: - overnight-tests +# These tests might be a bit more flaky since they randomize the order +# They will also take longer to run. Hence run them only nightly +overnight-randomized: + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:31-master-97748867 + <<: *tests + + script: + - mkdir -p "${INTEGRATION_CACHE}" + - useradd -Um buildstream + - chown -R buildstream:buildstream . + + # Don't run tests multiprocessed here, the randomized order doesn't like that + - su buildstream -c "tox -e py35-randomized,py36-randomized,py37-randomized -- --color=yes --integration" + - su buildstream -c "tox -e py35-randomized,py36-randomized,py37-randomized-external -- --color=yes --integration" + + # We need to override the exclusion from the template in order to run on schedules + except: [] + only: + - schedules + ##################################################### # Post stage # ##################################################### @@ -34,6 +34,9 @@ deps = # Only require coverage and pytest-cov when using it !nocover: -rrequirements/cov-requirements.txt + + # Install pytest-random-order for '-randomized' + randomized: pytest-random-order passenv = ARTIFACT_CACHE_SERVICE BST_FORCE_BACKEND @@ -61,6 +64,7 @@ setenv = py{35,36,37}: BST_TEST_XDG_DATA_HOME = {envtmpdir}/share # This is required to get coverage for Cython py{35,36,37}-!nocover: BST_CYTHON_TRACE = 1 + randomized: PYTEST_ADDOPTS="--random-order-bucket=global" whitelist_externals = py{35,36,37}: |