diff options
| author | Benjamin Schubert <contact@benschubert.me> | 2019-12-10 14:28:03 +0000 |
|---|---|---|
| committer | Benjamin Schubert <contact@benschubert.me> | 2019-12-10 14:28:03 +0000 |
| commit | 7ca05baa9fecaf263458650019aa908a26235858 (patch) | |
| tree | 2df37fb8e949a157abf7d75aaca4f841d631426a | |
| parent | 397a8fe5e8d32a8ec6a9ef252cd7a55007ad4f66 (diff) | |
| parent | 6c5c271f254e492b470d9ec553e1866007eca4b2 (diff) | |
| download | buildstream-7ca05baa9fecaf263458650019aa908a26235858.tar.gz | |
Merge branch 'bschubert/add-randomized-order-tests' into 'master'
tox.ini: Add ability to run tests in a randomized order
See merge request BuildStream/buildstream!1760
| -rw-r--r-- | .gitlab-ci.yml | 20 | ||||
| -rw-r--r-- | tox.ini | 4 |
2 files changed, 24 insertions, 0 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 05932c443..8cf593f0a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -366,6 +366,26 @@ overnight-tests-no-cache: tags: - overnight-tests +# These tests might be a bit more flaky since they randomize the order +# They will also take longer to run. Hence run them only nightly +overnight-randomized: + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:31-master-97748867 + <<: *tests + + script: + - mkdir -p "${INTEGRATION_CACHE}" + - useradd -Um buildstream + - chown -R buildstream:buildstream . + + # Don't run tests multiprocessed here, the randomized order doesn't like that + - su buildstream -c "tox -e py35-randomized,py36-randomized,py37-randomized -- --color=yes --integration" + - su buildstream -c "tox -e py35-randomized,py36-randomized,py37-randomized-external -- --color=yes --integration" + + # We need to override the exclusion from the template in order to run on schedules + except: [] + only: + - schedules + ##################################################### # Post stage # ##################################################### @@ -34,6 +34,9 @@ deps = # Only require coverage and pytest-cov when using it !nocover: -rrequirements/cov-requirements.txt + + # Install pytest-random-order for '-randomized' + randomized: pytest-random-order passenv = ARTIFACT_CACHE_SERVICE BST_FORCE_BACKEND @@ -61,6 +64,7 @@ setenv = py{35,36,37}: BST_TEST_XDG_DATA_HOME = {envtmpdir}/share # This is required to get coverage for Cython py{35,36,37}-!nocover: BST_CYTHON_TRACE = 1 + randomized: PYTEST_ADDOPTS="--random-order-bucket=global" whitelist_externals = py{35,36,37}: |