From 6c5c271f254e492b470d9ec553e1866007eca4b2 Mon Sep 17 00:00:00 2001 From: Benjamin Schubert Date: Wed, 4 Dec 2019 10:10:15 +0000 Subject: tox.ini: Add ability to run tests in a randomized order Also register them to run nightly. Running tests in a randomized order is a good way of checking whether our tests depend on each others or not. Since we order them globally, tests will take longer to run, but that will give us higher confidence that they are correct. --- .gitlab-ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to '.gitlab-ci.yml') diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 05932c443..8cf593f0a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -366,6 +366,26 @@ overnight-tests-no-cache: tags: - overnight-tests +# These tests might be a bit more flaky since they randomize the order +# They will also take longer to run. Hence run them only nightly +overnight-randomized: + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:31-master-97748867 + <<: *tests + + script: + - mkdir -p "${INTEGRATION_CACHE}" + - useradd -Um buildstream + - chown -R buildstream:buildstream . + + # Don't run tests multiprocessed here, the randomized order doesn't like that + - su buildstream -c "tox -e py35-randomized,py36-randomized,py37-randomized -- --color=yes --integration" + - su buildstream -c "tox -e py35-randomized,py36-randomized,py37-randomized-external -- --color=yes --integration" + + # We need to override the exclusion from the template in order to run on schedules + except: [] + only: + - schedules + ##################################################### # Post stage # ##################################################### -- cgit v1.2.1