From 7fa5ccb60a6ec0c38da2bfde0118ef531bfd3697 Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Tue, 24 Dec 2019 11:55:57 +0000 Subject: .gitlab-ci.yml: Fix command for randomized external tests The command for running external tests only had the `-external` qualifier for one of the tests, so it ended up duplicating the first command for the first two environments. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7025a9f7a..e06984c46 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -398,7 +398,7 @@ overnight-randomized: # Don't run tests multiprocessed here, the randomized order doesn't like that - su buildstream -c "tox -e py35-randomized,py36-randomized,py37-randomized -- --color=yes --integration" - - su buildstream -c "tox -e py35-randomized,py36-randomized,py37-randomized-external -- --color=yes --integration" + - su buildstream -c "tox -e py35-randomized-external,py36-randomized-external,py37-randomized-external -- --color=yes --integration" # We need to override the exclusion from the template in order to run on schedules except: [] -- cgit v1.2.1 From 0eca9e6a4a01d1094987773bce58d9d3e501f1e8 Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Mon, 21 Oct 2019 14:38:07 +0100 Subject: _scheduler/scheduler.py: Enforce SafeChildWatcher In Python 3.8, `ThreadedChildWatcher` is the default watcher that causes issues with our scheduler. Enforce use of `SafeChildWatcher`. --- src/buildstream/_scheduler/scheduler.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/buildstream/_scheduler/scheduler.py b/src/buildstream/_scheduler/scheduler.py index a45da82f9..ca7e9279c 100644 --- a/src/buildstream/_scheduler/scheduler.py +++ b/src/buildstream/_scheduler/scheduler.py @@ -171,6 +171,12 @@ class Scheduler: # Hold on to the queues to process self.queues = queues + # NOTE: Enforce use of `SafeChildWatcher` as we generally don't want + # background threads. + # In Python 3.8+, `ThreadedChildWatcher` is the default watcher, and + # not `SafeChildWatcher`. + asyncio.set_child_watcher(asyncio.SafeChildWatcher()) + # Ensure that we have a fresh new event loop, in case we want # to run another test in this thread. self.loop = asyncio.new_event_loop() -- cgit v1.2.1 From 435880fdcd77739d15ecd7355415915c739c2795 Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Wed, 16 Oct 2019 16:53:25 +0100 Subject: tox.ini: Add python3.8 to default environment list --- tox.ini | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/tox.ini b/tox.ini index 62737b0ca..85697085d 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ # Tox global configuration # [tox] -envlist = py{35,36,37} +envlist = py{35,36,37,38} skip_missing_interpreters = true isolated_build = true @@ -14,22 +14,22 @@ isolated_build = true [testenv] usedevelop = # This is required by Cython in order to get coverage for cython files. - py{35,36,37}-!nocover: True + py{35,36,37,38}-!nocover: True commands = # Running with coverage reporting enabled - py{35,36,37}-!external-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc {posargs} - py{35,36,37}-!nocover: mkdir -p .coverage-reports + py{35,36,37,38}-!external-!nocover: pytest --basetemp {envtmpdir} --cov=buildstream --cov-config .coveragerc {posargs} + py{35,36,37,38}-!nocover: mkdir -p .coverage-reports external-!nocover: {envpython} {toxinidir}/tests/external_plugins.py {envtmpdir} --cov=buildstream --cov-append {posargs} - py{35,36,37}-!external-!nocover: mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname} - py{35,36,37}-external-!nocover: - mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname} + py{35,36,37,38}-!external-!nocover: mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname} + py{35,36,37,38}-external-!nocover: - mv {envtmpdir}/.coverage {toxinidir}/.coverage-reports/.coverage.{env:COVERAGE_PREFIX:}{envname} # Running with coverage reporting disabled - py{35,36,37}-!external-nocover: pytest --basetemp {envtmpdir} {posargs} + py{35,36,37,38}-!external-nocover: pytest --basetemp {envtmpdir} {posargs} external-nocover: {envpython} {toxinidir}/tests/external_plugins.py {envtmpdir} {posargs} deps = - py{35,36,37}: -rrequirements/requirements.txt - py{35,36,37}: -rrequirements/dev-requirements.txt - py{35,36,37}: -rrequirements/plugin-requirements.txt + py{35,36,37,38}: -rrequirements/requirements.txt + py{35,36,37,38}: -rrequirements/dev-requirements.txt + py{35,36,37,38}: -rrequirements/plugin-requirements.txt git+https://gitlab.com/BuildStream/bst-plugins-experimental.git@5b004e5850ab0e987c00c681b2c768f6ae02586b # Only require coverage and pytest-cov when using it @@ -58,17 +58,17 @@ passenv = # These keys are not inherited by any other sections # setenv = - py{35,36,37}: COVERAGE_FILE = {envtmpdir}/.coverage - py{35,36,37}: BST_TEST_HOME = {envtmpdir} - py{35,36,37}: BST_TEST_XDG_CACHE_HOME = {envtmpdir}/cache - py{35,36,37}: BST_TEST_XDG_CONFIG_HOME = {envtmpdir}/config - py{35,36,37}: BST_TEST_XDG_DATA_HOME = {envtmpdir}/share + py{35,36,37,38}: COVERAGE_FILE = {envtmpdir}/.coverage + py{35,36,37,38}: BST_TEST_HOME = {envtmpdir} + py{35,36,37,38}: BST_TEST_XDG_CACHE_HOME = {envtmpdir}/cache + py{35,36,37,38}: BST_TEST_XDG_CONFIG_HOME = {envtmpdir}/config + py{35,36,37,38}: BST_TEST_XDG_DATA_HOME = {envtmpdir}/share # This is required to get coverage for Cython - py{35,36,37}-!nocover: BST_CYTHON_TRACE = 1 + py{35,36,37,38}-!nocover: BST_CYTHON_TRACE = 1 randomized: PYTEST_ADDOPTS="--random-order-bucket=global" whitelist_externals = - py{35,36,37}: + py{35,36,37,38}: mv mkdir -- cgit v1.2.1 From 807f275a6639d4596d050753d488537511df38fe Mon Sep 17 00:00:00 2001 From: Chandan Singh Date: Wed, 16 Oct 2019 16:55:28 +0100 Subject: .gitlab-ci.yml: Add test environment for Python 3.8 * Bump the testsuite image versions in order to get the latest Python 3.8 image, as we need at least Python 3.8.1. Python 3.8.0 suffers from an issue that causes recursion errors when using `shutil.copytree`. See upstream issue (https://bugs.python.org/issue38688) for more details on that. * Add partial support for running tests on Python 3.8. Coverage still doesn't play well with our tests on 3.8. https://gitlab.com/BuildStream/buildstream/issues/1173 is tracking that. --- .gitlab-ci.yml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e06984c46..d535c184e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,12 +24,12 @@ variables: # Our own variables # Version of the docker images we should use for all the images. # This is taken from buildstream/buildstream-docker-images - DOCKER_IMAGE_VERSION: master-103717922 + DOCKER_IMAGE_VERSION: master-105004115 PYTEST_ADDOPTS: "--color=yes" INTEGRATION_CACHE: "${CI_PROJECT_DIR}/cache/integration-cache" PYTEST_ARGS: "--color=yes --integration -n 2" TEST_COMMAND: "tox -- ${PYTEST_ARGS}" - EXTERNAL_TESTS_COMMAND: "tox -e py35-external,py36-external,py37-external -- ${PYTEST_ARGS}" + EXTERNAL_TESTS_COMMAND: "tox -e py35-external,py36-external,py37-external,py38-external -- ${PYTEST_ARGS}" COVERAGE_PREFIX: "${CI_JOB_NAME}." @@ -88,6 +88,21 @@ tests-centos-7.6: <<: *tests image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-centos:7.6.1810-${DOCKER_IMAGE_VERSION} +tests-python-3.8-buster: + image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-python:3.8-buster-${DOCKER_IMAGE_VERSION} + <<: *tests + variables: + # Our testsuite has issues with coverage on Python 3.8 so disable coverage + # in the meantime. For more details, see + # https://gitlab.com/BuildStream/buildstream/issues/1173. + TEST_COMMAND: "tox -e py38-nocover -- ${PYTEST_ARGS}" + + # This particular testsuite image has both Python 3.7 and Python 3.8 so we + # need to explicitly force the 3.8 environment. + # Once Python 3.8 is available in distros, we should switch to such an + # image, and remove the following override. + EXTERNAL_TESTS_COMMAND: "tox -e py38-external -- ${PYTEST_ARGS}" + overnight-fedora-30-aarch64: image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:aarch64-30-${DOCKER_IMAGE_VERSION} tags: @@ -397,8 +412,8 @@ overnight-randomized: - chown -R buildstream:buildstream . # Don't run tests multiprocessed here, the randomized order doesn't like that - - su buildstream -c "tox -e py35-randomized,py36-randomized,py37-randomized -- --color=yes --integration" - - su buildstream -c "tox -e py35-randomized-external,py36-randomized-external,py37-randomized-external -- --color=yes --integration" + - su buildstream -c "tox -e py35-randomized,py36-randomized,py37-randomized,py38-randomized -- --color=yes --integration" + - su buildstream -c "tox -e py35-randomized-external,py36-randomized-external,py37-randomized-external,py38-randomized-external -- --color=yes --integration" # We need to override the exclusion from the template in order to run on schedules except: [] -- cgit v1.2.1