diff options
| -rw-r--r-- | .gitlab-ci.yml | 3 | ||||
| -rwxr-xr-x | .gitlab-ci/run-tests.sh | 70 | ||||
| -rw-r--r-- | testsuite/a11y/meson.build | 7 | ||||
| -rw-r--r-- | testsuite/gdk/meson.build | 4 | ||||
| -rw-r--r-- | testsuite/gtk/meson.build | 24 | ||||
| -rw-r--r-- | testsuite/meson.build | 68 | ||||
| -rw-r--r-- | testsuite/reftests/meson.build | 81 |
7 files changed, 147 insertions, 110 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 389a761c5c..a78287b8b5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -51,8 +51,11 @@ style-check-diff: reports: junit: - "${CI_PROJECT_DIR}/_build/report-x11.xml" + - "${CI_PROJECT_DIR}/_build/report-x11_unstable.xml" - "${CI_PROJECT_DIR}/_build/report-wayland.xml" + - "${CI_PROJECT_DIR}/_build/report-wayland_unstable.xml" - "${CI_PROJECT_DIR}/_build/report-broadway.xml" + - "${CI_PROJECT_DIR}/_build/report-broadway_unstable.xml" name: "gtk-${CI_COMMIT_REF_NAME}" paths: - "${CI_PROJECT_DIR}/_build/meson-logs" diff --git a/.gitlab-ci/run-tests.sh b/.gitlab-ci/run-tests.sh index e68cf5a6c9..b60c3b38cc 100755 --- a/.gitlab-ci/run-tests.sh +++ b/.gitlab-ci/run-tests.sh @@ -19,11 +19,21 @@ case "${backend}" in --print-errorlogs \ --setup=${backend} \ --suite=gtk \ + --no-suite=failing \ + --no-suite=flaky \ --no-suite=gsk-compare-broadway # Store the exit code for the CI run, but always # generate the reports exit_code=$? + + xvfb-run -a -s "-screen 0 1024x768x24 -noreset" \ + meson test -C ${builddir} \ + --timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \ + --print-errorlogs \ + --setup=${backend}_unstable \ + --suite=flaky \ + --suite=failing || true ;; wayland) @@ -38,9 +48,18 @@ case "${backend}" in --print-errorlogs \ --setup=${backend} \ --suite=gtk \ + --no-suite=failing \ + --no-suite=flaky \ --no-suite=gsk-compare-broadway - exit_code=$? + + meson test -C ${builddir} \ + --timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \ + --print-errorlogs \ + --setup=${backend}_unstable \ + --suite=flaky \ + --suite=failing || true + kill ${compositor} ;; @@ -56,9 +75,18 @@ case "${backend}" in --print-errorlogs \ --setup=${backend} \ --suite=gtk \ + --no-suite=failing \ + --no-suite=flaky \ --no-suite=gsk-compare-broadway - exit_code=$? + + meson test -C ${builddir} \ + --timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \ + --print-errorlogs \ + --setup=${backend}_unstable \ + --suite=flaky \ + --suite=failing || true + kill ${compositor} ;; @@ -74,10 +102,20 @@ case "${backend}" in --print-errorlogs \ --setup=${backend} \ --suite=gtk \ + --no-suite=failing \ + --no-suite=flaky \ --no-suite=gsk-compare-opengl # don't let Broadway failures fail the run, for now exit_code=0 + + meson test -C ${builddir} \ + --timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \ + --print-errorlogs \ + --setup=${backend}_unstable \ + --suite=flaky \ + --suite=failing || true + kill ${server} ;; @@ -90,18 +128,20 @@ esac cd ${builddir} -$srcdir/.gitlab-ci/meson-junit-report.py \ - --project-name=gtk \ - --backend=${backend} \ - --job-id="${CI_JOB_NAME}" \ - --output=report-${backend}.xml \ - meson-logs/testlog-${backend}.json -$srcdir/.gitlab-ci/meson-html-report.py \ - --project-name=gtk \ - --backend=${backend} \ - --job-id="${CI_JOB_NAME}" \ - --reftest-output-dir="testsuite/reftests/output/${backend}" \ - --output=report-${backend}.html \ - meson-logs/testlog-${backend}.json +for suffix in "" "_unstable"; do + $srcdir/.gitlab-ci/meson-junit-report.py \ + --project-name=gtk \ + --backend="${backend}${suffix}" \ + --job-id="${CI_JOB_NAME}" \ + --output="report-${backend}${suffix}.xml" \ + "meson-logs/testlog-${backend}${suffix}.json" + $srcdir/.gitlab-ci/meson-html-report.py \ + --project-name=gtk \ + --backend="${backend}${suffix}" \ + --job-id="${CI_JOB_NAME}" \ + --reftest-output-dir="testsuite/reftests/output/${backend}${suffix}" \ + --output="report-${backend}${suffix}.html" \ + "meson-logs/testlog-${backend}${suffix}.json" +done exit $exit_code diff --git a/testsuite/a11y/meson.build b/testsuite/a11y/meson.build index 4f50d3a606..4547b21ba2 100644 --- a/testsuite/a11y/meson.build +++ b/testsuite/a11y/meson.build @@ -35,10 +35,6 @@ tests = [ ] -# Tests that are expected to fail -xfail = [ -] - is_debug = get_option('buildtype').startswith('debug') test_cargs = [] @@ -76,8 +72,6 @@ foreach t : tests install_dir: testexecdir, ) - expect_fail = xfail.contains(test_name) - if test_extra_suites.contains('slow') test_timeout = 90 endif @@ -88,6 +82,5 @@ foreach t : tests timeout: test_timeout, env: test_env, suite: ['a11y'] + test_extra_suites, - should_fail: expect_fail, ) endforeach diff --git a/testsuite/gdk/meson.build b/testsuite/gdk/meson.build index 03528a3905..7aa0cf6668 100644 --- a/testsuite/gdk/meson.build +++ b/testsuite/gdk/meson.build @@ -39,6 +39,8 @@ foreach t : tests install_dir: testexecdir, ) + suites = ['gdk'] + t.get('suites', []) + test(test_name, test_exe, args: [ '--tap', '-k' ], protocol: 'tap', @@ -48,7 +50,7 @@ foreach t : tests 'G_TEST_BUILDDIR=@0@'.format(meson.current_build_dir()), 'DBUS_SESSION_BUS_ADDRESS=', ], - suite: 'gdk', + suite: suites, ) endforeach diff --git a/testsuite/gtk/meson.build b/testsuite/gtk/meson.build index fc986b09fd..dccea45a48 100644 --- a/testsuite/gtk/meson.build +++ b/testsuite/gtk/meson.build @@ -23,8 +23,9 @@ endif # - 'suites': (array): additional test suites tests = [ { 'name': 'accel' }, -# sadly, mesons xfail support seems busted -# { 'name': 'accessor-apis' }, + # we are still missing some accessors + { 'name': 'accessor-apis', + 'suites': ['failing'] }, { 'name': 'action' }, { 'name': 'adjustment' }, { 'name': 'bitset' }, @@ -106,6 +107,9 @@ tests = [ { 'name': 'revealer-size' }, { 'name': 'widgetorder' }, { 'name': 'widget-refcount' }, + # This test was disabled for long enough that it no longer compiles + #{ 'name': 'window', + # 'suites': ['failing'] }, ] # Tests that test private apis and therefore are linked against libgtk-4.a @@ -129,16 +133,6 @@ internal_tests = [ { 'name': 'fnmatch' }, ] -# Tests that are expected to fail -xfail = [ - # we are still missing some accessors - 'accessor-apis', - # one of the window resizing tests fails after - # the GdkToplevel refactoring, and needs a big - # gtkwindow.c configure request cleanup - 'window', -] - is_debug = get_option('buildtype').startswith('debug') test_cargs = [] @@ -182,8 +176,6 @@ foreach t : tests install_dir: testexecdir, ) - expect_fail = xfail.contains(test_name) - if test_extra_suites.contains('slow') test_timeout = 90 endif @@ -194,7 +186,6 @@ foreach t : tests timeout: test_timeout, env: test_env, suite: ['gtk'] + test_extra_suites, - should_fail: expect_fail, ) endforeach @@ -215,8 +206,6 @@ foreach t : internal_tests install_dir: testexecdir, ) - expect_fail = xfail.contains(test_name) - if test_extra_suites.contains('slow') test_timeout = 90 endif @@ -227,7 +216,6 @@ foreach t : internal_tests timeout: test_timeout, env: test_env, suite: ['gtk'] + test_extra_suites, - should_fail: expect_fail, ) endforeach diff --git a/testsuite/meson.build b/testsuite/meson.build index 97344f3062..688a253978 100644 --- a/testsuite/meson.build +++ b/testsuite/meson.build @@ -10,47 +10,45 @@ common_env = [ 'GSETTINGS_SCHEMA_DIR=@0@'.format(gtk_schema_build_dir), 'GDK_DEBUG=default-settings', ] +exclude_unstable = ['flaky', 'failing'] -if x11_enabled - add_test_setup ('x11', - env: common_env + [ - 'GDK_BACKEND=x11', - 'TEST_OUTPUT_SUBDIR=x11', - ]) -endif +setups = [ + { 'backend': 'x11', 'if': x11_enabled, }, + { 'backend': 'wayland', 'if': wayland_enabled, 'is_default': true, }, + { 'name': 'waylandgles', 'backend': 'wayland', 'if': wayland_enabled, + 'env': ['GDK_DEBUG=gl-gles,default-settings'], }, + { 'backend': 'win32', 'if': os_win32 }, + { 'backend': 'broadway', 'if': broadway_enabled, }, +] -if wayland_enabled - add_test_setup ('wayland', - is_default: true, - env: common_env + [ - 'GDK_BACKEND=wayland', - 'TEST_OUTPUT_SUBDIR=wayland', - ]) +foreach setup : setups + if setup.get('if') + backend = setup.get('backend') + name = setup.get('name', backend) + exclude = [] - add_test_setup ('waylandgles', - env: common_env + [ - 'GDK_BACKEND=wayland', - 'TEST_OUTPUT_SUBDIR=waylandgles', - 'GDK_DEBUG=gl-gles,default-settings', - ]) + if backend != 'broadway' + exclude += 'gsk-compare-broadway' + endif -endif + env = common_env + [ + 'GDK_BACKEND=@0@'.format(backend), + ] + setup.get('env', []) -if os_win32 - add_test_setup ('win32', - env: common_env + [ - 'GDK_BACKEND=win32', - 'TEST_OUTPUT_SUBDIR=win32', - ]) -endif + add_test_setup( + name, + env: env + ['TEST_OUTPUT_SUBDIR=@0@'.format(name)], + exclude_suites: exclude_unstable + exclude, + is_default: setup.get('is_default', false), + ) -if broadway_enabled - add_test_setup ('broadway', - env: common_env + [ - 'GDK_BACKEND=broadway', - 'TEST_OUTPUT_SUBDIR=broadway', - ]) -endif + add_test_setup( + '@0@_unstable'.format(name), + env: env + ['TEST_OUTPUT_SUBDIR=@0@_unstable'.format(name)], + exclude_suites: exclude, + ) + endif +endforeach subdir('performance') subdir('gdk') diff --git a/testsuite/reftests/meson.build b/testsuite/reftests/meson.build index 2cd31db1c5..329348d64f 100644 --- a/testsuite/reftests/meson.build +++ b/testsuite/reftests/meson.build @@ -137,11 +137,9 @@ testdata = [ 'border-image-url-scaled.css', 'border-image-url-scaled.ref.ui', 'border-image-url-scaled.ui', - # this seems to make assumptions on text positioning - # that are not valid with subpixel positioning - #'border-image-url.css', - #'border-image-url.ref.ui', - #'border-image-url.ui', + 'border-image-url.css', + 'border-image-url.ref.ui', + 'border-image-url.ui', 'border-radius-clamp.css', 'border-radius-clamp.ref.ui', 'border-radius-clamp.ui', @@ -382,11 +380,9 @@ testdata = [ 'label-attribute-preference.css', 'label-attribute-preference.ref.ui', 'label-attribute-preference.ui', - # makes assumptions about text positioning that are not - # valid with subpixel positioning - #'label-background.css', - #'label-background.ref.ui', - #'label-background.ui', + 'label-background.css', + 'label-background.ref.ui', + 'label-background.ui', 'label-box-shadow-clip.css', 'label-box-shadow-clip.ref.ui', 'label-box-shadow-clip.ui', @@ -432,10 +428,8 @@ testdata = [ 'label-wrap-word-char-natural-size.ui', 'label-wrapped-huge-max-width-chars.ref.ui', 'label-wrapped-huge-max-width-chars.ui', - # this seems to make assumptions on text positioning - # that are not valid with subpixel positioning - #'label-wrap-justify.ref.ui', - #'label-wrap-justify.ui', + 'label-wrap-justify.ref.ui', + 'label-wrap-justify.ui', 'late-binding.ui', 'late-binding.ref.ui', 'late-property.ui', @@ -559,23 +553,18 @@ testdata = [ 'textview-border-windows.css', 'textview-border-windows.ref.ui', 'textview-border-windows.ui', - # these tests needs a better way to perform delayed actions - # they are not in xfail since they succeed on some platforms - #'textview-margins.css', - #'textview-margins.ref.ui', - #'textview-margins.ui', - #'textview-tags.ref.ui', - #'textview-tags.ui', + 'textview-margins.css', + 'textview-margins.ref.ui', + 'textview-margins.ui', + 'textview-tags.ref.ui', + 'textview-tags.ui', 'treeview-crash-too-wide.ref.ui', 'treeview-crash-too-wide.ui', 'treeview-fixed-height.css', 'treeview-fixed-height.ref.ui', 'treeview-fixed-height.ui', - # this test fails with an off-by-one in ci too frequently - # to be left enabled. Remove it until somebody figures out - # what is going on there. - #'treeview-headers-hidden.ref.ui', - #'treeview-headers-hidden.ui', + 'treeview-headers-hidden.ref.ui', + 'treeview-headers-hidden.ui', 'unresolvable.css', 'unresolvable.ref.ui', 'unresolvable.ui', @@ -589,21 +578,36 @@ testdata = [ 'window-default-size.ui', 'window-height-for-width.ref.ui', 'window-height-for-width.ui', - # this test needs a better way to perform delayed actions - # it is not in xfail since it succeeds on some platforms - #'window-show-contents-on-map.ref.ui', - #'window-show-contents-on-map.ui', + 'window-show-contents-on-map.ref.ui', + 'window-show-contents-on-map.ui', 'wrap-margin-align-critical.ref.ui', 'wrap-margin-align-critical.ui', 'wrapping-in-boxes-in-boxes.ref.ui', 'wrapping-in-boxes-in-boxes.ui', ] -# These need to be fixed but the issue hasn't been tracked down. xfails = [ + # needs to be fixed but the issue hasn't been tracked down 'sizegroups-evolution-identity-page.ui', # the NGL renderer can't deal with non-integer sizes - 'border-half-pixel.ui' + 'border-half-pixel.ui', + + # makes assumptions about text positioning that are not + # valid with subpixel positioning + 'border-image-url.ui', + 'label-background.ui', + 'label-wrap-justify.ui', +] +flaky = [ + # these tests need a better way to perform delayed actions + # they are not in xfails since they succeed on some platforms + 'textview-margins.ui', + 'textview-tags.ui', + 'window-show-contents-on-map.ui', + # this test fails with an off-by-one in ci too frequently + # to be left enabled. Remove it until somebody figures out + # what is going on there. + 'treeview-headers-hidden.ui', ] reftest_env = environment() @@ -616,6 +620,16 @@ reftest_env.set('G_ENABLE_DIAGNOSTIC', '0') reftest_env.set('REFTEST_MODULE_DIR', meson.current_build_dir()) foreach testname : testdata + suites = ['reftest'] + + if flaky.contains(testname) + suites += 'flaky' + endif + + if xfails.contains(testname) + suites += 'failing' + endif + if testname.endswith('.ui') and not testname.endswith('.ref.ui') test('reftest ' + testname, gtk_reftest, args: [ @@ -626,8 +640,7 @@ foreach testname : testdata ], protocol: 'tap', env: reftest_env, - suite: 'reftest', - should_fail: xfails.contains(testname), + suite: suites, ) endif endforeach |