ci: implement unified sections

in after_script, variable $SCRIPTS_DIR is lost

Acked-by: Daniel Stone <daniels@collabora.com>
Signed-off-by: David Heidelberg <david.heidelberg@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/20272>

20 files changed, 192 insertions, 59 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5a4a4b03da9..347d1b46fb1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -35,10 +35,13 @@ variables:
 
 default:
   before_script:
-    - echo -e "\e[0Ksection_start:$(date +%s):unset_env_vars_section[collapsed=true]\r\e[0KUnsetting vulnerable environment variables"
-    - echo -n "${CI_JOB_JWT}" > "${CI_JOB_JWT_FILE}"
-    - unset CI_JOB_JWT
-    - echo -e "\e[0Ksection_end:$(date +%s):unset_env_vars_section\r\e[0K"
+    - >
+      export SCRIPTS_DIR=$(mktemp -d) &&
+      curl -L -s --retry 4 -f --retry-all-errors --retry-delay 60 -O --output-dir "${SCRIPTS_DIR}" "${CI_PROJECT_URL}/-/raw/${CI_COMMIT_SHA}/.gitlab-ci/setup-test-env.sh" &&
+      chmod +x ${SCRIPTS_DIR}/setup-test-env.sh &&
+      . ${SCRIPTS_DIR}/setup-test-env.sh &&
+      echo -n "${CI_JOB_JWT}" > "${CI_JOB_JWT_FILE}" &&
+      unset CI_JOB_JWT  # Unsetting vulnerable env variables
 
   after_script:
     - >
diff --git a/.gitlab-ci/bare-metal/fastboot.sh b/.gitlab-ci/bare-metal/fastboot.sh
index bc4453f7b84..e4ea361b788 100755
--- a/.gitlab-ci/bare-metal/fastboot.sh
+++ b/.gitlab-ci/bare-metal/fastboot.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+. "$SCRIPTS_DIR"/setup-test-env.sh
+
 BM=$CI_PROJECT_DIR/install/bare-metal
 CI_COMMON=$CI_PROJECT_DIR/install/common
 
diff --git a/.gitlab-ci/bare-metal/poe-powered.sh b/.gitlab-ci/bare-metal/poe-powered.sh
index 4fcd3ba947d..b075aaec1fe 100755
--- a/.gitlab-ci/bare-metal/poe-powered.sh
+++ b/.gitlab-ci/bare-metal/poe-powered.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+. "$SCRIPTS_DIR"/setup-test-env.sh
+
 # Boot script for devices attached to a PoE switch, using NFS for the root
 # filesystem.
 
diff --git a/.gitlab-ci/bare-metal/rootfs-setup.sh b/.gitlab-ci/bare-metal/rootfs-setup.sh
index e6272b72e1e..7d0c9b4aac6 100644
--- a/.gitlab-ci/bare-metal/rootfs-setup.sh
+++ b/.gitlab-ci/bare-metal/rootfs-setup.sh
@@ -18,6 +18,7 @@ date +'%F %T'
 
 cp $CI_COMMON/capture-devcoredump.sh $rootfs_dst/
 cp $CI_COMMON/intel-gpu-freq.sh $rootfs_dst/
+cp "$SCRIPTS_DIR/setup-test-env.sh" "$rootfs_dst/"
 
 set +x
 
diff --git a/.gitlab-ci/build/gitlab-ci.yml b/.gitlab-ci/build/gitlab-ci.yml
index 4436788e624..ae07eb13b83 100644
--- a/.gitlab-ci/build/gitlab-ci.yml
+++ b/.gitlab-ci/build/gitlab-ci.yml
@@ -21,15 +21,14 @@
   # Use ccache transparently, and print stats before/after
   before_script:
     - !reference [default, before_script]
-    - export PATH="/usr/lib/ccache:$PATH"
-    - export CCACHE_BASEDIR="$PWD"
-    - echo -e "\e[0Ksection_start:$(date +%s):ccache_before[collapsed=true]\r\e[0Kccache stats before build"
-    - ccache --show-stats
-    - echo -e "\e[0Ksection_end:$(date +%s):ccache_before\r\e[0K"
+    - |
+      export PATH="/usr/lib/ccache:$PATH"
+      export CCACHE_BASEDIR="$PWD"
+      section_start ccache_before "ccache stats before build"
+      ccache --show-stats
+      section_end ccache_before
   after_script:
-    - echo -e "\e[0Ksection_start:$(date +%s):ccache_after[collapsed=true]\r\e[0Kccache stats after build"
-    - ccache --show-stats
-    - echo -e "\e[0Ksection_end:$(date +%s):ccache_after\r\e[0K"
+    - ccache --show-stats | grep "cache hit rate"
     - !reference [default, after_script]
 
 .build-windows:
@@ -186,12 +185,17 @@ debian-build-testing:
       -D tools=drm-shim,etnaviv,freedreno,glsl,intel,intel-ui,nir,nouveau,lima,panfrost,asahi
       -D b_lto=true
     LLVM_VERSION: 13
-  script:
-    - .gitlab-ci/lava/lava-pytest.sh
-    - .gitlab-ci/run-shellcheck.sh
-    - .gitlab-ci/run-yamllint.sh
-    - .gitlab-ci/meson/build.sh
-    - .gitlab-ci/run-shader-db.sh
+  script: |
+    section_start lava-pytest "lava-pytest"
+    .gitlab-ci/lava/lava-pytest.sh
+    section_switch shellcheck "shellcheck"
+    .gitlab-ci/run-shellcheck.sh
+    section_switch yamllint "yamllint"
+    .gitlab-ci/run-yamllint.sh
+    section_switch meson "meson"
+    .gitlab-ci/meson/build.sh
+    section_switch shader-db "shader-db"
+    .gitlab-ci/run-shader-db.sh
 
 # Test a release build with -Werror so new warnings don't sneak in.
 debian-release:
diff --git a/.gitlab-ci/common/generate-env.sh b/.gitlab-ci/common/generate-env.sh
index e08905030b8..d9f18c1a100 100755
--- a/.gitlab-ci/common/generate-env.sh
+++ b/.gitlab-ci/common/generate-env.sh
@@ -10,6 +10,7 @@ for var in \
     CI_COMMIT_TITLE \
     CI_JOB_ID \
     CI_JOB_JWT_FILE \
+    CI_JOB_STARTED_AT \
     CI_JOB_NAME \
     CI_JOB_URL \
     CI_MERGE_REQUEST_SOURCE_BRANCH_NAME \
@@ -27,6 +28,7 @@ for var in \
     CI_SERVER_URL \
     CROSVM_GALLIUM_DRIVER \
     CROSVM_GPU_ARGS \
+    CURRENT_SECTION \
     DEQP_BIN_DIR \
     DEQP_CONFIG \
     DEQP_EXPECTED_RENDERER \
diff --git a/.gitlab-ci/common/init-stage2.sh b/.gitlab-ci/common/init-stage2.sh
index 64a57304c66..a19d47960e0 100755
--- a/.gitlab-ci/common/init-stage2.sh
+++ b/.gitlab-ci/common/init-stage2.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Make sure to kill itself and all the children process from this script on
 # exiting, since any console output may interfere with LAVA signals handling,
@@ -37,6 +37,7 @@ BACKGROUND_PIDS=
 # running tests.
 
 . /set-job-env-vars.sh
+. "$SCRIPTS_DIR"/setup-test-env.sh
 
 set -ex
 
@@ -160,7 +161,7 @@ fi
 
 RESULT=fail
 set +e
-sh -c "$HWCI_TEST_SCRIPT"
+bash -c ". $SCRIPTS_DIR/setup-test-env.sh && $HWCI_TEST_SCRIPT"
 EXIT_CODE=$?
 set -e
 
diff --git a/.gitlab-ci/deqp-runner.sh b/.gitlab-ci/deqp-runner.sh
index 7a34553089b..819039710d7 100755
--- a/.gitlab-ci/deqp-runner.sh
+++ b/.gitlab-ci/deqp-runner.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-echo -e "\e[0Ksection_start:$(date +%s):test_setup[collapsed=true]\r\e[0Kpreparing test setup"
+section_start test_setup "deqp: preparing test setup"
 
 set -ex
 
@@ -160,11 +160,7 @@ if [ -z "$DEQP_SUITE" ]; then
     fi
 fi
 
-set +x
-echo -e "\e[0Ksection_end:$(date +%s):test_setup\r\e[0K"
-
-echo -e "\e[0Ksection_start:$(date +%s):deqp[collapsed=false]\r\e[0Kdeqp-runner"
-set -x
+uncollapsed_section_switch deqp "deqp: deqp-runner"
 
 set +e
 if [ -z "$DEQP_SUITE" ]; then
@@ -197,11 +193,10 @@ fi
 DEQP_EXITCODE=$?
 
 set +x
-echo -e "\e[0Ksection_end:$(date +%s):deqp\r\e[0K"
 
 report_load
 
-echo -e "\e[0Ksection_start:$(date +%s):test_post_process[collapsed=true]\r\e[0Kpost-processing test results"
+section_switch test_post_process "deqp: post-processing test results"
 set -x
 
 # Remove all but the first 50 individual XML files uploaded as artifacts, to
@@ -243,6 +238,6 @@ fi
 # 0.17s on a Ryzen 5950X (16 threads, 0.95s when limited to 1 thread).
 zstd --rm -T0 -8qc $RESULTS/results.csv -o $RESULTS/results.csv.zst
 
-echo -e "\e[0Ksection_end:$(date +%s):test_post_process\r\e[0K"
+section_end test_post_process
 
 exit $DEQP_EXITCODE
diff --git a/.gitlab-ci/image-tags.yml b/.gitlab-ci/image-tags.yml
index 75a0533e8e4..c701dedc226 100644
--- a/.gitlab-ci/image-tags.yml
+++ b/.gitlab-ci/image-tags.yml
@@ -21,7 +21,7 @@ variables:
 
    ALPINE_X86_BUILD_TAG: "2023-02-26-add-bash-coreutils"
    FEDORA_X86_BUILD_TAG: "2023-02-09-f36"
-   KERNEL_ROOTFS_TAG: "2023-02-23-virglrenderer"
+   KERNEL_ROOTFS_TAG: "2023-02-28-add-CI_JOB_STARTED_AT"
 
    WINDOWS_X64_VS_PATH: "windows/x64_vs"
    WINDOWS_X64_VS_TAG: "2022-10-20-upgrade-zlib"
diff --git a/.gitlab-ci/lava/lava-pytest.sh b/.gitlab-ci/lava/lava-pytest.sh
index cb105385983..9ace8a05f1e 100755
--- a/.gitlab-ci/lava/lava-pytest.sh
+++ b/.gitlab-ci/lava/lava-pytest.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 #
 # Copyright (C) 2022 Collabora Limited
 # Author: Guilherme Gallo <guilherme.gallo@collabora.com>
diff --git a/.gitlab-ci/lava/lava-submit.sh b/.gitlab-ci/lava/lava-submit.sh
index af93c02a179..c8bbd8f9362 100755
--- a/.gitlab-ci/lava/lava-submit.sh
+++ b/.gitlab-ci/lava/lava-submit.sh
@@ -18,13 +18,14 @@ mkdir -p results/job-rootfs-overlay/
 cp artifacts/ci-common/capture-devcoredump.sh results/job-rootfs-overlay/
 cp artifacts/ci-common/init-*.sh results/job-rootfs-overlay/
 cp artifacts/ci-common/intel-gpu-freq.sh results/job-rootfs-overlay/
+cp "$SCRIPTS_DIR"/setup-test-env.sh results/job-rootfs-overlay/
 
 # Prepare env vars for upload.
 KERNEL_IMAGE_BASE_URL="https://${BASE_SYSTEM_HOST_PATH}" \
 	artifacts/ci-common/generate-env.sh > results/job-rootfs-overlay/set-job-env-vars.sh
-echo -e "\e[0Ksection_start:$(date +%s):variables[collapsed=true]\r\e[0KVariables passed through:"
+section_start variables "Variables passed through:"
 cat results/job-rootfs-overlay/set-job-env-vars.sh
-echo -e "\e[0Ksection_end:$(date +%s):variables\r\e[0K"
+section_end variables
 
 tar zcf job-rootfs-overlay.tar.gz -C results/job-rootfs-overlay/ .
 ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" job-rootfs-overlay.tar.gz "https://${JOB_ROOTFS_OVERLAY_PATH}"
diff --git a/.gitlab-ci/meson/build.sh b/.gitlab-ci/meson/build.sh
index d3ce66303f1..8809d3269b8 100755
--- a/.gitlab-ci/meson/build.sh
+++ b/.gitlab-ci/meson/build.sh
@@ -1,4 +1,6 @@
-#!/bin/bash
+#!/usr/bin/env bash
+
+section_switch meson-configure "meson: configure"
 
 set -e
 set -o xtrace
@@ -81,11 +83,17 @@ meson setup _build \
       ${EXTRA_OPTION}
 cd _build
 meson configure
+
+section_switch meson-build "meson: build"
+
 if command -V mold &> /dev/null ; then
     mold --run ninja
 else
     ninja
 fi
+
+
+section_switch meson-test "meson: test"
 LC_ALL=C.UTF-8 meson test --num-processes ${FDO_CI_CONCURRENT:-4} --print-errorlogs ${MESON_TEST_ARGS}
 if command -V mold &> /dev/null ; then
     mold --run ninja install
@@ -93,3 +101,4 @@ else
     ninja install
 fi
 cd ..
+section_end meson-test
diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh
index 1f243566a99..3df83151c6f 100755
--- a/.gitlab-ci/prepare-artifacts.sh
+++ b/.gitlab-ci/prepare-artifacts.sh
@@ -1,4 +1,6 @@
-#!/bin/bash
+#!/usr/bin/env bash
+
+section_switch prepare-artifacts "artifacts: prepare"
 
 set -e
 set -o xtrace
@@ -56,3 +58,5 @@ if [ -n "$MINIO_ARTIFACT_NAME" ]; then
     zstd artifacts/install.tar -o ${MINIO_ARTIFACT_NAME}
     ci-fairy s3cp --token-file "${CI_JOB_JWT_FILE}" ${MINIO_ARTIFACT_NAME} https://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME}
 fi
+
+section_end prepare-artifacts
diff --git a/.gitlab-ci/run-shader-db.sh b/.gitlab-ci/run-shader-db.sh
index 2c6a6267763..70cc4fa381d 100755
--- a/.gitlab-ci/run-shader-db.sh
+++ b/.gitlab-ci/run-shader-db.sh
@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 set -e
 
-echo -e "\e[0Ksection_start:$(date +%s):shader-db-prepare[collapsed=true]\r\e[0KPreparing shader-db"
 ARTIFACTSDIR=$(pwd)/shader-db
 mkdir -p "$ARTIFACTSDIR"
 export DRM_SHIM_DEBUG=true
@@ -10,32 +9,31 @@ LIBDIR=$(pwd)/install/lib
 export LD_LIBRARY_PATH=$LIBDIR
 
 cd /usr/local/shader-db
-echo -e "\e[0Ksection_end:$(date +%s):shader-db-prepare\r\e[0K"
 
 for driver in freedreno intel v3d vc4; do
-    echo -e "\e[0Ksection_start:$(date +%s):shader-db-${driver}[collapsed=true]\r\e[0KRunning shader-db for $driver"
+    section_start shader-db-${driver} "Running shader-db for $driver"
     env LD_PRELOAD="$LIBDIR/lib${driver}_noop_drm_shim.so" \
         ./run -j"${FDO_CI_CONCURRENT:-4}" ./shaders \
             > "$ARTIFACTSDIR/${driver}-shader-db.txt"
-    echo -e "\e[0Ksection_end:$(date +%s):shader-db-${driver}\r\e[0K"
+    section_end shader-db-${driver}
 done
 
 # Run shader-db over a number of supported chipsets for nouveau
 for chipset in 40 a3 c0 e4 f0 134 162; do
-    echo -e "\e[0Ksection_start:$(date +%s):shader-db-nouveau-${chipset}[collapsed=true]\r\e[0KRunning shader-db for nouveau - ${chipset}"
+    section_start shader-db-nouveau-${chipset} "Running shader-db for nouveau - ${chipset}"
     env LD_PRELOAD="$LIBDIR/libnouveau_noop_drm_shim.so" \
         NOUVEAU_CHIPSET=${chipset} \
         ./run -j"${FDO_CI_CONCURRENT:-4}" ./shaders \
             > "$ARTIFACTSDIR/nouveau-${chipset}-shader-db.txt"
-    echo -e "\e[0Ksection_end:$(date +%s):shader-db-nouveau-${chipset}\r\e[0K"
+    section_end shader-db-nouveau-${chipset}
 done
 
 # Run shader-db for r300 (RV370 and RV515)
 for chipset in 0x5460 0x7140; do
-    echo -e "\e[0Ksection_start:$(date +%s):shader-db-r300-${chipset}[collapsed=true]\r\e[0KRunning shader-db for r300 - ${chipset}"
+    section_start shader-db-r300-${chipset} "Running shader-db for r300 - ${chipset}"
     env LD_PRELOAD="$LIBDIR/libradeon_noop_drm_shim.so" \
         RADEON_GPU_ID=${chipset} \
         ./run -j"${FDO_CI_CONCURRENT:-4}" -o r300 ./shaders \
             > "$ARTIFACTSDIR/r300-${chipset}-shader-db.txt"
-    echo -e "\e[0Ksection_end:$(date +%s):shader-db-r300-${chipset}\r\e[0K"
+    section_end shader-db-r300-${chipset}
 done
diff --git a/.gitlab-ci/setup-test-env.sh b/.gitlab-ci/setup-test-env.sh
new file mode 100644
index 00000000000..ff8d676ec04
--- /dev/null
+++ b/.gitlab-ci/setup-test-env.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+
+function x_off {
+    if [[ "$-" == *"x"* ]]; then
+      state_x=1
+      set +x
+    else
+      state_x=0
+    fi
+}
+
+# TODO: implement x_on !
+
+function error {
+    x_off 2>/dev/null
+    RED="\e[0;31m"
+    ENDCOLOR="\e[0m"
+    # we force the following to be not in a section
+    section_end $CURRENT_SECTION
+
+    DATE_S=$(date -u +"%s")
+    JOB_START_S=$(date -u +"%s" -d "${CI_JOB_STARTED_AT:?}")
+    CURR_TIME=$((DATE_S-JOB_START_S))
+    CURR_MINSEC="$(printf "%02d" $((CURR_TIME/60))):$(printf "%02d" $((CURR_TIME%60)))"
+    echo -e "\n${RED}[${CURR_MINSEC}] ERROR: $*${ENDCOLOR}\n"
+    [ "$state_x" -eq 0 ] || set -x
+}
+
+function trap_exit {
+    local ret=$1
+    shift
+    error $CURRENT_SECTION: $*
+    exit $ret
+}
+
+function build_section_start {
+    local section_params=$1
+    shift
+    local section_name=$1
+    CURRENT_SECTION=$section_name
+    shift
+    CYAN="\e[0;36m"
+    ENDCOLOR="\e[0m"
+
+    DATE_S=$(date -u +"%s")
+    JOB_START_S=$(date -u +"%s" -d "${CI_JOB_STARTED_AT:?}")
+    CURR_TIME=$((DATE_S-JOB_START_S))
+    CURR_MINSEC="$(printf "%02d" $((CURR_TIME/60))):$(printf "%02d" $((CURR_TIME%60)))"
+    echo -e "\n\e[0Ksection_start:$(date +%s):$section_name$section_params\r\e[0K${CYAN}[${CURR_MINSEC}] $*${ENDCOLOR}\n"
+}
+
+function section_start {
+    x_off 2>/dev/null
+    build_section_start "[collapsed=true]" $*
+    [ "$state_x" -eq 0 ] || set -x
+}
+
+function build_section_end {
+    echo -e "\e[0Ksection_end:$(date +%s):$1\r\e[0K"
+    CURRENT_SECTION=""
+}
+
+function section_end {
+    x_off >/dev/null
+    build_section_end $*
+    [ "$state_x" -eq 0 ] || set -x
+}
+
+function section_switch {
+    x_off 2>/dev/null
+    if [ -n "$CURRENT_SECTION" ]
+    then
+	build_section_end $CURRENT_SECTION
+    fi
+    build_section_start "[collapsed=true]" $*
+    [ "$state_x" -eq 0 ] || set -x
+}
+
+function uncollapsed_section_switch {
+    x_off 2>/dev/null
+    if [ -n "$CURRENT_SECTION" ]
+    then
+	build_section_end $CURRENT_SECTION
+    fi
+    build_section_start "" $*
+    [ "$state_x" -eq 0 ] || set -x
+}
+
+export -f x_off
+export -f error
+export -f trap_exit
+export -f build_section_start
+export -f section_start
+export -f build_section_end
+export -f section_end
+export -f section_switch
+export -f uncollapsed_section_switch
+
+set -E
+trap 'trap_exit $?' ERR
diff --git a/.gitlab-ci/test/gitlab-ci.yml b/.gitlab-ci/test/gitlab-ci.yml
index 03c94f52b8a..7579990c1a5 100644
--- a/.gitlab-ci/test/gitlab-ci.yml
+++ b/.gitlab-ci/test/gitlab-ci.yml
@@ -8,9 +8,9 @@
     # Note: Build dir (and thus install) may be dirty due to GIT_STRATEGY
     - rm -rf install
     - tar -xf artifacts/install.tar
-    - echo -e "\e[0Ksection_start:$(date +%s):ldd_section[collapsed=true]\r\e[0KChecking ldd on driver build"
+    - section_start ldd_section "Checking ldd on driver build"
     - LD_LIBRARY_PATH=install/lib find install/lib -name "*.so" -print -exec ldd {} \;
-    - echo -e "\e[0Ksection_end:$(date +%s):ldd_section\r\e[0K"
+    - section_end ldd_section
   artifacts:
     when: always
     name: "mesa_${CI_JOB_NAME}"
@@ -108,9 +108,9 @@ rustfmt:
   variables:
     PIGLIT_REPLAY_EXTRA_ARGS: --keep-image --db-path ${CI_PROJECT_DIR}/replayer-db/ --minio_bucket=mesa-tracie-public --jwt-file=${CI_JOB_JWT_FILE}
   script:
-    - echo -e "\e[0Ksection_start:$(date +%s):variables[collapsed=true]\r\e[0KVariables passed through:"
+    - section_start variables "Variables passed through:"
     - install/common/generate-env.sh
-    - echo -e "\e[0Ksection_end:$(date +%s):variables\r\e[0K"
+    - section_end variables
     - install/piglit/piglit-traces.sh
 
 .deqp-test:
@@ -148,11 +148,12 @@ rustfmt:
     # instead of fd.o.  Set FDO_HTTP_CACHE_URI to an http cache for your test lab to
     # improve it even more (see https://docs.mesa3d.org/ci/bare-metal.html for
     # setup).
-    - echo -e "\e[0Ksection_start:$(date +%s):artifacts_download[collapsed=true]\r\e[0KDownloading artifacts from minio"
+    - section_start artifacts_download "Downloading artifacts from s3"
     # Note: Build dir (and thus install) may be dirty due to GIT_STRATEGY
     - rm -rf install
     - curl -L --retry 4 -f --retry-all-errors --retry-delay 60 ${FDO_HTTP_CACHE_URI:-}https://${PIPELINE_ARTIFACTS_BASE}/${MINIO_ARTIFACT_NAME}.tar.zst | tar --zstd -x
     - echo -e "\e[0Ksection_end:$(date +%s):artifacts_download\r\e[0K"
+    - section_end artifacts_download
   artifacts:
     when: always
     name: "mesa_${CI_JOB_NAME}"
@@ -220,7 +221,7 @@ rustfmt:
   # like FDO_DISTRIBUTION_TAG for *the* image, there is no way to
   # depend on more than one image per job. So, the job container is
   # built as part of the CI in the boot2container project.
-  image: registry.freedesktop.org/mupuf/valve-infra/mesa-trigger:2022-12-08.1
+  image: registry.freedesktop.org/mupuf/valve-infra/mesa-trigger:2023-02-27.1
   timeout: 1h 40m
   variables:
     # No need by default to pull the whole repo
@@ -283,7 +284,7 @@ rustfmt:
       [ -d "$CI_COMMON_SCRIPTS" ] || exit 1
 
 
-      B2C_TEST_SCRIPT="bash -c 'source ./set-job-env-vars.sh; tar xf ${INSTALL_TARBALL_NAME}; ${B2C_TEST_SCRIPT}'"
+      B2C_TEST_SCRIPT="bash -c 'source ./set-job-env-vars.sh; source ./setup-test-env.sh; tar xf ${INSTALL_TARBALL_NAME}; ${B2C_TEST_SCRIPT}'"
 
       # The Valve CI gateway receives jobs in a YAML format. Create a
       # job description from the CI environment.
@@ -324,6 +325,7 @@ rustfmt:
       echo "Variables passed through:"
       cat ${JOB_FOLDER}/set-job-env-vars.sh
       echo "export CI_JOB_JWT=${CI_JOB_JWT}" >> ${JOB_FOLDER}/set-job-env-vars.sh
+      cp ${SCRIPTS_DIR}/setup-test-env.sh ${JOB_FOLDER}/setup-test-env.sh
       set -x
 
       # Copy the mesa install tarball to the job folder, for later extraction
diff --git a/src/gallium/drivers/llvmpipe/ci/gitlab-ci.yml b/src/gallium/drivers/llvmpipe/ci/gitlab-ci.yml
index 8db8c1a2721..fa98a416106 100644
--- a/src/gallium/drivers/llvmpipe/ci/gitlab-ci.yml
+++ b/src/gallium/drivers/llvmpipe/ci/gitlab-ci.yml
@@ -66,11 +66,15 @@ llvmpipe:
   variables:
     DEQP_SUITE: llvmpipe
     XDG_RUNTIME_DIR: /run/user
+    XVFB_SCRIPT: "install/deqp-runner.sh"
   extends: .llvmpipe-deqp-test
-  script:
-    - mkdir -p $XDG_RUNTIME_DIR
-    - LD_LIBRARY_PATH=`pwd`/install weston -Bheadless-backend.so -Swayland-0 &
-    - LD_LIBRARY_PATH=`pwd`/install WAYLAND_DISPLAY=wayland-0 xvfb-run --server-args='-noreset' install/deqp-runner.sh
+  script: |
+    . "$SCRIPTS_DIR"/setup-test-env.sh
+    section_start weston "weston: prepare"
+    mkdir -p $XDG_RUNTIME_DIR
+    LD_LIBRARY_PATH=`pwd`/install weston -Bheadless-backend.so -Swayland-0 &
+    section_end weston
+    LD_LIBRARY_PATH=`pwd`/install WAYLAND_DISPLAY=wayland-0 xvfb-run --server-args='-noreset' bash -c ". $SCRIPTS_DIR/setup-test-env.sh && ${XVFB_SCRIPT}"
 
 llvmpipe-deqp-asan:
   variables:
diff --git a/src/gallium/drivers/softpipe/ci/gitlab-ci.yml b/src/gallium/drivers/softpipe/ci/gitlab-ci.yml
index 5561a839ca0..c3468bb1a05 100644
--- a/src/gallium/drivers/softpipe/ci/gitlab-ci.yml
+++ b/src/gallium/drivers/softpipe/ci/gitlab-ci.yml
@@ -20,9 +20,11 @@ softpipe:
     - mesa-swrast
   variables:
     DEQP_SUITE: softpipe
+    XVFB_SCRIPT: "install/deqp-runner.sh"
   extends: .softpipe-deqp-test
-  script:
-    - LD_LIBRARY_PATH=`pwd`/install/lib xvfb-run --server-args='-noreset' install/deqp-runner.sh
+  script: |
+    . "$SCRIPTS_DIR"/setup-test-env.sh
+    LD_LIBRARY_PATH=`pwd`/install/lib xvfb-run --server-args='-noreset' bash -c ". $SCRIPTS_DIR/setup-test-env.sh && ${XVFB_SCRIPT}"
 
 softpipe-asan-gles31:
   variables:
diff --git a/src/gallium/drivers/virgl/ci/gitlab-ci.yml b/src/gallium/drivers/virgl/ci/gitlab-ci.yml
index 50b0524ed39..14dfa38768f 100644
--- a/src/gallium/drivers/virgl/ci/gitlab-ci.yml
+++ b/src/gallium/drivers/virgl/ci/gitlab-ci.yml
@@ -6,7 +6,7 @@
     GALLIVM_PERF: nopt
     FLAKES_CHANNEL: "#virgl-ci"
   script:
-    - xvfb-run --server-args='-noreset' sh -c "GALLIUM_DRIVER=virpipe install/deqp-runner.sh"
+    - xvfb-run --server-args='-noreset' bash -c ". $SCRIPTS_DIR/setup-test-env.sh && GALLIUM_DRIVER=virpipe install/deqp-runner.sh"
 
 virpipe-on-gl:
   extends:
@@ -64,6 +64,7 @@ virgl-traces:
   tags:
     - kvm
   script:
+    - . "$SCRIPTS_DIR"/setup-test-env.sh
     # Use all threads for rendering and only run one job at a time
     # Couldn't get GitLab CI to correctly substitute the variable in the yaml
     - LP_NUM_THREADS=${FDO_CI_CONCURRENT} FDO_CI_CONCURRENT=1 install/crosvm-runner.sh install/piglit/piglit-traces.sh
diff --git a/src/gallium/drivers/zink/ci/gitlab-ci.yml b/src/gallium/drivers/zink/ci/gitlab-ci.yml
index 2ed7ea79f40..5a6edfc9106 100644
--- a/src/gallium/drivers/zink/ci/gitlab-ci.yml
+++ b/src/gallium/drivers/zink/ci/gitlab-ci.yml
@@ -41,8 +41,10 @@ zink-lvp:
     # to stdout and aborting on unknown failures.
     ZINK_DEBUG: validation
     VK_LAYER_SETTINGS_PATH: ${CI_PROJECT_DIR}/install/zink-lvp-validation-settings.txt
-  script:
-    - xvfb-run --server-args='-noreset' sh -c "GALLIUM_DRIVER=zink VK_DRIVER=lvp install/deqp-runner.sh"
+    XVFB_SCRIPT: "GALLIUM_DRIVER=zink VK_DRIVER=lvp install/deqp-runner.sh"
+  script: |
+    . "$SCRIPTS_DIR"/setup-test-env.sh
+    xvfb-run --server-args='-noreset' bash -c ". $SCRIPTS_DIR/setup-test-env.sh ${XVFB_SCRIPT}"
 
 .zink-anv-test:
   extends: