gitlab-ci: Automated testing with OpenGL traces

Introduce automated testing of Mesa by replaying traces with Renderdoc
or Apitrace.

For now only LLVMPipe is tested, but other drivers can be tested if
there's runners with the necessary hardware.

Signed-off-by: Alexandros Frantzis <alexandros.frantzis@collabora.com>
Signed-off-by: Tomeu Vizoso <tomeu.vizoso@collabora.com>
Reviewed-by: Eric Anholt <eric@anholt.net>
Tested-by: Marge Bot <https://gitlab.freedesktop.org/mesa/mesa/merge_requests/2935>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/merge_requests/2935>

19 files changed, 1056 insertions, 4 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9e70ce88233..4b6c97a6b69 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -125,7 +125,7 @@ x86_build:
 x86_test-gl:
   extends: x86_build
   variables:
-    DEBIAN_TAG: &x86_test-gl "2020-01-30"
+    DEBIAN_TAG: &x86_test-gl "2020-02-14"
 
 # Debian 10 based x86 test image for VK
 x86_test-vk:
@@ -705,3 +705,20 @@ radv_polaris10_vkcts:
     DEQP_SKIPS: deqp-radv-polaris10-skips.txt
   tags:
     - polaris10
+
+.traces-test:
+  extends:
+    - .test-gl
+  cache:
+    key: ${CI_JOB_NAME}
+    paths:
+      - .git-lfs-storage/
+  script:
+    - ./artifacts/tracie-runner.sh
+
+llvmpipe-traces:
+  variables:
+    LIBGL_ALWAYS_SOFTWARE: "true"
+    GALLIUM_DRIVER: "llvmpipe"
+    DEVICE_NAME: "vmware-llvmpipe"
+  extends: .traces-test
diff --git a/.gitlab-ci/build-apitrace.sh b/.gitlab-ci/build-apitrace.sh
new file mode 100644
index 00000000000..785a5ae52e7
--- /dev/null
+++ b/.gitlab-ci/build-apitrace.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -ex
+
+APITRACE_VERSION="9.0"
+
+git clone https://github.com/apitrace/apitrace.git --single-branch --no-checkout /apitrace
+pushd /apitrace
+git checkout "$APITRACE_VERSION"
+cmake -G Ninja -B_build -H. -DCMAKE_BUILD_TYPE=Release -DENABLE_GUI=False
+ninja -C _build -j4
+mkdir build
+cp _build/apitrace build
+cp _build/glretrace build
+cp _build/eglretrace build
+strip build/*
+find . -not -path './build' -not -path './build/*' -delete
+popd
diff --git a/.gitlab-ci/build-deqp-gl.sh b/.gitlab-ci/build-deqp-gl.sh
index 13c684b31a6..57ce554f660 100644
--- a/.gitlab-ci/build-deqp-gl.sh
+++ b/.gitlab-ci/build-deqp-gl.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
 git config --global user.email "mesa@example.com"
 git config --global user.name "Mesa CI"
 git clone \
diff --git a/.gitlab-ci/build-renderdoc.sh b/.gitlab-ci/build-renderdoc.sh
new file mode 100644
index 00000000000..31c8c014b8a
--- /dev/null
+++ b/.gitlab-ci/build-renderdoc.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -ex
+
+RENDERDOC_VERSION=6653316a62f6168b3e45040358cb77612dcffcb8
+
+git clone https://github.com/baldurk/renderdoc.git --single-branch --no-checkout /renderdoc
+pushd /renderdoc
+git checkout "$RENDERDOC_VERSION"
+cmake -G Ninja -B_build -H. -DENABLE_QRENDERDOC=false -DCMAKE_BUILD_TYPE=Release
+ninja -C _build -j4
+mkdir -p build/lib
+cp _build/lib/renderdoc.so build/lib
+cp _build/lib/librenderdoc.so build/lib
+strip build/lib/*
+find . -not -path './build' -not -path './build/*' -delete
+popd
diff --git a/.gitlab-ci/container/x86_test-gl.sh b/.gitlab-ci/container/x86_test-gl.sh
index 961ad2accc6..6a0a78183e0 100644
--- a/.gitlab-ci/container/x86_test-gl.sh
+++ b/.gitlab-ci/container/x86_test-gl.sh
@@ -28,33 +28,49 @@ EOF
 apt-get dist-upgrade -y
 
 apt-get install -y --no-remove \
+      autoconf \
+      automake \
       cmake \
       g++ \
       git \
+      git-lfs \
       gcc \
       libexpat1 \
       libgbm-dev \
       libgles2-mesa-dev \
+      libpcre32-3 \
+      libpcre3-dev \
       libpng16-16 \
       libpng-dev \
+      libpython3.7 \
       libvulkan1 \
       libvulkan-dev \
       libwaffle-dev \
       libwayland-server0 \
+      libxcb-keysyms1 \
+      libxcb-keysyms1-dev \
       libxcb-xfixes0 \
       libxkbcommon0 \
       libxkbcommon-dev \
       libxrender1 \
       libxrender-dev \
       libllvm9 \
+      make \
       meson \
       patch \
       pkg-config \
+      python \
+      python3.7 \
+      python3.7-dev \
       python3-distutils \
       python3-mako \
       python3-numpy \
+      python3-pil \
+      python3-pilkit \
       python3-six \
-      python \
+      python3-yaml \
+      qt5-default \
+      qt5-qmake \
       waffle-utils \
       xauth \
       xvfb \
@@ -73,26 +89,37 @@ apt-get install -y --no-remove \
 
 . .gitlab-ci/build-deqp-gl.sh
 
+############### Build apitrace
+
+. .gitlab-ci/build-apitrace.sh
+
+############### Build renderdoc
+
+. .gitlab-ci/build-renderdoc.sh
 
 ############### Uninstall the build software
 
 apt-get purge -y \
+      autoconf \
+      automake \
       cmake \
       g++ \
       gcc \
-      git \
       gnupg \
       libc6-dev \
       libgbm-dev \
       libgles2-mesa-dev \
+      libpcre3-dev \
       libpng-dev \
       libwaffle-dev \
+      libxcb-keysyms1-dev \
       libxkbcommon-dev \
       libxrender-dev \
+      make \
       meson \
       patch \
       pkg-config \
-      python \
+      python3.7-dev \
       python3-distutils
 
 apt-get autoremove -y --purge
diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh
index b1424b12102..f6a3f1beee4 100755
--- a/.gitlab-ci/prepare-artifacts.sh
+++ b/.gitlab-ci/prepare-artifacts.sh
@@ -26,6 +26,9 @@ mkdir -p artifacts/
 cp VERSION artifacts/
 cp -Rp .gitlab-ci/deqp* artifacts/
 cp -Rp .gitlab-ci/piglit artifacts/
+cp -Rp .gitlab-ci/traces.yml artifacts/
+cp -Rp .gitlab-ci/tracie artifacts/
+cp -Rp .gitlab-ci/tracie-runner.sh artifacts/
 
 # Tar up the install dir so that symlinks and hardlinks aren't each
 # packed separately in the zip file.
diff --git a/.gitlab-ci/traces.yml b/.gitlab-ci/traces.yml
new file mode 100644
index 00000000000..d390f185c20
--- /dev/null
+++ b/.gitlab-ci/traces.yml
@@ -0,0 +1,17 @@
+traces-db:
+  repo: "https://gitlab.freedesktop.org/gfx-ci/tracie/traces-db"
+  commit: "595235059fc84d7b03930aa0262ebca091d8260f"
+
+traces:
+  - path: glmark2/desktop-blur-radius=5:effect=blur:passes=1:separable=true:windows=4.rdc
+    expectations:
+      - device: vmware-llvmpipe
+        checksum: 8867f3a41f180626d0d4b7661ff5c0f4
+  - path: glmark2/jellyfish.rdc
+    expectations:
+      - device: vmware-llvmpipe
+        checksum: e0fe979fee129c0ed42a3059d1a4e1c9
+  - path: glxgears/glxgears.trace
+    expectations:
+      - device: vmware-llvmpipe
+        checksum: 02aca9b4b4ad6fd60331df6e4f87f2cd
diff --git a/.gitlab-ci/tracie-runner.sh b/.gitlab-ci/tracie-runner.sh
new file mode 100755
index 00000000000..7b5db08e105
--- /dev/null
+++ b/.gitlab-ci/tracie-runner.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+set -ex
+
+ARTIFACTS="$(pwd)/artifacts"
+
+# Set up the driver environment.
+export LD_LIBRARY_PATH="$(pwd)/install/lib/"
+
+# Set environment for renderdoc libraries.
+export PYTHONPATH="$PYTHONPATH:/renderdoc/build/lib"
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/renderdoc/build/lib"
+
+# Perform a self-test to ensure tracie is working properly.
+"$ARTIFACTS/tracie/tests/test.sh"
+
+ret=0
+
+# The renderdoc version we use can handle surfaceless.
+EGL_PLATFORM=surfaceless DISPLAY= \
+    "$ARTIFACTS/tracie/tracie.sh" "$ARTIFACTS/traces.yml" renderdoc \
+    || ret=1
+
+# We need a newer waffle to use surfaceless with apitrace. For now run with
+# xvfb.
+xvfb-run --server-args="-noreset" sh -c \
+    "set -ex; \
+     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH; \
+     export PATH=/apitrace/build:\$PATH; \
+    \"$ARTIFACTS/tracie/tracie.sh\" \"$ARTIFACTS/traces.yml\" apitrace" \
+    || ret=1
+
+exit $ret
diff --git a/.gitlab-ci/tracie/README.md b/.gitlab-ci/tracie/README.md
new file mode 100644
index 00000000000..5c1f8a881ac
--- /dev/null
+++ b/.gitlab-ci/tracie/README.md
@@ -0,0 +1,126 @@
+Tracie - Mesa Traces Continuous Integration System
+==================================================
+
+Home of the Mesa trace testing effort.
+
+### Traces definition file
+
+The trace definition file contains information about the git repo/commit to get
+the traces from, and a list of the traces to run along with their expected image
+checksums on each device. An example:
+
+```yaml
+traces-db:
+  repo: https://gitlab.freedesktop.org/gfx-ci/tracie/traces-db
+  commit: master
+
+traces:
+  - path: glmark2/jellyfish.rdc
+    expectations:
+      - device: intel-0x3185
+        checksum: 58359ea4caf6ad44c6b65526881bbd17
+      - device: vmware-llvmpipe
+        checksum: d82267c25a0decdad7b563c56bb81106
+  - path: supertuxkart/supertuxkart-antediluvian-abyss.rdc
+    expectations:
+      - device: intel-0x3185
+        checksum: ff827f7eb069afd87cc305a422cba939
+```
+
+The traces-db entry can be absent, in which case it is assumed that the
+current directory is the traces-db directory.
+
+Traces that don't have an expectation for the current device are skipped
+during trace replay.
+
+Adding a new trace to the list involves commiting the trace to the git repo and
+adding an entry to the `traces` list. The reference checksums can be calculated
+with the [image_checksum.py](.gitlab-ci/tracie/image_checksum.py) script.
+Alternatively, an arbitrary checksum can be used, and during replay (see below)
+the scripts will report the mismatch and expected checksum.
+
+### Trace-db repos
+
+The trace-db repos are assumed to be git repositories using LFS for their trace
+files. This is so that trace files can be potentially checked out and replayed
+individually, thus reducing storage requirements during CI runs.
+
+### Enabling trace testing on a new device
+
+To enable trace testing on a new device:
+
+1. Create a new job in .gitlab-ci.yml. The job will need to be tagged
+   to run on runners with the appropriate hardware. Use the `.traces-test`
+   template job as a base, and make sure you set a unique value for the
+   `DEVICE_NAME` variable:
+
+   ```yaml
+   my-hardware-traces:
+     variables:
+       DEVICE_NAME: "myhardware"
+     extends: .traces-test
+   ```
+
+2. Update the .gitlab-ci/traces.yml file with expectations for the new device.
+   Ensure that the device name used in the expectations matches the one
+   set in the job. For more information, and tips about how to calculate
+   the checksums, see the section describing the trace definition files.
+
+### Trace files
+
+Tracie supports both renderdoc (.rdc) and apitrace (.trace) files. Trace files
+need to have the correct extension so that tracie can detect them properly.
+
+The trace files that are contained in public traces-db repositories must be
+legally redistributable. This is typically true for FOSS games and
+applications. Traces for proprietary games and application are typically not
+redistributable, unless specific redistribution rights have been granted by the
+publisher.
+
+### Replaying traces
+
+Mesa traces CI uses a set of scripts to replay traces and check the output
+against reference checksums.
+
+The high level script [tracie.sh](.gitlab-ci/tracie/tracie.sh) accepts
+a traces definition file and the type of traces (apitrace/renderdoc) to run:
+
+    tracie.sh .gitlab-ci/traces.yml renderdoc
+
+tracie.sh copies produced artifacts to the `$CI_PROJECT_DIR/result`
+directory. By default, created images from traces are only stored in case of a
+checksum mismatch. The `TRACIE_STORE_IMAGES` CI/environment variable can be set
+to `1` to force storing images, e.g., to get a complete set of reference
+images.
+
+The `tracie.sh` script requires that the environment variable `DEVICE_NAME` is
+properly set for the target machine, and matches the `device` field of the
+relevant trace expectations in the used `traces.yml` file.
+
+At a lower level the
+[dump_trace_images.py](.gitlab-ci/tracie/dump_trace_images.py) script is
+called, which replays a trace, dumping a set of images in the process. By
+default only the image corresponding to the last frame of the trace is dumped,
+but this can be changed with the `--calls` parameter. The dumped images are
+stored in a subdirectory `test/<device-name>` next to the trace file itself,
+with names of the form `tracefilename-callnum.png`.  The full log of any
+commands used while dumping the images is also saved in a file in the
+'test/<device-name>' subdirectory, named after the trace name with '.log'
+appended.
+
+Examples:
+
+    python3 dump_traces_images.py --device-name=vmware-llvmpipe mytrace.trace
+    python3 dump_traces_images.py --device-name=vmware-llvmpipe --calls=2075,3300 mytrace.trace
+
+### Running the replay scripts locally
+
+It's often useful, especially during development, to be able to run the scripts
+locally. The scripts require a recent version of apitrace being in the path,
+and also the renderdoc python module being available.
+
+To ensure python3 can find the renderdoc python module you need to set
+`PYTHONPATH` to point to the location of `renderdoc.so` (binary python modules)
+and `LD_LIBRARY_PATH` to point to the location of `librenderdoc.so`. In the
+renderdoc build tree, both of these are in `renderdoc/<builddir>/lib`. Note
+that renderdoc doesn't install the `renderdoc.so` python module.
diff --git a/.gitlab-ci/tracie/dump_trace_images.py b/.gitlab-ci/tracie/dump_trace_images.py
new file mode 100644
index 00000000000..66a99f7efe2
--- /dev/null
+++ b/.gitlab-ci/tracie/dump_trace_images.py
@@ -0,0 +1,134 @@
+#!/usr/bin/python3
+
+# Copyright (c) 2019 Collabora Ltd
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+import os
+import sys
+import subprocess
+from pathlib import Path
+from traceutil import trace_type_from_filename, TraceType
+
+def log(severity, msg, end='\n'):
+    print("[dump_trace_images] %s: %s" % (severity, msg), flush=True, end=end)
+
+def log_result(msg):
+    print(msg, flush=True)
+
+def run_logged_command(cmd, log_path):
+    ret = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    logoutput = ("[dump_trace_images] Running: %s\n" % " ".join(cmd)).encode() + \
+                ret.stdout
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    with log_path.open(mode='wb') as log:
+        log.write(logoutput)
+    if ret.returncode:
+        raise RuntimeError(
+            logoutput.decode(errors='replace') +
+            "[dump_traces_images] Process failed with error code: %d" % ret.returncode)
+
+def get_last_apitrace_frame_call(trace_path):
+    cmd = ["apitrace", "dump", "--calls=frame", str(trace_path)]
+    ret = subprocess.run(cmd, stdout=subprocess.PIPE)
+    for l in reversed(ret.stdout.decode(errors='replace').splitlines()):
+        s = l.split(None, 1)
+        if len(s) >= 1 and s[0].isnumeric():
+            return int(s[0])
+    return -1
+
+def dump_with_apitrace(trace_path, calls, device_name):
+    outputdir = str(trace_path.parent / "test" / device_name)
+    os.makedirs(outputdir, exist_ok=True)
+    outputprefix = str(Path(outputdir) / trace_path.name) + "-"
+    if len(calls) == 0:
+        calls = [str(get_last_apitrace_frame_call(trace_path))]
+    cmd = ["apitrace", "dump-images", "--calls=" + ','.join(calls),
+           "-o", outputprefix, str(trace_path)]
+    log_path = Path(outputdir) / (trace_path.name + ".log")
+    run_logged_command(cmd, log_path)
+
+def dump_with_renderdoc(trace_path, calls, device_name):
+    outputdir = str(trace_path.parent / "test" / device_name)
+    script_path = Path(os.path.dirname(os.path.abspath(__file__)))
+    cmd = [str(script_path / "renderdoc_dump_images.py"), str(trace_path), outputdir]
+    cmd.extend(calls)
+    log_path = Path(outputdir) / (trace_path.name + ".log")
+    run_logged_command(cmd, log_path)
+
+def dump_with_testtrace(trace_path, calls, device_name):
+    from PIL import Image
+    outputdir_path = trace_path.parent / "test" / device_name
+    outputdir_path.mkdir(parents=True, exist_ok=True)
+    with trace_path.open() as f:
+        rgba = f.read()
+    color = [int(rgba[0:2], 16), int(rgba[2:4], 16),
+             int(rgba[4:6], 16), int(rgba[6:8], 16)]
+    if len(calls) == 0: calls = ["0"]
+    for c in calls:
+        outputfile = str(outputdir_path / trace_path.name) + "-" + c + ".png"
+        log_path = outputdir_path / (trace_path.name + ".log")
+        with log_path.open(mode='w') as log:
+            log.write("Writing RGBA: %s to %s" % (rgba, outputfile))
+        Image.frombytes('RGBA', (32, 32), bytes(color * 32 * 32)).save(outputfile)
+
+def dump_from_trace(trace_path, calls, device_name):
+    log("Info", "Dumping trace %s" % trace_path, end='... ')
+    trace_type = trace_type_from_filename(trace_path.name)
+    try:
+        if trace_type == TraceType.APITRACE:
+            dump_with_apitrace(trace_path, calls, device_name)
+        elif trace_type == TraceType.RENDERDOC:
+            dump_with_renderdoc(trace_path, calls, device_name)
+        elif trace_type == TraceType.TESTTRACE:
+            dump_with_testtrace(trace_path, calls, device_name)
+        else:
+            raise RuntimeError("Unknown tracefile extension")
+        log_result("OK")
+        return True
+    except Exception as e:
+        log_result("ERROR")
+        log("Debug", "=== Failure log start ===")
+        print(e)
+        log("Debug", "=== Failure log end ===")
+        return False
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('tracepath', help="trace to dump")
+    parser.add_argument('--device-name', required=True,
+                        help="the name of the graphics device used to produce images")
+    parser.add_argument('--calls', required=False,
+                        help="the call numbers from the trace to dump (default: last frame)")
+
+    args = parser.parse_args()
+    if args.calls is not None:
+        args.calls = args.calls.split(",")
+    else:
+        args.calls = []
+
+    success = dump_from_trace(Path(args.tracepath), args.calls, args.device_name)
+
+    sys.exit(0 if success else 1)
+
+if __name__ == "__main__":
+    main()
diff --git a/.gitlab-ci/tracie/image_checksum.py b/.gitlab-ci/tracie/image_checksum.py
new file mode 100644
index 00000000000..e920c1e7341
--- /dev/null
+++ b/.gitlab-ci/tracie/image_checksum.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2019 Collabora Ltd
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+import hashlib
+from PIL import Image
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('imagefile', help='image file to calculate checksum for')
+
+    args = parser.parse_args()
+
+    md5 = hashlib.md5(Image.open(args.imagefile).tobytes())
+    print(md5.hexdigest())
+
+if __name__ == "__main__":
+    main()
diff --git a/.gitlab-ci/tracie/query_traces_yaml.py b/.gitlab-ci/tracie/query_traces_yaml.py
new file mode 100644
index 00000000000..6ed069ba804
--- /dev/null
+++ b/.gitlab-ci/tracie/query_traces_yaml.py
@@ -0,0 +1,107 @@
+#!/usr/bin/python3
+
+# Copyright (c) 2019 Collabora Ltd
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+import yaml
+from traceutil import all_trace_type_names, trace_type_from_name
+from traceutil import trace_type_from_filename
+
+def trace_devices(trace):
+    return [e['device'] for e in trace['expectations']]
+
+def cmd_traces_db_repo(args):
+    with open(args.file, 'r') as f:
+        y = yaml.safe_load(f)
+    print(y['traces-db']['repo'])
+
+def cmd_traces_db_commit(args):
+    with open(args.file, 'r') as f:
+        y = yaml.safe_load(f)
+    print(y['traces-db']['commit'])
+
+def cmd_traces(args):
+    with open(args.file, 'r') as f:
+        y = yaml.safe_load(f)
+
+    traces = y['traces']
+    traces = filter(lambda t: trace_type_from_filename(t['path']) in args.trace_types,
+                    traces)
+    if args.device_name:
+        traces = filter(lambda t: args.device_name in trace_devices(t), traces)
+
+    traces = list(traces)
+
+    if len(traces) == 0:
+        return
+
+    print('\n'.join((t['path'] for t in traces)))
+
+def cmd_checksum(args):
+    with open(args.file, 'r') as f:
+        y = yaml.safe_load(f)
+
+    traces = y['traces']
+    trace = next(t for t in traces if t['path'] == args.trace_path)
+    expectation = next(e for e in trace['expectations'] if e['device'] == args.device_name)
+
+    print(expectation['checksum'])
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--file', required=True,
+                        help='the name of the yaml file')
+
+    subparsers = parser.add_subparsers(help='sub-command help')
+
+    parser_traces_db_repo = subparsers.add_parser('traces_db_repo')
+    parser_traces_db_repo.set_defaults(func=cmd_traces_db_repo)
+
+    parser_traces_db_commit = subparsers.add_parser('traces_db_commit')
+    parser_traces_db_commit.set_defaults(func=cmd_traces_db_commit)
+
+    parser_traces = subparsers.add_parser('traces')
+    parser_traces.add_argument('--device-name', required=False,
+                               help="the name of the graphics device used to "
+                                     "produce images")
+    parser_traces.add_argument('--trace-types', required=False,
+                               default=",".join(all_trace_type_names()),
+                               help="the types of traces to look for in recursive "
+                                    "dir walks " "(by default all types)")
+    parser_traces.set_defaults(func=cmd_traces)
+
+    parser_checksum = subparsers.add_parser('checksum')
+    parser_checksum.add_argument('--device-name', required=True,
+                               help="the name of the graphics device used to "
+                                     "produce images")
+    parser_checksum.add_argument('trace_path')
+    parser_checksum.set_defaults(func=cmd_checksum)
+
+    args = parser.parse_args()
+    if hasattr(args, 'trace_types'):
+        args.trace_types = [trace_type_from_name(t) for t in args.trace_types.split(",")]
+
+    args.func(args)
+
+if __name__ == "__main__":
+    main()
diff --git a/.gitlab-ci/tracie/renderdoc_dump_images.py b/.gitlab-ci/tracie/renderdoc_dump_images.py
new file mode 100755
index 00000000000..f1252c11f25
--- /dev/null
+++ b/.gitlab-ci/tracie/renderdoc_dump_images.py
@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2019 Collabora Ltd
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# SPDX-License-Identifier: MIT
+
+import sys
+from pathlib import Path
+
+import renderdoc as rd
+
+def findDrawWithEventId(controller, eventId):
+   for d in controller.GetDrawcalls():
+      if d.eventId == eventId:
+         return d
+
+   return None
+
+def dumpImage(controller, eventId, outputDir, tracefile):
+    draw = findDrawWithEventId(controller, eventId)
+    if draw is None:
+        raise RuntimeError("Couldn't find draw call with eventId " + str(eventId))
+
+    controller.SetFrameEvent(draw.eventId, True)
+
+    texsave = rd.TextureSave()
+
+    # Select the first color output
+    texsave.resourceId = draw.outputs[0]
+
+    if texsave.resourceId == rd.ResourceId.Null():
+        return
+
+    filepath = Path(outputDir)
+    filepath.mkdir(parents = True, exist_ok = True)
+    filepath = filepath / (tracefile + "-" + str(int(draw.eventId)) + ".png")
+
+    print("Saving image at eventId %d: %s to %s" % (draw.eventId, draw.name, filepath))
+
+    # Most formats can only display a single image per file, so we select the
+    # first mip and first slice
+    texsave.mip = 0
+    texsave.slice.sliceIndex = 0
+
+    # For formats with an alpha channel, preserve it
+    texsave.alpha = rd.AlphaMapping.Preserve
+    texsave.destType = rd.FileType.PNG
+    controller.SaveTexture(texsave, str(filepath))
+
+def loadCapture(filename):
+    cap = rd.OpenCaptureFile()
+
+    status = cap.OpenFile(filename, '', None)
+
+    if status != rd.ReplayStatus.Succeeded:
+        raise RuntimeError("Couldn't open file: " + str(status))
+    if not cap.LocalReplaySupport():
+        raise RuntimeError("Capture cannot be replayed")
+
+    status,controller = cap.OpenCapture(rd.ReplayOptions(), None)
+
+    if status != rd.ReplayStatus.Succeeded:
+        raise RuntimeError("Couldn't initialise replay: " + str(status))
+
+    return (cap, controller)
+
+def renderdoc_dump_images(filename, eventIds, outputDir):
+   rd.InitGlobalEnv(rd.GlobalEnvironment(), [])
+   cap,controller = loadCapture(filename);
+
+   tracefile = Path(filename).name
+
+   if len(eventIds) == 0:
+      eventIds.append(controller.GetDrawcalls()[-1].eventId)
+
+   for eventId in eventIds:
+      dumpImage(controller, eventId, outputDir, tracefile)
+
+   controller.Shutdown()
+   cap.Shutdown()
+
+if __name__ == "__main__":
+   if len(sys.argv) < 3:
+      raise RuntimeError("Usage: renderdoc_dump_images.py <trace> <outputdir> [<draw-id>...]")
+
+   eventIds = [int(e) for e in sys.argv[3:]]
+
+   renderdoc_dump_images(sys.argv[1], eventIds, sys.argv[2])
diff --git a/.gitlab-ci/tracie/tests/test-data/trace1/magenta.testtrace b/.gitlab-ci/tracie/tests/test-data/trace1/magenta.testtrace
new file mode 100644
index 00000000000..2354cb56da6
--- /dev/null
+++ b/.gitlab-ci/tracie/tests/test-data/trace1/magenta.testtrace
@@ -0,0 +1 @@
+ff00ffff
diff --git a/.gitlab-ci/tracie/tests/test-data/trace2/olive.testtrace b/.gitlab-ci/tracie/tests/test-data/trace2/olive.testtrace
new file mode 100644
index 00000000000..825890f66c5
--- /dev/null
+++ b/.gitlab-ci/tracie/tests/test-data/trace2/olive.testtrace
@@ -0,0 +1 @@
+80800080
diff --git a/.gitlab-ci/tracie/tests/test.sh b/.gitlab-ci/tracie/tests/test.sh
new file mode 100755
index 00000000000..7b4c9e822ce
--- /dev/null
+++ b/.gitlab-ci/tracie/tests/test.sh
@@ -0,0 +1,214 @@
+#!/bin/sh
+
+TRACIE_DIR="$(dirname "$(readlink -f "$0")")/.."
+TEST_DIR=""
+TEST_EXIT=0
+
+create_repo() {
+    repo="$(mktemp -d $TEST_DIR/repo.XXXXXXXXXX)"
+    cp -R "$TEST_DIR"/tests/test-data/* "$repo"
+    (
+     cd "$repo";
+     git init -q .;
+     git config user.email "me@example.com"
+     git config user.name "Me me"
+     git lfs track '*.testtrace' > /dev/null;
+     git add .;
+     git commit -q -a -m 'initial';
+    )
+    echo $repo
+}
+
+destroy_repo() {
+    [ -d "$1"/.git ] && rm -rf "$1"
+}
+
+assert() {
+    if ! $1; then
+        echo "Assertion failed:  \"$1\""
+        exit 1
+    fi
+}
+
+run_tracie() {
+    # Run tests for the .testtrace types, using the "test-device" device name.
+    DEVICE_NAME=test-device CI_PROJECT_DIR="$TEST_DIR" \
+        "$TEST_DIR/tracie.sh" "$TEST_DIR/tests/traces.yml" testtrace
+}
+
+cleanup() {
+    rm -rf "$TEST_DIR"
+}
+
+prepare_for_run() {
+    TEST_DIR="$(mktemp -d -t tracie.test.XXXXXXXXXX)"
+    # Copy all the tracie scripts to the the test dir and later make that the
+    # CI_PROJECT_DIR for the run-tests.sh script. This avoids polluting the
+    # normal working dir with test result artifacts.
+    cp -R "$TRACIE_DIR"/. "$TEST_DIR"
+    trap cleanup EXIT
+    # Ensure we have a clean environment.
+    unset TRACIE_STORE_IMAGES
+}
+
+run_test() {
+    prepare_for_run
+    log=$(mktemp)
+    if ($1 > "$log" 2>&1 ;); then
+        if [ -t 1 ]; then
+            echo "$1: \e[0;32mSuccess\e[0m"
+        else
+            echo "$1: Success"
+        fi
+    else
+        if [ -t 1 ]; then
+            echo "$1: \e[0;31mFail\e[0m"
+        else
+            echo "$1: Fail"
+        fi
+        cat "$log"
+        TEST_EXIT=1
+    fi
+    rm "$log"
+    cleanup
+}
+
+tracie_succeeds_if_all_images_match() {
+    repo="$(create_repo)"
+    cd "$repo"
+
+    run_tracie
+    assert "[ $? = 0 ]"
+
+    destroy_repo "$repo"
+}
+
+tracie_fails_on_image_mismatch() {
+    repo="$(create_repo)"
+    cd "$repo"
+
+    sed -i 's/5efda83854befe0155ff8517a58d5b51/8e0a801367e1714463475a824dab363b/g' \
+        "$TEST_DIR/tests/traces.yml"
+
+    run_tracie
+    assert "[ $? != 0 ]"
+
+    destroy_repo "$repo"
+}
+
+tracie_ignores_unspecified_trace_types() {
+    repo="$(create_repo)"
+    cd "$repo"
+
+    echo "  - path: trace1/empty.trace" >> "$TEST_DIR/tests/traces.yml"
+    echo "    expectations:" >> "$TEST_DIR/tests/traces.yml"
+    echo "    - device: test-device" >> "$TEST_DIR/tests/traces.yml"
+    echo "      checksum: 000000000000000" >> "$TEST_DIR/tests/traces.yml"
+    # For the tests we only scan for the .testtrace type,
+    # so the .trace file added below should be ignored.
+    echo "empty" > trace1/empty.trace
+    git lfs track '*.trace'
+    git add trace1
+    git commit -a -m 'break'
+
+    run_tracie
+    assert "[ $? = 0 ]"
+
+    destroy_repo "$repo"
+}
+
+tracie_skips_traces_without_checksum() {
+    repo="$(create_repo)"
+    cd "$repo"
+
+    echo "  - path: trace1/red.testtrace" >> "$TEST_DIR/tests/traces.yml"
+    echo "    expectations:" >> "$TEST_DIR/tests/traces.yml"
+    echo "    - device: bla" >> "$TEST_DIR/tests/traces.yml"
+    echo "      checksum: 000000000000000" >> "$TEST_DIR/tests/traces.yml"
+    # red.testtrace should be skipped, since it doesn't
+    # have any checksums for our device
+    echo "ff0000ff" > trace1/red.testtrace
+    git add trace1
+    git commit -a -m 'red'
+
+    run_tracie
+    assert "[ $? = 0 ]"
+
+    destroy_repo "$repo"
+}
+
+tracie_fails_on_dump_image_error() {
+    repo="$(create_repo)"
+    cd "$repo"
+
+    # "invalid" should fail to parse as rgba and
+    # cause an error
+    echo "invalid" > trace1/magenta.testtrace
+    git add trace1
+    git commit -a -m 'invalid'
+
+    run_tracie
+    assert "[ $? != 0 ]"
+
+    destroy_repo "$repo"
+}
+
+tracie_stores_only_logs_on_checksum_match() {
+    repo="$(create_repo)"
+    cd "$repo"
+
+    run_tracie
+    assert "[ $? = 0 ]"
+
+    assert "[ -f "$TEST_DIR/results/trace1/test/test-device/magenta.testtrace.log" ]"
+    assert "[ -f "$TEST_DIR/results/trace2/test/test-device/olive.testtrace.log" ]"
+
+    assert "[ ! -f "$TEST_DIR/results/trace1/test/test-device/magenta.testtrace-0.png" ]"
+    assert "[ ! -f "$TEST_DIR/results/trace2/test/test-device/olive.testtrace-0.png" ]"
+
+    ls -lR "$TEST_DIR"
+
+    destroy_repo "$repo"
+}
+
+tracie_stores_images_on_checksum_mismatch() {
+    repo="$(create_repo)"
+    cd "$repo"
+
+    sed -i 's/5efda83854befe0155ff8517a58d5b51/8e0a801367e1714463475a824dab363b/g' \
+        "$TEST_DIR/tests/traces.yml"
+
+    run_tracie
+    assert "[ $? != 0 ]"
+
+    assert "[ ! -f "$TEST_DIR/results/trace1/test/test-device/magenta.testtrace-0.png" ]"
+    assert "[ -f "$TEST_DIR/results/trace2/test/test-device/olive.testtrace-0.png" ]"
+
+    destroy_repo "$repo"
+}
+
+tracie_stores_images_on_request() {
+    repo="$(create_repo)"
+    cd "$repo"
+
+    (export TRACIE_STORE_IMAGES=1; run_tracie)
+    assert "[ $? = 0 ]"
+
+    assert "[ -f "$TEST_DIR/results/trace1/test/test-device/magenta.testtrace-0.png" ]"
+    assert "[ -f "$TEST_DIR/results/trace2/test/test-device/olive.testtrace-0.png" ]"
+
+    ls -lR "$TEST_DIR"
+
+    destroy_repo "$repo"
+}
+
+run_test tracie_succeeds_if_all_images_match
+run_test tracie_fails_on_image_mismatch
+run_test tracie_ignores_unspecified_trace_types
+run_test tracie_skips_traces_without_checksum
+run_test tracie_fails_on_dump_image_error
+run_test tracie_stores_only_logs_on_checksum_match
+run_test tracie_stores_images_on_checksum_mismatch
+run_test tracie_stores_images_on_request
+
+exit $TEST_EXIT
diff --git a/.gitlab-ci/tracie/tests/traces.yml b/.gitlab-ci/tracie/tests/traces.yml
new file mode 100644
index 00000000000..dc6dd04b526
--- /dev/null
+++ b/.gitlab-ci/tracie/tests/traces.yml
@@ -0,0 +1,9 @@
+traces:
+  - path: trace1/magenta.testtrace
+    expectations:
+      - device: test-device
+        checksum: 8e0a801367e1714463475a824dab363b
+  - path: trace2/olive.testtrace
+    expectations:
+      - device: test-device
+        checksum: 5efda83854befe0155ff8517a58d5b51
diff --git a/.gitlab-ci/tracie/traceutil.py b/.gitlab-ci/tracie/traceutil.py
new file mode 100644
index 00000000000..1b4de23ba5d
--- /dev/null
+++ b/.gitlab-ci/tracie/traceutil.py
@@ -0,0 +1,58 @@
+# Copyright (c) 2019 Collabora Ltd
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# SPDX-License-Identifier: MIT
+
+import os
+from pathlib import Path
+from enum import Enum, auto
+
+class TraceType(Enum):
+    UNKNOWN = auto()
+    APITRACE = auto()
+    RENDERDOC = auto()
+    TESTTRACE = auto()
+
+_trace_type_info_map = {
+    TraceType.APITRACE : ("apitrace", ".trace"),
+    TraceType.RENDERDOC : ("renderdoc", ".rdc"),
+    TraceType.TESTTRACE : ("testtrace", ".testtrace")
+}
+
+def all_trace_type_names():
+    s = []
+    for t,(name, ext) in _trace_type_info_map.items():
+        if t != TraceType.UNKNOWN:
+            s.append(name)
+    return s
+
+def trace_type_from_name(tt_name):
+    for t,(name, ext) in _trace_type_info_map.items():
+        if tt_name == name:
+            return t
+
+    return TraceType.UNKNOWN
+
+def trace_type_from_filename(trace_file):
+    for t,(name, ext) in _trace_type_info_map.items():
+        if trace_file.endswith(ext):
+            return t
+
+    return TraceType.UNKNOWN
diff --git a/.gitlab-ci/tracie/tracie.sh b/.gitlab-ci/tracie/tracie.sh
new file mode 100755
index 00000000000..afae5be3365
--- /dev/null
+++ b/.gitlab-ci/tracie/tracie.sh
@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+
+TRACIE_SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+TRACES_YAML="$(readlink -f "$1")"
+TRACE_TYPE="$2"
+
+# Clone the traces-db repo without a checkout. Since we are dealing with
+# git-lfs repositories, such clones are very lightweight. We check out
+# individual files as needed at a later stage (see fetch_trace).
+clone_traces_db_no_checkout()
+{
+    local repo="$1"
+    local commit="$2"
+    rm -rf traces-db
+    git clone --no-checkout -c lfs.storage="$CI_PROJECT_DIR/.git-lfs-storage" "$repo" traces-db
+    (cd traces-db; git reset "$commit" || git reset "origin/$commit")
+}
+
+query_traces_yaml()
+{
+    python3 "$TRACIE_SCRIPT_DIR/query_traces_yaml.py" \
+        --file "$TRACES_YAML" "$@"
+}
+
+create_clean_git()
+{
+    rm -rf .clean_git
+    cp -R .git .clean_git
+}
+
+restore_clean_git()
+{
+    rm -rf .git
+    cp -R .clean_git .git
+}
+
+fetch_trace()
+{
+    local trace="${1//,/?}"
+    echo -n "[fetch_trace] Fetching $1... "
+    local output=$(git lfs pull -I "$trace" 2>&1)
+    local ret=0
+    if [[ $? -ne 0 || ! -f "$1" ]]; then
+        echo "ERROR"
+        echo "$output"
+        ret=1
+    else
+        echo "OK"
+    fi
+    # Restore a clean .git directory, effectively removing any downloaded
+    # git-lfs objects, in order to limit required storage. Note that the
+    # checked out trace file is still present at this point. We remove it
+    # when we are done with the trace replay at a later stage.
+    restore_clean_git
+    return $ret
+}
+
+get_dumped_file()
+{
+    local trace="$1"
+    local tracedir="$(dirname "$trace")"
+    local tracename="$(basename "$trace")"
+
+    find "$tracedir/test/$DEVICE_NAME" -name "$tracename*.$2"
+}
+
+check_image()
+{
+    local trace="$1"
+    local image="$2"
+
+    checksum=$(python3 "$TRACIE_SCRIPT_DIR/image_checksum.py" "$image")
+    expected=$(query_traces_yaml checksum --device-name "$DEVICE_NAME" "$trace")
+    if [[ "$checksum" = "$expected" ]]; then
+        echo "[check_image] Images match for $trace"
+        return 0
+    else
+        echo "[check_image] Images differ for $trace (expected: $expected, actual: $checksum)"
+        echo "[check_image] For more information see https://gitlab.freedesktop.org/mesa/mesa/blob/master/.gitlab-ci/tracie/README.md"
+        return 1
+    fi
+}
+
+archive_artifact()
+{
+    mkdir -p "$CI_PROJECT_DIR/results"
+    cp --parents "$1" "$CI_PROJECT_DIR/results"
+}
+
+if [[ -n "$(query_traces_yaml traces_db_repo)" ]]; then
+    clone_traces_db_no_checkout "$(query_traces_yaml traces_db_repo)" \
+                                "$(query_traces_yaml traces_db_commit)"
+    cd traces-db
+else
+    echo "Warning: No traces-db entry in $TRACES_YAML, assuming traces-db is current directory"
+fi
+
+# During git operations various git objects get created which
+# may take up significant space. Store a clean .git instance,
+# which we restore after various git operations to keep our
+# storage consumption low.
+create_clean_git
+
+ret=0
+
+for trace in $(query_traces_yaml traces --device-name "$DEVICE_NAME" --trace-types "$TRACE_TYPE")
+do
+    [[ -n "$(query_traces_yaml checksum --device-name "$DEVICE_NAME" "$trace")" ]] ||
+        { echo "[fetch_trace] Skipping $trace since it has no checksums for $DEVICE_NAME"; continue; }
+    fetch_trace "$trace" || exit $?
+    python3 "$TRACIE_SCRIPT_DIR/dump_trace_images.py" --device-name "$DEVICE_NAME" "$trace" || exit $?
+    image="$(get_dumped_file "$trace" png)"
+    check_image "$trace" "$image" && check_succeeded=true || { ret=1; check_succeeded=false; }
+    if [[ "$check_succeeded" = false || "$TRACIE_STORE_IMAGES" = "1" ]]; then
+        archive_artifact "$image"
+    fi
+    archive_artifact "$(get_dumped_file "$trace" log)"
+    # Remove the downloaded trace file to reduce the total amount of storage
+    # that is required.
+    rm "$trace"
+done
+
+exit $ret