116 files changed, 3321 insertions, 506 deletions
diff --git a/render-test/android-manifest.json b/render-test/android-manifest.json
new file mode 100644
index 0000000000..56223d4753
--- /dev/null
+++ b/render-test/android-manifest.json
@@ -0,0 +1,7 @@
+{
+    "base_test_path":"mapbox-gl-js/test/integration",
+    "expectation_paths":["render-test/expected/render-tests"],
+    "ignore_paths":["platform/node/test/ignores.json", "render-test/linux-ignores.json", "render-test/tests/should-fail.json"],
+    "vendor_path":"vendor",
+    "asset_path": "mapbox-gl-js/test/integration"
+}
+\ No newline at end of file
diff --git a/render-test/android/README.md b/render-test/android/README.md
new file mode 100644
index 0000000000..270c970fce
--- /dev/null
+++ b/render-test/android/README.md
@@ -0,0 +1,9 @@
+# RenderTestRunner
+
+This app is a purely native application, with no Java source code, that can run **mbgl-render-test-runner** on android devices.
+
+
+## Setup the test environment
+- Run render_test_setup.sh so that all the necessary test resources are pushed to the device.
+
+- Switch on storage permission of the app so that it can read/write data on SD card.
+\ No newline at end of file
diff --git a/render-test/android/app/build.gradle b/render-test/android/app/build.gradle
new file mode 100644
index 0000000000..60609e3ba2
--- /dev/null
+++ b/render-test/android/app/build.gradle
@@ -0,0 +1,29 @@
+apply plugin: 'com.android.application'
+
+android {
+    compileSdkVersion 28
+
+    defaultConfig {
+        applicationId = 'com.mapbox.mapboxsdk.maps.render_test_runner'
+        minSdkVersion 14
+        targetSdkVersion 28
+        externalNativeBuild {
+            cmake {
+                arguments '-DANDROID_CCACHE=ccache'
+                arguments '-DANDROID_STL=c++_static'
+                targets 'mbgl-render-test-runner'
+            }
+        }
+    }
+    externalNativeBuild {
+        cmake {
+            version '3.10.2'
+            path '../../../next/CMakeLists.txt'
+        }
+    }
+}
+
+dependencies {
+    implementation 'androidx.appcompat:appcompat:1.0.2'
+    implementation 'androidx.constraintlayout:constraintlayout:1.1.3'
+}
diff --git a/render-test/android/app/src/main/AndroidManifest.xml b/render-test/android/app/src/main/AndroidManifest.xml
new file mode 100644
index 0000000000..6c7af7ed8f
--- /dev/null
+++ b/render-test/android/app/src/main/AndroidManifest.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+          package="com.example.native_activity"
+          android:versionCode="1"
+          android:versionName="1.0">
+  <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+  <application
+      android:allowBackup="false"
+      android:fullBackupContent="false"
+      android:icon="@mipmap/ic_launcher"
+      android:label="@string/app_name"
+      android:hasCode="false">
+
+    <activity android:name="android.app.NativeActivity"
+              android:label="@string/app_name"
+              android:screenOrientation="portrait">
+      <meta-data android:name="android.app.lib_name"
+                 android:value="mbgl-render-test-runner" />
+      <intent-filter>
+        <action android:name="android.intent.action.MAIN" />
+        <category android:name="android.intent.category.LAUNCHER" />
+      </intent-filter>
+    </activity>
+  </application>
+
+</manifest>
diff --git a/render-test/android/app/src/main/res/mipmap-hdpi/ic_launcher.png b/render-test/android/app/src/main/res/mipmap-hdpi/ic_launcher.png
new file mode 100644
index 0000000000..cde69bccce
--- /dev/null
+++ b/render-test/android/app/src/main/res/mipmap-hdpi/ic_launcher.png
diff --git a/render-test/android/app/src/main/res/mipmap-mdpi/ic_launcher.png b/render-test/android/app/src/main/res/mipmap-mdpi/ic_launcher.png
new file mode 100644
index 0000000000..c133a0cbd3
--- /dev/null
+++ b/render-test/android/app/src/main/res/mipmap-mdpi/ic_launcher.png
diff --git a/render-test/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png b/render-test/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png
new file mode 100644
index 0000000000..bfa42f0e7b
--- /dev/null
+++ b/render-test/android/app/src/main/res/mipmap-xhdpi/ic_launcher.png
diff --git a/render-test/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png b/render-test/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png
new file mode 100644
index 0000000000..324e72cdd7
--- /dev/null
+++ b/render-test/android/app/src/main/res/mipmap-xxhdpi/ic_launcher.png
diff --git a/render-test/android/app/src/main/res/values/strings.xml b/render-test/android/app/src/main/res/values/strings.xml
new file mode 100644
index 0000000000..00b181ca34
--- /dev/null
+++ b/render-test/android/app/src/main/res/values/strings.xml
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <string name="app_name">RenderTestRunner</string>
+</resources>
diff --git a/render-test/android/build.gradle b/render-test/android/build.gradle
new file mode 100644
index 0000000000..e58b831f71
--- /dev/null
+++ b/render-test/android/build.gradle
@@ -0,0 +1,16 @@
+buildscript {
+    repositories {
+       google()
+       jcenter()
+    }
+    dependencies {
+        classpath 'com.android.tools.build:gradle:3.5.1'
+    }
+}
+
+allprojects {
+    repositories {
+        google()
+        jcenter()
+    }
+}
diff --git a/render-test/android/gradle.properties b/render-test/android/gradle.properties
new file mode 100644
index 0000000000..2427fd0bf4
--- /dev/null
+++ b/render-test/android/gradle.properties
@@ -0,0 +1 @@
+org.gradle.jvmargs=-Xmx1536m
+\ No newline at end of file
diff --git a/render-test/android/gradle/wrapper/gradle-wrapper.jar b/render-test/android/gradle/wrapper/gradle-wrapper.jar
new file mode 100644
index 0000000000..8c0fb64a86
--- /dev/null
+++ b/render-test/android/gradle/wrapper/gradle-wrapper.jar
diff --git a/render-test/android/gradle/wrapper/gradle-wrapper.properties b/render-test/android/gradle/wrapper/gradle-wrapper.properties
new file mode 100644
index 0000000000..2e4993ee89
--- /dev/null
+++ b/render-test/android/gradle/wrapper/gradle-wrapper.properties
@@ -0,0 +1,6 @@
+#Mon Oct 21 12:45:47 EEST 2019
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-all.zip
diff --git a/render-test/android/gradlew b/render-test/android/gradlew
new file mode 100755
index 0000000000..91a7e269e1
--- /dev/null
+++ b/render-test/android/gradlew
@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn ( ) {
+    echo "$*"
+}
+
+die ( ) {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+esac
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched.
+if $cygwin ; then
+    [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
+fi
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >&-
+APP_HOME="`pwd -P`"
+cd "$SAVED" >&-
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=$((i+1))
+    done
+    case $i in
+        (0) set -- ;;
+        (1) set -- "$args0" ;;
+        (2) set -- "$args0" "$args1" ;;
+        (3) set -- "$args0" "$args1" "$args2" ;;
+        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
+function splitJvmOpts() {
+    JVM_OPTS=("$@")
+}
+eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
+JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+
+exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
diff --git a/render-test/android/gradlew.bat b/render-test/android/gradlew.bat
new file mode 100644
index 0000000000..aec99730b4
--- /dev/null
+++ b/render-test/android/gradlew.bat
@@ -0,0 +1,90 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windowz variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+if "%@eval[2+2]" == "4" goto 4NT_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+goto execute
+
+:4NT_args
+@rem Get arguments from the 4NT Shell from JP Software
+set CMD_LINE_ARGS=%$
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega
diff --git a/render-test/android/render_test_setup.sh b/render-test/android/render_test_setup.sh
new file mode 100755
index 0000000000..1dea44399e
--- /dev/null
+++ b/render-test/android/render_test_setup.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+adb shell rm -rf /sdcard/render-test
+adb shell mkdir /sdcard/render-test
+adb shell mkdir /sdcard/render-test/vendor
+adb shell mkdir /sdcard/render-test/ignores
+adb shell mkdir /sdcard/render-test/render-test/tests
+
+# push test sources
+adb push ../../mapbox-gl-js/test/integration/render-tests /sdcard/render-test/mapbox-gl-js/test/integration/render-tests
+adb push ../../mapbox-gl-js/test/integration/query-tests /sdcard/render-test/mapbox-gl-js/test/integration/query-tests 
+adb push ../../mapbox-gl-js/test/integration/tiles /sdcard/render-test/mapbox-gl-js/test/integration/tiles
+adb push ../../mapbox-gl-js/test/integration/glyphs /sdcard/render-test/mapbox-gl-js/test/integration/glyphs
+adb push ../../mapbox-gl-js/test/integration/styles /sdcard/render-test/mapbox-gl-js/test/integration/styles 
+adb push ../../mapbox-gl-js/test/integration/tilesets /sdcard/render-test/mapbox-gl-js/test/integration/tilesets 
+adb push ../../mapbox-gl-js/test/integration/image /sdcard/render-test/mapbox-gl-js/test/integration/image 
+adb push ../../mapbox-gl-js/test/integration/video /sdcard/render-test/mapbox-gl-js/test/integration/video 
+adb push ../../vendor/mapbox-gl-styles/styles /sdcard/render-test/vendor/mapbox-gl-styles/styles 
+adb push ../../vendor/mapbox-gl-styles/sprites /sdcard/render-test/vendor/mapbox-gl-styles/sprites 
+adb push ../../mapbox-gl-js/test/integration/data /sdcard/render-test/mapbox-gl-js/test/integration/data 
+adb push ../../mapbox-gl-js/test/integration/geojson /sdcard/render-test/mapbox-gl-js/test/integration/geojson
+mkdir sprites
+cp -r ../../mapbox-gl-js/test/integration/sprites/ sprites
+adb push sprites /sdcard/render-test/mapbox-gl-js/test/integration/sprites
+rm -rf sprites
+
+# push extra expectations
+adb push ../../render-test/expected/render-tests /sdcard/render-test/render-test/expected/render-tests
+
+# push default ignore lists
+adb shell mkdir /sdcard/render-test/platform
+adb shell mkdir /sdcard/render-test/platform/node
+adb shell mkdir /sdcard/render-test/platform/node/test
+adb push ../../platform/node/test/ignores.json /sdcard/render-test/platform/node/test
+adb shell mkdir /sdcard/render-test/render-test
+adb push ../linux-ignores.json /sdcard/render-test/render-test
+adb push ../tests/should-fail.json /sdcard/render-test/render-test/tests
+
+# push manifest
+adb push ../android-manifest.json /sdcard/render-test
+
+adb shell ls /sdcard/render-test/
+\ No newline at end of file
diff --git a/render-test/android/settings.gradle b/render-test/android/settings.gradle
new file mode 100644
index 0000000000..573abcb323
--- /dev/null
+++ b/render-test/android/settings.gradle
@@ -0,0 +1,2 @@
+include ':app'
+
diff --git a/render-test/expected/debug/collision-lines-overscaled/expected.png b/render-test/expected/render-tests/debug/collision-lines-overscaled/expected.png
index 38eb0d2da6..38eb0d2da6 100644
--- a/render-test/expected/debug/collision-lines-overscaled/expected.png
+++ b/render-test/expected/render-tests/debug/collision-lines-overscaled/expected.png
diff --git a/render-test/expected/debug/collision-lines-pitched/expected.png b/render-test/expected/render-tests/debug/collision-lines-pitched/expected.png
index 416d7d5715..416d7d5715 100644
--- a/render-test/expected/debug/collision-lines-pitched/expected.png
+++ b/render-test/expected/render-tests/debug/collision-lines-pitched/expected.png
diff --git a/render-test/expected/debug/collision-lines/expected.png b/render-test/expected/render-tests/debug/collision-lines/expected.png
index 3f4790a585..3f4790a585 100644
--- a/render-test/expected/debug/collision-lines/expected.png
+++ b/render-test/expected/render-tests/debug/collision-lines/expected.png
diff --git a/render-test/expected/debug/collision-pitched-wrapped/expected.png b/render-test/expected/render-tests/debug/collision-pitched-wrapped/expected.png
index 9b718c09c0..9b718c09c0 100644
--- a/render-test/expected/debug/collision-pitched-wrapped/expected.png
+++ b/render-test/expected/render-tests/debug/collision-pitched-wrapped/expected.png
diff --git a/render-test/expected/symbol-visibility/visible/expected.png b/render-test/expected/render-tests/symbol-visibility/visible/expected.png
index 8da157772a..8da157772a 100644
--- a/render-test/expected/symbol-visibility/visible/expected.png
+++ b/render-test/expected/render-tests/symbol-visibility/visible/expected.png
diff --git a/render-test/expected/text-pitch-alignment/auto-text-rotation-alignment-map/expected.png b/render-test/expected/render-tests/text-pitch-alignment/auto-text-rotation-alignment-map/expected.png
index cd690ca152..cd690ca152 100644
--- a/render-test/expected/text-pitch-alignment/auto-text-rotation-alignment-map/expected.png
+++ b/render-test/expected/render-tests/text-pitch-alignment/auto-text-rotation-alignment-map/expected.png
diff --git a/render-test/expected/text-pitch-alignment/map-text-rotation-alignment-map/expected.png b/render-test/expected/render-tests/text-pitch-alignment/map-text-rotation-alignment-map/expected.png
index cd690ca152..cd690ca152 100644
--- a/render-test/expected/text-pitch-alignment/map-text-rotation-alignment-map/expected.png
+++ b/render-test/expected/render-tests/text-pitch-alignment/map-text-rotation-alignment-map/expected.png
diff --git a/render-test/expected/text-pitch-alignment/viewport-text-rotation-alignment-map/expected.png b/render-test/expected/render-tests/text-pitch-alignment/viewport-text-rotation-alignment-map/expected.png
index 764d4a0b24..764d4a0b24 100644
--- a/render-test/expected/text-pitch-alignment/viewport-text-rotation-alignment-map/expected.png
+++ b/render-test/expected/render-tests/text-pitch-alignment/viewport-text-rotation-alignment-map/expected.png
diff --git a/render-test/expected/text-variable-anchor/pitched-rotated-debug/expected.png b/render-test/expected/render-tests/text-variable-anchor/pitched-rotated-debug/expected.png
index 4e3d012844..4e3d012844 100644
--- a/render-test/expected/text-variable-anchor/pitched-rotated-debug/expected.png
+++ b/render-test/expected/render-tests/text-variable-anchor/pitched-rotated-debug/expected.png
diff --git a/render-test/expected/text-variable-anchor/rotated-offset/expected.png b/render-test/expected/render-tests/text-variable-anchor/rotated-offset/expected.png
index 13690d147c..13690d147c 100644
--- a/render-test/expected/text-variable-anchor/rotated-offset/expected.png
+++ b/render-test/expected/render-tests/text-variable-anchor/rotated-offset/expected.png
diff --git a/render-test/file_source.cpp b/render-test/file_source.cpp
new file mode 100644
index 0000000000..0968f1d2f0
--- /dev/null
+++ b/render-test/file_source.cpp
@@ -0,0 +1,65 @@
+#include <mbgl/storage/resource_options.hpp>
+
+#include "file_source.hpp"
+
+namespace mbgl {
+
+std::atomic_size_t requestCount{0};
+std::atomic_size_t transferredSize{0};
+std::atomic_bool active{false};
+
+ProxyFileSource::ProxyFileSource(const std::string& cachePath,
+                                 const std::string& assetPath,
+                                 bool supportCacheOnlyRequests_)
+    : DefaultFileSource(cachePath, assetPath, supportCacheOnlyRequests_) {}
+
+ProxyFileSource::ProxyFileSource(const std::string& cachePath,
+                                 std::unique_ptr<FileSource>&& assetFileSource_,
+                                 bool supportCacheOnlyRequests_)
+    : DefaultFileSource(cachePath, std::move(assetFileSource_), supportCacheOnlyRequests_) {}
+
+ProxyFileSource::~ProxyFileSource() = default;
+
+std::unique_ptr<AsyncRequest> ProxyFileSource::request(const Resource& resource, Callback callback) {
+    auto result = DefaultFileSource::request(resource, [=](Response response) {
+        std::size_t size = response.data != nullptr ? response.data->size() : 0;
+        if (active) {
+            requestCount++;
+            transferredSize += size;
+        }
+        callback(response);
+    });
+    return result;
+}
+
+std::shared_ptr<FileSource> FileSource::createPlatformFileSource(const ResourceOptions& options) {
+    auto fileSource = std::make_shared<ProxyFileSource>(
+        options.cachePath(), options.assetPath(), options.supportsCacheOnlyRequests());
+    fileSource->setAccessToken(options.accessToken());
+    fileSource->setAPIBaseURL(options.baseURL());
+    return fileSource;
+}
+
+// static
+void ProxyFileSource::setTrackingActive(bool active_) {
+    active = active_;
+    requestCount = 0;
+    transferredSize = 0;
+}
+
+// static
+bool ProxyFileSource::isTrackingActive() {
+    return active;
+}
+
+// static
+size_t ProxyFileSource::getRequestCount() {
+    return requestCount;
+}
+
+// static
+size_t ProxyFileSource::getTransferredSize() {
+    return transferredSize;
+}
+
+} // namespace mbgl
diff --git a/render-test/file_source.hpp b/render-test/file_source.hpp
new file mode 100644
index 0000000000..58acf7b6ad
--- /dev/null
+++ b/render-test/file_source.hpp
@@ -0,0 +1,40 @@
+#pragma once
+
+#include <mbgl/storage/default_file_source.hpp>
+
+namespace mbgl {
+
+class ProxyFileSource : public DefaultFileSource {
+public:
+    ProxyFileSource(const std::string& cachePath, const std::string& assetPath, bool supportCacheOnlyRequests = true);
+    ProxyFileSource(const std::string& cachePath,
+                    std::unique_ptr<FileSource>&& assetFileSource,
+                    bool supportCacheOnlyRequests = true);
+    ~ProxyFileSource();
+
+    std::unique_ptr<AsyncRequest> request(const Resource&, Callback);
+
+    /**
+     * @brief Starts/stops metrics tracking.
+     */
+    static void setTrackingActive(bool);
+    /**
+     * @brief Returns metrics tracking status.
+     */
+    static bool isTrackingActive();
+    /**
+     * @brief Returns the total amount of requests.
+     *
+     * @return size_t
+     */
+    static size_t getRequestCount();
+
+    /**
+     * @brief Returns the size of transferred data (in bytes).
+     *
+     * @return size_t
+     */
+    static size_t getTransferredSize();
+};
+
+} // namespace mbgl
diff --git a/render-test/include/mbgl/render_test.hpp b/render-test/include/mbgl/render_test.hpp
new file mode 100644
index 0000000000..8a82079bee
--- /dev/null
+++ b/render-test/include/mbgl/render_test.hpp
@@ -0,0 +1,6 @@
+#pragma once
+namespace mbgl {
+
+int runRenderTests(int argc, char* argv[]);
+
+} // namespace mbgl
diff --git a/render-test/linux-manifest.json b/render-test/linux-manifest.json
new file mode 100644
index 0000000000..32a5afdbdb
--- /dev/null
+++ b/render-test/linux-manifest.json
@@ -0,0 +1,7 @@
+{
+    "base_test_path":"../mapbox-gl-js/test/integration",
+    "expectation_paths":["expected/render-tests"],
+    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json", "../render-test/tests/should-fail.json"],
+    "vendor_path":"../vendor",
+    "asset_path": "../mapbox-gl-js/test/integration"
+}
+\ No newline at end of file
diff --git a/render-test/linux-probe-manifest.json b/render-test/linux-probe-manifest.json
new file mode 100644
index 0000000000..0b2ca88988
--- /dev/null
+++ b/render-test/linux-probe-manifest.json
@@ -0,0 +1,7 @@
+{
+    "probe_test_path":".",
+    "expectation_paths":["expected/render-tests"],
+    "ignore_paths":["../render-test/linux-ignores.json", "../render-test/tests/should-fail.json"],
+    "vendor_path":"../vendor",
+    "asset_path": "../mapbox-gl-js/test/integration"
+}
+\ No newline at end of file
diff --git a/render-test/mac-manifest.json b/render-test/mac-manifest.json
new file mode 100644
index 0000000000..224df81298
--- /dev/null
+++ b/render-test/mac-manifest.json
@@ -0,0 +1,7 @@
+{
+    "base_test_path":"../mapbox-gl-js/test/integration",
+    "expectation_paths":["expected/render-tests", "tests/mac"],
+    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/mac-ignores.json", "../render-test/tests/should-fail.json"],
+    "vendor_path":"../vendor",
+    "asset_path": "../mapbox-gl-js/test/integration"
+}
+\ No newline at end of file
diff --git a/render-test/mac-probe-manifest.json b/render-test/mac-probe-manifest.json
new file mode 100644
index 0000000000..f3cc56d0a8
--- /dev/null
+++ b/render-test/mac-probe-manifest.json
@@ -0,0 +1,7 @@
+{
+    "probe_test_path":".",
+    "expectation_paths":["expected/render-tests", "tests/mac"],
+    "ignore_paths":["../render-test/mac-ignores.json", "../render-test/tests/should-fail.json"],
+    "vendor_path":"../vendor",
+    "asset_path": "../mapbox-gl-js/test/integration"
+}
+\ No newline at end of file
diff --git a/render-test/manifest_parser.cpp b/render-test/manifest_parser.cpp
new file mode 100644
index 0000000000..f1884634e5
--- /dev/null
+++ b/render-test/manifest_parser.cpp
@@ -0,0 +1,400 @@
+#include "manifest_parser.hpp"
+#include "filesystem.hpp"
+#include "parser.hpp"
+
+#include <mbgl/util/logging.hpp>
+
+#include <algorithm>
+#include <random>
+
+namespace {
+std::string removeURLArguments(const std::string& url) {
+    std::string::size_type index = url.find('?');
+    if (index != std::string::npos) {
+        return url.substr(0, index);
+    }
+    return url;
+}
+
+std::string prependFileScheme(const std::string& url) {
+    static const std::string fileScheme("file://");
+    return fileScheme + url;
+}
+} // namespace
+
+Manifest::Manifest() = default;
+Manifest::~Manifest() = default;
+
+const std::vector<TestPaths>& Manifest::getTestPaths() const {
+    return testPaths;
+}
+const std::vector<std::pair<std::string, std::string>>& Manifest::getIgnores() const {
+    return ignores;
+}
+const std::string& Manifest::getTestRootPath() const {
+    return testRootPath;
+}
+const std::string& Manifest::getManifestPath() const {
+    return manifestPath;
+}
+
+void Manifest::doShuffle(uint32_t seed) {
+    std::seed_seq sequence{seed};
+    std::mt19937 shuffler(sequence);
+    std::shuffle(testPaths.begin(), testPaths.end(), shuffler);
+}
+
+std::string Manifest::localizeURL(const std::string& url) const {
+    static const std::regex regex{"local://"};
+    if (auto vendorPath = getVendorPath(url, regex)) {
+        return *vendorPath;
+    }
+    return getIntegrationPath(url, "", regex).value_or(url);
+}
+
+void Manifest::localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const {
+    if (root.HasMember("urls") && root["urls"].IsArray()) {
+        for (auto& urlValue : root["urls"].GetArray()) {
+            const std::string path =
+                prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
+                                      .value_or(localizeLocalURL(urlValue.GetString()).value_or(urlValue.GetString())));
+            urlValue.Set<std::string>(path, document.GetAllocator());
+        }
+    }
+
+    if (root.HasMember("url")) {
+        static const std::string image("image");
+        static const std::string video("video");
+
+        mbgl::JSValue& urlValue = root["url"];
+        const std::string path =
+            prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
+                                  .value_or(localizeLocalURL(urlValue.GetString()).value_or(urlValue.GetString())));
+        urlValue.Set<std::string>(path, document.GetAllocator());
+
+        if (root["type"].GetString() != image && root["type"].GetString() != video) {
+            const auto tilesetPath = std::string(urlValue.GetString()).erase(0u, 7u); // remove "file://"
+            auto maybeTileset = readJson(tilesetPath);
+            if (maybeTileset.is<mbgl::JSDocument>()) {
+                const auto& tileset = maybeTileset.get<mbgl::JSDocument>();
+                assert(tileset.HasMember("tiles"));
+                root.AddMember("tiles", (mbgl::JSValue&)tileset["tiles"], document.GetAllocator());
+                root.RemoveMember("url");
+            }
+        }
+    }
+
+    if (root.HasMember("tiles")) {
+        mbgl::JSValue& tilesValue = root["tiles"];
+        assert(tilesValue.IsArray());
+        for (auto& tileValue : tilesValue.GetArray()) {
+            const std::string path = prependFileScheme(
+                localizeMapboxTilesURL(tileValue.GetString())
+                    .value_or(localizeLocalURL(tileValue.GetString())
+                                  .value_or(localizeHttpURL(tileValue.GetString()).value_or(tileValue.GetString()))));
+            tileValue.Set<std::string>(path, document.GetAllocator());
+        }
+    }
+
+    if (root.HasMember("data") && root["data"].IsString()) {
+        mbgl::JSValue& dataValue = root["data"];
+        const std::string path =
+            prependFileScheme(localizeLocalURL(dataValue.GetString()).value_or(dataValue.GetString()));
+        dataValue.Set<std::string>(path, document.GetAllocator());
+    }
+}
+
+void Manifest::localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const {
+    if (root.HasMember("sources")) {
+        mbgl::JSValue& sourcesValue = root["sources"];
+        for (auto& sourceProperty : sourcesValue.GetObject()) {
+            localizeSourceURLs(sourceProperty.value, document);
+        }
+    }
+
+    if (root.HasMember("glyphs")) {
+        mbgl::JSValue& glyphsValue = root["glyphs"];
+        const std::string path = prependFileScheme(
+            localizeMapboxFontsURL(glyphsValue.GetString())
+                .value_or(localizeLocalURL(glyphsValue.GetString(), true).value_or(glyphsValue.GetString())));
+        glyphsValue.Set<std::string>(path, document.GetAllocator());
+    }
+
+    if (root.HasMember("sprite")) {
+        mbgl::JSValue& spriteValue = root["sprite"];
+        const std::string path = prependFileScheme(
+            localizeMapboxSpriteURL(spriteValue.GetString())
+                .value_or(localizeLocalURL(spriteValue.GetString()).value_or(spriteValue.GetString())));
+        spriteValue.Set<std::string>(path, document.GetAllocator());
+    }
+}
+
+mbgl::optional<std::string> Manifest::localizeLocalURL(const std::string& url, bool glyphsPath) const {
+    static const std::regex regex{"local://"};
+    if (auto vendorPath = getVendorPath(url, regex, glyphsPath)) {
+        return vendorPath;
+    }
+    return getIntegrationPath(url, "", regex, glyphsPath);
+}
+
+mbgl::optional<std::string> Manifest::localizeHttpURL(const std::string& url) const {
+    static const std::regex regex{"http://localhost:2900"};
+    if (auto vendorPath = getVendorPath(url, regex)) {
+        return vendorPath;
+    }
+    return getIntegrationPath(url, "", regex);
+}
+
+mbgl::optional<std::string> Manifest::localizeMapboxSpriteURL(const std::string& url) const {
+    static const std::regex regex{"mapbox://"};
+    return getIntegrationPath(url, "", regex);
+}
+
+mbgl::optional<std::string> Manifest::localizeMapboxFontsURL(const std::string& url) const {
+    static const std::regex regex{"mapbox://fonts"};
+    return getIntegrationPath(url, "glyphs/", regex, true);
+}
+
+mbgl::optional<std::string> Manifest::localizeMapboxTilesURL(const std::string& url) const {
+    static const std::regex regex{"mapbox://"};
+    if (auto vendorPath = getVendorPath(url, regex)) {
+        return vendorPath;
+    }
+    return getIntegrationPath(url, "tiles/", regex);
+}
+
+mbgl::optional<std::string> Manifest::localizeMapboxTilesetURL(const std::string& url) const {
+    static const std::regex regex{"mapbox://"};
+    return getIntegrationPath(url, "tilesets/", regex);
+}
+
+mbgl::optional<std::string> Manifest::getVendorPath(const std::string& url,
+                                                    const std::regex& regex,
+                                                    bool glyphsPath) const {
+    mbgl::filesystem::path file = std::regex_replace(url, regex, vendorPath);
+    if (mbgl::filesystem::exists(file.parent_path())) {
+        return removeURLArguments(file.string());
+    }
+
+    if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
+        return removeURLArguments(file.string());
+    }
+
+    return mbgl::nullopt;
+}
+
+mbgl::optional<std::string> Manifest::getIntegrationPath(const std::string& url,
+                                                         const std::string& parent,
+                                                         const std::regex& regex,
+                                                         bool glyphsPath) const {
+    mbgl::filesystem::path file = std::regex_replace(url, regex, assetPath + parent);
+    if (mbgl::filesystem::exists(file.parent_path())) {
+        return removeURLArguments(file.string());
+    }
+
+    if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
+        return removeURLArguments(file.string());
+    }
+
+    return mbgl::nullopt;
+}
+
+namespace {
+std::vector<std::pair<std::string, std::string>> parseIgnores(const std::vector<mbgl::filesystem::path>& ignoresPaths) {
+    std::vector<std::pair<std::string, std::string>> ignores;
+    for (const auto& path : ignoresPaths) {
+        auto maybeIgnores = readJson(path);
+        if (!maybeIgnores.is<mbgl::JSDocument>()) {
+            continue;
+        }
+        for (const auto& property : maybeIgnores.get<mbgl::JSDocument>().GetObject()) {
+            const std::string ignore = {property.name.GetString(), property.name.GetStringLength()};
+            const std::string reason = {property.value.GetString(), property.value.GetStringLength()};
+            ignores.emplace_back(std::make_pair(ignore, reason));
+        }
+    }
+
+    return ignores;
+}
+
+std::vector<mbgl::filesystem::path> getTestExpectations(mbgl::filesystem::path testPath,
+                                                        const mbgl::filesystem::path& testsRootPath,
+                                                        std::vector<mbgl::filesystem::path> expectationsPaths) {
+    std::vector<mbgl::filesystem::path> expectations{std::move(testPath.remove_filename())};
+    const auto& defaultTestExpectationsPath = expectations.front().string();
+
+    const std::regex regex{testsRootPath.string()};
+    for (const auto& path : expectationsPaths) {
+        expectations.emplace_back(std::regex_replace(defaultTestExpectationsPath, regex, path.string()));
+        assert(!expectations.back().empty());
+    }
+
+    return expectations;
+}
+
+mbgl::filesystem::path getValidPath(const std::string& manifestPath, const std::string& path) {
+    const static mbgl::filesystem::path BasePath{manifestPath};
+    mbgl::filesystem::path result{path};
+    if (result.is_relative()) {
+        result = BasePath / result;
+    }
+    if (mbgl::filesystem::exists(result)) {
+        return result;
+    }
+    mbgl::Log::Warning(mbgl::Event::General, "Invalid path is provoided inside the manifest file: %s", path.c_str());
+    return mbgl::filesystem::path{};
+}
+
+} // namespace
+
+mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manifestPath,
+                                                       const std::vector<std::string>& testNames,
+                                                       const std::string& testFilter) {
+    Manifest manifest;
+    const auto filePath = mbgl::filesystem::path(manifestPath);
+    manifest.manifestPath = manifestPath.substr(0, manifestPath.find(filePath.filename()));
+
+    auto contents = readJson(filePath);
+    if (!contents.is<mbgl::JSDocument>()) {
+        mbgl::Log::Error(mbgl::Event::General, "Provided manifest file: %s is not a valid json", filePath.c_str());
+        return mbgl::nullopt;
+    }
+
+    auto document = std::move(contents.get<mbgl::JSDocument>());
+    if (document.HasMember("asset_path")) {
+        const auto& assetPathValue = document["asset_path"];
+        if (!assetPathValue.IsString()) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Invalid assetPath is provoided inside the manifest file: %s", filePath.c_str());
+            return mbgl::nullopt;
+        }
+        manifest.assetPath = (getValidPath(manifest.manifestPath, assetPathValue.GetString()) / "").string();
+        if (manifest.assetPath.empty()) {
+            return mbgl::nullopt;
+        }
+    }
+    if (document.HasMember("vendor_path")) {
+        const auto& vendorPathValue = document["vendor_path"];
+        if (!vendorPathValue.IsString()) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Invalid vendorPath is provoided inside the manifest file: %s", filePath.c_str());
+            return mbgl::nullopt;
+        }
+        manifest.vendorPath = (getValidPath(manifest.manifestPath, vendorPathValue.GetString()) / "").string();
+        if (manifest.vendorPath.empty()) {
+            return mbgl::nullopt;
+        }
+    }
+    mbgl::filesystem::path baseTestPath;
+    if (document.HasMember("base_test_path")) {
+        const auto& testPathValue = document["base_test_path"];
+        if (!testPathValue.IsString()) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Invalid testPath is provoided inside the manifest file: %s", filePath.c_str());
+            return mbgl::nullopt;
+        }
+        baseTestPath = getValidPath(manifest.manifestPath, testPathValue.GetString());
+        if (baseTestPath.empty()) {
+            return mbgl::nullopt;
+        }
+    }
+    mbgl::filesystem::path probeTestPath;
+    bool enbaleProbeTest{false};
+    if (document.HasMember("probe_test_path")) {
+        const auto& testPathValue = document["probe_test_path"];
+        if (!testPathValue.IsString()) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Invalid testPath is provoided inside the manifest file: %s", filePath.c_str());
+            return mbgl::nullopt;
+        }
+        probeTestPath = getValidPath(manifest.manifestPath, testPathValue.GetString());
+        if (probeTestPath.empty()) {
+            return mbgl::nullopt;
+        }
+        enbaleProbeTest = true;
+    }
+    std::vector<mbgl::filesystem::path> expectationPaths{};
+    if (document.HasMember("expectation_paths")) {
+        const auto& expectationPathValue = document["expectation_paths"];
+        if (!expectationPathValue.IsArray()) {
+            mbgl::Log::Warning(mbgl::Event::General,
+                               "Provided expectation_paths inside the manifest file: %s is not a valid array",
+                               filePath.c_str());
+            return mbgl::nullopt;
+        }
+        for (const auto& value : expectationPathValue.GetArray()) {
+            if (!value.IsString()) {
+                mbgl::Log::Warning(mbgl::Event::General,
+                                   "Invalid expectation path item is provoided inside the manifest file: %s",
+                                   filePath.c_str());
+                return mbgl::nullopt;
+            }
+            expectationPaths.emplace_back(getValidPath(manifest.manifestPath, value.GetString()));
+            if (expectationPaths.back().empty()) {
+                return mbgl::nullopt;
+            }
+        }
+    }
+    std::vector<mbgl::filesystem::path> ignorePaths{};
+    if (document.HasMember("ignore_paths")) {
+        const auto& ignorePathValue = document["ignore_paths"];
+        if (!ignorePathValue.IsArray()) {
+            mbgl::Log::Warning(mbgl::Event::General,
+                               "Provided ignore_paths inside the manifest file: %s is not a valid array",
+                               filePath.c_str());
+            return mbgl::nullopt;
+        }
+        for (const auto& value : ignorePathValue.GetArray()) {
+            if (!value.IsString()) {
+                mbgl::Log::Warning(mbgl::Event::General,
+                                   "Invalid ignore path item is provoided inside the manifest file: %s",
+                                   filePath.c_str());
+                return mbgl::nullopt;
+            }
+            ignorePaths.emplace_back(getValidPath(manifest.manifestPath, value.GetString()));
+            if (ignorePaths.back().empty()) {
+                return mbgl::nullopt;
+            }
+        }
+        manifest.ignores = parseIgnores(ignorePaths);
+    }
+
+    manifest.testRootPath = enbaleProbeTest ? probeTestPath.string() : baseTestPath.string();
+    if (manifest.testRootPath.back() == '/') {
+        manifest.testRootPath.pop_back();
+    }
+    if (manifest.manifestPath.back() == '/') {
+        manifest.manifestPath.pop_back();
+    }
+
+    std::vector<mbgl::filesystem::path> paths;
+    for (const auto& id : testNames) {
+        paths.emplace_back(manifest.testRootPath + "/" + id);
+    }
+    if (paths.empty()) {
+        paths.emplace_back(manifest.testRootPath);
+    }
+
+    // Recursively traverse through the test paths and collect test directories containing "style.json".
+    auto& testPaths = manifest.testPaths;
+    testPaths.reserve(paths.size());
+    for (const auto& path : paths) {
+        if (!mbgl::filesystem::exists(path)) {
+            mbgl::Log::Warning(
+                mbgl::Event::General, "Provided test folder '%s' does not exist.", path.string().c_str());
+            continue;
+        }
+        for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(path)) {
+            // Skip paths that fail regexp match.
+            if (!testFilter.empty() && !std::regex_match(testPath.path().string(), std::regex(testFilter))) {
+                continue;
+            }
+
+            if (testPath.path().filename() == "style.json") {
+                testPaths.emplace_back(testPath, getTestExpectations(testPath, path, expectationPaths));
+            }
+        }
+    }
+
+    return mbgl::optional<Manifest>(manifest);
+}
diff --git a/render-test/manifest_parser.hpp b/render-test/manifest_parser.hpp
new file mode 100644
index 0000000000..bc5adf1091
--- /dev/null
+++ b/render-test/manifest_parser.hpp
@@ -0,0 +1,55 @@
+#pragma once
+
+#include "metadata.hpp"
+
+#include <mbgl/util/optional.hpp>
+#include <mbgl/util/rapidjson.hpp>
+
+#include <regex>
+#include <string>
+#include <utility>
+#include <vector>
+
+class Manifest {
+public:
+    Manifest();
+    ~Manifest();
+    const std::vector<std::pair<std::string, std::string>>& getIgnores() const;
+    const std::vector<TestPaths>& getTestPaths() const;
+    const std::string& getTestRootPath() const;
+    const std::string& getManifestPath() const;
+    void doShuffle(uint32_t seed);
+
+    std::string localizeURL(const std::string& url) const;
+    void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const;
+    void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const;
+
+private:
+    friend class ManifestParser;
+    mbgl::optional<std::string> localizeLocalURL(const std::string& url, bool glyphsPath = false) const;
+    mbgl::optional<std::string> localizeHttpURL(const std::string& url) const;
+    mbgl::optional<std::string> localizeMapboxSpriteURL(const std::string& url) const;
+    mbgl::optional<std::string> localizeMapboxFontsURL(const std::string& url) const;
+    mbgl::optional<std::string> localizeMapboxTilesURL(const std::string& url) const;
+    mbgl::optional<std::string> localizeMapboxTilesetURL(const std::string& url) const;
+    mbgl::optional<std::string> getVendorPath(const std::string& url,
+                                              const std::regex& regex,
+                                              bool glyphsPath = false) const;
+    mbgl::optional<std::string> getIntegrationPath(const std::string& url,
+                                                   const std::string& parent,
+                                                   const std::regex& regex,
+                                                   bool glyphsPath = false) const;
+    std::string manifestPath;
+    std::string testRootPath;
+    std::string vendorPath;
+    std::string assetPath;
+    std::vector<std::pair<std::string, std::string>> ignores;
+    std::vector<TestPaths> testPaths;
+};
+
+class ManifestParser {
+public:
+    static mbgl::optional<Manifest> parseManifest(const std::string& manifestPath,
+                                                  const std::vector<std::string>& testNames,
+                                                  const std::string& testFilter);
+};
diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp
index d23a0fb296..567c89e3fc 100644
--- a/render-test/metadata.hpp
+++ b/render-test/metadata.hpp
@@ -1,14 +1,22 @@
 #pragma once
 
+#include <mbgl/util/geo.hpp>
 #include <mbgl/util/rapidjson.hpp>
 #include <mbgl/util/size.hpp>
 
 #include <mbgl/map/mode.hpp>
+#include <mbgl/renderer/query.hpp>
 
 #include "filesystem.hpp"
 
 #include <map>
 
+namespace mbgl {
+namespace gfx {
+struct RenderingStats;
+}
+} // namespace mbgl
+
 struct TestStatistics {
     TestStatistics() = default;
 
@@ -20,6 +28,10 @@ struct TestStatistics {
 };
 
 struct TestPaths {
+    TestPaths() = default;
+    TestPaths(mbgl::filesystem::path stylePath_, std::vector<mbgl::filesystem::path> expectations_)
+        : stylePath(std::move(stylePath_)), expectations(std::move(expectations_)) {}
+
     mbgl::filesystem::path stylePath;
     std::vector<mbgl::filesystem::path> expectations;
 
@@ -29,20 +41,83 @@ struct TestPaths {
     }
 };
 
+inline std::tuple<bool, float> checkValue(float expected, float actual, float tolerance) {
+    float delta = expected * tolerance;
+    assert(delta >= 0.0f);
+    return std::make_tuple(std::abs(expected - actual) <= delta, delta);
+}
+
+struct FileSizeProbe {
+    FileSizeProbe() = default;
+    FileSizeProbe(std::string path_, uintmax_t size_, float tolerance_)
+        : path(std::move(path_)), size(size_), tolerance(tolerance_) {}
+
+    std::string path;
+    uintmax_t size;
+    float tolerance;
+};
+
 struct MemoryProbe {
     MemoryProbe() = default;
-    MemoryProbe(size_t peak_, size_t allocations_)
-        : peak(peak_)
-        , allocations(allocations_) {}
+    MemoryProbe(size_t peak_, size_t allocations_) : peak(peak_), allocations(allocations_), tolerance(0.0f) {}
 
     size_t peak;
     size_t allocations;
+    float tolerance;
+
+    static std::tuple<bool, float> checkPeak(const MemoryProbe& expected, const MemoryProbe& actual) {
+        return checkValue(expected.peak, actual.peak, actual.tolerance);
+    }
+
+    static std::tuple<bool, float> checkAllocations(const MemoryProbe& expected, const MemoryProbe& actual) {
+        return checkValue(expected.allocations, actual.allocations, actual.tolerance);
+    }
+};
+
+struct FpsProbe {
+    float average = 0.0;
+    float minOnePc = 0.0;
+    float tolerance = 0.0f;
+};
+
+struct NetworkProbe {
+    NetworkProbe() = default;
+    NetworkProbe(size_t requests_, size_t transferred_) : requests(requests_), transferred(transferred_) {}
+
+    size_t requests;
+    size_t transferred;
+};
+
+struct GfxProbe {
+    struct Memory {
+        Memory() = default;
+        Memory(int allocated_, int peak_) : allocated(allocated_), peak(peak_) {}
+
+        int allocated;
+        int peak;
+    };
+
+    GfxProbe() = default;
+    GfxProbe(const mbgl::gfx::RenderingStats&, const GfxProbe&);
+
+    int numDrawCalls;
+    int numTextures;
+    int numBuffers;
+    int numFrameBuffers;
+
+    Memory memTextures;
+    Memory memIndexBuffers;
+    Memory memVertexBuffers;
 };
 
 class TestMetrics {
 public:
-    bool isEmpty() const { return memory.empty(); }
+    bool isEmpty() const { return fileSize.empty() && memory.empty() && network.empty() && fps.empty() && gfx.empty(); }
+    std::map<std::string, FileSizeProbe> fileSize;
     std::map<std::string, MemoryProbe> memory;
+    std::map<std::string, NetworkProbe> network;
+    std::map<std::string, FpsProbe> fps;
+    std::map<std::string, GfxProbe> gfx;
 };
 
 struct TestMetadata {
@@ -50,6 +125,8 @@ struct TestMetadata {
 
     TestPaths paths;
     mbgl::JSDocument document;
+    bool renderTest = true;
+    bool outputsImage = true;
 
     mbgl::Size size{ 512u, 512u };
     float pixelRatio = 1.0f;
@@ -61,6 +138,9 @@ struct TestMetadata {
     bool axonometric = false;
     double xSkew = 0.0;
     double ySkew = 1.0;
+    mbgl::ScreenCoordinate queryGeometry{0u, 0u};
+    mbgl::ScreenBox queryGeometryBox{{0u, 0u}, {0u, 0u}};
+    mbgl::RenderedQueryOptions queryOptions;
 
     // TODO
     uint32_t fadeDuration = 0;
@@ -72,6 +152,7 @@ struct TestMetadata {
     std::string color;
 
     std::string actual;
+    std::string actualJson;
     std::string expected;
     std::string diff;
 
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index 9b462dee72..b5d48d23a1 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -3,11 +3,14 @@
 #include <mbgl/util/rapidjson.hpp>
 #include <mbgl/util/string.hpp>
 
-#include <args.hxx>
-
+#include <rapidjson/prettywriter.h>
 #include <rapidjson/stringbuffer.h>
 #include <rapidjson/writer.h>
 
+#include <mapbox/geojson_impl.hpp>
+#include <mbgl/style/conversion/filter.hpp>
+#include <mbgl/style/conversion/json.hpp>
+
 #include <boost/archive/iterators/base64_from_binary.hpp>
 #include <boost/archive/iterators/insert_linebreaks.hpp>
 #include <boost/archive/iterators/transform_width.hpp>
@@ -18,8 +21,8 @@
 #include "parser.hpp"
 #include "runner.hpp"
 
-#include <sstream>
 #include <regex>
+#include <sstream>
 
 namespace {
 
@@ -78,109 +81,73 @@ const char* resultsHeaderButtons = R"HTML(
 </h1>
 )HTML";
 
-std::string removeURLArguments(const std::string &url) {
-    std::string::size_type index = url.find('?');
-    if (index != std::string::npos) {
-        return url.substr(0, index);
-    }
-    return url;
-}
-
-std::string prependFileScheme(const std::string &url) {
-    static const std::string fileScheme("file://");
-    return fileScheme + url;
-}
-
-mbgl::optional<std::string> getVendorPath(const std::string& url, const std::regex& regex, bool glyphsPath = false) {
-    static const mbgl::filesystem::path vendorPath(std::string(TEST_RUNNER_ROOT_PATH) + "/vendor/");
-
-    mbgl::filesystem::path file = std::regex_replace(url, regex, vendorPath.string());
-    if (mbgl::filesystem::exists(file.parent_path())) {
-        return removeURLArguments(file.string());
-    }
-
-    if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
-        return removeURLArguments(file.string());
-    }
-
-    return {};
-}
-
-mbgl::optional<std::string> getIntegrationPath(const std::string& url, const std::string& parent, const std::regex& regex, bool glyphsPath = false) {
-    static const mbgl::filesystem::path integrationPath(std::string(TEST_RUNNER_ROOT_PATH) + "/mapbox-gl-js/test/integration/");
-
-    mbgl::filesystem::path file = std::regex_replace(url, regex, integrationPath.string() + parent);
-    if (mbgl::filesystem::exists(file.parent_path())) {
-        return removeURLArguments(file.string());
-    }
-
-    if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
-        return removeURLArguments(file.string());
-    }
-
-    return {};
+void writeJSON(rapidjson::PrettyWriter<rapidjson::StringBuffer>& writer, const mbgl::Value& value) {
+    value.match([&writer](const mbgl::NullValue&) { writer.Null(); },
+                [&writer](bool b) { writer.Bool(b); },
+                [&writer](uint64_t u) { writer.Uint64(u); },
+                [&writer](int64_t i) { writer.Int64(i); },
+                [&writer](double d) { d == std::floor(d) ? writer.Int64(d) : writer.Double(d); },
+                [&writer](const std::string& s) { writer.String(s); },
+                [&writer](const std::vector<mbgl::Value>& arr) {
+                    writer.StartArray();
+                    for (const auto& item : arr) {
+                        writeJSON(writer, item);
+                    }
+                    writer.EndArray();
+                },
+                [&writer](const std::unordered_map<std::string, mbgl::Value>& obj) {
+                    writer.StartObject();
+                    std::map<std::string, mbgl::Value> sorted(obj.begin(), obj.end());
+                    for (const auto& entry : sorted) {
+                        writer.Key(entry.first.c_str());
+                        writeJSON(writer, entry.second);
+                    }
+                    writer.EndObject();
+                });
 }
 
-mbgl::optional<std::string> localizeLocalURL(const std::string& url, bool glyphsPath = false) {
-    static const std::regex regex { "local://" };
-    if (auto vendorPath = getVendorPath(url, regex, glyphsPath)) {
-        return vendorPath;
-    } else {
-        return getIntegrationPath(url, "", regex, glyphsPath);
-    }
-}
+} // namespace
 
-mbgl::optional<std::string> localizeHttpURL(const std::string& url) {
-    static const std::regex regex { "http://localhost:2900" };
-    if (auto vendorPath = getVendorPath(url, regex)) {
-        return vendorPath;
-    } else {
-        return getIntegrationPath(url, "", regex);
+std::string toJSON(const mbgl::Value& value, unsigned indent, bool singleLine) {
+    rapidjson::StringBuffer buffer;
+    rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
+    if (singleLine) {
+        writer.SetFormatOptions(rapidjson::kFormatSingleLineArray);
     }
+    writer.SetIndent(' ', indent);
+    writeJSON(writer, value);
+    return buffer.GetString();
 }
 
-mbgl::optional<std::string> localizeMapboxSpriteURL(const std::string& url) {
-    static const std::regex regex { "mapbox://" };
-    return getIntegrationPath(url, "", regex);
-}
-
-mbgl::optional<std::string> localizeMapboxFontsURL(const std::string& url) {
-    static const std::regex regex { "mapbox://fonts" };
-    return getIntegrationPath(url, "glyphs/", regex, true);
-}
-
-mbgl::optional<std::string> localizeMapboxTilesURL(const std::string& url) {
-    static const std::regex regex { "mapbox://" };
-    if (auto vendorPath = getVendorPath(url, regex)) {
-        return vendorPath;
-    } else {
-        return getIntegrationPath(url, "tiles/", regex);
+std::string toJSON(const std::vector<mbgl::Feature>& features, unsigned indent, bool singleLine) {
+    rapidjson::CrtAllocator allocator;
+    rapidjson::StringBuffer buffer;
+    rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
+    if (singleLine) {
+        writer.SetFormatOptions(rapidjson::kFormatSingleLineArray);
     }
-}
-
-mbgl::optional<std::string> localizeMapboxTilesetURL(const std::string& url) {
-    static const std::regex regex { "mapbox://" };
-    return getIntegrationPath(url, "tilesets/", regex);
-}
-
-TestPaths makeTestPaths(mbgl::filesystem::path stylePath) {
-    std::vector<mbgl::filesystem::path> expectations{ stylePath };
-    expectations.front().remove_filename();
-
-    const static std::regex regex{ TestRunner::getBasePath() };
-    for (const std::string& path : TestRunner::getPlatformExpectationsPaths()) {
-        expectations.emplace_back(std::regex_replace(expectations.front().string(), regex, path));
-        assert(!expectations.back().empty());
+    writer.SetIndent(' ', indent);
+    writer.StartArray();
+    for (const auto& feature : features) {
+        mbgl::JSValue result(rapidjson::kObjectType);
+        result.AddMember("type", "Feature", allocator);
+        if (!feature.id.is<mbgl::NullValue>()) {
+            result.AddMember(
+                "id", mapbox::geojson::identifier::visit(feature.id, mapbox::geojson::to_value{allocator}), allocator);
+        }
+        result.AddMember("geometry", mapbox::geojson::convert(feature.geometry, allocator), allocator);
+        result.AddMember("properties", mapbox::geojson::to_value{allocator}(feature.properties), allocator);
+        result.AddMember("source", feature.source, allocator);
+        if (!feature.sourceLayer.empty()) {
+            result.AddMember("sourceLayer", feature.sourceLayer, allocator);
+        }
+        result.AddMember("state", mapbox::geojson::to_value{allocator}(feature.state), allocator);
+        result.Accept(writer);
     }
-
-    return {
-        std::move(stylePath),
-        std::move(expectations)
-    };
+    writer.EndArray();
+    return buffer.GetString();
 }
 
-} // namespace
-
 JSONReply readJson(const mbgl::filesystem::path& jsonPath) {
     auto maybeJSON = mbgl::util::readFile(jsonPath);
     if (!maybeJSON) {
@@ -193,7 +160,7 @@ JSONReply readJson(const mbgl::filesystem::path& jsonPath) {
         return { mbgl::formatJSONParseError(document) };
     }
 
-    return { std::move(document) };
+    return {std::move(document)};
 }
 
 std::string serializeJsonValue(const mbgl::JSValue& value) {
@@ -206,30 +173,108 @@ std::string serializeJsonValue(const mbgl::JSValue& value) {
 
 std::string serializeMetrics(const TestMetrics& metrics) {
     rapidjson::StringBuffer s;
-    rapidjson::Writer<rapidjson::StringBuffer> writer(s);
+    rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(s);
 
     writer.StartObject();
+
+    // Start file-size section.
+    if (!metrics.fileSize.empty()) {
+        writer.Key("file-size");
+        writer.StartArray();
+        for (const auto& fileSizeProbe : metrics.fileSize) {
+            assert(!fileSizeProbe.first.empty());
+            writer.StartArray();
+            writer.String(fileSizeProbe.first.c_str());
+            writer.String(fileSizeProbe.second.path);
+            writer.Uint64(fileSizeProbe.second.size);
+            writer.EndArray();
+        }
+        writer.EndArray();
+    }
+
     // Start memory section.
-    writer.Key("memory");
-    writer.StartArray();
-    for (const auto& memoryProbe : metrics.memory) {
-        assert(!memoryProbe.first.empty());       
+    if (!metrics.memory.empty()) {
+        writer.Key("memory");
         writer.StartArray();
-        writer.String(memoryProbe.first.c_str());
-        writer.Uint64(memoryProbe.second.peak);
-        writer.Uint64(memoryProbe.second.allocations);
+        for (const auto& memoryProbe : metrics.memory) {
+            assert(!memoryProbe.first.empty());
+            writer.StartArray();
+            writer.String(memoryProbe.first.c_str());
+            writer.Uint64(memoryProbe.second.peak);
+            writer.Uint64(memoryProbe.second.allocations);
+            writer.EndArray();
+        }
         writer.EndArray();
     }
-    // End memory section.
-    writer.EndArray();
+
+    // Start network section
+    if (!metrics.network.empty()) {
+        writer.Key("network");
+        writer.StartArray();
+        for (const auto& networkProbe : metrics.network) {
+            assert(!networkProbe.first.empty());
+            writer.StartArray();
+            writer.String(networkProbe.first.c_str());
+            writer.Uint64(networkProbe.second.requests);
+            writer.Uint64(networkProbe.second.transferred);
+            writer.EndArray();
+        }
+        writer.EndArray();
+    }
+
+    if (!metrics.fps.empty()) {
+        // Start fps section
+        writer.Key("fps");
+        writer.StartArray();
+        for (const auto& fpsProbe : metrics.fps) {
+            assert(!fpsProbe.first.empty());
+            writer.StartArray();
+            writer.String(fpsProbe.first.c_str());
+            writer.Double(fpsProbe.second.average);
+            writer.Double(fpsProbe.second.minOnePc);
+            writer.EndArray();
+        }
+        writer.EndArray();
+        // End fps section
+    }
+
+    if (!metrics.gfx.empty()) {
+        // Start gfx section
+        writer.Key("gfx");
+        writer.StartArray();
+        for (const auto& gfxProbe : metrics.gfx) {
+            assert(!gfxProbe.first.empty());
+            writer.StartArray();
+            writer.String(gfxProbe.first.c_str());
+            writer.Int(gfxProbe.second.numDrawCalls);
+            writer.Int(gfxProbe.second.numTextures);
+            writer.Int(gfxProbe.second.numBuffers);
+            writer.Int(gfxProbe.second.numFrameBuffers);
+            writer.StartArray();
+            writer.Int(gfxProbe.second.memTextures.allocated);
+            writer.Int(gfxProbe.second.memTextures.peak);
+            writer.EndArray();
+            writer.StartArray();
+            writer.Int(gfxProbe.second.memIndexBuffers.allocated);
+            writer.Int(gfxProbe.second.memIndexBuffers.peak);
+            writer.EndArray();
+            writer.StartArray();
+            writer.Int(gfxProbe.second.memVertexBuffers.allocated);
+            writer.Int(gfxProbe.second.memVertexBuffers.peak);
+            writer.EndArray();
+            writer.EndArray();
+        }
+        writer.EndArray();
+        // End gfx section
+    }
+
     writer.EndObject();
 
     return s.GetString();
 }
 
-std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base) {
-    static const std::regex regex(".*expected.*.png");
-
+namespace {
+std::vector<std::string> readExpectedEntries(const std::regex& regex, const mbgl::filesystem::path& base) {
     std::vector<std::string> expectedImages;
     for (const auto& entry : mbgl::filesystem::directory_iterator(base)) {
         if (entry.is_regular_file()) {
@@ -241,120 +286,16 @@ std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base)
     }
     return expectedImages;
 }
+} // namespace
 
-
-ArgumentsTuple parseArguments(int argc, char** argv) {
-    args::ArgumentParser argumentParser("Mapbox GL Test Runner");
-
-    args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", { 'h', "help" });
-
-    args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object",
-                              { 'r', "recycle-map" });
-    args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order",
-                           { 's', "shuffle" });
-    args::ValueFlag<uint32_t> seedValue(argumentParser, "seed", "Shuffle seed (default: random)",
-                                        { "seed" });
-    args::ValueFlag<std::string> testPathValue(argumentParser, "rootPath", "Test root rootPath",
-                                               { 'p', "rootPath" });
-    args::ValueFlag<std::regex> testFilterValue(argumentParser, "filter", "Test filter regex",
-                                               { 'f', "filter" });
-    args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
-
-    try {
-        argumentParser.ParseCLI(argc, argv);
-    } catch (const args::Help&) {
-        std::ostringstream stream;
-        stream << argumentParser;
-        mbgl::Log::Info(mbgl::Event::General, stream.str());
-        exit(0);
-    } catch (const args::ParseError& e) {
-        std::ostringstream stream;
-        stream << argumentParser;
-        mbgl::Log::Info(mbgl::Event::General, stream.str());
-        mbgl::Log::Error(mbgl::Event::General, e.what());
-        exit(1);
-    } catch (const args::ValidationError& e) {
-        std::ostringstream stream;
-        stream << argumentParser;
-        mbgl::Log::Info(mbgl::Event::General, stream.str());
-        mbgl::Log::Error(mbgl::Event::General, e.what());
-        exit(2);
-    } catch (const std::regex_error& e) {
-        mbgl::Log::Error(mbgl::Event::General, "Invalid filter regular expression: %s", e.what());
-        exit(3);
-    }
-
-    mbgl::filesystem::path rootPath {testPathValue ? args::get(testPathValue) : TestRunner::getBasePath()};
-    if (!mbgl::filesystem::exists(rootPath)) {
-        mbgl::Log::Error(mbgl::Event::General, "Provided rootPath '%s' does not exist.", rootPath.string().c_str());
-        exit(4);
-    }
-
-    std::vector<mbgl::filesystem::path> paths;
-    for (const auto& id : args::get(testNameValues)) {
-        paths.emplace_back(TestRunner::getBasePath() + "/" + id);
-    }
-
-    if (paths.empty()) {
-        paths.emplace_back(TestRunner::getBasePath());
-    }
-
-    // Recursively traverse through the test paths and collect test directories containing "style.json".
-    std::vector<TestPaths> testPaths;
-    testPaths.reserve(paths.size());
-    for (const auto& path : paths) {
-        if (!mbgl::filesystem::exists(path)) {
-            mbgl::Log::Warning(mbgl::Event::General, "Provided test folder '%s' does not exist.", path.string().c_str());
-            continue;
-        }
-        for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(path)) {
-            // Skip paths that fail regexp match.
-            if (testFilterValue && !std::regex_match(testPath.path().string(), args::get(testFilterValue))) {
-                continue;
-            }
-            if (testPath.path().filename() == "style.json") {
-                testPaths.emplace_back(makeTestPaths(testPath));
-            }
-        }
-    }
-
-    return ArgumentsTuple {
-        recycleMapFlag ? args::get(recycleMapFlag) : false,
-        shuffleFlag ? args::get(shuffleFlag) : false, seedValue ? args::get(seedValue) : 1u,
-        testPathValue ? args::get(testPathValue) : TestRunner::getBasePath(),
-        std::move(testPaths)
-    };
+std::vector<std::string> readExpectedImageEntries(const mbgl::filesystem::path& base) {
+    static const std::regex regex(".*expected.*.png");
+    return readExpectedEntries(regex, base);
 }
 
-std::vector<std::pair<std::string, std::string>> parseIgnores() {
-    std::vector<std::pair<std::string, std::string>> ignores;
-
-    auto mainIgnoresPath = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("platform/node/test/ignores.json");
-
-    mbgl::filesystem::path platformSpecificIgnores;
-
-#ifdef __APPLE__
-    platformSpecificIgnores = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/mac-ignores.json");
-#elif __linux__
-    platformSpecificIgnores = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/linux-ignores.json");
-#endif
-    
-    std::vector<mbgl::filesystem::path> ignoresPaths = { mainIgnoresPath, platformSpecificIgnores };
-    for (auto path: ignoresPaths) {
-        auto maybeIgnores = readJson(path);
-        if (!maybeIgnores.is<mbgl::JSDocument>()) {
-            continue;
-        }
-        for (const auto& property : maybeIgnores.get<mbgl::JSDocument>().GetObject()) {
-            const std::string ignore = { property.name.GetString(),
-                                         property.name.GetStringLength() };
-            const std::string reason = { property.value.GetString(),
-                                         property.value.GetStringLength() };
-            ignores.emplace_back(std::make_pair(ignore, reason));
-        }
-    }
-
-    return ignores;
+std::vector<std::string> readExpectedJSONEntries(const mbgl::filesystem::path& base) {
+    static const std::regex regex(".*expected.*.json");
+    return readExpectedEntries(regex, base);
 }
 
 TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
@@ -366,6 +307,29 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
     }
 
     const auto& document = maybeJson.get<mbgl::JSDocument>();
+
+    if (document.HasMember("file-size")) {
+        const mbgl::JSValue& fileSizeValue = document["file-size"];
+        assert(fileSizeValue.IsArray());
+        for (auto& probeValue : fileSizeValue.GetArray()) {
+            assert(probeValue.IsArray());
+            assert(probeValue.Size() >= 3u);
+            assert(probeValue[0].IsString());
+            assert(probeValue[1].IsString());
+            assert(probeValue[2].IsNumber());
+
+            std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()};
+            assert(!mark.empty());
+
+            std::string filePath{probeValue[1].GetString(), probeValue[1].GetStringLength()};
+            assert(!filePath.empty());
+
+            result.fileSize.emplace(std::piecewise_construct,
+                                    std::forward_as_tuple(std::move(mark)),
+                                    std::forward_as_tuple(std::move(filePath), probeValue[2].GetUint64(), 0.f));
+        }
+    }
+
     if (document.HasMember("memory")) {
         const mbgl::JSValue& memoryValue = document["memory"];
         assert(memoryValue.IsArray());
@@ -376,18 +340,88 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
             assert(probeValue[1].IsNumber());
             assert(probeValue[2].IsNumber());
 
-            const std::string mark { probeValue[0].GetString(), probeValue[0].GetStringLength() };
+            std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()};
             assert(!mark.empty());
             result.memory.emplace(std::piecewise_construct,
-                                  std::forward_as_tuple(std::move(mark)), 
+                                  std::forward_as_tuple(std::move(mark)),
                                   std::forward_as_tuple(probeValue[1].GetUint64(), probeValue[2].GetUint64()));
         }
     }
 
+    if (document.HasMember("network")) {
+        const mbgl::JSValue& networkValue = document["network"];
+        assert(networkValue.IsArray());
+        for (auto& probeValue : networkValue.GetArray()) {
+            assert(probeValue.IsArray());
+            assert(probeValue.Size() >= 3u);
+            assert(probeValue[0].IsString());
+            assert(probeValue[1].IsNumber());
+            assert(probeValue[2].IsNumber());
+
+            std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()};
+            assert(!mark.empty());
+
+            result.network.emplace(std::piecewise_construct,
+                                   std::forward_as_tuple(std::move(mark)),
+                                   std::forward_as_tuple(probeValue[1].GetUint64(), probeValue[2].GetUint64()));
+        }
+    }
+
+    if (document.HasMember("fps")) {
+        const mbgl::JSValue& fpsValue = document["fps"];
+        assert(fpsValue.IsArray());
+        for (auto& probeValue : fpsValue.GetArray()) {
+            assert(probeValue.IsArray());
+            assert(probeValue.Size() >= 4u);
+            assert(probeValue[0].IsString());
+            assert(probeValue[1].IsNumber()); // Average
+            assert(probeValue[2].IsNumber()); // Minimum
+            assert(probeValue[3].IsNumber()); // Tolerance
+            const std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()};
+            assert(!mark.empty());
+            result.fps.insert(
+                {std::move(mark), {probeValue[1].GetFloat(), probeValue[2].GetFloat(), probeValue[3].GetFloat()}});
+        }
+    }
+
+    if (document.HasMember("gfx")) {
+        const mbgl::JSValue& gfxValue = document["gfx"];
+        assert(gfxValue.IsArray());
+        for (auto& probeValue : gfxValue.GetArray()) {
+            assert(probeValue.IsArray());
+            assert(probeValue.Size() >= 8u);
+            assert(probeValue[0].IsString());
+            assert(probeValue[1].IsInt());
+            assert(probeValue[2].IsInt());
+            assert(probeValue[3].IsInt());
+            assert(probeValue[4].IsInt());
+            assert(probeValue[5].IsArray());
+            assert(probeValue[6].IsArray());
+            assert(probeValue[7].IsArray());
+
+            const std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()};
+            assert(!mark.empty());
+
+            GfxProbe probe;
+            probe.numDrawCalls = probeValue[1].GetInt();
+            probe.numTextures = probeValue[2].GetInt();
+            probe.numBuffers = probeValue[3].GetInt();
+            probe.numFrameBuffers = probeValue[4].GetInt();
+            probe.memTextures.allocated = probeValue[5].GetArray()[0].GetInt();
+            probe.memTextures.peak = probeValue[5].GetArray()[1].GetInt();
+            probe.memIndexBuffers.allocated = probeValue[6].GetArray()[0].GetInt();
+            probe.memIndexBuffers.peak = probeValue[6].GetArray()[1].GetInt();
+            probe.memVertexBuffers.allocated = probeValue[7].GetArray()[0].GetInt();
+            probe.memVertexBuffers.peak = probeValue[7].GetArray()[1].GetInt();
+
+            result.gfx.insert({mark, std::move(probe)});
+        }
+    }
+
     return result;
 }
 
-TestMetadata parseTestMetadata(const TestPaths& paths) {
+TestMetadata parseTestMetadata(const TestPaths& paths, const Manifest& manifest) {
     TestMetadata metadata;
     metadata.paths = paths;
 
@@ -398,18 +432,16 @@ TestMetadata parseTestMetadata(const TestPaths& paths) {
     }
 
     metadata.document = std::move(maybeJson.get<mbgl::JSDocument>());
-    localizeStyleURLs(metadata.document, metadata.document);
+    manifest.localizeStyleURLs(metadata.document, metadata.document);
 
     if (!metadata.document.HasMember("metadata")) {
-        mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s",
-                           paths.stylePath.c_str());
+        mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s", paths.stylePath.c_str());
         return metadata;
     }
 
     const mbgl::JSValue& metadataValue = metadata.document["metadata"];
     if (!metadataValue.HasMember("test")) {
-        mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata.test': %s",
-                           paths.stylePath.c_str());
+        mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata.test': %s", paths.stylePath.c_str());
         return metadata;
     }
 
@@ -437,13 +469,26 @@ TestMetadata parseTestMetadata(const TestPaths& paths) {
 
     if (testValue.HasMember("description")) {
         assert(testValue["description"].IsString());
-        metadata.description = std::string{ testValue["description"].GetString(),
-                                                testValue["description"].GetStringLength() };
+        metadata.description =
+            std::string{testValue["description"].GetString(), testValue["description"].GetStringLength()};
     }
 
     if (testValue.HasMember("mapMode")) {
+        metadata.outputsImage = true;
         assert(testValue["mapMode"].IsString());
-        metadata.mapMode = testValue["mapMode"].GetString() == std::string("tile") ? mbgl::MapMode::Tile : mbgl::MapMode::Static;
+        std::string mapModeStr = testValue["mapMode"].GetString();
+        if (mapModeStr == "tile")
+            metadata.mapMode = mbgl::MapMode::Tile;
+        else if (mapModeStr == "continuous") {
+            metadata.mapMode = mbgl::MapMode::Continuous;
+            metadata.outputsImage = false;
+        } else if (mapModeStr == "static")
+            metadata.mapMode = mbgl::MapMode::Static;
+        else {
+            mbgl::Log::Warning(
+                mbgl::Event::ParseStyle, "Unknown map mode: %s. Falling back to static mode", mapModeStr.c_str());
+            metadata.mapMode = mbgl::MapMode::Static;
+        }
     }
 
     // Test operations handled in runner.cpp.
@@ -476,6 +521,45 @@ TestMetadata parseTestMetadata(const TestPaths& paths) {
         metadata.ySkew = testValue["skew"][1].GetDouble();
     }
 
+    if (testValue.HasMember("queryGeometry")) {
+        assert(testValue["queryGeometry"].IsArray());
+        if (testValue["queryGeometry"][0].IsNumber() && testValue["queryGeometry"][1].IsNumber()) {
+            metadata.queryGeometry.x = testValue["queryGeometry"][0].GetDouble();
+            metadata.queryGeometry.y = testValue["queryGeometry"][1].GetDouble();
+        } else if (testValue["queryGeometry"][0].IsArray() && testValue["queryGeometry"][1].IsArray()) {
+            metadata.queryGeometryBox.min.x = testValue["queryGeometry"][0][0].GetDouble();
+            metadata.queryGeometryBox.min.y = testValue["queryGeometry"][0][1].GetDouble();
+            metadata.queryGeometryBox.max.x = testValue["queryGeometry"][1][0].GetDouble();
+            metadata.queryGeometryBox.max.y = testValue["queryGeometry"][1][1].GetDouble();
+        }
+        metadata.renderTest = false;
+    }
+
+    if (testValue.HasMember("queryOptions")) {
+        assert(testValue["queryOptions"].IsObject());
+
+        if (testValue["queryOptions"].HasMember("layers")) {
+            assert(testValue["queryOptions"]["layers"].IsArray());
+            auto layersArray = testValue["queryOptions"]["layers"].GetArray();
+            std::vector<std::string> layersVec;
+            for (uint32_t i = 0; i < layersArray.Size(); i++) {
+                layersVec.emplace_back(testValue["queryOptions"]["layers"][i].GetString());
+            }
+            metadata.queryOptions.layerIDs = layersVec;
+        }
+
+        using namespace mbgl::style;
+        using namespace mbgl::style::conversion;
+        if (testValue["queryOptions"].HasMember("filter")) {
+            assert(testValue["queryOptions"]["filter"].IsArray());
+            auto& filterVal = testValue["queryOptions"]["filter"];
+            Error error;
+            mbgl::optional<Filter> converted = convert<Filter>(filterVal, error);
+            assert(converted);
+            metadata.queryOptions.filter = std::move(*converted);
+        }
+    }
+
     // TODO: fadeDuration
     // TODO: addFakeCanvas
 
@@ -493,26 +577,41 @@ std::string encodeBase64(const std::string& data) {
 }
 
 std::string createResultItem(const TestMetadata& metadata, bool hasFailedTests) {
-    const bool shouldHide = (hasFailedTests && metadata.status == "passed") || (metadata.status.find("ignored") != std::string::npos);
-    
+    const bool shouldHide =
+        (hasFailedTests && metadata.status == "passed") || (metadata.status.find("ignored") != std::string::npos);
+
     std::string html;
     html.append("<div class=\"test " + metadata.status + (shouldHide ? " hide" : "") + "\">\n");
     html.append(R"(<h2><span class="label" style="background: )" + metadata.color + "\">" + metadata.status + "</span> " + metadata.id + "</h2>\n");
     if (metadata.status != "errored") {
-        html.append("<img width=" + mbgl::util::toString(metadata.size.width));
-        html.append(" height=" + mbgl::util::toString(metadata.size.height));
-        html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.actual) + "\"");
-        html.append(" data-alt-src=\"data:image/png;base64," + encodeBase64(metadata.expected) + "\">\n");
-
-        html.append("<img width=" + mbgl::util::toString(metadata.size.width));
-        html.append(" height=" + mbgl::util::toString(metadata.size.height));
-        html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.diff) + "\">\n");
+        if (metadata.outputsImage) {
+            if (metadata.renderTest) {
+                html.append("<img width=" + mbgl::util::toString(metadata.size.width));
+                html.append(" height=" + mbgl::util::toString(metadata.size.height));
+                html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.actual) + "\"");
+                html.append(" data-alt-src=\"data:image/png;base64," + encodeBase64(metadata.expected) + "\">\n");
+
+                html.append("<img width=" + mbgl::util::toString(metadata.size.width));
+                html.append(" height=" + mbgl::util::toString(metadata.size.height));
+                html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.diff) + "\">\n");
+            } else {
+                html.append("<img width=" + mbgl::util::toString(metadata.size.width));
+                html.append(" height=" + mbgl::util::toString(metadata.size.height));
+                html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.actual) + "\">\n");
+            }
+        }
     } else {
-        assert(!metadata.errorMessage.empty());
+        // FIXME: there are several places that errorMessage is not filled
+        // comment out assert(!metadata.errorMessage.empty());
         html.append("<p style=\"color: red\"><strong>Error:</strong> " + metadata.errorMessage + "</p>\n");
     }
     if (metadata.difference != 0.0) {
-        html.append("<p class=\"diff\"><strong>Diff:</strong> " + mbgl::util::toString(metadata.difference) + "</p>\n");
+        if (metadata.renderTest) {
+            html.append("<p class=\"diff\"><strong>Diff:</strong> " + mbgl::util::toString(metadata.difference) +
+                        "</p>\n");
+        } else {
+            html.append("<p class=\"diff\"><strong>Diff:</strong> " + metadata.diff + "</p>\n");
+        }
     }
     html.append("</div>\n");
 
@@ -595,89 +694,3 @@ std::string createResultPage(const TestStatistics& stats, const std::vector<Test
 
     return resultsPage;
 }
-
-std::string localizeURL(const std::string& url) {
-    static const std::regex regex { "local://" };
-    if (auto vendorPath = getVendorPath(url, regex)) {
-        return *vendorPath;
-    } else {
-        return getIntegrationPath(url, "", regex).value_or(url);
-    }
-}
-
-void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) {
-    if (root.HasMember("urls") && root["urls"].IsArray()) {
-        for (auto& urlValue : root["urls"].GetArray()) {
-            const std::string path = prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
-                .value_or(localizeLocalURL(urlValue.GetString())
-                    .value_or(urlValue.GetString())));
-            urlValue.Set<std::string>(path, document.GetAllocator());
-        }
-    }
-
-    if (root.HasMember("url")) {
-        static const std::string image("image");
-        static const std::string video("video");
-
-        mbgl::JSValue& urlValue = root["url"];
-        const std::string path = prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
-            .value_or(localizeLocalURL(urlValue.GetString())
-                .value_or(urlValue.GetString())));
-        urlValue.Set<std::string>(path, document.GetAllocator());
-
-        if (root["type"].GetString() != image && root["type"].GetString() != video) {
-            const auto tilesetPath = std::string(urlValue.GetString()).erase(0u, 7u); // remove "file://"
-            auto maybeTileset = readJson(tilesetPath);
-            if (maybeTileset.is<mbgl::JSDocument>()) {
-                const auto& tileset = maybeTileset.get<mbgl::JSDocument>();
-                assert(tileset.HasMember("tiles"));
-                root.AddMember("tiles", (mbgl::JSValue&)tileset["tiles"], document.GetAllocator());
-                root.RemoveMember("url");
-            }
-        }
-    }
-
-    if (root.HasMember("tiles")) {
-        mbgl::JSValue& tilesValue = root["tiles"];
-        assert(tilesValue.IsArray());
-        for (auto& tileValue : tilesValue.GetArray()) {
-            const std::string path = prependFileScheme(localizeMapboxTilesURL(tileValue.GetString())
-                .value_or(localizeLocalURL(tileValue.GetString())
-                    .value_or(localizeHttpURL(tileValue.GetString())
-                        .value_or(tileValue.GetString()))));
-            tileValue.Set<std::string>(path, document.GetAllocator());
-        }
-    }
-
-    if (root.HasMember("data") && root["data"].IsString()) {
-        mbgl::JSValue& dataValue = root["data"];
-        const std::string path = prependFileScheme(localizeLocalURL(dataValue.GetString())
-            .value_or(dataValue.GetString()));
-        dataValue.Set<std::string>(path, document.GetAllocator());
-    }
-}
-
-void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) {
-    if (root.HasMember("sources")) {
-        mbgl::JSValue& sourcesValue = root["sources"];
-        for (auto& sourceProperty : sourcesValue.GetObject()) {
-            localizeSourceURLs(sourceProperty.value, document);
-        }
-    }
-
-    if (root.HasMember("glyphs")) {
-        mbgl::JSValue& glyphsValue = root["glyphs"];
-        const std::string path = prependFileScheme(localizeMapboxFontsURL(glyphsValue.GetString())
-            .value_or(localizeLocalURL(glyphsValue.GetString(), true)
-                .value_or(glyphsValue.GetString())));
-        glyphsValue.Set<std::string>(path, document.GetAllocator());
-    }
-
-    if (root.HasMember("sprite")) {
-        mbgl::JSValue& spriteValue = root["sprite"];
-        const std::string path = prependFileScheme(localizeMapboxSpriteURL(spriteValue.GetString())
-            .value_or(localizeLocalURL(spriteValue.GetString())
-                .value_or(spriteValue.GetString())));
-        spriteValue.Set<std::string>(path, document.GetAllocator());
-    }
-}
diff --git a/render-test/parser.hpp b/render-test/parser.hpp
index 94fb212944..3d79ac668a 100644
--- a/render-test/parser.hpp
+++ b/render-test/parser.hpp
@@ -5,31 +5,27 @@
 #include <mbgl/util/rapidjson.hpp>
 #include <mbgl/util/variant.hpp>
 
-#include <tuple>
 #include <string>
+#include <tuple>
 #include <vector>
 
+class Manifest;
+
 using ErrorMessage = std::string;
 using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>;
 
-using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector<TestPaths>>;
-
 JSONReply readJson(const mbgl::filesystem::path&);
 std::string serializeJsonValue(const mbgl::JSValue&);
 std::string serializeMetrics(const TestMetrics&);
 
-std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base);
+std::vector<std::string> readExpectedImageEntries(const mbgl::filesystem::path& base);
+std::vector<std::string> readExpectedJSONEntries(const mbgl::filesystem::path& base);
 
 TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path);
 
-ArgumentsTuple parseArguments(int argc, char** argv);
-std::vector<std::pair<std::string, std::string>> parseIgnores();
-
-TestMetadata parseTestMetadata(const TestPaths& paths);
+TestMetadata parseTestMetadata(const TestPaths& paths, const Manifest& manifest);
 
 std::string createResultPage(const TestStatistics&, const std::vector<TestMetadata>&, bool shuffle, uint32_t seed);
 
-std::string localizeURL(const std::string& url);
-
-void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document);
-void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document);
-\ No newline at end of file
+std::string toJSON(const mbgl::Value& value, unsigned indent, bool singleLine);
+std::string toJSON(const std::vector<mbgl::Feature>& features, unsigned indent, bool singleLine);
diff --git a/render-test/main.cpp b/render-test/render_test.cpp
index fcdbe3ab55..38d6c15f3f 100644
--- a/render-test/main.cpp
+++ b/render-test/render_test.cpp
@@ -1,14 +1,17 @@
 #include "allocation_index.hpp"
 
-#include <mbgl/util/run_loop.hpp>
+#include <mbgl/render_test.hpp>
 #include <mbgl/util/io.hpp>
+#include <mbgl/util/logging.hpp>
+#include <mbgl/util/run_loop.hpp>
+
+#include <args.hxx>
 
+#include "manifest_parser.hpp"
 #include "metadata.hpp"
 #include "parser.hpp"
 #include "runner.hpp"
 
-#include <random>
-
 #define ANSI_COLOR_RED        "\x1b[31m"
 #define ANSI_COLOR_GREEN      "\x1b[32m"
 #define ANSI_COLOR_YELLOW     "\x1b[33m"
@@ -36,36 +39,97 @@ void operator delete(void* ptr, size_t) noexcept {
 }
 #endif
 
-int main(int argc, char** argv) {
-    bool recycleMap;
-    bool shuffle;
-    uint32_t seed;
-    std::string testRootPath;
-    std::vector<TestPaths> testPaths;
+namespace {
+using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector<std::string>, std::string>;
+ArgumentsTuple parseArguments(int argc, char** argv) {
+    args::ArgumentParser argumentParser("Mapbox GL Test Runner");
+
+    args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", {'h', "help"});
+
+    args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", {'r', "recycle-map"});
+    args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order", {'s', "shuffle"});
+    args::ValueFlag<uint32_t> seedValue(argumentParser, "seed", "Shuffle seed (default: random)", {"seed"});
+    args::ValueFlag<std::string> testPathValue(
+        argumentParser, "manifestPath", "Test manifest file path", {'p', "manifestPath"});
+    args::ValueFlag<std::string> testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"});
+    args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
+
+    try {
+        argumentParser.ParseCLI(argc, argv);
+    } catch (const args::Help&) {
+        std::ostringstream stream;
+        stream << argumentParser;
+        mbgl::Log::Info(mbgl::Event::General, stream.str());
+        exit(0);
+    } catch (const args::ParseError& e) {
+        std::ostringstream stream;
+        stream << argumentParser;
+        mbgl::Log::Info(mbgl::Event::General, stream.str());
+        mbgl::Log::Error(mbgl::Event::General, e.what());
+        exit(1);
+    } catch (const args::ValidationError& e) {
+        std::ostringstream stream;
+        stream << argumentParser;
+        mbgl::Log::Info(mbgl::Event::General, stream.str());
+        mbgl::Log::Error(mbgl::Event::General, e.what());
+        exit(2);
+    } catch (const std::regex_error& e) {
+        mbgl::Log::Error(mbgl::Event::General, "Invalid filter regular expression: %s", e.what());
+        exit(3);
+    }
 
-    std::tie(recycleMap, shuffle, seed, testRootPath, testPaths) = parseArguments(argc, argv);
-    const std::string::size_type rootLength = testRootPath.length();
+    mbgl::filesystem::path manifestPath{testPathValue ? args::get(testPathValue) : std::string{TEST_RUNNER_ROOT_PATH}};
+    if (!mbgl::filesystem::exists(manifestPath) || !manifestPath.has_filename()) {
+        mbgl::Log::Error(mbgl::Event::General,
+                         "Provided test manifest file path '%s' does not exist",
+                         manifestPath.string().c_str());
+        exit(4);
+    }
 
-    const auto ignores = parseIgnores();
+    auto testNames = testNameValues ? args::get(testNameValues) : std::vector<std::string>{};
+    auto testFilter = testFilterValue ? args::get(testFilterValue) : std::string{};
+    const auto shuffle = shuffleFlag ? args::get(shuffleFlag) : false;
+    const auto seed = seedValue ? args::get(seedValue) : 1u;
+    return ArgumentsTuple{recycleMapFlag ? args::get(recycleMapFlag) : false,
+                          shuffle,
+                          seed,
+                          manifestPath.string(),
+                          std::move(testNames),
+                          std::move(testFilter)};
+}
+} // namespace
+namespace mbgl {
 
+int runRenderTests(int argc, char** argv) {
+    bool recycleMap;
+    bool shuffle;
+    uint32_t seed;
+    std::string manifestPath;
+    std::vector<std::string> testNames;
+    std::string testFilter;
+
+    std::tie(recycleMap, shuffle, seed, manifestPath, testNames, testFilter) = parseArguments(argc, argv);
+    auto manifestData = ManifestParser::parseManifest(manifestPath, testNames, testFilter);
+    if (!manifestData) {
+        exit(5);
+    }
+    mbgl::util::RunLoop runLoop;
+    TestRunner runner(std::move(*manifestData));
     if (shuffle) {
         printf(ANSI_COLOR_YELLOW "Shuffle seed: %d" ANSI_COLOR_RESET "\n", seed);
-
-        std::seed_seq sequence { seed };
-        std::mt19937 shuffler(sequence);
-        std::shuffle(testPaths.begin(), testPaths.end(), shuffler);
+        runner.doShuffle(seed);
     }
 
-    mbgl::util::RunLoop runLoop;
-    TestRunner runner;
-
+    const auto& manifest = runner.getManifest();
+    const auto& ignores = manifest.getIgnores();
+    const auto& testPaths = manifest.getTestPaths();
     std::vector<TestMetadata> metadatas;
     metadatas.reserve(testPaths.size());
 
     TestStatistics stats;
 
     for (auto& testPath : testPaths) {
-        TestMetadata metadata = parseTestMetadata(testPath);
+        TestMetadata metadata = parseTestMetadata(testPath, manifest);
 
         if (!recycleMap) {
             runner.reset();
@@ -75,13 +139,14 @@ int main(int argc, char** argv) {
         std::string& status = metadata.status;
         std::string& color = metadata.color;
 
+        const std::string::size_type rootLength = manifest.getTestRootPath().length();
         id = testPath.defaultExpectations();
         id = id.substr(rootLength + 1, id.length() - rootLength - 2);
 
         bool shouldIgnore = false;
         std::string ignoreReason;
 
-        const std::string ignoreName = "render-tests/" + id;
+        const std::string ignoreName = id;
         const auto it = std::find_if(ignores.cbegin(), ignores.cend(), [&ignoreName](auto pair) { return pair.first == ignoreName; });
         if (it != ignores.end()) {
             shouldIgnore = true;
@@ -97,7 +162,8 @@ int main(int argc, char** argv) {
             errored = !runner.run(metadata) || !metadata.errorMessage.empty();
         }
 
-        bool passed = !errored && !metadata.diff.empty() && metadata.difference <= metadata.allowed;
+        bool passed =
+            !errored && (!metadata.outputsImage || !metadata.diff.empty()) && metadata.difference <= metadata.allowed;
 
         if (shouldIgnore) {
             if (passed) {
@@ -122,6 +188,7 @@ int main(int argc, char** argv) {
                 color = "red";
                 stats.erroredTests++;
                 printf(ANSI_COLOR_RED "* errored %s" ANSI_COLOR_RESET "\n", id.c_str());
+                printf(ANSI_COLOR_RED "* error: %s" ANSI_COLOR_RESET "\n", metadata.errorMessage.c_str());
             } else {
                 status = "failed";
                 color = "red";
@@ -132,13 +199,14 @@ int main(int argc, char** argv) {
 
         metadatas.push_back(std::move(metadata));
     }
-
+    const auto& testRootPath = manifest.getManifestPath();
+    const auto resultPath =
+        testRootPath + "/" + (testNames.empty() ? "render-tests" : testNames.front()) + "_index.html";
     std::string resultsHTML = createResultPage(stats, metadatas, shuffle, seed);
-    mbgl::util::write_file(testRootPath + "/index.html", resultsHTML);
+    mbgl::util::write_file(resultPath, resultsHTML);
 
-    const uint32_t count = stats.erroredTests + stats.failedTests +
-                           stats.ignoreFailedTests + stats.ignorePassedTests +
-                           stats.passedTests;
+    const uint32_t count =
+        stats.erroredTests + stats.failedTests + stats.ignoreFailedTests + stats.ignorePassedTests + stats.passedTests;
 
     if (stats.passedTests) {
         printf(ANSI_COLOR_GREEN "%u passed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.passedTests, 100.0 * stats.passedTests / count);
@@ -156,7 +224,9 @@ int main(int argc, char** argv) {
         printf(ANSI_COLOR_RED "%u errored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.erroredTests, 100.0 * stats.erroredTests / count);
     }
 
-    printf("Results at: %s%s\n", testRootPath.c_str(), "/index.html");
+    printf("Results at: %s\n", resultPath.c_str());
 
     return stats.failedTests + stats.erroredTests == 0 ? 0 : 1;
 }
+
+} // namespace mbgl
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 5c8f53759f..8a4b0b3b0e 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -1,6 +1,7 @@
 #include <mbgl/map/camera.hpp>
 #include <mbgl/map/map_observer.hpp>
 #include <mbgl/renderer/renderer.hpp>
+#include <mbgl/renderer/renderer_observer.hpp>
 #include <mbgl/style/conversion/filter.hpp>
 #include <mbgl/style/conversion/layer.hpp>
 #include <mbgl/style/conversion/light.hpp>
@@ -19,34 +20,117 @@
 
 #include <mapbox/pixelmatch.hpp>
 
+#include <../expression-test/test_runner_common.hpp>
 #include "allocation_index.hpp"
+#include "file_source.hpp"
 #include "metadata.hpp"
 #include "parser.hpp"
 #include "runner.hpp"
 
 #include <algorithm>
 #include <cassert>
-#include <regex>
 #include <utility>
 #include <sstream>
 
-// static 
-const std::string& TestRunner::getBasePath() {
-    const static std::string result =
-        std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration/render-tests");
-    return result;
-}
+using namespace mbgl;
+
+GfxProbe::GfxProbe(const mbgl::gfx::RenderingStats& stats, const GfxProbe& prev)
+    : numBuffers(stats.numBuffers),
+      numDrawCalls(stats.numDrawCalls),
+      numFrameBuffers(stats.numFrameBuffers),
+      numTextures(stats.numActiveTextures),
+      memIndexBuffers(stats.memIndexBuffers, std::max(stats.memIndexBuffers, prev.memIndexBuffers.peak)),
+      memVertexBuffers(stats.memVertexBuffers, std::max(stats.memVertexBuffers, prev.memVertexBuffers.peak)),
+      memTextures(stats.memTextures, std::max(stats.memTextures, prev.memTextures.peak)) {}
+
+struct RunContext {
+    RunContext() = default;
+
+    GfxProbe activeGfxProbe;
+    GfxProbe baselineGfxProbe;
+    bool gfxProbeActive;
+};
+
+class TestRunnerMapObserver : public MapObserver {
+public:
+    TestRunnerMapObserver() : mapLoadFailure(false), finishRenderingMap(false), idle(false) {}
+
+    void onDidFailLoadingMap(MapLoadError, const std::string&) override { mapLoadFailure = true; }
+
+    void onDidFinishRenderingMap(RenderMode mode) override final {
+        if (!finishRenderingMap) finishRenderingMap = mode == RenderMode::Full;
+    }
+
+    void onDidBecomeIdle() override final { idle = true; }
+
+    void reset() {
+        mapLoadFailure = false;
+        finishRenderingMap = false;
+        idle = false;
+    }
+
+    bool mapLoadFailure;
+    bool finishRenderingMap;
+    bool idle;
+};
 
 // static
-const std::vector<std::string>& TestRunner::getPlatformExpectationsPaths() {
-    // TODO: Populate from command line.
-    const static std::vector<std::string> result {
-        std::string(TEST_RUNNER_ROOT_PATH).append("/render-test/expected")
-    };
-    return result;
+gfx::HeadlessBackend::SwapBehaviour swapBehavior(MapMode mode) {
+    return mode == MapMode::Continuous ? gfx::HeadlessBackend::SwapBehaviour::Flush
+                                       : gfx::HeadlessBackend::SwapBehaviour::NoFlush;
+}
+
+std::string simpleDiff(const Value& result, const Value& expected) {
+    std::vector<std::string> resultTokens{tokenize(toJSON(result, 2, false))};
+    std::vector<std::string> expectedTokens{tokenize(toJSON(expected, 2, false))};
+    std::size_t maxLength = std::max(resultTokens.size(), expectedTokens.size());
+    std::ostringstream diff;
+
+    diff << "<pre>" << std::endl;
+    const auto flush =
+        [](const std::vector<std::string>& vec, std::size_t pos, std::ostringstream& out, std::string separator) {
+            for (std::size_t j = pos; j < vec.size(); ++j) {
+                out << separator << vec[j] << std::endl;
+            }
+        };
+
+    for (std::size_t i = 0; i < maxLength; ++i) {
+        if (resultTokens.size() <= i) {
+            flush(expectedTokens, i, diff, "-");
+            break;
+        }
+
+        if (expectedTokens.size() <= i) {
+            flush(resultTokens, i, diff, "+");
+            break;
+        }
+
+        if (!deepEqual(resultTokens[i], expectedTokens[i])) {
+            diff << "<b>"
+                 << "-" << expectedTokens[i] << "</b>" << std::endl;
+            diff << "<b>"
+                 << "+" << resultTokens[i] << "</b>" << std::endl;
+        } else {
+            diff << resultTokens[i] << std::endl;
+        }
+    }
+    diff << "</pre>" << std::endl;
+    return diff.str();
+}
+
+TestRunner::TestRunner(Manifest manifest_) : manifest(std::move(manifest_)) {}
+
+const Manifest& TestRunner::getManifest() const {
+    return manifest;
+}
+
+void TestRunner::doShuffle(uint32_t seed) {
+    manifest.doShuffle(seed);
 }
 
-bool TestRunner::checkResults(mbgl::PremultipliedImage&& actualImage, TestMetadata& metadata) {
+bool TestRunner::checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
+                                       std::vector<mbgl::Feature>&& features,
+                                       TestMetadata& metadata) {
     const std::string& base = metadata.paths.defaultExpectations();
     const std::vector<mbgl::filesystem::path>& expectations = metadata.paths.expectations;
 
@@ -57,74 +141,193 @@ bool TestRunner::checkResults(mbgl::PremultipliedImage&& actualImage, TestMetada
         return false;
     }
 
+    metadata.actualJson = toJSON(features, 2, false);
+
+    if (metadata.actualJson.empty()) {
+        metadata.errorMessage = "Invalid size for actual JSON";
+        return false;
+    }
+
 #if !TEST_READ_ONLY
     if (getenv("UPDATE_PLATFORM")) {
         mbgl::filesystem::create_directories(expectations.back());
-        mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actualImage));
+        mbgl::util::write_file(expectations.back().string() + "/expected.json", metadata.actualJson);
         return true;
     } else if (getenv("UPDATE_DEFAULT")) {
-        mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actualImage));
+        mbgl::util::write_file(base + "/expected.json", metadata.actualJson);
         return true;
-    } else if (getenv("UPDATE_METRICS")) {
-        if (!metadata.metrics.isEmpty()) {
-            mbgl::filesystem::create_directories(expectations.back());
-            mbgl::util::write_file(expectations.back().string() + "/metrics.json", serializeMetrics(metadata.metrics));
-            return true;
-        }
     }
 
-    mbgl::util::write_file(base + "/actual.png", metadata.actual);
+    mbgl::util::write_file(base + "/actual.json", metadata.actualJson);
 #endif
 
-    mbgl::PremultipliedImage expectedImage { actualImage.size };
-    mbgl::PremultipliedImage imageDiff { actualImage.size };
-
-    double pixels = 0.0;
-    std::vector<std::string> expectedImagesPaths;
+    std::vector<std::string> expectedJsonPaths;
     mbgl::filesystem::path expectedMetricsPath;
-    for (auto rit = expectations.rbegin(); rit!= expectations.rend(); ++rit) {
+    for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) {
         if (mbgl::filesystem::exists(*rit)) {
-            if (metadata.expectedMetrics.isEmpty()) {
-                mbgl::filesystem::path maybeExpectedMetricsPath{ *rit };
-                maybeExpectedMetricsPath.replace_filename("metrics.json");
-                metadata.expectedMetrics = readExpectedMetrics(maybeExpectedMetricsPath);
-            }
-            expectedImagesPaths = readExpectedEntries(*rit);
-            if (!expectedImagesPaths.empty()) break;
+            expectedJsonPaths = readExpectedJSONEntries(*rit);
+            if (!expectedJsonPaths.empty()) break;
         }
     }
 
-    if (expectedImagesPaths.empty()) {
+    if (expectedJsonPaths.empty()) {
         metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string();
         return false;
     }
-    
-    for (const auto& entry: expectedImagesPaths) {
-        mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
-        if (!maybeExpectedImage) {
-            metadata.errorMessage = "Failed to load expected image " + entry;
+
+    for (const auto& entry : expectedJsonPaths) {
+        auto maybeExpectedJson = readJson(entry);
+        if (maybeExpectedJson.is<mbgl::JSDocument>()) {
+            auto& expected = maybeExpectedJson.get<mbgl::JSDocument>();
+
+            mbgl::JSDocument actual;
+            actual.Parse<0>(metadata.actualJson);
+            if (actual.HasParseError()) {
+                metadata.errorMessage = "Error parsing actual JSON for: " + metadata.paths.stylePath.string();
+                return false;
+            }
+
+            auto actualVal = mapbox::geojson::convert<mapbox::geojson::value>(actual);
+            auto expectedVal = mapbox::geojson::convert<mapbox::geojson::value>(expected);
+            bool equal = deepEqual(actualVal, expectedVal);
+
+            metadata.difference = !equal;
+            if (equal) {
+                metadata.diff = "Match";
+            } else {
+                metadata.diff = simpleDiff(actualVal, expectedVal);
+            }
+        } else {
+            metadata.errorMessage = "Failed to load expected JSON " + entry;
+            return false;
+        }
+    }
+
+    return true;
+}
+
+bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage, TestMetadata& metadata) {
+    const std::string& base = metadata.paths.defaultExpectations();
+    const std::vector<mbgl::filesystem::path>& expectations = metadata.paths.expectations;
+
+    if (metadata.outputsImage) {
+        metadata.actual = mbgl::encodePNG(actualImage);
+
+        if (actualImage.size.isEmpty()) {
+            metadata.errorMessage = "Invalid size for actual image";
+            return false;
+        }
+
+#if !TEST_READ_ONLY
+        if (getenv("UPDATE_PLATFORM")) {
+            mbgl::filesystem::create_directories(expectations.back());
+            mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actualImage));
+            return true;
+        } else if (getenv("UPDATE_DEFAULT")) {
+            mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actualImage));
+            return true;
+        }
+
+        mbgl::util::write_file(base + "/actual.png", metadata.actual);
+#endif
+
+        mbgl::PremultipliedImage expectedImage{actualImage.size};
+        mbgl::PremultipliedImage imageDiff{actualImage.size};
+
+        double pixels = 0.0;
+        std::vector<std::string> expectedImagesPaths;
+        for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) {
+            if (mbgl::filesystem::exists(*rit)) {
+                expectedImagesPaths = readExpectedImageEntries(*rit);
+                if (!expectedImagesPaths.empty()) break;
+            }
+        }
+
+        if (expectedImagesPaths.empty()) {
+            metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string();
             return false;
         }
 
-        metadata.expected = *maybeExpectedImage;
+        for (const auto& entry : expectedImagesPaths) {
+            mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
+            if (!maybeExpectedImage) {
+                metadata.errorMessage = "Failed to load expected image " + entry;
+                return false;
+            }
 
-        expectedImage = mbgl::decodeImage(*maybeExpectedImage);
+            metadata.expected = *maybeExpectedImage;
 
-        pixels = // implicitly converting from uint64_t
-            mapbox::pixelmatch(actualImage.data.get(), expectedImage.data.get(), expectedImage.size.width,
-                               expectedImage.size.height, imageDiff.data.get(), 0.1285); // Defined in GL JS
+            expectedImage = mbgl::decodeImage(*maybeExpectedImage);
 
-        metadata.diff = mbgl::encodePNG(imageDiff);
+            pixels = // implicitly converting from uint64_t
+                mapbox::pixelmatch(actualImage.data.get(),
+                                   expectedImage.data.get(),
+                                   expectedImage.size.width,
+                                   expectedImage.size.height,
+                                   imageDiff.data.get(),
+                                   0.1285); // Defined in GL JS
+
+            metadata.diff = mbgl::encodePNG(imageDiff);
 
 #if !TEST_READ_ONLY
-        mbgl::util::write_file(base + "/diff.png", metadata.diff);
+            mbgl::util::write_file(base + "/diff.png", metadata.diff);
 #endif
 
-        metadata.difference = pixels / expectedImage.size.area();
-        if (metadata.difference <= metadata.allowed) {
-            break;
+            metadata.difference = pixels / expectedImage.size.area();
+            if (metadata.difference <= metadata.allowed) {
+                break;
+            }
+        }
+    }
+
+#if !TEST_READ_ONLY
+    if (getenv("UPDATE_METRICS")) {
+        if (!metadata.metrics.isEmpty()) {
+            mbgl::filesystem::create_directories(expectations.back());
+            mbgl::util::write_file(expectations.back().string() + "/metrics.json", serializeMetrics(metadata.metrics));
+            return true;
+        }
+    }
+#endif
+
+    mbgl::filesystem::path expectedMetricsPath;
+    for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) {
+        if (mbgl::filesystem::exists(*rit)) {
+            if (metadata.expectedMetrics.isEmpty()) {
+                mbgl::filesystem::path maybeExpectedMetricsPath{*rit};
+                maybeExpectedMetricsPath.replace_filename("metrics.json");
+                metadata.expectedMetrics = readExpectedMetrics(maybeExpectedMetricsPath);
+            }
+        }
+    }
+
+    // Check file size metrics.
+    for (const auto& expected : metadata.expectedMetrics.fileSize) {
+        auto actual = metadata.metrics.fileSize.find(expected.first);
+        if (actual == metadata.metrics.fileSize.end()) {
+            metadata.errorMessage = "Failed to find fileSize probe: " + expected.first;
+            return false;
+        }
+        if (actual->second.path != expected.second.path) {
+            std::stringstream ss;
+            ss << "Comparing different files at probe \"" << expected.first << "\": " << actual->second.path
+               << ", expected is " << expected.second.path << ".";
+            metadata.errorMessage = ss.str();
+
+            return false;
+        }
+
+        auto result = checkValue(expected.second.size, actual->second.size, actual->second.tolerance);
+        if (!std::get<bool>(result)) {
+            std::stringstream ss;
+            ss << "File size does not match at probe \"" << expected.first << "\": " << actual->second.size
+               << ", expected is " << expected.second.size << ".";
+
+            metadata.errorMessage = ss.str();
+            return false;
         }
     }
+#if !defined(SANITIZE)
     // Check memory metrics.
     for (const auto& expected : metadata.expectedMetrics.memory) {
         auto actual = metadata.metrics.memory.find(expected.first);
@@ -132,35 +335,168 @@ bool TestRunner::checkResults(mbgl::PremultipliedImage&& actualImage, TestMetada
             metadata.errorMessage = "Failed to find memory probe: " + expected.first;
             return false;
         }
-        if (actual->second.peak > expected.second.peak) {
-            std::stringstream ss;
-            ss << "Allocated memory peak size at probe \"" << expected.first << "\" is "
-               << actual->second.peak << " bytes, expected is " << expected.second.peak << " bytes.";
+        bool passed{false};
+        float delta{0.0f};
+        std::stringstream errorStream;
+        std::tie(passed, delta) = MemoryProbe::checkPeak(expected.second, actual->second);
+        if (!passed) {
+            errorStream << "Allocated memory peak size at probe \"" << expected.first << "\" is " << actual->second.peak
+                        << " bytes, expected is " << expected.second.peak << "±" << delta << " bytes.";
+        }
 
-            metadata.errorMessage = ss.str();
+        std::tie(passed, delta) = MemoryProbe::checkAllocations(expected.second, actual->second);
+        if (!passed) {
+            errorStream << "Number of allocations at probe \"" << expected.first << "\" is "
+                        << actual->second.allocations << ", expected is " << expected.second.allocations << "±"
+                        << std::round(delta) << " allocations.";
+        }
+
+        metadata.errorMessage = errorStream.str();
+        if (!metadata.errorMessage.empty()) return false;
+    }
+
+    // Check network metrics.
+    for (const auto& expected : metadata.expectedMetrics.network) {
+        auto actual = metadata.metrics.network.find(expected.first);
+        if (actual == metadata.metrics.network.end()) {
+            metadata.errorMessage = "Failed to find network probe: " + expected.first;
             return false;
         }
+        bool failed = false;
+        if (actual->second.requests != expected.second.requests) {
+            std::stringstream ss;
+            ss << "Number of requests at probe \"" << expected.first << "\" is " << actual->second.requests
+               << ", expected is " << expected.second.requests << ". ";
 
-        if (actual->second.allocations > expected.second.allocations) {
+            metadata.errorMessage = ss.str();
+            failed = true;
+        }
+        if (actual->second.transferred != expected.second.transferred) {
             std::stringstream ss;
-            ss << "Number of allocations at probe \"" << expected.first << "\" is "
-               << actual->second.allocations << ", expected is " << expected.second.allocations << ".";
+            ss << "Transferred data at probe \"" << expected.first << "\" is " << actual->second.transferred
+               << " bytes, expected is " << expected.second.transferred << " bytes.";
 
+            metadata.errorMessage += ss.str();
+            failed = true;
+        }
+        if (failed) {
+            return false;
+        }
+    }
+#endif // !defined(SANITIZE)
+    // Check fps metrics
+    for (const auto& expected : metadata.expectedMetrics.fps) {
+        auto actual = metadata.metrics.fps.find(expected.first);
+        if (actual == metadata.metrics.fps.end()) {
+            metadata.errorMessage = "Failed to find fps probe: " + expected.first;
+            return false;
+        }
+        auto result = checkValue(expected.second.average, actual->second.average, expected.second.tolerance);
+        if (!std::get<bool>(result)) {
+            std::stringstream ss;
+            ss << "Average fps at probe \"" << expected.first << "\" is " << actual->second.average
+               << ", expected to be " << expected.second.average << " with tolerance of " << expected.second.tolerance;
+            metadata.errorMessage = ss.str();
+            return false;
+        }
+        result = checkValue(expected.second.minOnePc, actual->second.minOnePc, expected.second.tolerance);
+        if (!std::get<bool>(result)) {
+            std::stringstream ss;
+            ss << "Minimum(1%) fps at probe \"" << expected.first << "\" is " << actual->second.minOnePc
+               << ", expected to be " << expected.second.minOnePc << " with tolerance of " << expected.second.tolerance;
             metadata.errorMessage = ss.str();
             return false;
         }
     }
+    // Check gfx metrics
+    for (const auto& expected : metadata.expectedMetrics.gfx) {
+        auto actual = metadata.metrics.gfx.find(expected.first);
+        if (actual == metadata.metrics.gfx.end()) {
+            metadata.errorMessage = "Failed to find gfx probe: " + expected.first;
+            return false;
+        }
+
+        const auto& probeName = expected.first;
+        const auto& expectedValue = expected.second;
+        const auto& actualValue = actual->second;
+        bool failed = false;
+
+        if (expectedValue.numDrawCalls != actualValue.numDrawCalls) {
+            std::stringstream ss;
+            if (!metadata.errorMessage.empty()) ss << std::endl;
+            ss << "Number of draw calls at probe\"" << probeName << "\" is " << actualValue.numDrawCalls
+               << ", expected is " << expectedValue.numDrawCalls;
+            metadata.errorMessage += ss.str();
+            failed = true;
+        }
+
+        if (expectedValue.numTextures != actualValue.numTextures) {
+            std::stringstream ss;
+            if (!metadata.errorMessage.empty()) ss << std::endl;
+            ss << "Number of textures at probe \"" << probeName << "\" is " << actualValue.numTextures
+               << ", expected is " << expectedValue.numTextures;
+            metadata.errorMessage += ss.str();
+            failed = true;
+        }
+
+        if (expectedValue.numBuffers != actualValue.numBuffers) {
+            std::stringstream ss;
+            if (!metadata.errorMessage.empty()) ss << std::endl;
+            ss << "Number of vertex and index buffers at probe \"" << probeName << "\" is " << actualValue.numBuffers
+               << ", expected is " << expectedValue.numBuffers;
+            metadata.errorMessage += ss.str();
+            failed = true;
+        }
+
+        if (expectedValue.numFrameBuffers != actualValue.numFrameBuffers) {
+            std::stringstream ss;
+            if (!metadata.errorMessage.empty()) ss << std::endl;
+            ss << "Number of frame buffers at probe \"" << probeName << "\" is " << actualValue.numFrameBuffers
+               << ", expected is " << expectedValue.numFrameBuffers;
+            metadata.errorMessage += ss.str();
+            failed = true;
+        }
+
+        if (expectedValue.memTextures.peak != actualValue.memTextures.peak) {
+            std::stringstream ss;
+            if (!metadata.errorMessage.empty()) ss << std::endl;
+            ss << "Allocated texture memory peak size at probe \"" << probeName << "\" is "
+               << actualValue.memTextures.peak << " bytes, expected is " << expectedValue.memTextures.peak << " bytes";
+            metadata.errorMessage += ss.str();
+            failed = true;
+        }
+
+        if (expectedValue.memIndexBuffers.peak != actualValue.memIndexBuffers.peak) {
+            std::stringstream ss;
+            if (!metadata.errorMessage.empty()) ss << std::endl;
+            ss << "Allocated index buffer memory peak size at probe \"" << probeName << "\" is "
+               << actualValue.memIndexBuffers.peak << " bytes, expected is " << expectedValue.memIndexBuffers.peak
+               << " bytes";
+            metadata.errorMessage += ss.str();
+            failed = true;
+        }
+
+        if (expectedValue.memVertexBuffers.peak != actualValue.memVertexBuffers.peak) {
+            std::stringstream ss;
+            if (!metadata.errorMessage.empty()) ss << std::endl;
+            ss << "Allocated vertex buffer memory peak size at probe \"" << probeName << "\" is "
+               << actualValue.memVertexBuffers.peak << " bytes, expected is " << expectedValue.memVertexBuffers.peak
+               << " bytes";
+            metadata.errorMessage += ss.str();
+            failed = true;
+        }
+
+        if (failed) return false;
+    }
 
     return true;
 }
 
-bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
-    if (!metadata.document.HasMember("metadata") ||
-        !metadata.document["metadata"].HasMember("test") ||
+bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata, RunContext& ctx) {
+    if (!metadata.document.HasMember("metadata") || !metadata.document["metadata"].HasMember("test") ||
         !metadata.document["metadata"]["test"].HasMember("operations")) {
         return true;
     }
-
     assert(metadata.document["metadata"]["test"]["operations"].IsArray());
 
     const auto& operationsArray = metadata.document["metadata"]["test"]["operations"].GetArray();
@@ -176,6 +512,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
 
     auto& frontend = maps[key]->frontend;
     auto& map = maps[key]->map;
+    auto& observer = maps[key]->observer;
 
     static const std::string waitOp("wait");
     static const std::string sleepOp("sleep");
@@ -186,6 +523,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
     static const std::string setCenterOp("setCenter");
     static const std::string setZoomOp("setZoom");
     static const std::string setBearingOp("setBearing");
+    static const std::string setPitchOp("setPitch");
     static const std::string setFilterOp("setFilter");
     static const std::string setLayerZoomRangeOp("setLayerZoomRange");
     static const std::string setLightOp("setLight");
@@ -195,22 +533,30 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
     static const std::string removeSourceOp("removeSource");
     static const std::string setPaintPropertyOp("setPaintProperty");
     static const std::string setLayoutPropertyOp("setLayoutProperty");
+    static const std::string fileSizeProbeOp("probeFileSize");
     static const std::string memoryProbeOp("probeMemory");
     static const std::string memoryProbeStartOp("probeMemoryStart");
     static const std::string memoryProbeEndOp("probeMemoryEnd");
+    static const std::string networkProbeOp("probeNetwork");
+    static const std::string networkProbeStartOp("probeNetworkStart");
+    static const std::string networkProbeEndOp("probeNetworkEnd");
     static const std::string setFeatureStateOp("setFeatureState");
     static const std::string getFeatureStateOp("getFeatureState");
     static const std::string removeFeatureStateOp("removeFeatureState");
+    static const std::string panGestureOp("panGesture");
+    static const std::string gfxProbeOp("probeGFX");
+    static const std::string gfxProbeStartOp("probeGFXStart");
+    static const std::string gfxProbeEndOp("probeGFXEnd");
 
-    // wait
     if (operationArray[0].GetString() == waitOp) {
+        // wait
         try {
             frontend.render(map);
         } catch (const std::exception&) {
             return false;
         }
-    // sleep
     } else if (operationArray[0].GetString() == sleepOp) {
+        // sleep
         mbgl::util::Timer sleepTimer;
         bool sleeping = true;
 
@@ -226,9 +572,8 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         while (sleeping) {
             mbgl::util::RunLoop::Get()->runOnce();
         }
-
-    // addImage | updateImage
     } else if (operationArray[0].GetString() == addImageOp || operationArray[0].GetString() == updateImageOp) {
+        // addImage | updateImage
         assert(operationArray.Size() >= 3u);
 
         float pixelRatio = 1.0f;
@@ -251,7 +596,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         std::string imagePath = operationArray[2].GetString();
         imagePath.erase(std::remove(imagePath.begin(), imagePath.end(), '"'), imagePath.end());
 
-        const mbgl::filesystem::path filePath(std::string(TEST_RUNNER_ROOT_PATH) + "/mapbox-gl-js/test/integration/" + imagePath);
+        const mbgl::filesystem::path filePath = mbgl::filesystem::path(manifest.getTestRootPath()) / imagePath;
 
         mbgl::optional<std::string> maybeImage = mbgl::util::readFile(filePath.string());
         if (!maybeImage) {
@@ -260,33 +605,30 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         }
 
         map.getStyle().addImage(std::make_unique<mbgl::style::Image>(imageName, mbgl::decodeImage(*maybeImage), pixelRatio, sdf));
-
-    // removeImage
     } else if (operationArray[0].GetString() == removeImageOp) {
+        // removeImage
         assert(operationArray.Size() >= 2u);
         assert(operationArray[1].IsString());
 
         const std::string imageName { operationArray[1].GetString(), operationArray[1].GetStringLength() };
         map.getStyle().removeImage(imageName);
-
-    // setStyle
     } else if (operationArray[0].GetString() == setStyleOp) {
+        // setStyle
         assert(operationArray.Size() >= 2u);
         if (operationArray[1].IsString()) {
-            std::string stylePath = localizeURL(operationArray[1].GetString());
+            std::string stylePath = manifest.localizeURL(operationArray[1].GetString());
             auto maybeStyle = readJson(stylePath);
             if (maybeStyle.is<mbgl::JSDocument>()) {
                 auto& style = maybeStyle.get<mbgl::JSDocument>();
-                localizeStyleURLs((mbgl::JSValue&)style, style);
+                manifest.localizeStyleURLs((mbgl::JSValue&)style, style);
                 map.getStyle().loadJSON(serializeJsonValue(style));
             }
         } else {
-            localizeStyleURLs(operationArray[1], metadata.document);
+            manifest.localizeStyleURLs(operationArray[1], metadata.document);
             map.getStyle().loadJSON(serializeJsonValue(operationArray[1]));
         }
-
-    // setCenter
     } else if (operationArray[0].GetString() == setCenterOp) {
+        // setCenter
         assert(operationArray.Size() >= 2u);
         assert(operationArray[1].IsArray());
 
@@ -294,21 +636,23 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         assert(centerArray.Size() == 2u);
 
         map.jumpTo(mbgl::CameraOptions().withCenter(mbgl::LatLng(centerArray[1].GetDouble(), centerArray[0].GetDouble())));
-
-    // setZoom
     } else if (operationArray[0].GetString() == setZoomOp) {
+        // setZoom
         assert(operationArray.Size() >= 2u);
         assert(operationArray[1].IsNumber());
         map.jumpTo(mbgl::CameraOptions().withZoom(operationArray[1].GetDouble()));
-
-    // setBearing
     } else if (operationArray[0].GetString() == setBearingOp) {
+        // setBearing
         assert(operationArray.Size() >= 2u);
         assert(operationArray[1].IsNumber());
         map.jumpTo(mbgl::CameraOptions().withBearing(operationArray[1].GetDouble()));
-
-    // setFilter
+    } else if (operationArray[0].GetString() == setPitchOp) {
+        // setPitch
+        assert(operationArray.Size() >= 2u);
+        assert(operationArray[1].IsNumber());
+        map.jumpTo(mbgl::CameraOptions().withPitch(operationArray[1].GetDouble()));
     } else if (operationArray[0].GetString() == setFilterOp) {
+        // setFilter
         assert(operationArray.Size() >= 3u);
         assert(operationArray[1].IsString());
 
@@ -328,9 +672,8 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
                 layer->setFilter(std::move(*converted));
             }
         }
-
-    // setLayerZoomRange
     } else if (operationArray[0].GetString() == setLayerZoomRangeOp) {
+        // setLayerZoomRange
         assert(operationArray.Size() >= 4u);
         assert(operationArray[1].IsString());
         assert(operationArray[2].IsNumber());
@@ -345,9 +688,8 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
             layer->setMinZoom(operationArray[2].GetFloat());
             layer->setMaxZoom(operationArray[3].GetFloat());
         }
-
-    // setLight
     } else if (operationArray[0].GetString() == setLightOp) {
+        // setLight
         assert(operationArray.Size() >= 2u);
         assert(operationArray[1].IsObject());
 
@@ -359,9 +701,8 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         } else {
             map.getStyle().setLight(std::make_unique<mbgl::style::Light>(std::move(*converted)));
         }
-
-    // addLayer
     } else if (operationArray[0].GetString() == addLayerOp) {
+        // addLayer
         assert(operationArray.Size() >= 2u);
         assert(operationArray[1].IsObject());
 
@@ -373,20 +714,18 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         } else {
             map.getStyle().addLayer(std::move(*converted));
         }
-
-    // removeLayer
     } else if (operationArray[0].GetString() == removeLayerOp) {
+        // removeLayer
         assert(operationArray.Size() >= 2u);
         assert(operationArray[1].IsString());
         map.getStyle().removeLayer(operationArray[1].GetString());
-
-    // addSource
     } else if (operationArray[0].GetString() == addSourceOp) {
+        // addSource
         assert(operationArray.Size() >= 3u);
         assert(operationArray[1].IsString());
         assert(operationArray[2].IsObject());
 
-        localizeSourceURLs(operationArray[2], metadata.document);
+        manifest.localizeSourceURLs(operationArray[2], metadata.document);
 
         mbgl::style::conversion::Error error;
         auto converted = mbgl::style::conversion::convert<std::unique_ptr<mbgl::style::Source>>(operationArray[2], error, operationArray[1].GetString());
@@ -396,15 +735,13 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         } else {
             map.getStyle().addSource(std::move(*converted));
         }
-
-    // removeSource
     } else if (operationArray[0].GetString() == removeSourceOp) {
+        // removeSource
         assert(operationArray.Size() >= 2u);
         assert(operationArray[1].IsString());
         map.getStyle().removeSource(operationArray[1].GetString());
-
-    // setPaintProperty
     } else if (operationArray[0].GetString() == setPaintPropertyOp) {
+        // setPaintProperty
         assert(operationArray.Size() >= 4u);
         assert(operationArray[1].IsString());
         assert(operationArray[2].IsString());
@@ -420,9 +757,8 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
             const mbgl::JSValue* propertyValue = &operationArray[3];
             layer->setPaintProperty(propertyName, propertyValue);
         }
-
-    // setLayoutProperty
     } else if (operationArray[0].GetString() == setLayoutPropertyOp) {
+        // setLayoutProperty
         assert(operationArray.Size() >= 4u);
         assert(operationArray[1].IsString());
         assert(operationArray[2].IsString());
@@ -438,28 +774,79 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
             const mbgl::JSValue* propertyValue = &operationArray[3];
             layer->setLayoutProperty(propertyName, propertyValue);
         }
-    // probeMemoryStart
+    } else if (operationArray[0].GetString() == fileSizeProbeOp) {
+        // probeFileSize
+        assert(operationArray.Size() >= 4u);
+        assert(operationArray[1].IsString());
+        assert(operationArray[2].IsString());
+        assert(operationArray[3].IsNumber());
+
+        std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
+        std::string path = std::string(operationArray[2].GetString(), operationArray[2].GetStringLength());
+        assert(!path.empty());
+
+        float tolerance = operationArray[3].GetDouble();
+        mbgl::filesystem::path filePath(path);
+
+        if (!filePath.is_absolute()) {
+            filePath = metadata.paths.defaultExpectations() / filePath;
+        }
+
+        if (mbgl::filesystem::exists(filePath)) {
+            auto size = mbgl::filesystem::file_size(filePath);
+            metadata.metrics.fileSize.emplace(std::piecewise_construct,
+                                              std::forward_as_tuple(std::move(mark)),
+                                              std::forward_as_tuple(std::move(path), size, tolerance));
+        } else {
+            metadata.errorMessage = std::string("File not found: ") + path;
+            return false;
+        }
     } else if (operationArray[0].GetString() == memoryProbeStartOp) {
+        // probeMemoryStart
         assert(!AllocationIndex::isActive());
         AllocationIndex::setActive(true);
-    // probeMemory
     } else if (operationArray[0].GetString() == memoryProbeOp) {
+        // probeMemory
         assert(AllocationIndex::isActive());
         assert(operationArray.Size() >= 2u);
         assert(operationArray[1].IsString());
         std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
 
-        metadata.metrics.memory.emplace(std::piecewise_construct,
-                                        std::forward_as_tuple(std::move(mark)), 
-                                        std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(), AllocationIndex::getAllocationsCount()));
-    // probeMemoryEnd
+        auto emplaced = metadata.metrics.memory.emplace(
+            std::piecewise_construct,
+            std::forward_as_tuple(std::move(mark)),
+            std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(), AllocationIndex::getAllocationsCount()));
+        assert(emplaced.second);
+        if (operationArray.Size() >= 3u) {
+            assert(operationArray[2].IsNumber());
+            emplaced.first->second.tolerance = float(operationArray[2].GetDouble());
+        }
     } else if (operationArray[0].GetString() == memoryProbeEndOp) {
+        // probeMemoryEnd
         assert(AllocationIndex::isActive());
         AllocationIndex::setActive(false);
         AllocationIndex::reset();
+    } else if (operationArray[0].GetString() == networkProbeStartOp) {
+        // probeNetworkStart
+        assert(!ProxyFileSource::isTrackingActive());
+        ProxyFileSource::setTrackingActive(true);
+    } else if (operationArray[0].GetString() == networkProbeOp) {
+        // probeNetwork
+        assert(ProxyFileSource::isTrackingActive());
+        assert(operationArray.Size() >= 2u);
+        assert(operationArray[1].IsString());
+        std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
 
-        // setFeatureState
+        metadata.metrics.network.emplace(
+            std::piecewise_construct,
+            std::forward_as_tuple(std::move(mark)),
+            std::forward_as_tuple(ProxyFileSource::getRequestCount(), ProxyFileSource::getTransferredSize()));
+    } else if (operationArray[0].GetString() == networkProbeEndOp) {
+        // probeNetworkEnd
+        assert(ProxyFileSource::isTrackingActive());
+        ProxyFileSource::setTrackingActive(false);
     } else if (operationArray[0].GetString() == setFeatureStateOp) {
+        // setFeatureState
         assert(operationArray.Size() >= 3u);
         assert(operationArray[1].IsObject());
         assert(operationArray[2].IsObject());
@@ -483,7 +870,11 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
             sourceLayer = {featureOptions["sourceLayer"].GetString()};
         }
         if (featureOptions.HasMember("id")) {
-            featureID = featureOptions["id"].GetString();
+            if (featureOptions["id"].IsString()) {
+                featureID = featureOptions["id"].GetString();
+            } else if (featureOptions["id"].IsNumber()) {
+                featureID = mbgl::util::toString(featureOptions["id"].GetUint64());
+            }
         }
         const JSValue* state = &operationArray[2];
 
@@ -507,7 +898,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
                 result[k] = std::move(array);
                 stateValue = std::move(result);
                 valueParsed = true;
-                return {};
+                return nullopt;
 
             } else if (isObject(v)) {
                 eachMember(v, convertFn);
@@ -530,9 +921,8 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
             return false;
         }
         frontend.getRenderer()->setFeatureState(sourceID, sourceLayer, featureID, parsedState);
-
-        // getFeatureState
     } else if (operationArray[0].GetString() == getFeatureStateOp) {
+        // getFeatureState
         assert(operationArray.Size() >= 2u);
         assert(operationArray[1].IsObject());
 
@@ -548,7 +938,11 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
             sourceLayer = {featureOptions["sourceLayer"].GetString()};
         }
         if (featureOptions.HasMember("id")) {
-            featureID = featureOptions["id"].GetString();
+            if (featureOptions["id"].IsString()) {
+                featureID = featureOptions["id"].GetString();
+            } else if (featureOptions["id"].IsNumber()) {
+                featureID = mbgl::util::toString(featureOptions["id"].GetUint64());
+            }
         }
 
         try {
@@ -558,9 +952,8 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         }
         mbgl::FeatureState state;
         frontend.getRenderer()->getFeatureState(state, sourceID, sourceLayer, featureID);
-
-        // removeFeatureState
     } else if (operationArray[0].GetString() == removeFeatureStateOp) {
+        // removeFeatureState
         assert(operationArray.Size() >= 2u);
         assert(operationArray[1].IsObject());
 
@@ -577,7 +970,11 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
             sourceLayer = {featureOptions["sourceLayer"].GetString()};
         }
         if (featureOptions.HasMember("id")) {
-            featureID = featureOptions["id"].GetString();
+            if (featureOptions["id"].IsString()) {
+                featureID = featureOptions["id"].GetString();
+            } else if (featureOptions["id"].IsNumber()) {
+                featureID = mbgl::util::toString(featureOptions["id"].GetUint64());
+            }
         }
 
         if (operationArray.Size() >= 3u) {
@@ -591,20 +988,122 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
             return false;
         }
         frontend.getRenderer()->removeFeatureState(sourceID, sourceLayer, featureID, stateKey);
+    } else if (operationArray[0].GetString() == panGestureOp) {
+        // benchmarkPanGesture
+        assert(operationArray.Size() >= 4u);
+        assert(operationArray[1].IsString()); // identifier
+        assert(operationArray[2].IsNumber()); // duration
+        assert(operationArray[3].IsArray());  // start [lat, lng, zoom]
+        assert(operationArray[4].IsArray());  // end [lat, lng, zoom]
+
+        if (metadata.mapMode != mbgl::MapMode::Continuous) {
+            metadata.errorMessage = "Map mode must be Continous for " + panGestureOp + " operation";
+            return false;
+        }
+
+        std::string mark = operationArray[1].GetString();
+        int duration = operationArray[2].GetFloat();
+        LatLng startPos, endPos;
+        double startZoom, endZoom;
+        std::vector<float> samples;
+
+        auto parsePosition = [](auto arr) -> std::tuple<LatLng, double> {
+            assert(arr.Size() >= 3);
+            return {{arr[1].GetDouble(), arr[0].GetDouble()}, arr[2].GetDouble()};
+        };
+
+        std::tie(startPos, startZoom) = parsePosition(operationArray[3].GetArray());
+        std::tie(endPos, endZoom) = parsePosition(operationArray[4].GetArray());
+
+        // Jump to the starting point of the segment and make sure there's something to render
+        map.jumpTo(mbgl::CameraOptions().withCenter(startPos).withZoom(startZoom));
+
+        observer->reset();
+        while (!observer->finishRenderingMap) {
+            frontend.renderOnce(map);
+        }
+
+        if (observer->mapLoadFailure) return false;
+
+        size_t frames = 0;
+        float totalTime = 0.0;
+        bool transitionFinished = false;
+
+        mbgl::AnimationOptions animationOptions(mbgl::Milliseconds(duration * 1000));
+        animationOptions.minZoom = util::min(startZoom, endZoom);
+        animationOptions.transitionFinishFn = [&]() { transitionFinished = true; };
+
+        map.flyTo(mbgl::CameraOptions().withCenter(endPos).withZoom(endZoom), animationOptions);
+
+        for (; !transitionFinished; frames++) {
+            frontend.renderOnce(map);
+            float frameTime = (float)frontend.getFrameTime();
+            totalTime += frameTime;
+
+            samples.push_back(frameTime);
+        }
+
+        float averageFps = totalTime > 0.0 ? frames / totalTime : 0.0;
+        float minFrameTime = 0.0;
+
+        // Use 1% of the longest frames to compute the minimum fps
+        std::sort(samples.begin(), samples.end());
+
+        int sampleCount = util::max(1, (int)samples.size() / 100);
+        for (auto it = samples.rbegin(); it != samples.rbegin() + sampleCount; it++) minFrameTime += *it;
+
+        float minOnePcFps = sampleCount / minFrameTime;
+
+        metadata.metrics.fps.insert({std::move(mark), {averageFps, minOnePcFps, 0.0f}});
+
+    } else if (operationArray[0].GetString() == gfxProbeStartOp) {
+        // probeGFXStart
+        assert(!ctx.gfxProbeActive);
+        ctx.gfxProbeActive = true;
+        ctx.baselineGfxProbe = ctx.activeGfxProbe;
+    } else if (operationArray[0].GetString() == gfxProbeEndOp) {
+        // probeGFXEnd
+        assert(ctx.gfxProbeActive);
+        ctx.gfxProbeActive = false;
+    } else if (operationArray[0].GetString() == gfxProbeOp) {
+        // probeGFX
+        assert(operationArray.Size() >= 2u);
+        assert(operationArray[1].IsString());
+
+        std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
+
+        // Render the map and fetch rendering stats
+        gfx::RenderingStats stats;
+
+        try {
+            stats = frontend.render(map).stats;
+        } catch (const std::exception&) {
+            return false;
+        }
+
+        ctx.activeGfxProbe = GfxProbe(stats, ctx.activeGfxProbe);
+
+        // Compare memory allocations to the baseline probe
+        GfxProbe metricProbe = ctx.activeGfxProbe;
+        metricProbe.memIndexBuffers.peak -= ctx.baselineGfxProbe.memIndexBuffers.peak;
+        metricProbe.memVertexBuffers.peak -= ctx.baselineGfxProbe.memVertexBuffers.peak;
+        metricProbe.memTextures.peak -= ctx.baselineGfxProbe.memTextures.peak;
+        metadata.metrics.gfx.insert({mark, metricProbe});
 
     } else {
-        metadata.errorMessage = std::string("Unsupported operation: ")  + operationArray[0].GetString();
+        metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString();
         return false;
     }
 
     operationsArray.Erase(operationIt);
-    return runOperations(key, metadata);
+    return runOperations(key, metadata, ctx);
 }
 
 TestRunner::Impl::Impl(const TestMetadata& metadata)
-    : frontend(metadata.size, metadata.pixelRatio),
+    : observer(std::make_unique<TestRunnerMapObserver>()),
+      frontend(metadata.size, metadata.pixelRatio, swapBehavior(metadata.mapMode)),
       map(frontend,
-          mbgl::MapObserver::nullObserver(),
+          *observer.get(),
           mbgl::MapOptions()
               .withMapMode(metadata.mapMode)
               .withSize(metadata.size)
@@ -612,9 +1111,12 @@ TestRunner::Impl::Impl(const TestMetadata& metadata)
               .withCrossSourceCollisions(metadata.crossSourceCollisions),
           mbgl::ResourceOptions().withCacheOnlyRequestsSupport(false)) {}
 
+TestRunner::Impl::~Impl() {}
+
 bool TestRunner::run(TestMetadata& metadata) {
     AllocationIndex::setActive(false);
     AllocationIndex::reset();
+    ProxyFileSource::setTrackingActive(false);
     std::string key = mbgl::util::toString(uint32_t(metadata.mapMode))
         + "/" + mbgl::util::toString(metadata.pixelRatio)
         + "/" + mbgl::util::toString(uint32_t(metadata.crossSourceCollisions));
@@ -635,18 +1137,32 @@ bool TestRunner::run(TestMetadata& metadata) {
     map.getStyle().loadJSON(serializeJsonValue(metadata.document));
     map.jumpTo(map.getStyle().getDefaultCamera());
 
-    if (!runOperations(key, metadata)) {
+    RunContext ctx{};
+
+    if (!runOperations(key, metadata, ctx)) {
         return false;
     }
 
     mbgl::PremultipliedImage image;
     try {
-        image = frontend.render(map);
+        if (metadata.outputsImage) image = frontend.render(map).image;
     } catch (const std::exception&) {
         return false;
     }
 
-    return checkResults(std::move(image), metadata);
+    if (metadata.renderTest) {
+        return checkRenderTestResults(std::move(image), metadata);
+    } else {
+        std::vector<mbgl::Feature> features;
+        assert(metadata.document["metadata"]["test"]["queryGeometry"].IsArray());
+        if (metadata.document["metadata"]["test"]["queryGeometry"][0].IsNumber() &&
+            metadata.document["metadata"]["test"]["queryGeometry"][1].IsNumber()) {
+            features = frontend.getRenderer()->queryRenderedFeatures(metadata.queryGeometry, metadata.queryOptions);
+        } else {
+            features = frontend.getRenderer()->queryRenderedFeatures(metadata.queryGeometryBox, metadata.queryOptions);
+        }
+        return checkQueryTestResults(std::move(image), std::move(features), metadata);
+    }
 }
 
 void TestRunner::reset() {
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
index fdea65e104..4f80286c0b 100644
--- a/render-test/runner.hpp
+++ b/render-test/runner.hpp
@@ -3,31 +3,40 @@
 #include <mbgl/gfx/headless_frontend.hpp>
 #include <mbgl/map/map.hpp>
 
+#include "manifest_parser.hpp"
+
 #include <memory>
+#include <string>
 
+struct RunContext;
+class TestRunnerMapObserver;
 struct TestMetadata;
 
 class TestRunner {
 public:
-    TestRunner() = default;
-
+    explicit TestRunner(Manifest);
     bool run(TestMetadata&);
     void reset();
 
-    /// Returns path of the render tests root directory.
-    static const std::string& getBasePath();
-    /// Returns path of mapbox-gl-native expectations directory.
-    static const std::vector<std::string>& getPlatformExpectationsPaths();
+    // Manifest
+    const Manifest& getManifest() const;
+    void doShuffle(uint32_t seed);
 
 private:
-    bool runOperations(const std::string& key, TestMetadata&);
-    bool checkResults(mbgl::PremultipliedImage&& image, TestMetadata&);
+    bool runOperations(const std::string& key, TestMetadata&, RunContext&);
+    bool checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
+                               std::vector<mbgl::Feature>&& features,
+                               TestMetadata&);
+    bool checkRenderTestResults(mbgl::PremultipliedImage&& image, TestMetadata&);
 
     struct Impl {
         Impl(const TestMetadata&);
+        ~Impl();
 
+        std::unique_ptr<TestRunnerMapObserver> observer;
         mbgl::HeadlessFrontend frontend;
         mbgl::Map map;
     };
     std::unordered_map<std::string, std::unique_ptr<Impl>> maps;
-};
-\ No newline at end of file
+    Manifest manifest;
+};
diff --git a/render-test/tests/file-size/fail-file-doesnt-match/expected.png b/render-test/tests/file-size/fail-file-doesnt-match/expected.png
new file mode 100644
index 0000000000..83d01c4e5d
--- /dev/null
+++ b/render-test/tests/file-size/fail-file-doesnt-match/expected.png
diff --git a/render-test/tests/file-size/fail-file-doesnt-match/metrics.json b/render-test/tests/file-size/fail-file-doesnt-match/metrics.json
new file mode 100644
index 0000000000..c0002163d6
--- /dev/null
+++ b/render-test/tests/file-size/fail-file-doesnt-match/metrics.json
@@ -0,0 +1,14 @@
+{
+    "file-size": [
+        [
+            "image",
+            "expected.aaa",
+            169
+        ],
+        [
+            "style",
+            "style.bbbb",
+            510
+        ]
+    ]
+}
diff --git a/render-test/tests/file-size/fail-file-doesnt-match/style.json b/render-test/tests/file-size/fail-file-doesnt-match/style.json
new file mode 100644
index 0000000000..839a8d4a12
--- /dev/null
+++ b/render-test/tests/file-size/fail-file-doesnt-match/style.json
@@ -0,0 +1,32 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeFileSize", "style", "style.json", 0],
+        ["probeFileSize", "image", "expected.png", 0]
+      ],
+      "width": 64,
+      "height": 64
+    }
+  },
+  "sources": {
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Point",
+        "coordinates": [
+          0,
+          0
+        ]
+      }
+    }
+  },
+  "layers": [
+    {
+      "id": "circle",
+      "type": "circle",
+      "source": "geojson"
+    }
+  ]
+}
diff --git a/render-test/tests/file-size/fail-file-not-found/expected.png b/render-test/tests/file-size/fail-file-not-found/expected.png
new file mode 100644
index 0000000000..83d01c4e5d
--- /dev/null
+++ b/render-test/tests/file-size/fail-file-not-found/expected.png
diff --git a/render-test/tests/file-size/fail-file-not-found/style.json b/render-test/tests/file-size/fail-file-not-found/style.json
new file mode 100644
index 0000000000..74054e1f40
--- /dev/null
+++ b/render-test/tests/file-size/fail-file-not-found/style.json
@@ -0,0 +1,32 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeFileSize", "style", "style.aaaa", 0],
+        ["probeFileSize", "image", "expected.bbb", 0]
+      ],
+      "width": 64,
+      "height": 64
+    }
+  },
+  "sources": {
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Point",
+        "coordinates": [
+          0,
+          0
+        ]
+      }
+    }
+  },
+  "layers": [
+    {
+      "id": "circle",
+      "type": "circle",
+      "source": "geojson"
+    }
+  ]
+}
diff --git a/render-test/tests/file-size/fail-size-is-over/expected.png b/render-test/tests/file-size/fail-size-is-over/expected.png
new file mode 100644
index 0000000000..83d01c4e5d
--- /dev/null
+++ b/render-test/tests/file-size/fail-size-is-over/expected.png
diff --git a/render-test/tests/file-size/fail-size-is-over/metrics.json b/render-test/tests/file-size/fail-size-is-over/metrics.json
new file mode 100644
index 0000000000..bc194081cf
--- /dev/null
+++ b/render-test/tests/file-size/fail-size-is-over/metrics.json
@@ -0,0 +1,14 @@
+{
+    "file-size": [
+        [
+            "image",
+            "expected.png",
+            999
+        ],
+        [
+            "style",
+            "style.json",
+            999
+        ]
+    ]
+}
diff --git a/render-test/tests/file-size/fail-size-is-over/style.json b/render-test/tests/file-size/fail-size-is-over/style.json
new file mode 100644
index 0000000000..839a8d4a12
--- /dev/null
+++ b/render-test/tests/file-size/fail-size-is-over/style.json
@@ -0,0 +1,32 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeFileSize", "style", "style.json", 0],
+        ["probeFileSize", "image", "expected.png", 0]
+      ],
+      "width": 64,
+      "height": 64
+    }
+  },
+  "sources": {
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Point",
+        "coordinates": [
+          0,
+          0
+        ]
+      }
+    }
+  },
+  "layers": [
+    {
+      "id": "circle",
+      "type": "circle",
+      "source": "geojson"
+    }
+  ]
+}
diff --git a/render-test/tests/file-size/fail-size-is-under/expected.png b/render-test/tests/file-size/fail-size-is-under/expected.png
new file mode 100644
index 0000000000..83d01c4e5d
--- /dev/null
+++ b/render-test/tests/file-size/fail-size-is-under/expected.png
diff --git a/render-test/tests/file-size/fail-size-is-under/metrics.json b/render-test/tests/file-size/fail-size-is-under/metrics.json
new file mode 100644
index 0000000000..d288c2ceee
--- /dev/null
+++ b/render-test/tests/file-size/fail-size-is-under/metrics.json
@@ -0,0 +1,14 @@
+{
+    "file-size": [
+        [
+            "image",
+            "expected.png",
+            100
+        ],
+        [
+            "style",
+            "style.json",
+            100
+        ]
+    ]
+}
diff --git a/render-test/tests/file-size/fail-size-is-under/style.json b/render-test/tests/file-size/fail-size-is-under/style.json
new file mode 100644
index 0000000000..839a8d4a12
--- /dev/null
+++ b/render-test/tests/file-size/fail-size-is-under/style.json
@@ -0,0 +1,32 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeFileSize", "style", "style.json", 0],
+        ["probeFileSize", "image", "expected.png", 0]
+      ],
+      "width": 64,
+      "height": 64
+    }
+  },
+  "sources": {
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Point",
+        "coordinates": [
+          0,
+          0
+        ]
+      }
+    }
+  },
+  "layers": [
+    {
+      "id": "circle",
+      "type": "circle",
+      "source": "geojson"
+    }
+  ]
+}
diff --git a/render-test/tests/file-size/pass-size-is-in-tolerance-higher/expected.png b/render-test/tests/file-size/pass-size-is-in-tolerance-higher/expected.png
new file mode 100644
index 0000000000..83d01c4e5d
--- /dev/null
+++ b/render-test/tests/file-size/pass-size-is-in-tolerance-higher/expected.png
diff --git a/render-test/tests/file-size/pass-size-is-in-tolerance-higher/metrics.json b/render-test/tests/file-size/pass-size-is-in-tolerance-higher/metrics.json
new file mode 100644
index 0000000000..d0c0c9d7b6
--- /dev/null
+++ b/render-test/tests/file-size/pass-size-is-in-tolerance-higher/metrics.json
@@ -0,0 +1,14 @@
+{
+    "file-size": [
+        [
+            "image",
+            "expected.png",
+            177
+        ],
+        [
+            "style",
+            "style.json",
+            548
+        ]
+    ]
+}
diff --git a/render-test/tests/file-size/pass-size-is-in-tolerance-higher/style.json b/render-test/tests/file-size/pass-size-is-in-tolerance-higher/style.json
new file mode 100644
index 0000000000..d021673ba5
--- /dev/null
+++ b/render-test/tests/file-size/pass-size-is-in-tolerance-higher/style.json
@@ -0,0 +1,32 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeFileSize", "style", "style.json", 0.05],
+        ["probeFileSize", "image", "expected.png", 0.05]
+      ],
+      "width": 64,
+      "height": 64
+    }
+  },
+  "sources": {
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Point",
+        "coordinates": [
+          0,
+          0
+        ]
+      }
+    }
+  },
+  "layers": [
+    {
+      "id": "circle",
+      "type": "circle",
+      "source": "geojson"
+    }
+  ]
+}
diff --git a/render-test/tests/file-size/pass-size-is-in-tolerance-lower/expected.png b/render-test/tests/file-size/pass-size-is-in-tolerance-lower/expected.png
new file mode 100644
index 0000000000..83d01c4e5d
--- /dev/null
+++ b/render-test/tests/file-size/pass-size-is-in-tolerance-lower/expected.png
diff --git a/render-test/tests/file-size/pass-size-is-in-tolerance-lower/metrics.json b/render-test/tests/file-size/pass-size-is-in-tolerance-lower/metrics.json
new file mode 100644
index 0000000000..d62b751d5a
--- /dev/null
+++ b/render-test/tests/file-size/pass-size-is-in-tolerance-lower/metrics.json
@@ -0,0 +1,14 @@
+{
+    "file-size": [
+        [
+            "image",
+            "expected.png",
+            161
+        ],
+        [
+            "style",
+            "style.json",
+            498
+        ]
+    ]
+}
diff --git a/render-test/tests/file-size/pass-size-is-in-tolerance-lower/style.json b/render-test/tests/file-size/pass-size-is-in-tolerance-lower/style.json
new file mode 100644
index 0000000000..d021673ba5
--- /dev/null
+++ b/render-test/tests/file-size/pass-size-is-in-tolerance-lower/style.json
@@ -0,0 +1,32 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeFileSize", "style", "style.json", 0.05],
+        ["probeFileSize", "image", "expected.png", 0.05]
+      ],
+      "width": 64,
+      "height": 64
+    }
+  },
+  "sources": {
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Point",
+        "coordinates": [
+          0,
+          0
+        ]
+      }
+    }
+  },
+  "layers": [
+    {
+      "id": "circle",
+      "type": "circle",
+      "source": "geojson"
+    }
+  ]
+}
diff --git a/render-test/tests/file-size/pass-size-is-same/expected.png b/render-test/tests/file-size/pass-size-is-same/expected.png
new file mode 100644
index 0000000000..83d01c4e5d
--- /dev/null
+++ b/render-test/tests/file-size/pass-size-is-same/expected.png
diff --git a/render-test/tests/file-size/pass-size-is-same/metrics.json b/render-test/tests/file-size/pass-size-is-same/metrics.json
new file mode 100644
index 0000000000..3d560bd610
--- /dev/null
+++ b/render-test/tests/file-size/pass-size-is-same/metrics.json
@@ -0,0 +1,14 @@
+{
+    "file-size": [
+        [
+            "image",
+            "expected.png",
+            169
+        ],
+        [
+            "style",
+            "style.json",
+            516
+        ]
+    ]
+}
+\ No newline at end of file
diff --git a/render-test/tests/file-size/pass-size-is-same/style.json b/render-test/tests/file-size/pass-size-is-same/style.json
new file mode 100644
index 0000000000..839a8d4a12
--- /dev/null
+++ b/render-test/tests/file-size/pass-size-is-same/style.json
@@ -0,0 +1,32 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeFileSize", "style", "style.json", 0],
+        ["probeFileSize", "image", "expected.png", 0]
+      ],
+      "width": 64,
+      "height": 64
+    }
+  },
+  "sources": {
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Point",
+        "coordinates": [
+          0,
+          0
+        ]
+      }
+    }
+  },
+  "layers": [
+    {
+      "id": "circle",
+      "type": "circle",
+      "source": "geojson"
+    }
+  ]
+}
diff --git a/render-test/tests/gfx/fail-ib-mem-mismatch/expected.png b/render-test/tests/gfx/fail-ib-mem-mismatch/expected.png
new file mode 100644
index 0000000000..4b5ea75a25
--- /dev/null
+++ b/render-test/tests/gfx/fail-ib-mem-mismatch/expected.png
diff --git a/render-test/tests/gfx/fail-ib-mem-mismatch/metrics.json b/render-test/tests/gfx/fail-ib-mem-mismatch/metrics.json
new file mode 100644
index 0000000000..2b5c9c3eda
--- /dev/null
+++ b/render-test/tests/gfx/fail-ib-mem-mismatch/metrics.json
@@ -0,0 +1,5 @@
+{
+    "gfx":[
+        ["gfx 0", 36, 14, 63, 1, [371208, 371208], [68598, 65536], [74592, 74592]]
+    ]
+} 
diff --git a/render-test/tests/gfx/fail-ib-mem-mismatch/style.json b/render-test/tests/gfx/fail-ib-mem-mismatch/style.json
new file mode 100644
index 0000000000..6ae3eecb8c
--- /dev/null
+++ b/render-test/tests/gfx/fail-ib-mem-mismatch/style.json
@@ -0,0 +1,20 @@
+{
+    "version": 8,
+    "metadata": {
+      "test": {
+        "width": 512,
+        "height": 512,
+        "operations": [
+          ["setStyle", "local://styles/uruguay.json"],
+          ["setZoom", 9 ],
+          ["probeGFXStart"],
+          ["setCenter", [-56.509552, -32.865788] ],
+          ["probeGFX", "gfx 0"],
+          ["probeGFXEnd"]
+        ]
+      }
+    },
+    "sources": {},
+    "layers": []
+  }
+  
+\ No newline at end of file
diff --git a/render-test/tests/gfx/fail-negative-framebuffer-count/expected.png b/render-test/tests/gfx/fail-negative-framebuffer-count/expected.png
new file mode 100644
index 0000000000..4b5ea75a25
--- /dev/null
+++ b/render-test/tests/gfx/fail-negative-framebuffer-count/expected.png
diff --git a/render-test/tests/gfx/fail-negative-framebuffer-count/metrics.json b/render-test/tests/gfx/fail-negative-framebuffer-count/metrics.json
new file mode 100644
index 0000000000..e92cffa571
--- /dev/null
+++ b/render-test/tests/gfx/fail-negative-framebuffer-count/metrics.json
@@ -0,0 +1,5 @@
+{
+    "gfx":[
+        ["gfx 0", 36, 14, 63, -1, [371208, 371208], [68598, 68598], [74592, 74592]]
+    ]
+} 
diff --git a/render-test/tests/gfx/fail-negative-framebuffer-count/style.json b/render-test/tests/gfx/fail-negative-framebuffer-count/style.json
new file mode 100644
index 0000000000..6ae3eecb8c
--- /dev/null
+++ b/render-test/tests/gfx/fail-negative-framebuffer-count/style.json
@@ -0,0 +1,20 @@
+{
+    "version": 8,
+    "metadata": {
+      "test": {
+        "width": 512,
+        "height": 512,
+        "operations": [
+          ["setStyle", "local://styles/uruguay.json"],
+          ["setZoom", 9 ],
+          ["probeGFXStart"],
+          ["setCenter", [-56.509552, -32.865788] ],
+          ["probeGFX", "gfx 0"],
+          ["probeGFXEnd"]
+        ]
+      }
+    },
+    "sources": {},
+    "layers": []
+  }
+  
+\ No newline at end of file
diff --git a/render-test/tests/gfx/fail-texture-mem-mismatch/expected.png b/render-test/tests/gfx/fail-texture-mem-mismatch/expected.png
new file mode 100644
index 0000000000..4b5ea75a25
--- /dev/null
+++ b/render-test/tests/gfx/fail-texture-mem-mismatch/expected.png
diff --git a/render-test/tests/gfx/fail-texture-mem-mismatch/metrics.json b/render-test/tests/gfx/fail-texture-mem-mismatch/metrics.json
new file mode 100644
index 0000000000..fc6f55025f
--- /dev/null
+++ b/render-test/tests/gfx/fail-texture-mem-mismatch/metrics.json
@@ -0,0 +1,5 @@
+{
+    "gfx":[
+        ["gfx 0", 36, 14, 63, 1, [371208, 370000], [68598, 68598], [74592, 74592]]
+    ]
+} 
diff --git a/render-test/tests/gfx/fail-texture-mem-mismatch/style.json b/render-test/tests/gfx/fail-texture-mem-mismatch/style.json
new file mode 100644
index 0000000000..6ae3eecb8c
--- /dev/null
+++ b/render-test/tests/gfx/fail-texture-mem-mismatch/style.json
@@ -0,0 +1,20 @@
+{
+    "version": 8,
+    "metadata": {
+      "test": {
+        "width": 512,
+        "height": 512,
+        "operations": [
+          ["setStyle", "local://styles/uruguay.json"],
+          ["setZoom", 9 ],
+          ["probeGFXStart"],
+          ["setCenter", [-56.509552, -32.865788] ],
+          ["probeGFX", "gfx 0"],
+          ["probeGFXEnd"]
+        ]
+      }
+    },
+    "sources": {},
+    "layers": []
+  }
+  
+\ No newline at end of file
diff --git a/render-test/tests/gfx/fail-too-few-buffers/expected.png b/render-test/tests/gfx/fail-too-few-buffers/expected.png
new file mode 100644
index 0000000000..4b5ea75a25
--- /dev/null
+++ b/render-test/tests/gfx/fail-too-few-buffers/expected.png
diff --git a/render-test/tests/gfx/fail-too-few-buffers/metrics.json b/render-test/tests/gfx/fail-too-few-buffers/metrics.json
new file mode 100644
index 0000000000..07b59adeed
--- /dev/null
+++ b/render-test/tests/gfx/fail-too-few-buffers/metrics.json
@@ -0,0 +1,5 @@
+{
+    "gfx":[
+        ["gfx 0", 36, 14, 1000, 1, [371208, 371208], [68598, 68598], [74592, 74592]]
+    ]
+} 
diff --git a/render-test/tests/gfx/fail-too-few-buffers/style.json b/render-test/tests/gfx/fail-too-few-buffers/style.json
new file mode 100644
index 0000000000..6ae3eecb8c
--- /dev/null
+++ b/render-test/tests/gfx/fail-too-few-buffers/style.json
@@ -0,0 +1,20 @@
+{
+    "version": 8,
+    "metadata": {
+      "test": {
+        "width": 512,
+        "height": 512,
+        "operations": [
+          ["setStyle", "local://styles/uruguay.json"],
+          ["setZoom", 9 ],
+          ["probeGFXStart"],
+          ["setCenter", [-56.509552, -32.865788] ],
+          ["probeGFX", "gfx 0"],
+          ["probeGFXEnd"]
+        ]
+      }
+    },
+    "sources": {},
+    "layers": []
+  }
+  
+\ No newline at end of file
diff --git a/render-test/tests/gfx/fail-too-few-textures/expected.png b/render-test/tests/gfx/fail-too-few-textures/expected.png
new file mode 100644
index 0000000000..4b5ea75a25
--- /dev/null
+++ b/render-test/tests/gfx/fail-too-few-textures/expected.png
diff --git a/render-test/tests/gfx/fail-too-few-textures/metrics.json b/render-test/tests/gfx/fail-too-few-textures/metrics.json
new file mode 100644
index 0000000000..805d7035e2
--- /dev/null
+++ b/render-test/tests/gfx/fail-too-few-textures/metrics.json
@@ -0,0 +1,5 @@
+{
+    "gfx":[
+        ["gfx 0", 36, 1000, 63, 1, [371208, 371208], [68598, 68598], [74592, 74592]]
+    ]
+} 
diff --git a/render-test/tests/gfx/fail-too-few-textures/style.json b/render-test/tests/gfx/fail-too-few-textures/style.json
new file mode 100644
index 0000000000..6ae3eecb8c
--- /dev/null
+++ b/render-test/tests/gfx/fail-too-few-textures/style.json
@@ -0,0 +1,20 @@
+{
+    "version": 8,
+    "metadata": {
+      "test": {
+        "width": 512,
+        "height": 512,
+        "operations": [
+          ["setStyle", "local://styles/uruguay.json"],
+          ["setZoom", 9 ],
+          ["probeGFXStart"],
+          ["setCenter", [-56.509552, -32.865788] ],
+          ["probeGFX", "gfx 0"],
+          ["probeGFXEnd"]
+        ]
+      }
+    },
+    "sources": {},
+    "layers": []
+  }
+  
+\ No newline at end of file
diff --git a/render-test/tests/gfx/fail-too-many-drawcalls/expected.png b/render-test/tests/gfx/fail-too-many-drawcalls/expected.png
new file mode 100644
index 0000000000..4b5ea75a25
--- /dev/null
+++ b/render-test/tests/gfx/fail-too-many-drawcalls/expected.png
diff --git a/render-test/tests/gfx/fail-too-many-drawcalls/metrics.json b/render-test/tests/gfx/fail-too-many-drawcalls/metrics.json
new file mode 100644
index 0000000000..9abb546450
--- /dev/null
+++ b/render-test/tests/gfx/fail-too-many-drawcalls/metrics.json
@@ -0,0 +1,5 @@
+{
+    "gfx":[
+        ["gfx 0", 28, 14, 63, 1, [371208, 371208], [68598, 68598], [74592, 74592]]
+    ]
+} 
diff --git a/render-test/tests/gfx/fail-too-many-drawcalls/style.json b/render-test/tests/gfx/fail-too-many-drawcalls/style.json
new file mode 100644
index 0000000000..6ae3eecb8c
--- /dev/null
+++ b/render-test/tests/gfx/fail-too-many-drawcalls/style.json
@@ -0,0 +1,20 @@
+{
+    "version": 8,
+    "metadata": {
+      "test": {
+        "width": 512,
+        "height": 512,
+        "operations": [
+          ["setStyle", "local://styles/uruguay.json"],
+          ["setZoom", 9 ],
+          ["probeGFXStart"],
+          ["setCenter", [-56.509552, -32.865788] ],
+          ["probeGFX", "gfx 0"],
+          ["probeGFXEnd"]
+        ]
+      }
+    },
+    "sources": {},
+    "layers": []
+  }
+  
+\ No newline at end of file
diff --git a/render-test/tests/gfx/fail-vb-mem-mismatch/expected.png b/render-test/tests/gfx/fail-vb-mem-mismatch/expected.png
new file mode 100644
index 0000000000..4b5ea75a25
--- /dev/null
+++ b/render-test/tests/gfx/fail-vb-mem-mismatch/expected.png
diff --git a/render-test/tests/gfx/fail-vb-mem-mismatch/metrics.json b/render-test/tests/gfx/fail-vb-mem-mismatch/metrics.json
new file mode 100644
index 0000000000..fd25dbd68b
--- /dev/null
+++ b/render-test/tests/gfx/fail-vb-mem-mismatch/metrics.json
@@ -0,0 +1,5 @@
+{
+    "gfx":[
+        ["gfx 0", 36, 14, 63, 1, [371208, 371208], [68598, 68598], [74592, 7654321]]
+    ]
+} 
diff --git a/render-test/tests/gfx/fail-vb-mem-mismatch/style.json b/render-test/tests/gfx/fail-vb-mem-mismatch/style.json
new file mode 100644
index 0000000000..6ae3eecb8c
--- /dev/null
+++ b/render-test/tests/gfx/fail-vb-mem-mismatch/style.json
@@ -0,0 +1,20 @@
+{
+    "version": 8,
+    "metadata": {
+      "test": {
+        "width": 512,
+        "height": 512,
+        "operations": [
+          ["setStyle", "local://styles/uruguay.json"],
+          ["setZoom", 9 ],
+          ["probeGFXStart"],
+          ["setCenter", [-56.509552, -32.865788] ],
+          ["probeGFX", "gfx 0"],
+          ["probeGFXEnd"]
+        ]
+      }
+    },
+    "sources": {},
+    "layers": []
+  }
+  
+\ No newline at end of file
diff --git a/render-test/tests/gfx/pass-double-probe/expected.png b/render-test/tests/gfx/pass-double-probe/expected.png
new file mode 100644
index 0000000000..9f4e48ab24
--- /dev/null
+++ b/render-test/tests/gfx/pass-double-probe/expected.png
diff --git a/render-test/tests/gfx/pass-double-probe/metrics.json b/render-test/tests/gfx/pass-double-probe/metrics.json
new file mode 100644
index 0000000000..f2641a9a16
--- /dev/null
+++ b/render-test/tests/gfx/pass-double-probe/metrics.json
@@ -0,0 +1,6 @@
+{
+    "gfx":[
+        ["gfx 0", 36, 13, 63, 1, [240136, 240136], [68598, 68598], [74592, 74592]],
+        ["gfx 1", 32, 17, 85, 1, [325008, 325008], [84926, 84926], [100224, 100224]]
+    ]
+} 
diff --git a/render-test/tests/gfx/pass-double-probe/style.json b/render-test/tests/gfx/pass-double-probe/style.json
new file mode 100644
index 0000000000..395f4ce50a
--- /dev/null
+++ b/render-test/tests/gfx/pass-double-probe/style.json
@@ -0,0 +1,22 @@
+{
+    "version": 8,
+    "metadata": {
+      "test": {
+        "width": 512,
+        "height": 512,
+        "operations": [
+          ["setStyle", "local://styles/uruguay.json"],
+          ["setZoom", 9 ],
+          ["probeGFXStart"],
+          ["setCenter", [-56.509552, -32.865788] ],
+          ["probeGFX", "gfx 0"],
+          ["setCenter", [-56.509552, -32.745788] ],
+          ["probeGFX", "gfx 1"],
+          ["probeGFXEnd"]
+        ]
+      }
+    },
+    "sources": {},
+    "layers": []
+  }
+  
+\ No newline at end of file
diff --git a/render-test/tests/gfx/pass-probe-reset/expected.png b/render-test/tests/gfx/pass-probe-reset/expected.png
new file mode 100644
index 0000000000..9f4e48ab24
--- /dev/null
+++ b/render-test/tests/gfx/pass-probe-reset/expected.png
diff --git a/render-test/tests/gfx/pass-probe-reset/metrics.json b/render-test/tests/gfx/pass-probe-reset/metrics.json
new file mode 100644
index 0000000000..0e42bc1697
--- /dev/null
+++ b/render-test/tests/gfx/pass-probe-reset/metrics.json
@@ -0,0 +1,6 @@
+{
+    "gfx":[
+        ["gfx 0", 36, 13, 63, 1, [240136, 240136], [68598, 68598], [74592, 74592]],
+        ["gfx 1", 32, 17, 85, 1, [84872, 84872], [16328, 16328], [25632, 25632]]
+    ]
+} 
diff --git a/render-test/tests/gfx/pass-probe-reset/style.json b/render-test/tests/gfx/pass-probe-reset/style.json
new file mode 100644
index 0000000000..698d95bb74
--- /dev/null
+++ b/render-test/tests/gfx/pass-probe-reset/style.json
@@ -0,0 +1,24 @@
+{
+    "version": 8,
+    "metadata": {
+      "test": {
+        "width": 512,
+        "height": 512,
+        "operations": [
+          ["setStyle", "local://styles/uruguay.json"],
+          ["setZoom", 9 ],
+          ["probeGFXStart"],
+          ["setCenter", [-56.509552, -32.865788] ],
+          ["probeGFX", "gfx 0"],
+          ["probeGFXEnd"],
+          ["probeGFXStart"],
+          ["setCenter", [-56.509552, -32.745788] ],
+          ["probeGFX", "gfx 1"],
+          ["probeGFXEnd"]
+        ]
+      }
+    },
+    "sources": {},
+    "layers": []
+  }
+  
+\ No newline at end of file
diff --git a/render-test/tests/gfx/pass/expected.png b/render-test/tests/gfx/pass/expected.png
new file mode 100644
index 0000000000..4b5ea75a25
--- /dev/null
+++ b/render-test/tests/gfx/pass/expected.png
diff --git a/render-test/tests/gfx/pass/metrics.json b/render-test/tests/gfx/pass/metrics.json
new file mode 100644
index 0000000000..67ecec5d78
--- /dev/null
+++ b/render-test/tests/gfx/pass/metrics.json
@@ -0,0 +1,5 @@
+{
+    "gfx":[
+        ["gfx 0", 36, 13, 63, 1, [240136, 240136], [68598, 68598], [74592, 74592]]
+    ]
+} 
diff --git a/render-test/tests/gfx/pass/style.json b/render-test/tests/gfx/pass/style.json
new file mode 100644
index 0000000000..6ae3eecb8c
--- /dev/null
+++ b/render-test/tests/gfx/pass/style.json
@@ -0,0 +1,20 @@
+{
+    "version": 8,
+    "metadata": {
+      "test": {
+        "width": 512,
+        "height": 512,
+        "operations": [
+          ["setStyle", "local://styles/uruguay.json"],
+          ["setZoom", 9 ],
+          ["probeGFXStart"],
+          ["setCenter", [-56.509552, -32.865788] ],
+          ["probeGFX", "gfx 0"],
+          ["probeGFXEnd"]
+        ]
+      }
+    },
+    "sources": {},
+    "layers": []
+  }
+  
+\ No newline at end of file
diff --git a/render-test/tests/mac/memory/pass-memory-size-is-same/expected.png b/render-test/tests/mac/memory/pass-memory-size-is-same/expected.png
new file mode 100644
index 0000000000..0858c19f05
--- /dev/null
+++ b/render-test/tests/mac/memory/pass-memory-size-is-same/expected.png
diff --git a/render-test/tests/mac/memory/pass-memory-size-is-same/metrics.json b/render-test/tests/mac/memory/pass-memory-size-is-same/metrics.json
new file mode 100644
index 0000000000..cc7d1c9fab
--- /dev/null
+++ b/render-test/tests/mac/memory/pass-memory-size-is-same/metrics.json
@@ -0,0 +1,19 @@
+{
+    "memory": [
+        [
+            "after setZoom 0.9",
+            33271,
+            51
+        ],
+        [
+            "end",
+            47552,
+            101
+        ],
+        [
+            "start",
+            0,
+            0
+        ]
+    ]
+}
diff --git a/render-test/tests/memory/fail-memory-size-is-too-big/expected.png b/render-test/tests/memory/fail-memory-size-is-too-big/expected.png
new file mode 100644
index 0000000000..0858c19f05
--- /dev/null
+++ b/render-test/tests/memory/fail-memory-size-is-too-big/expected.png
diff --git a/render-test/tests/memory/fail-memory-size-is-too-big/metrics.json b/render-test/tests/memory/fail-memory-size-is-too-big/metrics.json
new file mode 100644
index 0000000000..3e55e0c60e
--- /dev/null
+++ b/render-test/tests/memory/fail-memory-size-is-too-big/metrics.json
@@ -0,0 +1,19 @@
+{
+    "memory": [
+        [
+            "after setZoom 0.9",
+            33271,
+            62
+        ],
+        [
+            "end",
+            40000,
+            118
+        ],
+        [
+            "start",
+            0,
+            0
+        ]
+    ]
+}
diff --git a/render-test/tests/memory/fail-memory-size-is-too-big/style.json b/render-test/tests/memory/fail-memory-size-is-too-big/style.json
new file mode 100644
index 0000000000..b2805ef34f
--- /dev/null
+++ b/render-test/tests/memory/fail-memory-size-is-too-big/style.json
@@ -0,0 +1,52 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "width": 64,
+      "height": 64,
+      "operations": [
+        [ "wait" ],
+        [ "probeMemoryStart" ],
+        [ "probeMemory", "start" ],
+        [
+          "setZoom",
+          0.9
+        ],
+        [
+          "wait"
+        ],
+        [ "probeMemory", "after setZoom 0.9", 0.005 ],
+        [
+          "setLayerZoomRange",
+          "circle",
+          1,
+          2
+        ],
+        [
+          "wait"
+        ],
+        [ "probeMemory", "end", 0.005 ],
+        [ "probeMemoryEnd" ]
+      ]
+    }
+  },
+  "sources": {
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Point",
+        "coordinates": [
+          0,
+          0
+        ]
+      }
+    }
+  },
+  "layers": [
+    {
+      "id": "circle",
+      "type": "circle",
+      "source": "geojson"
+    }
+  ]
+}
diff --git a/render-test/tests/memory/fail-memory-size-is-too-small/expected.png b/render-test/tests/memory/fail-memory-size-is-too-small/expected.png
new file mode 100644
index 0000000000..0858c19f05
--- /dev/null
+++ b/render-test/tests/memory/fail-memory-size-is-too-small/expected.png
diff --git a/render-test/tests/memory/fail-memory-size-is-too-small/metrics.json b/render-test/tests/memory/fail-memory-size-is-too-small/metrics.json
new file mode 100644
index 0000000000..7e3997675b
--- /dev/null
+++ b/render-test/tests/memory/fail-memory-size-is-too-small/metrics.json
@@ -0,0 +1,19 @@
+{
+    "memory": [
+        [
+            "after setZoom 0.9",
+            33271,
+            62
+        ],
+        [
+            "end",
+            60000,
+            118
+        ],
+        [
+            "start",
+            0,
+            0
+        ]
+    ]
+}
diff --git a/render-test/tests/memory/fail-memory-size-is-too-small/style.json b/render-test/tests/memory/fail-memory-size-is-too-small/style.json
new file mode 100644
index 0000000000..b2805ef34f
--- /dev/null
+++ b/render-test/tests/memory/fail-memory-size-is-too-small/style.json
@@ -0,0 +1,52 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "width": 64,
+      "height": 64,
+      "operations": [
+        [ "wait" ],
+        [ "probeMemoryStart" ],
+        [ "probeMemory", "start" ],
+        [
+          "setZoom",
+          0.9
+        ],
+        [
+          "wait"
+        ],
+        [ "probeMemory", "after setZoom 0.9", 0.005 ],
+        [
+          "setLayerZoomRange",
+          "circle",
+          1,
+          2
+        ],
+        [
+          "wait"
+        ],
+        [ "probeMemory", "end", 0.005 ],
+        [ "probeMemoryEnd" ]
+      ]
+    }
+  },
+  "sources": {
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Point",
+        "coordinates": [
+          0,
+          0
+        ]
+      }
+    }
+  },
+  "layers": [
+    {
+      "id": "circle",
+      "type": "circle",
+      "source": "geojson"
+    }
+  ]
+}
diff --git a/render-test/tests/memory/pass-memory-size-is-same/expected.png b/render-test/tests/memory/pass-memory-size-is-same/expected.png
new file mode 100644
index 0000000000..0858c19f05
--- /dev/null
+++ b/render-test/tests/memory/pass-memory-size-is-same/expected.png
diff --git a/render-test/tests/memory/pass-memory-size-is-same/metrics.json b/render-test/tests/memory/pass-memory-size-is-same/metrics.json
new file mode 100644
index 0000000000..4baa38c7f7
--- /dev/null
+++ b/render-test/tests/memory/pass-memory-size-is-same/metrics.json
@@ -0,0 +1,19 @@
+{
+    "memory": [
+        [
+            "after setZoom 0.9",
+            33271,
+            62
+        ],
+        [
+            "end",
+            47833,
+            118
+        ],
+        [
+            "start",
+            0,
+            0
+        ]
+    ]
+}
+\ No newline at end of file
diff --git a/render-test/tests/memory/pass-memory-size-is-same/style.json b/render-test/tests/memory/pass-memory-size-is-same/style.json
new file mode 100644
index 0000000000..d1c09f69f9
--- /dev/null
+++ b/render-test/tests/memory/pass-memory-size-is-same/style.json
@@ -0,0 +1,52 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "width": 64,
+      "height": 64,
+      "operations": [
+        [ "wait" ],
+        [ "probeMemoryStart" ],
+        [ "probeMemory", "start" ],
+        [
+          "setZoom",
+          0.9
+        ],
+        [
+          "wait"
+        ],
+        [ "probeMemory", "after setZoom 0.9", 0.01 ],
+        [
+          "setLayerZoomRange",
+          "circle",
+          1,
+          2
+        ],
+        [
+          "wait"
+        ],
+        [ "probeMemory", "end", 0.01 ],
+        [ "probeMemoryEnd" ]
+      ]
+    }
+  },
+  "sources": {
+    "geojson": {
+      "type": "geojson",
+      "data": {
+        "type": "Point",
+        "coordinates": [
+          0,
+          0
+        ]
+      }
+    }
+  },
+  "layers": [
+    {
+      "id": "circle",
+      "type": "circle",
+      "source": "geojson"
+    }
+  ]
+}
diff --git a/render-test/tests/network/fail-requests-transferred/expected.png b/render-test/tests/network/fail-requests-transferred/expected.png
new file mode 100644
index 0000000000..b63b151765
--- /dev/null
+++ b/render-test/tests/network/fail-requests-transferred/expected.png
diff --git a/render-test/tests/network/fail-requests-transferred/metrics.json b/render-test/tests/network/fail-requests-transferred/metrics.json
new file mode 100644
index 0000000000..1a200ca38f
--- /dev/null
+++ b/render-test/tests/network/fail-requests-transferred/metrics.json
@@ -0,0 +1,14 @@
+{
+    "network": [
+        [
+            "end",
+            2,
+            200000
+        ],
+        [
+            "start",
+            0,
+            0
+        ]
+    ]
+}
+\ No newline at end of file
diff --git a/render-test/tests/network/fail-requests-transferred/style.json b/render-test/tests/network/fail-requests-transferred/style.json
new file mode 100644
index 0000000000..ef94ddc748
--- /dev/null
+++ b/render-test/tests/network/fail-requests-transferred/style.json
@@ -0,0 +1,67 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeNetworkStart"],
+        ["probeNetwork", "start"],
+        ["wait"],
+        ["probeNetwork", "end"],
+        ["probeNetworkEnd"]
+      ],
+      "height": 256,
+      "width": 1024
+    }
+  },
+  "center": [
+    -73,
+    15
+  ],
+  "zoom": 4.5,
+  "sources": {
+    "mapbox": {
+      "type": "vector",
+      "maxzoom": 14,
+      "tiles": [
+        "local://tiles/mapbox.mapbox-streets-v7/{z}-{x}-{y}.mvt"
+      ]
+    }
+  },
+  "glyphs": "local://glyphs/{fontstack}/{range}.pbf",
+  "layers": [
+    {
+      "id": "background",
+      "type": "background",
+      "paint": {
+        "background-color": "white"
+      }
+    },
+    {
+      "id": "line-center",
+      "type": "symbol",
+      "source": "mapbox",
+      "source-layer": "marine_label",
+      "layout": {
+        "text-field": "{name_en}",
+        "symbol-placement": "line-center",
+        "text-allow-overlap": true,
+        "text-size": 35,
+        "text-letter-spacing": 0.4,
+        "text-offset": [3, 0],
+        "text-font": [
+          "Open Sans Semibold",
+          "Arial Unicode MS Bold"
+        ]
+      }
+    },
+    {
+      "id": "line",
+      "type": "line",
+      "source": "mapbox",
+      "source-layer": "marine_label",
+      "paint": {
+        "line-width": 1
+      }
+    }
+  ]
+}
diff --git a/render-test/tests/network/fail-requests/expected.png b/render-test/tests/network/fail-requests/expected.png
new file mode 100644
index 0000000000..b63b151765
--- /dev/null
+++ b/render-test/tests/network/fail-requests/expected.png
diff --git a/render-test/tests/network/fail-requests/metrics.json b/render-test/tests/network/fail-requests/metrics.json
new file mode 100644
index 0000000000..81c9b8a5d4
--- /dev/null
+++ b/render-test/tests/network/fail-requests/metrics.json
@@ -0,0 +1,14 @@
+{
+    "network": [
+        [
+            "end",
+            2,
+            183111
+        ],
+        [
+            "start",
+            0,
+            0
+        ]
+    ]
+}
+\ No newline at end of file
diff --git a/render-test/tests/network/fail-requests/style.json b/render-test/tests/network/fail-requests/style.json
new file mode 100644
index 0000000000..ef94ddc748
--- /dev/null
+++ b/render-test/tests/network/fail-requests/style.json
@@ -0,0 +1,67 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeNetworkStart"],
+        ["probeNetwork", "start"],
+        ["wait"],
+        ["probeNetwork", "end"],
+        ["probeNetworkEnd"]
+      ],
+      "height": 256,
+      "width": 1024
+    }
+  },
+  "center": [
+    -73,
+    15
+  ],
+  "zoom": 4.5,
+  "sources": {
+    "mapbox": {
+      "type": "vector",
+      "maxzoom": 14,
+      "tiles": [
+        "local://tiles/mapbox.mapbox-streets-v7/{z}-{x}-{y}.mvt"
+      ]
+    }
+  },
+  "glyphs": "local://glyphs/{fontstack}/{range}.pbf",
+  "layers": [
+    {
+      "id": "background",
+      "type": "background",
+      "paint": {
+        "background-color": "white"
+      }
+    },
+    {
+      "id": "line-center",
+      "type": "symbol",
+      "source": "mapbox",
+      "source-layer": "marine_label",
+      "layout": {
+        "text-field": "{name_en}",
+        "symbol-placement": "line-center",
+        "text-allow-overlap": true,
+        "text-size": 35,
+        "text-letter-spacing": 0.4,
+        "text-offset": [3, 0],
+        "text-font": [
+          "Open Sans Semibold",
+          "Arial Unicode MS Bold"
+        ]
+      }
+    },
+    {
+      "id": "line",
+      "type": "line",
+      "source": "mapbox",
+      "source-layer": "marine_label",
+      "paint": {
+        "line-width": 1
+      }
+    }
+  ]
+}
diff --git a/render-test/tests/network/fail-transferred/expected.png b/render-test/tests/network/fail-transferred/expected.png
new file mode 100644
index 0000000000..b63b151765
--- /dev/null
+++ b/render-test/tests/network/fail-transferred/expected.png
diff --git a/render-test/tests/network/fail-transferred/metrics.json b/render-test/tests/network/fail-transferred/metrics.json
new file mode 100644
index 0000000000..20b42d5b9b
--- /dev/null
+++ b/render-test/tests/network/fail-transferred/metrics.json
@@ -0,0 +1,14 @@
+{
+    "network": [
+        [
+            "end",
+            3,
+            100000
+        ],
+        [
+            "start",
+            0,
+            0
+        ]
+    ]
+}
+\ No newline at end of file
diff --git a/render-test/tests/network/fail-transferred/style.json b/render-test/tests/network/fail-transferred/style.json
new file mode 100644
index 0000000000..ef94ddc748
--- /dev/null
+++ b/render-test/tests/network/fail-transferred/style.json
@@ -0,0 +1,67 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeNetworkStart"],
+        ["probeNetwork", "start"],
+        ["wait"],
+        ["probeNetwork", "end"],
+        ["probeNetworkEnd"]
+      ],
+      "height": 256,
+      "width": 1024
+    }
+  },
+  "center": [
+    -73,
+    15
+  ],
+  "zoom": 4.5,
+  "sources": {
+    "mapbox": {
+      "type": "vector",
+      "maxzoom": 14,
+      "tiles": [
+        "local://tiles/mapbox.mapbox-streets-v7/{z}-{x}-{y}.mvt"
+      ]
+    }
+  },
+  "glyphs": "local://glyphs/{fontstack}/{range}.pbf",
+  "layers": [
+    {
+      "id": "background",
+      "type": "background",
+      "paint": {
+        "background-color": "white"
+      }
+    },
+    {
+      "id": "line-center",
+      "type": "symbol",
+      "source": "mapbox",
+      "source-layer": "marine_label",
+      "layout": {
+        "text-field": "{name_en}",
+        "symbol-placement": "line-center",
+        "text-allow-overlap": true,
+        "text-size": 35,
+        "text-letter-spacing": 0.4,
+        "text-offset": [3, 0],
+        "text-font": [
+          "Open Sans Semibold",
+          "Arial Unicode MS Bold"
+        ]
+      }
+    },
+    {
+      "id": "line",
+      "type": "line",
+      "source": "mapbox",
+      "source-layer": "marine_label",
+      "paint": {
+        "line-width": 1
+      }
+    }
+  ]
+}
diff --git a/render-test/tests/network/pass/expected.png b/render-test/tests/network/pass/expected.png
new file mode 100644
index 0000000000..b63b151765
--- /dev/null
+++ b/render-test/tests/network/pass/expected.png
diff --git a/render-test/tests/network/pass/metrics.json b/render-test/tests/network/pass/metrics.json
new file mode 100644
index 0000000000..6afd106a45
--- /dev/null
+++ b/render-test/tests/network/pass/metrics.json
@@ -0,0 +1,14 @@
+{
+    "network": [
+        [
+            "end",
+            3,
+            183111
+        ],
+        [
+            "start",
+            0,
+            0
+        ]
+    ]
+}
+\ No newline at end of file
diff --git a/render-test/tests/network/pass/style.json b/render-test/tests/network/pass/style.json
new file mode 100644
index 0000000000..ef94ddc748
--- /dev/null
+++ b/render-test/tests/network/pass/style.json
@@ -0,0 +1,67 @@
+{
+  "version": 8,
+  "metadata": {
+    "test": {
+      "operations": [
+        ["probeNetworkStart"],
+        ["probeNetwork", "start"],
+        ["wait"],
+        ["probeNetwork", "end"],
+        ["probeNetworkEnd"]
+      ],
+      "height": 256,
+      "width": 1024
+    }
+  },
+  "center": [
+    -73,
+    15
+  ],
+  "zoom": 4.5,
+  "sources": {
+    "mapbox": {
+      "type": "vector",
+      "maxzoom": 14,
+      "tiles": [
+        "local://tiles/mapbox.mapbox-streets-v7/{z}-{x}-{y}.mvt"
+      ]
+    }
+  },
+  "glyphs": "local://glyphs/{fontstack}/{range}.pbf",
+  "layers": [
+    {
+      "id": "background",
+      "type": "background",
+      "paint": {
+        "background-color": "white"
+      }
+    },
+    {
+      "id": "line-center",
+      "type": "symbol",
+      "source": "mapbox",
+      "source-layer": "marine_label",
+      "layout": {
+        "text-field": "{name_en}",
+        "symbol-placement": "line-center",
+        "text-allow-overlap": true,
+        "text-size": 35,
+        "text-letter-spacing": 0.4,
+        "text-offset": [3, 0],
+        "text-font": [
+          "Open Sans Semibold",
+          "Arial Unicode MS Bold"
+        ]
+      }
+    },
+    {
+      "id": "line",
+      "type": "line",
+      "source": "mapbox",
+      "source-layer": "marine_label",
+      "paint": {
+        "line-width": 1
+      }
+    }
+  ]
+}
diff --git a/render-test/tests/should-fail.json b/render-test/tests/should-fail.json
new file mode 100644
index 0000000000..b539d66019
--- /dev/null
+++ b/render-test/tests/should-fail.json
@@ -0,0 +1,18 @@
+{
+  "tests/file-size/fail-size-is-over": "Should fail, size is bigger than expected.",
+  "tests/file-size/fail-size-is-under": "Should fail, size is smaller than expected.",
+  "tests/file-size/fail-file-doesnt-match": "Should fail, doesn't match the expectation.",
+  "tests/file-size/fail-file-not-found": "Should fail, file not found.",
+  "tests/network/fail-requests": "Should fail, number of requests higher than expected.",
+  "tests/network/fail-transferred": "Should fail, amount of transferred data higher than expected.",
+  "tests/network/fail-requests-transferred": "Should fail, number of requests higher than expected and amount of transferred data less than expected.",
+  "tests/memory/fail-memory-size-is-too-big": "Should fail, memory size is bigger than expected.",
+  "tests/memory/fail-memory-size-is-too-small": "Should fail, memory size is smaller than expected.",
+  "tests/gfx/fail-ib-mem-mismatch": "Should fail, combined byte size of index buffers doesn't match the expectation.",
+  "tests/gfx/fail-negative-framebuffer-count": "Should fail, number of frame buffers is higher than expected.",
+  "tests/gfx/fail-texture-mem-mismatch": "Should fail, combined byte size of textures doesn't match the expectation.",
+  "tests/gfx/fail-too-many-drawcalls": "Should fail, number of draw calls higher than expected.",
+  "tests/gfx/fail-too-few-buffers": "Should fail, number of vertex and index buffers is smaller than expected.",
+  "tests/gfx/fail-too-few-textures": "Should fail, number of textures is smaller than expected.",
+  "tests/gfx/fail-vb-mem-mismatch": "Should fail, combined byte size of index buffers doesn't match the expectation."
+}