[core] Implement C++ render test runner

14 files changed, 1356 insertions, 23 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index ca33c36be2..df2e4940fd 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -184,6 +184,7 @@ include(cmake/core.cmake)
 
 if(COMMAND mbgl_platform_test)
     include(cmake/test.cmake)
+    include(cmake/render-test.cmake)
 endif()
 
 if(COMMAND mbgl_platform_benchmark)
diff --git a/Makefile b/Makefile
index 5b5e878280..35bf097486 100644
--- a/Makefile
+++ b/Makefile
@@ -552,12 +552,6 @@ qt-docs:
 .PHONY: test-node
 test-node: node
 	npm test
-	npm run test-suite
-
-.PHONY: test-node-recycle-map
-test-node-recycle-map: node
-	npm test
-	npm run test-render -- --recycle-map --shuffle
 	npm run test-query
 	npm run test-expressions
 
diff --git a/circle.yml b/circle.yml
index b04b2029da..da5b5ce264 100644
--- a/circle.yml
+++ b/circle.yml
@@ -43,6 +43,7 @@ workflows:
           name: linux-gcc4.9-debug
       - linux-gcc5-debug-coverage
       - linux-doxygen
+      - linux-render-tests
       - ios-debug
       - ios-release-template:
           name: ios-release
@@ -53,6 +54,7 @@ workflows:
             branches:
               ignore: /.*/
       - macos-debug
+      - macos-render-tests
       - qt5-linux-gcc5-release
       - qt5-macos-debug
   nightly:
@@ -241,11 +243,25 @@ commands:
           sed -i 's/"$@" 2>&1/"$@"/' /usr/bin/xvfb-run
 
 
+  configure-cmake:
+    steps:
+    - run:
+        name: CMake configuration step
+        command: |
+          mkdir -p build
+          cd build
+          cmake -DWITH_CXX11ABI=${WITH_CXX11ABI:0} -DWITH_COVERAGE=${WITH_COVERAGE:0} -DWITH_OSMESA=${WITH_OSMESA:0} -DWITH_EGL=${WITH_EGL:0} ..
+          cd ..
   build-node:
     steps:
     - run:
         name: Build node
         command: make node-all
+  build-mbgl-render-test:
+    steps:
+    - run:
+        name: Build mbgl-render-test
+        command: cmake --build build --config ${BUILDTYPE} --target mbgl-render-test -- -j${JOBS}
   build-linux:
     steps:
     - run:
@@ -371,18 +387,25 @@ commands:
           xvfb-run --server-args="-screen 0 1024x768x24" \
             logbt -- apitrace trace --api=egl -v make test-node
 
-  run-node-linux-tests-recycle-map:
+  run-macos-render-tests:
+    steps:
+    - run:
+        name: Run render tests (mbgl-render-test)
+        command: |
+          build/mbgl-render-test --recycle-map --shuffle
+        no_output_timeout: 2m
+
+  run-linux-render-tests:
     parameters:
       node_version:
         type: string
         default: v8
     steps:
     - run:
-        name: Run node tests (recycling the map object)
+        name: Run render tests (mbgl-render-test)
         command: |
-          . "$NVM_DIR/nvm.sh" && nvm use << parameters.node_version >>
           xvfb-run --server-args="-screen 0 1024x768x24" \
-            logbt -- apitrace trace --api=egl -v make test-node-recycle-map
+            logbt -- apitrace trace --api=egl -v build/mbgl-render-test --recycle-map --shuffle
 
   run-unit-tests:
     steps:
@@ -408,17 +431,11 @@ commands:
         when: on_success
         command: platform/node/scripts/publish.sh
 
-
   upload-render-tests:
     steps:
     - store_artifacts:
         path: mapbox-gl-js/test/integration/render-tests/index.html
         destination: render-tests
-  upload-render-tests-recycle-map:
-    steps:
-    - store_artifacts:
-        path: mapbox-gl-js/test/integration/render-tests/index-recycle-map.html
-        destination: render-tests
 
   collect-xcode-build-logs:
     steps:
@@ -728,7 +745,6 @@ jobs:
       - save-dependencies
       - run-node-linux-tests
       - publish-node-package
-      - upload-render-tests
 
 # ------------------------------------------------------------------------------
   node-gcc8-debug:
@@ -745,9 +761,7 @@ jobs:
       - install-dependencies
       - build-node
       - save-dependencies
-      - run-node-linux-tests-recycle-map
       - publish-node-package
-      - upload-render-tests-recycle-map
 
 # ------------------------------------------------------------------------------
   node-macos-release:
@@ -764,7 +778,6 @@ jobs:
       - save-dependencies
       - run-node-macos-tests
       - publish-node-package
-      - upload-render-tests
       - collect-xcode-build-logs
       - upload-xcode-build-logs
 
@@ -912,6 +925,25 @@ jobs:
               scripts/publish_doxygen_coverage.js "build/linux-$(uname -m)/Debug/doxygen-coverage.json"
 
 # ------------------------------------------------------------------------------
+  linux-render-tests:
+    docker:
+      - image: mbgl/linux-clang-7:a5a3c52107
+    resource_class: large
+    working_directory: /src
+    environment:
+      LIBSYSCONFCPUS: 4
+      JOBS: 4
+      BUILDTYPE: RelWithDebInfo
+      WITH_EGL: 1
+    steps:
+      - install-dependencies
+      - configure-cmake
+      - build-mbgl-render-test
+      - run-linux-render-tests
+      - save-dependencies
+      - upload-render-tests
+
+# ------------------------------------------------------------------------------
   ios-debug:
     macos:
       xcode: "10.2.1"
@@ -1121,6 +1153,23 @@ jobs:
       - upload-xcode-build-logs
 
 # ------------------------------------------------------------------------------
+  macos-render-tests:
+    macos:
+      xcode: "10.2.1"
+    environment:
+      BUILDTYPE: RelWithDebInfo
+      HOMEBREW_NO_AUTO_UPDATE: 1
+      JOBS: 2
+    steps:
+      - install-macos-dependencies
+      - install-dependencies
+      - configure-cmake
+      - build-mbgl-render-test
+      - save-dependencies
+      - run-macos-render-tests
+      - upload-render-tests
+
+# ------------------------------------------------------------------------------
   qt5-linux-gcc5-release:
     docker:
       - image: mbgl/linux-gcc-5-qt-5.9:5132cfd29f
diff --git a/cmake/render-test.cmake b/cmake/render-test.cmake
new file mode 100644
index 0000000000..6505cc73f8
--- /dev/null
+++ b/cmake/render-test.cmake
@@ -0,0 +1,29 @@
+add_executable(mbgl-render-test
+    render-test/main.cpp
+    render-test/parser.cpp
+    render-test/runner.cpp
+)
+
+if(APPLE)
+    target_link_libraries(mbgl-render-test PRIVATE mbgl-loop-darwin)
+else()
+    target_link_libraries(mbgl-render-test PRIVATE mbgl-loop-uv)
+endif()
+
+target_include_directories(mbgl-render-test
+    PRIVATE src
+    PRIVATE platform/default/include
+    PRIVATE render-test
+)
+
+target_link_libraries(mbgl-render-test PRIVATE
+    mbgl-core
+    mbgl-filesource
+    args
+    expected
+    filesystem
+    pixelmatch-cpp
+    rapidjson
+)
+
+add_definitions(-DTEST_RUNNER_ROOT_PATH="${CMAKE_SOURCE_DIR}")
diff --git a/cmake/sqlite.cmake b/cmake/sqlite.cmake
index fd2d98d3e2..c38f36d23d 100644
--- a/cmake/sqlite.cmake
+++ b/cmake/sqlite.cmake
@@ -34,5 +34,10 @@ target_compile_definitions(sqlite
 target_compile_options(sqlite
     PRIVATE "-Wno-int-conversion"
     PRIVATE "-Wno-implicit-fallthrough"
-    PRIVATE "-Wno-cast-function-type"
 )
+
+if(CMAKE_COMPILER_IS_GNUCXX)
+    target_compile_options(sqlite PRIVATE "-Wno-bad-function-cast")
+elseif(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang")
+    target_compile_options(sqlite PRIVATE "-Wno-cast-function-type")
+endif()
diff --git a/package.json b/package.json
index 7b12cdcb79..991b115446 100644
--- a/package.json
+++ b/package.json
@@ -45,7 +45,6 @@
     "install": "node-pre-gyp install --fallback-to-build=false || make node",
     "test": "tape platform/node/test/js/**/*.test.js",
     "test-memory": "node --expose-gc platform/node/test/memory.test.js",
-    "test-suite": "run-s test-render test-query test-expressions",
     "test-expressions": "node -r esm platform/node/test/expression.test.js",
     "test-render": "node -r esm platform/node/test/render.test.js",
     "test-query": "node -r esm platform/node/test/query.test.js"
diff --git a/platform/node/test/ignores.json b/platform/node/test/ignores.json
index b3fb4b5a51..63b9764f22 100644
--- a/platform/node/test/ignores.json
+++ b/platform/node/test/ignores.json
@@ -77,7 +77,6 @@
   "render-tests/regressions/mapbox-gl-js#7271": "https://github.com/mapbox/mapbox-gl-native/issues/12888",
   "render-tests/regressions/mapbox-gl-js#7302": "skip - js specific",
   "render-tests/regressions/mapbox-gl-native#7357": "https://github.com/mapbox/mapbox-gl-native/issues/7357",
-  "render-tests/runtime-styling/image-add-sdf": "https://github.com/mapbox/mapbox-gl-native/issues/9847",
   "render-tests/runtime-styling/paint-property-fill-flat-to-extrude": "https://github.com/mapbox/mapbox-gl-native/issues/6745",
   "render-tests/runtime-styling/set-style-paint-property-fill-flat-to-extrude": "https://github.com/mapbox/mapbox-gl-native/issues/6745",
   "render-tests/symbol-cross-fade/chinese": "https://github.com/mapbox/mapbox-gl-native/issues/10619",
diff --git a/render-test/filesystem.hpp b/render-test/filesystem.hpp
new file mode 100644
index 0000000000..cee7e9d911
--- /dev/null
+++ b/render-test/filesystem.hpp
@@ -0,0 +1,9 @@
+#pragma once
+
+#include <ghc/filesystem.hpp>
+
+namespace mbgl {
+
+namespace filesystem = ghc::filesystem;
+
+} // namespace mbgl
diff --git a/render-test/main.cpp b/render-test/main.cpp
new file mode 100644
index 0000000000..35f0cdea30
--- /dev/null
+++ b/render-test/main.cpp
@@ -0,0 +1,154 @@
+#include <mbgl/util/run_loop.hpp>
+#include <mbgl/util/io.hpp>
+
+#include "filesystem.hpp"
+#include "metadata.hpp"
+#include "parser.hpp"
+#include "runner.hpp"
+
+#include <random>
+
+#define ANSI_COLOR_RED        "\x1b[31m"
+#define ANSI_COLOR_GREEN      "\x1b[32m"
+#define ANSI_COLOR_YELLOW     "\x1b[33m"
+#define ANSI_COLOR_BLUE       "\x1b[34m"
+#define ANSI_COLOR_MAGENTA    "\x1b[35m"
+#define ANSI_COLOR_CYAN       "\x1b[36m"
+#define ANSI_COLOR_GRAY       "\x1b[37m"
+#define ANSI_COLOR_LIGHT_GRAY "\x1b[90m"
+#define ANSI_COLOR_RESET      "\x1b[0m"
+
+int main(int argc, char** argv) {
+    bool recycleMap;
+    bool shuffle;
+    uint32_t seed;
+    std::string testRootPath;
+    std::vector<std::string> ids;
+
+    std::tie(recycleMap, shuffle, seed, testRootPath, ids) = parseArguments(argc, argv);
+    const std::string::size_type rootLength = testRootPath.length();
+
+    const auto ignores = parseIgnores();
+
+    // Recursively traverse through the test paths and collect test directories containing "style.json".
+    std::vector<mbgl::filesystem::path> testPaths;
+    for (const auto& id : ids) {
+        for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(mbgl::filesystem::path(id))) {
+            if (testPath.path().filename() == "style.json") {
+                testPaths.push_back(testPath);
+            }
+        }
+    }
+
+    if (shuffle) {
+        printf(ANSI_COLOR_YELLOW "Shuffle seed: %d" ANSI_COLOR_RESET "\n", seed);
+
+        std::seed_seq sequence { seed };
+        std::mt19937 shuffler(sequence);
+        std::shuffle(testPaths.begin(), testPaths.end(), shuffler);
+    }
+
+    mbgl::util::RunLoop runLoop;
+    TestRunner runner;
+
+    std::vector<TestMetadata> metadatas;
+    metadatas.reserve(testPaths.size());
+
+    TestStatistics stats;
+
+    for (auto& testPath : testPaths) {
+        TestMetadata metadata = parseTestMetadata(testPath);
+
+        if (!recycleMap) {
+            runner.reset();
+        }
+
+        std::string& id = metadata.id;
+        std::string& status = metadata.status;
+        std::string& color = metadata.color;
+
+        id = testPath.remove_filename().string();
+        id = id.substr(rootLength + 1, id.length() - rootLength - 2);
+
+        bool shouldIgnore = false;
+        std::string ignoreReason;
+
+        const std::string ignoreName = "render-tests/" + id;
+        const auto it = std::find_if(ignores.cbegin(), ignores.cend(), [&ignoreName](auto pair) { return pair.first == ignoreName; });
+        if (it != ignores.end()) {
+            shouldIgnore = true;
+            ignoreReason = it->second;
+            if (ignoreReason.rfind("skip", 0) == 0) {
+                printf(ANSI_COLOR_GRAY "* skipped %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str());
+                continue;
+            }
+        }
+
+        bool errored = !metadata.errorMessage.empty();
+        if (!errored) {
+            errored = runner.run(metadata) && !metadata.errorMessage.empty();
+        }
+
+        bool passed = !errored && !metadata.diff.empty() && metadata.difference <= metadata.allowed;
+
+        if (shouldIgnore) {
+            if (passed) {
+                status = "ignored passed";
+                color = "#E8A408";
+                stats.ignorePassedTests++;
+                printf(ANSI_COLOR_YELLOW "* ignore %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str());
+            } else {
+                status = "ignored failed";
+                color = "#9E9E9E";
+                stats.ignoreFailedTests++;
+                printf(ANSI_COLOR_LIGHT_GRAY "* ignore %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str());
+            }
+        } else {
+            if (passed) {
+                status = "passed";
+                color = "green";
+                stats.passedTests++;
+                printf(ANSI_COLOR_GREEN "* passed %s" ANSI_COLOR_RESET "\n", id.c_str());
+            } else if (errored) {
+                status = "errored";
+                color = "red";
+                stats.erroredTests++;
+                printf(ANSI_COLOR_RED "* errored %s" ANSI_COLOR_RESET "\n", id.c_str());
+            } else {
+                status = "failed";
+                color = "red";
+                stats.failedTests++;
+                printf(ANSI_COLOR_RED "* failed %s" ANSI_COLOR_RESET "\n", id.c_str());
+            }
+        }
+
+        metadatas.push_back(std::move(metadata));
+    }
+
+    std::string resultsHTML = createResultPage(stats, metadatas, shuffle, seed);
+    mbgl::util::write_file(testRootPath + "/index.html", resultsHTML);
+
+    const uint32_t count = stats.erroredTests + stats.failedTests +
+                           stats.ignoreFailedTests + stats.ignorePassedTests +
+                           stats.passedTests;
+
+    if (stats.passedTests) {
+        printf(ANSI_COLOR_GREEN "%u passed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.passedTests, 100.0 * stats.passedTests / count);
+    }
+    if (stats.ignorePassedTests) {
+        printf(ANSI_COLOR_YELLOW "%u passed but were ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.ignorePassedTests, 100.0 * stats.ignorePassedTests / count);
+    }
+    if (stats.ignoreFailedTests) {
+        printf(ANSI_COLOR_LIGHT_GRAY "%u ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.ignoreFailedTests, 100.0 * stats.ignoreFailedTests / count);
+    }
+    if (stats.failedTests) {
+        printf(ANSI_COLOR_RED "%u failed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.failedTests, 100.0 * stats.failedTests / count);
+    }
+    if (stats.erroredTests) {
+        printf(ANSI_COLOR_RED "%u errored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.erroredTests, 100.0 * stats.erroredTests / count);
+    }
+
+    printf("Results at: %s%s\n", testRootPath.c_str(), "/index.html");
+
+    return stats.failedTests + stats.erroredTests == 0 ? 0 : 1;
+}
diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp
new file mode 100644
index 0000000000..4be83a5436
--- /dev/null
+++ b/render-test/metadata.hpp
@@ -0,0 +1,52 @@
+#pragma once
+
+#include <mbgl/util/rapidjson.hpp>
+#include <mbgl/util/size.hpp>
+
+#include <mbgl/map/mode.hpp>
+
+#include "filesystem.hpp"
+
+struct TestStatistics {
+    TestStatistics() = default;
+
+    uint32_t ignoreFailedTests = 0;
+    uint32_t ignorePassedTests = 0;
+    uint32_t erroredTests = 0;
+    uint32_t failedTests = 0;
+    uint32_t passedTests = 0;
+};
+
+struct TestMetadata {
+    TestMetadata() = default;
+
+    mbgl::filesystem::path path;
+    mbgl::JSDocument document;
+
+    mbgl::Size size{ 512u, 512u };
+    float pixelRatio = 1.0f;
+    double allowed = 0.00015; // diff
+    std::string description;
+    mbgl::MapMode mapMode = mbgl::MapMode::Static;
+    mbgl::MapDebugOptions debug = mbgl::MapDebugOptions::NoDebug;
+    bool crossSourceCollisions = true;
+    bool axonometric = false;
+    double xSkew = 0.0;
+    double ySkew = 1.0;
+
+    // TODO
+    uint32_t fadeDuration = 0;
+    bool addFakeCanvas = false;
+
+    // HTML
+    std::string id;
+    std::string status;
+    std::string color;
+
+    std::string actual;
+    std::string expected;
+    std::string diff;
+
+    std::string errorMessage;
+    double difference = 0.0;
+};
\ No newline at end of file
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
new file mode 100644
index 0000000000..089a5c45c9
--- /dev/null
+++ b/render-test/parser.cpp
@@ -0,0 +1,572 @@
+#include <mbgl/util/logging.hpp>
+#include <mbgl/util/io.hpp>
+#include <mbgl/util/rapidjson.hpp>
+#include <mbgl/util/string.hpp>
+
+#include <args.hxx>
+
+#include <rapidjson/stringbuffer.h>
+#include <rapidjson/writer.h>
+
+#include <boost/archive/iterators/base64_from_binary.hpp>
+#include <boost/archive/iterators/insert_linebreaks.hpp>
+#include <boost/archive/iterators/transform_width.hpp>
+#include <boost/archive/iterators/ostream_iterator.hpp>
+
+#include "parser.hpp"
+#include "metadata.hpp"
+
+#include <sstream>
+#include <regex>
+
+namespace {
+
+const char* resultsStyle = R"HTML(
+<style>
+    body { font: 18px/1.2 -apple-system, BlinkMacSystemFont, "Helvetica Neue", Helvetica, Arial, sans-serif; padding: 10px; }
+    h1 { font-size: 32px; margin-bottom: 0; }
+    button { vertical-align: middle; }
+    h2 { font-size: 24px; font-weight: normal; margin: 10px 0 10px; line-height: 1; }
+    img { margin: 0 10px 10px 0; border: 1px dotted #ccc; }
+    .stats { margin-top: 10px; }
+    .test { border-bottom: 1px dotted #bbb; padding-bottom: 5px; }
+    .tests { border-top: 1px dotted #bbb; margin-top: 10px; }
+    .diff { color: #777; }
+    .test p, .test pre { margin: 0 0 10px; }
+    .test pre { font-size: 14px; }
+    .label { color: white; font-size: 18px; padding: 2px 6px 3px; border-radius: 3px; margin-right: 3px; vertical-align: bottom; display: inline-block; }
+    .hide { display: none; }
+</style>
+)HTML";
+
+const char* resultsScript = R"HTML(
+<script>
+document.addEventListener('mouseover', handleHover);
+document.addEventListener('mouseout', handleHover);
+
+function handleHover(e) {
+    var el = e.target;
+    if (el.tagName === 'IMG' && el.dataset.altSrc) {
+        var tmp = el.src;
+        el.src = el.dataset.altSrc;
+        el.dataset.altSrc = tmp;
+    }
+}
+
+document.getElementById('toggle-passed').addEventListener('click', function (e) {
+    for (const row of document.querySelectorAll('.test.passed')) {
+        row.classList.toggle('hide');
+    }
+});
+document.getElementById('toggle-ignored').addEventListener('click', function (e) {
+    for (const row of document.querySelectorAll('.test.ignored')) {
+        row.classList.toggle('hide');
+    }
+});
+document.getElementById('toggle-sequence').addEventListener('click', function (e) {
+    document.getElementById('test-sequence').classList.toggle('hide');
+});
+</script>
+)HTML";
+
+const char* resultsHeaderButtons = R"HTML(
+    <button id='toggle-sequence'>Toggle test sequence</button>
+    <button id='toggle-passed'>Toggle passed tests</button>
+    <button id='toggle-ignored'>Toggle ignored tests</button>
+</h1>
+)HTML";
+
+std::string removeURLArguments(const std::string &url) {
+    std::string::size_type index = url.find('?');
+    if (index != std::string::npos) {
+        return url.substr(0, index);
+    }
+    return url;
+}
+
+std::string prependFileScheme(const std::string &url) {
+    static const std::string fileScheme("file://");
+    return fileScheme + url;
+}
+
+mbgl::optional<std::string> getVendorPath(const std::string& url, const std::regex& regex, bool glyphsPath = false) {
+    static const mbgl::filesystem::path vendorPath(std::string(TEST_RUNNER_ROOT_PATH) + "/vendor/");
+
+    mbgl::filesystem::path file = std::regex_replace(url, regex, vendorPath.string());
+    if (mbgl::filesystem::exists(file.parent_path())) {
+        return removeURLArguments(file.string());
+    }
+
+    if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
+        return removeURLArguments(file.string());
+    }
+
+    return {};
+}
+
+mbgl::optional<std::string> getIntegrationPath(const std::string& url, const std::string& parent, const std::regex& regex, bool glyphsPath = false) {
+    static const mbgl::filesystem::path integrationPath(std::string(TEST_RUNNER_ROOT_PATH) + "/mapbox-gl-js/test/integration/");
+
+    mbgl::filesystem::path file = std::regex_replace(url, regex, integrationPath.string() + parent);
+    if (mbgl::filesystem::exists(file.parent_path())) {
+        return removeURLArguments(file.string());
+    }
+
+    if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
+        return removeURLArguments(file.string());
+    }
+
+    return {};
+}
+
+mbgl::optional<std::string> localizeLocalURL(const std::string& url, bool glyphsPath = false) {
+    static const std::regex regex { "local://" };
+    if (auto vendorPath = getVendorPath(url, regex, glyphsPath)) {
+        return vendorPath;
+    } else {
+        return getIntegrationPath(url, "", regex, glyphsPath);
+    }
+}
+
+mbgl::optional<std::string> localizeHttpURL(const std::string& url) {
+    static const std::regex regex { "http://localhost:2900" };
+    if (auto vendorPath = getVendorPath(url, regex)) {
+        return vendorPath;
+    } else {
+        return getIntegrationPath(url, "", regex);
+    }
+}
+
+mbgl::optional<std::string> localizeMapboxSpriteURL(const std::string& url) {
+    static const std::regex regex { "mapbox://" };
+    return getIntegrationPath(url, "", regex);
+}
+
+mbgl::optional<std::string> localizeMapboxFontsURL(const std::string& url) {
+    static const std::regex regex { "mapbox://fonts" };
+    return getIntegrationPath(url, "glyphs/", regex, true);
+}
+
+mbgl::optional<std::string> localizeMapboxTilesURL(const std::string& url) {
+    static const std::regex regex { "mapbox://" };
+    if (auto vendorPath = getVendorPath(url, regex)) {
+        return vendorPath;
+    } else {
+        return getIntegrationPath(url, "tiles/", regex);
+    }
+}
+
+mbgl::optional<std::string> localizeMapboxTilesetURL(const std::string& url) {
+    static const std::regex regex { "mapbox://" };
+    return getIntegrationPath(url, "tilesets/", regex);
+}
+
+} // namespace
+
+JSONReply readJson(const mbgl::filesystem::path& jsonPath) {
+    auto maybeJSON = mbgl::util::readFile(jsonPath);
+    if (!maybeJSON) {
+        return { std::string("Unable to open file ") + jsonPath.string() };
+    }
+
+    mbgl::JSDocument document;
+    document.Parse<0>(*maybeJSON);
+    if (document.HasParseError()) {
+        return { mbgl::formatJSONParseError(document) };
+    }
+
+    return { std::move(document) };
+}
+
+std::string serializeJsonValue(const mbgl::JSValue& value) {
+    rapidjson::StringBuffer buffer;
+    buffer.Clear();
+    rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
+    value.Accept(writer);
+    return buffer.GetString();
+}
+
+std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base) {
+    static const std::regex regex(".*expected.*.png");
+
+    std::vector<std::string> expectedImages;
+    for (const auto& entry : mbgl::filesystem::directory_iterator(base)) {
+        if (entry.is_regular_file()) {
+            const std::string path = entry.path().string();
+            if (std::regex_match(path, regex)) {
+                expectedImages.emplace_back(std::move(path));
+            }
+        }
+    }
+    return expectedImages;
+}
+
+
+ArgumentsTuple parseArguments(int argc, char** argv) {
+    args::ArgumentParser argumentParser("Mapbox GL Test Runner");
+
+    args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", { 'h', "help" });
+
+    args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object",
+                              { 'r', "recycle-map" });
+    args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order",
+                           { 's', "shuffle" });
+    args::ValueFlag<uint32_t> seedValue(argumentParser, "seed", "Shuffle seed (default: random)",
+                                        { "seed" });
+    args::ValueFlag<std::string> testPathValue(argumentParser, "rootPath", "Test root rootPath",
+                                               { 'p', "rootPath" });
+    args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
+
+    try {
+        argumentParser.ParseCLI(argc, argv);
+    } catch (const args::Help&) {
+        std::ostringstream stream;
+        stream << argumentParser;
+        mbgl::Log::Info(mbgl::Event::General, stream.str());
+        exit(0);
+    } catch (const args::ParseError& e) {
+        std::ostringstream stream;
+        stream << argumentParser;
+        mbgl::Log::Info(mbgl::Event::General, stream.str());
+        mbgl::Log::Error(mbgl::Event::General, e.what());
+        exit(1);
+    } catch (const args::ValidationError& e) {
+        std::ostringstream stream;
+        stream << argumentParser;
+        mbgl::Log::Info(mbgl::Event::General, stream.str());
+        mbgl::Log::Error(mbgl::Event::General, e.what());
+        exit(2);
+    }
+
+    const std::string testDefaultPath =
+        std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration/render-tests");
+
+    std::vector<std::string> ids;
+    for (const auto& id : args::get(testNameValues)) {
+        ids.emplace_back(testDefaultPath + "/" + id);
+    }
+
+    if (ids.empty()) {
+        ids.emplace_back(testDefaultPath);
+    }
+
+    return ArgumentsTuple {
+        recycleMapFlag ? args::get(recycleMapFlag) : false,
+        shuffleFlag ? args::get(shuffleFlag) : false, seedValue ? args::get(seedValue) : 1u,
+        testPathValue ? args::get(testPathValue) : testDefaultPath, ids
+    };
+}
+
+std::vector<std::pair<std::string, std::string>> parseIgnores() {
+    std::vector<std::pair<std::string, std::string>> ignores;
+
+    auto path = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("platform/node/test/ignores.json");
+
+    auto maybeIgnores = readJson(path.string());
+    if (maybeIgnores.is<mbgl::JSDocument>()) {
+        for (const auto& property : maybeIgnores.get<mbgl::JSDocument>().GetObject()) {
+            const std::string ignore = { property.name.GetString(),
+                                         property.name.GetStringLength() };
+            const std::string reason = { property.value.GetString(),
+                                         property.value.GetStringLength() };
+            ignores.emplace_back(std::make_pair(ignore, reason));
+        }
+    }
+
+    return ignores;
+}
+
+TestMetadata parseTestMetadata(const mbgl::filesystem::path& path) {
+    TestMetadata metadata;
+    metadata.path = path;
+
+    auto maybeJson = readJson(path.string());
+    if (!maybeJson.is<mbgl::JSDocument>()) { // NOLINT
+        metadata.errorMessage = std::string("Unable to parse: ") + path.string();
+        return metadata;
+    }
+
+    metadata.document = std::move(maybeJson.get<mbgl::JSDocument>());
+    localizeStyleURLs(metadata.document, metadata.document);
+
+    if (!metadata.document.HasMember("metadata")) {
+        mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s",
+                           path.c_str());
+        return metadata;
+    }
+
+    const mbgl::JSValue& metadataValue = metadata.document["metadata"];
+    if (!metadataValue.HasMember("test")) {
+        mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata.test': %s",
+                           path.c_str());
+        return metadata;
+    }
+
+    const mbgl::JSValue& testValue = metadataValue["test"];
+
+    if (testValue.HasMember("width")) {
+        assert(testValue["width"].IsNumber());
+        metadata.size.width = testValue["width"].GetInt();
+    }
+
+    if (testValue.HasMember("height")) {
+        assert(testValue["height"].IsNumber());
+        metadata.size.height = testValue["height"].GetInt();
+    }
+
+    if (testValue.HasMember("pixelRatio")) {
+        assert(testValue["pixelRatio"].IsNumber());
+        metadata.pixelRatio = testValue["pixelRatio"].GetFloat();
+    }
+
+    if (testValue.HasMember("allowed")) {
+        assert(testValue["allowed"].IsNumber());
+        metadata.allowed = testValue["allowed"].GetDouble();
+    }
+
+    if (testValue.HasMember("description")) {
+        assert(testValue["description"].IsString());
+        metadata.description = std::string{ testValue["description"].GetString(),
+                                                testValue["description"].GetStringLength() };
+    }
+
+    if (testValue.HasMember("mapMode")) {
+        assert(testValue["mapMode"].IsString());
+        metadata.mapMode = testValue["mapMode"].GetString() == std::string("tile") ? mbgl::MapMode::Tile : mbgl::MapMode::Static;
+    }
+
+    // Test operations handled in runner.cpp.
+
+    if (testValue.HasMember("debug")) {
+        metadata.debug |= mbgl::MapDebugOptions::TileBorders;
+    }
+
+    if (testValue.HasMember("collisionDebug")) {
+        metadata.debug |= mbgl::MapDebugOptions::Collision;
+    }
+
+    if (testValue.HasMember("showOverdrawInspector")) {
+        metadata.debug |= mbgl::MapDebugOptions::Overdraw;
+    }
+
+    if (testValue.HasMember("crossSourceCollisions")) {
+        assert(testValue["crossSourceCollisions"].IsBool());
+        metadata.crossSourceCollisions = testValue["crossSourceCollisions"].GetBool();
+    }
+
+    if (testValue.HasMember("axonometric")) {
+        assert(testValue["axonometric"].IsBool());
+        metadata.axonometric = testValue["axonometric"].GetBool();
+    }
+
+    if (testValue.HasMember("skew")) {
+        assert(testValue["skew"].IsArray());
+        metadata.xSkew = testValue["skew"][0].GetDouble();
+        metadata.ySkew = testValue["skew"][1].GetDouble();
+    }
+
+    // TODO: fadeDuration
+    // TODO: addFakeCanvas
+
+    return metadata;
+}
+
+// https://stackoverflow.com/questions/7053538/how-do-i-encode-a-string-to-base64-using-only-boost
+std::string encodeBase64(const std::string& data) {
+    using namespace boost::archive::iterators;
+    using base64 = insert_linebreaks<base64_from_binary<transform_width<const char*, 6, 8>>, 72>;
+
+    std::stringstream os;
+    std::copy(base64(data.c_str()), base64(data.c_str() + data.size()), ostream_iterator<char>(os));
+    return os.str();
+}
+
+std::string createResultItem(const TestMetadata& metadata, bool hasFailedTests) {
+    const bool shouldHide = (hasFailedTests && metadata.status == "passed") || (metadata.status.find("ignored") != std::string::npos);
+    
+    std::string html;
+    html.append("<div class=\"test " + metadata.status + (shouldHide ? " hide" : "") + "\">\n");
+    html.append(R"(<h2><span class="label" style="background: )" + metadata.color + "\">" + metadata.status + "</span> " + metadata.id + "</h2>\n");
+    if (metadata.status != "errored") {
+        html.append("<img width=" + mbgl::util::toString(metadata.size.width));
+        html.append(" height=" + mbgl::util::toString(metadata.size.height));
+        html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.actual) + "\"");
+        html.append(" data-alt-src=\"data:image/png;base64," + encodeBase64(metadata.expected) + "\">\n");
+
+        html.append("<img width=" + mbgl::util::toString(metadata.size.width));
+        html.append(" height=" + mbgl::util::toString(metadata.size.height));
+        html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.diff) + "\">\n");
+    } else {
+        assert(!metadata.errorMessage.empty());
+        html.append("<p style=\"color: red\"><strong>Error:</strong> " + metadata.errorMessage + "</p>\n");
+    }
+    if (metadata.difference != 0.0) {
+        html.append("<p class=\"diff\"><strong>Diff:</strong> " + mbgl::util::toString(metadata.difference) + "</p>\n");
+    }
+    html.append("</div>\n");
+
+    return html;
+}
+
+std::string createResultPage(const TestStatistics& stats, const std::vector<TestMetadata>& metadatas, bool shuffle, uint32_t seed) {
+    const uint32_t unsuccessful = stats.erroredTests + stats.failedTests;
+    std::string resultsPage;
+
+    // Style
+    resultsPage.append(resultsStyle);
+
+    // Header
+    if (unsuccessful) {
+        resultsPage.append(R"HTML(<h1 style="color: red;">)HTML");
+        resultsPage.append(mbgl::util::toString(unsuccessful) + " tests failed.");
+    } else {
+        resultsPage.append(R"HTML(<h1 style="color: green;">)HTML");
+        resultsPage.append("All tests passed!");
+    }
+    resultsPage.append(resultsHeaderButtons);
+
+    // stats
+    resultsPage.append(R"HTML(<p class="stats">)HTML");
+    if (stats.ignoreFailedTests) {
+        resultsPage.append(mbgl::util::toString(stats.ignoreFailedTests) + " ignored failed, ");
+    }
+    if (stats.ignorePassedTests) {
+        resultsPage.append(mbgl::util::toString(stats.ignorePassedTests) + " ignored passed, ");
+    }
+    if (stats.erroredTests) {
+        resultsPage.append(mbgl::util::toString(stats.erroredTests) + " errored, ");
+    }
+    if (stats.failedTests) {
+        resultsPage.append(mbgl::util::toString(stats.failedTests) + " failed, ");
+    }
+    resultsPage.append(mbgl::util::toString(stats.passedTests) + " passed.\n");
+    resultsPage.append("</p>\n");
+
+    // Test sequence
+    {
+        resultsPage.append("<div id='test-sequence' class='hide'>\n");
+
+        // Failed tests
+        if (unsuccessful) {
+            resultsPage.append("<p><strong>Failed tests:</strong>");
+            for (const auto& metadata : metadatas) {
+                if (metadata.status == "failed" || metadata.status == "errored") {
+                    resultsPage.append(metadata.id + " ");
+                }
+            }
+            resultsPage.append("</p>\n");
+        }
+
+        // Test sequence
+        resultsPage.append("<p><strong>Test sequence:</strong>");
+        for (const auto& metadata : metadatas) {
+            resultsPage.append(metadata.id + " ");
+        }
+        resultsPage.append("</p>\n");
+
+        // Shuffle
+        if (shuffle) {
+            resultsPage.append("<p><strong>Shuffle seed</strong>: " + mbgl::util::toString(seed) + "</p>\n");
+        }
+
+        resultsPage.append("</div>\n");
+    }
+
+    // Script
+    resultsPage.append(resultsScript);
+
+    // Tests
+    resultsPage.append("<div class=\"tests\">\n");
+    for (const auto& metadata : metadatas) {
+        resultsPage.append(createResultItem(metadata, unsuccessful));
+    }
+    resultsPage.append("</div>\n");
+
+    return resultsPage;
+}
+
+std::string localizeURL(const std::string& url) {
+    static const std::regex regex { "local://" };
+    if (auto vendorPath = getVendorPath(url, regex)) {
+        return *vendorPath;
+    } else {
+        return getIntegrationPath(url, "", regex).value_or(url);
+    }
+}
+
+void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) {
+    if (root.HasMember("urls") && root["urls"].IsArray()) {
+        for (auto& urlValue : root["urls"].GetArray()) {
+            const std::string path = prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
+                .value_or(localizeLocalURL(urlValue.GetString())
+                    .value_or(urlValue.GetString())));
+            urlValue.Set<std::string>(path, document.GetAllocator());
+        }
+    }
+
+    if (root.HasMember("url")) {
+        static const std::string image("image");
+        static const std::string video("video");
+
+        mbgl::JSValue& urlValue = root["url"];
+        const std::string path = prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
+            .value_or(localizeLocalURL(urlValue.GetString())
+                .value_or(urlValue.GetString())));
+        urlValue.Set<std::string>(path, document.GetAllocator());
+
+        if (root["type"].GetString() != image && root["type"].GetString() != video) {
+            const auto tilesetPath = std::string(urlValue.GetString()).erase(0u, 7u); // remove "file://"
+            auto maybeTileset = readJson(tilesetPath);
+            if (maybeTileset.is<mbgl::JSDocument>()) {
+                const auto& tileset = maybeTileset.get<mbgl::JSDocument>();
+                assert(tileset.HasMember("tiles"));
+                root.AddMember("tiles", (mbgl::JSValue&)tileset["tiles"], document.GetAllocator());
+                root.RemoveMember("url");
+            }
+        }
+    }
+
+    if (root.HasMember("tiles")) {
+        mbgl::JSValue& tilesValue = root["tiles"];
+        assert(tilesValue.IsArray());
+        for (auto& tileValue : tilesValue.GetArray()) {
+            const std::string path = prependFileScheme(localizeMapboxTilesURL(tileValue.GetString())
+                .value_or(localizeLocalURL(tileValue.GetString())
+                    .value_or(localizeHttpURL(tileValue.GetString())
+                        .value_or(tileValue.GetString()))));
+            tileValue.Set<std::string>(path, document.GetAllocator());
+        }
+    }
+
+    if (root.HasMember("data") && root["data"].IsString()) {
+        mbgl::JSValue& dataValue = root["data"];
+        const std::string path = prependFileScheme(localizeLocalURL(dataValue.GetString())
+            .value_or(dataValue.GetString()));
+        dataValue.Set<std::string>(path, document.GetAllocator());
+    }
+}
+
+void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) {
+    if (root.HasMember("sources")) {
+        mbgl::JSValue& sourcesValue = root["sources"];
+        for (auto& sourceProperty : sourcesValue.GetObject()) {
+            localizeSourceURLs(sourceProperty.value, document);
+        }
+    }
+
+    if (root.HasMember("glyphs")) {
+        mbgl::JSValue& glyphsValue = root["glyphs"];
+        const std::string path = prependFileScheme(localizeMapboxFontsURL(glyphsValue.GetString())
+            .value_or(localizeLocalURL(glyphsValue.GetString(), true)
+                .value_or(glyphsValue.GetString())));
+        glyphsValue.Set<std::string>(path, document.GetAllocator());
+    }
+
+    if (root.HasMember("sprite")) {
+        mbgl::JSValue& spriteValue = root["sprite"];
+        const std::string path = prependFileScheme(localizeMapboxSpriteURL(spriteValue.GetString())
+            .value_or(localizeLocalURL(spriteValue.GetString())
+                .value_or(spriteValue.GetString())));
+        spriteValue.Set<std::string>(path, document.GetAllocator());
+    }
+}
diff --git a/render-test/parser.hpp b/render-test/parser.hpp
new file mode 100644
index 0000000000..be98719ab5
--- /dev/null
+++ b/render-test/parser.hpp
@@ -0,0 +1,34 @@
+#pragma once
+
+#include <mbgl/util/rapidjson.hpp>
+#include <mbgl/util/variant.hpp>
+
+#include <tuple>
+#include <string>
+#include <vector>
+
+#include "filesystem.hpp"
+
+struct TestMetadata;
+struct TestStatistics;
+
+using ErrorMessage = std::string;
+using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>;
+
+using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector<std::string>>;
+
+JSONReply readJson(const mbgl::filesystem::path&);
+std::string serializeJsonValue(const mbgl::JSValue&);
+
+std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base);
+
+ArgumentsTuple parseArguments(int argc, char** argv);
+std::vector<std::pair<std::string, std::string>> parseIgnores();
+TestMetadata parseTestMetadata(const mbgl::filesystem::path& path);
+
+std::string createResultPage(const TestStatistics&, const std::vector<TestMetadata>&, bool shuffle, uint32_t seed);
+
+std::string localizeURL(const std::string& url);
+
+void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document);
+void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document);
\ No newline at end of file
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
new file mode 100644
index 0000000000..1aa347f1a8
--- /dev/null
+++ b/render-test/runner.cpp
@@ -0,0 +1,408 @@
+#include <mbgl/map/camera.hpp>
+#include <mbgl/map/map_observer.hpp>
+#include <mbgl/style/conversion/filter.hpp>
+#include <mbgl/style/conversion/layer.hpp>
+#include <mbgl/style/conversion/light.hpp>
+#include <mbgl/style/conversion/source.hpp>
+#include <mbgl/style/image.hpp>
+#include <mbgl/style/layer.hpp>
+#include <mbgl/style/light.hpp>
+#include <mbgl/style/style.hpp>
+#include <mbgl/style/rapidjson_conversion.hpp>
+#include <mbgl/util/chrono.hpp>
+#include <mbgl/util/io.hpp>
+#include <mbgl/util/image.hpp>
+#include <mbgl/util/run_loop.hpp>
+#include <mbgl/util/string.hpp>
+#include <mbgl/util/timer.hpp>
+
+#include <mapbox/pixelmatch.hpp>
+
+#include "metadata.hpp"
+#include "parser.hpp"
+#include "runner.hpp"
+
+#include <algorithm>
+#include <cassert>
+#include <regex>
+
+bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& metadata) {
+    const std::string base = metadata.path.remove_filename().string();
+    metadata.actual = mbgl::encodePNG(actual);
+
+    if (actual.size.isEmpty()) {
+        metadata.errorMessage = "Invalid size for actual image";
+        return false;
+    }
+
+#if !TEST_READ_ONLY
+    if (getenv("UPDATE")) {
+        mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actual));
+        return true;
+    }
+
+    mbgl::util::write_file(base + "/actual.png", metadata.actual);
+#endif
+
+    mbgl::PremultipliedImage expected { actual.size };
+    mbgl::PremultipliedImage diff { actual.size };
+
+    double pixels = 0.0;
+
+    for (const auto& entry: readExpectedEntries(base)) {
+        mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
+        if (!maybeExpectedImage) {
+            metadata.errorMessage = "Failed to load expected image " + entry;
+            return false;
+        }
+
+        metadata.expected = *maybeExpectedImage;
+
+        expected = mbgl::decodeImage(*maybeExpectedImage);
+
+        pixels = // implicitly converting from uint64_t
+            mapbox::pixelmatch(actual.data.get(), expected.data.get(), expected.size.width,
+                               expected.size.height, diff.data.get(), 0.16); // GL JS uses 0.1285
+
+        metadata.diff = mbgl::encodePNG(diff);
+
+#if !TEST_READ_ONLY
+        mbgl::util::write_file(base + "/diff.png", metadata.diff);
+#endif
+
+        metadata.difference = pixels / expected.size.area();
+        if (metadata.difference <= metadata.allowed) {
+            break;
+        }
+    }
+
+    return true;
+}
+
+bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
+    if (!metadata.document.HasMember("metadata") ||
+        !metadata.document["metadata"].HasMember("test") ||
+        !metadata.document["metadata"]["test"].HasMember("operations")) {
+        return true;
+    }
+
+    assert(metadata.document["metadata"]["test"]["operations"].IsArray());
+
+    const auto& operationsArray = metadata.document["metadata"]["test"]["operations"].GetArray();
+    if (operationsArray.Empty()) {
+        return true;
+    }
+
+    const auto& operationIt = operationsArray.Begin();
+    assert(operationIt->IsArray());
+
+    const auto& operationArray = operationIt->GetArray();
+    assert(operationArray.Size() >= 1u);
+
+    auto& frontend = maps[key]->frontend;
+    auto& map = maps[key]->map;
+
+    static const std::string waitOp("wait");
+    static const std::string sleepOp("sleep");
+    static const std::string addImageOp("addImage");
+    static const std::string updateImageOp("updateImage");
+    static const std::string removeImageOp("removeImage");
+    static const std::string setStyleOp("setStyle");
+    static const std::string setCenterOp("setCenter");
+    static const std::string setZoomOp("setZoom");
+    static const std::string setBearingOp("setBearing");
+    static const std::string setFilterOp("setFilter");
+    static const std::string setLayerZoomRangeOp("setLayerZoomRange");
+    static const std::string setLightOp("setLight");
+    static const std::string addLayerOp("addLayer");
+    static const std::string removeLayerOp("removeLayer");
+    static const std::string addSourceOp("addSource");
+    static const std::string removeSourceOp("removeSource");
+    static const std::string setPaintPropertyOp("setPaintProperty");
+    static const std::string setLayoutPropertyOp("setLayoutProperty");
+
+    // wait
+    if (operationArray[0].GetString() == waitOp) {
+        frontend.render(map);
+
+    // sleep
+    } else if (operationArray[0].GetString() == sleepOp) {
+        mbgl::util::Timer sleepTimer;
+        bool sleeping = true;
+
+        mbgl::Duration duration = mbgl::Seconds(3);
+        if (operationArray.Size() >= 2u) {
+            duration = mbgl::Milliseconds(operationArray[1].GetUint());
+        }
+
+        sleepTimer.start(duration, mbgl::Duration::zero(), [&]() {
+            sleeping = false;
+        });
+
+        while (sleeping) {
+            mbgl::util::RunLoop::Get()->runOnce();
+        }
+
+    // addImage | updateImage
+    } else if (operationArray[0].GetString() == addImageOp || operationArray[0].GetString() == updateImageOp) {
+        assert(operationArray.Size() >= 3u);
+
+        float pixelRatio = 1.0f;
+        bool sdf = false;
+
+        if (operationArray.Size() == 4u) {
+            assert(operationArray[3].IsObject());
+            const auto& imageOptions = operationArray[3].GetObject();
+            if (imageOptions.HasMember("pixelRatio")) {
+                pixelRatio = imageOptions["pixelRatio"].GetFloat();
+            }
+            if (imageOptions.HasMember("sdf")) {
+                sdf = imageOptions["sdf"].GetBool();
+            }
+        }
+
+        std::string imageName = operationArray[1].GetString();
+        imageName.erase(std::remove(imageName.begin(), imageName.end(), '"'), imageName.end());
+
+        std::string imagePath = operationArray[2].GetString();
+        imagePath.erase(std::remove(imagePath.begin(), imagePath.end(), '"'), imagePath.end());
+
+        const mbgl::filesystem::path filePath(std::string(TEST_RUNNER_ROOT_PATH) + "/mapbox-gl-js/test/integration/" + imagePath);
+
+        mbgl::optional<std::string> maybeImage = mbgl::util::readFile(filePath.string());
+        if (!maybeImage) {
+            metadata.errorMessage = std::string("Failed to load expected image ") + filePath.string();
+            return false;
+        }
+
+        map.getStyle().addImage(std::make_unique<mbgl::style::Image>(imageName, mbgl::decodeImage(*maybeImage), pixelRatio, sdf));
+
+    // removeImage
+    } else if (operationArray[0].GetString() == removeImageOp) {
+        assert(operationArray.Size() >= 2u);
+        assert(operationArray[1].IsString());
+
+        const std::string imageName { operationArray[1].GetString(), operationArray[1].GetStringLength() };
+        map.getStyle().removeImage(imageName);
+
+    // setStyle
+    } else if (operationArray[0].GetString() == setStyleOp) {
+        assert(operationArray.Size() >= 2u);
+        if (operationArray[1].IsString()) {
+            std::string stylePath = localizeURL(operationArray[1].GetString());
+            auto maybeStyle = readJson(stylePath);
+            if (maybeStyle.is<mbgl::JSDocument>()) {
+                auto& style = maybeStyle.get<mbgl::JSDocument>();
+                localizeStyleURLs((mbgl::JSValue&)style, style);
+                map.getStyle().loadJSON(serializeJsonValue(style));
+            }
+        } else {
+            localizeStyleURLs(operationArray[1], metadata.document);
+            map.getStyle().loadJSON(serializeJsonValue(operationArray[1]));
+        }
+
+    // setCenter
+    } else if (operationArray[0].GetString() == setCenterOp) {
+        assert(operationArray.Size() >= 2u);
+        assert(operationArray[1].IsArray());
+
+        const auto& centerArray = operationArray[1].GetArray();
+        assert(centerArray.Size() == 2u);
+
+        map.jumpTo(mbgl::CameraOptions().withCenter(mbgl::LatLng(centerArray[1].GetDouble(), centerArray[0].GetDouble())));
+
+    // setZoom
+    } else if (operationArray[0].GetString() == setZoomOp) {
+        assert(operationArray.Size() >= 2u);
+        assert(operationArray[1].IsNumber());
+        map.jumpTo(mbgl::CameraOptions().withZoom(operationArray[1].GetDouble()));
+
+    // setBearing
+    } else if (operationArray[0].GetString() == setBearingOp) {
+        assert(operationArray.Size() >= 2u);
+        assert(operationArray[1].IsNumber());
+        map.jumpTo(mbgl::CameraOptions().withBearing(operationArray[1].GetDouble()));
+
+    // setFilter
+    } else if (operationArray[0].GetString() == setFilterOp) {
+        assert(operationArray.Size() >= 3u);
+        assert(operationArray[1].IsString());
+
+        const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() };
+
+        mbgl::style::conversion::Error error;
+        auto converted = mbgl::style::conversion::convert<mbgl::style::Filter>(operationArray[2], error);
+        if (!converted) {
+            metadata.errorMessage = std::string("Unable to convert filter: ")  + error.message;
+            return false;
+        } else {
+            auto layer = map.getStyle().getLayer(layerName);
+            if (!layer) {
+                metadata.errorMessage = std::string("Layer not found: ")  + layerName;
+                return false;
+            } else {
+                layer->setFilter(std::move(*converted));
+            }
+        }
+
+    // setLayerZoomRange
+    } else if (operationArray[0].GetString() == setLayerZoomRangeOp) {
+        assert(operationArray.Size() >= 4u);
+        assert(operationArray[1].IsString());
+        assert(operationArray[2].IsNumber());
+        assert(operationArray[3].IsNumber());
+
+        const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() };
+        auto layer = map.getStyle().getLayer(layerName);
+        if (!layer) {
+            metadata.errorMessage = std::string("Layer not found: ")  + layerName;
+            return false;
+        } else {
+            layer->setMinZoom(operationArray[2].GetFloat());
+            layer->setMaxZoom(operationArray[3].GetFloat());
+        }
+
+    // setLight
+    } else if (operationArray[0].GetString() == setLightOp) {
+        assert(operationArray.Size() >= 2u);
+        assert(operationArray[1].IsObject());
+
+        mbgl::style::conversion::Error error;
+        auto converted = mbgl::style::conversion::convert<mbgl::style::Light>(operationArray[1], error);
+        if (!converted) {
+            metadata.errorMessage = std::string("Unable to convert light: ")  + error.message;
+            return false;
+        } else {
+            map.getStyle().setLight(std::make_unique<mbgl::style::Light>(std::move(*converted)));
+        }
+
+    // addLayer
+    } else if (operationArray[0].GetString() == addLayerOp) {
+        assert(operationArray.Size() >= 2u);
+        assert(operationArray[1].IsObject());
+
+        mbgl::style::conversion::Error error;
+        auto converted = mbgl::style::conversion::convert<std::unique_ptr<mbgl::style::Layer>>(operationArray[1], error);
+        if (!converted) {
+            metadata.errorMessage = std::string("Unable to convert layer: ")  + error.message;
+            return false;
+        } else {
+            map.getStyle().addLayer(std::move(*converted));
+        }
+
+    // removeLayer
+    } else if (operationArray[0].GetString() == removeLayerOp) {
+        assert(operationArray.Size() >= 2u);
+        assert(operationArray[1].IsString());
+        map.getStyle().removeLayer(operationArray[1].GetString());
+
+    // addSource
+    } else if (operationArray[0].GetString() == addSourceOp) {
+        assert(operationArray.Size() >= 3u);
+        assert(operationArray[1].IsString());
+        assert(operationArray[2].IsObject());
+
+        localizeSourceURLs(operationArray[2], metadata.document);
+
+        mbgl::style::conversion::Error error;
+        auto converted = mbgl::style::conversion::convert<std::unique_ptr<mbgl::style::Source>>(operationArray[2], error, operationArray[1].GetString());
+        if (!converted) {
+            metadata.errorMessage = std::string("Unable to convert source: ")  + error.message;
+            return false;
+        } else {
+            map.getStyle().addSource(std::move(*converted));
+        }
+
+    // removeSource
+    } else if (operationArray[0].GetString() == removeSourceOp) {
+        assert(operationArray.Size() >= 2u);
+        assert(operationArray[1].IsString());
+        map.getStyle().removeSource(operationArray[1].GetString());
+
+    // setPaintProperty
+    } else if (operationArray[0].GetString() == setPaintPropertyOp) {
+        assert(operationArray.Size() >= 4u);
+        assert(operationArray[1].IsString());
+        assert(operationArray[2].IsString());
+
+        const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() };
+        const std::string propertyName { operationArray[2].GetString(), operationArray[2].GetStringLength() };
+
+        auto layer = map.getStyle().getLayer(layerName);
+        if (!layer) {
+            metadata.errorMessage = std::string("Layer not found: ")  + layerName;
+            return false;
+        } else {
+            const mbgl::JSValue* propertyValue = &operationArray[3];
+            layer->setPaintProperty(propertyName, propertyValue);
+        }
+
+    // setLayoutProperty
+    } else if (operationArray[0].GetString() == setLayoutPropertyOp) {
+        assert(operationArray.Size() >= 4u);
+        assert(operationArray[1].IsString());
+        assert(operationArray[2].IsString());
+
+        const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() };
+        const std::string propertyName { operationArray[2].GetString(), operationArray[2].GetStringLength() };
+
+        auto layer = map.getStyle().getLayer(layerName);
+        if (!layer) {
+            metadata.errorMessage = std::string("Layer not found: ")  + layerName;
+            return false;
+        } else {
+            const mbgl::JSValue* propertyValue = &operationArray[3];
+            layer->setLayoutProperty(propertyName, propertyValue);
+        }
+
+    } else {
+        metadata.errorMessage = std::string("Unsupported operation: ")  + operationArray[0].GetString();
+        return false;
+    }
+
+    operationsArray.Erase(operationIt);
+    return runOperations(key, metadata);
+}
+
+TestRunner::Impl::Impl(const TestMetadata& metadata)
+    : frontend(metadata.size, metadata.pixelRatio),
+      map(frontend,
+          mbgl::MapObserver::nullObserver(),
+          mbgl::MapOptions()
+              .withMapMode(metadata.mapMode)
+              .withSize(metadata.size)
+              .withPixelRatio(metadata.pixelRatio)
+              .withCrossSourceCollisions(metadata.crossSourceCollisions),
+          mbgl::ResourceOptions().withCacheOnlyRequestsSupport(false)) {}
+
+bool TestRunner::run(TestMetadata& metadata) {
+    std::string key = mbgl::util::toString(uint32_t(metadata.mapMode))
+        + "/" + mbgl::util::toString(metadata.pixelRatio)
+        + "/" + mbgl::util::toString(uint32_t(metadata.crossSourceCollisions));
+
+    if (maps.find(key) == maps.end()) {
+        maps[key] = std::make_unique<TestRunner::Impl>(metadata);
+    }
+
+    auto& frontend = maps[key]->frontend;
+    auto& map = maps[key]->map;
+
+    frontend.setSize(metadata.size);
+    map.setSize(metadata.size);
+
+    map.setProjectionMode(mbgl::ProjectionMode().withAxonometric(metadata.axonometric).withXSkew(metadata.xSkew).withYSkew(metadata.ySkew));
+    map.setDebug(metadata.debug);
+
+    map.getStyle().loadJSON(serializeJsonValue(metadata.document));
+    map.jumpTo(map.getStyle().getDefaultCamera());
+
+    if (!runOperations(key, metadata)) {
+        return false;
+    }
+
+    return checkImage(frontend.render(map), metadata);
+}
+
+void TestRunner::reset() {
+    maps.clear();
+}
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
new file mode 100644
index 0000000000..cbc0f42546
--- /dev/null
+++ b/render-test/runner.hpp
@@ -0,0 +1,28 @@
+#pragma once
+
+#include <mbgl/gfx/headless_frontend.hpp>
+#include <mbgl/map/map.hpp>
+
+#include <memory>
+
+struct TestMetadata;
+
+class TestRunner {
+public:
+    TestRunner() = default;
+
+    bool run(TestMetadata&);
+    void reset();
+
+private:
+    bool runOperations(const std::string& key, TestMetadata&);
+    bool checkImage(mbgl::PremultipliedImage&& image, TestMetadata&);
+
+    struct Impl {
+        Impl(const TestMetadata&);
+
+        mbgl::HeadlessFrontend frontend;
+        mbgl::Map map;
+    };
+    std::unordered_map<std::string, std::unique_ptr<Impl>> maps;
+};
\ No newline at end of file