summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--CMakeLists.txt1
-rw-r--r--Makefile6
-rw-r--r--circle.yml77
-rw-r--r--cmake/render-test.cmake29
-rw-r--r--cmake/sqlite.cmake7
-rw-r--r--package.json1
-rw-r--r--platform/node/test/ignores.json1
-rw-r--r--render-test/filesystem.hpp9
-rw-r--r--render-test/main.cpp154
-rw-r--r--render-test/metadata.hpp52
-rw-r--r--render-test/parser.cpp572
-rw-r--r--render-test/parser.hpp34
-rw-r--r--render-test/runner.cpp408
-rw-r--r--render-test/runner.hpp28
14 files changed, 1356 insertions, 23 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt
index ca33c36be2..df2e4940fd 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -184,6 +184,7 @@ include(cmake/core.cmake)
if(COMMAND mbgl_platform_test)
include(cmake/test.cmake)
+ include(cmake/render-test.cmake)
endif()
if(COMMAND mbgl_platform_benchmark)
diff --git a/Makefile b/Makefile
index 5b5e878280..35bf097486 100644
--- a/Makefile
+++ b/Makefile
@@ -552,12 +552,6 @@ qt-docs:
.PHONY: test-node
test-node: node
npm test
- npm run test-suite
-
-.PHONY: test-node-recycle-map
-test-node-recycle-map: node
- npm test
- npm run test-render -- --recycle-map --shuffle
npm run test-query
npm run test-expressions
diff --git a/circle.yml b/circle.yml
index b04b2029da..da5b5ce264 100644
--- a/circle.yml
+++ b/circle.yml
@@ -43,6 +43,7 @@ workflows:
name: linux-gcc4.9-debug
- linux-gcc5-debug-coverage
- linux-doxygen
+ - linux-render-tests
- ios-debug
- ios-release-template:
name: ios-release
@@ -53,6 +54,7 @@ workflows:
branches:
ignore: /.*/
- macos-debug
+ - macos-render-tests
- qt5-linux-gcc5-release
- qt5-macos-debug
nightly:
@@ -241,11 +243,25 @@ commands:
sed -i 's/"$@" 2>&1/"$@"/' /usr/bin/xvfb-run
+ configure-cmake:
+ steps:
+ - run:
+ name: CMake configuration step
+ command: |
+ mkdir -p build
+ cd build
+ cmake -DWITH_CXX11ABI=${WITH_CXX11ABI:0} -DWITH_COVERAGE=${WITH_COVERAGE:0} -DWITH_OSMESA=${WITH_OSMESA:0} -DWITH_EGL=${WITH_EGL:0} ..
+ cd ..
build-node:
steps:
- run:
name: Build node
command: make node-all
+ build-mbgl-render-test:
+ steps:
+ - run:
+ name: Build mbgl-render-test
+ command: cmake --build build --config ${BUILDTYPE} --target mbgl-render-test -- -j${JOBS}
build-linux:
steps:
- run:
@@ -371,18 +387,25 @@ commands:
xvfb-run --server-args="-screen 0 1024x768x24" \
logbt -- apitrace trace --api=egl -v make test-node
- run-node-linux-tests-recycle-map:
+ run-macos-render-tests:
+ steps:
+ - run:
+ name: Run render tests (mbgl-render-test)
+ command: |
+ build/mbgl-render-test --recycle-map --shuffle
+ no_output_timeout: 2m
+
+ run-linux-render-tests:
parameters:
node_version:
type: string
default: v8
steps:
- run:
- name: Run node tests (recycling the map object)
+ name: Run render tests (mbgl-render-test)
command: |
- . "$NVM_DIR/nvm.sh" && nvm use << parameters.node_version >>
xvfb-run --server-args="-screen 0 1024x768x24" \
- logbt -- apitrace trace --api=egl -v make test-node-recycle-map
+ logbt -- apitrace trace --api=egl -v build/mbgl-render-test --recycle-map --shuffle
run-unit-tests:
steps:
@@ -408,17 +431,11 @@ commands:
when: on_success
command: platform/node/scripts/publish.sh
-
upload-render-tests:
steps:
- store_artifacts:
path: mapbox-gl-js/test/integration/render-tests/index.html
destination: render-tests
- upload-render-tests-recycle-map:
- steps:
- - store_artifacts:
- path: mapbox-gl-js/test/integration/render-tests/index-recycle-map.html
- destination: render-tests
collect-xcode-build-logs:
steps:
@@ -728,7 +745,6 @@ jobs:
- save-dependencies
- run-node-linux-tests
- publish-node-package
- - upload-render-tests
# ------------------------------------------------------------------------------
node-gcc8-debug:
@@ -745,9 +761,7 @@ jobs:
- install-dependencies
- build-node
- save-dependencies
- - run-node-linux-tests-recycle-map
- publish-node-package
- - upload-render-tests-recycle-map
# ------------------------------------------------------------------------------
node-macos-release:
@@ -764,7 +778,6 @@ jobs:
- save-dependencies
- run-node-macos-tests
- publish-node-package
- - upload-render-tests
- collect-xcode-build-logs
- upload-xcode-build-logs
@@ -912,6 +925,25 @@ jobs:
scripts/publish_doxygen_coverage.js "build/linux-$(uname -m)/Debug/doxygen-coverage.json"
# ------------------------------------------------------------------------------
+ linux-render-tests:
+ docker:
+ - image: mbgl/linux-clang-7:a5a3c52107
+ resource_class: large
+ working_directory: /src
+ environment:
+ LIBSYSCONFCPUS: 4
+ JOBS: 4
+ BUILDTYPE: RelWithDebInfo
+ WITH_EGL: 1
+ steps:
+ - install-dependencies
+ - configure-cmake
+ - build-mbgl-render-test
+ - run-linux-render-tests
+ - save-dependencies
+ - upload-render-tests
+
+# ------------------------------------------------------------------------------
ios-debug:
macos:
xcode: "10.2.1"
@@ -1121,6 +1153,23 @@ jobs:
- upload-xcode-build-logs
# ------------------------------------------------------------------------------
+ macos-render-tests:
+ macos:
+ xcode: "10.2.1"
+ environment:
+ BUILDTYPE: RelWithDebInfo
+ HOMEBREW_NO_AUTO_UPDATE: 1
+ JOBS: 2
+ steps:
+ - install-macos-dependencies
+ - install-dependencies
+ - configure-cmake
+ - build-mbgl-render-test
+ - save-dependencies
+ - run-macos-render-tests
+ - upload-render-tests
+
+# ------------------------------------------------------------------------------
qt5-linux-gcc5-release:
docker:
- image: mbgl/linux-gcc-5-qt-5.9:5132cfd29f
diff --git a/cmake/render-test.cmake b/cmake/render-test.cmake
new file mode 100644
index 0000000000..6505cc73f8
--- /dev/null
+++ b/cmake/render-test.cmake
@@ -0,0 +1,29 @@
+add_executable(mbgl-render-test
+ render-test/main.cpp
+ render-test/parser.cpp
+ render-test/runner.cpp
+)
+
+if(APPLE)
+ target_link_libraries(mbgl-render-test PRIVATE mbgl-loop-darwin)
+else()
+ target_link_libraries(mbgl-render-test PRIVATE mbgl-loop-uv)
+endif()
+
+target_include_directories(mbgl-render-test
+ PRIVATE src
+ PRIVATE platform/default/include
+ PRIVATE render-test
+)
+
+target_link_libraries(mbgl-render-test PRIVATE
+ mbgl-core
+ mbgl-filesource
+ args
+ expected
+ filesystem
+ pixelmatch-cpp
+ rapidjson
+)
+
+add_definitions(-DTEST_RUNNER_ROOT_PATH="${CMAKE_SOURCE_DIR}")
diff --git a/cmake/sqlite.cmake b/cmake/sqlite.cmake
index fd2d98d3e2..c38f36d23d 100644
--- a/cmake/sqlite.cmake
+++ b/cmake/sqlite.cmake
@@ -34,5 +34,10 @@ target_compile_definitions(sqlite
target_compile_options(sqlite
PRIVATE "-Wno-int-conversion"
PRIVATE "-Wno-implicit-fallthrough"
- PRIVATE "-Wno-cast-function-type"
)
+
+if(CMAKE_COMPILER_IS_GNUCXX)
+ target_compile_options(sqlite PRIVATE "-Wno-bad-function-cast")
+elseif(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang")
+ target_compile_options(sqlite PRIVATE "-Wno-cast-function-type")
+endif()
diff --git a/package.json b/package.json
index 7b12cdcb79..991b115446 100644
--- a/package.json
+++ b/package.json
@@ -45,7 +45,6 @@
"install": "node-pre-gyp install --fallback-to-build=false || make node",
"test": "tape platform/node/test/js/**/*.test.js",
"test-memory": "node --expose-gc platform/node/test/memory.test.js",
- "test-suite": "run-s test-render test-query test-expressions",
"test-expressions": "node -r esm platform/node/test/expression.test.js",
"test-render": "node -r esm platform/node/test/render.test.js",
"test-query": "node -r esm platform/node/test/query.test.js"
diff --git a/platform/node/test/ignores.json b/platform/node/test/ignores.json
index b3fb4b5a51..63b9764f22 100644
--- a/platform/node/test/ignores.json
+++ b/platform/node/test/ignores.json
@@ -77,7 +77,6 @@
"render-tests/regressions/mapbox-gl-js#7271": "https://github.com/mapbox/mapbox-gl-native/issues/12888",
"render-tests/regressions/mapbox-gl-js#7302": "skip - js specific",
"render-tests/regressions/mapbox-gl-native#7357": "https://github.com/mapbox/mapbox-gl-native/issues/7357",
- "render-tests/runtime-styling/image-add-sdf": "https://github.com/mapbox/mapbox-gl-native/issues/9847",
"render-tests/runtime-styling/paint-property-fill-flat-to-extrude": "https://github.com/mapbox/mapbox-gl-native/issues/6745",
"render-tests/runtime-styling/set-style-paint-property-fill-flat-to-extrude": "https://github.com/mapbox/mapbox-gl-native/issues/6745",
"render-tests/symbol-cross-fade/chinese": "https://github.com/mapbox/mapbox-gl-native/issues/10619",
diff --git a/render-test/filesystem.hpp b/render-test/filesystem.hpp
new file mode 100644
index 0000000000..cee7e9d911
--- /dev/null
+++ b/render-test/filesystem.hpp
@@ -0,0 +1,9 @@
+#pragma once
+
+#include <ghc/filesystem.hpp>
+
+namespace mbgl {
+
+namespace filesystem = ghc::filesystem;
+
+} // namespace mbgl
diff --git a/render-test/main.cpp b/render-test/main.cpp
new file mode 100644
index 0000000000..35f0cdea30
--- /dev/null
+++ b/render-test/main.cpp
@@ -0,0 +1,154 @@
+#include <mbgl/util/run_loop.hpp>
+#include <mbgl/util/io.hpp>
+
+#include "filesystem.hpp"
+#include "metadata.hpp"
+#include "parser.hpp"
+#include "runner.hpp"
+
+#include <random>
+
+#define ANSI_COLOR_RED "\x1b[31m"
+#define ANSI_COLOR_GREEN "\x1b[32m"
+#define ANSI_COLOR_YELLOW "\x1b[33m"
+#define ANSI_COLOR_BLUE "\x1b[34m"
+#define ANSI_COLOR_MAGENTA "\x1b[35m"
+#define ANSI_COLOR_CYAN "\x1b[36m"
+#define ANSI_COLOR_GRAY "\x1b[37m"
+#define ANSI_COLOR_LIGHT_GRAY "\x1b[90m"
+#define ANSI_COLOR_RESET "\x1b[0m"
+
+int main(int argc, char** argv) {
+ bool recycleMap;
+ bool shuffle;
+ uint32_t seed;
+ std::string testRootPath;
+ std::vector<std::string> ids;
+
+ std::tie(recycleMap, shuffle, seed, testRootPath, ids) = parseArguments(argc, argv);
+ const std::string::size_type rootLength = testRootPath.length();
+
+ const auto ignores = parseIgnores();
+
+ // Recursively traverse through the test paths and collect test directories containing "style.json".
+ std::vector<mbgl::filesystem::path> testPaths;
+ for (const auto& id : ids) {
+ for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(mbgl::filesystem::path(id))) {
+ if (testPath.path().filename() == "style.json") {
+ testPaths.push_back(testPath);
+ }
+ }
+ }
+
+ if (shuffle) {
+ printf(ANSI_COLOR_YELLOW "Shuffle seed: %d" ANSI_COLOR_RESET "\n", seed);
+
+ std::seed_seq sequence { seed };
+ std::mt19937 shuffler(sequence);
+ std::shuffle(testPaths.begin(), testPaths.end(), shuffler);
+ }
+
+ mbgl::util::RunLoop runLoop;
+ TestRunner runner;
+
+ std::vector<TestMetadata> metadatas;
+ metadatas.reserve(testPaths.size());
+
+ TestStatistics stats;
+
+ for (auto& testPath : testPaths) {
+ TestMetadata metadata = parseTestMetadata(testPath);
+
+ if (!recycleMap) {
+ runner.reset();
+ }
+
+ std::string& id = metadata.id;
+ std::string& status = metadata.status;
+ std::string& color = metadata.color;
+
+ id = testPath.remove_filename().string();
+ id = id.substr(rootLength + 1, id.length() - rootLength - 2);
+
+ bool shouldIgnore = false;
+ std::string ignoreReason;
+
+ const std::string ignoreName = "render-tests/" + id;
+ const auto it = std::find_if(ignores.cbegin(), ignores.cend(), [&ignoreName](auto pair) { return pair.first == ignoreName; });
+ if (it != ignores.end()) {
+ shouldIgnore = true;
+ ignoreReason = it->second;
+ if (ignoreReason.rfind("skip", 0) == 0) {
+ printf(ANSI_COLOR_GRAY "* skipped %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str());
+ continue;
+ }
+ }
+
+ bool errored = !metadata.errorMessage.empty();
+ if (!errored) {
+ errored = runner.run(metadata) && !metadata.errorMessage.empty();
+ }
+
+ bool passed = !errored && !metadata.diff.empty() && metadata.difference <= metadata.allowed;
+
+ if (shouldIgnore) {
+ if (passed) {
+ status = "ignored passed";
+ color = "#E8A408";
+ stats.ignorePassedTests++;
+ printf(ANSI_COLOR_YELLOW "* ignore %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str());
+ } else {
+ status = "ignored failed";
+ color = "#9E9E9E";
+ stats.ignoreFailedTests++;
+ printf(ANSI_COLOR_LIGHT_GRAY "* ignore %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str());
+ }
+ } else {
+ if (passed) {
+ status = "passed";
+ color = "green";
+ stats.passedTests++;
+ printf(ANSI_COLOR_GREEN "* passed %s" ANSI_COLOR_RESET "\n", id.c_str());
+ } else if (errored) {
+ status = "errored";
+ color = "red";
+ stats.erroredTests++;
+ printf(ANSI_COLOR_RED "* errored %s" ANSI_COLOR_RESET "\n", id.c_str());
+ } else {
+ status = "failed";
+ color = "red";
+ stats.failedTests++;
+ printf(ANSI_COLOR_RED "* failed %s" ANSI_COLOR_RESET "\n", id.c_str());
+ }
+ }
+
+ metadatas.push_back(std::move(metadata));
+ }
+
+ std::string resultsHTML = createResultPage(stats, metadatas, shuffle, seed);
+ mbgl::util::write_file(testRootPath + "/index.html", resultsHTML);
+
+ const uint32_t count = stats.erroredTests + stats.failedTests +
+ stats.ignoreFailedTests + stats.ignorePassedTests +
+ stats.passedTests;
+
+ if (stats.passedTests) {
+ printf(ANSI_COLOR_GREEN "%u passed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.passedTests, 100.0 * stats.passedTests / count);
+ }
+ if (stats.ignorePassedTests) {
+ printf(ANSI_COLOR_YELLOW "%u passed but were ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.ignorePassedTests, 100.0 * stats.ignorePassedTests / count);
+ }
+ if (stats.ignoreFailedTests) {
+ printf(ANSI_COLOR_LIGHT_GRAY "%u ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.ignoreFailedTests, 100.0 * stats.ignoreFailedTests / count);
+ }
+ if (stats.failedTests) {
+ printf(ANSI_COLOR_RED "%u failed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.failedTests, 100.0 * stats.failedTests / count);
+ }
+ if (stats.erroredTests) {
+ printf(ANSI_COLOR_RED "%u errored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.erroredTests, 100.0 * stats.erroredTests / count);
+ }
+
+ printf("Results at: %s%s\n", testRootPath.c_str(), "/index.html");
+
+ return stats.failedTests + stats.erroredTests == 0 ? 0 : 1;
+}
diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp
new file mode 100644
index 0000000000..4be83a5436
--- /dev/null
+++ b/render-test/metadata.hpp
@@ -0,0 +1,52 @@
+#pragma once
+
+#include <mbgl/util/rapidjson.hpp>
+#include <mbgl/util/size.hpp>
+
+#include <mbgl/map/mode.hpp>
+
+#include "filesystem.hpp"
+
+struct TestStatistics {
+ TestStatistics() = default;
+
+ uint32_t ignoreFailedTests = 0;
+ uint32_t ignorePassedTests = 0;
+ uint32_t erroredTests = 0;
+ uint32_t failedTests = 0;
+ uint32_t passedTests = 0;
+};
+
+struct TestMetadata {
+ TestMetadata() = default;
+
+ mbgl::filesystem::path path;
+ mbgl::JSDocument document;
+
+ mbgl::Size size{ 512u, 512u };
+ float pixelRatio = 1.0f;
+ double allowed = 0.00015; // diff
+ std::string description;
+ mbgl::MapMode mapMode = mbgl::MapMode::Static;
+ mbgl::MapDebugOptions debug = mbgl::MapDebugOptions::NoDebug;
+ bool crossSourceCollisions = true;
+ bool axonometric = false;
+ double xSkew = 0.0;
+ double ySkew = 1.0;
+
+ // TODO
+ uint32_t fadeDuration = 0;
+ bool addFakeCanvas = false;
+
+ // HTML
+ std::string id;
+ std::string status;
+ std::string color;
+
+ std::string actual;
+ std::string expected;
+ std::string diff;
+
+ std::string errorMessage;
+ double difference = 0.0;
+}; \ No newline at end of file
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
new file mode 100644
index 0000000000..089a5c45c9
--- /dev/null
+++ b/render-test/parser.cpp
@@ -0,0 +1,572 @@
+#include <mbgl/util/logging.hpp>
+#include <mbgl/util/io.hpp>
+#include <mbgl/util/rapidjson.hpp>
+#include <mbgl/util/string.hpp>
+
+#include <args.hxx>
+
+#include <rapidjson/stringbuffer.h>
+#include <rapidjson/writer.h>
+
+#include <boost/archive/iterators/base64_from_binary.hpp>
+#include <boost/archive/iterators/insert_linebreaks.hpp>
+#include <boost/archive/iterators/transform_width.hpp>
+#include <boost/archive/iterators/ostream_iterator.hpp>
+
+#include "parser.hpp"
+#include "metadata.hpp"
+
+#include <sstream>
+#include <regex>
+
+namespace {
+
+const char* resultsStyle = R"HTML(
+<style>
+ body { font: 18px/1.2 -apple-system, BlinkMacSystemFont, "Helvetica Neue", Helvetica, Arial, sans-serif; padding: 10px; }
+ h1 { font-size: 32px; margin-bottom: 0; }
+ button { vertical-align: middle; }
+ h2 { font-size: 24px; font-weight: normal; margin: 10px 0 10px; line-height: 1; }
+ img { margin: 0 10px 10px 0; border: 1px dotted #ccc; }
+ .stats { margin-top: 10px; }
+ .test { border-bottom: 1px dotted #bbb; padding-bottom: 5px; }
+ .tests { border-top: 1px dotted #bbb; margin-top: 10px; }
+ .diff { color: #777; }
+ .test p, .test pre { margin: 0 0 10px; }
+ .test pre { font-size: 14px; }
+ .label { color: white; font-size: 18px; padding: 2px 6px 3px; border-radius: 3px; margin-right: 3px; vertical-align: bottom; display: inline-block; }
+ .hide { display: none; }
+</style>
+)HTML";
+
+const char* resultsScript = R"HTML(
+<script>
+document.addEventListener('mouseover', handleHover);
+document.addEventListener('mouseout', handleHover);
+
+function handleHover(e) {
+ var el = e.target;
+ if (el.tagName === 'IMG' && el.dataset.altSrc) {
+ var tmp = el.src;
+ el.src = el.dataset.altSrc;
+ el.dataset.altSrc = tmp;
+ }
+}
+
+document.getElementById('toggle-passed').addEventListener('click', function (e) {
+ for (const row of document.querySelectorAll('.test.passed')) {
+ row.classList.toggle('hide');
+ }
+});
+document.getElementById('toggle-ignored').addEventListener('click', function (e) {
+ for (const row of document.querySelectorAll('.test.ignored')) {
+ row.classList.toggle('hide');
+ }
+});
+document.getElementById('toggle-sequence').addEventListener('click', function (e) {
+ document.getElementById('test-sequence').classList.toggle('hide');
+});
+</script>
+)HTML";
+
+const char* resultsHeaderButtons = R"HTML(
+ <button id='toggle-sequence'>Toggle test sequence</button>
+ <button id='toggle-passed'>Toggle passed tests</button>
+ <button id='toggle-ignored'>Toggle ignored tests</button>
+</h1>
+)HTML";
+
+std::string removeURLArguments(const std::string &url) {
+ std::string::size_type index = url.find('?');
+ if (index != std::string::npos) {
+ return url.substr(0, index);
+ }
+ return url;
+}
+
+std::string prependFileScheme(const std::string &url) {
+ static const std::string fileScheme("file://");
+ return fileScheme + url;
+}
+
+mbgl::optional<std::string> getVendorPath(const std::string& url, const std::regex& regex, bool glyphsPath = false) {
+ static const mbgl::filesystem::path vendorPath(std::string(TEST_RUNNER_ROOT_PATH) + "/vendor/");
+
+ mbgl::filesystem::path file = std::regex_replace(url, regex, vendorPath.string());
+ if (mbgl::filesystem::exists(file.parent_path())) {
+ return removeURLArguments(file.string());
+ }
+
+ if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
+ return removeURLArguments(file.string());
+ }
+
+ return {};
+}
+
+mbgl::optional<std::string> getIntegrationPath(const std::string& url, const std::string& parent, const std::regex& regex, bool glyphsPath = false) {
+ static const mbgl::filesystem::path integrationPath(std::string(TEST_RUNNER_ROOT_PATH) + "/mapbox-gl-js/test/integration/");
+
+ mbgl::filesystem::path file = std::regex_replace(url, regex, integrationPath.string() + parent);
+ if (mbgl::filesystem::exists(file.parent_path())) {
+ return removeURLArguments(file.string());
+ }
+
+ if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
+ return removeURLArguments(file.string());
+ }
+
+ return {};
+}
+
+mbgl::optional<std::string> localizeLocalURL(const std::string& url, bool glyphsPath = false) {
+ static const std::regex regex { "local://" };
+ if (auto vendorPath = getVendorPath(url, regex, glyphsPath)) {
+ return vendorPath;
+ } else {
+ return getIntegrationPath(url, "", regex, glyphsPath);
+ }
+}
+
+mbgl::optional<std::string> localizeHttpURL(const std::string& url) {
+ static const std::regex regex { "http://localhost:2900" };
+ if (auto vendorPath = getVendorPath(url, regex)) {
+ return vendorPath;
+ } else {
+ return getIntegrationPath(url, "", regex);
+ }
+}
+
+mbgl::optional<std::string> localizeMapboxSpriteURL(const std::string& url) {
+ static const std::regex regex { "mapbox://" };
+ return getIntegrationPath(url, "", regex);
+}
+
+mbgl::optional<std::string> localizeMapboxFontsURL(const std::string& url) {
+ static const std::regex regex { "mapbox://fonts" };
+ return getIntegrationPath(url, "glyphs/", regex, true);
+}
+
+mbgl::optional<std::string> localizeMapboxTilesURL(const std::string& url) {
+ static const std::regex regex { "mapbox://" };
+ if (auto vendorPath = getVendorPath(url, regex)) {
+ return vendorPath;
+ } else {
+ return getIntegrationPath(url, "tiles/", regex);
+ }
+}
+
+mbgl::optional<std::string> localizeMapboxTilesetURL(const std::string& url) {
+ static const std::regex regex { "mapbox://" };
+ return getIntegrationPath(url, "tilesets/", regex);
+}
+
+} // namespace
+
+JSONReply readJson(const mbgl::filesystem::path& jsonPath) {
+ auto maybeJSON = mbgl::util::readFile(jsonPath);
+ if (!maybeJSON) {
+ return { std::string("Unable to open file ") + jsonPath.string() };
+ }
+
+ mbgl::JSDocument document;
+ document.Parse<0>(*maybeJSON);
+ if (document.HasParseError()) {
+ return { mbgl::formatJSONParseError(document) };
+ }
+
+ return { std::move(document) };
+}
+
+std::string serializeJsonValue(const mbgl::JSValue& value) {
+ rapidjson::StringBuffer buffer;
+ buffer.Clear();
+ rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
+ value.Accept(writer);
+ return buffer.GetString();
+}
+
+std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base) {
+ static const std::regex regex(".*expected.*.png");
+
+ std::vector<std::string> expectedImages;
+ for (const auto& entry : mbgl::filesystem::directory_iterator(base)) {
+ if (entry.is_regular_file()) {
+ const std::string path = entry.path().string();
+ if (std::regex_match(path, regex)) {
+ expectedImages.emplace_back(std::move(path));
+ }
+ }
+ }
+ return expectedImages;
+}
+
+
+ArgumentsTuple parseArguments(int argc, char** argv) {
+ args::ArgumentParser argumentParser("Mapbox GL Test Runner");
+
+ args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", { 'h', "help" });
+
+ args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object",
+ { 'r', "recycle-map" });
+ args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order",
+ { 's', "shuffle" });
+ args::ValueFlag<uint32_t> seedValue(argumentParser, "seed", "Shuffle seed (default: random)",
+ { "seed" });
+ args::ValueFlag<std::string> testPathValue(argumentParser, "rootPath", "Test root rootPath",
+ { 'p', "rootPath" });
+ args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)");
+
+ try {
+ argumentParser.ParseCLI(argc, argv);
+ } catch (const args::Help&) {
+ std::ostringstream stream;
+ stream << argumentParser;
+ mbgl::Log::Info(mbgl::Event::General, stream.str());
+ exit(0);
+ } catch (const args::ParseError& e) {
+ std::ostringstream stream;
+ stream << argumentParser;
+ mbgl::Log::Info(mbgl::Event::General, stream.str());
+ mbgl::Log::Error(mbgl::Event::General, e.what());
+ exit(1);
+ } catch (const args::ValidationError& e) {
+ std::ostringstream stream;
+ stream << argumentParser;
+ mbgl::Log::Info(mbgl::Event::General, stream.str());
+ mbgl::Log::Error(mbgl::Event::General, e.what());
+ exit(2);
+ }
+
+ const std::string testDefaultPath =
+ std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration/render-tests");
+
+ std::vector<std::string> ids;
+ for (const auto& id : args::get(testNameValues)) {
+ ids.emplace_back(testDefaultPath + "/" + id);
+ }
+
+ if (ids.empty()) {
+ ids.emplace_back(testDefaultPath);
+ }
+
+ return ArgumentsTuple {
+ recycleMapFlag ? args::get(recycleMapFlag) : false,
+ shuffleFlag ? args::get(shuffleFlag) : false, seedValue ? args::get(seedValue) : 1u,
+ testPathValue ? args::get(testPathValue) : testDefaultPath, ids
+ };
+}
+
+std::vector<std::pair<std::string, std::string>> parseIgnores() {
+ std::vector<std::pair<std::string, std::string>> ignores;
+
+ auto path = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("platform/node/test/ignores.json");
+
+ auto maybeIgnores = readJson(path.string());
+ if (maybeIgnores.is<mbgl::JSDocument>()) {
+ for (const auto& property : maybeIgnores.get<mbgl::JSDocument>().GetObject()) {
+ const std::string ignore = { property.name.GetString(),
+ property.name.GetStringLength() };
+ const std::string reason = { property.value.GetString(),
+ property.value.GetStringLength() };
+ ignores.emplace_back(std::make_pair(ignore, reason));
+ }
+ }
+
+ return ignores;
+}
+
+TestMetadata parseTestMetadata(const mbgl::filesystem::path& path) {
+ TestMetadata metadata;
+ metadata.path = path;
+
+ auto maybeJson = readJson(path.string());
+ if (!maybeJson.is<mbgl::JSDocument>()) { // NOLINT
+ metadata.errorMessage = std::string("Unable to parse: ") + path.string();
+ return metadata;
+ }
+
+ metadata.document = std::move(maybeJson.get<mbgl::JSDocument>());
+ localizeStyleURLs(metadata.document, metadata.document);
+
+ if (!metadata.document.HasMember("metadata")) {
+ mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s",
+ path.c_str());
+ return metadata;
+ }
+
+ const mbgl::JSValue& metadataValue = metadata.document["metadata"];
+ if (!metadataValue.HasMember("test")) {
+ mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata.test': %s",
+ path.c_str());
+ return metadata;
+ }
+
+ const mbgl::JSValue& testValue = metadataValue["test"];
+
+ if (testValue.HasMember("width")) {
+ assert(testValue["width"].IsNumber());
+ metadata.size.width = testValue["width"].GetInt();
+ }
+
+ if (testValue.HasMember("height")) {
+ assert(testValue["height"].IsNumber());
+ metadata.size.height = testValue["height"].GetInt();
+ }
+
+ if (testValue.HasMember("pixelRatio")) {
+ assert(testValue["pixelRatio"].IsNumber());
+ metadata.pixelRatio = testValue["pixelRatio"].GetFloat();
+ }
+
+ if (testValue.HasMember("allowed")) {
+ assert(testValue["allowed"].IsNumber());
+ metadata.allowed = testValue["allowed"].GetDouble();
+ }
+
+ if (testValue.HasMember("description")) {
+ assert(testValue["description"].IsString());
+ metadata.description = std::string{ testValue["description"].GetString(),
+ testValue["description"].GetStringLength() };
+ }
+
+ if (testValue.HasMember("mapMode")) {
+ assert(testValue["mapMode"].IsString());
+ metadata.mapMode = testValue["mapMode"].GetString() == std::string("tile") ? mbgl::MapMode::Tile : mbgl::MapMode::Static;
+ }
+
+ // Test operations handled in runner.cpp.
+
+ if (testValue.HasMember("debug")) {
+ metadata.debug |= mbgl::MapDebugOptions::TileBorders;
+ }
+
+ if (testValue.HasMember("collisionDebug")) {
+ metadata.debug |= mbgl::MapDebugOptions::Collision;
+ }
+
+ if (testValue.HasMember("showOverdrawInspector")) {
+ metadata.debug |= mbgl::MapDebugOptions::Overdraw;
+ }
+
+ if (testValue.HasMember("crossSourceCollisions")) {
+ assert(testValue["crossSourceCollisions"].IsBool());
+ metadata.crossSourceCollisions = testValue["crossSourceCollisions"].GetBool();
+ }
+
+ if (testValue.HasMember("axonometric")) {
+ assert(testValue["axonometric"].IsBool());
+ metadata.axonometric = testValue["axonometric"].GetBool();
+ }
+
+ if (testValue.HasMember("skew")) {
+ assert(testValue["skew"].IsArray());
+ metadata.xSkew = testValue["skew"][0].GetDouble();
+ metadata.ySkew = testValue["skew"][1].GetDouble();
+ }
+
+ // TODO: fadeDuration
+ // TODO: addFakeCanvas
+
+ return metadata;
+}
+
+// https://stackoverflow.com/questions/7053538/how-do-i-encode-a-string-to-base64-using-only-boost
+std::string encodeBase64(const std::string& data) {
+ using namespace boost::archive::iterators;
+ using base64 = insert_linebreaks<base64_from_binary<transform_width<const char*, 6, 8>>, 72>;
+
+ std::stringstream os;
+ std::copy(base64(data.c_str()), base64(data.c_str() + data.size()), ostream_iterator<char>(os));
+ return os.str();
+}
+
+std::string createResultItem(const TestMetadata& metadata, bool hasFailedTests) {
+ const bool shouldHide = (hasFailedTests && metadata.status == "passed") || (metadata.status.find("ignored") != std::string::npos);
+
+ std::string html;
+ html.append("<div class=\"test " + metadata.status + (shouldHide ? " hide" : "") + "\">\n");
+ html.append(R"(<h2><span class="label" style="background: )" + metadata.color + "\">" + metadata.status + "</span> " + metadata.id + "</h2>\n");
+ if (metadata.status != "errored") {
+ html.append("<img width=" + mbgl::util::toString(metadata.size.width));
+ html.append(" height=" + mbgl::util::toString(metadata.size.height));
+ html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.actual) + "\"");
+ html.append(" data-alt-src=\"data:image/png;base64," + encodeBase64(metadata.expected) + "\">\n");
+
+ html.append("<img width=" + mbgl::util::toString(metadata.size.width));
+ html.append(" height=" + mbgl::util::toString(metadata.size.height));
+ html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.diff) + "\">\n");
+ } else {
+ assert(!metadata.errorMessage.empty());
+ html.append("<p style=\"color: red\"><strong>Error:</strong> " + metadata.errorMessage + "</p>\n");
+ }
+ if (metadata.difference != 0.0) {
+ html.append("<p class=\"diff\"><strong>Diff:</strong> " + mbgl::util::toString(metadata.difference) + "</p>\n");
+ }
+ html.append("</div>\n");
+
+ return html;
+}
+
+std::string createResultPage(const TestStatistics& stats, const std::vector<TestMetadata>& metadatas, bool shuffle, uint32_t seed) {
+ const uint32_t unsuccessful = stats.erroredTests + stats.failedTests;
+ std::string resultsPage;
+
+ // Style
+ resultsPage.append(resultsStyle);
+
+ // Header
+ if (unsuccessful) {
+ resultsPage.append(R"HTML(<h1 style="color: red;">)HTML");
+ resultsPage.append(mbgl::util::toString(unsuccessful) + " tests failed.");
+ } else {
+ resultsPage.append(R"HTML(<h1 style="color: green;">)HTML");
+ resultsPage.append("All tests passed!");
+ }
+ resultsPage.append(resultsHeaderButtons);
+
+ // stats
+ resultsPage.append(R"HTML(<p class="stats">)HTML");
+ if (stats.ignoreFailedTests) {
+ resultsPage.append(mbgl::util::toString(stats.ignoreFailedTests) + " ignored failed, ");
+ }
+ if (stats.ignorePassedTests) {
+ resultsPage.append(mbgl::util::toString(stats.ignorePassedTests) + " ignored passed, ");
+ }
+ if (stats.erroredTests) {
+ resultsPage.append(mbgl::util::toString(stats.erroredTests) + " errored, ");
+ }
+ if (stats.failedTests) {
+ resultsPage.append(mbgl::util::toString(stats.failedTests) + " failed, ");
+ }
+ resultsPage.append(mbgl::util::toString(stats.passedTests) + " passed.\n");
+ resultsPage.append("</p>\n");
+
+ // Test sequence
+ {
+ resultsPage.append("<div id='test-sequence' class='hide'>\n");
+
+ // Failed tests
+ if (unsuccessful) {
+ resultsPage.append("<p><strong>Failed tests:</strong>");
+ for (const auto& metadata : metadatas) {
+ if (metadata.status == "failed" || metadata.status == "errored") {
+ resultsPage.append(metadata.id + " ");
+ }
+ }
+ resultsPage.append("</p>\n");
+ }
+
+ // Test sequence
+ resultsPage.append("<p><strong>Test sequence:</strong>");
+ for (const auto& metadata : metadatas) {
+ resultsPage.append(metadata.id + " ");
+ }
+ resultsPage.append("</p>\n");
+
+ // Shuffle
+ if (shuffle) {
+ resultsPage.append("<p><strong>Shuffle seed</strong>: " + mbgl::util::toString(seed) + "</p>\n");
+ }
+
+ resultsPage.append("</div>\n");
+ }
+
+ // Script
+ resultsPage.append(resultsScript);
+
+ // Tests
+ resultsPage.append("<div class=\"tests\">\n");
+ for (const auto& metadata : metadatas) {
+ resultsPage.append(createResultItem(metadata, unsuccessful));
+ }
+ resultsPage.append("</div>\n");
+
+ return resultsPage;
+}
+
+std::string localizeURL(const std::string& url) {
+ static const std::regex regex { "local://" };
+ if (auto vendorPath = getVendorPath(url, regex)) {
+ return *vendorPath;
+ } else {
+ return getIntegrationPath(url, "", regex).value_or(url);
+ }
+}
+
+void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) {
+ if (root.HasMember("urls") && root["urls"].IsArray()) {
+ for (auto& urlValue : root["urls"].GetArray()) {
+ const std::string path = prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
+ .value_or(localizeLocalURL(urlValue.GetString())
+ .value_or(urlValue.GetString())));
+ urlValue.Set<std::string>(path, document.GetAllocator());
+ }
+ }
+
+ if (root.HasMember("url")) {
+ static const std::string image("image");
+ static const std::string video("video");
+
+ mbgl::JSValue& urlValue = root["url"];
+ const std::string path = prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
+ .value_or(localizeLocalURL(urlValue.GetString())
+ .value_or(urlValue.GetString())));
+ urlValue.Set<std::string>(path, document.GetAllocator());
+
+ if (root["type"].GetString() != image && root["type"].GetString() != video) {
+ const auto tilesetPath = std::string(urlValue.GetString()).erase(0u, 7u); // remove "file://"
+ auto maybeTileset = readJson(tilesetPath);
+ if (maybeTileset.is<mbgl::JSDocument>()) {
+ const auto& tileset = maybeTileset.get<mbgl::JSDocument>();
+ assert(tileset.HasMember("tiles"));
+ root.AddMember("tiles", (mbgl::JSValue&)tileset["tiles"], document.GetAllocator());
+ root.RemoveMember("url");
+ }
+ }
+ }
+
+ if (root.HasMember("tiles")) {
+ mbgl::JSValue& tilesValue = root["tiles"];
+ assert(tilesValue.IsArray());
+ for (auto& tileValue : tilesValue.GetArray()) {
+ const std::string path = prependFileScheme(localizeMapboxTilesURL(tileValue.GetString())
+ .value_or(localizeLocalURL(tileValue.GetString())
+ .value_or(localizeHttpURL(tileValue.GetString())
+ .value_or(tileValue.GetString()))));
+ tileValue.Set<std::string>(path, document.GetAllocator());
+ }
+ }
+
+ if (root.HasMember("data") && root["data"].IsString()) {
+ mbgl::JSValue& dataValue = root["data"];
+ const std::string path = prependFileScheme(localizeLocalURL(dataValue.GetString())
+ .value_or(dataValue.GetString()));
+ dataValue.Set<std::string>(path, document.GetAllocator());
+ }
+}
+
+void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) {
+ if (root.HasMember("sources")) {
+ mbgl::JSValue& sourcesValue = root["sources"];
+ for (auto& sourceProperty : sourcesValue.GetObject()) {
+ localizeSourceURLs(sourceProperty.value, document);
+ }
+ }
+
+ if (root.HasMember("glyphs")) {
+ mbgl::JSValue& glyphsValue = root["glyphs"];
+ const std::string path = prependFileScheme(localizeMapboxFontsURL(glyphsValue.GetString())
+ .value_or(localizeLocalURL(glyphsValue.GetString(), true)
+ .value_or(glyphsValue.GetString())));
+ glyphsValue.Set<std::string>(path, document.GetAllocator());
+ }
+
+ if (root.HasMember("sprite")) {
+ mbgl::JSValue& spriteValue = root["sprite"];
+ const std::string path = prependFileScheme(localizeMapboxSpriteURL(spriteValue.GetString())
+ .value_or(localizeLocalURL(spriteValue.GetString())
+ .value_or(spriteValue.GetString())));
+ spriteValue.Set<std::string>(path, document.GetAllocator());
+ }
+}
diff --git a/render-test/parser.hpp b/render-test/parser.hpp
new file mode 100644
index 0000000000..be98719ab5
--- /dev/null
+++ b/render-test/parser.hpp
@@ -0,0 +1,34 @@
+#pragma once
+
+#include <mbgl/util/rapidjson.hpp>
+#include <mbgl/util/variant.hpp>
+
+#include <tuple>
+#include <string>
+#include <vector>
+
+#include "filesystem.hpp"
+
+struct TestMetadata;
+struct TestStatistics;
+
+using ErrorMessage = std::string;
+using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>;
+
+using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector<std::string>>;
+
+JSONReply readJson(const mbgl::filesystem::path&);
+std::string serializeJsonValue(const mbgl::JSValue&);
+
+std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base);
+
+ArgumentsTuple parseArguments(int argc, char** argv);
+std::vector<std::pair<std::string, std::string>> parseIgnores();
+TestMetadata parseTestMetadata(const mbgl::filesystem::path& path);
+
+std::string createResultPage(const TestStatistics&, const std::vector<TestMetadata>&, bool shuffle, uint32_t seed);
+
+std::string localizeURL(const std::string& url);
+
+void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document);
+void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document); \ No newline at end of file
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
new file mode 100644
index 0000000000..1aa347f1a8
--- /dev/null
+++ b/render-test/runner.cpp
@@ -0,0 +1,408 @@
+#include <mbgl/map/camera.hpp>
+#include <mbgl/map/map_observer.hpp>
+#include <mbgl/style/conversion/filter.hpp>
+#include <mbgl/style/conversion/layer.hpp>
+#include <mbgl/style/conversion/light.hpp>
+#include <mbgl/style/conversion/source.hpp>
+#include <mbgl/style/image.hpp>
+#include <mbgl/style/layer.hpp>
+#include <mbgl/style/light.hpp>
+#include <mbgl/style/style.hpp>
+#include <mbgl/style/rapidjson_conversion.hpp>
+#include <mbgl/util/chrono.hpp>
+#include <mbgl/util/io.hpp>
+#include <mbgl/util/image.hpp>
+#include <mbgl/util/run_loop.hpp>
+#include <mbgl/util/string.hpp>
+#include <mbgl/util/timer.hpp>
+
+#include <mapbox/pixelmatch.hpp>
+
+#include "metadata.hpp"
+#include "parser.hpp"
+#include "runner.hpp"
+
+#include <algorithm>
+#include <cassert>
+#include <regex>
+
+bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& metadata) {
+ const std::string base = metadata.path.remove_filename().string();
+ metadata.actual = mbgl::encodePNG(actual);
+
+ if (actual.size.isEmpty()) {
+ metadata.errorMessage = "Invalid size for actual image";
+ return false;
+ }
+
+#if !TEST_READ_ONLY
+ if (getenv("UPDATE")) {
+ mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actual));
+ return true;
+ }
+
+ mbgl::util::write_file(base + "/actual.png", metadata.actual);
+#endif
+
+ mbgl::PremultipliedImage expected { actual.size };
+ mbgl::PremultipliedImage diff { actual.size };
+
+ double pixels = 0.0;
+
+ for (const auto& entry: readExpectedEntries(base)) {
+ mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
+ if (!maybeExpectedImage) {
+ metadata.errorMessage = "Failed to load expected image " + entry;
+ return false;
+ }
+
+ metadata.expected = *maybeExpectedImage;
+
+ expected = mbgl::decodeImage(*maybeExpectedImage);
+
+ pixels = // implicitly converting from uint64_t
+ mapbox::pixelmatch(actual.data.get(), expected.data.get(), expected.size.width,
+ expected.size.height, diff.data.get(), 0.16); // GL JS uses 0.1285
+
+ metadata.diff = mbgl::encodePNG(diff);
+
+#if !TEST_READ_ONLY
+ mbgl::util::write_file(base + "/diff.png", metadata.diff);
+#endif
+
+ metadata.difference = pixels / expected.size.area();
+ if (metadata.difference <= metadata.allowed) {
+ break;
+ }
+ }
+
+ return true;
+}
+
+bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
+ if (!metadata.document.HasMember("metadata") ||
+ !metadata.document["metadata"].HasMember("test") ||
+ !metadata.document["metadata"]["test"].HasMember("operations")) {
+ return true;
+ }
+
+ assert(metadata.document["metadata"]["test"]["operations"].IsArray());
+
+ const auto& operationsArray = metadata.document["metadata"]["test"]["operations"].GetArray();
+ if (operationsArray.Empty()) {
+ return true;
+ }
+
+ const auto& operationIt = operationsArray.Begin();
+ assert(operationIt->IsArray());
+
+ const auto& operationArray = operationIt->GetArray();
+ assert(operationArray.Size() >= 1u);
+
+ auto& frontend = maps[key]->frontend;
+ auto& map = maps[key]->map;
+
+ static const std::string waitOp("wait");
+ static const std::string sleepOp("sleep");
+ static const std::string addImageOp("addImage");
+ static const std::string updateImageOp("updateImage");
+ static const std::string removeImageOp("removeImage");
+ static const std::string setStyleOp("setStyle");
+ static const std::string setCenterOp("setCenter");
+ static const std::string setZoomOp("setZoom");
+ static const std::string setBearingOp("setBearing");
+ static const std::string setFilterOp("setFilter");
+ static const std::string setLayerZoomRangeOp("setLayerZoomRange");
+ static const std::string setLightOp("setLight");
+ static const std::string addLayerOp("addLayer");
+ static const std::string removeLayerOp("removeLayer");
+ static const std::string addSourceOp("addSource");
+ static const std::string removeSourceOp("removeSource");
+ static const std::string setPaintPropertyOp("setPaintProperty");
+ static const std::string setLayoutPropertyOp("setLayoutProperty");
+
+ // wait
+ if (operationArray[0].GetString() == waitOp) {
+ frontend.render(map);
+
+ // sleep
+ } else if (operationArray[0].GetString() == sleepOp) {
+ mbgl::util::Timer sleepTimer;
+ bool sleeping = true;
+
+ mbgl::Duration duration = mbgl::Seconds(3);
+ if (operationArray.Size() >= 2u) {
+ duration = mbgl::Milliseconds(operationArray[1].GetUint());
+ }
+
+ sleepTimer.start(duration, mbgl::Duration::zero(), [&]() {
+ sleeping = false;
+ });
+
+ while (sleeping) {
+ mbgl::util::RunLoop::Get()->runOnce();
+ }
+
+ // addImage | updateImage
+ } else if (operationArray[0].GetString() == addImageOp || operationArray[0].GetString() == updateImageOp) {
+ assert(operationArray.Size() >= 3u);
+
+ float pixelRatio = 1.0f;
+ bool sdf = false;
+
+ if (operationArray.Size() == 4u) {
+ assert(operationArray[3].IsObject());
+ const auto& imageOptions = operationArray[3].GetObject();
+ if (imageOptions.HasMember("pixelRatio")) {
+ pixelRatio = imageOptions["pixelRatio"].GetFloat();
+ }
+ if (imageOptions.HasMember("sdf")) {
+ sdf = imageOptions["sdf"].GetBool();
+ }
+ }
+
+ std::string imageName = operationArray[1].GetString();
+ imageName.erase(std::remove(imageName.begin(), imageName.end(), '"'), imageName.end());
+
+ std::string imagePath = operationArray[2].GetString();
+ imagePath.erase(std::remove(imagePath.begin(), imagePath.end(), '"'), imagePath.end());
+
+ const mbgl::filesystem::path filePath(std::string(TEST_RUNNER_ROOT_PATH) + "/mapbox-gl-js/test/integration/" + imagePath);
+
+ mbgl::optional<std::string> maybeImage = mbgl::util::readFile(filePath.string());
+ if (!maybeImage) {
+ metadata.errorMessage = std::string("Failed to load expected image ") + filePath.string();
+ return false;
+ }
+
+ map.getStyle().addImage(std::make_unique<mbgl::style::Image>(imageName, mbgl::decodeImage(*maybeImage), pixelRatio, sdf));
+
+ // removeImage
+ } else if (operationArray[0].GetString() == removeImageOp) {
+ assert(operationArray.Size() >= 2u);
+ assert(operationArray[1].IsString());
+
+ const std::string imageName { operationArray[1].GetString(), operationArray[1].GetStringLength() };
+ map.getStyle().removeImage(imageName);
+
+ // setStyle
+ } else if (operationArray[0].GetString() == setStyleOp) {
+ assert(operationArray.Size() >= 2u);
+ if (operationArray[1].IsString()) {
+ std::string stylePath = localizeURL(operationArray[1].GetString());
+ auto maybeStyle = readJson(stylePath);
+ if (maybeStyle.is<mbgl::JSDocument>()) {
+ auto& style = maybeStyle.get<mbgl::JSDocument>();
+ localizeStyleURLs((mbgl::JSValue&)style, style);
+ map.getStyle().loadJSON(serializeJsonValue(style));
+ }
+ } else {
+ localizeStyleURLs(operationArray[1], metadata.document);
+ map.getStyle().loadJSON(serializeJsonValue(operationArray[1]));
+ }
+
+ // setCenter
+ } else if (operationArray[0].GetString() == setCenterOp) {
+ assert(operationArray.Size() >= 2u);
+ assert(operationArray[1].IsArray());
+
+ const auto& centerArray = operationArray[1].GetArray();
+ assert(centerArray.Size() == 2u);
+
+ map.jumpTo(mbgl::CameraOptions().withCenter(mbgl::LatLng(centerArray[1].GetDouble(), centerArray[0].GetDouble())));
+
+ // setZoom
+ } else if (operationArray[0].GetString() == setZoomOp) {
+ assert(operationArray.Size() >= 2u);
+ assert(operationArray[1].IsNumber());
+ map.jumpTo(mbgl::CameraOptions().withZoom(operationArray[1].GetDouble()));
+
+ // setBearing
+ } else if (operationArray[0].GetString() == setBearingOp) {
+ assert(operationArray.Size() >= 2u);
+ assert(operationArray[1].IsNumber());
+ map.jumpTo(mbgl::CameraOptions().withBearing(operationArray[1].GetDouble()));
+
+ // setFilter
+ } else if (operationArray[0].GetString() == setFilterOp) {
+ assert(operationArray.Size() >= 3u);
+ assert(operationArray[1].IsString());
+
+ const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() };
+
+ mbgl::style::conversion::Error error;
+ auto converted = mbgl::style::conversion::convert<mbgl::style::Filter>(operationArray[2], error);
+ if (!converted) {
+ metadata.errorMessage = std::string("Unable to convert filter: ") + error.message;
+ return false;
+ } else {
+ auto layer = map.getStyle().getLayer(layerName);
+ if (!layer) {
+ metadata.errorMessage = std::string("Layer not found: ") + layerName;
+ return false;
+ } else {
+ layer->setFilter(std::move(*converted));
+ }
+ }
+
+ // setLayerZoomRange
+ } else if (operationArray[0].GetString() == setLayerZoomRangeOp) {
+ assert(operationArray.Size() >= 4u);
+ assert(operationArray[1].IsString());
+ assert(operationArray[2].IsNumber());
+ assert(operationArray[3].IsNumber());
+
+ const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() };
+ auto layer = map.getStyle().getLayer(layerName);
+ if (!layer) {
+ metadata.errorMessage = std::string("Layer not found: ") + layerName;
+ return false;
+ } else {
+ layer->setMinZoom(operationArray[2].GetFloat());
+ layer->setMaxZoom(operationArray[3].GetFloat());
+ }
+
+ // setLight
+ } else if (operationArray[0].GetString() == setLightOp) {
+ assert(operationArray.Size() >= 2u);
+ assert(operationArray[1].IsObject());
+
+ mbgl::style::conversion::Error error;
+ auto converted = mbgl::style::conversion::convert<mbgl::style::Light>(operationArray[1], error);
+ if (!converted) {
+ metadata.errorMessage = std::string("Unable to convert light: ") + error.message;
+ return false;
+ } else {
+ map.getStyle().setLight(std::make_unique<mbgl::style::Light>(std::move(*converted)));
+ }
+
+ // addLayer
+ } else if (operationArray[0].GetString() == addLayerOp) {
+ assert(operationArray.Size() >= 2u);
+ assert(operationArray[1].IsObject());
+
+ mbgl::style::conversion::Error error;
+ auto converted = mbgl::style::conversion::convert<std::unique_ptr<mbgl::style::Layer>>(operationArray[1], error);
+ if (!converted) {
+ metadata.errorMessage = std::string("Unable to convert layer: ") + error.message;
+ return false;
+ } else {
+ map.getStyle().addLayer(std::move(*converted));
+ }
+
+ // removeLayer
+ } else if (operationArray[0].GetString() == removeLayerOp) {
+ assert(operationArray.Size() >= 2u);
+ assert(operationArray[1].IsString());
+ map.getStyle().removeLayer(operationArray[1].GetString());
+
+ // addSource
+ } else if (operationArray[0].GetString() == addSourceOp) {
+ assert(operationArray.Size() >= 3u);
+ assert(operationArray[1].IsString());
+ assert(operationArray[2].IsObject());
+
+ localizeSourceURLs(operationArray[2], metadata.document);
+
+ mbgl::style::conversion::Error error;
+ auto converted = mbgl::style::conversion::convert<std::unique_ptr<mbgl::style::Source>>(operationArray[2], error, operationArray[1].GetString());
+ if (!converted) {
+ metadata.errorMessage = std::string("Unable to convert source: ") + error.message;
+ return false;
+ } else {
+ map.getStyle().addSource(std::move(*converted));
+ }
+
+ // removeSource
+ } else if (operationArray[0].GetString() == removeSourceOp) {
+ assert(operationArray.Size() >= 2u);
+ assert(operationArray[1].IsString());
+ map.getStyle().removeSource(operationArray[1].GetString());
+
+ // setPaintProperty
+ } else if (operationArray[0].GetString() == setPaintPropertyOp) {
+ assert(operationArray.Size() >= 4u);
+ assert(operationArray[1].IsString());
+ assert(operationArray[2].IsString());
+
+ const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() };
+ const std::string propertyName { operationArray[2].GetString(), operationArray[2].GetStringLength() };
+
+ auto layer = map.getStyle().getLayer(layerName);
+ if (!layer) {
+ metadata.errorMessage = std::string("Layer not found: ") + layerName;
+ return false;
+ } else {
+ const mbgl::JSValue* propertyValue = &operationArray[3];
+ layer->setPaintProperty(propertyName, propertyValue);
+ }
+
+ // setLayoutProperty
+ } else if (operationArray[0].GetString() == setLayoutPropertyOp) {
+ assert(operationArray.Size() >= 4u);
+ assert(operationArray[1].IsString());
+ assert(operationArray[2].IsString());
+
+ const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() };
+ const std::string propertyName { operationArray[2].GetString(), operationArray[2].GetStringLength() };
+
+ auto layer = map.getStyle().getLayer(layerName);
+ if (!layer) {
+ metadata.errorMessage = std::string("Layer not found: ") + layerName;
+ return false;
+ } else {
+ const mbgl::JSValue* propertyValue = &operationArray[3];
+ layer->setLayoutProperty(propertyName, propertyValue);
+ }
+
+ } else {
+ metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString();
+ return false;
+ }
+
+ operationsArray.Erase(operationIt);
+ return runOperations(key, metadata);
+}
+
+TestRunner::Impl::Impl(const TestMetadata& metadata)
+ : frontend(metadata.size, metadata.pixelRatio),
+ map(frontend,
+ mbgl::MapObserver::nullObserver(),
+ mbgl::MapOptions()
+ .withMapMode(metadata.mapMode)
+ .withSize(metadata.size)
+ .withPixelRatio(metadata.pixelRatio)
+ .withCrossSourceCollisions(metadata.crossSourceCollisions),
+ mbgl::ResourceOptions().withCacheOnlyRequestsSupport(false)) {}
+
+bool TestRunner::run(TestMetadata& metadata) {
+ std::string key = mbgl::util::toString(uint32_t(metadata.mapMode))
+ + "/" + mbgl::util::toString(metadata.pixelRatio)
+ + "/" + mbgl::util::toString(uint32_t(metadata.crossSourceCollisions));
+
+ if (maps.find(key) == maps.end()) {
+ maps[key] = std::make_unique<TestRunner::Impl>(metadata);
+ }
+
+ auto& frontend = maps[key]->frontend;
+ auto& map = maps[key]->map;
+
+ frontend.setSize(metadata.size);
+ map.setSize(metadata.size);
+
+ map.setProjectionMode(mbgl::ProjectionMode().withAxonometric(metadata.axonometric).withXSkew(metadata.xSkew).withYSkew(metadata.ySkew));
+ map.setDebug(metadata.debug);
+
+ map.getStyle().loadJSON(serializeJsonValue(metadata.document));
+ map.jumpTo(map.getStyle().getDefaultCamera());
+
+ if (!runOperations(key, metadata)) {
+ return false;
+ }
+
+ return checkImage(frontend.render(map), metadata);
+}
+
+void TestRunner::reset() {
+ maps.clear();
+}
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
new file mode 100644
index 0000000000..cbc0f42546
--- /dev/null
+++ b/render-test/runner.hpp
@@ -0,0 +1,28 @@
+#pragma once
+
+#include <mbgl/gfx/headless_frontend.hpp>
+#include <mbgl/map/map.hpp>
+
+#include <memory>
+
+struct TestMetadata;
+
+class TestRunner {
+public:
+ TestRunner() = default;
+
+ bool run(TestMetadata&);
+ void reset();
+
+private:
+ bool runOperations(const std::string& key, TestMetadata&);
+ bool checkImage(mbgl::PremultipliedImage&& image, TestMetadata&);
+
+ struct Impl {
+ Impl(const TestMetadata&);
+
+ mbgl::HeadlessFrontend frontend;
+ mbgl::Map map;
+ };
+ std::unordered_map<std::string, std::unique_ptr<Impl>> maps;
+}; \ No newline at end of file