diff options
Diffstat (limited to 'render-test')
| -rw-r--r-- | render-test/filesystem.hpp | 9 | ||||
| -rw-r--r-- | render-test/main.cpp | 154 | ||||
| -rw-r--r-- | render-test/metadata.hpp | 52 | ||||
| -rw-r--r-- | render-test/parser.cpp | 572 | ||||
| -rw-r--r-- | render-test/parser.hpp | 34 | ||||
| -rw-r--r-- | render-test/runner.cpp | 408 | ||||
| -rw-r--r-- | render-test/runner.hpp | 28 |
7 files changed, 1257 insertions, 0 deletions
diff --git a/render-test/filesystem.hpp b/render-test/filesystem.hpp new file mode 100644 index 0000000000..cee7e9d911 --- /dev/null +++ b/render-test/filesystem.hpp @@ -0,0 +1,9 @@ +#pragma once + +#include <ghc/filesystem.hpp> + +namespace mbgl { + +namespace filesystem = ghc::filesystem; + +} // namespace mbgl diff --git a/render-test/main.cpp b/render-test/main.cpp new file mode 100644 index 0000000000..35f0cdea30 --- /dev/null +++ b/render-test/main.cpp @@ -0,0 +1,154 @@ +#include <mbgl/util/run_loop.hpp> +#include <mbgl/util/io.hpp> + +#include "filesystem.hpp" +#include "metadata.hpp" +#include "parser.hpp" +#include "runner.hpp" + +#include <random> + +#define ANSI_COLOR_RED "\x1b[31m" +#define ANSI_COLOR_GREEN "\x1b[32m" +#define ANSI_COLOR_YELLOW "\x1b[33m" +#define ANSI_COLOR_BLUE "\x1b[34m" +#define ANSI_COLOR_MAGENTA "\x1b[35m" +#define ANSI_COLOR_CYAN "\x1b[36m" +#define ANSI_COLOR_GRAY "\x1b[37m" +#define ANSI_COLOR_LIGHT_GRAY "\x1b[90m" +#define ANSI_COLOR_RESET "\x1b[0m" + +int main(int argc, char** argv) { + bool recycleMap; + bool shuffle; + uint32_t seed; + std::string testRootPath; + std::vector<std::string> ids; + + std::tie(recycleMap, shuffle, seed, testRootPath, ids) = parseArguments(argc, argv); + const std::string::size_type rootLength = testRootPath.length(); + + const auto ignores = parseIgnores(); + + // Recursively traverse through the test paths and collect test directories containing "style.json". + std::vector<mbgl::filesystem::path> testPaths; + for (const auto& id : ids) { + for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(mbgl::filesystem::path(id))) { + if (testPath.path().filename() == "style.json") { + testPaths.push_back(testPath); + } + } + } + + if (shuffle) { + printf(ANSI_COLOR_YELLOW "Shuffle seed: %d" ANSI_COLOR_RESET "\n", seed); + + std::seed_seq sequence { seed }; + std::mt19937 shuffler(sequence); + std::shuffle(testPaths.begin(), testPaths.end(), shuffler); + } + + mbgl::util::RunLoop runLoop; + TestRunner runner; + + std::vector<TestMetadata> metadatas; + metadatas.reserve(testPaths.size()); + + TestStatistics stats; + + for (auto& testPath : testPaths) { + TestMetadata metadata = parseTestMetadata(testPath); + + if (!recycleMap) { + runner.reset(); + } + + std::string& id = metadata.id; + std::string& status = metadata.status; + std::string& color = metadata.color; + + id = testPath.remove_filename().string(); + id = id.substr(rootLength + 1, id.length() - rootLength - 2); + + bool shouldIgnore = false; + std::string ignoreReason; + + const std::string ignoreName = "render-tests/" + id; + const auto it = std::find_if(ignores.cbegin(), ignores.cend(), [&ignoreName](auto pair) { return pair.first == ignoreName; }); + if (it != ignores.end()) { + shouldIgnore = true; + ignoreReason = it->second; + if (ignoreReason.rfind("skip", 0) == 0) { + printf(ANSI_COLOR_GRAY "* skipped %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str()); + continue; + } + } + + bool errored = !metadata.errorMessage.empty(); + if (!errored) { + errored = runner.run(metadata) && !metadata.errorMessage.empty(); + } + + bool passed = !errored && !metadata.diff.empty() && metadata.difference <= metadata.allowed; + + if (shouldIgnore) { + if (passed) { + status = "ignored passed"; + color = "#E8A408"; + stats.ignorePassedTests++; + printf(ANSI_COLOR_YELLOW "* ignore %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str()); + } else { + status = "ignored failed"; + color = "#9E9E9E"; + stats.ignoreFailedTests++; + printf(ANSI_COLOR_LIGHT_GRAY "* ignore %s (%s)" ANSI_COLOR_RESET "\n", id.c_str(), ignoreReason.c_str()); + } + } else { + if (passed) { + status = "passed"; + color = "green"; + stats.passedTests++; + printf(ANSI_COLOR_GREEN "* passed %s" ANSI_COLOR_RESET "\n", id.c_str()); + } else if (errored) { + status = "errored"; + color = "red"; + stats.erroredTests++; + printf(ANSI_COLOR_RED "* errored %s" ANSI_COLOR_RESET "\n", id.c_str()); + } else { + status = "failed"; + color = "red"; + stats.failedTests++; + printf(ANSI_COLOR_RED "* failed %s" ANSI_COLOR_RESET "\n", id.c_str()); + } + } + + metadatas.push_back(std::move(metadata)); + } + + std::string resultsHTML = createResultPage(stats, metadatas, shuffle, seed); + mbgl::util::write_file(testRootPath + "/index.html", resultsHTML); + + const uint32_t count = stats.erroredTests + stats.failedTests + + stats.ignoreFailedTests + stats.ignorePassedTests + + stats.passedTests; + + if (stats.passedTests) { + printf(ANSI_COLOR_GREEN "%u passed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.passedTests, 100.0 * stats.passedTests / count); + } + if (stats.ignorePassedTests) { + printf(ANSI_COLOR_YELLOW "%u passed but were ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.ignorePassedTests, 100.0 * stats.ignorePassedTests / count); + } + if (stats.ignoreFailedTests) { + printf(ANSI_COLOR_LIGHT_GRAY "%u ignored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.ignoreFailedTests, 100.0 * stats.ignoreFailedTests / count); + } + if (stats.failedTests) { + printf(ANSI_COLOR_RED "%u failed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.failedTests, 100.0 * stats.failedTests / count); + } + if (stats.erroredTests) { + printf(ANSI_COLOR_RED "%u errored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.erroredTests, 100.0 * stats.erroredTests / count); + } + + printf("Results at: %s%s\n", testRootPath.c_str(), "/index.html"); + + return stats.failedTests + stats.erroredTests == 0 ? 0 : 1; +} diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp new file mode 100644 index 0000000000..4be83a5436 --- /dev/null +++ b/render-test/metadata.hpp @@ -0,0 +1,52 @@ +#pragma once + +#include <mbgl/util/rapidjson.hpp> +#include <mbgl/util/size.hpp> + +#include <mbgl/map/mode.hpp> + +#include "filesystem.hpp" + +struct TestStatistics { + TestStatistics() = default; + + uint32_t ignoreFailedTests = 0; + uint32_t ignorePassedTests = 0; + uint32_t erroredTests = 0; + uint32_t failedTests = 0; + uint32_t passedTests = 0; +}; + +struct TestMetadata { + TestMetadata() = default; + + mbgl::filesystem::path path; + mbgl::JSDocument document; + + mbgl::Size size{ 512u, 512u }; + float pixelRatio = 1.0f; + double allowed = 0.00015; // diff + std::string description; + mbgl::MapMode mapMode = mbgl::MapMode::Static; + mbgl::MapDebugOptions debug = mbgl::MapDebugOptions::NoDebug; + bool crossSourceCollisions = true; + bool axonometric = false; + double xSkew = 0.0; + double ySkew = 1.0; + + // TODO + uint32_t fadeDuration = 0; + bool addFakeCanvas = false; + + // HTML + std::string id; + std::string status; + std::string color; + + std::string actual; + std::string expected; + std::string diff; + + std::string errorMessage; + double difference = 0.0; +};
\ No newline at end of file diff --git a/render-test/parser.cpp b/render-test/parser.cpp new file mode 100644 index 0000000000..089a5c45c9 --- /dev/null +++ b/render-test/parser.cpp @@ -0,0 +1,572 @@ +#include <mbgl/util/logging.hpp> +#include <mbgl/util/io.hpp> +#include <mbgl/util/rapidjson.hpp> +#include <mbgl/util/string.hpp> + +#include <args.hxx> + +#include <rapidjson/stringbuffer.h> +#include <rapidjson/writer.h> + +#include <boost/archive/iterators/base64_from_binary.hpp> +#include <boost/archive/iterators/insert_linebreaks.hpp> +#include <boost/archive/iterators/transform_width.hpp> +#include <boost/archive/iterators/ostream_iterator.hpp> + +#include "parser.hpp" +#include "metadata.hpp" + +#include <sstream> +#include <regex> + +namespace { + +const char* resultsStyle = R"HTML( +<style> + body { font: 18px/1.2 -apple-system, BlinkMacSystemFont, "Helvetica Neue", Helvetica, Arial, sans-serif; padding: 10px; } + h1 { font-size: 32px; margin-bottom: 0; } + button { vertical-align: middle; } + h2 { font-size: 24px; font-weight: normal; margin: 10px 0 10px; line-height: 1; } + img { margin: 0 10px 10px 0; border: 1px dotted #ccc; } + .stats { margin-top: 10px; } + .test { border-bottom: 1px dotted #bbb; padding-bottom: 5px; } + .tests { border-top: 1px dotted #bbb; margin-top: 10px; } + .diff { color: #777; } + .test p, .test pre { margin: 0 0 10px; } + .test pre { font-size: 14px; } + .label { color: white; font-size: 18px; padding: 2px 6px 3px; border-radius: 3px; margin-right: 3px; vertical-align: bottom; display: inline-block; } + .hide { display: none; } +</style> +)HTML"; + +const char* resultsScript = R"HTML( +<script> +document.addEventListener('mouseover', handleHover); +document.addEventListener('mouseout', handleHover); + +function handleHover(e) { + var el = e.target; + if (el.tagName === 'IMG' && el.dataset.altSrc) { + var tmp = el.src; + el.src = el.dataset.altSrc; + el.dataset.altSrc = tmp; + } +} + +document.getElementById('toggle-passed').addEventListener('click', function (e) { + for (const row of document.querySelectorAll('.test.passed')) { + row.classList.toggle('hide'); + } +}); +document.getElementById('toggle-ignored').addEventListener('click', function (e) { + for (const row of document.querySelectorAll('.test.ignored')) { + row.classList.toggle('hide'); + } +}); +document.getElementById('toggle-sequence').addEventListener('click', function (e) { + document.getElementById('test-sequence').classList.toggle('hide'); +}); +</script> +)HTML"; + +const char* resultsHeaderButtons = R"HTML( + <button id='toggle-sequence'>Toggle test sequence</button> + <button id='toggle-passed'>Toggle passed tests</button> + <button id='toggle-ignored'>Toggle ignored tests</button> +</h1> +)HTML"; + +std::string removeURLArguments(const std::string &url) { + std::string::size_type index = url.find('?'); + if (index != std::string::npos) { + return url.substr(0, index); + } + return url; +} + +std::string prependFileScheme(const std::string &url) { + static const std::string fileScheme("file://"); + return fileScheme + url; +} + +mbgl::optional<std::string> getVendorPath(const std::string& url, const std::regex& regex, bool glyphsPath = false) { + static const mbgl::filesystem::path vendorPath(std::string(TEST_RUNNER_ROOT_PATH) + "/vendor/"); + + mbgl::filesystem::path file = std::regex_replace(url, regex, vendorPath.string()); + if (mbgl::filesystem::exists(file.parent_path())) { + return removeURLArguments(file.string()); + } + + if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) { + return removeURLArguments(file.string()); + } + + return {}; +} + +mbgl::optional<std::string> getIntegrationPath(const std::string& url, const std::string& parent, const std::regex& regex, bool glyphsPath = false) { + static const mbgl::filesystem::path integrationPath(std::string(TEST_RUNNER_ROOT_PATH) + "/mapbox-gl-js/test/integration/"); + + mbgl::filesystem::path file = std::regex_replace(url, regex, integrationPath.string() + parent); + if (mbgl::filesystem::exists(file.parent_path())) { + return removeURLArguments(file.string()); + } + + if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) { + return removeURLArguments(file.string()); + } + + return {}; +} + +mbgl::optional<std::string> localizeLocalURL(const std::string& url, bool glyphsPath = false) { + static const std::regex regex { "local://" }; + if (auto vendorPath = getVendorPath(url, regex, glyphsPath)) { + return vendorPath; + } else { + return getIntegrationPath(url, "", regex, glyphsPath); + } +} + +mbgl::optional<std::string> localizeHttpURL(const std::string& url) { + static const std::regex regex { "http://localhost:2900" }; + if (auto vendorPath = getVendorPath(url, regex)) { + return vendorPath; + } else { + return getIntegrationPath(url, "", regex); + } +} + +mbgl::optional<std::string> localizeMapboxSpriteURL(const std::string& url) { + static const std::regex regex { "mapbox://" }; + return getIntegrationPath(url, "", regex); +} + +mbgl::optional<std::string> localizeMapboxFontsURL(const std::string& url) { + static const std::regex regex { "mapbox://fonts" }; + return getIntegrationPath(url, "glyphs/", regex, true); +} + +mbgl::optional<std::string> localizeMapboxTilesURL(const std::string& url) { + static const std::regex regex { "mapbox://" }; + if (auto vendorPath = getVendorPath(url, regex)) { + return vendorPath; + } else { + return getIntegrationPath(url, "tiles/", regex); + } +} + +mbgl::optional<std::string> localizeMapboxTilesetURL(const std::string& url) { + static const std::regex regex { "mapbox://" }; + return getIntegrationPath(url, "tilesets/", regex); +} + +} // namespace + +JSONReply readJson(const mbgl::filesystem::path& jsonPath) { + auto maybeJSON = mbgl::util::readFile(jsonPath); + if (!maybeJSON) { + return { std::string("Unable to open file ") + jsonPath.string() }; + } + + mbgl::JSDocument document; + document.Parse<0>(*maybeJSON); + if (document.HasParseError()) { + return { mbgl::formatJSONParseError(document) }; + } + + return { std::move(document) }; +} + +std::string serializeJsonValue(const mbgl::JSValue& value) { + rapidjson::StringBuffer buffer; + buffer.Clear(); + rapidjson::Writer<rapidjson::StringBuffer> writer(buffer); + value.Accept(writer); + return buffer.GetString(); +} + +std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base) { + static const std::regex regex(".*expected.*.png"); + + std::vector<std::string> expectedImages; + for (const auto& entry : mbgl::filesystem::directory_iterator(base)) { + if (entry.is_regular_file()) { + const std::string path = entry.path().string(); + if (std::regex_match(path, regex)) { + expectedImages.emplace_back(std::move(path)); + } + } + } + return expectedImages; +} + + +ArgumentsTuple parseArguments(int argc, char** argv) { + args::ArgumentParser argumentParser("Mapbox GL Test Runner"); + + args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", { 'h', "help" }); + + args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", + { 'r', "recycle-map" }); + args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order", + { 's', "shuffle" }); + args::ValueFlag<uint32_t> seedValue(argumentParser, "seed", "Shuffle seed (default: random)", + { "seed" }); + args::ValueFlag<std::string> testPathValue(argumentParser, "rootPath", "Test root rootPath", + { 'p', "rootPath" }); + args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)"); + + try { + argumentParser.ParseCLI(argc, argv); + } catch (const args::Help&) { + std::ostringstream stream; + stream << argumentParser; + mbgl::Log::Info(mbgl::Event::General, stream.str()); + exit(0); + } catch (const args::ParseError& e) { + std::ostringstream stream; + stream << argumentParser; + mbgl::Log::Info(mbgl::Event::General, stream.str()); + mbgl::Log::Error(mbgl::Event::General, e.what()); + exit(1); + } catch (const args::ValidationError& e) { + std::ostringstream stream; + stream << argumentParser; + mbgl::Log::Info(mbgl::Event::General, stream.str()); + mbgl::Log::Error(mbgl::Event::General, e.what()); + exit(2); + } + + const std::string testDefaultPath = + std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration/render-tests"); + + std::vector<std::string> ids; + for (const auto& id : args::get(testNameValues)) { + ids.emplace_back(testDefaultPath + "/" + id); + } + + if (ids.empty()) { + ids.emplace_back(testDefaultPath); + } + + return ArgumentsTuple { + recycleMapFlag ? args::get(recycleMapFlag) : false, + shuffleFlag ? args::get(shuffleFlag) : false, seedValue ? args::get(seedValue) : 1u, + testPathValue ? args::get(testPathValue) : testDefaultPath, ids + }; +} + +std::vector<std::pair<std::string, std::string>> parseIgnores() { + std::vector<std::pair<std::string, std::string>> ignores; + + auto path = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("platform/node/test/ignores.json"); + + auto maybeIgnores = readJson(path.string()); + if (maybeIgnores.is<mbgl::JSDocument>()) { + for (const auto& property : maybeIgnores.get<mbgl::JSDocument>().GetObject()) { + const std::string ignore = { property.name.GetString(), + property.name.GetStringLength() }; + const std::string reason = { property.value.GetString(), + property.value.GetStringLength() }; + ignores.emplace_back(std::make_pair(ignore, reason)); + } + } + + return ignores; +} + +TestMetadata parseTestMetadata(const mbgl::filesystem::path& path) { + TestMetadata metadata; + metadata.path = path; + + auto maybeJson = readJson(path.string()); + if (!maybeJson.is<mbgl::JSDocument>()) { // NOLINT + metadata.errorMessage = std::string("Unable to parse: ") + path.string(); + return metadata; + } + + metadata.document = std::move(maybeJson.get<mbgl::JSDocument>()); + localizeStyleURLs(metadata.document, metadata.document); + + if (!metadata.document.HasMember("metadata")) { + mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s", + path.c_str()); + return metadata; + } + + const mbgl::JSValue& metadataValue = metadata.document["metadata"]; + if (!metadataValue.HasMember("test")) { + mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata.test': %s", + path.c_str()); + return metadata; + } + + const mbgl::JSValue& testValue = metadataValue["test"]; + + if (testValue.HasMember("width")) { + assert(testValue["width"].IsNumber()); + metadata.size.width = testValue["width"].GetInt(); + } + + if (testValue.HasMember("height")) { + assert(testValue["height"].IsNumber()); + metadata.size.height = testValue["height"].GetInt(); + } + + if (testValue.HasMember("pixelRatio")) { + assert(testValue["pixelRatio"].IsNumber()); + metadata.pixelRatio = testValue["pixelRatio"].GetFloat(); + } + + if (testValue.HasMember("allowed")) { + assert(testValue["allowed"].IsNumber()); + metadata.allowed = testValue["allowed"].GetDouble(); + } + + if (testValue.HasMember("description")) { + assert(testValue["description"].IsString()); + metadata.description = std::string{ testValue["description"].GetString(), + testValue["description"].GetStringLength() }; + } + + if (testValue.HasMember("mapMode")) { + assert(testValue["mapMode"].IsString()); + metadata.mapMode = testValue["mapMode"].GetString() == std::string("tile") ? mbgl::MapMode::Tile : mbgl::MapMode::Static; + } + + // Test operations handled in runner.cpp. + + if (testValue.HasMember("debug")) { + metadata.debug |= mbgl::MapDebugOptions::TileBorders; + } + + if (testValue.HasMember("collisionDebug")) { + metadata.debug |= mbgl::MapDebugOptions::Collision; + } + + if (testValue.HasMember("showOverdrawInspector")) { + metadata.debug |= mbgl::MapDebugOptions::Overdraw; + } + + if (testValue.HasMember("crossSourceCollisions")) { + assert(testValue["crossSourceCollisions"].IsBool()); + metadata.crossSourceCollisions = testValue["crossSourceCollisions"].GetBool(); + } + + if (testValue.HasMember("axonometric")) { + assert(testValue["axonometric"].IsBool()); + metadata.axonometric = testValue["axonometric"].GetBool(); + } + + if (testValue.HasMember("skew")) { + assert(testValue["skew"].IsArray()); + metadata.xSkew = testValue["skew"][0].GetDouble(); + metadata.ySkew = testValue["skew"][1].GetDouble(); + } + + // TODO: fadeDuration + // TODO: addFakeCanvas + + return metadata; +} + +// https://stackoverflow.com/questions/7053538/how-do-i-encode-a-string-to-base64-using-only-boost +std::string encodeBase64(const std::string& data) { + using namespace boost::archive::iterators; + using base64 = insert_linebreaks<base64_from_binary<transform_width<const char*, 6, 8>>, 72>; + + std::stringstream os; + std::copy(base64(data.c_str()), base64(data.c_str() + data.size()), ostream_iterator<char>(os)); + return os.str(); +} + +std::string createResultItem(const TestMetadata& metadata, bool hasFailedTests) { + const bool shouldHide = (hasFailedTests && metadata.status == "passed") || (metadata.status.find("ignored") != std::string::npos); + + std::string html; + html.append("<div class=\"test " + metadata.status + (shouldHide ? " hide" : "") + "\">\n"); + html.append(R"(<h2><span class="label" style="background: )" + metadata.color + "\">" + metadata.status + "</span> " + metadata.id + "</h2>\n"); + if (metadata.status != "errored") { + html.append("<img width=" + mbgl::util::toString(metadata.size.width)); + html.append(" height=" + mbgl::util::toString(metadata.size.height)); + html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.actual) + "\""); + html.append(" data-alt-src=\"data:image/png;base64," + encodeBase64(metadata.expected) + "\">\n"); + + html.append("<img width=" + mbgl::util::toString(metadata.size.width)); + html.append(" height=" + mbgl::util::toString(metadata.size.height)); + html.append(" src=\"data:image/png;base64," + encodeBase64(metadata.diff) + "\">\n"); + } else { + assert(!metadata.errorMessage.empty()); + html.append("<p style=\"color: red\"><strong>Error:</strong> " + metadata.errorMessage + "</p>\n"); + } + if (metadata.difference != 0.0) { + html.append("<p class=\"diff\"><strong>Diff:</strong> " + mbgl::util::toString(metadata.difference) + "</p>\n"); + } + html.append("</div>\n"); + + return html; +} + +std::string createResultPage(const TestStatistics& stats, const std::vector<TestMetadata>& metadatas, bool shuffle, uint32_t seed) { + const uint32_t unsuccessful = stats.erroredTests + stats.failedTests; + std::string resultsPage; + + // Style + resultsPage.append(resultsStyle); + + // Header + if (unsuccessful) { + resultsPage.append(R"HTML(<h1 style="color: red;">)HTML"); + resultsPage.append(mbgl::util::toString(unsuccessful) + " tests failed."); + } else { + resultsPage.append(R"HTML(<h1 style="color: green;">)HTML"); + resultsPage.append("All tests passed!"); + } + resultsPage.append(resultsHeaderButtons); + + // stats + resultsPage.append(R"HTML(<p class="stats">)HTML"); + if (stats.ignoreFailedTests) { + resultsPage.append(mbgl::util::toString(stats.ignoreFailedTests) + " ignored failed, "); + } + if (stats.ignorePassedTests) { + resultsPage.append(mbgl::util::toString(stats.ignorePassedTests) + " ignored passed, "); + } + if (stats.erroredTests) { + resultsPage.append(mbgl::util::toString(stats.erroredTests) + " errored, "); + } + if (stats.failedTests) { + resultsPage.append(mbgl::util::toString(stats.failedTests) + " failed, "); + } + resultsPage.append(mbgl::util::toString(stats.passedTests) + " passed.\n"); + resultsPage.append("</p>\n"); + + // Test sequence + { + resultsPage.append("<div id='test-sequence' class='hide'>\n"); + + // Failed tests + if (unsuccessful) { + resultsPage.append("<p><strong>Failed tests:</strong>"); + for (const auto& metadata : metadatas) { + if (metadata.status == "failed" || metadata.status == "errored") { + resultsPage.append(metadata.id + " "); + } + } + resultsPage.append("</p>\n"); + } + + // Test sequence + resultsPage.append("<p><strong>Test sequence:</strong>"); + for (const auto& metadata : metadatas) { + resultsPage.append(metadata.id + " "); + } + resultsPage.append("</p>\n"); + + // Shuffle + if (shuffle) { + resultsPage.append("<p><strong>Shuffle seed</strong>: " + mbgl::util::toString(seed) + "</p>\n"); + } + + resultsPage.append("</div>\n"); + } + + // Script + resultsPage.append(resultsScript); + + // Tests + resultsPage.append("<div class=\"tests\">\n"); + for (const auto& metadata : metadatas) { + resultsPage.append(createResultItem(metadata, unsuccessful)); + } + resultsPage.append("</div>\n"); + + return resultsPage; +} + +std::string localizeURL(const std::string& url) { + static const std::regex regex { "local://" }; + if (auto vendorPath = getVendorPath(url, regex)) { + return *vendorPath; + } else { + return getIntegrationPath(url, "", regex).value_or(url); + } +} + +void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) { + if (root.HasMember("urls") && root["urls"].IsArray()) { + for (auto& urlValue : root["urls"].GetArray()) { + const std::string path = prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString()) + .value_or(localizeLocalURL(urlValue.GetString()) + .value_or(urlValue.GetString()))); + urlValue.Set<std::string>(path, document.GetAllocator()); + } + } + + if (root.HasMember("url")) { + static const std::string image("image"); + static const std::string video("video"); + + mbgl::JSValue& urlValue = root["url"]; + const std::string path = prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString()) + .value_or(localizeLocalURL(urlValue.GetString()) + .value_or(urlValue.GetString()))); + urlValue.Set<std::string>(path, document.GetAllocator()); + + if (root["type"].GetString() != image && root["type"].GetString() != video) { + const auto tilesetPath = std::string(urlValue.GetString()).erase(0u, 7u); // remove "file://" + auto maybeTileset = readJson(tilesetPath); + if (maybeTileset.is<mbgl::JSDocument>()) { + const auto& tileset = maybeTileset.get<mbgl::JSDocument>(); + assert(tileset.HasMember("tiles")); + root.AddMember("tiles", (mbgl::JSValue&)tileset["tiles"], document.GetAllocator()); + root.RemoveMember("url"); + } + } + } + + if (root.HasMember("tiles")) { + mbgl::JSValue& tilesValue = root["tiles"]; + assert(tilesValue.IsArray()); + for (auto& tileValue : tilesValue.GetArray()) { + const std::string path = prependFileScheme(localizeMapboxTilesURL(tileValue.GetString()) + .value_or(localizeLocalURL(tileValue.GetString()) + .value_or(localizeHttpURL(tileValue.GetString()) + .value_or(tileValue.GetString())))); + tileValue.Set<std::string>(path, document.GetAllocator()); + } + } + + if (root.HasMember("data") && root["data"].IsString()) { + mbgl::JSValue& dataValue = root["data"]; + const std::string path = prependFileScheme(localizeLocalURL(dataValue.GetString()) + .value_or(dataValue.GetString())); + dataValue.Set<std::string>(path, document.GetAllocator()); + } +} + +void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) { + if (root.HasMember("sources")) { + mbgl::JSValue& sourcesValue = root["sources"]; + for (auto& sourceProperty : sourcesValue.GetObject()) { + localizeSourceURLs(sourceProperty.value, document); + } + } + + if (root.HasMember("glyphs")) { + mbgl::JSValue& glyphsValue = root["glyphs"]; + const std::string path = prependFileScheme(localizeMapboxFontsURL(glyphsValue.GetString()) + .value_or(localizeLocalURL(glyphsValue.GetString(), true) + .value_or(glyphsValue.GetString()))); + glyphsValue.Set<std::string>(path, document.GetAllocator()); + } + + if (root.HasMember("sprite")) { + mbgl::JSValue& spriteValue = root["sprite"]; + const std::string path = prependFileScheme(localizeMapboxSpriteURL(spriteValue.GetString()) + .value_or(localizeLocalURL(spriteValue.GetString()) + .value_or(spriteValue.GetString()))); + spriteValue.Set<std::string>(path, document.GetAllocator()); + } +} diff --git a/render-test/parser.hpp b/render-test/parser.hpp new file mode 100644 index 0000000000..be98719ab5 --- /dev/null +++ b/render-test/parser.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include <mbgl/util/rapidjson.hpp> +#include <mbgl/util/variant.hpp> + +#include <tuple> +#include <string> +#include <vector> + +#include "filesystem.hpp" + +struct TestMetadata; +struct TestStatistics; + +using ErrorMessage = std::string; +using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>; + +using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector<std::string>>; + +JSONReply readJson(const mbgl::filesystem::path&); +std::string serializeJsonValue(const mbgl::JSValue&); + +std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base); + +ArgumentsTuple parseArguments(int argc, char** argv); +std::vector<std::pair<std::string, std::string>> parseIgnores(); +TestMetadata parseTestMetadata(const mbgl::filesystem::path& path); + +std::string createResultPage(const TestStatistics&, const std::vector<TestMetadata>&, bool shuffle, uint32_t seed); + +std::string localizeURL(const std::string& url); + +void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document); +void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document);
\ No newline at end of file diff --git a/render-test/runner.cpp b/render-test/runner.cpp new file mode 100644 index 0000000000..1aa347f1a8 --- /dev/null +++ b/render-test/runner.cpp @@ -0,0 +1,408 @@ +#include <mbgl/map/camera.hpp> +#include <mbgl/map/map_observer.hpp> +#include <mbgl/style/conversion/filter.hpp> +#include <mbgl/style/conversion/layer.hpp> +#include <mbgl/style/conversion/light.hpp> +#include <mbgl/style/conversion/source.hpp> +#include <mbgl/style/image.hpp> +#include <mbgl/style/layer.hpp> +#include <mbgl/style/light.hpp> +#include <mbgl/style/style.hpp> +#include <mbgl/style/rapidjson_conversion.hpp> +#include <mbgl/util/chrono.hpp> +#include <mbgl/util/io.hpp> +#include <mbgl/util/image.hpp> +#include <mbgl/util/run_loop.hpp> +#include <mbgl/util/string.hpp> +#include <mbgl/util/timer.hpp> + +#include <mapbox/pixelmatch.hpp> + +#include "metadata.hpp" +#include "parser.hpp" +#include "runner.hpp" + +#include <algorithm> +#include <cassert> +#include <regex> + +bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& metadata) { + const std::string base = metadata.path.remove_filename().string(); + metadata.actual = mbgl::encodePNG(actual); + + if (actual.size.isEmpty()) { + metadata.errorMessage = "Invalid size for actual image"; + return false; + } + +#if !TEST_READ_ONLY + if (getenv("UPDATE")) { + mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actual)); + return true; + } + + mbgl::util::write_file(base + "/actual.png", metadata.actual); +#endif + + mbgl::PremultipliedImage expected { actual.size }; + mbgl::PremultipliedImage diff { actual.size }; + + double pixels = 0.0; + + for (const auto& entry: readExpectedEntries(base)) { + mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry); + if (!maybeExpectedImage) { + metadata.errorMessage = "Failed to load expected image " + entry; + return false; + } + + metadata.expected = *maybeExpectedImage; + + expected = mbgl::decodeImage(*maybeExpectedImage); + + pixels = // implicitly converting from uint64_t + mapbox::pixelmatch(actual.data.get(), expected.data.get(), expected.size.width, + expected.size.height, diff.data.get(), 0.16); // GL JS uses 0.1285 + + metadata.diff = mbgl::encodePNG(diff); + +#if !TEST_READ_ONLY + mbgl::util::write_file(base + "/diff.png", metadata.diff); +#endif + + metadata.difference = pixels / expected.size.area(); + if (metadata.difference <= metadata.allowed) { + break; + } + } + + return true; +} + +bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { + if (!metadata.document.HasMember("metadata") || + !metadata.document["metadata"].HasMember("test") || + !metadata.document["metadata"]["test"].HasMember("operations")) { + return true; + } + + assert(metadata.document["metadata"]["test"]["operations"].IsArray()); + + const auto& operationsArray = metadata.document["metadata"]["test"]["operations"].GetArray(); + if (operationsArray.Empty()) { + return true; + } + + const auto& operationIt = operationsArray.Begin(); + assert(operationIt->IsArray()); + + const auto& operationArray = operationIt->GetArray(); + assert(operationArray.Size() >= 1u); + + auto& frontend = maps[key]->frontend; + auto& map = maps[key]->map; + + static const std::string waitOp("wait"); + static const std::string sleepOp("sleep"); + static const std::string addImageOp("addImage"); + static const std::string updateImageOp("updateImage"); + static const std::string removeImageOp("removeImage"); + static const std::string setStyleOp("setStyle"); + static const std::string setCenterOp("setCenter"); + static const std::string setZoomOp("setZoom"); + static const std::string setBearingOp("setBearing"); + static const std::string setFilterOp("setFilter"); + static const std::string setLayerZoomRangeOp("setLayerZoomRange"); + static const std::string setLightOp("setLight"); + static const std::string addLayerOp("addLayer"); + static const std::string removeLayerOp("removeLayer"); + static const std::string addSourceOp("addSource"); + static const std::string removeSourceOp("removeSource"); + static const std::string setPaintPropertyOp("setPaintProperty"); + static const std::string setLayoutPropertyOp("setLayoutProperty"); + + // wait + if (operationArray[0].GetString() == waitOp) { + frontend.render(map); + + // sleep + } else if (operationArray[0].GetString() == sleepOp) { + mbgl::util::Timer sleepTimer; + bool sleeping = true; + + mbgl::Duration duration = mbgl::Seconds(3); + if (operationArray.Size() >= 2u) { + duration = mbgl::Milliseconds(operationArray[1].GetUint()); + } + + sleepTimer.start(duration, mbgl::Duration::zero(), [&]() { + sleeping = false; + }); + + while (sleeping) { + mbgl::util::RunLoop::Get()->runOnce(); + } + + // addImage | updateImage + } else if (operationArray[0].GetString() == addImageOp || operationArray[0].GetString() == updateImageOp) { + assert(operationArray.Size() >= 3u); + + float pixelRatio = 1.0f; + bool sdf = false; + + if (operationArray.Size() == 4u) { + assert(operationArray[3].IsObject()); + const auto& imageOptions = operationArray[3].GetObject(); + if (imageOptions.HasMember("pixelRatio")) { + pixelRatio = imageOptions["pixelRatio"].GetFloat(); + } + if (imageOptions.HasMember("sdf")) { + sdf = imageOptions["sdf"].GetBool(); + } + } + + std::string imageName = operationArray[1].GetString(); + imageName.erase(std::remove(imageName.begin(), imageName.end(), '"'), imageName.end()); + + std::string imagePath = operationArray[2].GetString(); + imagePath.erase(std::remove(imagePath.begin(), imagePath.end(), '"'), imagePath.end()); + + const mbgl::filesystem::path filePath(std::string(TEST_RUNNER_ROOT_PATH) + "/mapbox-gl-js/test/integration/" + imagePath); + + mbgl::optional<std::string> maybeImage = mbgl::util::readFile(filePath.string()); + if (!maybeImage) { + metadata.errorMessage = std::string("Failed to load expected image ") + filePath.string(); + return false; + } + + map.getStyle().addImage(std::make_unique<mbgl::style::Image>(imageName, mbgl::decodeImage(*maybeImage), pixelRatio, sdf)); + + // removeImage + } else if (operationArray[0].GetString() == removeImageOp) { + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsString()); + + const std::string imageName { operationArray[1].GetString(), operationArray[1].GetStringLength() }; + map.getStyle().removeImage(imageName); + + // setStyle + } else if (operationArray[0].GetString() == setStyleOp) { + assert(operationArray.Size() >= 2u); + if (operationArray[1].IsString()) { + std::string stylePath = localizeURL(operationArray[1].GetString()); + auto maybeStyle = readJson(stylePath); + if (maybeStyle.is<mbgl::JSDocument>()) { + auto& style = maybeStyle.get<mbgl::JSDocument>(); + localizeStyleURLs((mbgl::JSValue&)style, style); + map.getStyle().loadJSON(serializeJsonValue(style)); + } + } else { + localizeStyleURLs(operationArray[1], metadata.document); + map.getStyle().loadJSON(serializeJsonValue(operationArray[1])); + } + + // setCenter + } else if (operationArray[0].GetString() == setCenterOp) { + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsArray()); + + const auto& centerArray = operationArray[1].GetArray(); + assert(centerArray.Size() == 2u); + + map.jumpTo(mbgl::CameraOptions().withCenter(mbgl::LatLng(centerArray[1].GetDouble(), centerArray[0].GetDouble()))); + + // setZoom + } else if (operationArray[0].GetString() == setZoomOp) { + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsNumber()); + map.jumpTo(mbgl::CameraOptions().withZoom(operationArray[1].GetDouble())); + + // setBearing + } else if (operationArray[0].GetString() == setBearingOp) { + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsNumber()); + map.jumpTo(mbgl::CameraOptions().withBearing(operationArray[1].GetDouble())); + + // setFilter + } else if (operationArray[0].GetString() == setFilterOp) { + assert(operationArray.Size() >= 3u); + assert(operationArray[1].IsString()); + + const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() }; + + mbgl::style::conversion::Error error; + auto converted = mbgl::style::conversion::convert<mbgl::style::Filter>(operationArray[2], error); + if (!converted) { + metadata.errorMessage = std::string("Unable to convert filter: ") + error.message; + return false; + } else { + auto layer = map.getStyle().getLayer(layerName); + if (!layer) { + metadata.errorMessage = std::string("Layer not found: ") + layerName; + return false; + } else { + layer->setFilter(std::move(*converted)); + } + } + + // setLayerZoomRange + } else if (operationArray[0].GetString() == setLayerZoomRangeOp) { + assert(operationArray.Size() >= 4u); + assert(operationArray[1].IsString()); + assert(operationArray[2].IsNumber()); + assert(operationArray[3].IsNumber()); + + const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() }; + auto layer = map.getStyle().getLayer(layerName); + if (!layer) { + metadata.errorMessage = std::string("Layer not found: ") + layerName; + return false; + } else { + layer->setMinZoom(operationArray[2].GetFloat()); + layer->setMaxZoom(operationArray[3].GetFloat()); + } + + // setLight + } else if (operationArray[0].GetString() == setLightOp) { + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsObject()); + + mbgl::style::conversion::Error error; + auto converted = mbgl::style::conversion::convert<mbgl::style::Light>(operationArray[1], error); + if (!converted) { + metadata.errorMessage = std::string("Unable to convert light: ") + error.message; + return false; + } else { + map.getStyle().setLight(std::make_unique<mbgl::style::Light>(std::move(*converted))); + } + + // addLayer + } else if (operationArray[0].GetString() == addLayerOp) { + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsObject()); + + mbgl::style::conversion::Error error; + auto converted = mbgl::style::conversion::convert<std::unique_ptr<mbgl::style::Layer>>(operationArray[1], error); + if (!converted) { + metadata.errorMessage = std::string("Unable to convert layer: ") + error.message; + return false; + } else { + map.getStyle().addLayer(std::move(*converted)); + } + + // removeLayer + } else if (operationArray[0].GetString() == removeLayerOp) { + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsString()); + map.getStyle().removeLayer(operationArray[1].GetString()); + + // addSource + } else if (operationArray[0].GetString() == addSourceOp) { + assert(operationArray.Size() >= 3u); + assert(operationArray[1].IsString()); + assert(operationArray[2].IsObject()); + + localizeSourceURLs(operationArray[2], metadata.document); + + mbgl::style::conversion::Error error; + auto converted = mbgl::style::conversion::convert<std::unique_ptr<mbgl::style::Source>>(operationArray[2], error, operationArray[1].GetString()); + if (!converted) { + metadata.errorMessage = std::string("Unable to convert source: ") + error.message; + return false; + } else { + map.getStyle().addSource(std::move(*converted)); + } + + // removeSource + } else if (operationArray[0].GetString() == removeSourceOp) { + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsString()); + map.getStyle().removeSource(operationArray[1].GetString()); + + // setPaintProperty + } else if (operationArray[0].GetString() == setPaintPropertyOp) { + assert(operationArray.Size() >= 4u); + assert(operationArray[1].IsString()); + assert(operationArray[2].IsString()); + + const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() }; + const std::string propertyName { operationArray[2].GetString(), operationArray[2].GetStringLength() }; + + auto layer = map.getStyle().getLayer(layerName); + if (!layer) { + metadata.errorMessage = std::string("Layer not found: ") + layerName; + return false; + } else { + const mbgl::JSValue* propertyValue = &operationArray[3]; + layer->setPaintProperty(propertyName, propertyValue); + } + + // setLayoutProperty + } else if (operationArray[0].GetString() == setLayoutPropertyOp) { + assert(operationArray.Size() >= 4u); + assert(operationArray[1].IsString()); + assert(operationArray[2].IsString()); + + const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() }; + const std::string propertyName { operationArray[2].GetString(), operationArray[2].GetStringLength() }; + + auto layer = map.getStyle().getLayer(layerName); + if (!layer) { + metadata.errorMessage = std::string("Layer not found: ") + layerName; + return false; + } else { + const mbgl::JSValue* propertyValue = &operationArray[3]; + layer->setLayoutProperty(propertyName, propertyValue); + } + + } else { + metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString(); + return false; + } + + operationsArray.Erase(operationIt); + return runOperations(key, metadata); +} + +TestRunner::Impl::Impl(const TestMetadata& metadata) + : frontend(metadata.size, metadata.pixelRatio), + map(frontend, + mbgl::MapObserver::nullObserver(), + mbgl::MapOptions() + .withMapMode(metadata.mapMode) + .withSize(metadata.size) + .withPixelRatio(metadata.pixelRatio) + .withCrossSourceCollisions(metadata.crossSourceCollisions), + mbgl::ResourceOptions().withCacheOnlyRequestsSupport(false)) {} + +bool TestRunner::run(TestMetadata& metadata) { + std::string key = mbgl::util::toString(uint32_t(metadata.mapMode)) + + "/" + mbgl::util::toString(metadata.pixelRatio) + + "/" + mbgl::util::toString(uint32_t(metadata.crossSourceCollisions)); + + if (maps.find(key) == maps.end()) { + maps[key] = std::make_unique<TestRunner::Impl>(metadata); + } + + auto& frontend = maps[key]->frontend; + auto& map = maps[key]->map; + + frontend.setSize(metadata.size); + map.setSize(metadata.size); + + map.setProjectionMode(mbgl::ProjectionMode().withAxonometric(metadata.axonometric).withXSkew(metadata.xSkew).withYSkew(metadata.ySkew)); + map.setDebug(metadata.debug); + + map.getStyle().loadJSON(serializeJsonValue(metadata.document)); + map.jumpTo(map.getStyle().getDefaultCamera()); + + if (!runOperations(key, metadata)) { + return false; + } + + return checkImage(frontend.render(map), metadata); +} + +void TestRunner::reset() { + maps.clear(); +} diff --git a/render-test/runner.hpp b/render-test/runner.hpp new file mode 100644 index 0000000000..cbc0f42546 --- /dev/null +++ b/render-test/runner.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include <mbgl/gfx/headless_frontend.hpp> +#include <mbgl/map/map.hpp> + +#include <memory> + +struct TestMetadata; + +class TestRunner { +public: + TestRunner() = default; + + bool run(TestMetadata&); + void reset(); + +private: + bool runOperations(const std::string& key, TestMetadata&); + bool checkImage(mbgl::PremultipliedImage&& image, TestMetadata&); + + struct Impl { + Impl(const TestMetadata&); + + mbgl::HeadlessFrontend frontend; + mbgl::Map map; + }; + std::unordered_map<std::string, std::unique_ptr<Impl>> maps; +};
\ No newline at end of file |