#include #include #include #include #include #include #include #include #include #include #include #include "parser.hpp" #include "metadata.hpp" #include #include namespace { const char* resultsStyle = R"HTML( )HTML"; const char* resultsScript = R"HTML( )HTML"; const char* resultsHeaderButtons = R"HTML( )HTML"; std::string removeURLArguments(const std::string &url) { std::string::size_type index = url.find('?'); if (index != std::string::npos) { return url.substr(0, index); } return url; } std::string prependFileScheme(const std::string &url) { static const std::string fileScheme("file://"); return fileScheme + url; } mbgl::optional getVendorPath(const std::string& url, const std::regex& regex, bool glyphsPath = false) { static const mbgl::filesystem::path vendorPath(std::string(TEST_RUNNER_ROOT_PATH) + "/vendor/"); mbgl::filesystem::path file = std::regex_replace(url, regex, vendorPath.string()); if (mbgl::filesystem::exists(file.parent_path())) { return removeURLArguments(file.string()); } if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) { return removeURLArguments(file.string()); } return {}; } mbgl::optional getIntegrationPath(const std::string& url, const std::string& parent, const std::regex& regex, bool glyphsPath = false) { static const mbgl::filesystem::path integrationPath(std::string(TEST_RUNNER_ROOT_PATH) + "/mapbox-gl-js/test/integration/"); mbgl::filesystem::path file = std::regex_replace(url, regex, integrationPath.string() + parent); if (mbgl::filesystem::exists(file.parent_path())) { return removeURLArguments(file.string()); } if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) { return removeURLArguments(file.string()); } return {}; } mbgl::optional localizeLocalURL(const std::string& url, bool glyphsPath = false) { static const std::regex regex { "local://" }; if (auto vendorPath = getVendorPath(url, regex, glyphsPath)) { return vendorPath; } else { return getIntegrationPath(url, "", regex, glyphsPath); } } mbgl::optional localizeHttpURL(const std::string& url) { static const std::regex regex { "http://localhost:2900" }; if (auto vendorPath = getVendorPath(url, regex)) { return vendorPath; } else { return getIntegrationPath(url, "", regex); } } mbgl::optional localizeMapboxSpriteURL(const std::string& url) { static const std::regex regex { "mapbox://" }; return getIntegrationPath(url, "", regex); } mbgl::optional localizeMapboxFontsURL(const std::string& url) { static const std::regex regex { "mapbox://fonts" }; return getIntegrationPath(url, "glyphs/", regex, true); } mbgl::optional localizeMapboxTilesURL(const std::string& url) { static const std::regex regex { "mapbox://" }; if (auto vendorPath = getVendorPath(url, regex)) { return vendorPath; } else { return getIntegrationPath(url, "tiles/", regex); } } mbgl::optional localizeMapboxTilesetURL(const std::string& url) { static const std::regex regex { "mapbox://" }; return getIntegrationPath(url, "tilesets/", regex); } } // namespace JSONReply readJson(const mbgl::filesystem::path& jsonPath) { auto maybeJSON = mbgl::util::readFile(jsonPath); if (!maybeJSON) { return { std::string("Unable to open file ") + jsonPath.string() }; } mbgl::JSDocument document; document.Parse<0>(*maybeJSON); if (document.HasParseError()) { return { mbgl::formatJSONParseError(document) }; } return { std::move(document) }; } std::string serializeJsonValue(const mbgl::JSValue& value) { rapidjson::StringBuffer buffer; buffer.Clear(); rapidjson::Writer writer(buffer); value.Accept(writer); return buffer.GetString(); } std::vector readExpectedEntries(const mbgl::filesystem::path& base) { static const std::regex regex(".*expected.*.png"); std::vector expectedImages; for (const auto& entry : mbgl::filesystem::directory_iterator(base)) { if (entry.is_regular_file()) { const std::string path = entry.path().string(); if (std::regex_match(path, regex)) { expectedImages.emplace_back(std::move(path)); } } } return expectedImages; } ArgumentsTuple parseArguments(int argc, char** argv) { args::ArgumentParser argumentParser("Mapbox GL Test Runner"); args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", { 'h', "help" }); args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", { 'r', "recycle-map" }); args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order", { 's', "shuffle" }); args::ValueFlag seedValue(argumentParser, "seed", "Shuffle seed (default: random)", { "seed" }); args::ValueFlag testPathValue(argumentParser, "rootPath", "Test root rootPath", { 'p', "rootPath" }); args::PositionalList testNameValues(argumentParser, "URL", "Test name(s)"); try { argumentParser.ParseCLI(argc, argv); } catch (const args::Help&) { std::ostringstream stream; stream << argumentParser; mbgl::Log::Info(mbgl::Event::General, stream.str()); exit(0); } catch (const args::ParseError& e) { std::ostringstream stream; stream << argumentParser; mbgl::Log::Info(mbgl::Event::General, stream.str()); mbgl::Log::Error(mbgl::Event::General, e.what()); exit(1); } catch (const args::ValidationError& e) { std::ostringstream stream; stream << argumentParser; mbgl::Log::Info(mbgl::Event::General, stream.str()); mbgl::Log::Error(mbgl::Event::General, e.what()); exit(2); } const std::string testDefaultPath = std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration/render-tests"); std::vector ids; for (const auto& id : args::get(testNameValues)) { ids.emplace_back(testDefaultPath + "/" + id); } if (ids.empty()) { ids.emplace_back(testDefaultPath); } return ArgumentsTuple { recycleMapFlag ? args::get(recycleMapFlag) : false, shuffleFlag ? args::get(shuffleFlag) : false, seedValue ? args::get(seedValue) : 1u, testPathValue ? args::get(testPathValue) : testDefaultPath, ids }; } std::vector> parseIgnores() { std::vector> ignores; auto mainIgnoresPath = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("platform/node/test/ignores.json"); mbgl::filesystem::path platformSpecificIgnores; #ifdef __APPLE__ platformSpecificIgnores = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/mac-ignores.json"); #elif __linux__ platformSpecificIgnores = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH).append("render-test/linux-ignores.json"); #endif std::vector ignoresPaths = { mainIgnoresPath, platformSpecificIgnores }; for (auto path: ignoresPaths) { auto maybeIgnores = readJson(path); if (!maybeIgnores.is()) { continue; } for (const auto& property : maybeIgnores.get().GetObject()) { const std::string ignore = { property.name.GetString(), property.name.GetStringLength() }; const std::string reason = { property.value.GetString(), property.value.GetStringLength() }; ignores.emplace_back(std::make_pair(ignore, reason)); } } return ignores; } TestMetadata parseTestMetadata(const mbgl::filesystem::path& path) { TestMetadata metadata; metadata.path = path; auto maybeJson = readJson(path.string()); if (!maybeJson.is()) { // NOLINT metadata.errorMessage = std::string("Unable to parse: ") + path.string(); return metadata; } metadata.document = std::move(maybeJson.get()); localizeStyleURLs(metadata.document, metadata.document); if (!metadata.document.HasMember("metadata")) { mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s", path.c_str()); return metadata; } const mbgl::JSValue& metadataValue = metadata.document["metadata"]; if (!metadataValue.HasMember("test")) { mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata.test': %s", path.c_str()); return metadata; } const mbgl::JSValue& testValue = metadataValue["test"]; if (testValue.HasMember("width")) { assert(testValue["width"].IsNumber()); metadata.size.width = testValue["width"].GetInt(); } if (testValue.HasMember("height")) { assert(testValue["height"].IsNumber()); metadata.size.height = testValue["height"].GetInt(); } if (testValue.HasMember("pixelRatio")) { assert(testValue["pixelRatio"].IsNumber()); metadata.pixelRatio = testValue["pixelRatio"].GetFloat(); } if (testValue.HasMember("allowed")) { assert(testValue["allowed"].IsNumber()); metadata.allowed = testValue["allowed"].GetDouble(); } if (testValue.HasMember("description")) { assert(testValue["description"].IsString()); metadata.description = std::string{ testValue["description"].GetString(), testValue["description"].GetStringLength() }; } if (testValue.HasMember("mapMode")) { assert(testValue["mapMode"].IsString()); metadata.mapMode = testValue["mapMode"].GetString() == std::string("tile") ? mbgl::MapMode::Tile : mbgl::MapMode::Static; } // Test operations handled in runner.cpp. if (testValue.HasMember("debug")) { metadata.debug |= mbgl::MapDebugOptions::TileBorders; } if (testValue.HasMember("collisionDebug")) { metadata.debug |= mbgl::MapDebugOptions::Collision; } if (testValue.HasMember("showOverdrawInspector")) { metadata.debug |= mbgl::MapDebugOptions::Overdraw; } if (testValue.HasMember("crossSourceCollisions")) { assert(testValue["crossSourceCollisions"].IsBool()); metadata.crossSourceCollisions = testValue["crossSourceCollisions"].GetBool(); } if (testValue.HasMember("axonometric")) { assert(testValue["axonometric"].IsBool()); metadata.axonometric = testValue["axonometric"].GetBool(); } if (testValue.HasMember("skew")) { assert(testValue["skew"].IsArray()); metadata.xSkew = testValue["skew"][0].GetDouble(); metadata.ySkew = testValue["skew"][1].GetDouble(); } // TODO: fadeDuration // TODO: addFakeCanvas return metadata; } // https://stackoverflow.com/questions/7053538/how-do-i-encode-a-string-to-base64-using-only-boost std::string encodeBase64(const std::string& data) { using namespace boost::archive::iterators; using base64 = insert_linebreaks>, 72>; std::stringstream os; std::copy(base64(data.c_str()), base64(data.c_str() + data.size()), ostream_iterator(os)); return os.str(); } std::string createResultItem(const TestMetadata& metadata, bool hasFailedTests) { const bool shouldHide = (hasFailedTests && metadata.status == "passed") || (metadata.status.find("ignored") != std::string::npos); std::string html; html.append("
\n"); html.append(R"(

" + metadata.status + " " + metadata.id + "

\n"); if (metadata.status != "errored") { html.append("\n"); html.append("\n"); } else { assert(!metadata.errorMessage.empty()); html.append("

Error: " + metadata.errorMessage + "

\n"); } if (metadata.difference != 0.0) { html.append("

Diff: " + mbgl::util::toString(metadata.difference) + "

\n"); } html.append("
\n"); return html; } std::string createResultPage(const TestStatistics& stats, const std::vector& metadatas, bool shuffle, uint32_t seed) { const uint32_t unsuccessful = stats.erroredTests + stats.failedTests; std::string resultsPage; // Style resultsPage.append(resultsStyle); // Header if (unsuccessful) { resultsPage.append(R"HTML(

)HTML"); resultsPage.append(mbgl::util::toString(unsuccessful) + " tests failed."); } else { resultsPage.append(R"HTML(

)HTML"); resultsPage.append("All tests passed!"); } resultsPage.append(resultsHeaderButtons); // stats resultsPage.append(R"HTML(

)HTML"); if (stats.ignoreFailedTests) { resultsPage.append(mbgl::util::toString(stats.ignoreFailedTests) + " ignored failed, "); } if (stats.ignorePassedTests) { resultsPage.append(mbgl::util::toString(stats.ignorePassedTests) + " ignored passed, "); } if (stats.erroredTests) { resultsPage.append(mbgl::util::toString(stats.erroredTests) + " errored, "); } if (stats.failedTests) { resultsPage.append(mbgl::util::toString(stats.failedTests) + " failed, "); } resultsPage.append(mbgl::util::toString(stats.passedTests) + " passed.\n"); resultsPage.append("

\n"); // Test sequence { resultsPage.append("
\n"); // Failed tests if (unsuccessful) { resultsPage.append("

Failed tests:"); for (const auto& metadata : metadatas) { if (metadata.status == "failed" || metadata.status == "errored") { resultsPage.append(metadata.id + " "); } } resultsPage.append("

\n"); } // Test sequence resultsPage.append("

Test sequence:"); for (const auto& metadata : metadatas) { resultsPage.append(metadata.id + " "); } resultsPage.append("

\n"); // Shuffle if (shuffle) { resultsPage.append("

Shuffle seed: " + mbgl::util::toString(seed) + "

\n"); } resultsPage.append("
\n"); } // Script resultsPage.append(resultsScript); // Tests resultsPage.append("
\n"); for (const auto& metadata : metadatas) { resultsPage.append(createResultItem(metadata, unsuccessful)); } resultsPage.append("
\n"); return resultsPage; } std::string localizeURL(const std::string& url) { static const std::regex regex { "local://" }; if (auto vendorPath = getVendorPath(url, regex)) { return *vendorPath; } else { return getIntegrationPath(url, "", regex).value_or(url); } } void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) { if (root.HasMember("urls") && root["urls"].IsArray()) { for (auto& urlValue : root["urls"].GetArray()) { const std::string path = prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString()) .value_or(localizeLocalURL(urlValue.GetString()) .value_or(urlValue.GetString()))); urlValue.Set(path, document.GetAllocator()); } } if (root.HasMember("url")) { static const std::string image("image"); static const std::string video("video"); mbgl::JSValue& urlValue = root["url"]; const std::string path = prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString()) .value_or(localizeLocalURL(urlValue.GetString()) .value_or(urlValue.GetString()))); urlValue.Set(path, document.GetAllocator()); if (root["type"].GetString() != image && root["type"].GetString() != video) { const auto tilesetPath = std::string(urlValue.GetString()).erase(0u, 7u); // remove "file://" auto maybeTileset = readJson(tilesetPath); if (maybeTileset.is()) { const auto& tileset = maybeTileset.get(); assert(tileset.HasMember("tiles")); root.AddMember("tiles", (mbgl::JSValue&)tileset["tiles"], document.GetAllocator()); root.RemoveMember("url"); } } } if (root.HasMember("tiles")) { mbgl::JSValue& tilesValue = root["tiles"]; assert(tilesValue.IsArray()); for (auto& tileValue : tilesValue.GetArray()) { const std::string path = prependFileScheme(localizeMapboxTilesURL(tileValue.GetString()) .value_or(localizeLocalURL(tileValue.GetString()) .value_or(localizeHttpURL(tileValue.GetString()) .value_or(tileValue.GetString())))); tileValue.Set(path, document.GetAllocator()); } } if (root.HasMember("data") && root["data"].IsString()) { mbgl::JSValue& dataValue = root["data"]; const std::string path = prependFileScheme(localizeLocalURL(dataValue.GetString()) .value_or(dataValue.GetString())); dataValue.Set(path, document.GetAllocator()); } } void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) { if (root.HasMember("sources")) { mbgl::JSValue& sourcesValue = root["sources"]; for (auto& sourceProperty : sourcesValue.GetObject()) { localizeSourceURLs(sourceProperty.value, document); } } if (root.HasMember("glyphs")) { mbgl::JSValue& glyphsValue = root["glyphs"]; const std::string path = prependFileScheme(localizeMapboxFontsURL(glyphsValue.GetString()) .value_or(localizeLocalURL(glyphsValue.GetString(), true) .value_or(glyphsValue.GetString()))); glyphsValue.Set(path, document.GetAllocator()); } if (root.HasMember("sprite")) { mbgl::JSValue& spriteValue = root["sprite"]; const std::string path = prependFileScheme(localizeMapboxSpriteURL(spriteValue.GetString()) .value_or(localizeLocalURL(spriteValue.GetString()) .value_or(spriteValue.GetString()))); spriteValue.Set(path, document.GetAllocator()); } }