From c8576e089f12438a2384a4fe822af650e94ace74 Mon Sep 17 00:00:00 2001 From: zmiao Date: Tue, 29 Oct 2019 20:59:29 +0200 Subject: [render-test] Add manifest parser for render-test-runner (#15861) * [render-test] Add manifest parser for render-test-runner * [render-test] Refactory manifest parser * [render-test] Parse full manifest file through CLI * [render-test] Add linux probe manifest --- circle.yml | 6 +- cmake/render-test.cmake | 2 + next/platform/linux/linux.cmake | 8 +- next/platform/macos/macos.cmake | 15 +- next/render-test/CMakeLists.txt | 2 + platform/android/src/test/render_test_runner.cpp | 2 +- render-test/android-manifest.json | 7 + render-test/android/render_test_setup.sh | 10 +- render-test/linux-manifest.json | 7 + render-test/linux-probe-manifest.json | 7 + render-test/mac-manifest.json | 7 + render-test/mac-probe-manifest.json | 7 + render-test/manifest_parser.cpp | 400 +++++++++++++++++++++++ render-test/manifest_parser.hpp | 55 ++++ render-test/parser.cpp | 398 +--------------------- render-test/parser.hpp | 20 +- render-test/render_test.cpp | 112 +++++-- render-test/runner.cpp | 20 +- render-test/runner.hpp | 11 +- 19 files changed, 635 insertions(+), 461 deletions(-) create mode 100644 render-test/android-manifest.json create mode 100644 render-test/linux-manifest.json create mode 100644 render-test/linux-probe-manifest.json create mode 100644 render-test/mac-manifest.json create mode 100644 render-test/mac-probe-manifest.json create mode 100644 render-test/manifest_parser.cpp create mode 100644 render-test/manifest_parser.hpp diff --git a/circle.yml b/circle.yml index 103ea8c685..ad4dc8bfd0 100644 --- a/circle.yml +++ b/circle.yml @@ -267,9 +267,9 @@ commands: when: on_fail command: | mkdir -p /tmp/tests/render - mkdir -p /tmp/tests/probe - if [ -f index.html ]; then cp index.html /tmp/tests/render; fi - if [ -f render-test/index.html ]; then cp render-test/index.html /tmp/tests/probe; fi + if [ -f render-test/tests_index.html ]; then cp render-test/tests_index.html /tmp/tests/render; fi + if [ -f render-test/render-tests_index.html ]; then cp render-test/render-tests_index.html /tmp/tests/render; fi + if [ -f render-test/query-tests_index.html ]; then cp render-test/query-tests_index.html /tmp/tests/render; fi mkdir -p /tmp/tests/coredumps if ls core* 1> /dev/null 2>&1; then cp core* /tmp/tests/coredumps; fi - store_artifacts: diff --git a/cmake/render-test.cmake b/cmake/render-test.cmake index 8726fb4650..ac17be559c 100644 --- a/cmake/render-test.cmake +++ b/cmake/render-test.cmake @@ -8,6 +8,8 @@ add_executable( render-test/filesystem.hpp render-test/filesystem.hpp render-test/include/mbgl/render_test.hpp + render-test/manifest_parser.cpp + render-test/manifest_parser.hpp render-test/metadata.hpp render-test/parser.cpp render-test/parser.hpp diff --git a/next/platform/linux/linux.cmake b/next/platform/linux/linux.cmake index 5bde30a61a..396b9a0ffd 100644 --- a/next/platform/linux/linux.cmake +++ b/next/platform/linux/linux.cmake @@ -144,14 +144,12 @@ add_test( render-tests --recycle-map --shuffle - --expectationsPath=render-test/expected/render-tests + --manifestPath=${MBGL_ROOT}/render-test/linux-manifest.json --seed=${MBGL_RENDER_TEST_SEED} - WORKING_DIRECTORY ${MBGL_ROOT} ) add_test( NAME mbgl-render-test-probes - COMMAND mbgl-render-test-runner tests --rootPath=${MBGL_ROOT}/render-test - WORKING_DIRECTORY ${MBGL_ROOT} + COMMAND mbgl-render-test-runner tests --manifestPath=${MBGL_ROOT}/render-test/linux-probe-manifest.json ) -add_test(NAME mbgl-query-test COMMAND mbgl-render-test-runner query-tests WORKING_DIRECTORY ${MBGL_ROOT}) +add_test(NAME mbgl-query-test COMMAND mbgl-render-test-runner query-tests --manifestPath=${MBGL_ROOT}/render-test/linux-manifest.json) diff --git a/next/platform/macos/macos.cmake b/next/platform/macos/macos.cmake index cfabcb1cfa..721d19e625 100644 --- a/next/platform/macos/macos.cmake +++ b/next/platform/macos/macos.cmake @@ -208,19 +208,10 @@ add_test( render-tests --recycle-map --shuffle - --expectationsPath=render-test/expected/render-tests + --manifestPath=${MBGL_ROOT}/render-test/mac-manifest.json --seed=${MBGL_RENDER_TEST_SEED} - WORKING_DIRECTORY ${MBGL_ROOT} ) -add_test( - NAME mbgl-render-test-probes - COMMAND - mbgl-render-test-runner - tests - --rootPath=${MBGL_ROOT}/render-test - --expectationsPath=tests/mac - WORKING_DIRECTORY ${MBGL_ROOT} -) +add_test(NAME mbgl-render-test-probes COMMAND mbgl-render-test-runner tests --manifestPath=${MBGL_ROOT}/render-test/mac-probe-manifest.json) -add_test(NAME mbgl-query-test COMMAND mbgl-render-test-runner query-tests WORKING_DIRECTORY ${MBGL_ROOT}) +add_test(NAME mbgl-query-test COMMAND mbgl-render-test-runner query-tests --manifestPath=${MBGL_ROOT}/render-test/mac-manifest.json) diff --git a/next/render-test/CMakeLists.txt b/next/render-test/CMakeLists.txt index 02ee6141cf..fa4bb0d615 100644 --- a/next/render-test/CMakeLists.txt +++ b/next/render-test/CMakeLists.txt @@ -8,6 +8,8 @@ add_library( ${MBGL_ROOT}/render-test/file_source.hpp ${MBGL_ROOT}/render-test/filesystem.hpp ${MBGL_ROOT}/render-test/include/mbgl/render_test.hpp + ${MBGL_ROOT}/render-test/manifest_parser.cpp + ${MBGL_ROOT}/render-test/manifest_parser.hpp ${MBGL_ROOT}/render-test/metadata.hpp ${MBGL_ROOT}/render-test/parser.cpp ${MBGL_ROOT}/render-test/parser.hpp diff --git a/platform/android/src/test/render_test_runner.cpp b/platform/android/src/test/render_test_runner.cpp index 3f85140128..d4554aa9de 100644 --- a/platform/android/src/test/render_test_runner.cpp +++ b/platform/android/src/test/render_test_runner.cpp @@ -46,7 +46,7 @@ void android_main(struct android_app* app) { JNIEnv* env; app->activity->vm->AttachCurrentThread(&env, NULL); - std::vector arguments = {"mbgl-render-test-runner", "-p", "/sdcard/render-test"}; + std::vector arguments = {"mbgl-render-test-runner", "-p", "/sdcard/render-test/android-manifest.json"}; std::vector argv; for (const auto& arg : arguments) { argv.push_back((char*)arg.data()); diff --git a/render-test/android-manifest.json b/render-test/android-manifest.json new file mode 100644 index 0000000000..56223d4753 --- /dev/null +++ b/render-test/android-manifest.json @@ -0,0 +1,7 @@ +{ + "base_test_path":"mapbox-gl-js/test/integration", + "expectation_paths":["render-test/expected/render-tests"], + "ignore_paths":["platform/node/test/ignores.json", "render-test/linux-ignores.json", "render-test/tests/should-fail.json"], + "vendor_path":"vendor", + "asset_path": "mapbox-gl-js/test/integration" +} \ No newline at end of file diff --git a/render-test/android/render_test_setup.sh b/render-test/android/render_test_setup.sh index e34d71df3b..1dea44399e 100755 --- a/render-test/android/render_test_setup.sh +++ b/render-test/android/render_test_setup.sh @@ -3,8 +3,8 @@ adb shell rm -rf /sdcard/render-test adb shell mkdir /sdcard/render-test adb shell mkdir /sdcard/render-test/vendor -adb shell mkdir /sdcard/render-test/expected adb shell mkdir /sdcard/render-test/ignores +adb shell mkdir /sdcard/render-test/render-test/tests # push test sources adb push ../../mapbox-gl-js/test/integration/render-tests /sdcard/render-test/mapbox-gl-js/test/integration/render-tests @@ -25,7 +25,7 @@ adb push sprites /sdcard/render-test/mapbox-gl-js/test/integration/sprites rm -rf sprites # push extra expectations -adb push ../../render-test/expected /sdcard/render-test/render-test/expected +adb push ../../render-test/expected/render-tests /sdcard/render-test/render-test/expected/render-tests # push default ignore lists adb shell mkdir /sdcard/render-test/platform @@ -33,6 +33,10 @@ adb shell mkdir /sdcard/render-test/platform/node adb shell mkdir /sdcard/render-test/platform/node/test adb push ../../platform/node/test/ignores.json /sdcard/render-test/platform/node/test adb shell mkdir /sdcard/render-test/render-test -adb push ../../render-test/linux-ignores.json /sdcard/render-test/render-test +adb push ../linux-ignores.json /sdcard/render-test/render-test +adb push ../tests/should-fail.json /sdcard/render-test/render-test/tests + +# push manifest +adb push ../android-manifest.json /sdcard/render-test adb shell ls /sdcard/render-test/ \ No newline at end of file diff --git a/render-test/linux-manifest.json b/render-test/linux-manifest.json new file mode 100644 index 0000000000..32a5afdbdb --- /dev/null +++ b/render-test/linux-manifest.json @@ -0,0 +1,7 @@ +{ + "base_test_path":"../mapbox-gl-js/test/integration", + "expectation_paths":["expected/render-tests"], + "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json", "../render-test/tests/should-fail.json"], + "vendor_path":"../vendor", + "asset_path": "../mapbox-gl-js/test/integration" +} \ No newline at end of file diff --git a/render-test/linux-probe-manifest.json b/render-test/linux-probe-manifest.json new file mode 100644 index 0000000000..0b2ca88988 --- /dev/null +++ b/render-test/linux-probe-manifest.json @@ -0,0 +1,7 @@ +{ + "probe_test_path":".", + "expectation_paths":["expected/render-tests"], + "ignore_paths":["../render-test/linux-ignores.json", "../render-test/tests/should-fail.json"], + "vendor_path":"../vendor", + "asset_path": "../mapbox-gl-js/test/integration" +} \ No newline at end of file diff --git a/render-test/mac-manifest.json b/render-test/mac-manifest.json new file mode 100644 index 0000000000..224df81298 --- /dev/null +++ b/render-test/mac-manifest.json @@ -0,0 +1,7 @@ +{ + "base_test_path":"../mapbox-gl-js/test/integration", + "expectation_paths":["expected/render-tests", "tests/mac"], + "ignore_paths":["../platform/node/test/ignores.json", "../render-test/mac-ignores.json", "../render-test/tests/should-fail.json"], + "vendor_path":"../vendor", + "asset_path": "../mapbox-gl-js/test/integration" +} \ No newline at end of file diff --git a/render-test/mac-probe-manifest.json b/render-test/mac-probe-manifest.json new file mode 100644 index 0000000000..f3cc56d0a8 --- /dev/null +++ b/render-test/mac-probe-manifest.json @@ -0,0 +1,7 @@ +{ + "probe_test_path":".", + "expectation_paths":["expected/render-tests", "tests/mac"], + "ignore_paths":["../render-test/mac-ignores.json", "../render-test/tests/should-fail.json"], + "vendor_path":"../vendor", + "asset_path": "../mapbox-gl-js/test/integration" +} \ No newline at end of file diff --git a/render-test/manifest_parser.cpp b/render-test/manifest_parser.cpp new file mode 100644 index 0000000000..f1884634e5 --- /dev/null +++ b/render-test/manifest_parser.cpp @@ -0,0 +1,400 @@ +#include "manifest_parser.hpp" +#include "filesystem.hpp" +#include "parser.hpp" + +#include + +#include +#include + +namespace { +std::string removeURLArguments(const std::string& url) { + std::string::size_type index = url.find('?'); + if (index != std::string::npos) { + return url.substr(0, index); + } + return url; +} + +std::string prependFileScheme(const std::string& url) { + static const std::string fileScheme("file://"); + return fileScheme + url; +} +} // namespace + +Manifest::Manifest() = default; +Manifest::~Manifest() = default; + +const std::vector& Manifest::getTestPaths() const { + return testPaths; +} +const std::vector>& Manifest::getIgnores() const { + return ignores; +} +const std::string& Manifest::getTestRootPath() const { + return testRootPath; +} +const std::string& Manifest::getManifestPath() const { + return manifestPath; +} + +void Manifest::doShuffle(uint32_t seed) { + std::seed_seq sequence{seed}; + std::mt19937 shuffler(sequence); + std::shuffle(testPaths.begin(), testPaths.end(), shuffler); +} + +std::string Manifest::localizeURL(const std::string& url) const { + static const std::regex regex{"local://"}; + if (auto vendorPath = getVendorPath(url, regex)) { + return *vendorPath; + } + return getIntegrationPath(url, "", regex).value_or(url); +} + +void Manifest::localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const { + if (root.HasMember("urls") && root["urls"].IsArray()) { + for (auto& urlValue : root["urls"].GetArray()) { + const std::string path = + prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString()) + .value_or(localizeLocalURL(urlValue.GetString()).value_or(urlValue.GetString()))); + urlValue.Set(path, document.GetAllocator()); + } + } + + if (root.HasMember("url")) { + static const std::string image("image"); + static const std::string video("video"); + + mbgl::JSValue& urlValue = root["url"]; + const std::string path = + prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString()) + .value_or(localizeLocalURL(urlValue.GetString()).value_or(urlValue.GetString()))); + urlValue.Set(path, document.GetAllocator()); + + if (root["type"].GetString() != image && root["type"].GetString() != video) { + const auto tilesetPath = std::string(urlValue.GetString()).erase(0u, 7u); // remove "file://" + auto maybeTileset = readJson(tilesetPath); + if (maybeTileset.is()) { + const auto& tileset = maybeTileset.get(); + assert(tileset.HasMember("tiles")); + root.AddMember("tiles", (mbgl::JSValue&)tileset["tiles"], document.GetAllocator()); + root.RemoveMember("url"); + } + } + } + + if (root.HasMember("tiles")) { + mbgl::JSValue& tilesValue = root["tiles"]; + assert(tilesValue.IsArray()); + for (auto& tileValue : tilesValue.GetArray()) { + const std::string path = prependFileScheme( + localizeMapboxTilesURL(tileValue.GetString()) + .value_or(localizeLocalURL(tileValue.GetString()) + .value_or(localizeHttpURL(tileValue.GetString()).value_or(tileValue.GetString())))); + tileValue.Set(path, document.GetAllocator()); + } + } + + if (root.HasMember("data") && root["data"].IsString()) { + mbgl::JSValue& dataValue = root["data"]; + const std::string path = + prependFileScheme(localizeLocalURL(dataValue.GetString()).value_or(dataValue.GetString())); + dataValue.Set(path, document.GetAllocator()); + } +} + +void Manifest::localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const { + if (root.HasMember("sources")) { + mbgl::JSValue& sourcesValue = root["sources"]; + for (auto& sourceProperty : sourcesValue.GetObject()) { + localizeSourceURLs(sourceProperty.value, document); + } + } + + if (root.HasMember("glyphs")) { + mbgl::JSValue& glyphsValue = root["glyphs"]; + const std::string path = prependFileScheme( + localizeMapboxFontsURL(glyphsValue.GetString()) + .value_or(localizeLocalURL(glyphsValue.GetString(), true).value_or(glyphsValue.GetString()))); + glyphsValue.Set(path, document.GetAllocator()); + } + + if (root.HasMember("sprite")) { + mbgl::JSValue& spriteValue = root["sprite"]; + const std::string path = prependFileScheme( + localizeMapboxSpriteURL(spriteValue.GetString()) + .value_or(localizeLocalURL(spriteValue.GetString()).value_or(spriteValue.GetString()))); + spriteValue.Set(path, document.GetAllocator()); + } +} + +mbgl::optional Manifest::localizeLocalURL(const std::string& url, bool glyphsPath) const { + static const std::regex regex{"local://"}; + if (auto vendorPath = getVendorPath(url, regex, glyphsPath)) { + return vendorPath; + } + return getIntegrationPath(url, "", regex, glyphsPath); +} + +mbgl::optional Manifest::localizeHttpURL(const std::string& url) const { + static const std::regex regex{"http://localhost:2900"}; + if (auto vendorPath = getVendorPath(url, regex)) { + return vendorPath; + } + return getIntegrationPath(url, "", regex); +} + +mbgl::optional Manifest::localizeMapboxSpriteURL(const std::string& url) const { + static const std::regex regex{"mapbox://"}; + return getIntegrationPath(url, "", regex); +} + +mbgl::optional Manifest::localizeMapboxFontsURL(const std::string& url) const { + static const std::regex regex{"mapbox://fonts"}; + return getIntegrationPath(url, "glyphs/", regex, true); +} + +mbgl::optional Manifest::localizeMapboxTilesURL(const std::string& url) const { + static const std::regex regex{"mapbox://"}; + if (auto vendorPath = getVendorPath(url, regex)) { + return vendorPath; + } + return getIntegrationPath(url, "tiles/", regex); +} + +mbgl::optional Manifest::localizeMapboxTilesetURL(const std::string& url) const { + static const std::regex regex{"mapbox://"}; + return getIntegrationPath(url, "tilesets/", regex); +} + +mbgl::optional Manifest::getVendorPath(const std::string& url, + const std::regex& regex, + bool glyphsPath) const { + mbgl::filesystem::path file = std::regex_replace(url, regex, vendorPath); + if (mbgl::filesystem::exists(file.parent_path())) { + return removeURLArguments(file.string()); + } + + if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) { + return removeURLArguments(file.string()); + } + + return mbgl::nullopt; +} + +mbgl::optional Manifest::getIntegrationPath(const std::string& url, + const std::string& parent, + const std::regex& regex, + bool glyphsPath) const { + mbgl::filesystem::path file = std::regex_replace(url, regex, assetPath + parent); + if (mbgl::filesystem::exists(file.parent_path())) { + return removeURLArguments(file.string()); + } + + if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) { + return removeURLArguments(file.string()); + } + + return mbgl::nullopt; +} + +namespace { +std::vector> parseIgnores(const std::vector& ignoresPaths) { + std::vector> ignores; + for (const auto& path : ignoresPaths) { + auto maybeIgnores = readJson(path); + if (!maybeIgnores.is()) { + continue; + } + for (const auto& property : maybeIgnores.get().GetObject()) { + const std::string ignore = {property.name.GetString(), property.name.GetStringLength()}; + const std::string reason = {property.value.GetString(), property.value.GetStringLength()}; + ignores.emplace_back(std::make_pair(ignore, reason)); + } + } + + return ignores; +} + +std::vector getTestExpectations(mbgl::filesystem::path testPath, + const mbgl::filesystem::path& testsRootPath, + std::vector expectationsPaths) { + std::vector expectations{std::move(testPath.remove_filename())}; + const auto& defaultTestExpectationsPath = expectations.front().string(); + + const std::regex regex{testsRootPath.string()}; + for (const auto& path : expectationsPaths) { + expectations.emplace_back(std::regex_replace(defaultTestExpectationsPath, regex, path.string())); + assert(!expectations.back().empty()); + } + + return expectations; +} + +mbgl::filesystem::path getValidPath(const std::string& manifestPath, const std::string& path) { + const static mbgl::filesystem::path BasePath{manifestPath}; + mbgl::filesystem::path result{path}; + if (result.is_relative()) { + result = BasePath / result; + } + if (mbgl::filesystem::exists(result)) { + return result; + } + mbgl::Log::Warning(mbgl::Event::General, "Invalid path is provoided inside the manifest file: %s", path.c_str()); + return mbgl::filesystem::path{}; +} + +} // namespace + +mbgl::optional ManifestParser::parseManifest(const std::string& manifestPath, + const std::vector& testNames, + const std::string& testFilter) { + Manifest manifest; + const auto filePath = mbgl::filesystem::path(manifestPath); + manifest.manifestPath = manifestPath.substr(0, manifestPath.find(filePath.filename())); + + auto contents = readJson(filePath); + if (!contents.is()) { + mbgl::Log::Error(mbgl::Event::General, "Provided manifest file: %s is not a valid json", filePath.c_str()); + return mbgl::nullopt; + } + + auto document = std::move(contents.get()); + if (document.HasMember("asset_path")) { + const auto& assetPathValue = document["asset_path"]; + if (!assetPathValue.IsString()) { + mbgl::Log::Warning( + mbgl::Event::General, "Invalid assetPath is provoided inside the manifest file: %s", filePath.c_str()); + return mbgl::nullopt; + } + manifest.assetPath = (getValidPath(manifest.manifestPath, assetPathValue.GetString()) / "").string(); + if (manifest.assetPath.empty()) { + return mbgl::nullopt; + } + } + if (document.HasMember("vendor_path")) { + const auto& vendorPathValue = document["vendor_path"]; + if (!vendorPathValue.IsString()) { + mbgl::Log::Warning( + mbgl::Event::General, "Invalid vendorPath is provoided inside the manifest file: %s", filePath.c_str()); + return mbgl::nullopt; + } + manifest.vendorPath = (getValidPath(manifest.manifestPath, vendorPathValue.GetString()) / "").string(); + if (manifest.vendorPath.empty()) { + return mbgl::nullopt; + } + } + mbgl::filesystem::path baseTestPath; + if (document.HasMember("base_test_path")) { + const auto& testPathValue = document["base_test_path"]; + if (!testPathValue.IsString()) { + mbgl::Log::Warning( + mbgl::Event::General, "Invalid testPath is provoided inside the manifest file: %s", filePath.c_str()); + return mbgl::nullopt; + } + baseTestPath = getValidPath(manifest.manifestPath, testPathValue.GetString()); + if (baseTestPath.empty()) { + return mbgl::nullopt; + } + } + mbgl::filesystem::path probeTestPath; + bool enbaleProbeTest{false}; + if (document.HasMember("probe_test_path")) { + const auto& testPathValue = document["probe_test_path"]; + if (!testPathValue.IsString()) { + mbgl::Log::Warning( + mbgl::Event::General, "Invalid testPath is provoided inside the manifest file: %s", filePath.c_str()); + return mbgl::nullopt; + } + probeTestPath = getValidPath(manifest.manifestPath, testPathValue.GetString()); + if (probeTestPath.empty()) { + return mbgl::nullopt; + } + enbaleProbeTest = true; + } + std::vector expectationPaths{}; + if (document.HasMember("expectation_paths")) { + const auto& expectationPathValue = document["expectation_paths"]; + if (!expectationPathValue.IsArray()) { + mbgl::Log::Warning(mbgl::Event::General, + "Provided expectation_paths inside the manifest file: %s is not a valid array", + filePath.c_str()); + return mbgl::nullopt; + } + for (const auto& value : expectationPathValue.GetArray()) { + if (!value.IsString()) { + mbgl::Log::Warning(mbgl::Event::General, + "Invalid expectation path item is provoided inside the manifest file: %s", + filePath.c_str()); + return mbgl::nullopt; + } + expectationPaths.emplace_back(getValidPath(manifest.manifestPath, value.GetString())); + if (expectationPaths.back().empty()) { + return mbgl::nullopt; + } + } + } + std::vector ignorePaths{}; + if (document.HasMember("ignore_paths")) { + const auto& ignorePathValue = document["ignore_paths"]; + if (!ignorePathValue.IsArray()) { + mbgl::Log::Warning(mbgl::Event::General, + "Provided ignore_paths inside the manifest file: %s is not a valid array", + filePath.c_str()); + return mbgl::nullopt; + } + for (const auto& value : ignorePathValue.GetArray()) { + if (!value.IsString()) { + mbgl::Log::Warning(mbgl::Event::General, + "Invalid ignore path item is provoided inside the manifest file: %s", + filePath.c_str()); + return mbgl::nullopt; + } + ignorePaths.emplace_back(getValidPath(manifest.manifestPath, value.GetString())); + if (ignorePaths.back().empty()) { + return mbgl::nullopt; + } + } + manifest.ignores = parseIgnores(ignorePaths); + } + + manifest.testRootPath = enbaleProbeTest ? probeTestPath.string() : baseTestPath.string(); + if (manifest.testRootPath.back() == '/') { + manifest.testRootPath.pop_back(); + } + if (manifest.manifestPath.back() == '/') { + manifest.manifestPath.pop_back(); + } + + std::vector paths; + for (const auto& id : testNames) { + paths.emplace_back(manifest.testRootPath + "/" + id); + } + if (paths.empty()) { + paths.emplace_back(manifest.testRootPath); + } + + // Recursively traverse through the test paths and collect test directories containing "style.json". + auto& testPaths = manifest.testPaths; + testPaths.reserve(paths.size()); + for (const auto& path : paths) { + if (!mbgl::filesystem::exists(path)) { + mbgl::Log::Warning( + mbgl::Event::General, "Provided test folder '%s' does not exist.", path.string().c_str()); + continue; + } + for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(path)) { + // Skip paths that fail regexp match. + if (!testFilter.empty() && !std::regex_match(testPath.path().string(), std::regex(testFilter))) { + continue; + } + + if (testPath.path().filename() == "style.json") { + testPaths.emplace_back(testPath, getTestExpectations(testPath, path, expectationPaths)); + } + } + } + + return mbgl::optional(manifest); +} diff --git a/render-test/manifest_parser.hpp b/render-test/manifest_parser.hpp new file mode 100644 index 0000000000..bc5adf1091 --- /dev/null +++ b/render-test/manifest_parser.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include "metadata.hpp" + +#include +#include + +#include +#include +#include +#include + +class Manifest { +public: + Manifest(); + ~Manifest(); + const std::vector>& getIgnores() const; + const std::vector& getTestPaths() const; + const std::string& getTestRootPath() const; + const std::string& getManifestPath() const; + void doShuffle(uint32_t seed); + + std::string localizeURL(const std::string& url) const; + void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const; + void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const; + +private: + friend class ManifestParser; + mbgl::optional localizeLocalURL(const std::string& url, bool glyphsPath = false) const; + mbgl::optional localizeHttpURL(const std::string& url) const; + mbgl::optional localizeMapboxSpriteURL(const std::string& url) const; + mbgl::optional localizeMapboxFontsURL(const std::string& url) const; + mbgl::optional localizeMapboxTilesURL(const std::string& url) const; + mbgl::optional localizeMapboxTilesetURL(const std::string& url) const; + mbgl::optional getVendorPath(const std::string& url, + const std::regex& regex, + bool glyphsPath = false) const; + mbgl::optional getIntegrationPath(const std::string& url, + const std::string& parent, + const std::regex& regex, + bool glyphsPath = false) const; + std::string manifestPath; + std::string testRootPath; + std::string vendorPath; + std::string assetPath; + std::vector> ignores; + std::vector testPaths; +}; + +class ManifestParser { +public: + static mbgl::optional parseManifest(const std::string& manifestPath, + const std::vector& testNames, + const std::string& testFilter); +}; diff --git a/render-test/parser.cpp b/render-test/parser.cpp index 11a42f3202..5a91fc7a58 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -3,8 +3,6 @@ #include #include -#include - #include #include #include @@ -23,8 +21,8 @@ #include "parser.hpp" #include "runner.hpp" -#include #include +#include namespace { @@ -83,101 +81,6 @@ const char* resultsHeaderButtons = R"HTML( )HTML"; -std::string removeURLArguments(const std::string &url) { - std::string::size_type index = url.find('?'); - if (index != std::string::npos) { - return url.substr(0, index); - } - return url; -} - -std::string prependFileScheme(const std::string &url) { - static const std::string fileScheme("file://"); - return fileScheme + url; -} - -mbgl::optional getVendorPath(const std::string& url, - const std::regex& regex, - const std::string& testRootPath, - bool glyphsPath = false) { - static const mbgl::filesystem::path vendorPath = getValidPath(testRootPath, std::string("vendor/")); - - mbgl::filesystem::path file = std::regex_replace(url, regex, vendorPath.string()); - if (mbgl::filesystem::exists(file.parent_path())) { - return removeURLArguments(file.string()); - } - - if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) { - return removeURLArguments(file.string()); - } - - return {}; -} - -mbgl::optional getIntegrationPath(const std::string& url, - const std::string& parent, - const std::regex& regex, - const std::string& testRootPath, - bool glyphsPath = false) { - static const mbgl::filesystem::path integrationPath = - getValidPath(testRootPath, std::string("mapbox-gl-js/test/integration/")); - - mbgl::filesystem::path file = std::regex_replace(url, regex, integrationPath.string() + parent); - if (mbgl::filesystem::exists(file.parent_path())) { - return removeURLArguments(file.string()); - } - - if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) { - return removeURLArguments(file.string()); - } - - return {}; -} - -mbgl::optional localizeLocalURL(const std::string& url, - const std::string& testRootPath, - bool glyphsPath = false) { - static const std::regex regex{"local://"}; - if (auto vendorPath = getVendorPath(url, regex, testRootPath, glyphsPath)) { - return vendorPath; - } else { - return getIntegrationPath(url, "", regex, testRootPath, glyphsPath); - } -} - -mbgl::optional localizeHttpURL(const std::string& url, const std::string& testRootPath) { - static const std::regex regex{"http://localhost:2900"}; - if (auto vendorPath = getVendorPath(url, regex, testRootPath)) { - return vendorPath; - } else { - return getIntegrationPath(url, "", regex, testRootPath); - } -} - -mbgl::optional localizeMapboxSpriteURL(const std::string& url, const std::string& testRootPath) { - static const std::regex regex{"mapbox://"}; - return getIntegrationPath(url, "", regex, testRootPath); -} - -mbgl::optional localizeMapboxFontsURL(const std::string& url, const std::string& testRootPath) { - static const std::regex regex{"mapbox://fonts"}; - return getIntegrationPath(url, "glyphs/", regex, testRootPath, true); -} - -mbgl::optional localizeMapboxTilesURL(const std::string& url, const std::string& testRootPath) { - static const std::regex regex{"mapbox://"}; - if (auto vendorPath = getVendorPath(url, regex, testRootPath)) { - return vendorPath; - } else { - return getIntegrationPath(url, "tiles/", regex, testRootPath); - } -} - -mbgl::optional localizeMapboxTilesetURL(const std::string& url, const std::string& testRootPath) { - static const std::regex regex{"mapbox://"}; - return getIntegrationPath(url, "tilesets/", regex, testRootPath); -} - void writeJSON(rapidjson::PrettyWriter& writer, const mbgl::Value& value) { value.match([&writer](const mbgl::NullValue&) { writer.Null(); }, [&writer](bool b) { writer.Bool(b); }, @@ -205,33 +108,6 @@ void writeJSON(rapidjson::PrettyWriter& writer, const m } // namespace -static const mbgl::filesystem::path DefaultRootPath{std::string(TEST_RUNNER_ROOT_PATH)}; - -const mbgl::filesystem::path getValidPath(const std::string& basePath, const std::string& subPath) { - auto filePath = mbgl::filesystem::path(basePath) / subPath; - if (mbgl::filesystem::exists(filePath)) { - return filePath; - } - // Fall back to check default path - filePath = DefaultRootPath / subPath; - if (mbgl::filesystem::exists(filePath)) { - return filePath; - } - mbgl::Log::Warning(mbgl::Event::General, "Failed to find path: %s", subPath.c_str()); - return mbgl::filesystem::path{}; -} - -/// Returns path of the render test cases directory. -const std::string getTestPath(const std::string& rootTestPath) { - // Check if sub-directory exits or not - auto testBasePath = mbgl::filesystem::path(rootTestPath) / ("mapbox-gl-js/test/integration"); - if (mbgl::filesystem::exists(testBasePath)) { - return testBasePath.string(); - } - // Use root test path for further processing - return rootTestPath; -} - std::string toJSON(const mbgl::Value& value, unsigned indent, bool singleLine) { rapidjson::StringBuffer buffer; rapidjson::PrettyWriter writer(buffer); @@ -284,7 +160,7 @@ JSONReply readJson(const mbgl::filesystem::path& jsonPath) { return { mbgl::formatJSONParseError(document) }; } - return { std::move(document) }; + return {std::move(document)}; } std::string serializeJsonValue(const mbgl::JSValue& value) { @@ -392,170 +268,6 @@ std::vector readExpectedJSONEntries(const mbgl::filesystem::path& b return readExpectedEntries(regex, base); } -namespace { - -std::vector getTestExpectations(mbgl::filesystem::path testPath, - const mbgl::filesystem::path& testsRootPath, - std::vector expectationsPaths) { - std::vector expectations{std::move(testPath.remove_filename())}; - const auto& defaultTestExpectationsPath = expectations.front().string(); - - const std::regex regex{testsRootPath.string()}; - for (const auto& path : expectationsPaths) { - expectations.emplace_back(std::regex_replace(defaultTestExpectationsPath, regex, path.string())); - assert(!expectations.back().empty()); - } - - return expectations; -} - -} // namespace - -ArgumentsTuple parseArguments(int argc, char** argv) { - args::ArgumentParser argumentParser("Mapbox GL Test Runner"); - - args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", { 'h', "help" }); - - args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", {'r', "recycle-map"}); - args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order", {'s', "shuffle"}); - args::ValueFlag seedValue(argumentParser, "seed", "Shuffle seed (default: random)", - { "seed" }); - args::ValueFlag testPathValue(argumentParser, "rootPath", "Test root rootPath", {'p', "rootPath"}); - args::ValueFlag testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"}); - args::ValueFlag expectationsPathValue( - argumentParser, "expectationsPath", "Test expectations path", {'e', "expectationsPath"}); - args::ValueFlag ignoresPathValue( - argumentParser, "ignoresPath", "Test ignore list path", {'i', "ignoresPath"}); - args::PositionalList testNameValues(argumentParser, "URL", "Test name(s)"); - - try { - argumentParser.ParseCLI(argc, argv); - } catch (const args::Help&) { - std::ostringstream stream; - stream << argumentParser; - mbgl::Log::Info(mbgl::Event::General, stream.str()); - exit(0); - } catch (const args::ParseError& e) { - std::ostringstream stream; - stream << argumentParser; - mbgl::Log::Info(mbgl::Event::General, stream.str()); - mbgl::Log::Error(mbgl::Event::General, e.what()); - exit(1); - } catch (const args::ValidationError& e) { - std::ostringstream stream; - stream << argumentParser; - mbgl::Log::Info(mbgl::Event::General, stream.str()); - mbgl::Log::Error(mbgl::Event::General, e.what()); - exit(2); - } catch (const std::regex_error& e) { - mbgl::Log::Error(mbgl::Event::General, "Invalid filter regular expression: %s", e.what()); - exit(3); - } - - const auto testRootPath = testPathValue ? args::get(testPathValue) : std::string{TEST_RUNNER_ROOT_PATH}; - mbgl::filesystem::path rootPath{testRootPath}; - if (!mbgl::filesystem::exists(rootPath)) { - mbgl::Log::Error( - mbgl::Event::General, "Provided test rootPath '%s' does not exist.", rootPath.string().c_str()); - exit(4); - } - std::vector expectationsPaths; - if (expectationsPathValue) { - auto expectationsPath = mbgl::filesystem::path(testRootPath) / args::get(expectationsPathValue); - if (!mbgl::filesystem::exists(expectationsPath)) { - mbgl::Log::Error(mbgl::Event::General, - "Provided expectationsPath '%s' does not exist.", - expectationsPath.string().c_str()); - exit(5); - } - expectationsPaths.emplace_back(std::move(expectationsPath)); - } - - std::string ignoresPath{}; - if (ignoresPathValue) { - auto path = mbgl::filesystem::path(testRootPath) / args::get(ignoresPathValue); - if (!mbgl::filesystem::exists(path)) { - mbgl::Log::Error( - mbgl::Event::General, "Provided ignore list path '%s' does not exist.", path.string().c_str()); - exit(6); - } - ignoresPath = path.string(); - } - - std::vector paths; - auto testBasePath = mbgl::filesystem::path(getTestPath(testRootPath)); - for (const auto& id : args::get(testNameValues)) { - paths.emplace_back(testBasePath / id); - } - - if (paths.empty()) { - paths.emplace_back(testBasePath); - } - - // Recursively traverse through the test paths and collect test directories containing "style.json". - std::vector testPaths; - testPaths.reserve(paths.size()); - for (const auto& path : paths) { - if (!mbgl::filesystem::exists(path)) { - mbgl::Log::Warning(mbgl::Event::General, "Provided test folder '%s' does not exist.", path.string().c_str()); - continue; - } - for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(path)) { - // Skip paths that fail regexp match. - if (testFilterValue && !std::regex_match(testPath.path().string(), args::get(testFilterValue))) { - continue; - } - - if (testPath.path().filename() == "style.json") { - testPaths.emplace_back(testPath, getTestExpectations(testPath, path, expectationsPaths)); - } - } - } - - return ArgumentsTuple{recycleMapFlag ? args::get(recycleMapFlag) : false, - shuffleFlag ? args::get(shuffleFlag) : false, - seedValue ? args::get(seedValue) : 1u, - testRootPath, - ignoresPath, - std::move(testPaths)}; -} - -std::vector> parseIgnores(const std::string& testRootPath, - const std::string& ignoresPath) { - std::vector> ignores; - auto mainIgnoresPath = getValidPath(testRootPath, "platform/node/test/ignores.json"); - - mbgl::filesystem::path platformSpecificIgnores; - mbgl::filesystem::path ownTestsIgnores = getValidPath(testRootPath, "render-test/tests/should-fail.json"); - -#ifdef __APPLE__ - platformSpecificIgnores = getValidPath(testRootPath, "render-test/mac-ignores.json"); -#elif __linux__ - platformSpecificIgnores = getValidPath(testRootPath, "render-test/linux-ignores.json"); -#endif - - std::vector ignoresPaths = {mainIgnoresPath, platformSpecificIgnores, ownTestsIgnores}; - - if (!ignoresPath.empty()) { - ignoresPaths.emplace_back(ignoresPath); - } - for (const auto& path : ignoresPaths) { - auto maybeIgnores = readJson(path); - if (!maybeIgnores.is()) { - continue; - } - for (const auto& property : maybeIgnores.get().GetObject()) { - const std::string ignore = { property.name.GetString(), - property.name.GetStringLength() }; - const std::string reason = { property.value.GetString(), - property.value.GetStringLength() }; - ignores.emplace_back(std::make_pair(ignore, reason)); - } - } - - return ignores; -} - TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) { TestMetrics result; @@ -601,7 +313,7 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) { std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()}; assert(!mark.empty()); result.memory.emplace(std::piecewise_construct, - std::forward_as_tuple(std::move(mark)), + std::forward_as_tuple(std::move(mark)), std::forward_as_tuple(probeValue[1].GetUint64(), probeValue[2].GetUint64())); } } @@ -645,7 +357,7 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) { return result; } -TestMetadata parseTestMetadata(const TestPaths& paths, const std::string& testRootPath) { +TestMetadata parseTestMetadata(const TestPaths& paths, const Manifest& manifest) { TestMetadata metadata; metadata.paths = paths; @@ -656,7 +368,7 @@ TestMetadata parseTestMetadata(const TestPaths& paths, const std::string& testRo } metadata.document = std::move(maybeJson.get()); - localizeStyleURLs(metadata.document, metadata.document, testRootPath); + manifest.localizeStyleURLs(metadata.document, metadata.document); if (!metadata.document.HasMember("metadata")) { mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s", paths.stylePath.c_str()); @@ -665,8 +377,7 @@ TestMetadata parseTestMetadata(const TestPaths& paths, const std::string& testRo const mbgl::JSValue& metadataValue = metadata.document["metadata"]; if (!metadataValue.HasMember("test")) { - mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata.test': %s", - paths.stylePath.c_str()); + mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata.test': %s", paths.stylePath.c_str()); return metadata; } @@ -694,8 +405,8 @@ TestMetadata parseTestMetadata(const TestPaths& paths, const std::string& testRo if (testValue.HasMember("description")) { assert(testValue["description"].IsString()); - metadata.description = std::string{ testValue["description"].GetString(), - testValue["description"].GetStringLength() }; + metadata.description = + std::string{testValue["description"].GetString(), testValue["description"].GetStringLength()}; } if (testValue.HasMember("mapMode")) { @@ -802,8 +513,9 @@ std::string encodeBase64(const std::string& data) { } std::string createResultItem(const TestMetadata& metadata, bool hasFailedTests) { - const bool shouldHide = (hasFailedTests && metadata.status == "passed") || (metadata.status.find("ignored") != std::string::npos); - + const bool shouldHide = + (hasFailedTests && metadata.status == "passed") || (metadata.status.find("ignored") != std::string::npos); + std::string html; html.append("
\n"); html.append(R"(

" + metadata.status + " " + metadata.id + "

\n"); @@ -918,91 +630,3 @@ std::string createResultPage(const TestStatistics& stats, const std::vector(path, document.GetAllocator()); - } - } - - if (root.HasMember("url")) { - static const std::string image("image"); - static const std::string video("video"); - - mbgl::JSValue& urlValue = root["url"]; - const std::string path = prependFileScheme( - localizeMapboxTilesetURL(urlValue.GetString(), testRootPath) - .value_or(localizeLocalURL(urlValue.GetString(), testRootPath).value_or(urlValue.GetString()))); - urlValue.Set(path, document.GetAllocator()); - - if (root["type"].GetString() != image && root["type"].GetString() != video) { - const auto tilesetPath = std::string(urlValue.GetString()).erase(0u, 7u); // remove "file://" - auto maybeTileset = readJson(tilesetPath); - if (maybeTileset.is()) { - const auto& tileset = maybeTileset.get(); - assert(tileset.HasMember("tiles")); - root.AddMember("tiles", (mbgl::JSValue&)tileset["tiles"], document.GetAllocator()); - root.RemoveMember("url"); - } - } - } - - if (root.HasMember("tiles")) { - mbgl::JSValue& tilesValue = root["tiles"]; - assert(tilesValue.IsArray()); - for (auto& tileValue : tilesValue.GetArray()) { - const std::string path = - prependFileScheme(localizeMapboxTilesURL(tileValue.GetString(), testRootPath) - .value_or(localizeLocalURL(tileValue.GetString(), testRootPath) - .value_or(localizeHttpURL(tileValue.GetString(), testRootPath) - .value_or(tileValue.GetString())))); - tileValue.Set(path, document.GetAllocator()); - } - } - - if (root.HasMember("data") && root["data"].IsString()) { - mbgl::JSValue& dataValue = root["data"]; - const std::string path = - prependFileScheme(localizeLocalURL(dataValue.GetString(), testRootPath).value_or(dataValue.GetString())); - dataValue.Set(path, document.GetAllocator()); - } -} - -void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const std::string& testRootPath) { - if (root.HasMember("sources")) { - mbgl::JSValue& sourcesValue = root["sources"]; - for (auto& sourceProperty : sourcesValue.GetObject()) { - localizeSourceURLs(sourceProperty.value, document, testRootPath); - } - } - - if (root.HasMember("glyphs")) { - mbgl::JSValue& glyphsValue = root["glyphs"]; - const std::string path = prependFileScheme( - localizeMapboxFontsURL(glyphsValue.GetString(), testRootPath) - .value_or( - localizeLocalURL(glyphsValue.GetString(), testRootPath, true).value_or(glyphsValue.GetString()))); - glyphsValue.Set(path, document.GetAllocator()); - } - - if (root.HasMember("sprite")) { - mbgl::JSValue& spriteValue = root["sprite"]; - const std::string path = prependFileScheme( - localizeMapboxSpriteURL(spriteValue.GetString(), testRootPath) - .value_or(localizeLocalURL(spriteValue.GetString(), testRootPath).value_or(spriteValue.GetString()))); - spriteValue.Set(path, document.GetAllocator()); - } -} diff --git a/render-test/parser.hpp b/render-test/parser.hpp index 1985b9cffc..3d79ac668a 100644 --- a/render-test/parser.hpp +++ b/render-test/parser.hpp @@ -5,15 +5,15 @@ #include #include -#include #include +#include #include +class Manifest; + using ErrorMessage = std::string; using JSONReply = mbgl::variant; -using ArgumentsTuple = std::tuple>; - JSONReply readJson(const mbgl::filesystem::path&); std::string serializeJsonValue(const mbgl::JSValue&); std::string serializeMetrics(const TestMetrics&); @@ -23,21 +23,9 @@ std::vector readExpectedJSONEntries(const mbgl::filesystem::path& b TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path); -const std::string getTestPath(const std::string& rootTestPath); -const mbgl::filesystem::path getValidPath(const std::string& basePath, const std::string& subPath); - -ArgumentsTuple parseArguments(int argc, char** argv); -std::vector> parseIgnores(const std::string& testRootPath, - const std::string& ignoresPath); - -TestMetadata parseTestMetadata(const TestPaths& paths, const std::string& testRootPath); +TestMetadata parseTestMetadata(const TestPaths& paths, const Manifest& manifest); std::string createResultPage(const TestStatistics&, const std::vector&, bool shuffle, uint32_t seed); -std::string localizeURL(const std::string& url, const std::string& testPath); - std::string toJSON(const mbgl::Value& value, unsigned indent, bool singleLine); std::string toJSON(const std::vector& features, unsigned indent, bool singleLine); - -void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const std::string& testPath); -void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document, const std::string& testPath); diff --git a/render-test/render_test.cpp b/render-test/render_test.cpp index 2cf3f5fd65..38d6c15f3f 100644 --- a/render-test/render_test.cpp +++ b/render-test/render_test.cpp @@ -2,14 +2,16 @@ #include #include +#include #include +#include + +#include "manifest_parser.hpp" #include "metadata.hpp" #include "parser.hpp" #include "runner.hpp" -#include - #define ANSI_COLOR_RED "\x1b[31m" #define ANSI_COLOR_GREEN "\x1b[32m" #define ANSI_COLOR_YELLOW "\x1b[33m" @@ -37,38 +39,97 @@ void operator delete(void* ptr, size_t) noexcept { } #endif +namespace { +using ArgumentsTuple = std::tuple, std::string>; +ArgumentsTuple parseArguments(int argc, char** argv) { + args::ArgumentParser argumentParser("Mapbox GL Test Runner"); + + args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", {'h', "help"}); + + args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", {'r', "recycle-map"}); + args::Flag shuffleFlag(argumentParser, "shuffle", "Toggle shuffling the tests order", {'s', "shuffle"}); + args::ValueFlag seedValue(argumentParser, "seed", "Shuffle seed (default: random)", {"seed"}); + args::ValueFlag testPathValue( + argumentParser, "manifestPath", "Test manifest file path", {'p', "manifestPath"}); + args::ValueFlag testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"}); + args::PositionalList testNameValues(argumentParser, "URL", "Test name(s)"); + + try { + argumentParser.ParseCLI(argc, argv); + } catch (const args::Help&) { + std::ostringstream stream; + stream << argumentParser; + mbgl::Log::Info(mbgl::Event::General, stream.str()); + exit(0); + } catch (const args::ParseError& e) { + std::ostringstream stream; + stream << argumentParser; + mbgl::Log::Info(mbgl::Event::General, stream.str()); + mbgl::Log::Error(mbgl::Event::General, e.what()); + exit(1); + } catch (const args::ValidationError& e) { + std::ostringstream stream; + stream << argumentParser; + mbgl::Log::Info(mbgl::Event::General, stream.str()); + mbgl::Log::Error(mbgl::Event::General, e.what()); + exit(2); + } catch (const std::regex_error& e) { + mbgl::Log::Error(mbgl::Event::General, "Invalid filter regular expression: %s", e.what()); + exit(3); + } + + mbgl::filesystem::path manifestPath{testPathValue ? args::get(testPathValue) : std::string{TEST_RUNNER_ROOT_PATH}}; + if (!mbgl::filesystem::exists(manifestPath) || !manifestPath.has_filename()) { + mbgl::Log::Error(mbgl::Event::General, + "Provided test manifest file path '%s' does not exist", + manifestPath.string().c_str()); + exit(4); + } + + auto testNames = testNameValues ? args::get(testNameValues) : std::vector{}; + auto testFilter = testFilterValue ? args::get(testFilterValue) : std::string{}; + const auto shuffle = shuffleFlag ? args::get(shuffleFlag) : false; + const auto seed = seedValue ? args::get(seedValue) : 1u; + return ArgumentsTuple{recycleMapFlag ? args::get(recycleMapFlag) : false, + shuffle, + seed, + manifestPath.string(), + std::move(testNames), + std::move(testFilter)}; +} +} // namespace namespace mbgl { int runRenderTests(int argc, char** argv) { bool recycleMap; bool shuffle; uint32_t seed; - std::string testRootPath; - std::string ignoresPath; - std::vector testPaths; - - std::tie(recycleMap, shuffle, seed, testRootPath, ignoresPath, testPaths) = parseArguments(argc, argv); - - const auto ignores = parseIgnores(testRootPath, ignoresPath); - + std::string manifestPath; + std::vector testNames; + std::string testFilter; + + std::tie(recycleMap, shuffle, seed, manifestPath, testNames, testFilter) = parseArguments(argc, argv); + auto manifestData = ManifestParser::parseManifest(manifestPath, testNames, testFilter); + if (!manifestData) { + exit(5); + } + mbgl::util::RunLoop runLoop; + TestRunner runner(std::move(*manifestData)); if (shuffle) { printf(ANSI_COLOR_YELLOW "Shuffle seed: %d" ANSI_COLOR_RESET "\n", seed); - - std::seed_seq sequence { seed }; - std::mt19937 shuffler(sequence); - std::shuffle(testPaths.begin(), testPaths.end(), shuffler); + runner.doShuffle(seed); } - mbgl::util::RunLoop runLoop; - TestRunner runner(testRootPath); - + const auto& manifest = runner.getManifest(); + const auto& ignores = manifest.getIgnores(); + const auto& testPaths = manifest.getTestPaths(); std::vector metadatas; metadatas.reserve(testPaths.size()); TestStatistics stats; for (auto& testPath : testPaths) { - TestMetadata metadata = parseTestMetadata(testPath, testRootPath); + TestMetadata metadata = parseTestMetadata(testPath, manifest); if (!recycleMap) { runner.reset(); @@ -78,7 +139,7 @@ int runRenderTests(int argc, char** argv) { std::string& status = metadata.status; std::string& color = metadata.color; - const std::string::size_type rootLength = getTestPath(testRootPath).length(); + const std::string::size_type rootLength = manifest.getTestRootPath().length(); id = testPath.defaultExpectations(); id = id.substr(rootLength + 1, id.length() - rootLength - 2); @@ -138,13 +199,14 @@ int runRenderTests(int argc, char** argv) { metadatas.push_back(std::move(metadata)); } - + const auto& testRootPath = manifest.getManifestPath(); + const auto resultPath = + testRootPath + "/" + (testNames.empty() ? "render-tests" : testNames.front()) + "_index.html"; std::string resultsHTML = createResultPage(stats, metadatas, shuffle, seed); - mbgl::util::write_file(testRootPath + "/index.html", resultsHTML); + mbgl::util::write_file(resultPath, resultsHTML); - const uint32_t count = stats.erroredTests + stats.failedTests + - stats.ignoreFailedTests + stats.ignorePassedTests + - stats.passedTests; + const uint32_t count = + stats.erroredTests + stats.failedTests + stats.ignoreFailedTests + stats.ignorePassedTests + stats.passedTests; if (stats.passedTests) { printf(ANSI_COLOR_GREEN "%u passed (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.passedTests, 100.0 * stats.passedTests / count); @@ -162,7 +224,7 @@ int runRenderTests(int argc, char** argv) { printf(ANSI_COLOR_RED "%u errored (%.1lf%%)" ANSI_COLOR_RESET "\n", stats.erroredTests, 100.0 * stats.erroredTests / count); } - printf("Results at: %s%s\n", testRootPath.c_str(), "/index.html"); + printf("Results at: %s\n", resultPath.c_str()); return stats.failedTests + stats.erroredTests == 0 ? 0 : 1; } diff --git a/render-test/runner.cpp b/render-test/runner.cpp index 4c0967a9e9..e882e394be 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -101,7 +101,15 @@ std::string simpleDiff(const Value& result, const Value& expected) { return diff.str(); } -TestRunner::TestRunner(const std::string& testRootPath_) : maps(), testRootPath(testRootPath_) {} +TestRunner::TestRunner(Manifest manifest_) : manifest(std::move(manifest_)) {} + +const Manifest& TestRunner::getManifest() const { + return manifest; +} + +void TestRunner::doShuffle(uint32_t seed) { + manifest.doShuffle(seed); +} bool TestRunner::checkQueryTestResults(mbgl::PremultipliedImage&& actualImage, std::vector&& features, @@ -487,7 +495,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { std::string imagePath = operationArray[2].GetString(); imagePath.erase(std::remove(imagePath.begin(), imagePath.end(), '"'), imagePath.end()); - const mbgl::filesystem::path filePath = mbgl::filesystem::path(getTestPath(testRootPath)) / imagePath; + const mbgl::filesystem::path filePath = mbgl::filesystem::path(manifest.getTestRootPath()) / imagePath; mbgl::optional maybeImage = mbgl::util::readFile(filePath.string()); if (!maybeImage) { @@ -507,15 +515,15 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { // setStyle assert(operationArray.Size() >= 2u); if (operationArray[1].IsString()) { - std::string stylePath = localizeURL(operationArray[1].GetString(), testRootPath); + std::string stylePath = manifest.localizeURL(operationArray[1].GetString()); auto maybeStyle = readJson(stylePath); if (maybeStyle.is()) { auto& style = maybeStyle.get(); - localizeStyleURLs((mbgl::JSValue&)style, style, testRootPath); + manifest.localizeStyleURLs((mbgl::JSValue&)style, style); map.getStyle().loadJSON(serializeJsonValue(style)); } } else { - localizeStyleURLs(operationArray[1], metadata.document, testRootPath); + manifest.localizeStyleURLs(operationArray[1], metadata.document); map.getStyle().loadJSON(serializeJsonValue(operationArray[1])); } } else if (operationArray[0].GetString() == setCenterOp) { @@ -616,7 +624,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { assert(operationArray[1].IsString()); assert(operationArray[2].IsObject()); - localizeSourceURLs(operationArray[2], metadata.document, testRootPath); + manifest.localizeSourceURLs(operationArray[2], metadata.document); mbgl::style::conversion::Error error; auto converted = mbgl::style::conversion::convert>(operationArray[2], error, operationArray[1].GetString()); diff --git a/render-test/runner.hpp b/render-test/runner.hpp index 3efd17bf1e..bc97f8300b 100644 --- a/render-test/runner.hpp +++ b/render-test/runner.hpp @@ -3,6 +3,8 @@ #include #include +#include "manifest_parser.hpp" + #include #include @@ -11,11 +13,14 @@ struct TestMetadata; class TestRunner { public: - TestRunner() = default; - explicit TestRunner(const std::string& testRootPath); + explicit TestRunner(Manifest); bool run(TestMetadata&); void reset(); + // Manifest + const Manifest& getManifest() const; + void doShuffle(uint32_t seed); + private: bool runOperations(const std::string& key, TestMetadata&); bool checkQueryTestResults(mbgl::PremultipliedImage&& actualImage, @@ -32,5 +37,5 @@ private: mbgl::Map map; }; std::unordered_map> maps; - std::string testRootPath{TEST_RUNNER_ROOT_PATH}; + Manifest manifest; }; -- cgit v1.2.1