From a1904e802c61e5b5a0cef8cc9de9bf7c2c617552 Mon Sep 17 00:00:00 2001 From: Mikhail Pozdnyakov Date: Wed, 14 Aug 2019 17:07:00 +0300 Subject: [test-runner] Introduce mapbox-gl-native test expectations The `mapbox-gl-native`-specific render tests expected results will be stored at the `render-test/expected/` subfolder. Test runner will automatically put new expected results there if invoked with `UPDATE=true`. --- render-test/main.cpp | 27 ++++++++++++++++++++++++--- render-test/metadata.hpp | 8 +++++++- render-test/parser.cpp | 20 +++++++++++--------- render-test/parser.hpp | 4 +++- render-test/runner.cpp | 25 ++++++++++++++++++++++--- render-test/runner.hpp | 5 +++++ 6 files changed, 72 insertions(+), 17 deletions(-) diff --git a/render-test/main.cpp b/render-test/main.cpp index 35f0cdea30..db141649f2 100644 --- a/render-test/main.cpp +++ b/render-test/main.cpp @@ -7,6 +7,7 @@ #include "runner.hpp" #include +#include #define ANSI_COLOR_RED "\x1b[31m" #define ANSI_COLOR_GREEN "\x1b[32m" @@ -18,6 +19,25 @@ #define ANSI_COLOR_LIGHT_GRAY "\x1b[90m" #define ANSI_COLOR_RESET "\x1b[0m" +namespace { + +TestPaths makeTestPaths(mbgl::filesystem::path stylePath) { + auto defaultExpectations = stylePath; + defaultExpectations.remove_filename(); + const static std::regex regex{ TestRunner::getBasePath() }; + auto platformExpectations = std::regex_replace(defaultExpectations.string(), regex, TestRunner::getPlatformExpectationsPath()); + assert(!defaultExpectations.empty()); + assert(!platformExpectations.empty()); + + return { + std::move(stylePath), + std::move(defaultExpectations), + std::move(platformExpectations) + }; +} + +} // namespace + int main(int argc, char** argv) { bool recycleMap; bool shuffle; @@ -31,11 +51,12 @@ int main(int argc, char** argv) { const auto ignores = parseIgnores(); // Recursively traverse through the test paths and collect test directories containing "style.json". - std::vector testPaths; + std::vector testPaths; + testPaths.reserve(ids.size()); for (const auto& id : ids) { for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(mbgl::filesystem::path(id))) { if (testPath.path().filename() == "style.json") { - testPaths.push_back(testPath); + testPaths.emplace_back(makeTestPaths(testPath)); } } } @@ -67,7 +88,7 @@ int main(int argc, char** argv) { std::string& status = metadata.status; std::string& color = metadata.color; - id = testPath.remove_filename().string(); + id = testPath.defaultExpectations.string(); id = id.substr(rootLength + 1, id.length() - rootLength - 2); bool shouldIgnore = false; diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp index 4be83a5436..2f568359ab 100644 --- a/render-test/metadata.hpp +++ b/render-test/metadata.hpp @@ -17,10 +17,16 @@ struct TestStatistics { uint32_t passedTests = 0; }; +struct TestPaths { + mbgl::filesystem::path stylePath; + mbgl::filesystem::path defaultExpectations; + mbgl::filesystem::path platformExpectations; +}; + struct TestMetadata { TestMetadata() = default; - mbgl::filesystem::path path; + TestPaths paths; mbgl::JSDocument document; mbgl::Size size{ 512u, 512u }; diff --git a/render-test/parser.cpp b/render-test/parser.cpp index fd048f08c6..49f084465c 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -15,6 +15,7 @@ #include "parser.hpp" #include "metadata.hpp" +#include "runner.hpp" #include #include @@ -243,17 +244,18 @@ ArgumentsTuple parseArguments(int argc, char** argv) { std::vector ids; for (const auto& id : args::get(testNameValues)) { - ids.emplace_back(testDefaultPath + "/" + id); + ids.emplace_back(TestRunner::getBasePath() + "/" + id); } if (ids.empty()) { - ids.emplace_back(testDefaultPath); + ids.emplace_back(TestRunner::getBasePath()); } return ArgumentsTuple { recycleMapFlag ? args::get(recycleMapFlag) : false, shuffleFlag ? args::get(shuffleFlag) : false, seedValue ? args::get(seedValue) : 1u, - testPathValue ? args::get(testPathValue) : testDefaultPath, ids + testPathValue ? args::get(testPathValue) : testDefaultPath, + std::move(ids) }; } @@ -288,13 +290,13 @@ std::vector> parseIgnores() { return ignores; } -TestMetadata parseTestMetadata(const mbgl::filesystem::path& path) { +TestMetadata parseTestMetadata(const TestPaths& paths) { TestMetadata metadata; - metadata.path = path; + metadata.paths = paths; - auto maybeJson = readJson(path.string()); + auto maybeJson = readJson(paths.stylePath.string()); if (!maybeJson.is()) { // NOLINT - metadata.errorMessage = std::string("Unable to parse: ") + path.string(); + metadata.errorMessage = std::string("Unable to parse: ") + metadata.paths.stylePath.string(); return metadata; } @@ -303,14 +305,14 @@ TestMetadata parseTestMetadata(const mbgl::filesystem::path& path) { if (!metadata.document.HasMember("metadata")) { mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s", - path.c_str()); + paths.stylePath.c_str()); return metadata; } const mbgl::JSValue& metadataValue = metadata.document["metadata"]; if (!metadataValue.HasMember("test")) { mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata.test': %s", - path.c_str()); + paths.stylePath.c_str()); return metadata; } diff --git a/render-test/parser.hpp b/render-test/parser.hpp index be98719ab5..3e69968152 100644 --- a/render-test/parser.hpp +++ b/render-test/parser.hpp @@ -11,6 +11,7 @@ struct TestMetadata; struct TestStatistics; +struct TestPaths; using ErrorMessage = std::string; using JSONReply = mbgl::variant; @@ -24,7 +25,8 @@ std::vector readExpectedEntries(const mbgl::filesystem::path& base) ArgumentsTuple parseArguments(int argc, char** argv); std::vector> parseIgnores(); -TestMetadata parseTestMetadata(const mbgl::filesystem::path& path); + +TestMetadata parseTestMetadata(const TestPaths& paths); std::string createResultPage(const TestStatistics&, const std::vector&, bool shuffle, uint32_t seed); diff --git a/render-test/runner.cpp b/render-test/runner.cpp index 5bd855c43a..7b0f210b62 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -26,8 +26,24 @@ #include #include + +// static +const std::string& TestRunner::getBasePath() { + const static std::string result = + std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration/render-tests"); + return result; +} + +// static +const std::string& TestRunner::getPlatformExpectationsPath() { + const static std::string result = + std::string(TEST_RUNNER_ROOT_PATH).append("/render-test/expected"); + return result; +} + bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& metadata) { - const std::string base = metadata.path.remove_filename().string(); + const std::string& base = metadata.paths.defaultExpectations.string(); + metadata.actual = mbgl::encodePNG(actual); if (actual.size.isEmpty()) { @@ -37,7 +53,8 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met #if !TEST_READ_ONLY if (getenv("UPDATE")) { - mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actual)); + mbgl::filesystem::create_directories(metadata.paths.platformExpectations); + mbgl::util::write_file(metadata.paths.platformExpectations.string() + "/expected.png", mbgl::encodePNG(actual)); return true; } @@ -48,8 +65,10 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met mbgl::PremultipliedImage diff { actual.size }; double pixels = 0.0; + const auto& expectedPath = mbgl::filesystem::exists(metadata.paths.platformExpectations) ? + metadata.paths.platformExpectations : metadata.paths.defaultExpectations; - for (const auto& entry: readExpectedEntries(base)) { + for (const auto& entry: readExpectedEntries(expectedPath)) { mbgl::optional maybeExpectedImage = mbgl::util::readFile(entry); if (!maybeExpectedImage) { metadata.errorMessage = "Failed to load expected image " + entry; diff --git a/render-test/runner.hpp b/render-test/runner.hpp index cbc0f42546..920a9ee372 100644 --- a/render-test/runner.hpp +++ b/render-test/runner.hpp @@ -14,6 +14,11 @@ public: bool run(TestMetadata&); void reset(); + /// Returns path of the render tests root directory. + static const std::string& getBasePath(); + /// Returns path of mapbox-gl-native expectations directory. + static const std::string& getPlatformExpectationsPath(); + private: bool runOperations(const std::string& key, TestMetadata&); bool checkImage(mbgl::PremultipliedImage&& image, TestMetadata&); -- cgit v1.2.1