From b03982730e6c02dcc166a609bf4b8fd17b3cf01d Mon Sep 17 00:00:00 2001 From: Mikhail Pozdnyakov Date: Thu, 17 Oct 2019 20:13:45 +0300 Subject: [test runner] Command line option for the tests expectations path -e[expectationsPath], --expectationsPath=[expectationsPath] Test expectations path. --- next/platform/linux/linux.cmake | 1 + next/platform/macos/macos.cmake | 1 + render-test/metadata.hpp | 4 ++++ render-test/parser.cpp | 53 ++++++++++++++++++++++++++--------------- render-test/runner.cpp | 9 ------- render-test/runner.hpp | 2 -- 6 files changed, 40 insertions(+), 30 deletions(-) diff --git a/next/platform/linux/linux.cmake b/next/platform/linux/linux.cmake index be905c204f..7b77e134ea 100644 --- a/next/platform/linux/linux.cmake +++ b/next/platform/linux/linux.cmake @@ -143,6 +143,7 @@ add_test( render-tests --recycle-map --shuffle + --expectationsPath=render-test/expected/render-tests --seed=${MBGL_RENDER_TEST_SEED} WORKING_DIRECTORY ${MBGL_ROOT} ) diff --git a/next/platform/macos/macos.cmake b/next/platform/macos/macos.cmake index cbe386a1ac..69f41027c6 100644 --- a/next/platform/macos/macos.cmake +++ b/next/platform/macos/macos.cmake @@ -207,6 +207,7 @@ add_test( render-tests --recycle-map --shuffle + --expectationsPath=render-test/expected/render-tests --seed=${MBGL_RENDER_TEST_SEED} WORKING_DIRECTORY ${MBGL_ROOT} ) diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp index 20cc440fd4..1d5a346f22 100644 --- a/render-test/metadata.hpp +++ b/render-test/metadata.hpp @@ -22,6 +22,10 @@ struct TestStatistics { }; struct TestPaths { + TestPaths() = default; + TestPaths(mbgl::filesystem::path stylePath_, std::vector expectations_) + : stylePath(std::move(stylePath_)), expectations(std::move(expectations_)) {} + mbgl::filesystem::path stylePath; std::vector expectations; diff --git a/render-test/parser.cpp b/render-test/parser.cpp index 7300530b07..da5e6bea81 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -168,22 +168,6 @@ mbgl::optional localizeMapboxTilesetURL(const std::string& url) { return getIntegrationPath(url, "tilesets/", regex); } -TestPaths makeTestPaths(mbgl::filesystem::path stylePath) { - std::vector expectations{ stylePath }; - expectations.front().remove_filename(); - - const static std::regex regex{ TestRunner::getBasePath() }; - for (const std::string& path : TestRunner::getPlatformExpectationsPaths()) { - expectations.emplace_back(std::regex_replace(expectations.front().string(), regex, path)); - assert(!expectations.back().empty()); - } - - return { - std::move(stylePath), - std::move(expectations) - }; -} - void writeJSON(rapidjson::PrettyWriter& writer, const mbgl::Value& value) { value.match([&writer](const mbgl::NullValue&) { writer.Null(); }, [&writer](bool b) { writer.Bool(b); }, @@ -349,6 +333,25 @@ std::vector readExpectedJSONEntries(const mbgl::filesystem::path& b return readExpectedEntries(regex, base); } +namespace { + +std::vector getTestExpectations(mbgl::filesystem::path testPath, + const mbgl::filesystem::path& testsRootPath, + std::vector expectationsPaths) { + std::vector expectations{std::move(testPath.remove_filename())}; + const auto& defaultTestExpectationsPath = expectations.front().string(); + + const std::regex regex{testsRootPath.string()}; + for (const auto& path : expectationsPaths) { + expectations.emplace_back(std::regex_replace(defaultTestExpectationsPath, regex, path.string())); + assert(!expectations.back().empty()); + } + + return expectations; +} + +} // namespace + ArgumentsTuple parseArguments(int argc, char** argv) { args::ArgumentParser argumentParser("Mapbox GL Test Runner"); @@ -362,8 +365,9 @@ ArgumentsTuple parseArguments(int argc, char** argv) { { "seed" }); args::ValueFlag testPathValue(argumentParser, "rootPath", "Test root rootPath", { 'p', "rootPath" }); - args::ValueFlag testFilterValue(argumentParser, "filter", "Test filter regex", - { 'f', "filter" }); + args::ValueFlag testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"}); + args::ValueFlag expectationsPathValue( + argumentParser, "expectationsPath", "Test expectations path", {'e', "expectationsPath"}); args::PositionalList testNameValues(argumentParser, "URL", "Test name(s)"); try { @@ -395,6 +399,17 @@ ArgumentsTuple parseArguments(int argc, char** argv) { mbgl::Log::Error(mbgl::Event::General, "Provided rootPath '%s' does not exist.", rootPath.string().c_str()); exit(4); } + std::vector expectationsPaths; + if (expectationsPathValue) { + auto expectationsPath = mbgl::filesystem::path(TEST_RUNNER_ROOT_PATH) / args::get(expectationsPathValue); + if (!mbgl::filesystem::exists(expectationsPath)) { + mbgl::Log::Error(mbgl::Event::General, + "Provided expectationsPath '%s' does not exist.", + expectationsPath.string().c_str()); + exit(5); + } + expectationsPaths.emplace_back(std::move(expectationsPath)); + } std::vector paths; for (const auto& id : args::get(testNameValues)) { @@ -419,7 +434,7 @@ ArgumentsTuple parseArguments(int argc, char** argv) { continue; } if (testPath.path().filename() == "style.json") { - testPaths.emplace_back(makeTestPaths(testPath)); + testPaths.emplace_back(testPath, getTestExpectations(testPath, path, expectationsPaths)); } } } diff --git a/render-test/runner.cpp b/render-test/runner.cpp index 651361994a..0464fd861a 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -39,15 +39,6 @@ const std::string& TestRunner::getBasePath() { return result; } -// static -const std::vector& TestRunner::getPlatformExpectationsPaths() { - // TODO: Populate from command line. - const static std::vector result { - std::string(TEST_RUNNER_ROOT_PATH).append("/render-test/expected") - }; - return result; -} - std::string simpleDiff(const Value& result, const Value& expected) { std::vector resultTokens{tokenize(toJSON(result, 2, false))}; std::vector expectedTokens{tokenize(toJSON(expected, 2, false))}; diff --git a/render-test/runner.hpp b/render-test/runner.hpp index d8e5275f61..ea593bcc61 100644 --- a/render-test/runner.hpp +++ b/render-test/runner.hpp @@ -16,8 +16,6 @@ public: /// Returns path of the render tests root directory. static const std::string& getBasePath(); - /// Returns path of mapbox-gl-native expectations directory. - static const std::vector& getPlatformExpectationsPaths(); private: bool runOperations(const std::string& key, TestMetadata&); -- cgit v1.2.1