summaryrefslogtreecommitdiff
path: root/render-test
diff options
context:
space:
mode:
authorMikhail Pozdnyakov <mikhail.pozdnyakov@mapbox.com>2019-08-14 17:07:00 +0300
committerMikhail Pozdnyakov <mikhail.pozdnyakov@mapbox.com>2019-08-16 16:25:59 +0300
commita1904e802c61e5b5a0cef8cc9de9bf7c2c617552 (patch)
treee5c71ef187ca93be40a8228b59295358261873bf /render-test
parent1165f62df6dd344d83f88a6396cfba153e9271f3 (diff)
downloadqtlocation-mapboxgl-a1904e802c61e5b5a0cef8cc9de9bf7c2c617552.tar.gz
[test-runner] Introduce mapbox-gl-native test expectations
The `mapbox-gl-native`-specific render tests expected results will be stored at the `render-test/expected/` subfolder. Test runner will automatically put new expected results there if invoked with `UPDATE=true`.
Diffstat (limited to 'render-test')
-rw-r--r--render-test/main.cpp27
-rw-r--r--render-test/metadata.hpp8
-rw-r--r--render-test/parser.cpp20
-rw-r--r--render-test/parser.hpp4
-rw-r--r--render-test/runner.cpp25
-rw-r--r--render-test/runner.hpp5
6 files changed, 72 insertions, 17 deletions
diff --git a/render-test/main.cpp b/render-test/main.cpp
index 35f0cdea30..db141649f2 100644
--- a/render-test/main.cpp
+++ b/render-test/main.cpp
@@ -7,6 +7,7 @@
#include "runner.hpp"
#include <random>
+#include <regex>
#define ANSI_COLOR_RED "\x1b[31m"
#define ANSI_COLOR_GREEN "\x1b[32m"
@@ -18,6 +19,25 @@
#define ANSI_COLOR_LIGHT_GRAY "\x1b[90m"
#define ANSI_COLOR_RESET "\x1b[0m"
+namespace {
+
+TestPaths makeTestPaths(mbgl::filesystem::path stylePath) {
+ auto defaultExpectations = stylePath;
+ defaultExpectations.remove_filename();
+ const static std::regex regex{ TestRunner::getBasePath() };
+ auto platformExpectations = std::regex_replace(defaultExpectations.string(), regex, TestRunner::getPlatformExpectationsPath());
+ assert(!defaultExpectations.empty());
+ assert(!platformExpectations.empty());
+
+ return {
+ std::move(stylePath),
+ std::move(defaultExpectations),
+ std::move(platformExpectations)
+ };
+}
+
+} // namespace
+
int main(int argc, char** argv) {
bool recycleMap;
bool shuffle;
@@ -31,11 +51,12 @@ int main(int argc, char** argv) {
const auto ignores = parseIgnores();
// Recursively traverse through the test paths and collect test directories containing "style.json".
- std::vector<mbgl::filesystem::path> testPaths;
+ std::vector<TestPaths> testPaths;
+ testPaths.reserve(ids.size());
for (const auto& id : ids) {
for (auto& testPath : mbgl::filesystem::recursive_directory_iterator(mbgl::filesystem::path(id))) {
if (testPath.path().filename() == "style.json") {
- testPaths.push_back(testPath);
+ testPaths.emplace_back(makeTestPaths(testPath));
}
}
}
@@ -67,7 +88,7 @@ int main(int argc, char** argv) {
std::string& status = metadata.status;
std::string& color = metadata.color;
- id = testPath.remove_filename().string();
+ id = testPath.defaultExpectations.string();
id = id.substr(rootLength + 1, id.length() - rootLength - 2);
bool shouldIgnore = false;
diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp
index 4be83a5436..2f568359ab 100644
--- a/render-test/metadata.hpp
+++ b/render-test/metadata.hpp
@@ -17,10 +17,16 @@ struct TestStatistics {
uint32_t passedTests = 0;
};
+struct TestPaths {
+ mbgl::filesystem::path stylePath;
+ mbgl::filesystem::path defaultExpectations;
+ mbgl::filesystem::path platformExpectations;
+};
+
struct TestMetadata {
TestMetadata() = default;
- mbgl::filesystem::path path;
+ TestPaths paths;
mbgl::JSDocument document;
mbgl::Size size{ 512u, 512u };
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index fd048f08c6..49f084465c 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -15,6 +15,7 @@
#include "parser.hpp"
#include "metadata.hpp"
+#include "runner.hpp"
#include <sstream>
#include <regex>
@@ -243,17 +244,18 @@ ArgumentsTuple parseArguments(int argc, char** argv) {
std::vector<std::string> ids;
for (const auto& id : args::get(testNameValues)) {
- ids.emplace_back(testDefaultPath + "/" + id);
+ ids.emplace_back(TestRunner::getBasePath() + "/" + id);
}
if (ids.empty()) {
- ids.emplace_back(testDefaultPath);
+ ids.emplace_back(TestRunner::getBasePath());
}
return ArgumentsTuple {
recycleMapFlag ? args::get(recycleMapFlag) : false,
shuffleFlag ? args::get(shuffleFlag) : false, seedValue ? args::get(seedValue) : 1u,
- testPathValue ? args::get(testPathValue) : testDefaultPath, ids
+ testPathValue ? args::get(testPathValue) : testDefaultPath,
+ std::move(ids)
};
}
@@ -288,13 +290,13 @@ std::vector<std::pair<std::string, std::string>> parseIgnores() {
return ignores;
}
-TestMetadata parseTestMetadata(const mbgl::filesystem::path& path) {
+TestMetadata parseTestMetadata(const TestPaths& paths) {
TestMetadata metadata;
- metadata.path = path;
+ metadata.paths = paths;
- auto maybeJson = readJson(path.string());
+ auto maybeJson = readJson(paths.stylePath.string());
if (!maybeJson.is<mbgl::JSDocument>()) { // NOLINT
- metadata.errorMessage = std::string("Unable to parse: ") + path.string();
+ metadata.errorMessage = std::string("Unable to parse: ") + metadata.paths.stylePath.string();
return metadata;
}
@@ -303,14 +305,14 @@ TestMetadata parseTestMetadata(const mbgl::filesystem::path& path) {
if (!metadata.document.HasMember("metadata")) {
mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s",
- path.c_str());
+ paths.stylePath.c_str());
return metadata;
}
const mbgl::JSValue& metadataValue = metadata.document["metadata"];
if (!metadataValue.HasMember("test")) {
mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata.test': %s",
- path.c_str());
+ paths.stylePath.c_str());
return metadata;
}
diff --git a/render-test/parser.hpp b/render-test/parser.hpp
index be98719ab5..3e69968152 100644
--- a/render-test/parser.hpp
+++ b/render-test/parser.hpp
@@ -11,6 +11,7 @@
struct TestMetadata;
struct TestStatistics;
+struct TestPaths;
using ErrorMessage = std::string;
using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>;
@@ -24,7 +25,8 @@ std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base)
ArgumentsTuple parseArguments(int argc, char** argv);
std::vector<std::pair<std::string, std::string>> parseIgnores();
-TestMetadata parseTestMetadata(const mbgl::filesystem::path& path);
+
+TestMetadata parseTestMetadata(const TestPaths& paths);
std::string createResultPage(const TestStatistics&, const std::vector<TestMetadata>&, bool shuffle, uint32_t seed);
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 5bd855c43a..7b0f210b62 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -26,8 +26,24 @@
#include <cassert>
#include <regex>
+
+// static
+const std::string& TestRunner::getBasePath() {
+ const static std::string result =
+ std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration/render-tests");
+ return result;
+}
+
+// static
+const std::string& TestRunner::getPlatformExpectationsPath() {
+ const static std::string result =
+ std::string(TEST_RUNNER_ROOT_PATH).append("/render-test/expected");
+ return result;
+}
+
bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& metadata) {
- const std::string base = metadata.path.remove_filename().string();
+ const std::string& base = metadata.paths.defaultExpectations.string();
+
metadata.actual = mbgl::encodePNG(actual);
if (actual.size.isEmpty()) {
@@ -37,7 +53,8 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met
#if !TEST_READ_ONLY
if (getenv("UPDATE")) {
- mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actual));
+ mbgl::filesystem::create_directories(metadata.paths.platformExpectations);
+ mbgl::util::write_file(metadata.paths.platformExpectations.string() + "/expected.png", mbgl::encodePNG(actual));
return true;
}
@@ -48,8 +65,10 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met
mbgl::PremultipliedImage diff { actual.size };
double pixels = 0.0;
+ const auto& expectedPath = mbgl::filesystem::exists(metadata.paths.platformExpectations) ?
+ metadata.paths.platformExpectations : metadata.paths.defaultExpectations;
- for (const auto& entry: readExpectedEntries(base)) {
+ for (const auto& entry: readExpectedEntries(expectedPath)) {
mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
if (!maybeExpectedImage) {
metadata.errorMessage = "Failed to load expected image " + entry;
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
index cbc0f42546..920a9ee372 100644
--- a/render-test/runner.hpp
+++ b/render-test/runner.hpp
@@ -14,6 +14,11 @@ public:
bool run(TestMetadata&);
void reset();
+ /// Returns path of the render tests root directory.
+ static const std::string& getBasePath();
+ /// Returns path of mapbox-gl-native expectations directory.
+ static const std::string& getPlatformExpectationsPath();
+
private:
bool runOperations(const std::string& key, TestMetadata&);
bool checkImage(mbgl::PremultipliedImage&& image, TestMetadata&);