From 13801f1b80e21e128b851caa58a9f4bbd2421844 Mon Sep 17 00:00:00 2001 From: Juha Alanen Date: Wed, 2 Oct 2019 13:26:39 +0300 Subject: [render-test] Add support for query tests --- .../debug/collision-lines-overscaled/expected.png | Bin 7774 -> 0 bytes .../debug/collision-lines-pitched/expected.png | Bin 165500 -> 0 bytes .../expected/debug/collision-lines/expected.png | Bin 196454 -> 0 bytes .../debug/collision-pitched-wrapped/expected.png | Bin 69725 -> 0 bytes .../debug/collision-lines-overscaled/expected.png | Bin 0 -> 7774 bytes .../debug/collision-lines-pitched/expected.png | Bin 0 -> 165500 bytes .../debug/collision-lines/expected.png | Bin 0 -> 196454 bytes .../debug/collision-pitched-wrapped/expected.png | Bin 0 -> 69725 bytes .../symbol-visibility/visible/expected.png | Bin 0 -> 14729 bytes .../auto-text-rotation-alignment-map/expected.png | Bin 0 -> 26157 bytes .../map-text-rotation-alignment-map/expected.png | Bin 0 -> 26157 bytes .../expected.png | Bin 0 -> 26866 bytes .../pitched-rotated-debug/expected.png | Bin 0 -> 43135 bytes .../rotated-offset/expected.png | Bin 0 -> 35553 bytes .../symbol-visibility/visible/expected.png | Bin 14729 -> 0 bytes .../auto-text-rotation-alignment-map/expected.png | Bin 26157 -> 0 bytes .../map-text-rotation-alignment-map/expected.png | Bin 26157 -> 0 bytes .../expected.png | Bin 26866 -> 0 bytes .../pitched-rotated-debug/expected.png | Bin 43135 -> 0 bytes .../rotated-offset/expected.png | Bin 35553 -> 0 bytes render-test/main.cpp | 2 +- render-test/metadata.hpp | 7 + render-test/parser.cpp | 134 ++++++++++- render-test/parser.hpp | 3 + render-test/runner.cpp | 254 ++++++++++++++++++++- render-test/runner.hpp | 5 +- 26 files changed, 387 insertions(+), 18 deletions(-) delete mode 100644 render-test/expected/debug/collision-lines-overscaled/expected.png delete mode 100644 render-test/expected/debug/collision-lines-pitched/expected.png delete mode 100644 render-test/expected/debug/collision-lines/expected.png delete mode 100644 render-test/expected/debug/collision-pitched-wrapped/expected.png create mode 100644 render-test/expected/render-tests/debug/collision-lines-overscaled/expected.png create mode 100644 render-test/expected/render-tests/debug/collision-lines-pitched/expected.png create mode 100644 render-test/expected/render-tests/debug/collision-lines/expected.png create mode 100644 render-test/expected/render-tests/debug/collision-pitched-wrapped/expected.png create mode 100644 render-test/expected/render-tests/symbol-visibility/visible/expected.png create mode 100644 render-test/expected/render-tests/text-pitch-alignment/auto-text-rotation-alignment-map/expected.png create mode 100644 render-test/expected/render-tests/text-pitch-alignment/map-text-rotation-alignment-map/expected.png create mode 100644 render-test/expected/render-tests/text-pitch-alignment/viewport-text-rotation-alignment-map/expected.png create mode 100644 render-test/expected/render-tests/text-variable-anchor/pitched-rotated-debug/expected.png create mode 100644 render-test/expected/render-tests/text-variable-anchor/rotated-offset/expected.png delete mode 100644 render-test/expected/symbol-visibility/visible/expected.png delete mode 100644 render-test/expected/text-pitch-alignment/auto-text-rotation-alignment-map/expected.png delete mode 100644 render-test/expected/text-pitch-alignment/map-text-rotation-alignment-map/expected.png delete mode 100644 render-test/expected/text-pitch-alignment/viewport-text-rotation-alignment-map/expected.png delete mode 100644 render-test/expected/text-variable-anchor/pitched-rotated-debug/expected.png delete mode 100644 render-test/expected/text-variable-anchor/rotated-offset/expected.png diff --git a/render-test/expected/debug/collision-lines-overscaled/expected.png b/render-test/expected/debug/collision-lines-overscaled/expected.png deleted file mode 100644 index 38eb0d2da6..0000000000 Binary files a/render-test/expected/debug/collision-lines-overscaled/expected.png and /dev/null differ diff --git a/render-test/expected/debug/collision-lines-pitched/expected.png b/render-test/expected/debug/collision-lines-pitched/expected.png deleted file mode 100644 index 416d7d5715..0000000000 Binary files a/render-test/expected/debug/collision-lines-pitched/expected.png and /dev/null differ diff --git a/render-test/expected/debug/collision-lines/expected.png b/render-test/expected/debug/collision-lines/expected.png deleted file mode 100644 index 3f4790a585..0000000000 Binary files a/render-test/expected/debug/collision-lines/expected.png and /dev/null differ diff --git a/render-test/expected/debug/collision-pitched-wrapped/expected.png b/render-test/expected/debug/collision-pitched-wrapped/expected.png deleted file mode 100644 index 9b718c09c0..0000000000 Binary files a/render-test/expected/debug/collision-pitched-wrapped/expected.png and /dev/null differ diff --git a/render-test/expected/render-tests/debug/collision-lines-overscaled/expected.png b/render-test/expected/render-tests/debug/collision-lines-overscaled/expected.png new file mode 100644 index 0000000000..38eb0d2da6 Binary files /dev/null and b/render-test/expected/render-tests/debug/collision-lines-overscaled/expected.png differ diff --git a/render-test/expected/render-tests/debug/collision-lines-pitched/expected.png b/render-test/expected/render-tests/debug/collision-lines-pitched/expected.png new file mode 100644 index 0000000000..416d7d5715 Binary files /dev/null and b/render-test/expected/render-tests/debug/collision-lines-pitched/expected.png differ diff --git a/render-test/expected/render-tests/debug/collision-lines/expected.png b/render-test/expected/render-tests/debug/collision-lines/expected.png new file mode 100644 index 0000000000..3f4790a585 Binary files /dev/null and b/render-test/expected/render-tests/debug/collision-lines/expected.png differ diff --git a/render-test/expected/render-tests/debug/collision-pitched-wrapped/expected.png b/render-test/expected/render-tests/debug/collision-pitched-wrapped/expected.png new file mode 100644 index 0000000000..9b718c09c0 Binary files /dev/null and b/render-test/expected/render-tests/debug/collision-pitched-wrapped/expected.png differ diff --git a/render-test/expected/render-tests/symbol-visibility/visible/expected.png b/render-test/expected/render-tests/symbol-visibility/visible/expected.png new file mode 100644 index 0000000000..8da157772a Binary files /dev/null and b/render-test/expected/render-tests/symbol-visibility/visible/expected.png differ diff --git a/render-test/expected/render-tests/text-pitch-alignment/auto-text-rotation-alignment-map/expected.png b/render-test/expected/render-tests/text-pitch-alignment/auto-text-rotation-alignment-map/expected.png new file mode 100644 index 0000000000..cd690ca152 Binary files /dev/null and b/render-test/expected/render-tests/text-pitch-alignment/auto-text-rotation-alignment-map/expected.png differ diff --git a/render-test/expected/render-tests/text-pitch-alignment/map-text-rotation-alignment-map/expected.png b/render-test/expected/render-tests/text-pitch-alignment/map-text-rotation-alignment-map/expected.png new file mode 100644 index 0000000000..cd690ca152 Binary files /dev/null and b/render-test/expected/render-tests/text-pitch-alignment/map-text-rotation-alignment-map/expected.png differ diff --git a/render-test/expected/render-tests/text-pitch-alignment/viewport-text-rotation-alignment-map/expected.png b/render-test/expected/render-tests/text-pitch-alignment/viewport-text-rotation-alignment-map/expected.png new file mode 100644 index 0000000000..764d4a0b24 Binary files /dev/null and b/render-test/expected/render-tests/text-pitch-alignment/viewport-text-rotation-alignment-map/expected.png differ diff --git a/render-test/expected/render-tests/text-variable-anchor/pitched-rotated-debug/expected.png b/render-test/expected/render-tests/text-variable-anchor/pitched-rotated-debug/expected.png new file mode 100644 index 0000000000..4e3d012844 Binary files /dev/null and b/render-test/expected/render-tests/text-variable-anchor/pitched-rotated-debug/expected.png differ diff --git a/render-test/expected/render-tests/text-variable-anchor/rotated-offset/expected.png b/render-test/expected/render-tests/text-variable-anchor/rotated-offset/expected.png new file mode 100644 index 0000000000..13690d147c Binary files /dev/null and b/render-test/expected/render-tests/text-variable-anchor/rotated-offset/expected.png differ diff --git a/render-test/expected/symbol-visibility/visible/expected.png b/render-test/expected/symbol-visibility/visible/expected.png deleted file mode 100644 index 8da157772a..0000000000 Binary files a/render-test/expected/symbol-visibility/visible/expected.png and /dev/null differ diff --git a/render-test/expected/text-pitch-alignment/auto-text-rotation-alignment-map/expected.png b/render-test/expected/text-pitch-alignment/auto-text-rotation-alignment-map/expected.png deleted file mode 100644 index cd690ca152..0000000000 Binary files a/render-test/expected/text-pitch-alignment/auto-text-rotation-alignment-map/expected.png and /dev/null differ diff --git a/render-test/expected/text-pitch-alignment/map-text-rotation-alignment-map/expected.png b/render-test/expected/text-pitch-alignment/map-text-rotation-alignment-map/expected.png deleted file mode 100644 index cd690ca152..0000000000 Binary files a/render-test/expected/text-pitch-alignment/map-text-rotation-alignment-map/expected.png and /dev/null differ diff --git a/render-test/expected/text-pitch-alignment/viewport-text-rotation-alignment-map/expected.png b/render-test/expected/text-pitch-alignment/viewport-text-rotation-alignment-map/expected.png deleted file mode 100644 index 764d4a0b24..0000000000 Binary files a/render-test/expected/text-pitch-alignment/viewport-text-rotation-alignment-map/expected.png and /dev/null differ diff --git a/render-test/expected/text-variable-anchor/pitched-rotated-debug/expected.png b/render-test/expected/text-variable-anchor/pitched-rotated-debug/expected.png deleted file mode 100644 index 4e3d012844..0000000000 Binary files a/render-test/expected/text-variable-anchor/pitched-rotated-debug/expected.png and /dev/null differ diff --git a/render-test/expected/text-variable-anchor/rotated-offset/expected.png b/render-test/expected/text-variable-anchor/rotated-offset/expected.png deleted file mode 100644 index 13690d147c..0000000000 Binary files a/render-test/expected/text-variable-anchor/rotated-offset/expected.png and /dev/null differ diff --git a/render-test/main.cpp b/render-test/main.cpp index fcdbe3ab55..6f5e2449a9 100644 --- a/render-test/main.cpp +++ b/render-test/main.cpp @@ -81,7 +81,7 @@ int main(int argc, char** argv) { bool shouldIgnore = false; std::string ignoreReason; - const std::string ignoreName = "render-tests/" + id; + const std::string ignoreName = id; const auto it = std::find_if(ignores.cbegin(), ignores.cend(), [&ignoreName](auto pair) { return pair.first == ignoreName; }); if (it != ignores.end()) { shouldIgnore = true; diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp index d23a0fb296..d25b81c7ab 100644 --- a/render-test/metadata.hpp +++ b/render-test/metadata.hpp @@ -1,9 +1,11 @@ #pragma once +#include #include #include #include +#include #include "filesystem.hpp" @@ -50,6 +52,7 @@ struct TestMetadata { TestPaths paths; mbgl::JSDocument document; + bool renderTest = true; mbgl::Size size{ 512u, 512u }; float pixelRatio = 1.0f; @@ -61,6 +64,9 @@ struct TestMetadata { bool axonometric = false; double xSkew = 0.0; double ySkew = 1.0; + mbgl::ScreenCoordinate queryGeometry{0u, 0u}; + mbgl::ScreenBox queryGeometryBox{{0u, 0u}, {0u, 0u}}; + mbgl::RenderedQueryOptions queryOptions; // TODO uint32_t fadeDuration = 0; @@ -72,6 +78,7 @@ struct TestMetadata { std::string color; std::string actual; + std::string actualJson; std::string expected; std::string diff; diff --git a/render-test/parser.cpp b/render-test/parser.cpp index 9b462dee72..802c3c7f55 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -5,9 +5,14 @@ #include +#include #include #include +#include +#include +#include + #include #include #include @@ -179,8 +184,67 @@ TestPaths makeTestPaths(mbgl::filesystem::path stylePath) { }; } +void writeJSON(rapidjson::PrettyWriter& writer, const mbgl::Value& value) { + value.match([&writer](const mbgl::NullValue&) { writer.Null(); }, + [&writer](bool b) { writer.Bool(b); }, + [&writer](uint64_t u) { writer.Uint64(u); }, + [&writer](int64_t i) { writer.Int64(i); }, + [&writer](double d) { d == std::floor(d) ? writer.Int64(d) : writer.Double(d); }, + [&writer](const std::string& s) { writer.String(s); }, + [&writer](const std::vector& arr) { + writer.StartArray(); + for (const auto& item : arr) { + writeJSON(writer, item); + } + writer.EndArray(); + }, + [&writer](const std::unordered_map& obj) { + writer.StartObject(); + std::map sorted(obj.begin(), obj.end()); + for (const auto& entry : sorted) { + writer.Key(entry.first.c_str()); + writeJSON(writer, entry.second); + } + writer.EndObject(); + }); +} + } // namespace +std::string toJSON(const mbgl::Value& value, unsigned indent, bool singleLine) { + rapidjson::StringBuffer buffer; + rapidjson::PrettyWriter writer(buffer); + if (singleLine) { + writer.SetFormatOptions(rapidjson::kFormatSingleLineArray); + } + writer.SetIndent(' ', indent); + writeJSON(writer, value); + return buffer.GetString(); +} + +std::string toJSON(const std::vector& features, unsigned indent, bool singleLine) { + rapidjson::CrtAllocator allocator; + rapidjson::StringBuffer buffer; + rapidjson::PrettyWriter writer(buffer); + if (singleLine) { + writer.SetFormatOptions(rapidjson::kFormatSingleLineArray); + } + writer.SetIndent(' ', indent); + writer.StartArray(); + for (size_t i = 0; i < features.size(); ++i) { + auto result = mapbox::geojson::convert(features[i], allocator); + + result.AddMember("source", features[i].source, allocator); + if (!features[i].sourceLayer.empty()) { + result.AddMember("sourceLayer", features[i].sourceLayer, allocator); + } + result.AddMember("state", mapbox::geojson::to_value{allocator}(features[i].state), allocator); + result.Accept(writer); + } + writer.EndArray(); + return buffer.GetString(); +} + JSONReply readJson(const mbgl::filesystem::path& jsonPath) { auto maybeJSON = mbgl::util::readFile(jsonPath); if (!maybeJSON) { @@ -228,7 +292,7 @@ std::string serializeMetrics(const TestMetrics& metrics) { } std::vector readExpectedEntries(const mbgl::filesystem::path& base) { - static const std::regex regex(".*expected.*.png"); + static const std::regex regex(".*expected.*.png|.*expected.*.json"); std::vector expectedImages; for (const auto& entry : mbgl::filesystem::directory_iterator(base)) { @@ -476,6 +540,45 @@ TestMetadata parseTestMetadata(const TestPaths& paths) { metadata.ySkew = testValue["skew"][1].GetDouble(); } + if (testValue.HasMember("queryGeometry")) { + assert(testValue["queryGeometry"].IsArray()); + if (testValue["queryGeometry"][0].IsNumber() && testValue["queryGeometry"][1].IsNumber()) { + metadata.queryGeometry.x = testValue["queryGeometry"][0].GetDouble(); + metadata.queryGeometry.y = testValue["queryGeometry"][1].GetDouble(); + } else if (testValue["queryGeometry"][0].IsArray() && testValue["queryGeometry"][1].IsArray()) { + metadata.queryGeometryBox.min.x = testValue["queryGeometry"][0][0].GetDouble(); + metadata.queryGeometryBox.min.y = testValue["queryGeometry"][0][1].GetDouble(); + metadata.queryGeometryBox.max.x = testValue["queryGeometry"][1][0].GetDouble(); + metadata.queryGeometryBox.max.y = testValue["queryGeometry"][1][1].GetDouble(); + } + metadata.renderTest = false; + } + + if (testValue.HasMember("queryOptions")) { + assert(testValue["queryOptions"].IsObject()); + + if (testValue["queryOptions"].HasMember("layers")) { + assert(testValue["queryOptions"]["layers"].IsArray()); + auto layersArray = testValue["queryOptions"]["layers"].GetArray(); + std::vector layersVec; + for (uint32_t i = 0; i < layersArray.Size(); i++) { + layersVec.emplace_back(testValue["queryOptions"]["layers"][i].GetString()); + } + metadata.queryOptions.layerIDs = layersVec; + } + + using namespace mbgl::style; + using namespace mbgl::style::conversion; + if (testValue["queryOptions"].HasMember("filter")) { + assert(testValue["queryOptions"]["filter"].IsArray()); + auto& filterVal = testValue["queryOptions"]["filter"]; + Error error; + mbgl::optional converted = convert(filterVal, error); + assert(converted); + metadata.queryOptions.filter = std::move(*converted); + } + } + // TODO: fadeDuration // TODO: addFakeCanvas @@ -499,20 +602,31 @@ std::string createResultItem(const TestMetadata& metadata, bool hasFailedTests) html.append("
\n"); html.append(R"(

" + metadata.status + " " + metadata.id + "

\n"); if (metadata.status != "errored") { - html.append("\n"); - - html.append("\n"); + if (metadata.renderTest) { + html.append("\n"); + + html.append("\n"); + } else { + html.append("\n"); + } } else { assert(!metadata.errorMessage.empty()); html.append("

Error: " + metadata.errorMessage + "

\n"); } if (metadata.difference != 0.0) { - html.append("

Diff: " + mbgl::util::toString(metadata.difference) + "

\n"); + if (metadata.renderTest) { + html.append("

Diff: " + mbgl::util::toString(metadata.difference) + + "

\n"); + } else { + html.append("

Diff: " + metadata.diff + "

\n"); + } } html.append("
\n"); diff --git a/render-test/parser.hpp b/render-test/parser.hpp index 94fb212944..afa281ad30 100644 --- a/render-test/parser.hpp +++ b/render-test/parser.hpp @@ -31,5 +31,8 @@ std::string createResultPage(const TestStatistics&, const std::vector& features, unsigned indent, bool singleLine); + void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document); void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document); \ No newline at end of file diff --git a/render-test/runner.cpp b/render-test/runner.cpp index fa7a86e948..868e1a9a32 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -30,10 +30,11 @@ #include #include +using namespace mbgl; + // static const std::string& TestRunner::getBasePath() { - const static std::string result = - std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration/render-tests"); + const static std::string result = std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration"); return result; } @@ -46,7 +47,236 @@ const std::vector& TestRunner::getPlatformExpectationsPaths() { return result; } -bool TestRunner::checkResults(mbgl::PremultipliedImage&& actualImage, TestMetadata& metadata) { +// Strip precision for numbers, so that we can compare evaluated results with fixtures. +// Copied from JS expression harness. +Value stripPrecision(const Value& value) { + const double decimalSigFigs = 6; + if (auto num = numericValue(value)) { + if (*num == 0) { + return *num; + } + + const double multiplier = std::pow(10, std::max(0.0, decimalSigFigs - std::ceil(std::log10(std::fabs(*num))))); + + // We strip precision twice in a row here to avoid cases where + // stripping an already stripped number will modify its value + // due to bad floating point precision luck + // eg `Math.floor(8.16598 * 100000) / 100000` -> 8.16597 + const double firstStrip = std::floor(*num * multiplier) / multiplier; + return std::floor(firstStrip * multiplier) / multiplier; + } + + if (value.getArray()) { + std::vector stripped; + const auto& vec = *value.getArray(); + stripped.reserve(vec.size()); + for (const auto& val : vec) { + stripped.emplace_back(stripPrecision(val)); + } + return stripped; + } else if (value.getObject()) { + std::unordered_map stripped; + const auto& map = *value.getObject(); + for (const auto& pair : map) { + stripped.emplace(pair.first, stripPrecision(pair.second)); + } + return stripped; + } + + return value; +} + +bool deepEqual(const Value& a, const Value& b) { + const auto& anum = numericValue(a); + const auto& bnum = numericValue(b); + if (anum && bnum) { + return stripPrecision(*anum) == stripPrecision(*bnum); + } + + if (a.which() != b.which()) { + return false; + } + + if (a.getArray() && b.getArray()) { + const auto& avec = *a.getArray(); + const auto& bvec = *b.getArray(); + if (avec.size() != bvec.size()) { + return false; + } + for (std::size_t i = 0; i < avec.size(); ++i) { + if (!deepEqual(avec[i], bvec[i])) { + return false; + } + } + return true; + } + + if (a.getObject() && b.getObject()) { + const auto& amap = *a.getObject(); + const auto& bmap = *b.getObject(); + if (amap.size() != bmap.size()) { + return false; + } + for (const auto& pair : amap) { + auto it = bmap.find(pair.first); + if (it == bmap.end()) { + return false; + } + if (!deepEqual(pair.second, it->second)) { + return false; + } + } + return true; + } + + if (a == b) { + return true; + } + + if (a.getString() && b.getString()) { + const auto& strA = *a.getString(); + const auto& strB = *b.getString(); + if (strA == strB) { + return true; + } + + try { + double numA = std::stod(strA); + double numB = std::stod(strB); + return stripPrecision(numA) == stripPrecision(numB); + } catch (...) { + } + } + + return false; +} + +std::vector tokenize(std::string str) { + std::vector tokens; + std::regex re("\n"); + std::copy(std::regex_token_iterator(str.begin(), str.end(), re, -1), + std::regex_token_iterator(), + std::back_inserter(tokens)); + return tokens; +} + +std::string simpleDiff(const Value& result, const Value& expected) { + std::vector resultTokens{tokenize(toJSON(result, 2, false))}; + std::vector expectedTokens{tokenize(toJSON(expected, 2, false))}; + std::size_t maxLength = std::max(resultTokens.size(), expectedTokens.size()); + std::ostringstream diff; + + diff << "
" << std::endl;
+    const auto flush =
+        [](const std::vector& vec, std::size_t pos, std::ostringstream& out, std::string separator) {
+            for (std::size_t j = pos; j < vec.size(); ++j) {
+                out << separator << vec[j] << std::endl;
+            }
+        };
+
+    for (std::size_t i = 0; i < maxLength; ++i) {
+        if (resultTokens.size() <= i) {
+            flush(expectedTokens, i, diff, "-");
+            break;
+        }
+
+        if (expectedTokens.size() <= i) {
+            flush(resultTokens, i, diff, "+");
+            break;
+        }
+
+        if (!deepEqual(resultTokens[i], expectedTokens[i])) {
+            diff << ""
+                 << "-" << expectedTokens[i] << "" << std::endl;
+            diff << ""
+                 << "+" << resultTokens[i] << "" << std::endl;
+        } else {
+            diff << resultTokens[i] << std::endl;
+        }
+    }
+    diff << "
" << std::endl; + return diff.str(); +} + +bool TestRunner::checkQueryTestResults(mbgl::PremultipliedImage&& actualImage, + std::vector&& features, + TestMetadata& metadata) { + const std::string& base = metadata.paths.defaultExpectations(); + const std::vector& expectations = metadata.paths.expectations; + + metadata.actual = mbgl::encodePNG(actualImage); + + if (actualImage.size.isEmpty()) { + metadata.errorMessage = "Invalid size for actual image"; + return false; + } + + metadata.actualJson = toJSON(features, 2, false); + + if (metadata.actualJson.empty()) { + metadata.errorMessage = "Invalid size for actual JSON"; + return false; + } + +#if !TEST_READ_ONLY + if (getenv("UPDATE_PLATFORM")) { + mbgl::filesystem::create_directories(expectations.back()); + mbgl::util::write_file(expectations.back().string() + "/expected.json", metadata.actualJson); + return true; + } else if (getenv("UPDATE_DEFAULT")) { + mbgl::util::write_file(base + "/expected.json", metadata.actualJson); + return true; + } + + mbgl::util::write_file(base + "/actual.json", metadata.actualJson); +#endif + + std::vector expectedJsonPaths; + mbgl::filesystem::path expectedMetricsPath; + for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) { + if (mbgl::filesystem::exists(*rit)) { + expectedJsonPaths = readExpectedEntries(*rit); + if (!expectedJsonPaths.empty()) break; + } + } + + if (expectedJsonPaths.empty()) { + metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string(); + return false; + } + + for (const auto& entry : expectedJsonPaths) { + auto maybeExpectedJson = readJson(entry); + if (maybeExpectedJson.is()) { + auto& expected = maybeExpectedJson.get(); + + mbgl::JSDocument actual; + actual.Parse<0>(metadata.actualJson); + if (actual.HasParseError()) { + metadata.errorMessage = "Error parsing actual JSON for: " + metadata.paths.stylePath.string(); + return false; + } + + auto actualVal = mapbox::geojson::convert(actual); + auto expectedVal = mapbox::geojson::convert(expected); + bool equal = deepEqual(actualVal, expectedVal); + + metadata.difference = !equal; + if (equal) { + metadata.diff = "Match"; + } else { + metadata.diff = simpleDiff(actualVal, expectedVal); + } + } else { + metadata.errorMessage = "Failed to load expected JSON " + entry; + return false; + } + } + + return true; +} + +bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage, TestMetadata& metadata) { const std::string& base = metadata.paths.defaultExpectations(); const std::vector& expectations = metadata.paths.expectations; @@ -308,13 +538,13 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { assert(operationArray[1].IsNumber()); map.jumpTo(mbgl::CameraOptions().withBearing(operationArray[1].GetDouble())); - // setPitch + // setPitch } else if (operationArray[0].GetString() == setPitchOp) { assert(operationArray.Size() >= 2u); assert(operationArray[1].IsNumber()); map.jumpTo(mbgl::CameraOptions().withPitch(operationArray[1].GetDouble())); - // setFilter + // setFilter } else if (operationArray[0].GetString() == setFilterOp) { assert(operationArray.Size() >= 3u); assert(operationArray[1].IsString()); @@ -665,7 +895,19 @@ bool TestRunner::run(TestMetadata& metadata) { return false; } - return checkResults(std::move(image), metadata); + if (metadata.renderTest) { + return checkRenderTestResults(std::move(image), metadata); + } else { + std::vector features; + assert(metadata.document["metadata"]["test"]["queryGeometry"].IsArray()); + if (metadata.document["metadata"]["test"]["queryGeometry"][0].IsNumber() && + metadata.document["metadata"]["test"]["queryGeometry"][1].IsNumber()) { + features = frontend.getRenderer()->queryRenderedFeatures(metadata.queryGeometry, metadata.queryOptions); + } else { + features = frontend.getRenderer()->queryRenderedFeatures(metadata.queryGeometryBox, metadata.queryOptions); + } + return checkQueryTestResults(std::move(image), std::move(features), metadata); + } } void TestRunner::reset() { diff --git a/render-test/runner.hpp b/render-test/runner.hpp index fdea65e104..d8e5275f61 100644 --- a/render-test/runner.hpp +++ b/render-test/runner.hpp @@ -21,7 +21,10 @@ public: private: bool runOperations(const std::string& key, TestMetadata&); - bool checkResults(mbgl::PremultipliedImage&& image, TestMetadata&); + bool checkQueryTestResults(mbgl::PremultipliedImage&& actualImage, + std::vector&& features, + TestMetadata&); + bool checkRenderTestResults(mbgl::PremultipliedImage&& image, TestMetadata&); struct Impl { Impl(const TestMetadata&); -- cgit v1.2.1