summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--render-test/main.cpp2
-rw-r--r--render-test/metadata.hpp11
-rw-r--r--render-test/parser.cpp53
-rw-r--r--render-test/parser.hpp11
-rw-r--r--render-test/runner.cpp80
-rw-r--r--render-test/runner.hpp2
6 files changed, 128 insertions, 31 deletions
diff --git a/render-test/main.cpp b/render-test/main.cpp
index 46447f5853..c7b0b6be55 100644
--- a/render-test/main.cpp
+++ b/render-test/main.cpp
@@ -125,7 +125,7 @@ int main(int argc, char** argv) {
bool errored = !metadata.errorMessage.empty();
if (!errored) {
- errored = runner.run(metadata) && !metadata.errorMessage.empty();
+ errored = !runner.run(metadata) || !metadata.errorMessage.empty();
}
bool passed = !errored && !metadata.diff.empty() && metadata.difference <= metadata.allowed;
diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp
index 308c9074f0..5e96301e4d 100644
--- a/render-test/metadata.hpp
+++ b/render-test/metadata.hpp
@@ -39,6 +39,12 @@ struct MemoryProbe {
size_t allocations;
};
+class TestMetrics {
+public:
+ bool isEmpty() const { return memory.empty(); }
+ std::map<std::string, MemoryProbe> memory;
+};
+
struct TestMetadata {
TestMetadata() = default;
@@ -72,5 +78,6 @@ struct TestMetadata {
std::string errorMessage;
double difference = 0.0;
- std::map<std::string, MemoryProbe> memoryProbes;
-}; \ No newline at end of file
+ TestMetrics metrics;
+ TestMetrics expectedMetrics;
+};
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index 49f084465c..581529a745 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -187,6 +187,29 @@ std::string serializeJsonValue(const mbgl::JSValue& value) {
return buffer.GetString();
}
+std::string serializeMetrics(const TestMetrics& metrics) {
+ rapidjson::StringBuffer s;
+ rapidjson::Writer<rapidjson::StringBuffer> writer(s);
+
+ writer.StartObject();
+ // Start memory section.
+ writer.Key("memory");
+ writer.StartArray();
+ for (const auto& memoryProbe : metrics.memory) {
+ assert(!memoryProbe.first.empty());
+ writer.StartArray();
+ writer.String(memoryProbe.first.c_str());
+ writer.Uint64(memoryProbe.second.size);
+ writer.Uint64(memoryProbe.second.allocations);
+ writer.EndArray();
+ }
+ // End memory section.
+ writer.EndArray();
+ writer.EndObject();
+
+ return s.GetString();
+}
+
std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base) {
static const std::regex regex(".*expected.*.png");
@@ -290,6 +313,36 @@ std::vector<std::pair<std::string, std::string>> parseIgnores() {
return ignores;
}
+TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
+ TestMetrics result;
+
+ auto maybeJson = readJson(path.string());
+ if (!maybeJson.is<mbgl::JSDocument>()) { // NOLINT
+ return result;
+ }
+
+ const auto& document = maybeJson.get<mbgl::JSDocument>();
+ if (document.HasMember("memory")) {
+ const mbgl::JSValue& memoryValue = document["memory"];
+ assert(memoryValue.IsArray());
+ for (auto& probeValue : memoryValue.GetArray()) {
+ assert(probeValue.IsArray());
+ assert(probeValue.Size() >= 3u);
+ assert(probeValue[0].IsString());
+ assert(probeValue[1].IsNumber());
+ assert(probeValue[2].IsNumber());
+
+ const std::string mark { probeValue[0].GetString(), probeValue[0].GetStringLength() };
+ assert(!mark.empty());
+ result.memory.emplace(std::piecewise_construct,
+ std::forward_as_tuple(std::move(mark)),
+ std::forward_as_tuple(probeValue[1].GetUint64(), probeValue[2].GetUint64()));
+ }
+ }
+
+ return result;
+}
+
TestMetadata parseTestMetadata(const TestPaths& paths) {
TestMetadata metadata;
metadata.paths = paths;
diff --git a/render-test/parser.hpp b/render-test/parser.hpp
index 3e69968152..483089266e 100644
--- a/render-test/parser.hpp
+++ b/render-test/parser.hpp
@@ -1,5 +1,7 @@
#pragma once
+#include "metadata.hpp"
+
#include <mbgl/util/rapidjson.hpp>
#include <mbgl/util/variant.hpp>
@@ -7,12 +9,6 @@
#include <string>
#include <vector>
-#include "filesystem.hpp"
-
-struct TestMetadata;
-struct TestStatistics;
-struct TestPaths;
-
using ErrorMessage = std::string;
using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>;
@@ -20,9 +16,12 @@ using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector
JSONReply readJson(const mbgl::filesystem::path&);
std::string serializeJsonValue(const mbgl::JSValue&);
+std::string serializeMetrics(const TestMetrics&);
std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base);
+TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path);
+
ArgumentsTuple parseArguments(int argc, char** argv);
std::vector<std::pair<std::string, std::string>> parseIgnores();
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 5424eaa447..e325c82b1d 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -27,6 +27,7 @@
#include <cassert>
#include <regex>
#include <utility>
+#include <sstream>
// static
const std::string& TestRunner::getBasePath() {
@@ -44,13 +45,13 @@ const std::vector<std::string>& TestRunner::getPlatformExpectationsPaths() {
return result;
}
-bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& metadata) {
+bool TestRunner::checkResults(mbgl::PremultipliedImage&& actualImage, TestMetadata& metadata) {
const std::string& base = metadata.paths.defaultExpectations();
const std::vector<mbgl::filesystem::path>& expectations = metadata.paths.expectations;
- metadata.actual = mbgl::encodePNG(actual);
+ metadata.actual = mbgl::encodePNG(actualImage);
- if (actual.size.isEmpty()) {
+ if (actualImage.size.isEmpty()) {
metadata.errorMessage = "Invalid size for actual image";
return false;
}
@@ -58,34 +59,46 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met
#if !TEST_READ_ONLY
if (getenv("UPDATE_PLATFORM")) {
mbgl::filesystem::create_directories(expectations.back());
- mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actual));
+ mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actualImage));
return true;
} else if (getenv("UPDATE_DEFAULT")) {
- mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actual));
+ mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actualImage));
return true;
+ } else if (getenv("UPDATE_METRICS")) {
+ if (!metadata.metrics.isEmpty()) {
+ mbgl::filesystem::create_directories(expectations.back());
+ mbgl::util::write_file(expectations.back().string() + "/metrics.json", serializeMetrics(metadata.metrics));
+ return true;
+ }
}
mbgl::util::write_file(base + "/actual.png", metadata.actual);
#endif
- mbgl::PremultipliedImage expected { actual.size };
- mbgl::PremultipliedImage diff { actual.size };
+ mbgl::PremultipliedImage expectedImage { actualImage.size };
+ mbgl::PremultipliedImage imageDiff { actualImage.size };
double pixels = 0.0;
- mbgl::filesystem::path expectedPath;
+ std::vector<std::string> expectedImagesPaths;
+ mbgl::filesystem::path expectedMetricsPath;
for (auto rit = expectations.rbegin(); rit!= expectations.rend(); ++rit) {
if (mbgl::filesystem::exists(*rit)) {
- expectedPath = *rit;
- break;
+ if (metadata.expectedMetrics.isEmpty()) {
+ mbgl::filesystem::path maybeExpectedMetricsPath{ *rit };
+ maybeExpectedMetricsPath.replace_filename("metrics.json");
+ metadata.expectedMetrics = readExpectedMetrics(maybeExpectedMetricsPath);
+ }
+ expectedImagesPaths = readExpectedEntries(*rit);
+ if (!expectedImagesPaths.empty()) break;
}
}
- if (expectedPath.empty()) {
+ if (expectedImagesPaths.empty()) {
metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string();
return false;
}
- for (const auto& entry: readExpectedEntries(expectedPath)) {
+ for (const auto& entry: expectedImagesPaths) {
mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
if (!maybeExpectedImage) {
metadata.errorMessage = "Failed to load expected image " + entry;
@@ -94,23 +107,48 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met
metadata.expected = *maybeExpectedImage;
- expected = mbgl::decodeImage(*maybeExpectedImage);
+ expectedImage = mbgl::decodeImage(*maybeExpectedImage);
pixels = // implicitly converting from uint64_t
- mapbox::pixelmatch(actual.data.get(), expected.data.get(), expected.size.width,
- expected.size.height, diff.data.get(), 0.1285); // Defined in GL JS
+ mapbox::pixelmatch(actualImage.data.get(), expectedImage.data.get(), expectedImage.size.width,
+ expectedImage.size.height, imageDiff.data.get(), 0.1285); // Defined in GL JS
- metadata.diff = mbgl::encodePNG(diff);
+ metadata.diff = mbgl::encodePNG(imageDiff);
#if !TEST_READ_ONLY
mbgl::util::write_file(base + "/diff.png", metadata.diff);
#endif
- metadata.difference = pixels / expected.size.area();
+ metadata.difference = pixels / expectedImage.size.area();
if (metadata.difference <= metadata.allowed) {
break;
}
}
+ // Check memory metrics.
+ for (const auto& expected : metadata.expectedMetrics.memory) {
+ auto actual = metadata.metrics.memory.find(expected.first);
+ if (actual == metadata.metrics.memory.end()) {
+ metadata.errorMessage = "Failed to find memory probe: " + expected.first;
+ return false;
+ }
+ if (actual->second.size > expected.second.size) {
+ std::stringstream ss;
+ ss << "Allocated size at memory probe \" " << expected.first << "\" is "
+ << actual->second.size << " bytes, expected is " << expected.second.size << " bytes.";
+
+ metadata.errorMessage = ss.str();
+ return false;
+ }
+
+ if (actual->second.allocations > expected.second.allocations) {
+ std::stringstream ss;
+ ss << "Number of allocations at memory probe \" " << expected.first << "\" is "
+ << actual->second.allocations << ", expected is " << expected.second.allocations << ".";
+
+ metadata.errorMessage = ss.str();
+ return false;
+ }
+ }
return true;
}
@@ -407,9 +445,9 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
assert(operationArray[1].IsString());
std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
- metadata.memoryProbes.emplace(std::piecewise_construct,
- std::forward_as_tuple(std::move(mark)),
- std::forward_as_tuple(AllocationIndex::getAllocatedSize(), AllocationIndex::getAllocationsCount()));
+ metadata.metrics.memory.emplace(std::piecewise_construct,
+ std::forward_as_tuple(std::move(mark)),
+ std::forward_as_tuple(AllocationIndex::getAllocatedSize(), AllocationIndex::getAllocationsCount()));
// probeMemoryEnd
} else if (operationArray[0].GetString() == memoryProbeEndOp) {
assert(AllocationIndex::isActive());
@@ -469,7 +507,7 @@ bool TestRunner::run(TestMetadata& metadata) {
return false;
}
- return checkImage(std::move(image), metadata);
+ return checkResults(std::move(image), metadata);
}
void TestRunner::reset() {
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
index 74cc03ba03..fdea65e104 100644
--- a/render-test/runner.hpp
+++ b/render-test/runner.hpp
@@ -21,7 +21,7 @@ public:
private:
bool runOperations(const std::string& key, TestMetadata&);
- bool checkImage(mbgl::PremultipliedImage&& image, TestMetadata&);
+ bool checkResults(mbgl::PremultipliedImage&& image, TestMetadata&);
struct Impl {
Impl(const TestMetadata&);
View Lorry Controller Status