From 1b020eeb1195a7afcac35c263220cd3568a4bc69 Mon Sep 17 00:00:00 2001 From: Mikhail Pozdnyakov Date: Mon, 26 Aug 2019 18:30:22 +0300 Subject: [test runner] Test metrics reports. Check metrics results. --- render-test/main.cpp | 2 +- render-test/metadata.hpp | 11 +++++-- render-test/parser.cpp | 53 ++++++++++++++++++++++++++++++++ render-test/parser.hpp | 11 +++---- render-test/runner.cpp | 80 +++++++++++++++++++++++++++++++++++------------- render-test/runner.hpp | 2 +- 6 files changed, 128 insertions(+), 31 deletions(-) diff --git a/render-test/main.cpp b/render-test/main.cpp index 46447f5853..c7b0b6be55 100644 --- a/render-test/main.cpp +++ b/render-test/main.cpp @@ -125,7 +125,7 @@ int main(int argc, char** argv) { bool errored = !metadata.errorMessage.empty(); if (!errored) { - errored = runner.run(metadata) && !metadata.errorMessage.empty(); + errored = !runner.run(metadata) || !metadata.errorMessage.empty(); } bool passed = !errored && !metadata.diff.empty() && metadata.difference <= metadata.allowed; diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp index 308c9074f0..5e96301e4d 100644 --- a/render-test/metadata.hpp +++ b/render-test/metadata.hpp @@ -39,6 +39,12 @@ struct MemoryProbe { size_t allocations; }; +class TestMetrics { +public: + bool isEmpty() const { return memory.empty(); } + std::map memory; +}; + struct TestMetadata { TestMetadata() = default; @@ -72,5 +78,6 @@ struct TestMetadata { std::string errorMessage; double difference = 0.0; - std::map memoryProbes; -}; \ No newline at end of file + TestMetrics metrics; + TestMetrics expectedMetrics; +}; diff --git a/render-test/parser.cpp b/render-test/parser.cpp index 49f084465c..581529a745 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -187,6 +187,29 @@ std::string serializeJsonValue(const mbgl::JSValue& value) { return buffer.GetString(); } +std::string serializeMetrics(const TestMetrics& metrics) { + rapidjson::StringBuffer s; + rapidjson::Writer writer(s); + + writer.StartObject(); + // Start memory section. + writer.Key("memory"); + writer.StartArray(); + for (const auto& memoryProbe : metrics.memory) { + assert(!memoryProbe.first.empty()); + writer.StartArray(); + writer.String(memoryProbe.first.c_str()); + writer.Uint64(memoryProbe.second.size); + writer.Uint64(memoryProbe.second.allocations); + writer.EndArray(); + } + // End memory section. + writer.EndArray(); + writer.EndObject(); + + return s.GetString(); +} + std::vector readExpectedEntries(const mbgl::filesystem::path& base) { static const std::regex regex(".*expected.*.png"); @@ -290,6 +313,36 @@ std::vector> parseIgnores() { return ignores; } +TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) { + TestMetrics result; + + auto maybeJson = readJson(path.string()); + if (!maybeJson.is()) { // NOLINT + return result; + } + + const auto& document = maybeJson.get(); + if (document.HasMember("memory")) { + const mbgl::JSValue& memoryValue = document["memory"]; + assert(memoryValue.IsArray()); + for (auto& probeValue : memoryValue.GetArray()) { + assert(probeValue.IsArray()); + assert(probeValue.Size() >= 3u); + assert(probeValue[0].IsString()); + assert(probeValue[1].IsNumber()); + assert(probeValue[2].IsNumber()); + + const std::string mark { probeValue[0].GetString(), probeValue[0].GetStringLength() }; + assert(!mark.empty()); + result.memory.emplace(std::piecewise_construct, + std::forward_as_tuple(std::move(mark)), + std::forward_as_tuple(probeValue[1].GetUint64(), probeValue[2].GetUint64())); + } + } + + return result; +} + TestMetadata parseTestMetadata(const TestPaths& paths) { TestMetadata metadata; metadata.paths = paths; diff --git a/render-test/parser.hpp b/render-test/parser.hpp index 3e69968152..483089266e 100644 --- a/render-test/parser.hpp +++ b/render-test/parser.hpp @@ -1,5 +1,7 @@ #pragma once +#include "metadata.hpp" + #include #include @@ -7,12 +9,6 @@ #include #include -#include "filesystem.hpp" - -struct TestMetadata; -struct TestStatistics; -struct TestPaths; - using ErrorMessage = std::string; using JSONReply = mbgl::variant; @@ -20,9 +16,12 @@ using ArgumentsTuple = std::tuple readExpectedEntries(const mbgl::filesystem::path& base); +TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path); + ArgumentsTuple parseArguments(int argc, char** argv); std::vector> parseIgnores(); diff --git a/render-test/runner.cpp b/render-test/runner.cpp index 5424eaa447..e325c82b1d 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -27,6 +27,7 @@ #include #include #include +#include // static const std::string& TestRunner::getBasePath() { @@ -44,13 +45,13 @@ const std::vector& TestRunner::getPlatformExpectationsPaths() { return result; } -bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& metadata) { +bool TestRunner::checkResults(mbgl::PremultipliedImage&& actualImage, TestMetadata& metadata) { const std::string& base = metadata.paths.defaultExpectations(); const std::vector& expectations = metadata.paths.expectations; - metadata.actual = mbgl::encodePNG(actual); + metadata.actual = mbgl::encodePNG(actualImage); - if (actual.size.isEmpty()) { + if (actualImage.size.isEmpty()) { metadata.errorMessage = "Invalid size for actual image"; return false; } @@ -58,34 +59,46 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met #if !TEST_READ_ONLY if (getenv("UPDATE_PLATFORM")) { mbgl::filesystem::create_directories(expectations.back()); - mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actual)); + mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actualImage)); return true; } else if (getenv("UPDATE_DEFAULT")) { - mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actual)); + mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actualImage)); return true; + } else if (getenv("UPDATE_METRICS")) { + if (!metadata.metrics.isEmpty()) { + mbgl::filesystem::create_directories(expectations.back()); + mbgl::util::write_file(expectations.back().string() + "/metrics.json", serializeMetrics(metadata.metrics)); + return true; + } } mbgl::util::write_file(base + "/actual.png", metadata.actual); #endif - mbgl::PremultipliedImage expected { actual.size }; - mbgl::PremultipliedImage diff { actual.size }; + mbgl::PremultipliedImage expectedImage { actualImage.size }; + mbgl::PremultipliedImage imageDiff { actualImage.size }; double pixels = 0.0; - mbgl::filesystem::path expectedPath; + std::vector expectedImagesPaths; + mbgl::filesystem::path expectedMetricsPath; for (auto rit = expectations.rbegin(); rit!= expectations.rend(); ++rit) { if (mbgl::filesystem::exists(*rit)) { - expectedPath = *rit; - break; + if (metadata.expectedMetrics.isEmpty()) { + mbgl::filesystem::path maybeExpectedMetricsPath{ *rit }; + maybeExpectedMetricsPath.replace_filename("metrics.json"); + metadata.expectedMetrics = readExpectedMetrics(maybeExpectedMetricsPath); + } + expectedImagesPaths = readExpectedEntries(*rit); + if (!expectedImagesPaths.empty()) break; } } - if (expectedPath.empty()) { + if (expectedImagesPaths.empty()) { metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string(); return false; } - for (const auto& entry: readExpectedEntries(expectedPath)) { + for (const auto& entry: expectedImagesPaths) { mbgl::optional maybeExpectedImage = mbgl::util::readFile(entry); if (!maybeExpectedImage) { metadata.errorMessage = "Failed to load expected image " + entry; @@ -94,23 +107,48 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met metadata.expected = *maybeExpectedImage; - expected = mbgl::decodeImage(*maybeExpectedImage); + expectedImage = mbgl::decodeImage(*maybeExpectedImage); pixels = // implicitly converting from uint64_t - mapbox::pixelmatch(actual.data.get(), expected.data.get(), expected.size.width, - expected.size.height, diff.data.get(), 0.1285); // Defined in GL JS + mapbox::pixelmatch(actualImage.data.get(), expectedImage.data.get(), expectedImage.size.width, + expectedImage.size.height, imageDiff.data.get(), 0.1285); // Defined in GL JS - metadata.diff = mbgl::encodePNG(diff); + metadata.diff = mbgl::encodePNG(imageDiff); #if !TEST_READ_ONLY mbgl::util::write_file(base + "/diff.png", metadata.diff); #endif - metadata.difference = pixels / expected.size.area(); + metadata.difference = pixels / expectedImage.size.area(); if (metadata.difference <= metadata.allowed) { break; } } + // Check memory metrics. + for (const auto& expected : metadata.expectedMetrics.memory) { + auto actual = metadata.metrics.memory.find(expected.first); + if (actual == metadata.metrics.memory.end()) { + metadata.errorMessage = "Failed to find memory probe: " + expected.first; + return false; + } + if (actual->second.size > expected.second.size) { + std::stringstream ss; + ss << "Allocated size at memory probe \" " << expected.first << "\" is " + << actual->second.size << " bytes, expected is " << expected.second.size << " bytes."; + + metadata.errorMessage = ss.str(); + return false; + } + + if (actual->second.allocations > expected.second.allocations) { + std::stringstream ss; + ss << "Number of allocations at memory probe \" " << expected.first << "\" is " + << actual->second.allocations << ", expected is " << expected.second.allocations << "."; + + metadata.errorMessage = ss.str(); + return false; + } + } return true; } @@ -407,9 +445,9 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { assert(operationArray[1].IsString()); std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); - metadata.memoryProbes.emplace(std::piecewise_construct, - std::forward_as_tuple(std::move(mark)), - std::forward_as_tuple(AllocationIndex::getAllocatedSize(), AllocationIndex::getAllocationsCount())); + metadata.metrics.memory.emplace(std::piecewise_construct, + std::forward_as_tuple(std::move(mark)), + std::forward_as_tuple(AllocationIndex::getAllocatedSize(), AllocationIndex::getAllocationsCount())); // probeMemoryEnd } else if (operationArray[0].GetString() == memoryProbeEndOp) { assert(AllocationIndex::isActive()); @@ -469,7 +507,7 @@ bool TestRunner::run(TestMetadata& metadata) { return false; } - return checkImage(std::move(image), metadata); + return checkResults(std::move(image), metadata); } void TestRunner::reset() { diff --git a/render-test/runner.hpp b/render-test/runner.hpp index 74cc03ba03..fdea65e104 100644 --- a/render-test/runner.hpp +++ b/render-test/runner.hpp @@ -21,7 +21,7 @@ public: private: bool runOperations(const std::string& key, TestMetadata&); - bool checkImage(mbgl::PremultipliedImage&& image, TestMetadata&); + bool checkResults(mbgl::PremultipliedImage&& image, TestMetadata&); struct Impl { Impl(const TestMetadata&); -- cgit v1.2.1