[test runner] Test metrics reports. Check metrics results.

6 files changed, 128 insertions, 31 deletions

diff --git a/render-test/main.cpp b/render-test/main.cpp
index 46447f5853..c7b0b6be55 100644
--- a/render-test/main.cpp
+++ b/render-test/main.cpp
@@ -125,7 +125,7 @@ int main(int argc, char** argv) {
 
         bool errored = !metadata.errorMessage.empty();
         if (!errored) {
-            errored = runner.run(metadata) && !metadata.errorMessage.empty();
+            errored = !runner.run(metadata) || !metadata.errorMessage.empty();
         }
 
         bool passed = !errored && !metadata.diff.empty() && metadata.difference <= metadata.allowed;
diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp
index 308c9074f0..5e96301e4d 100644
--- a/render-test/metadata.hpp
+++ b/render-test/metadata.hpp
@@ -39,6 +39,12 @@ struct MemoryProbe {
     size_t allocations;
 };
 
+class TestMetrics {
+public:
+    bool isEmpty() const { return memory.empty(); }
+    std::map<std::string, MemoryProbe> memory;
+};
+
 struct TestMetadata {
     TestMetadata() = default;
 
@@ -72,5 +78,6 @@ struct TestMetadata {
     std::string errorMessage;
     double difference = 0.0;
 
-    std::map<std::string, MemoryProbe> memoryProbes; 
-};
\ No newline at end of file
+    TestMetrics metrics;
+    TestMetrics expectedMetrics;
+};
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index 49f084465c..581529a745 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -187,6 +187,29 @@ std::string serializeJsonValue(const mbgl::JSValue& value) {
     return buffer.GetString();
 }
 
+std::string serializeMetrics(const TestMetrics& metrics) {
+    rapidjson::StringBuffer s;
+    rapidjson::Writer<rapidjson::StringBuffer> writer(s);
+
+    writer.StartObject();
+    // Start memory section.
+    writer.Key("memory");
+    writer.StartArray();
+    for (const auto& memoryProbe : metrics.memory) {
+        assert(!memoryProbe.first.empty());       
+        writer.StartArray();
+        writer.String(memoryProbe.first.c_str());
+        writer.Uint64(memoryProbe.second.size);
+        writer.Uint64(memoryProbe.second.allocations);
+        writer.EndArray();
+    }
+    // End memory section.
+    writer.EndArray();
+    writer.EndObject();
+
+    return s.GetString();
+}
+
 std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base) {
     static const std::regex regex(".*expected.*.png");
 
@@ -290,6 +313,36 @@ std::vector<std::pair<std::string, std::string>> parseIgnores() {
     return ignores;
 }
 
+TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
+    TestMetrics result;
+
+    auto maybeJson = readJson(path.string());
+    if (!maybeJson.is<mbgl::JSDocument>()) { // NOLINT
+        return result;
+    }
+
+    const auto& document = maybeJson.get<mbgl::JSDocument>();
+    if (document.HasMember("memory")) {
+        const mbgl::JSValue& memoryValue = document["memory"];
+        assert(memoryValue.IsArray());
+        for (auto& probeValue : memoryValue.GetArray()) {
+            assert(probeValue.IsArray());
+            assert(probeValue.Size() >= 3u);
+            assert(probeValue[0].IsString());
+            assert(probeValue[1].IsNumber());
+            assert(probeValue[2].IsNumber());
+
+            const std::string mark { probeValue[0].GetString(), probeValue[0].GetStringLength() };
+            assert(!mark.empty());
+            result.memory.emplace(std::piecewise_construct,
+                                  std::forward_as_tuple(std::move(mark)), 
+                                  std::forward_as_tuple(probeValue[1].GetUint64(), probeValue[2].GetUint64()));
+        }
+    }
+
+    return result;
+}
+
 TestMetadata parseTestMetadata(const TestPaths& paths) {
     TestMetadata metadata;
     metadata.paths = paths;
diff --git a/render-test/parser.hpp b/render-test/parser.hpp
index 3e69968152..483089266e 100644
--- a/render-test/parser.hpp
+++ b/render-test/parser.hpp
@@ -1,5 +1,7 @@
 #pragma once
 
+#include "metadata.hpp"
+
 #include <mbgl/util/rapidjson.hpp>
 #include <mbgl/util/variant.hpp>
 
@@ -7,12 +9,6 @@
 #include <string>
 #include <vector>
 
-#include "filesystem.hpp"
-
-struct TestMetadata;
-struct TestStatistics;
-struct TestPaths;
-
 using ErrorMessage = std::string;
 using JSONReply = mbgl::variant<mbgl::JSDocument, ErrorMessage>;
 
@@ -20,9 +16,12 @@ using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector
 
 JSONReply readJson(const mbgl::filesystem::path&);
 std::string serializeJsonValue(const mbgl::JSValue&);
+std::string serializeMetrics(const TestMetrics&);
 
 std::vector<std::string> readExpectedEntries(const mbgl::filesystem::path& base);
 
+TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path);
+
 ArgumentsTuple parseArguments(int argc, char** argv);
 std::vector<std::pair<std::string, std::string>> parseIgnores();
 
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 5424eaa447..e325c82b1d 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -27,6 +27,7 @@
 #include <cassert>
 #include <regex>
 #include <utility>
+#include <sstream>
 
 // static 
 const std::string& TestRunner::getBasePath() {
@@ -44,13 +45,13 @@ const std::vector<std::string>& TestRunner::getPlatformExpectationsPaths() {
     return result;
 }
 
-bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& metadata) {
+bool TestRunner::checkResults(mbgl::PremultipliedImage&& actualImage, TestMetadata& metadata) {
     const std::string& base = metadata.paths.defaultExpectations();
     const std::vector<mbgl::filesystem::path>& expectations = metadata.paths.expectations;
 
-    metadata.actual = mbgl::encodePNG(actual);
+    metadata.actual = mbgl::encodePNG(actualImage);
 
-    if (actual.size.isEmpty()) {
+    if (actualImage.size.isEmpty()) {
         metadata.errorMessage = "Invalid size for actual image";
         return false;
     }
@@ -58,34 +59,46 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met
 #if !TEST_READ_ONLY
     if (getenv("UPDATE_PLATFORM")) {
         mbgl::filesystem::create_directories(expectations.back());
-        mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actual));
+        mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actualImage));
         return true;
     } else if (getenv("UPDATE_DEFAULT")) {
-        mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actual));
+        mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actualImage));
         return true;
+    } else if (getenv("UPDATE_METRICS")) {
+        if (!metadata.metrics.isEmpty()) {
+            mbgl::filesystem::create_directories(expectations.back());
+            mbgl::util::write_file(expectations.back().string() + "/metrics.json", serializeMetrics(metadata.metrics));
+            return true;
+        }
     }
 
     mbgl::util::write_file(base + "/actual.png", metadata.actual);
 #endif
 
-    mbgl::PremultipliedImage expected { actual.size };
-    mbgl::PremultipliedImage diff { actual.size };
+    mbgl::PremultipliedImage expectedImage { actualImage.size };
+    mbgl::PremultipliedImage imageDiff { actualImage.size };
 
     double pixels = 0.0;
-    mbgl::filesystem::path expectedPath;
+    std::vector<std::string> expectedImagesPaths;
+    mbgl::filesystem::path expectedMetricsPath;
     for (auto rit = expectations.rbegin(); rit!= expectations.rend(); ++rit) {
         if (mbgl::filesystem::exists(*rit)) {
-            expectedPath = *rit;
-            break;
+            if (metadata.expectedMetrics.isEmpty()) {
+                mbgl::filesystem::path maybeExpectedMetricsPath{ *rit };
+                maybeExpectedMetricsPath.replace_filename("metrics.json");
+                metadata.expectedMetrics = readExpectedMetrics(maybeExpectedMetricsPath);
+            }
+            expectedImagesPaths = readExpectedEntries(*rit);
+            if (!expectedImagesPaths.empty()) break;
         }
     }
 
-    if (expectedPath.empty()) {
+    if (expectedImagesPaths.empty()) {
         metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string();
         return false;
     }
     
-    for (const auto& entry: readExpectedEntries(expectedPath)) {
+    for (const auto& entry: expectedImagesPaths) {
         mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry);
         if (!maybeExpectedImage) {
             metadata.errorMessage = "Failed to load expected image " + entry;
@@ -94,23 +107,48 @@ bool TestRunner::checkImage(mbgl::PremultipliedImage&& actual, TestMetadata& met
 
         metadata.expected = *maybeExpectedImage;
 
-        expected = mbgl::decodeImage(*maybeExpectedImage);
+        expectedImage = mbgl::decodeImage(*maybeExpectedImage);
 
         pixels = // implicitly converting from uint64_t
-            mapbox::pixelmatch(actual.data.get(), expected.data.get(), expected.size.width,
-                               expected.size.height, diff.data.get(), 0.1285); // Defined in GL JS
+            mapbox::pixelmatch(actualImage.data.get(), expectedImage.data.get(), expectedImage.size.width,
+                               expectedImage.size.height, imageDiff.data.get(), 0.1285); // Defined in GL JS
 
-        metadata.diff = mbgl::encodePNG(diff);
+        metadata.diff = mbgl::encodePNG(imageDiff);
 
 #if !TEST_READ_ONLY
         mbgl::util::write_file(base + "/diff.png", metadata.diff);
 #endif
 
-        metadata.difference = pixels / expected.size.area();
+        metadata.difference = pixels / expectedImage.size.area();
         if (metadata.difference <= metadata.allowed) {
             break;
         }
     }
+    // Check memory metrics.
+    for (const auto& expected : metadata.expectedMetrics.memory) {
+        auto actual = metadata.metrics.memory.find(expected.first);
+        if (actual == metadata.metrics.memory.end()) {
+            metadata.errorMessage = "Failed to find memory probe: " + expected.first;
+            return false;
+        }
+        if (actual->second.size > expected.second.size) {
+            std::stringstream ss;
+            ss << "Allocated size at memory probe \" " << expected.first << "\" is "
+               << actual->second.size << " bytes, expected is " << expected.second.size << " bytes.";
+
+            metadata.errorMessage = ss.str();
+            return false;
+        }
+
+        if (actual->second.allocations > expected.second.allocations) {
+            std::stringstream ss;
+            ss << "Number of allocations at memory probe \" " << expected.first << "\" is "
+               << actual->second.allocations << ", expected is " << expected.second.allocations << ".";
+
+            metadata.errorMessage = ss.str();
+            return false;
+        }
+    }
 
     return true;
 }
@@ -407,9 +445,9 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         assert(operationArray[1].IsString());
         std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
 
-        metadata.memoryProbes.emplace(std::piecewise_construct,
-                                      std::forward_as_tuple(std::move(mark)), 
-                                      std::forward_as_tuple(AllocationIndex::getAllocatedSize(), AllocationIndex::getAllocationsCount()));
+        metadata.metrics.memory.emplace(std::piecewise_construct,
+                                        std::forward_as_tuple(std::move(mark)), 
+                                        std::forward_as_tuple(AllocationIndex::getAllocatedSize(), AllocationIndex::getAllocationsCount()));
     // probeMemoryEnd
     } else if (operationArray[0].GetString() == memoryProbeEndOp) {
         assert(AllocationIndex::isActive());
@@ -469,7 +507,7 @@ bool TestRunner::run(TestMetadata& metadata) {
         return false;
     }
 
-    return checkImage(std::move(image), metadata);
+    return checkResults(std::move(image), metadata);
 }
 
 void TestRunner::reset() {
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
index 74cc03ba03..fdea65e104 100644
--- a/render-test/runner.hpp
+++ b/render-test/runner.hpp
@@ -21,7 +21,7 @@ public:
 
 private:
     bool runOperations(const std::string& key, TestMetadata&);
-    bool checkImage(mbgl::PremultipliedImage&& image, TestMetadata&);
+    bool checkResults(mbgl::PremultipliedImage&& image, TestMetadata&);
 
     struct Impl {
         Impl(const TestMetadata&);