[test runner] Threshold for memory metrics

3 files changed, 30 insertions, 13 deletions

diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp
index c26fde1d19..20cc440fd4 100644
--- a/render-test/metadata.hpp
+++ b/render-test/metadata.hpp
@@ -49,12 +49,19 @@ struct FileSizeProbe {
 
 struct MemoryProbe {
     MemoryProbe() = default;
-    MemoryProbe(size_t peak_, size_t allocations_)
-        : peak(peak_)
-        , allocations(allocations_) {}
+    MemoryProbe(size_t peak_, size_t allocations_) : peak(peak_), allocations(allocations_), tolerance(0.0f) {}
 
     size_t peak;
     size_t allocations;
+    float tolerance;
+
+    static std::tuple<bool, float> checkPeak(const MemoryProbe& expected, const MemoryProbe& actual) {
+        return checkValue(expected.peak, actual.peak, actual.tolerance);
+    }
+
+    static std::tuple<bool, float> checkAllocations(const MemoryProbe& expected, const MemoryProbe& actual) {
+        return checkValue(expected.allocations, actual.allocations, actual.tolerance);
+    }
 };
 
 struct NetworkProbe {
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index 41c9b83298..7300530b07 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -507,7 +507,7 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
             assert(probeValue[1].IsNumber());
             assert(probeValue[2].IsNumber());
 
-            const std::string mark { probeValue[0].GetString(), probeValue[0].GetStringLength() };
+            std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()};
             assert(!mark.empty());
             result.memory.emplace(std::piecewise_construct,
                                   std::forward_as_tuple(std::move(mark)), 
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 810806d231..925599dfb8 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -276,19 +276,23 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage,
             metadata.errorMessage = "Failed to find memory probe: " + expected.first;
             return false;
         }
-        if (actual->second.peak > expected.second.peak) {
+        bool passed{false};
+        float delta{0.0f};
+        std::tie(passed, delta) = MemoryProbe::checkPeak(expected.second, actual->second);
+        if (!passed) {
             std::stringstream ss;
-            ss << "Allocated memory peak size at probe \"" << expected.first << "\" is "
-               << actual->second.peak << " bytes, expected is " << expected.second.peak << " bytes.";
+            ss << "Allocated memory peak size at probe \"" << expected.first << "\" is " << actual->second.peak
+               << " bytes, expected is " << expected.second.peak << "±" << delta << " bytes.";
 
             metadata.errorMessage = ss.str();
             return false;
         }
 
-        if (actual->second.allocations > expected.second.allocations) {
+        std::tie(passed, delta) = MemoryProbe::checkAllocations(expected.second, actual->second);
+        if (!passed) {
             std::stringstream ss;
-            ss << "Number of allocations at probe \"" << expected.first << "\" is "
-               << actual->second.allocations << ", expected is " << expected.second.allocations << ".";
+            ss << "Number of allocations at probe \"" << expected.first << "\" is " << actual->second.allocations
+               << ", expected is " << expected.second.allocations << "±" << delta << " allocations.";
 
             metadata.errorMessage = ss.str();
             return false;
@@ -644,9 +648,15 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         assert(operationArray[1].IsString());
         std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
 
-        metadata.metrics.memory.emplace(std::piecewise_construct,
-                                        std::forward_as_tuple(std::move(mark)), 
-                                        std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(), AllocationIndex::getAllocationsCount()));
+        auto emplaced = metadata.metrics.memory.emplace(
+            std::piecewise_construct,
+            std::forward_as_tuple(std::move(mark)),
+            std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(), AllocationIndex::getAllocationsCount()));
+        assert(emplaced.second);
+        if (operationArray.Size() >= 3u) {
+            assert(operationArray[2].IsNumber());
+            emplaced.first->second.tolerance = float(operationArray[2].GetDouble());
+        }
     } else if (operationArray[0].GetString() == memoryProbeEndOp) {
         // probeMemoryEnd
         assert(AllocationIndex::isActive());