[render-test] Add tolerance to file-size tests

9 files changed, 27 insertions, 16 deletions

diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp
index e881bce208..bd26e6a7ba 100644
--- a/render-test/metadata.hpp
+++ b/render-test/metadata.hpp
@@ -31,12 +31,20 @@ struct TestPaths {
     }
 };
 
+inline std::tuple<bool, float> checkValue(float expected, float actual, float tolerance) {
+    float delta = expected * tolerance;
+    assert(delta >= 0.0f);
+    return std::make_tuple(std::abs(expected - actual) <= delta, delta);
+}
+
 struct FileSizeProbe {
     FileSizeProbe() = default;
-    FileSizeProbe(std::string path_, uintmax_t size_) : path(std::move(path_)), size(size_) {}
+    FileSizeProbe(std::string path_, uintmax_t size_, float tolerance_)
+        : path(std::move(path_)), size(size_), tolerance(tolerance_) {}
 
     std::string path;
     uintmax_t size;
+    float tolerance;
 };
 
 struct MemoryProbe {
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index cbbc43293a..f4e54493eb 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -478,7 +478,7 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
 
             result.fileSize.emplace(std::piecewise_construct,
                                     std::forward_as_tuple(std::move(mark)),
-                                    std::forward_as_tuple(std::move(filePath), probeValue[2].GetUint64()));
+                                    std::forward_as_tuple(std::move(filePath), probeValue[2].GetUint64(), 0.f));
         }
     }
 
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 103a14a079..3594c9488b 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -258,7 +258,8 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage,
             return false;
         }
 
-        if (actual->second.size != expected.second.size) {
+        auto result = checkValue(expected.second.size, actual->second.size, actual->second.tolerance);
+        if (!std::get<bool>(result)) {
             std::stringstream ss;
             ss << "File size does not match at probe \"" << expected.first << "\": " << actual->second.size
                << ", expected is " << expected.second.size << ".";
@@ -574,14 +575,16 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
         }
     } else if (operationArray[0].GetString() == fileSizeProbeOp) {
         // probeFileSize
-        assert(operationArray.Size() >= 3u);
+        assert(operationArray.Size() >= 4u);
         assert(operationArray[1].IsString());
         assert(operationArray[2].IsString());
+        assert(operationArray[3].IsNumber());
 
         std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
         std::string path = std::string(operationArray[2].GetString(), operationArray[2].GetStringLength());
         assert(!path.empty());
 
+        float tolerance = operationArray[3].GetDouble();
         mbgl::filesystem::path filePath(path);
 
         if (!filePath.is_absolute()) {
@@ -592,7 +595,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
             auto size = mbgl::filesystem::file_size(filePath);
             metadata.metrics.fileSize.emplace(std::piecewise_construct,
                                               std::forward_as_tuple(std::move(mark)),
-                                              std::forward_as_tuple(std::move(path), size));
+                                              std::forward_as_tuple(std::move(path), size, tolerance));
         } else {
             metadata.errorMessage = std::string("File not found: ") + path;
             return false;
diff --git a/render-test/tests/file-size/fail-file-doesnt-match/style.json b/render-test/tests/file-size/fail-file-doesnt-match/style.json
index a3267fcf5e..839a8d4a12 100644
--- a/render-test/tests/file-size/fail-file-doesnt-match/style.json
+++ b/render-test/tests/file-size/fail-file-doesnt-match/style.json
@@ -3,8 +3,8 @@
   "metadata": {
     "test": {
       "operations": [
-        ["probeFileSize", "style", "style.json"],
-        ["probeFileSize", "image", "expected.png"]
+        ["probeFileSize", "style", "style.json", 0],
+        ["probeFileSize", "image", "expected.png", 0]
       ],
       "width": 64,
       "height": 64
diff --git a/render-test/tests/file-size/fail-file-not-found/style.json b/render-test/tests/file-size/fail-file-not-found/style.json
index 17dbd1e3d7..74054e1f40 100644
--- a/render-test/tests/file-size/fail-file-not-found/style.json
+++ b/render-test/tests/file-size/fail-file-not-found/style.json
@@ -3,8 +3,8 @@
   "metadata": {
     "test": {
       "operations": [
-        ["probeFileSize", "style", "style.aaaa"],
-        ["probeFileSize", "image", "expected.bbb"]
+        ["probeFileSize", "style", "style.aaaa", 0],
+        ["probeFileSize", "image", "expected.bbb", 0]
       ],
       "width": 64,
       "height": 64
diff --git a/render-test/tests/file-size/fail-size-is-over/style.json b/render-test/tests/file-size/fail-size-is-over/style.json
index a3267fcf5e..839a8d4a12 100644
--- a/render-test/tests/file-size/fail-size-is-over/style.json
+++ b/render-test/tests/file-size/fail-size-is-over/style.json
@@ -3,8 +3,8 @@
   "metadata": {
     "test": {
       "operations": [
-        ["probeFileSize", "style", "style.json"],
-        ["probeFileSize", "image", "expected.png"]
+        ["probeFileSize", "style", "style.json", 0],
+        ["probeFileSize", "image", "expected.png", 0]
       ],
       "width": 64,
       "height": 64
diff --git a/render-test/tests/file-size/fail-size-is-under/style.json b/render-test/tests/file-size/fail-size-is-under/style.json
index a3267fcf5e..839a8d4a12 100644
--- a/render-test/tests/file-size/fail-size-is-under/style.json
+++ b/render-test/tests/file-size/fail-size-is-under/style.json
@@ -3,8 +3,8 @@
   "metadata": {
     "test": {
       "operations": [
-        ["probeFileSize", "style", "style.json"],
-        ["probeFileSize", "image", "expected.png"]
+        ["probeFileSize", "style", "style.json", 0],
+        ["probeFileSize", "image", "expected.png", 0]
       ],
       "width": 64,
       "height": 64
diff --git a/render-test/tests/file-size/pass-size-is-same/metrics.json b/render-test/tests/file-size/pass-size-is-same/metrics.json
index b98328f72e..3d560bd610 100644
--- a/render-test/tests/file-size/pass-size-is-same/metrics.json
+++ b/render-test/tests/file-size/pass-size-is-same/metrics.json
@@ -8,7 +8,7 @@
         [
             "style",
             "style.json",
-            510
+            516
         ]
     ]
 }
\ No newline at end of file
diff --git a/render-test/tests/file-size/pass-size-is-same/style.json b/render-test/tests/file-size/pass-size-is-same/style.json
index a3267fcf5e..839a8d4a12 100644
--- a/render-test/tests/file-size/pass-size-is-same/style.json
+++ b/render-test/tests/file-size/pass-size-is-same/style.json
@@ -3,8 +3,8 @@
   "metadata": {
     "test": {
       "operations": [
-        ["probeFileSize", "style", "style.json"],
-        ["probeFileSize", "image", "expected.png"]
+        ["probeFileSize", "style", "style.json", 0],
+        ["probeFileSize", "image", "expected.png", 0]
       ],
       "width": 64,
       "height": 64