From 8b3a222c81abdfacddd6019135c1f655273ecaa5 Mon Sep 17 00:00:00 2001 From: "Thiago Marcos P. Santos" Date: Sun, 13 Oct 2019 17:49:49 +0300 Subject: [render-test] Add tolerance to file-size tests --- render-test/metadata.hpp | 10 +++++++++- render-test/parser.cpp | 2 +- render-test/runner.cpp | 9 ++++++--- render-test/tests/file-size/fail-file-doesnt-match/style.json | 4 ++-- render-test/tests/file-size/fail-file-not-found/style.json | 4 ++-- render-test/tests/file-size/fail-size-is-over/style.json | 4 ++-- render-test/tests/file-size/fail-size-is-under/style.json | 4 ++-- render-test/tests/file-size/pass-size-is-same/metrics.json | 2 +- render-test/tests/file-size/pass-size-is-same/style.json | 4 ++-- 9 files changed, 27 insertions(+), 16 deletions(-) diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp index e881bce208..bd26e6a7ba 100644 --- a/render-test/metadata.hpp +++ b/render-test/metadata.hpp @@ -31,12 +31,20 @@ struct TestPaths { } }; +inline std::tuple checkValue(float expected, float actual, float tolerance) { + float delta = expected * tolerance; + assert(delta >= 0.0f); + return std::make_tuple(std::abs(expected - actual) <= delta, delta); +} + struct FileSizeProbe { FileSizeProbe() = default; - FileSizeProbe(std::string path_, uintmax_t size_) : path(std::move(path_)), size(size_) {} + FileSizeProbe(std::string path_, uintmax_t size_, float tolerance_) + : path(std::move(path_)), size(size_), tolerance(tolerance_) {} std::string path; uintmax_t size; + float tolerance; }; struct MemoryProbe { diff --git a/render-test/parser.cpp b/render-test/parser.cpp index cbbc43293a..f4e54493eb 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -478,7 +478,7 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) { result.fileSize.emplace(std::piecewise_construct, std::forward_as_tuple(std::move(mark)), - std::forward_as_tuple(std::move(filePath), probeValue[2].GetUint64())); + std::forward_as_tuple(std::move(filePath), probeValue[2].GetUint64(), 0.f)); } } diff --git a/render-test/runner.cpp b/render-test/runner.cpp index 103a14a079..3594c9488b 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -258,7 +258,8 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage, return false; } - if (actual->second.size != expected.second.size) { + auto result = checkValue(expected.second.size, actual->second.size, actual->second.tolerance); + if (!std::get(result)) { std::stringstream ss; ss << "File size does not match at probe \"" << expected.first << "\": " << actual->second.size << ", expected is " << expected.second.size << "."; @@ -574,14 +575,16 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { } } else if (operationArray[0].GetString() == fileSizeProbeOp) { // probeFileSize - assert(operationArray.Size() >= 3u); + assert(operationArray.Size() >= 4u); assert(operationArray[1].IsString()); assert(operationArray[2].IsString()); + assert(operationArray[3].IsNumber()); std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); std::string path = std::string(operationArray[2].GetString(), operationArray[2].GetStringLength()); assert(!path.empty()); + float tolerance = operationArray[3].GetDouble(); mbgl::filesystem::path filePath(path); if (!filePath.is_absolute()) { @@ -592,7 +595,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { auto size = mbgl::filesystem::file_size(filePath); metadata.metrics.fileSize.emplace(std::piecewise_construct, std::forward_as_tuple(std::move(mark)), - std::forward_as_tuple(std::move(path), size)); + std::forward_as_tuple(std::move(path), size, tolerance)); } else { metadata.errorMessage = std::string("File not found: ") + path; return false; diff --git a/render-test/tests/file-size/fail-file-doesnt-match/style.json b/render-test/tests/file-size/fail-file-doesnt-match/style.json index a3267fcf5e..839a8d4a12 100644 --- a/render-test/tests/file-size/fail-file-doesnt-match/style.json +++ b/render-test/tests/file-size/fail-file-doesnt-match/style.json @@ -3,8 +3,8 @@ "metadata": { "test": { "operations": [ - ["probeFileSize", "style", "style.json"], - ["probeFileSize", "image", "expected.png"] + ["probeFileSize", "style", "style.json", 0], + ["probeFileSize", "image", "expected.png", 0] ], "width": 64, "height": 64 diff --git a/render-test/tests/file-size/fail-file-not-found/style.json b/render-test/tests/file-size/fail-file-not-found/style.json index 17dbd1e3d7..74054e1f40 100644 --- a/render-test/tests/file-size/fail-file-not-found/style.json +++ b/render-test/tests/file-size/fail-file-not-found/style.json @@ -3,8 +3,8 @@ "metadata": { "test": { "operations": [ - ["probeFileSize", "style", "style.aaaa"], - ["probeFileSize", "image", "expected.bbb"] + ["probeFileSize", "style", "style.aaaa", 0], + ["probeFileSize", "image", "expected.bbb", 0] ], "width": 64, "height": 64 diff --git a/render-test/tests/file-size/fail-size-is-over/style.json b/render-test/tests/file-size/fail-size-is-over/style.json index a3267fcf5e..839a8d4a12 100644 --- a/render-test/tests/file-size/fail-size-is-over/style.json +++ b/render-test/tests/file-size/fail-size-is-over/style.json @@ -3,8 +3,8 @@ "metadata": { "test": { "operations": [ - ["probeFileSize", "style", "style.json"], - ["probeFileSize", "image", "expected.png"] + ["probeFileSize", "style", "style.json", 0], + ["probeFileSize", "image", "expected.png", 0] ], "width": 64, "height": 64 diff --git a/render-test/tests/file-size/fail-size-is-under/style.json b/render-test/tests/file-size/fail-size-is-under/style.json index a3267fcf5e..839a8d4a12 100644 --- a/render-test/tests/file-size/fail-size-is-under/style.json +++ b/render-test/tests/file-size/fail-size-is-under/style.json @@ -3,8 +3,8 @@ "metadata": { "test": { "operations": [ - ["probeFileSize", "style", "style.json"], - ["probeFileSize", "image", "expected.png"] + ["probeFileSize", "style", "style.json", 0], + ["probeFileSize", "image", "expected.png", 0] ], "width": 64, "height": 64 diff --git a/render-test/tests/file-size/pass-size-is-same/metrics.json b/render-test/tests/file-size/pass-size-is-same/metrics.json index b98328f72e..3d560bd610 100644 --- a/render-test/tests/file-size/pass-size-is-same/metrics.json +++ b/render-test/tests/file-size/pass-size-is-same/metrics.json @@ -8,7 +8,7 @@ [ "style", "style.json", - 510 + 516 ] ] } \ No newline at end of file diff --git a/render-test/tests/file-size/pass-size-is-same/style.json b/render-test/tests/file-size/pass-size-is-same/style.json index a3267fcf5e..839a8d4a12 100644 --- a/render-test/tests/file-size/pass-size-is-same/style.json +++ b/render-test/tests/file-size/pass-size-is-same/style.json @@ -3,8 +3,8 @@ "metadata": { "test": { "operations": [ - ["probeFileSize", "style", "style.json"], - ["probeFileSize", "image", "expected.png"] + ["probeFileSize", "style", "style.json", 0], + ["probeFileSize", "image", "expected.png", 0] ], "width": 64, "height": 64 -- cgit v1.2.1