[render-test] Add metric_paths into Manifest (#15973)

* [render-test]Add expected metrics path in manifest file

* seperate metrics.json and expected.png

* fix expected path parsing

* keep expectation path if update is enabled

12 files changed, 276 insertions, 197 deletions

diff --git a/circle.yml b/circle.yml
index 1644bc49c0..8ad976c37d 100644
--- a/circle.yml
+++ b/circle.yml
@@ -262,6 +262,7 @@ commands:
             - ~/.gradle
       - run:
           name: Collecting artifacts
+          when: always
           command: |
             mkdir -p /tmp/tests/clang-tidy
             if [ -f clang-tidy.log ]; then cp clang-tidy.log /tmp/tests/clang-tidy; fi
diff --git a/render-test/tests/mac/memory/pass-memory-size-is-same/expected.png b/render-test/expected/tests/memory/pass-memory-size-is-same/expected.png
index 0858c19f05..0858c19f05 100644
--- a/render-test/tests/mac/memory/pass-memory-size-is-same/expected.png
+++ b/render-test/expected/tests/memory/pass-memory-size-is-same/expected.png
diff --git a/render-test/linux-manifest.json b/render-test/linux-manifest.json
index 32a5afdbdb..cc72a7446c 100644
--- a/render-test/linux-manifest.json
+++ b/render-test/linux-manifest.json
@@ -1,7 +1,7 @@
 {
     "base_test_path":"../mapbox-gl-js/test/integration",
-    "expectation_paths":["expected/render-tests"],
-    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json", "../render-test/tests/should-fail.json"],
+    "expectation_paths":["expected/"],
+    "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"],
     "vendor_path":"../vendor",
     "asset_path": "../mapbox-gl-js/test/integration"
 }
\ No newline at end of file
diff --git a/render-test/mac-manifest.json b/render-test/mac-manifest.json
index 224df81298..13a225cf4e 100644
--- a/render-test/mac-manifest.json
+++ b/render-test/mac-manifest.json
@@ -1,6 +1,6 @@
 {
     "base_test_path":"../mapbox-gl-js/test/integration",
-    "expectation_paths":["expected/render-tests", "tests/mac"],
+    "expectation_paths":["expected/"],
     "ignore_paths":["../platform/node/test/ignores.json", "../render-test/mac-ignores.json", "../render-test/tests/should-fail.json"],
     "vendor_path":"../vendor",
     "asset_path": "../mapbox-gl-js/test/integration"
diff --git a/render-test/mac-probe-manifest.json b/render-test/mac-probe-manifest.json
index f3cc56d0a8..154f6676dd 100644
--- a/render-test/mac-probe-manifest.json
+++ b/render-test/mac-probe-manifest.json
@@ -1,7 +1,8 @@
 {
     "probe_test_path":".",
-    "expectation_paths":["expected/render-tests", "tests/mac"],
+    "expectation_paths":["expected/"],
     "ignore_paths":["../render-test/mac-ignores.json", "../render-test/tests/should-fail.json"],
     "vendor_path":"../vendor",
-    "asset_path": "../mapbox-gl-js/test/integration"
+    "asset_path": "../mapbox-gl-js/test/integration",
+    "metric_path":"metrics/mac/"
 }
\ No newline at end of file
diff --git a/render-test/manifest_parser.cpp b/render-test/manifest_parser.cpp
index 6bba33754a..01d2246685 100644
--- a/render-test/manifest_parser.cpp
+++ b/render-test/manifest_parser.cpp
@@ -223,18 +223,16 @@ std::vector<std::pair<std::string, std::string>> parseIgnores(const std::vector<
     return ignores;
 }
 
-std::vector<mbgl::filesystem::path> getTestExpectations(mbgl::filesystem::path testPath,
-                                                        const mbgl::filesystem::path& testsRootPath,
-                                                        std::vector<mbgl::filesystem::path> expectationsPaths) {
-    std::vector<mbgl::filesystem::path> expectations{std::move(testPath.remove_filename())};
-    const auto& defaultTestExpectationsPath = expectations.front().string();
-
-    const std::regex regex{testsRootPath.string()};
-    for (const auto& path : expectationsPaths) {
-        expectations.emplace_back(std::regex_replace(defaultTestExpectationsPath, regex, path.string()));
-        assert(!expectations.back().empty());
+// defaultExpectationPath: absolute path that constains the style.json file for testing
+// testId: Test case id that used for composing expectation path
+// expectatedPaths: absolute paths that constain possible expected.png/metrics.json files for result checking
+std::vector<mbgl::filesystem::path> getTestExpectations(const mbgl::filesystem::path& defaultExpectationPath,
+                                                        const std::string& testId,
+                                                        std::vector<mbgl::filesystem::path> expectatedPaths) {
+    std::vector<mbgl::filesystem::path> expectations{defaultExpectationPath};
+    for (const auto& expectedPath : expectatedPaths) {
+        expectations.emplace_back(expectedPath / testId);
     }
-
     return expectations;
 }
 
@@ -331,6 +329,20 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manife
         }
         enbaleProbeTest = true;
     }
+    mbgl::filesystem::path expectedMetricPath;
+    if (document.HasMember("metric_path")) {
+        const auto& metricPathValue = document["metric_path"];
+        if (!metricPathValue.IsString()) {
+            mbgl::Log::Warning(mbgl::Event::General,
+                               "Invalid metric_path is provoided inside the manifest file: %s",
+                               filePath.c_str());
+            return mbgl::nullopt;
+        }
+        expectedMetricPath = getValidPath(manifest.manifestPath, metricPathValue.GetString());
+        if (expectedMetricPath.empty()) {
+            return mbgl::nullopt;
+        }
+    }
     std::vector<mbgl::filesystem::path> expectationPaths{};
     if (document.HasMember("expectation_paths")) {
         const auto& expectationPathValue = document["expectation_paths"];
@@ -414,7 +426,14 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manife
             }
 
             if (testPath.path().filename() == "style.json") {
-                testPaths.emplace_back(testPath, getTestExpectations(testPath, path, expectationPaths));
+                const auto defaultExpectationPath{std::move(mbgl::filesystem::path(testPath).remove_filename())};
+                const auto rootLength = manifest.testRootPath.length();
+                auto testId = defaultExpectationPath.string();
+                testId = testId.substr(rootLength + 1, testId.length() - rootLength - 1);
+
+                testPaths.emplace_back(testPath,
+                                       getTestExpectations(defaultExpectationPath, testId, expectationPaths),
+                                       getTestExpectations(defaultExpectationPath, testId, {expectedMetricPath}));
             }
         }
     }
diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp
index 2b007a445c..32eb3455ad 100644
--- a/render-test/metadata.hpp
+++ b/render-test/metadata.hpp
@@ -29,11 +29,16 @@ struct TestStatistics {
 
 struct TestPaths {
     TestPaths() = default;
-    TestPaths(mbgl::filesystem::path stylePath_, std::vector<mbgl::filesystem::path> expectations_)
-        : stylePath(std::move(stylePath_)), expectations(std::move(expectations_)) {}
+    TestPaths(mbgl::filesystem::path stylePath_,
+              std::vector<mbgl::filesystem::path> expectations_,
+              std::vector<mbgl::filesystem::path> expectedMetrics_)
+        : stylePath(std::move(stylePath_)),
+          expectations(std::move(expectations_)),
+          expectedMetrics(std::move(expectedMetrics_)) {}
 
     mbgl::filesystem::path stylePath;
     std::vector<mbgl::filesystem::path> expectations;
+    std::vector<mbgl::filesystem::path> expectedMetrics;
 
     std::string defaultExpectations() const {
         assert(!expectations.empty());
diff --git a/render-test/tests/mac/memory/pass-memory-size-is-same/metrics.json b/render-test/metrics/mac/tests/memory/pass-memory-size-is-same/metrics.json
index cc7d1c9fab..cc7d1c9fab 100644
--- a/render-test/tests/mac/memory/pass-memory-size-is-same/metrics.json
+++ b/render-test/metrics/mac/tests/memory/pass-memory-size-is-same/metrics.json
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index b5d48d23a1..b5f4f9e759 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -275,16 +275,16 @@ std::string serializeMetrics(const TestMetrics& metrics) {
 
 namespace {
 std::vector<std::string> readExpectedEntries(const std::regex& regex, const mbgl::filesystem::path& base) {
-    std::vector<std::string> expectedImages;
+    std::vector<std::string> expectedEntries;
     for (const auto& entry : mbgl::filesystem::directory_iterator(base)) {
         if (entry.is_regular_file()) {
             const std::string path = entry.path().string();
             if (std::regex_match(path, regex)) {
-                expectedImages.emplace_back(std::move(path));
+                expectedEntries.emplace_back(std::move(path));
             }
         }
     }
-    return expectedImages;
+    return expectedEntries;
 }
 } // namespace
 
@@ -293,6 +293,11 @@ std::vector<std::string> readExpectedImageEntries(const mbgl::filesystem::path&
     return readExpectedEntries(regex, base);
 }
 
+std::vector<std::string> readExpectedMetricEntries(const mbgl::filesystem::path& base) {
+    static const std::regex regex(".*metrics.*.json");
+    return readExpectedEntries(regex, base);
+}
+
 std::vector<std::string> readExpectedJSONEntries(const mbgl::filesystem::path& base) {
     static const std::regex regex(".*expected.*.json");
     return readExpectedEntries(regex, base);
diff --git a/render-test/parser.hpp b/render-test/parser.hpp
index 3d79ac668a..b7b4e9201d 100644
--- a/render-test/parser.hpp
+++ b/render-test/parser.hpp
@@ -19,6 +19,7 @@ std::string serializeJsonValue(const mbgl::JSValue&);
 std::string serializeMetrics(const TestMetrics&);
 
 std::vector<std::string> readExpectedImageEntries(const mbgl::filesystem::path& base);
+std::vector<std::string> readExpectedMetricEntries(const mbgl::filesystem::path& base);
 std::vector<std::string> readExpectedJSONEntries(const mbgl::filesystem::path& base);
 
 TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path);
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 004db87743..5a28576d85 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -315,217 +315,263 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage,
             }
         }
     }
+    return true;
+}
 
+bool TestRunner::checkProbingResults(TestMetadata& metadata) {
+    if (metadata.metrics.isEmpty()) return true;
+    const std::vector<mbgl::filesystem::path>& expectedMetrics = metadata.paths.expectedMetrics;
 #if !TEST_READ_ONLY
     if (getenv("UPDATE_METRICS")) {
-        if (!metadata.metrics.isEmpty()) {
-            mbgl::filesystem::create_directories(expectations.back());
-            mbgl::util::write_file(expectations.back().string() + "/metrics.json", serializeMetrics(metadata.metrics));
-            return true;
-        }
+        mbgl::filesystem::create_directories(expectedMetrics.back());
+        mbgl::util::write_file(expectedMetrics.back().string() + "/metrics.json", serializeMetrics(metadata.metrics));
+        return true;
     }
 #endif
 
-    mbgl::filesystem::path expectedMetricsPath;
-    for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) {
+    // Check the possible paths in reverse order, so that the default path with the test style will only be checked in
+    // the very end.
+    std::vector<std::string> expectedMetricsPaths;
+    for (auto rit = expectedMetrics.rbegin(); rit != expectedMetrics.rend(); ++rit) {
         if (mbgl::filesystem::exists(*rit)) {
-            if (metadata.expectedMetrics.isEmpty()) {
-                mbgl::filesystem::path maybeExpectedMetricsPath{*rit};
-                maybeExpectedMetricsPath.replace_filename("metrics.json");
-                metadata.expectedMetrics = readExpectedMetrics(maybeExpectedMetricsPath);
-            }
+            expectedMetricsPaths = readExpectedMetricEntries(*rit);
+            if (!expectedMetricsPaths.empty()) break;
         }
     }
 
-    // Check file size metrics.
-    for (const auto& expected : metadata.expectedMetrics.fileSize) {
-        auto actual = metadata.metrics.fileSize.find(expected.first);
-        if (actual == metadata.metrics.fileSize.end()) {
-            metadata.errorMessage = "Failed to find fileSize probe: " + expected.first;
+    // In case no metrics.json is found, skip assigning the expectedMetrics to metadata, otherwise, take the first found
+    // metrics.
+    for (const auto& entry : expectedMetricsPaths) {
+        auto maybeExpectedMetrics = readExpectedMetrics(entry);
+        if (maybeExpectedMetrics.isEmpty()) {
+            metadata.errorMessage = "Failed to load expected metrics " + entry;
             return false;
         }
-        if (actual->second.path != expected.second.path) {
-            std::stringstream ss;
-            ss << "Comparing different files at probe \"" << expected.first << "\": " << actual->second.path
-               << ", expected is " << expected.second.path << ".";
-            metadata.errorMessage = ss.str();
+        metadata.expectedMetrics = maybeExpectedMetrics;
+        break;
+    }
 
-            return false;
-        }
+    if (metadata.expectedMetrics.isEmpty()) {
+        metadata.errorMessage = "Failed to find metric expectations for: " + metadata.paths.stylePath.string();
+        return false;
+    }
+    // Check file size metrics.
+    auto checkFileSize = [](TestMetadata& metadata) -> bool {
+        if (metadata.metrics.fileSize.empty()) return true;
+        for (const auto& expected : metadata.expectedMetrics.fileSize) {
+            auto actual = metadata.metrics.fileSize.find(expected.first);
+            if (actual == metadata.metrics.fileSize.end()) {
+                metadata.errorMessage = "Failed to find fileSize probe: " + expected.first;
+                return false;
+            }
+            if (actual->second.path != expected.second.path) {
+                std::stringstream ss;
+                ss << "Comparing different files at probe \"" << expected.first << "\": " << actual->second.path
+                   << ", expected is " << expected.second.path << ".";
+                metadata.errorMessage = ss.str();
 
-        auto result = checkValue(expected.second.size, actual->second.size, actual->second.tolerance);
-        if (!std::get<bool>(result)) {
-            std::stringstream ss;
-            ss << "File size does not match at probe \"" << expected.first << "\": " << actual->second.size
-               << ", expected is " << expected.second.size << ".";
+                return false;
+            }
 
-            metadata.errorMessage = ss.str();
-            return false;
+            auto result = checkValue(expected.second.size, actual->second.size, actual->second.tolerance);
+            if (!std::get<bool>(result)) {
+                std::stringstream ss;
+                ss << "File size does not match at probe \"" << expected.first << "\": " << actual->second.size
+                   << ", expected is " << expected.second.size << ".";
+
+                metadata.errorMessage = ss.str();
+                return false;
+            }
         }
-    }
+        return true;
+    };
+    auto checkMemory = [](TestMetadata& metadata) -> bool {
+        if (metadata.metrics.memory.empty()) return true;
 #if !defined(SANITIZE)
-    // Check memory metrics.
-    for (const auto& expected : metadata.expectedMetrics.memory) {
-        auto actual = metadata.metrics.memory.find(expected.first);
-        if (actual == metadata.metrics.memory.end()) {
-            metadata.errorMessage = "Failed to find memory probe: " + expected.first;
-            return false;
-        }
-        bool passed{false};
-        float delta{0.0f};
-        std::stringstream errorStream;
-        std::tie(passed, delta) = MemoryProbe::checkPeak(expected.second, actual->second);
-        if (!passed) {
-            errorStream << "Allocated memory peak size at probe \"" << expected.first << "\" is " << actual->second.peak
-                        << " bytes, expected is " << expected.second.peak << "±" << delta << " bytes.";
-        }
+        // Check memory metrics.
+        for (const auto& expected : metadata.expectedMetrics.memory) {
+            auto actual = metadata.metrics.memory.find(expected.first);
+            if (actual == metadata.metrics.memory.end()) {
+                metadata.errorMessage = "Failed to find memory probe: " + expected.first;
+                return false;
+            }
+            bool passed{false};
+            float delta{0.0f};
+            std::stringstream errorStream;
+            std::tie(passed, delta) = MemoryProbe::checkPeak(expected.second, actual->second);
+            if (!passed) {
+                errorStream << "Allocated memory peak size at probe \"" << expected.first << "\" is "
+                            << actual->second.peak << " bytes, expected is " << expected.second.peak << "±" << delta
+                            << " bytes.";
+            }
 
-        std::tie(passed, delta) = MemoryProbe::checkAllocations(expected.second, actual->second);
-        if (!passed) {
-            errorStream << "Number of allocations at probe \"" << expected.first << "\" is "
-                        << actual->second.allocations << ", expected is " << expected.second.allocations << "±"
-                        << std::round(delta) << " allocations.";
+            std::tie(passed, delta) = MemoryProbe::checkAllocations(expected.second, actual->second);
+            if (!passed) {
+                errorStream << "Number of allocations at probe \"" << expected.first << "\" is "
+                            << actual->second.allocations << ", expected is " << expected.second.allocations << "±"
+                            << std::round(delta) << " allocations.";
+            }
+
+            metadata.errorMessage = errorStream.str();
+            if (!metadata.errorMessage.empty()) return false;
         }
 
-        metadata.errorMessage = errorStream.str();
-        if (!metadata.errorMessage.empty()) return false;
-    }
+#endif // !defined(SANITIZE)
+        return true;
+    };
 
     // Check network metrics.
-    for (const auto& expected : metadata.expectedMetrics.network) {
-        auto actual = metadata.metrics.network.find(expected.first);
-        if (actual == metadata.metrics.network.end()) {
-            metadata.errorMessage = "Failed to find network probe: " + expected.first;
-            return false;
-        }
-        bool failed = false;
-        if (actual->second.requests != expected.second.requests) {
-            std::stringstream ss;
-            ss << "Number of requests at probe \"" << expected.first << "\" is " << actual->second.requests
-               << ", expected is " << expected.second.requests << ". ";
-
-            metadata.errorMessage = ss.str();
-            failed = true;
-        }
-        if (actual->second.transferred != expected.second.transferred) {
-            std::stringstream ss;
-            ss << "Transferred data at probe \"" << expected.first << "\" is " << actual->second.transferred
-               << " bytes, expected is " << expected.second.transferred << " bytes.";
+    auto checkNetwork = [](TestMetadata& metadata) -> bool {
+        if (metadata.metrics.network.empty()) return true;
+#if !defined(SANITIZE)
+        for (const auto& expected : metadata.expectedMetrics.network) {
+            auto actual = metadata.metrics.network.find(expected.first);
+            if (actual == metadata.metrics.network.end()) {
+                metadata.errorMessage = "Failed to find network probe: " + expected.first;
+                return false;
+            }
+            bool failed = false;
+            if (actual->second.requests != expected.second.requests) {
+                std::stringstream ss;
+                ss << "Number of requests at probe \"" << expected.first << "\" is " << actual->second.requests
+                   << ", expected is " << expected.second.requests << ". ";
+
+                metadata.errorMessage = ss.str();
+                failed = true;
+            }
+            if (actual->second.transferred != expected.second.transferred) {
+                std::stringstream ss;
+                ss << "Transferred data at probe \"" << expected.first << "\" is " << actual->second.transferred
+                   << " bytes, expected is " << expected.second.transferred << " bytes.";
 
-            metadata.errorMessage += ss.str();
-            failed = true;
-        }
-        if (failed) {
-            return false;
+                metadata.errorMessage += ss.str();
+                failed = true;
+            }
+            if (failed) {
+                return false;
+            }
         }
-    }
 #endif // !defined(SANITIZE)
+        return true;
+    };
     // Check fps metrics
-    for (const auto& expected : metadata.expectedMetrics.fps) {
-        auto actual = metadata.metrics.fps.find(expected.first);
-        if (actual == metadata.metrics.fps.end()) {
-            metadata.errorMessage = "Failed to find fps probe: " + expected.first;
-            return false;
-        }
-        auto result = checkValue(expected.second.average, actual->second.average, expected.second.tolerance);
-        if (!std::get<bool>(result)) {
-            std::stringstream ss;
-            ss << "Average fps at probe \"" << expected.first << "\" is " << actual->second.average
-               << ", expected to be " << expected.second.average << " with tolerance of " << expected.second.tolerance;
-            metadata.errorMessage = ss.str();
-            return false;
-        }
-        result = checkValue(expected.second.minOnePc, actual->second.minOnePc, expected.second.tolerance);
-        if (!std::get<bool>(result)) {
-            std::stringstream ss;
-            ss << "Minimum(1%) fps at probe \"" << expected.first << "\" is " << actual->second.minOnePc
-               << ", expected to be " << expected.second.minOnePc << " with tolerance of " << expected.second.tolerance;
-            metadata.errorMessage = ss.str();
-            return false;
+    auto checkFps = [](TestMetadata& metadata) -> bool {
+        if (metadata.metrics.fps.empty()) return true;
+        for (const auto& expected : metadata.expectedMetrics.fps) {
+            auto actual = metadata.metrics.fps.find(expected.first);
+            if (actual == metadata.metrics.fps.end()) {
+                metadata.errorMessage = "Failed to find fps probe: " + expected.first;
+                return false;
+            }
+            auto result = checkValue(expected.second.average, actual->second.average, expected.second.tolerance);
+            if (!std::get<bool>(result)) {
+                std::stringstream ss;
+                ss << "Average fps at probe \"" << expected.first << "\" is " << actual->second.average
+                   << ", expected to be " << expected.second.average << " with tolerance of "
+                   << expected.second.tolerance;
+                metadata.errorMessage = ss.str();
+                return false;
+            }
+            result = checkValue(expected.second.minOnePc, actual->second.minOnePc, expected.second.tolerance);
+            if (!std::get<bool>(result)) {
+                std::stringstream ss;
+                ss << "Minimum(1%) fps at probe \"" << expected.first << "\" is " << actual->second.minOnePc
+                   << ", expected to be " << expected.second.minOnePc << " with tolerance of "
+                   << expected.second.tolerance;
+                metadata.errorMessage = ss.str();
+                return false;
+            }
         }
-    }
+        return true;
+    };
     // Check gfx metrics
-    for (const auto& expected : metadata.expectedMetrics.gfx) {
-        auto actual = metadata.metrics.gfx.find(expected.first);
-        if (actual == metadata.metrics.gfx.end()) {
-            metadata.errorMessage = "Failed to find gfx probe: " + expected.first;
-            return false;
-        }
+    auto checkGfx = [](TestMetadata& metadata) -> bool {
+        if (metadata.metrics.gfx.empty()) return true;
+        for (const auto& expected : metadata.expectedMetrics.gfx) {
+            auto actual = metadata.metrics.gfx.find(expected.first);
+            if (actual == metadata.metrics.gfx.end()) {
+                metadata.errorMessage = "Failed to find gfx probe: " + expected.first;
+                return false;
+            }
 
-        const auto& probeName = expected.first;
-        const auto& expectedValue = expected.second;
-        const auto& actualValue = actual->second;
-        bool failed = false;
-
-        if (expectedValue.numDrawCalls != actualValue.numDrawCalls) {
-            std::stringstream ss;
-            if (!metadata.errorMessage.empty()) ss << std::endl;
-            ss << "Number of draw calls at probe\"" << probeName << "\" is " << actualValue.numDrawCalls
-               << ", expected is " << expectedValue.numDrawCalls;
-            metadata.errorMessage += ss.str();
-            failed = true;
-        }
+            const auto& probeName = expected.first;
+            const auto& expectedValue = expected.second;
+            const auto& actualValue = actual->second;
+            bool failed = false;
+
+            if (expectedValue.numDrawCalls != actualValue.numDrawCalls) {
+                std::stringstream ss;
+                if (!metadata.errorMessage.empty()) ss << std::endl;
+                ss << "Number of draw calls at probe\"" << probeName << "\" is " << actualValue.numDrawCalls
+                   << ", expected is " << expectedValue.numDrawCalls;
+                metadata.errorMessage += ss.str();
+                failed = true;
+            }
 
-        if (expectedValue.numTextures != actualValue.numTextures) {
-            std::stringstream ss;
-            if (!metadata.errorMessage.empty()) ss << std::endl;
-            ss << "Number of textures at probe \"" << probeName << "\" is " << actualValue.numTextures
-               << ", expected is " << expectedValue.numTextures;
-            metadata.errorMessage += ss.str();
-            failed = true;
-        }
+            if (expectedValue.numTextures != actualValue.numTextures) {
+                std::stringstream ss;
+                if (!metadata.errorMessage.empty()) ss << std::endl;
+                ss << "Number of textures at probe \"" << probeName << "\" is " << actualValue.numTextures
+                   << ", expected is " << expectedValue.numTextures;
+                metadata.errorMessage += ss.str();
+                failed = true;
+            }
 
-        if (expectedValue.numBuffers != actualValue.numBuffers) {
-            std::stringstream ss;
-            if (!metadata.errorMessage.empty()) ss << std::endl;
-            ss << "Number of vertex and index buffers at probe \"" << probeName << "\" is " << actualValue.numBuffers
-               << ", expected is " << expectedValue.numBuffers;
-            metadata.errorMessage += ss.str();
-            failed = true;
-        }
+            if (expectedValue.numBuffers != actualValue.numBuffers) {
+                std::stringstream ss;
+                if (!metadata.errorMessage.empty()) ss << std::endl;
+                ss << "Number of vertex and index buffers at probe \"" << probeName << "\" is "
+                   << actualValue.numBuffers << ", expected is " << expectedValue.numBuffers;
+                metadata.errorMessage += ss.str();
+                failed = true;
+            }
 
-        if (expectedValue.numFrameBuffers != actualValue.numFrameBuffers) {
-            std::stringstream ss;
-            if (!metadata.errorMessage.empty()) ss << std::endl;
-            ss << "Number of frame buffers at probe \"" << probeName << "\" is " << actualValue.numFrameBuffers
-               << ", expected is " << expectedValue.numFrameBuffers;
-            metadata.errorMessage += ss.str();
-            failed = true;
-        }
+            if (expectedValue.numFrameBuffers != actualValue.numFrameBuffers) {
+                std::stringstream ss;
+                if (!metadata.errorMessage.empty()) ss << std::endl;
+                ss << "Number of frame buffers at probe \"" << probeName << "\" is " << actualValue.numFrameBuffers
+                   << ", expected is " << expectedValue.numFrameBuffers;
+                metadata.errorMessage += ss.str();
+                failed = true;
+            }
 
-        if (expectedValue.memTextures.peak != actualValue.memTextures.peak) {
-            std::stringstream ss;
-            if (!metadata.errorMessage.empty()) ss << std::endl;
-            ss << "Allocated texture memory peak size at probe \"" << probeName << "\" is "
-               << actualValue.memTextures.peak << " bytes, expected is " << expectedValue.memTextures.peak << " bytes";
-            metadata.errorMessage += ss.str();
-            failed = true;
-        }
+            if (expectedValue.memTextures.peak != actualValue.memTextures.peak) {
+                std::stringstream ss;
+                if (!metadata.errorMessage.empty()) ss << std::endl;
+                ss << "Allocated texture memory peak size at probe \"" << probeName << "\" is "
+                   << actualValue.memTextures.peak << " bytes, expected is " << expectedValue.memTextures.peak
+                   << " bytes";
+                metadata.errorMessage += ss.str();
+                failed = true;
+            }
 
-        if (expectedValue.memIndexBuffers.peak != actualValue.memIndexBuffers.peak) {
-            std::stringstream ss;
-            if (!metadata.errorMessage.empty()) ss << std::endl;
-            ss << "Allocated index buffer memory peak size at probe \"" << probeName << "\" is "
-               << actualValue.memIndexBuffers.peak << " bytes, expected is " << expectedValue.memIndexBuffers.peak
-               << " bytes";
-            metadata.errorMessage += ss.str();
-            failed = true;
-        }
+            if (expectedValue.memIndexBuffers.peak != actualValue.memIndexBuffers.peak) {
+                std::stringstream ss;
+                if (!metadata.errorMessage.empty()) ss << std::endl;
+                ss << "Allocated index buffer memory peak size at probe \"" << probeName << "\" is "
+                   << actualValue.memIndexBuffers.peak << " bytes, expected is " << expectedValue.memIndexBuffers.peak
+                   << " bytes";
+                metadata.errorMessage += ss.str();
+                failed = true;
+            }
 
-        if (expectedValue.memVertexBuffers.peak != actualValue.memVertexBuffers.peak) {
-            std::stringstream ss;
-            if (!metadata.errorMessage.empty()) ss << std::endl;
-            ss << "Allocated vertex buffer memory peak size at probe \"" << probeName << "\" is "
-               << actualValue.memVertexBuffers.peak << " bytes, expected is " << expectedValue.memVertexBuffers.peak
-               << " bytes";
-            metadata.errorMessage += ss.str();
-            failed = true;
-        }
+            if (expectedValue.memVertexBuffers.peak != actualValue.memVertexBuffers.peak) {
+                std::stringstream ss;
+                if (!metadata.errorMessage.empty()) ss << std::endl;
+                ss << "Allocated vertex buffer memory peak size at probe \"" << probeName << "\" is "
+                   << actualValue.memVertexBuffers.peak << " bytes, expected is " << expectedValue.memVertexBuffers.peak
+                   << " bytes";
+                metadata.errorMessage += ss.str();
+                failed = true;
+            }
 
-        if (failed) return false;
-    }
+            if (failed) return false;
+        }
+        return true;
+    };
 
-    return true;
+    return checkFileSize(metadata) && checkMemory(metadata) && checkNetwork(metadata) && checkFps(metadata) &&
+           checkGfx(metadata);
 }
 
 bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata, RunContext& ctx) {
@@ -1162,7 +1208,7 @@ bool TestRunner::run(TestMetadata& metadata, const std::set<std::string>& inject
     }
 
     if (metadata.renderTest) {
-        return checkRenderTestResults(std::move(result.image), metadata);
+        return checkRenderTestResults(std::move(result.image), metadata) && checkProbingResults(metadata);
     } else {
         std::vector<mbgl::Feature> features;
         assert(metadata.document["metadata"]["test"]["queryGeometry"].IsArray());
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
index c31d672ba0..5dd529f057 100644
--- a/render-test/runner.hpp
+++ b/render-test/runner.hpp
@@ -31,6 +31,7 @@ private:
                                std::vector<mbgl::Feature>&& features,
                                TestMetadata&);
     bool checkRenderTestResults(mbgl::PremultipliedImage&& image, TestMetadata&);
+    bool checkProbingResults(TestMetadata&);
 
     struct Impl {
         Impl(const TestMetadata&);