From 11321497da66b1f4b05dc23a9b1b0b434018b661 Mon Sep 17 00:00:00 2001 From: zmiao Date: Wed, 27 Nov 2019 11:08:55 +0200 Subject: [render-test] Add metric_paths into Manifest (#15973) * [render-test]Add expected metrics path in manifest file * seperate metrics.json and expected.png * fix expected path parsing * keep expectation path if update is enabled --- .../memory/pass-memory-size-is-same/expected.png | Bin 0 -> 96 bytes render-test/linux-manifest.json | 4 +- render-test/mac-manifest.json | 2 +- render-test/mac-probe-manifest.json | 5 +- render-test/manifest_parser.cpp | 43 ++- render-test/metadata.hpp | 9 +- .../memory/pass-memory-size-is-same/metrics.json | 19 + render-test/parser.cpp | 11 +- render-test/parser.hpp | 1 + render-test/runner.cpp | 396 ++++++++++++--------- render-test/runner.hpp | 1 + .../memory/pass-memory-size-is-same/expected.png | Bin 96 -> 0 bytes .../memory/pass-memory-size-is-same/metrics.json | 19 - 13 files changed, 294 insertions(+), 216 deletions(-) create mode 100644 render-test/expected/tests/memory/pass-memory-size-is-same/expected.png create mode 100644 render-test/metrics/mac/tests/memory/pass-memory-size-is-same/metrics.json delete mode 100644 render-test/tests/mac/memory/pass-memory-size-is-same/expected.png delete mode 100644 render-test/tests/mac/memory/pass-memory-size-is-same/metrics.json (limited to 'render-test') diff --git a/render-test/expected/tests/memory/pass-memory-size-is-same/expected.png b/render-test/expected/tests/memory/pass-memory-size-is-same/expected.png new file mode 100644 index 0000000000..0858c19f05 Binary files /dev/null and b/render-test/expected/tests/memory/pass-memory-size-is-same/expected.png differ diff --git a/render-test/linux-manifest.json b/render-test/linux-manifest.json index 32a5afdbdb..cc72a7446c 100644 --- a/render-test/linux-manifest.json +++ b/render-test/linux-manifest.json @@ -1,7 +1,7 @@ { "base_test_path":"../mapbox-gl-js/test/integration", - "expectation_paths":["expected/render-tests"], - "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json", "../render-test/tests/should-fail.json"], + "expectation_paths":["expected/"], + "ignore_paths":["../platform/node/test/ignores.json", "../render-test/linux-ignores.json"], "vendor_path":"../vendor", "asset_path": "../mapbox-gl-js/test/integration" } \ No newline at end of file diff --git a/render-test/mac-manifest.json b/render-test/mac-manifest.json index 224df81298..13a225cf4e 100644 --- a/render-test/mac-manifest.json +++ b/render-test/mac-manifest.json @@ -1,6 +1,6 @@ { "base_test_path":"../mapbox-gl-js/test/integration", - "expectation_paths":["expected/render-tests", "tests/mac"], + "expectation_paths":["expected/"], "ignore_paths":["../platform/node/test/ignores.json", "../render-test/mac-ignores.json", "../render-test/tests/should-fail.json"], "vendor_path":"../vendor", "asset_path": "../mapbox-gl-js/test/integration" diff --git a/render-test/mac-probe-manifest.json b/render-test/mac-probe-manifest.json index f3cc56d0a8..154f6676dd 100644 --- a/render-test/mac-probe-manifest.json +++ b/render-test/mac-probe-manifest.json @@ -1,7 +1,8 @@ { "probe_test_path":".", - "expectation_paths":["expected/render-tests", "tests/mac"], + "expectation_paths":["expected/"], "ignore_paths":["../render-test/mac-ignores.json", "../render-test/tests/should-fail.json"], "vendor_path":"../vendor", - "asset_path": "../mapbox-gl-js/test/integration" + "asset_path": "../mapbox-gl-js/test/integration", + "metric_path":"metrics/mac/" } \ No newline at end of file diff --git a/render-test/manifest_parser.cpp b/render-test/manifest_parser.cpp index 6bba33754a..01d2246685 100644 --- a/render-test/manifest_parser.cpp +++ b/render-test/manifest_parser.cpp @@ -223,18 +223,16 @@ std::vector> parseIgnores(const std::vector< return ignores; } -std::vector getTestExpectations(mbgl::filesystem::path testPath, - const mbgl::filesystem::path& testsRootPath, - std::vector expectationsPaths) { - std::vector expectations{std::move(testPath.remove_filename())}; - const auto& defaultTestExpectationsPath = expectations.front().string(); - - const std::regex regex{testsRootPath.string()}; - for (const auto& path : expectationsPaths) { - expectations.emplace_back(std::regex_replace(defaultTestExpectationsPath, regex, path.string())); - assert(!expectations.back().empty()); +// defaultExpectationPath: absolute path that constains the style.json file for testing +// testId: Test case id that used for composing expectation path +// expectatedPaths: absolute paths that constain possible expected.png/metrics.json files for result checking +std::vector getTestExpectations(const mbgl::filesystem::path& defaultExpectationPath, + const std::string& testId, + std::vector expectatedPaths) { + std::vector expectations{defaultExpectationPath}; + for (const auto& expectedPath : expectatedPaths) { + expectations.emplace_back(expectedPath / testId); } - return expectations; } @@ -331,6 +329,20 @@ mbgl::optional ManifestParser::parseManifest(const std::string& manife } enbaleProbeTest = true; } + mbgl::filesystem::path expectedMetricPath; + if (document.HasMember("metric_path")) { + const auto& metricPathValue = document["metric_path"]; + if (!metricPathValue.IsString()) { + mbgl::Log::Warning(mbgl::Event::General, + "Invalid metric_path is provoided inside the manifest file: %s", + filePath.c_str()); + return mbgl::nullopt; + } + expectedMetricPath = getValidPath(manifest.manifestPath, metricPathValue.GetString()); + if (expectedMetricPath.empty()) { + return mbgl::nullopt; + } + } std::vector expectationPaths{}; if (document.HasMember("expectation_paths")) { const auto& expectationPathValue = document["expectation_paths"]; @@ -414,7 +426,14 @@ mbgl::optional ManifestParser::parseManifest(const std::string& manife } if (testPath.path().filename() == "style.json") { - testPaths.emplace_back(testPath, getTestExpectations(testPath, path, expectationPaths)); + const auto defaultExpectationPath{std::move(mbgl::filesystem::path(testPath).remove_filename())}; + const auto rootLength = manifest.testRootPath.length(); + auto testId = defaultExpectationPath.string(); + testId = testId.substr(rootLength + 1, testId.length() - rootLength - 1); + + testPaths.emplace_back(testPath, + getTestExpectations(defaultExpectationPath, testId, expectationPaths), + getTestExpectations(defaultExpectationPath, testId, {expectedMetricPath})); } } } diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp index 2b007a445c..32eb3455ad 100644 --- a/render-test/metadata.hpp +++ b/render-test/metadata.hpp @@ -29,11 +29,16 @@ struct TestStatistics { struct TestPaths { TestPaths() = default; - TestPaths(mbgl::filesystem::path stylePath_, std::vector expectations_) - : stylePath(std::move(stylePath_)), expectations(std::move(expectations_)) {} + TestPaths(mbgl::filesystem::path stylePath_, + std::vector expectations_, + std::vector expectedMetrics_) + : stylePath(std::move(stylePath_)), + expectations(std::move(expectations_)), + expectedMetrics(std::move(expectedMetrics_)) {} mbgl::filesystem::path stylePath; std::vector expectations; + std::vector expectedMetrics; std::string defaultExpectations() const { assert(!expectations.empty()); diff --git a/render-test/metrics/mac/tests/memory/pass-memory-size-is-same/metrics.json b/render-test/metrics/mac/tests/memory/pass-memory-size-is-same/metrics.json new file mode 100644 index 0000000000..cc7d1c9fab --- /dev/null +++ b/render-test/metrics/mac/tests/memory/pass-memory-size-is-same/metrics.json @@ -0,0 +1,19 @@ +{ + "memory": [ + [ + "after setZoom 0.9", + 33271, + 51 + ], + [ + "end", + 47552, + 101 + ], + [ + "start", + 0, + 0 + ] + ] +} diff --git a/render-test/parser.cpp b/render-test/parser.cpp index b5d48d23a1..b5f4f9e759 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -275,16 +275,16 @@ std::string serializeMetrics(const TestMetrics& metrics) { namespace { std::vector readExpectedEntries(const std::regex& regex, const mbgl::filesystem::path& base) { - std::vector expectedImages; + std::vector expectedEntries; for (const auto& entry : mbgl::filesystem::directory_iterator(base)) { if (entry.is_regular_file()) { const std::string path = entry.path().string(); if (std::regex_match(path, regex)) { - expectedImages.emplace_back(std::move(path)); + expectedEntries.emplace_back(std::move(path)); } } } - return expectedImages; + return expectedEntries; } } // namespace @@ -293,6 +293,11 @@ std::vector readExpectedImageEntries(const mbgl::filesystem::path& return readExpectedEntries(regex, base); } +std::vector readExpectedMetricEntries(const mbgl::filesystem::path& base) { + static const std::regex regex(".*metrics.*.json"); + return readExpectedEntries(regex, base); +} + std::vector readExpectedJSONEntries(const mbgl::filesystem::path& base) { static const std::regex regex(".*expected.*.json"); return readExpectedEntries(regex, base); diff --git a/render-test/parser.hpp b/render-test/parser.hpp index 3d79ac668a..b7b4e9201d 100644 --- a/render-test/parser.hpp +++ b/render-test/parser.hpp @@ -19,6 +19,7 @@ std::string serializeJsonValue(const mbgl::JSValue&); std::string serializeMetrics(const TestMetrics&); std::vector readExpectedImageEntries(const mbgl::filesystem::path& base); +std::vector readExpectedMetricEntries(const mbgl::filesystem::path& base); std::vector readExpectedJSONEntries(const mbgl::filesystem::path& base); TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path); diff --git a/render-test/runner.cpp b/render-test/runner.cpp index 004db87743..5a28576d85 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -315,217 +315,263 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage, } } } + return true; +} +bool TestRunner::checkProbingResults(TestMetadata& metadata) { + if (metadata.metrics.isEmpty()) return true; + const std::vector& expectedMetrics = metadata.paths.expectedMetrics; #if !TEST_READ_ONLY if (getenv("UPDATE_METRICS")) { - if (!metadata.metrics.isEmpty()) { - mbgl::filesystem::create_directories(expectations.back()); - mbgl::util::write_file(expectations.back().string() + "/metrics.json", serializeMetrics(metadata.metrics)); - return true; - } + mbgl::filesystem::create_directories(expectedMetrics.back()); + mbgl::util::write_file(expectedMetrics.back().string() + "/metrics.json", serializeMetrics(metadata.metrics)); + return true; } #endif - mbgl::filesystem::path expectedMetricsPath; - for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) { + // Check the possible paths in reverse order, so that the default path with the test style will only be checked in + // the very end. + std::vector expectedMetricsPaths; + for (auto rit = expectedMetrics.rbegin(); rit != expectedMetrics.rend(); ++rit) { if (mbgl::filesystem::exists(*rit)) { - if (metadata.expectedMetrics.isEmpty()) { - mbgl::filesystem::path maybeExpectedMetricsPath{*rit}; - maybeExpectedMetricsPath.replace_filename("metrics.json"); - metadata.expectedMetrics = readExpectedMetrics(maybeExpectedMetricsPath); - } + expectedMetricsPaths = readExpectedMetricEntries(*rit); + if (!expectedMetricsPaths.empty()) break; } } - // Check file size metrics. - for (const auto& expected : metadata.expectedMetrics.fileSize) { - auto actual = metadata.metrics.fileSize.find(expected.first); - if (actual == metadata.metrics.fileSize.end()) { - metadata.errorMessage = "Failed to find fileSize probe: " + expected.first; + // In case no metrics.json is found, skip assigning the expectedMetrics to metadata, otherwise, take the first found + // metrics. + for (const auto& entry : expectedMetricsPaths) { + auto maybeExpectedMetrics = readExpectedMetrics(entry); + if (maybeExpectedMetrics.isEmpty()) { + metadata.errorMessage = "Failed to load expected metrics " + entry; return false; } - if (actual->second.path != expected.second.path) { - std::stringstream ss; - ss << "Comparing different files at probe \"" << expected.first << "\": " << actual->second.path - << ", expected is " << expected.second.path << "."; - metadata.errorMessage = ss.str(); + metadata.expectedMetrics = maybeExpectedMetrics; + break; + } - return false; - } + if (metadata.expectedMetrics.isEmpty()) { + metadata.errorMessage = "Failed to find metric expectations for: " + metadata.paths.stylePath.string(); + return false; + } + // Check file size metrics. + auto checkFileSize = [](TestMetadata& metadata) -> bool { + if (metadata.metrics.fileSize.empty()) return true; + for (const auto& expected : metadata.expectedMetrics.fileSize) { + auto actual = metadata.metrics.fileSize.find(expected.first); + if (actual == metadata.metrics.fileSize.end()) { + metadata.errorMessage = "Failed to find fileSize probe: " + expected.first; + return false; + } + if (actual->second.path != expected.second.path) { + std::stringstream ss; + ss << "Comparing different files at probe \"" << expected.first << "\": " << actual->second.path + << ", expected is " << expected.second.path << "."; + metadata.errorMessage = ss.str(); - auto result = checkValue(expected.second.size, actual->second.size, actual->second.tolerance); - if (!std::get(result)) { - std::stringstream ss; - ss << "File size does not match at probe \"" << expected.first << "\": " << actual->second.size - << ", expected is " << expected.second.size << "."; + return false; + } - metadata.errorMessage = ss.str(); - return false; + auto result = checkValue(expected.second.size, actual->second.size, actual->second.tolerance); + if (!std::get(result)) { + std::stringstream ss; + ss << "File size does not match at probe \"" << expected.first << "\": " << actual->second.size + << ", expected is " << expected.second.size << "."; + + metadata.errorMessage = ss.str(); + return false; + } } - } + return true; + }; + auto checkMemory = [](TestMetadata& metadata) -> bool { + if (metadata.metrics.memory.empty()) return true; #if !defined(SANITIZE) - // Check memory metrics. - for (const auto& expected : metadata.expectedMetrics.memory) { - auto actual = metadata.metrics.memory.find(expected.first); - if (actual == metadata.metrics.memory.end()) { - metadata.errorMessage = "Failed to find memory probe: " + expected.first; - return false; - } - bool passed{false}; - float delta{0.0f}; - std::stringstream errorStream; - std::tie(passed, delta) = MemoryProbe::checkPeak(expected.second, actual->second); - if (!passed) { - errorStream << "Allocated memory peak size at probe \"" << expected.first << "\" is " << actual->second.peak - << " bytes, expected is " << expected.second.peak << "±" << delta << " bytes."; - } + // Check memory metrics. + for (const auto& expected : metadata.expectedMetrics.memory) { + auto actual = metadata.metrics.memory.find(expected.first); + if (actual == metadata.metrics.memory.end()) { + metadata.errorMessage = "Failed to find memory probe: " + expected.first; + return false; + } + bool passed{false}; + float delta{0.0f}; + std::stringstream errorStream; + std::tie(passed, delta) = MemoryProbe::checkPeak(expected.second, actual->second); + if (!passed) { + errorStream << "Allocated memory peak size at probe \"" << expected.first << "\" is " + << actual->second.peak << " bytes, expected is " << expected.second.peak << "±" << delta + << " bytes."; + } - std::tie(passed, delta) = MemoryProbe::checkAllocations(expected.second, actual->second); - if (!passed) { - errorStream << "Number of allocations at probe \"" << expected.first << "\" is " - << actual->second.allocations << ", expected is " << expected.second.allocations << "±" - << std::round(delta) << " allocations."; + std::tie(passed, delta) = MemoryProbe::checkAllocations(expected.second, actual->second); + if (!passed) { + errorStream << "Number of allocations at probe \"" << expected.first << "\" is " + << actual->second.allocations << ", expected is " << expected.second.allocations << "±" + << std::round(delta) << " allocations."; + } + + metadata.errorMessage = errorStream.str(); + if (!metadata.errorMessage.empty()) return false; } - metadata.errorMessage = errorStream.str(); - if (!metadata.errorMessage.empty()) return false; - } +#endif // !defined(SANITIZE) + return true; + }; // Check network metrics. - for (const auto& expected : metadata.expectedMetrics.network) { - auto actual = metadata.metrics.network.find(expected.first); - if (actual == metadata.metrics.network.end()) { - metadata.errorMessage = "Failed to find network probe: " + expected.first; - return false; - } - bool failed = false; - if (actual->second.requests != expected.second.requests) { - std::stringstream ss; - ss << "Number of requests at probe \"" << expected.first << "\" is " << actual->second.requests - << ", expected is " << expected.second.requests << ". "; - - metadata.errorMessage = ss.str(); - failed = true; - } - if (actual->second.transferred != expected.second.transferred) { - std::stringstream ss; - ss << "Transferred data at probe \"" << expected.first << "\" is " << actual->second.transferred - << " bytes, expected is " << expected.second.transferred << " bytes."; + auto checkNetwork = [](TestMetadata& metadata) -> bool { + if (metadata.metrics.network.empty()) return true; +#if !defined(SANITIZE) + for (const auto& expected : metadata.expectedMetrics.network) { + auto actual = metadata.metrics.network.find(expected.first); + if (actual == metadata.metrics.network.end()) { + metadata.errorMessage = "Failed to find network probe: " + expected.first; + return false; + } + bool failed = false; + if (actual->second.requests != expected.second.requests) { + std::stringstream ss; + ss << "Number of requests at probe \"" << expected.first << "\" is " << actual->second.requests + << ", expected is " << expected.second.requests << ". "; + + metadata.errorMessage = ss.str(); + failed = true; + } + if (actual->second.transferred != expected.second.transferred) { + std::stringstream ss; + ss << "Transferred data at probe \"" << expected.first << "\" is " << actual->second.transferred + << " bytes, expected is " << expected.second.transferred << " bytes."; - metadata.errorMessage += ss.str(); - failed = true; - } - if (failed) { - return false; + metadata.errorMessage += ss.str(); + failed = true; + } + if (failed) { + return false; + } } - } #endif // !defined(SANITIZE) + return true; + }; // Check fps metrics - for (const auto& expected : metadata.expectedMetrics.fps) { - auto actual = metadata.metrics.fps.find(expected.first); - if (actual == metadata.metrics.fps.end()) { - metadata.errorMessage = "Failed to find fps probe: " + expected.first; - return false; - } - auto result = checkValue(expected.second.average, actual->second.average, expected.second.tolerance); - if (!std::get(result)) { - std::stringstream ss; - ss << "Average fps at probe \"" << expected.first << "\" is " << actual->second.average - << ", expected to be " << expected.second.average << " with tolerance of " << expected.second.tolerance; - metadata.errorMessage = ss.str(); - return false; - } - result = checkValue(expected.second.minOnePc, actual->second.minOnePc, expected.second.tolerance); - if (!std::get(result)) { - std::stringstream ss; - ss << "Minimum(1%) fps at probe \"" << expected.first << "\" is " << actual->second.minOnePc - << ", expected to be " << expected.second.minOnePc << " with tolerance of " << expected.second.tolerance; - metadata.errorMessage = ss.str(); - return false; + auto checkFps = [](TestMetadata& metadata) -> bool { + if (metadata.metrics.fps.empty()) return true; + for (const auto& expected : metadata.expectedMetrics.fps) { + auto actual = metadata.metrics.fps.find(expected.first); + if (actual == metadata.metrics.fps.end()) { + metadata.errorMessage = "Failed to find fps probe: " + expected.first; + return false; + } + auto result = checkValue(expected.second.average, actual->second.average, expected.second.tolerance); + if (!std::get(result)) { + std::stringstream ss; + ss << "Average fps at probe \"" << expected.first << "\" is " << actual->second.average + << ", expected to be " << expected.second.average << " with tolerance of " + << expected.second.tolerance; + metadata.errorMessage = ss.str(); + return false; + } + result = checkValue(expected.second.minOnePc, actual->second.minOnePc, expected.second.tolerance); + if (!std::get(result)) { + std::stringstream ss; + ss << "Minimum(1%) fps at probe \"" << expected.first << "\" is " << actual->second.minOnePc + << ", expected to be " << expected.second.minOnePc << " with tolerance of " + << expected.second.tolerance; + metadata.errorMessage = ss.str(); + return false; + } } - } + return true; + }; // Check gfx metrics - for (const auto& expected : metadata.expectedMetrics.gfx) { - auto actual = metadata.metrics.gfx.find(expected.first); - if (actual == metadata.metrics.gfx.end()) { - metadata.errorMessage = "Failed to find gfx probe: " + expected.first; - return false; - } + auto checkGfx = [](TestMetadata& metadata) -> bool { + if (metadata.metrics.gfx.empty()) return true; + for (const auto& expected : metadata.expectedMetrics.gfx) { + auto actual = metadata.metrics.gfx.find(expected.first); + if (actual == metadata.metrics.gfx.end()) { + metadata.errorMessage = "Failed to find gfx probe: " + expected.first; + return false; + } - const auto& probeName = expected.first; - const auto& expectedValue = expected.second; - const auto& actualValue = actual->second; - bool failed = false; - - if (expectedValue.numDrawCalls != actualValue.numDrawCalls) { - std::stringstream ss; - if (!metadata.errorMessage.empty()) ss << std::endl; - ss << "Number of draw calls at probe\"" << probeName << "\" is " << actualValue.numDrawCalls - << ", expected is " << expectedValue.numDrawCalls; - metadata.errorMessage += ss.str(); - failed = true; - } + const auto& probeName = expected.first; + const auto& expectedValue = expected.second; + const auto& actualValue = actual->second; + bool failed = false; + + if (expectedValue.numDrawCalls != actualValue.numDrawCalls) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Number of draw calls at probe\"" << probeName << "\" is " << actualValue.numDrawCalls + << ", expected is " << expectedValue.numDrawCalls; + metadata.errorMessage += ss.str(); + failed = true; + } - if (expectedValue.numTextures != actualValue.numTextures) { - std::stringstream ss; - if (!metadata.errorMessage.empty()) ss << std::endl; - ss << "Number of textures at probe \"" << probeName << "\" is " << actualValue.numTextures - << ", expected is " << expectedValue.numTextures; - metadata.errorMessage += ss.str(); - failed = true; - } + if (expectedValue.numTextures != actualValue.numTextures) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Number of textures at probe \"" << probeName << "\" is " << actualValue.numTextures + << ", expected is " << expectedValue.numTextures; + metadata.errorMessage += ss.str(); + failed = true; + } - if (expectedValue.numBuffers != actualValue.numBuffers) { - std::stringstream ss; - if (!metadata.errorMessage.empty()) ss << std::endl; - ss << "Number of vertex and index buffers at probe \"" << probeName << "\" is " << actualValue.numBuffers - << ", expected is " << expectedValue.numBuffers; - metadata.errorMessage += ss.str(); - failed = true; - } + if (expectedValue.numBuffers != actualValue.numBuffers) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Number of vertex and index buffers at probe \"" << probeName << "\" is " + << actualValue.numBuffers << ", expected is " << expectedValue.numBuffers; + metadata.errorMessage += ss.str(); + failed = true; + } - if (expectedValue.numFrameBuffers != actualValue.numFrameBuffers) { - std::stringstream ss; - if (!metadata.errorMessage.empty()) ss << std::endl; - ss << "Number of frame buffers at probe \"" << probeName << "\" is " << actualValue.numFrameBuffers - << ", expected is " << expectedValue.numFrameBuffers; - metadata.errorMessage += ss.str(); - failed = true; - } + if (expectedValue.numFrameBuffers != actualValue.numFrameBuffers) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Number of frame buffers at probe \"" << probeName << "\" is " << actualValue.numFrameBuffers + << ", expected is " << expectedValue.numFrameBuffers; + metadata.errorMessage += ss.str(); + failed = true; + } - if (expectedValue.memTextures.peak != actualValue.memTextures.peak) { - std::stringstream ss; - if (!metadata.errorMessage.empty()) ss << std::endl; - ss << "Allocated texture memory peak size at probe \"" << probeName << "\" is " - << actualValue.memTextures.peak << " bytes, expected is " << expectedValue.memTextures.peak << " bytes"; - metadata.errorMessage += ss.str(); - failed = true; - } + if (expectedValue.memTextures.peak != actualValue.memTextures.peak) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Allocated texture memory peak size at probe \"" << probeName << "\" is " + << actualValue.memTextures.peak << " bytes, expected is " << expectedValue.memTextures.peak + << " bytes"; + metadata.errorMessage += ss.str(); + failed = true; + } - if (expectedValue.memIndexBuffers.peak != actualValue.memIndexBuffers.peak) { - std::stringstream ss; - if (!metadata.errorMessage.empty()) ss << std::endl; - ss << "Allocated index buffer memory peak size at probe \"" << probeName << "\" is " - << actualValue.memIndexBuffers.peak << " bytes, expected is " << expectedValue.memIndexBuffers.peak - << " bytes"; - metadata.errorMessage += ss.str(); - failed = true; - } + if (expectedValue.memIndexBuffers.peak != actualValue.memIndexBuffers.peak) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Allocated index buffer memory peak size at probe \"" << probeName << "\" is " + << actualValue.memIndexBuffers.peak << " bytes, expected is " << expectedValue.memIndexBuffers.peak + << " bytes"; + metadata.errorMessage += ss.str(); + failed = true; + } - if (expectedValue.memVertexBuffers.peak != actualValue.memVertexBuffers.peak) { - std::stringstream ss; - if (!metadata.errorMessage.empty()) ss << std::endl; - ss << "Allocated vertex buffer memory peak size at probe \"" << probeName << "\" is " - << actualValue.memVertexBuffers.peak << " bytes, expected is " << expectedValue.memVertexBuffers.peak - << " bytes"; - metadata.errorMessage += ss.str(); - failed = true; - } + if (expectedValue.memVertexBuffers.peak != actualValue.memVertexBuffers.peak) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Allocated vertex buffer memory peak size at probe \"" << probeName << "\" is " + << actualValue.memVertexBuffers.peak << " bytes, expected is " << expectedValue.memVertexBuffers.peak + << " bytes"; + metadata.errorMessage += ss.str(); + failed = true; + } - if (failed) return false; - } + if (failed) return false; + } + return true; + }; - return true; + return checkFileSize(metadata) && checkMemory(metadata) && checkNetwork(metadata) && checkFps(metadata) && + checkGfx(metadata); } bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata, RunContext& ctx) { @@ -1162,7 +1208,7 @@ bool TestRunner::run(TestMetadata& metadata, const std::set& inject } if (metadata.renderTest) { - return checkRenderTestResults(std::move(result.image), metadata); + return checkRenderTestResults(std::move(result.image), metadata) && checkProbingResults(metadata); } else { std::vector features; assert(metadata.document["metadata"]["test"]["queryGeometry"].IsArray()); diff --git a/render-test/runner.hpp b/render-test/runner.hpp index c31d672ba0..5dd529f057 100644 --- a/render-test/runner.hpp +++ b/render-test/runner.hpp @@ -31,6 +31,7 @@ private: std::vector&& features, TestMetadata&); bool checkRenderTestResults(mbgl::PremultipliedImage&& image, TestMetadata&); + bool checkProbingResults(TestMetadata&); struct Impl { Impl(const TestMetadata&); diff --git a/render-test/tests/mac/memory/pass-memory-size-is-same/expected.png b/render-test/tests/mac/memory/pass-memory-size-is-same/expected.png deleted file mode 100644 index 0858c19f05..0000000000 Binary files a/render-test/tests/mac/memory/pass-memory-size-is-same/expected.png and /dev/null differ diff --git a/render-test/tests/mac/memory/pass-memory-size-is-same/metrics.json b/render-test/tests/mac/memory/pass-memory-size-is-same/metrics.json deleted file mode 100644 index cc7d1c9fab..0000000000 --- a/render-test/tests/mac/memory/pass-memory-size-is-same/metrics.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "memory": [ - [ - "after setZoom 0.9", - 33271, - 51 - ], - [ - "end", - 47552, - 101 - ], - [ - "start", - 0, - 0 - ] - ] -} -- cgit v1.2.1