diff options
Diffstat (limited to 'render-test/runner.cpp')
| -rw-r--r-- | render-test/runner.cpp | 762 |
1 files changed, 639 insertions, 123 deletions
diff --git a/render-test/runner.cpp b/render-test/runner.cpp index 5c8f53759f..8a4b0b3b0e 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -1,6 +1,7 @@ #include <mbgl/map/camera.hpp> #include <mbgl/map/map_observer.hpp> #include <mbgl/renderer/renderer.hpp> +#include <mbgl/renderer/renderer_observer.hpp> #include <mbgl/style/conversion/filter.hpp> #include <mbgl/style/conversion/layer.hpp> #include <mbgl/style/conversion/light.hpp> @@ -19,34 +20,117 @@ #include <mapbox/pixelmatch.hpp> +#include <../expression-test/test_runner_common.hpp> #include "allocation_index.hpp" +#include "file_source.hpp" #include "metadata.hpp" #include "parser.hpp" #include "runner.hpp" #include <algorithm> #include <cassert> -#include <regex> #include <utility> #include <sstream> -// static -const std::string& TestRunner::getBasePath() { - const static std::string result = - std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration/render-tests"); - return result; -} +using namespace mbgl; + +GfxProbe::GfxProbe(const mbgl::gfx::RenderingStats& stats, const GfxProbe& prev) + : numBuffers(stats.numBuffers), + numDrawCalls(stats.numDrawCalls), + numFrameBuffers(stats.numFrameBuffers), + numTextures(stats.numActiveTextures), + memIndexBuffers(stats.memIndexBuffers, std::max(stats.memIndexBuffers, prev.memIndexBuffers.peak)), + memVertexBuffers(stats.memVertexBuffers, std::max(stats.memVertexBuffers, prev.memVertexBuffers.peak)), + memTextures(stats.memTextures, std::max(stats.memTextures, prev.memTextures.peak)) {} + +struct RunContext { + RunContext() = default; + + GfxProbe activeGfxProbe; + GfxProbe baselineGfxProbe; + bool gfxProbeActive; +}; + +class TestRunnerMapObserver : public MapObserver { +public: + TestRunnerMapObserver() : mapLoadFailure(false), finishRenderingMap(false), idle(false) {} + + void onDidFailLoadingMap(MapLoadError, const std::string&) override { mapLoadFailure = true; } + + void onDidFinishRenderingMap(RenderMode mode) override final { + if (!finishRenderingMap) finishRenderingMap = mode == RenderMode::Full; + } + + void onDidBecomeIdle() override final { idle = true; } + + void reset() { + mapLoadFailure = false; + finishRenderingMap = false; + idle = false; + } + + bool mapLoadFailure; + bool finishRenderingMap; + bool idle; +}; // static -const std::vector<std::string>& TestRunner::getPlatformExpectationsPaths() { - // TODO: Populate from command line. - const static std::vector<std::string> result { - std::string(TEST_RUNNER_ROOT_PATH).append("/render-test/expected") - }; - return result; +gfx::HeadlessBackend::SwapBehaviour swapBehavior(MapMode mode) { + return mode == MapMode::Continuous ? gfx::HeadlessBackend::SwapBehaviour::Flush + : gfx::HeadlessBackend::SwapBehaviour::NoFlush; +} + +std::string simpleDiff(const Value& result, const Value& expected) { + std::vector<std::string> resultTokens{tokenize(toJSON(result, 2, false))}; + std::vector<std::string> expectedTokens{tokenize(toJSON(expected, 2, false))}; + std::size_t maxLength = std::max(resultTokens.size(), expectedTokens.size()); + std::ostringstream diff; + + diff << "<pre>" << std::endl; + const auto flush = + [](const std::vector<std::string>& vec, std::size_t pos, std::ostringstream& out, std::string separator) { + for (std::size_t j = pos; j < vec.size(); ++j) { + out << separator << vec[j] << std::endl; + } + }; + + for (std::size_t i = 0; i < maxLength; ++i) { + if (resultTokens.size() <= i) { + flush(expectedTokens, i, diff, "-"); + break; + } + + if (expectedTokens.size() <= i) { + flush(resultTokens, i, diff, "+"); + break; + } + + if (!deepEqual(resultTokens[i], expectedTokens[i])) { + diff << "<b>" + << "-" << expectedTokens[i] << "</b>" << std::endl; + diff << "<b>" + << "+" << resultTokens[i] << "</b>" << std::endl; + } else { + diff << resultTokens[i] << std::endl; + } + } + diff << "</pre>" << std::endl; + return diff.str(); +} + +TestRunner::TestRunner(Manifest manifest_) : manifest(std::move(manifest_)) {} + +const Manifest& TestRunner::getManifest() const { + return manifest; +} + +void TestRunner::doShuffle(uint32_t seed) { + manifest.doShuffle(seed); } -bool TestRunner::checkResults(mbgl::PremultipliedImage&& actualImage, TestMetadata& metadata) { +bool TestRunner::checkQueryTestResults(mbgl::PremultipliedImage&& actualImage, + std::vector<mbgl::Feature>&& features, + TestMetadata& metadata) { const std::string& base = metadata.paths.defaultExpectations(); const std::vector<mbgl::filesystem::path>& expectations = metadata.paths.expectations; @@ -57,74 +141,193 @@ bool TestRunner::checkResults(mbgl::PremultipliedImage&& actualImage, TestMetada return false; } + metadata.actualJson = toJSON(features, 2, false); + + if (metadata.actualJson.empty()) { + metadata.errorMessage = "Invalid size for actual JSON"; + return false; + } + #if !TEST_READ_ONLY if (getenv("UPDATE_PLATFORM")) { mbgl::filesystem::create_directories(expectations.back()); - mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actualImage)); + mbgl::util::write_file(expectations.back().string() + "/expected.json", metadata.actualJson); return true; } else if (getenv("UPDATE_DEFAULT")) { - mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actualImage)); + mbgl::util::write_file(base + "/expected.json", metadata.actualJson); return true; - } else if (getenv("UPDATE_METRICS")) { - if (!metadata.metrics.isEmpty()) { - mbgl::filesystem::create_directories(expectations.back()); - mbgl::util::write_file(expectations.back().string() + "/metrics.json", serializeMetrics(metadata.metrics)); - return true; - } } - mbgl::util::write_file(base + "/actual.png", metadata.actual); + mbgl::util::write_file(base + "/actual.json", metadata.actualJson); #endif - mbgl::PremultipliedImage expectedImage { actualImage.size }; - mbgl::PremultipliedImage imageDiff { actualImage.size }; - - double pixels = 0.0; - std::vector<std::string> expectedImagesPaths; + std::vector<std::string> expectedJsonPaths; mbgl::filesystem::path expectedMetricsPath; - for (auto rit = expectations.rbegin(); rit!= expectations.rend(); ++rit) { + for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) { if (mbgl::filesystem::exists(*rit)) { - if (metadata.expectedMetrics.isEmpty()) { - mbgl::filesystem::path maybeExpectedMetricsPath{ *rit }; - maybeExpectedMetricsPath.replace_filename("metrics.json"); - metadata.expectedMetrics = readExpectedMetrics(maybeExpectedMetricsPath); - } - expectedImagesPaths = readExpectedEntries(*rit); - if (!expectedImagesPaths.empty()) break; + expectedJsonPaths = readExpectedJSONEntries(*rit); + if (!expectedJsonPaths.empty()) break; } } - if (expectedImagesPaths.empty()) { + if (expectedJsonPaths.empty()) { metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string(); return false; } - - for (const auto& entry: expectedImagesPaths) { - mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry); - if (!maybeExpectedImage) { - metadata.errorMessage = "Failed to load expected image " + entry; + + for (const auto& entry : expectedJsonPaths) { + auto maybeExpectedJson = readJson(entry); + if (maybeExpectedJson.is<mbgl::JSDocument>()) { + auto& expected = maybeExpectedJson.get<mbgl::JSDocument>(); + + mbgl::JSDocument actual; + actual.Parse<0>(metadata.actualJson); + if (actual.HasParseError()) { + metadata.errorMessage = "Error parsing actual JSON for: " + metadata.paths.stylePath.string(); + return false; + } + + auto actualVal = mapbox::geojson::convert<mapbox::geojson::value>(actual); + auto expectedVal = mapbox::geojson::convert<mapbox::geojson::value>(expected); + bool equal = deepEqual(actualVal, expectedVal); + + metadata.difference = !equal; + if (equal) { + metadata.diff = "Match"; + } else { + metadata.diff = simpleDiff(actualVal, expectedVal); + } + } else { + metadata.errorMessage = "Failed to load expected JSON " + entry; + return false; + } + } + + return true; +} + +bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage, TestMetadata& metadata) { + const std::string& base = metadata.paths.defaultExpectations(); + const std::vector<mbgl::filesystem::path>& expectations = metadata.paths.expectations; + + if (metadata.outputsImage) { + metadata.actual = mbgl::encodePNG(actualImage); + + if (actualImage.size.isEmpty()) { + metadata.errorMessage = "Invalid size for actual image"; + return false; + } + +#if !TEST_READ_ONLY + if (getenv("UPDATE_PLATFORM")) { + mbgl::filesystem::create_directories(expectations.back()); + mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actualImage)); + return true; + } else if (getenv("UPDATE_DEFAULT")) { + mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actualImage)); + return true; + } + + mbgl::util::write_file(base + "/actual.png", metadata.actual); +#endif + + mbgl::PremultipliedImage expectedImage{actualImage.size}; + mbgl::PremultipliedImage imageDiff{actualImage.size}; + + double pixels = 0.0; + std::vector<std::string> expectedImagesPaths; + for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) { + if (mbgl::filesystem::exists(*rit)) { + expectedImagesPaths = readExpectedImageEntries(*rit); + if (!expectedImagesPaths.empty()) break; + } + } + + if (expectedImagesPaths.empty()) { + metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string(); return false; } - metadata.expected = *maybeExpectedImage; + for (const auto& entry : expectedImagesPaths) { + mbgl::optional<std::string> maybeExpectedImage = mbgl::util::readFile(entry); + if (!maybeExpectedImage) { + metadata.errorMessage = "Failed to load expected image " + entry; + return false; + } - expectedImage = mbgl::decodeImage(*maybeExpectedImage); + metadata.expected = *maybeExpectedImage; - pixels = // implicitly converting from uint64_t - mapbox::pixelmatch(actualImage.data.get(), expectedImage.data.get(), expectedImage.size.width, - expectedImage.size.height, imageDiff.data.get(), 0.1285); // Defined in GL JS + expectedImage = mbgl::decodeImage(*maybeExpectedImage); - metadata.diff = mbgl::encodePNG(imageDiff); + pixels = // implicitly converting from uint64_t + mapbox::pixelmatch(actualImage.data.get(), + expectedImage.data.get(), + expectedImage.size.width, + expectedImage.size.height, + imageDiff.data.get(), + 0.1285); // Defined in GL JS + + metadata.diff = mbgl::encodePNG(imageDiff); #if !TEST_READ_ONLY - mbgl::util::write_file(base + "/diff.png", metadata.diff); + mbgl::util::write_file(base + "/diff.png", metadata.diff); #endif - metadata.difference = pixels / expectedImage.size.area(); - if (metadata.difference <= metadata.allowed) { - break; + metadata.difference = pixels / expectedImage.size.area(); + if (metadata.difference <= metadata.allowed) { + break; + } + } + } + +#if !TEST_READ_ONLY + if (getenv("UPDATE_METRICS")) { + if (!metadata.metrics.isEmpty()) { + mbgl::filesystem::create_directories(expectations.back()); + mbgl::util::write_file(expectations.back().string() + "/metrics.json", serializeMetrics(metadata.metrics)); + return true; + } + } +#endif + + mbgl::filesystem::path expectedMetricsPath; + for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) { + if (mbgl::filesystem::exists(*rit)) { + if (metadata.expectedMetrics.isEmpty()) { + mbgl::filesystem::path maybeExpectedMetricsPath{*rit}; + maybeExpectedMetricsPath.replace_filename("metrics.json"); + metadata.expectedMetrics = readExpectedMetrics(maybeExpectedMetricsPath); + } + } + } + + // Check file size metrics. + for (const auto& expected : metadata.expectedMetrics.fileSize) { + auto actual = metadata.metrics.fileSize.find(expected.first); + if (actual == metadata.metrics.fileSize.end()) { + metadata.errorMessage = "Failed to find fileSize probe: " + expected.first; + return false; + } + if (actual->second.path != expected.second.path) { + std::stringstream ss; + ss << "Comparing different files at probe \"" << expected.first << "\": " << actual->second.path + << ", expected is " << expected.second.path << "."; + metadata.errorMessage = ss.str(); + + return false; + } + + auto result = checkValue(expected.second.size, actual->second.size, actual->second.tolerance); + if (!std::get<bool>(result)) { + std::stringstream ss; + ss << "File size does not match at probe \"" << expected.first << "\": " << actual->second.size + << ", expected is " << expected.second.size << "."; + + metadata.errorMessage = ss.str(); + return false; } } +#if !defined(SANITIZE) // Check memory metrics. for (const auto& expected : metadata.expectedMetrics.memory) { auto actual = metadata.metrics.memory.find(expected.first); @@ -132,35 +335,168 @@ bool TestRunner::checkResults(mbgl::PremultipliedImage&& actualImage, TestMetada metadata.errorMessage = "Failed to find memory probe: " + expected.first; return false; } - if (actual->second.peak > expected.second.peak) { - std::stringstream ss; - ss << "Allocated memory peak size at probe \"" << expected.first << "\" is " - << actual->second.peak << " bytes, expected is " << expected.second.peak << " bytes."; + bool passed{false}; + float delta{0.0f}; + std::stringstream errorStream; + std::tie(passed, delta) = MemoryProbe::checkPeak(expected.second, actual->second); + if (!passed) { + errorStream << "Allocated memory peak size at probe \"" << expected.first << "\" is " << actual->second.peak + << " bytes, expected is " << expected.second.peak << "±" << delta << " bytes."; + } - metadata.errorMessage = ss.str(); + std::tie(passed, delta) = MemoryProbe::checkAllocations(expected.second, actual->second); + if (!passed) { + errorStream << "Number of allocations at probe \"" << expected.first << "\" is " + << actual->second.allocations << ", expected is " << expected.second.allocations << "±" + << std::round(delta) << " allocations."; + } + + metadata.errorMessage = errorStream.str(); + if (!metadata.errorMessage.empty()) return false; + } + + // Check network metrics. + for (const auto& expected : metadata.expectedMetrics.network) { + auto actual = metadata.metrics.network.find(expected.first); + if (actual == metadata.metrics.network.end()) { + metadata.errorMessage = "Failed to find network probe: " + expected.first; return false; } + bool failed = false; + if (actual->second.requests != expected.second.requests) { + std::stringstream ss; + ss << "Number of requests at probe \"" << expected.first << "\" is " << actual->second.requests + << ", expected is " << expected.second.requests << ". "; - if (actual->second.allocations > expected.second.allocations) { + metadata.errorMessage = ss.str(); + failed = true; + } + if (actual->second.transferred != expected.second.transferred) { std::stringstream ss; - ss << "Number of allocations at probe \"" << expected.first << "\" is " - << actual->second.allocations << ", expected is " << expected.second.allocations << "."; + ss << "Transferred data at probe \"" << expected.first << "\" is " << actual->second.transferred + << " bytes, expected is " << expected.second.transferred << " bytes."; + metadata.errorMessage += ss.str(); + failed = true; + } + if (failed) { + return false; + } + } +#endif // !defined(SANITIZE) + // Check fps metrics + for (const auto& expected : metadata.expectedMetrics.fps) { + auto actual = metadata.metrics.fps.find(expected.first); + if (actual == metadata.metrics.fps.end()) { + metadata.errorMessage = "Failed to find fps probe: " + expected.first; + return false; + } + auto result = checkValue(expected.second.average, actual->second.average, expected.second.tolerance); + if (!std::get<bool>(result)) { + std::stringstream ss; + ss << "Average fps at probe \"" << expected.first << "\" is " << actual->second.average + << ", expected to be " << expected.second.average << " with tolerance of " << expected.second.tolerance; + metadata.errorMessage = ss.str(); + return false; + } + result = checkValue(expected.second.minOnePc, actual->second.minOnePc, expected.second.tolerance); + if (!std::get<bool>(result)) { + std::stringstream ss; + ss << "Minimum(1%) fps at probe \"" << expected.first << "\" is " << actual->second.minOnePc + << ", expected to be " << expected.second.minOnePc << " with tolerance of " << expected.second.tolerance; metadata.errorMessage = ss.str(); return false; } } + // Check gfx metrics + for (const auto& expected : metadata.expectedMetrics.gfx) { + auto actual = metadata.metrics.gfx.find(expected.first); + if (actual == metadata.metrics.gfx.end()) { + metadata.errorMessage = "Failed to find gfx probe: " + expected.first; + return false; + } + + const auto& probeName = expected.first; + const auto& expectedValue = expected.second; + const auto& actualValue = actual->second; + bool failed = false; + + if (expectedValue.numDrawCalls != actualValue.numDrawCalls) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Number of draw calls at probe\"" << probeName << "\" is " << actualValue.numDrawCalls + << ", expected is " << expectedValue.numDrawCalls; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.numTextures != actualValue.numTextures) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Number of textures at probe \"" << probeName << "\" is " << actualValue.numTextures + << ", expected is " << expectedValue.numTextures; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.numBuffers != actualValue.numBuffers) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Number of vertex and index buffers at probe \"" << probeName << "\" is " << actualValue.numBuffers + << ", expected is " << expectedValue.numBuffers; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.numFrameBuffers != actualValue.numFrameBuffers) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Number of frame buffers at probe \"" << probeName << "\" is " << actualValue.numFrameBuffers + << ", expected is " << expectedValue.numFrameBuffers; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.memTextures.peak != actualValue.memTextures.peak) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Allocated texture memory peak size at probe \"" << probeName << "\" is " + << actualValue.memTextures.peak << " bytes, expected is " << expectedValue.memTextures.peak << " bytes"; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.memIndexBuffers.peak != actualValue.memIndexBuffers.peak) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Allocated index buffer memory peak size at probe \"" << probeName << "\" is " + << actualValue.memIndexBuffers.peak << " bytes, expected is " << expectedValue.memIndexBuffers.peak + << " bytes"; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.memVertexBuffers.peak != actualValue.memVertexBuffers.peak) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Allocated vertex buffer memory peak size at probe \"" << probeName << "\" is " + << actualValue.memVertexBuffers.peak << " bytes, expected is " << expectedValue.memVertexBuffers.peak + << " bytes"; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (failed) return false; + } return true; } -bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { - if (!metadata.document.HasMember("metadata") || - !metadata.document["metadata"].HasMember("test") || +bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata, RunContext& ctx) { + if (!metadata.document.HasMember("metadata") || !metadata.document["metadata"].HasMember("test") || !metadata.document["metadata"]["test"].HasMember("operations")) { return true; } - assert(metadata.document["metadata"]["test"]["operations"].IsArray()); const auto& operationsArray = metadata.document["metadata"]["test"]["operations"].GetArray(); @@ -176,6 +512,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { auto& frontend = maps[key]->frontend; auto& map = maps[key]->map; + auto& observer = maps[key]->observer; static const std::string waitOp("wait"); static const std::string sleepOp("sleep"); @@ -186,6 +523,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { static const std::string setCenterOp("setCenter"); static const std::string setZoomOp("setZoom"); static const std::string setBearingOp("setBearing"); + static const std::string setPitchOp("setPitch"); static const std::string setFilterOp("setFilter"); static const std::string setLayerZoomRangeOp("setLayerZoomRange"); static const std::string setLightOp("setLight"); @@ -195,22 +533,30 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { static const std::string removeSourceOp("removeSource"); static const std::string setPaintPropertyOp("setPaintProperty"); static const std::string setLayoutPropertyOp("setLayoutProperty"); + static const std::string fileSizeProbeOp("probeFileSize"); static const std::string memoryProbeOp("probeMemory"); static const std::string memoryProbeStartOp("probeMemoryStart"); static const std::string memoryProbeEndOp("probeMemoryEnd"); + static const std::string networkProbeOp("probeNetwork"); + static const std::string networkProbeStartOp("probeNetworkStart"); + static const std::string networkProbeEndOp("probeNetworkEnd"); static const std::string setFeatureStateOp("setFeatureState"); static const std::string getFeatureStateOp("getFeatureState"); static const std::string removeFeatureStateOp("removeFeatureState"); + static const std::string panGestureOp("panGesture"); + static const std::string gfxProbeOp("probeGFX"); + static const std::string gfxProbeStartOp("probeGFXStart"); + static const std::string gfxProbeEndOp("probeGFXEnd"); - // wait if (operationArray[0].GetString() == waitOp) { + // wait try { frontend.render(map); } catch (const std::exception&) { return false; } - // sleep } else if (operationArray[0].GetString() == sleepOp) { + // sleep mbgl::util::Timer sleepTimer; bool sleeping = true; @@ -226,9 +572,8 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { while (sleeping) { mbgl::util::RunLoop::Get()->runOnce(); } - - // addImage | updateImage } else if (operationArray[0].GetString() == addImageOp || operationArray[0].GetString() == updateImageOp) { + // addImage | updateImage assert(operationArray.Size() >= 3u); float pixelRatio = 1.0f; @@ -251,7 +596,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { std::string imagePath = operationArray[2].GetString(); imagePath.erase(std::remove(imagePath.begin(), imagePath.end(), '"'), imagePath.end()); - const mbgl::filesystem::path filePath(std::string(TEST_RUNNER_ROOT_PATH) + "/mapbox-gl-js/test/integration/" + imagePath); + const mbgl::filesystem::path filePath = mbgl::filesystem::path(manifest.getTestRootPath()) / imagePath; mbgl::optional<std::string> maybeImage = mbgl::util::readFile(filePath.string()); if (!maybeImage) { @@ -260,33 +605,30 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { } map.getStyle().addImage(std::make_unique<mbgl::style::Image>(imageName, mbgl::decodeImage(*maybeImage), pixelRatio, sdf)); - - // removeImage } else if (operationArray[0].GetString() == removeImageOp) { + // removeImage assert(operationArray.Size() >= 2u); assert(operationArray[1].IsString()); const std::string imageName { operationArray[1].GetString(), operationArray[1].GetStringLength() }; map.getStyle().removeImage(imageName); - - // setStyle } else if (operationArray[0].GetString() == setStyleOp) { + // setStyle assert(operationArray.Size() >= 2u); if (operationArray[1].IsString()) { - std::string stylePath = localizeURL(operationArray[1].GetString()); + std::string stylePath = manifest.localizeURL(operationArray[1].GetString()); auto maybeStyle = readJson(stylePath); if (maybeStyle.is<mbgl::JSDocument>()) { auto& style = maybeStyle.get<mbgl::JSDocument>(); - localizeStyleURLs((mbgl::JSValue&)style, style); + manifest.localizeStyleURLs((mbgl::JSValue&)style, style); map.getStyle().loadJSON(serializeJsonValue(style)); } } else { - localizeStyleURLs(operationArray[1], metadata.document); + manifest.localizeStyleURLs(operationArray[1], metadata.document); map.getStyle().loadJSON(serializeJsonValue(operationArray[1])); } - - // setCenter } else if (operationArray[0].GetString() == setCenterOp) { + // setCenter assert(operationArray.Size() >= 2u); assert(operationArray[1].IsArray()); @@ -294,21 +636,23 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { assert(centerArray.Size() == 2u); map.jumpTo(mbgl::CameraOptions().withCenter(mbgl::LatLng(centerArray[1].GetDouble(), centerArray[0].GetDouble()))); - - // setZoom } else if (operationArray[0].GetString() == setZoomOp) { + // setZoom assert(operationArray.Size() >= 2u); assert(operationArray[1].IsNumber()); map.jumpTo(mbgl::CameraOptions().withZoom(operationArray[1].GetDouble())); - - // setBearing } else if (operationArray[0].GetString() == setBearingOp) { + // setBearing assert(operationArray.Size() >= 2u); assert(operationArray[1].IsNumber()); map.jumpTo(mbgl::CameraOptions().withBearing(operationArray[1].GetDouble())); - - // setFilter + } else if (operationArray[0].GetString() == setPitchOp) { + // setPitch + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsNumber()); + map.jumpTo(mbgl::CameraOptions().withPitch(operationArray[1].GetDouble())); } else if (operationArray[0].GetString() == setFilterOp) { + // setFilter assert(operationArray.Size() >= 3u); assert(operationArray[1].IsString()); @@ -328,9 +672,8 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { layer->setFilter(std::move(*converted)); } } - - // setLayerZoomRange } else if (operationArray[0].GetString() == setLayerZoomRangeOp) { + // setLayerZoomRange assert(operationArray.Size() >= 4u); assert(operationArray[1].IsString()); assert(operationArray[2].IsNumber()); @@ -345,9 +688,8 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { layer->setMinZoom(operationArray[2].GetFloat()); layer->setMaxZoom(operationArray[3].GetFloat()); } - - // setLight } else if (operationArray[0].GetString() == setLightOp) { + // setLight assert(operationArray.Size() >= 2u); assert(operationArray[1].IsObject()); @@ -359,9 +701,8 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { } else { map.getStyle().setLight(std::make_unique<mbgl::style::Light>(std::move(*converted))); } - - // addLayer } else if (operationArray[0].GetString() == addLayerOp) { + // addLayer assert(operationArray.Size() >= 2u); assert(operationArray[1].IsObject()); @@ -373,20 +714,18 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { } else { map.getStyle().addLayer(std::move(*converted)); } - - // removeLayer } else if (operationArray[0].GetString() == removeLayerOp) { + // removeLayer assert(operationArray.Size() >= 2u); assert(operationArray[1].IsString()); map.getStyle().removeLayer(operationArray[1].GetString()); - - // addSource } else if (operationArray[0].GetString() == addSourceOp) { + // addSource assert(operationArray.Size() >= 3u); assert(operationArray[1].IsString()); assert(operationArray[2].IsObject()); - localizeSourceURLs(operationArray[2], metadata.document); + manifest.localizeSourceURLs(operationArray[2], metadata.document); mbgl::style::conversion::Error error; auto converted = mbgl::style::conversion::convert<std::unique_ptr<mbgl::style::Source>>(operationArray[2], error, operationArray[1].GetString()); @@ -396,15 +735,13 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { } else { map.getStyle().addSource(std::move(*converted)); } - - // removeSource } else if (operationArray[0].GetString() == removeSourceOp) { + // removeSource assert(operationArray.Size() >= 2u); assert(operationArray[1].IsString()); map.getStyle().removeSource(operationArray[1].GetString()); - - // setPaintProperty } else if (operationArray[0].GetString() == setPaintPropertyOp) { + // setPaintProperty assert(operationArray.Size() >= 4u); assert(operationArray[1].IsString()); assert(operationArray[2].IsString()); @@ -420,9 +757,8 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { const mbgl::JSValue* propertyValue = &operationArray[3]; layer->setPaintProperty(propertyName, propertyValue); } - - // setLayoutProperty } else if (operationArray[0].GetString() == setLayoutPropertyOp) { + // setLayoutProperty assert(operationArray.Size() >= 4u); assert(operationArray[1].IsString()); assert(operationArray[2].IsString()); @@ -438,28 +774,79 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { const mbgl::JSValue* propertyValue = &operationArray[3]; layer->setLayoutProperty(propertyName, propertyValue); } - // probeMemoryStart + } else if (operationArray[0].GetString() == fileSizeProbeOp) { + // probeFileSize + assert(operationArray.Size() >= 4u); + assert(operationArray[1].IsString()); + assert(operationArray[2].IsString()); + assert(operationArray[3].IsNumber()); + + std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); + std::string path = std::string(operationArray[2].GetString(), operationArray[2].GetStringLength()); + assert(!path.empty()); + + float tolerance = operationArray[3].GetDouble(); + mbgl::filesystem::path filePath(path); + + if (!filePath.is_absolute()) { + filePath = metadata.paths.defaultExpectations() / filePath; + } + + if (mbgl::filesystem::exists(filePath)) { + auto size = mbgl::filesystem::file_size(filePath); + metadata.metrics.fileSize.emplace(std::piecewise_construct, + std::forward_as_tuple(std::move(mark)), + std::forward_as_tuple(std::move(path), size, tolerance)); + } else { + metadata.errorMessage = std::string("File not found: ") + path; + return false; + } } else if (operationArray[0].GetString() == memoryProbeStartOp) { + // probeMemoryStart assert(!AllocationIndex::isActive()); AllocationIndex::setActive(true); - // probeMemory } else if (operationArray[0].GetString() == memoryProbeOp) { + // probeMemory assert(AllocationIndex::isActive()); assert(operationArray.Size() >= 2u); assert(operationArray[1].IsString()); std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); - metadata.metrics.memory.emplace(std::piecewise_construct, - std::forward_as_tuple(std::move(mark)), - std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(), AllocationIndex::getAllocationsCount())); - // probeMemoryEnd + auto emplaced = metadata.metrics.memory.emplace( + std::piecewise_construct, + std::forward_as_tuple(std::move(mark)), + std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(), AllocationIndex::getAllocationsCount())); + assert(emplaced.second); + if (operationArray.Size() >= 3u) { + assert(operationArray[2].IsNumber()); + emplaced.first->second.tolerance = float(operationArray[2].GetDouble()); + } } else if (operationArray[0].GetString() == memoryProbeEndOp) { + // probeMemoryEnd assert(AllocationIndex::isActive()); AllocationIndex::setActive(false); AllocationIndex::reset(); + } else if (operationArray[0].GetString() == networkProbeStartOp) { + // probeNetworkStart + assert(!ProxyFileSource::isTrackingActive()); + ProxyFileSource::setTrackingActive(true); + } else if (operationArray[0].GetString() == networkProbeOp) { + // probeNetwork + assert(ProxyFileSource::isTrackingActive()); + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsString()); + std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); - // setFeatureState + metadata.metrics.network.emplace( + std::piecewise_construct, + std::forward_as_tuple(std::move(mark)), + std::forward_as_tuple(ProxyFileSource::getRequestCount(), ProxyFileSource::getTransferredSize())); + } else if (operationArray[0].GetString() == networkProbeEndOp) { + // probeNetworkEnd + assert(ProxyFileSource::isTrackingActive()); + ProxyFileSource::setTrackingActive(false); } else if (operationArray[0].GetString() == setFeatureStateOp) { + // setFeatureState assert(operationArray.Size() >= 3u); assert(operationArray[1].IsObject()); assert(operationArray[2].IsObject()); @@ -483,7 +870,11 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { sourceLayer = {featureOptions["sourceLayer"].GetString()}; } if (featureOptions.HasMember("id")) { - featureID = featureOptions["id"].GetString(); + if (featureOptions["id"].IsString()) { + featureID = featureOptions["id"].GetString(); + } else if (featureOptions["id"].IsNumber()) { + featureID = mbgl::util::toString(featureOptions["id"].GetUint64()); + } } const JSValue* state = &operationArray[2]; @@ -507,7 +898,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { result[k] = std::move(array); stateValue = std::move(result); valueParsed = true; - return {}; + return nullopt; } else if (isObject(v)) { eachMember(v, convertFn); @@ -530,9 +921,8 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { return false; } frontend.getRenderer()->setFeatureState(sourceID, sourceLayer, featureID, parsedState); - - // getFeatureState } else if (operationArray[0].GetString() == getFeatureStateOp) { + // getFeatureState assert(operationArray.Size() >= 2u); assert(operationArray[1].IsObject()); @@ -548,7 +938,11 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { sourceLayer = {featureOptions["sourceLayer"].GetString()}; } if (featureOptions.HasMember("id")) { - featureID = featureOptions["id"].GetString(); + if (featureOptions["id"].IsString()) { + featureID = featureOptions["id"].GetString(); + } else if (featureOptions["id"].IsNumber()) { + featureID = mbgl::util::toString(featureOptions["id"].GetUint64()); + } } try { @@ -558,9 +952,8 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { } mbgl::FeatureState state; frontend.getRenderer()->getFeatureState(state, sourceID, sourceLayer, featureID); - - // removeFeatureState } else if (operationArray[0].GetString() == removeFeatureStateOp) { + // removeFeatureState assert(operationArray.Size() >= 2u); assert(operationArray[1].IsObject()); @@ -577,7 +970,11 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { sourceLayer = {featureOptions["sourceLayer"].GetString()}; } if (featureOptions.HasMember("id")) { - featureID = featureOptions["id"].GetString(); + if (featureOptions["id"].IsString()) { + featureID = featureOptions["id"].GetString(); + } else if (featureOptions["id"].IsNumber()) { + featureID = mbgl::util::toString(featureOptions["id"].GetUint64()); + } } if (operationArray.Size() >= 3u) { @@ -591,20 +988,122 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { return false; } frontend.getRenderer()->removeFeatureState(sourceID, sourceLayer, featureID, stateKey); + } else if (operationArray[0].GetString() == panGestureOp) { + // benchmarkPanGesture + assert(operationArray.Size() >= 4u); + assert(operationArray[1].IsString()); // identifier + assert(operationArray[2].IsNumber()); // duration + assert(operationArray[3].IsArray()); // start [lat, lng, zoom] + assert(operationArray[4].IsArray()); // end [lat, lng, zoom] + + if (metadata.mapMode != mbgl::MapMode::Continuous) { + metadata.errorMessage = "Map mode must be Continous for " + panGestureOp + " operation"; + return false; + } + + std::string mark = operationArray[1].GetString(); + int duration = operationArray[2].GetFloat(); + LatLng startPos, endPos; + double startZoom, endZoom; + std::vector<float> samples; + + auto parsePosition = [](auto arr) -> std::tuple<LatLng, double> { + assert(arr.Size() >= 3); + return {{arr[1].GetDouble(), arr[0].GetDouble()}, arr[2].GetDouble()}; + }; + + std::tie(startPos, startZoom) = parsePosition(operationArray[3].GetArray()); + std::tie(endPos, endZoom) = parsePosition(operationArray[4].GetArray()); + + // Jump to the starting point of the segment and make sure there's something to render + map.jumpTo(mbgl::CameraOptions().withCenter(startPos).withZoom(startZoom)); + + observer->reset(); + while (!observer->finishRenderingMap) { + frontend.renderOnce(map); + } + + if (observer->mapLoadFailure) return false; + + size_t frames = 0; + float totalTime = 0.0; + bool transitionFinished = false; + + mbgl::AnimationOptions animationOptions(mbgl::Milliseconds(duration * 1000)); + animationOptions.minZoom = util::min(startZoom, endZoom); + animationOptions.transitionFinishFn = [&]() { transitionFinished = true; }; + + map.flyTo(mbgl::CameraOptions().withCenter(endPos).withZoom(endZoom), animationOptions); + + for (; !transitionFinished; frames++) { + frontend.renderOnce(map); + float frameTime = (float)frontend.getFrameTime(); + totalTime += frameTime; + + samples.push_back(frameTime); + } + + float averageFps = totalTime > 0.0 ? frames / totalTime : 0.0; + float minFrameTime = 0.0; + + // Use 1% of the longest frames to compute the minimum fps + std::sort(samples.begin(), samples.end()); + + int sampleCount = util::max(1, (int)samples.size() / 100); + for (auto it = samples.rbegin(); it != samples.rbegin() + sampleCount; it++) minFrameTime += *it; + + float minOnePcFps = sampleCount / minFrameTime; + + metadata.metrics.fps.insert({std::move(mark), {averageFps, minOnePcFps, 0.0f}}); + + } else if (operationArray[0].GetString() == gfxProbeStartOp) { + // probeGFXStart + assert(!ctx.gfxProbeActive); + ctx.gfxProbeActive = true; + ctx.baselineGfxProbe = ctx.activeGfxProbe; + } else if (operationArray[0].GetString() == gfxProbeEndOp) { + // probeGFXEnd + assert(ctx.gfxProbeActive); + ctx.gfxProbeActive = false; + } else if (operationArray[0].GetString() == gfxProbeOp) { + // probeGFX + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsString()); + + std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); + + // Render the map and fetch rendering stats + gfx::RenderingStats stats; + + try { + stats = frontend.render(map).stats; + } catch (const std::exception&) { + return false; + } + + ctx.activeGfxProbe = GfxProbe(stats, ctx.activeGfxProbe); + + // Compare memory allocations to the baseline probe + GfxProbe metricProbe = ctx.activeGfxProbe; + metricProbe.memIndexBuffers.peak -= ctx.baselineGfxProbe.memIndexBuffers.peak; + metricProbe.memVertexBuffers.peak -= ctx.baselineGfxProbe.memVertexBuffers.peak; + metricProbe.memTextures.peak -= ctx.baselineGfxProbe.memTextures.peak; + metadata.metrics.gfx.insert({mark, metricProbe}); } else { - metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString(); + metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString(); return false; } operationsArray.Erase(operationIt); - return runOperations(key, metadata); + return runOperations(key, metadata, ctx); } TestRunner::Impl::Impl(const TestMetadata& metadata) - : frontend(metadata.size, metadata.pixelRatio), + : observer(std::make_unique<TestRunnerMapObserver>()), + frontend(metadata.size, metadata.pixelRatio, swapBehavior(metadata.mapMode)), map(frontend, - mbgl::MapObserver::nullObserver(), + *observer.get(), mbgl::MapOptions() .withMapMode(metadata.mapMode) .withSize(metadata.size) @@ -612,9 +1111,12 @@ TestRunner::Impl::Impl(const TestMetadata& metadata) .withCrossSourceCollisions(metadata.crossSourceCollisions), mbgl::ResourceOptions().withCacheOnlyRequestsSupport(false)) {} +TestRunner::Impl::~Impl() {} + bool TestRunner::run(TestMetadata& metadata) { AllocationIndex::setActive(false); AllocationIndex::reset(); + ProxyFileSource::setTrackingActive(false); std::string key = mbgl::util::toString(uint32_t(metadata.mapMode)) + "/" + mbgl::util::toString(metadata.pixelRatio) + "/" + mbgl::util::toString(uint32_t(metadata.crossSourceCollisions)); @@ -635,18 +1137,32 @@ bool TestRunner::run(TestMetadata& metadata) { map.getStyle().loadJSON(serializeJsonValue(metadata.document)); map.jumpTo(map.getStyle().getDefaultCamera()); - if (!runOperations(key, metadata)) { + RunContext ctx{}; + + if (!runOperations(key, metadata, ctx)) { return false; } mbgl::PremultipliedImage image; try { - image = frontend.render(map); + if (metadata.outputsImage) image = frontend.render(map).image; } catch (const std::exception&) { return false; } - return checkResults(std::move(image), metadata); + if (metadata.renderTest) { + return checkRenderTestResults(std::move(image), metadata); + } else { + std::vector<mbgl::Feature> features; + assert(metadata.document["metadata"]["test"]["queryGeometry"].IsArray()); + if (metadata.document["metadata"]["test"]["queryGeometry"][0].IsNumber() && + metadata.document["metadata"]["test"]["queryGeometry"][1].IsNumber()) { + features = frontend.getRenderer()->queryRenderedFeatures(metadata.queryGeometry, metadata.queryOptions); + } else { + features = frontend.getRenderer()->queryRenderedFeatures(metadata.queryGeometryBox, metadata.queryOptions); + } + return checkQueryTestResults(std::move(image), std::move(features), metadata); + } } void TestRunner::reset() { |