diff options
author | Alexander Shalamov <alexander.shalamov@mapbox.com> | 2019-11-21 15:52:23 +0200 |
---|---|---|
committer | Alexander Shalamov <alexander.shalamov@mapbox.com> | 2019-11-22 15:29:25 +0200 |
commit | 17a544176807538c9e7a85048b81b014e97566e9 (patch) | |
tree | d8f4c267edeb93bb4c097d778dff81921b1df3e4 | |
parent | a02ed76d5bfcb2a4623eb3dd1256252ab2cf9281 (diff) | |
download | qtlocation-mapboxgl-17a544176807538c9e7a85048b81b014e97566e9.tar.gz |
[test-runner] Add render test runner options for injecting probes
This allows injection of memory, gfx and network probes before and
after operations defined in render test are run.
-rw-r--r-- | render-test/render_test.cpp | 24 | ||||
-rw-r--r-- | render-test/runner.cpp | 196 | ||||
-rw-r--r-- | render-test/runner.hpp | 5 |
3 files changed, 177 insertions, 48 deletions
diff --git a/render-test/render_test.cpp b/render-test/render_test.cpp index 5753065a7e..657e507d1a 100644 --- a/render-test/render_test.cpp +++ b/render-test/render_test.cpp @@ -40,10 +40,14 @@ void operator delete(void* ptr, size_t) noexcept { #endif namespace { -using ArgumentsTuple = std::tuple<bool, bool, uint32_t, std::string, std::vector<std::string>, std::string>; +using ArgumentsTuple = + std::tuple<bool, bool, uint32_t, std::string, std::vector<std::string>, std::string, std::set<std::string>>; ArgumentsTuple parseArguments(int argc, char** argv) { args::ArgumentParser argumentParser("Mapbox GL Test Runner"); + static const std::unordered_map<std::string, std::string> probeMap{ + {"memory", "probeMemory"}, {"network", "probeNetwork"}, {"gfx", "probeGFX"}}; + args::HelpFlag helpFlag(argumentParser, "help", "Display this help menu", {'h', "help"}); args::Flag recycleMapFlag(argumentParser, "recycle map", "Toggle reusing the map object", {'r', "recycle-map"}); @@ -53,6 +57,12 @@ ArgumentsTuple parseArguments(int argc, char** argv) { argumentParser, "manifestPath", "Test manifest file path", {'p', "manifestPath"}); args::ValueFlag<std::string> testFilterValue(argumentParser, "filter", "Test filter regex", {'f', "filter"}); args::PositionalList<std::string> testNameValues(argumentParser, "URL", "Test name(s)"); + args::MapFlagList<std::string, std::string> probes( + argumentParser, + "probe", + "Probe to be injected into a test. Supported values are: [memory, gfx, network]", + {"probe"}, + probeMap); try { argumentParser.ParseCLI(argc, argv); @@ -86,6 +96,9 @@ ArgumentsTuple parseArguments(int argc, char** argv) { exit(4); } + const auto& probeValues = args::get(probes); + std::set<std::string> injectedProbes(probeValues.begin(), probeValues.end()); + auto testNames = testNameValues ? args::get(testNameValues) : std::vector<std::string>{}; auto testFilter = testFilterValue ? args::get(testFilterValue) : std::string{}; const auto shuffle = shuffleFlag ? args::get(shuffleFlag) : false; @@ -95,7 +108,8 @@ ArgumentsTuple parseArguments(int argc, char** argv) { seed, manifestPath.string(), std::move(testNames), - std::move(testFilter)}; + std::move(testFilter), + std::move(injectedProbes)}; } } // namespace namespace mbgl { @@ -107,8 +121,10 @@ int runRenderTests(int argc, char** argv, std::function<void()> testStatus) { std::string manifestPath; std::vector<std::string> testNames; std::string testFilter; + std::set<std::string> injectedProbes; - std::tie(recycleMap, shuffle, seed, manifestPath, testNames, testFilter) = parseArguments(argc, argv); + std::tie(recycleMap, shuffle, seed, manifestPath, testNames, testFilter, injectedProbes) = + parseArguments(argc, argv); auto manifestData = ManifestParser::parseManifest(manifestPath, testNames, testFilter); if (!manifestData) { exit(5); @@ -159,7 +175,7 @@ int runRenderTests(int argc, char** argv, std::function<void()> testStatus) { bool errored = !metadata.errorMessage.empty(); if (!errored) { - errored = !runner.run(metadata) || !metadata.errorMessage.empty(); + errored = !runner.run(metadata, injectedProbes) || !metadata.errorMessage.empty(); } bool passed = diff --git a/render-test/runner.cpp b/render-test/runner.cpp index ce76bda157..004db87743 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -34,6 +34,42 @@ using namespace mbgl; +namespace { +static const std::string waitOp("wait"); +static const std::string sleepOp("sleep"); +static const std::string addImageOp("addImage"); +static const std::string updateImageOp("updateImage"); +static const std::string removeImageOp("removeImage"); +static const std::string setStyleOp("setStyle"); +static const std::string setCenterOp("setCenter"); +static const std::string setZoomOp("setZoom"); +static const std::string setBearingOp("setBearing"); +static const std::string setPitchOp("setPitch"); +static const std::string setFilterOp("setFilter"); +static const std::string setLayerZoomRangeOp("setLayerZoomRange"); +static const std::string setLightOp("setLight"); +static const std::string addLayerOp("addLayer"); +static const std::string removeLayerOp("removeLayer"); +static const std::string addSourceOp("addSource"); +static const std::string removeSourceOp("removeSource"); +static const std::string setPaintPropertyOp("setPaintProperty"); +static const std::string setLayoutPropertyOp("setLayoutProperty"); +static const std::string fileSizeProbeOp("probeFileSize"); +static const std::string memoryProbeOp("probeMemory"); +static const std::string memoryProbeStartOp("probeMemoryStart"); +static const std::string memoryProbeEndOp("probeMemoryEnd"); +static const std::string networkProbeOp("probeNetwork"); +static const std::string networkProbeStartOp("probeNetworkStart"); +static const std::string networkProbeEndOp("probeNetworkEnd"); +static const std::string setFeatureStateOp("setFeatureState"); +static const std::string getFeatureStateOp("getFeatureState"); +static const std::string removeFeatureStateOp("removeFeatureState"); +static const std::string panGestureOp("panGesture"); +static const std::string gfxProbeOp("probeGFX"); +static const std::string gfxProbeStartOp("probeGFXStart"); +static const std::string gfxProbeEndOp("probeGFXEnd"); +} // namespace + GfxProbe::GfxProbe(const mbgl::gfx::RenderingStats& stats, const GfxProbe& prev) : numBuffers(stats.numBuffers), numDrawCalls(stats.numDrawCalls), @@ -514,40 +550,6 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata, R auto& map = maps[key]->map; auto& observer = maps[key]->observer; - static const std::string waitOp("wait"); - static const std::string sleepOp("sleep"); - static const std::string addImageOp("addImage"); - static const std::string updateImageOp("updateImage"); - static const std::string removeImageOp("removeImage"); - static const std::string setStyleOp("setStyle"); - static const std::string setCenterOp("setCenter"); - static const std::string setZoomOp("setZoom"); - static const std::string setBearingOp("setBearing"); - static const std::string setPitchOp("setPitch"); - static const std::string setFilterOp("setFilter"); - static const std::string setLayerZoomRangeOp("setLayerZoomRange"); - static const std::string setLightOp("setLight"); - static const std::string addLayerOp("addLayer"); - static const std::string removeLayerOp("removeLayer"); - static const std::string addSourceOp("addSource"); - static const std::string removeSourceOp("removeSource"); - static const std::string setPaintPropertyOp("setPaintProperty"); - static const std::string setLayoutPropertyOp("setLayoutProperty"); - static const std::string fileSizeProbeOp("probeFileSize"); - static const std::string memoryProbeOp("probeMemory"); - static const std::string memoryProbeStartOp("probeMemoryStart"); - static const std::string memoryProbeEndOp("probeMemoryEnd"); - static const std::string networkProbeOp("probeNetwork"); - static const std::string networkProbeStartOp("probeNetworkStart"); - static const std::string networkProbeEndOp("probeNetworkEnd"); - static const std::string setFeatureStateOp("setFeatureState"); - static const std::string getFeatureStateOp("getFeatureState"); - static const std::string removeFeatureStateOp("removeFeatureState"); - static const std::string panGestureOp("panGesture"); - static const std::string gfxProbeOp("probeGFX"); - static const std::string gfxProbeStartOp("probeGFXStart"); - static const std::string gfxProbeEndOp("probeGFXEnd"); - if (operationArray[0].GetString() == waitOp) { // wait try { @@ -1114,7 +1116,7 @@ TestRunner::Impl::Impl(const TestMetadata& metadata) TestRunner::Impl::~Impl() {} -bool TestRunner::run(TestMetadata& metadata) { +bool TestRunner::run(TestMetadata& metadata, const std::set<std::string>& injectedProbes) { AllocationIndex::setActive(false); AllocationIndex::reset(); ProxyFileSource::setTrackingActive(false); @@ -1129,6 +1131,13 @@ bool TestRunner::run(TestMetadata& metadata) { auto& frontend = maps[key]->frontend; auto& map = maps[key]->map; + RunContext ctx{}; + + // Run 'begin' probes provided via command line arguments. + if (!runInjectedProbesBegin(metadata, injectedProbes, ctx)) { + return false; + } + frontend.setSize(metadata.size); map.setSize(metadata.size); @@ -1138,21 +1147,22 @@ bool TestRunner::run(TestMetadata& metadata) { map.getStyle().loadJSON(serializeJsonValue(metadata.document)); map.jumpTo(map.getStyle().getDefaultCamera()); - RunContext ctx{}; + if (!runOperations(key, metadata, ctx)) return false; - if (!runOperations(key, metadata, ctx)) { + HeadlessFrontend::RenderResult result; + try { + if (metadata.outputsImage) result = frontend.render(map); + } catch (const std::exception&) { return false; } - mbgl::PremultipliedImage image; - try { - if (metadata.outputsImage) image = frontend.render(map).image; - } catch (const std::exception&) { + // Run 'end' probes provided via command line arguments + if (!runInjectedProbesEnd(metadata, injectedProbes, ctx, result.stats)) { return false; } if (metadata.renderTest) { - return checkRenderTestResults(std::move(image), metadata); + return checkRenderTestResults(std::move(result.image), metadata); } else { std::vector<mbgl::Feature> features; assert(metadata.document["metadata"]["test"]["queryGeometry"].IsArray()); @@ -1162,8 +1172,108 @@ bool TestRunner::run(TestMetadata& metadata) { } else { features = frontend.getRenderer()->queryRenderedFeatures(metadata.queryGeometryBox, metadata.queryOptions); } - return checkQueryTestResults(std::move(image), std::move(features), metadata); + return checkQueryTestResults(std::move(result.image), std::move(features), metadata); + } +} + +using InjectedProbeMap = std::map<std::string, std::function<void(TestMetadata&, RunContext&)>>; +bool runInjectedProbe(TestMetadata& metadata, + const std::set<std::string>& probes, + RunContext& ctx, + const InjectedProbeMap& probeMap) { + for (const auto& probe : probes) { + auto it = probeMap.find(probe); + if (it == probeMap.end()) { + metadata.errorMessage = std::string("Unsupported operation: ") + probe; + return false; + } + it->second(metadata, ctx); } + return true; +} + +bool TestRunner::runInjectedProbesBegin(TestMetadata& metadata, const std::set<std::string>& probes, RunContext& ctx) { + const std::string mark = " - default - start"; + static const InjectedProbeMap beginInjectedProbeMap = { + {// Injected memory probe begin + memoryProbeOp, + [&mark](TestMetadata& metadata, RunContext&) { + assert(!AllocationIndex::isActive()); + AllocationIndex::setActive(true); + metadata.metrics.memory.emplace(std::piecewise_construct, + std::forward_as_tuple(memoryProbeOp + mark), + std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(), + AllocationIndex::getAllocationsCount())); + }}, + {// Injected gfx probe begin + gfxProbeOp, + [](TestMetadata&, RunContext& ctx) { + assert(!ctx.gfxProbeActive); + ctx.gfxProbeActive = true; + ctx.baselineGfxProbe = ctx.activeGfxProbe; + }}, + {// Injected network probe begin + networkProbeOp, + [&mark](TestMetadata& metadata, RunContext&) { + assert(!ProxyFileSource::isTrackingActive()); + ProxyFileSource::setTrackingActive(true); + metadata.metrics.network.emplace( + std::piecewise_construct, + std::forward_as_tuple(networkProbeOp + mark), + std::forward_as_tuple(ProxyFileSource::getRequestCount(), ProxyFileSource::getTransferredSize())); + }}}; + + return runInjectedProbe(metadata, probes, ctx, beginInjectedProbeMap); +} + +bool TestRunner::runInjectedProbesEnd(TestMetadata& metadata, + const std::set<std::string>& probes, + RunContext& ctx, + mbgl::gfx::RenderingStats stats) { + const std::string mark = " - default - end"; + static const InjectedProbeMap endInjectedProbeMap = { + {// Injected memory probe end + memoryProbeOp, + [&mark](TestMetadata& metadata, RunContext&) { + assert(AllocationIndex::isActive()); + auto emplaced = + metadata.metrics.memory.emplace(std::piecewise_construct, + std::forward_as_tuple(memoryProbeOp + mark), + std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(), + AllocationIndex::getAllocationsCount())); + assert(emplaced.second); + // TODO: Improve tolerance handling for memory tests. + emplaced.first->second.tolerance = 0.2f; + AllocationIndex::setActive(false); + AllocationIndex::reset(); + }}, + {// Injected gfx probe end + gfxProbeOp, + [&mark, &stats](TestMetadata& metadata, RunContext& ctx) { + assert(ctx.gfxProbeActive); + ctx.activeGfxProbe = GfxProbe(stats, ctx.activeGfxProbe); + + // Compare memory allocations to the baseline probe + GfxProbe metricProbe = ctx.activeGfxProbe; + metricProbe.memIndexBuffers.peak -= ctx.baselineGfxProbe.memIndexBuffers.peak; + metricProbe.memVertexBuffers.peak -= ctx.baselineGfxProbe.memVertexBuffers.peak; + metricProbe.memTextures.peak -= ctx.baselineGfxProbe.memTextures.peak; + metadata.metrics.gfx.insert({gfxProbeOp + mark, metricProbe}); + + ctx.gfxProbeActive = false; + }}, + {// Injected network probe end + networkProbeOp, + [&mark](TestMetadata& metadata, RunContext&) { + assert(ProxyFileSource::isTrackingActive()); + metadata.metrics.network.emplace( + std::piecewise_construct, + std::forward_as_tuple(networkProbeOp + mark), + std::forward_as_tuple(ProxyFileSource::getRequestCount(), ProxyFileSource::getTransferredSize())); + ProxyFileSource::setTrackingActive(false); + }}}; + + return runInjectedProbe(metadata, probes, ctx, endInjectedProbeMap); } void TestRunner::reset() { diff --git a/render-test/runner.hpp b/render-test/runner.hpp index 4f80286c0b..c31d672ba0 100644 --- a/render-test/runner.hpp +++ b/render-test/runner.hpp @@ -15,7 +15,7 @@ struct TestMetadata; class TestRunner { public: explicit TestRunner(Manifest); - bool run(TestMetadata&); + bool run(TestMetadata&, const std::set<std::string>&); void reset(); // Manifest @@ -24,6 +24,9 @@ public: private: bool runOperations(const std::string& key, TestMetadata&, RunContext&); + bool runInjectedProbesBegin(TestMetadata&, const std::set<std::string>&, RunContext&); + bool runInjectedProbesEnd(TestMetadata&, const std::set<std::string>&, RunContext&, mbgl::gfx::RenderingStats); + bool checkQueryTestResults(mbgl::PremultipliedImage&& actualImage, std::vector<mbgl::Feature>&& features, TestMetadata&); |