From a7a7e8cbe29e672d020db1d150e887f33d584a2d Mon Sep 17 00:00:00 2001 From: Mikko Pulkki Date: Tue, 15 Oct 2019 12:03:04 +0300 Subject: [render-test] Implement gfx probe for tracking gpu resources --- bin/render.cpp | 2 +- .../default/include/mbgl/gfx/headless_frontend.hpp | 8 +- .../default/src/mbgl/gfx/headless_frontend.cpp | 11 +- render-test/metadata.hpp | 31 ++++- render-test/parser.cpp | 64 +++++++++ render-test/runner.cpp | 145 ++++++++++++++++++++- render-test/runner.hpp | 3 +- test/api/annotations.test.cpp | 3 +- test/api/custom_geometry_source.test.cpp | 2 +- test/api/custom_layer.test.cpp | 2 +- test/api/recycle_map.cpp | 4 +- test/gl/context.test.cpp | 2 +- test/map/map.test.cpp | 20 +-- test/text/local_glyph_rasterizer.test.cpp | 4 +- 14 files changed, 267 insertions(+), 34 deletions(-) diff --git a/bin/render.cpp b/bin/render.cpp index 85231c8cba..409dfa7922 100644 --- a/bin/render.cpp +++ b/bin/render.cpp @@ -99,7 +99,7 @@ int main(int argc, char *argv[]) { try { std::ofstream out(output, std::ios::binary); - out << encodePNG(frontend.render(map)); + out << encodePNG(frontend.render(map).image); out.close(); } catch(std::exception& e) { std::cout << "Error: " << e.what() << std::endl; diff --git a/platform/default/include/mbgl/gfx/headless_frontend.hpp b/platform/default/include/mbgl/gfx/headless_frontend.hpp index 353452123d..8a98b4112d 100644 --- a/platform/default/include/mbgl/gfx/headless_frontend.hpp +++ b/platform/default/include/mbgl/gfx/headless_frontend.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -17,6 +18,11 @@ class TransformState; class HeadlessFrontend : public RendererFrontend { public: + struct RenderResult { + PremultipliedImage image; + gfx::RenderingStats stats; + }; + HeadlessFrontend(float pixelRatio_, gfx::HeadlessBackend::SwapBehaviour swapBehviour = gfx::HeadlessBackend::SwapBehaviour::NoFlush, gfx::ContextMode mode = gfx::ContextMode::Unique, @@ -48,7 +54,7 @@ public: LatLng latLngForPixel(const ScreenCoordinate&); PremultipliedImage readStillImage(); - PremultipliedImage render(Map&); + RenderResult render(Map&); void renderOnce(Map&); optional getTransformState() const; diff --git a/platform/default/src/mbgl/gfx/headless_frontend.cpp b/platform/default/src/mbgl/gfx/headless_frontend.cpp index 9e819f6653..5235b2f408 100644 --- a/platform/default/src/mbgl/gfx/headless_frontend.cpp +++ b/platform/default/src/mbgl/gfx/headless_frontend.cpp @@ -1,6 +1,6 @@ #include -#include #include +#include #include #include #include @@ -140,19 +140,20 @@ PremultipliedImage HeadlessFrontend::readStillImage() { return backend->readStillImage(); } -PremultipliedImage HeadlessFrontend::render(Map& map) { - PremultipliedImage result; +HeadlessFrontend::RenderResult HeadlessFrontend::render(Map& map) { + HeadlessFrontend::RenderResult result; std::exception_ptr error; map.renderStill([&](std::exception_ptr e) { if (e) { error = e; } else { - result = backend->readStillImage(); + result.image = backend->readStillImage(); + result.stats = getBackend()->getContext().renderingStats(); } }); - while (!result.valid() && !error) { + while (!result.image.valid() && !error) { util::RunLoop::Get()->runOnce(); } diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp index 996a2bc429..567c89e3fc 100644 --- a/render-test/metadata.hpp +++ b/render-test/metadata.hpp @@ -11,6 +11,12 @@ #include +namespace mbgl { +namespace gfx { +struct RenderingStats; +} +} // namespace mbgl + struct TestStatistics { TestStatistics() = default; @@ -82,13 +88,36 @@ struct NetworkProbe { size_t transferred; }; +struct GfxProbe { + struct Memory { + Memory() = default; + Memory(int allocated_, int peak_) : allocated(allocated_), peak(peak_) {} + + int allocated; + int peak; + }; + + GfxProbe() = default; + GfxProbe(const mbgl::gfx::RenderingStats&, const GfxProbe&); + + int numDrawCalls; + int numTextures; + int numBuffers; + int numFrameBuffers; + + Memory memTextures; + Memory memIndexBuffers; + Memory memVertexBuffers; +}; + class TestMetrics { public: - bool isEmpty() const { return fileSize.empty() && memory.empty() && network.empty() && fps.empty(); } + bool isEmpty() const { return fileSize.empty() && memory.empty() && network.empty() && fps.empty() && gfx.empty(); } std::map fileSize; std::map memory; std::map network; std::map fps; + std::map gfx; }; struct TestMetadata { diff --git a/render-test/parser.cpp b/render-test/parser.cpp index 5a91fc7a58..b5d48d23a1 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -238,6 +238,36 @@ std::string serializeMetrics(const TestMetrics& metrics) { // End fps section } + if (!metrics.gfx.empty()) { + // Start gfx section + writer.Key("gfx"); + writer.StartArray(); + for (const auto& gfxProbe : metrics.gfx) { + assert(!gfxProbe.first.empty()); + writer.StartArray(); + writer.String(gfxProbe.first.c_str()); + writer.Int(gfxProbe.second.numDrawCalls); + writer.Int(gfxProbe.second.numTextures); + writer.Int(gfxProbe.second.numBuffers); + writer.Int(gfxProbe.second.numFrameBuffers); + writer.StartArray(); + writer.Int(gfxProbe.second.memTextures.allocated); + writer.Int(gfxProbe.second.memTextures.peak); + writer.EndArray(); + writer.StartArray(); + writer.Int(gfxProbe.second.memIndexBuffers.allocated); + writer.Int(gfxProbe.second.memIndexBuffers.peak); + writer.EndArray(); + writer.StartArray(); + writer.Int(gfxProbe.second.memVertexBuffers.allocated); + writer.Int(gfxProbe.second.memVertexBuffers.peak); + writer.EndArray(); + writer.EndArray(); + } + writer.EndArray(); + // End gfx section + } + writer.EndObject(); return s.GetString(); @@ -354,6 +384,40 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) { } } + if (document.HasMember("gfx")) { + const mbgl::JSValue& gfxValue = document["gfx"]; + assert(gfxValue.IsArray()); + for (auto& probeValue : gfxValue.GetArray()) { + assert(probeValue.IsArray()); + assert(probeValue.Size() >= 8u); + assert(probeValue[0].IsString()); + assert(probeValue[1].IsInt()); + assert(probeValue[2].IsInt()); + assert(probeValue[3].IsInt()); + assert(probeValue[4].IsInt()); + assert(probeValue[5].IsArray()); + assert(probeValue[6].IsArray()); + assert(probeValue[7].IsArray()); + + const std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()}; + assert(!mark.empty()); + + GfxProbe probe; + probe.numDrawCalls = probeValue[1].GetInt(); + probe.numTextures = probeValue[2].GetInt(); + probe.numBuffers = probeValue[3].GetInt(); + probe.numFrameBuffers = probeValue[4].GetInt(); + probe.memTextures.allocated = probeValue[5].GetArray()[0].GetInt(); + probe.memTextures.peak = probeValue[5].GetArray()[1].GetInt(); + probe.memIndexBuffers.allocated = probeValue[6].GetArray()[0].GetInt(); + probe.memIndexBuffers.peak = probeValue[6].GetArray()[1].GetInt(); + probe.memVertexBuffers.allocated = probeValue[7].GetArray()[0].GetInt(); + probe.memVertexBuffers.peak = probeValue[7].GetArray()[1].GetInt(); + + result.gfx.insert({mark, std::move(probe)}); + } + } + return result; } diff --git a/render-test/runner.cpp b/render-test/runner.cpp index e882e394be..8a4b0b3b0e 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -34,6 +34,23 @@ using namespace mbgl; +GfxProbe::GfxProbe(const mbgl::gfx::RenderingStats& stats, const GfxProbe& prev) + : numBuffers(stats.numBuffers), + numDrawCalls(stats.numDrawCalls), + numFrameBuffers(stats.numFrameBuffers), + numTextures(stats.numActiveTextures), + memIndexBuffers(stats.memIndexBuffers, std::max(stats.memIndexBuffers, prev.memIndexBuffers.peak)), + memVertexBuffers(stats.memVertexBuffers, std::max(stats.memVertexBuffers, prev.memVertexBuffers.peak)), + memTextures(stats.memTextures, std::max(stats.memTextures, prev.memTextures.peak)) {} + +struct RunContext { + RunContext() = default; + + GfxProbe activeGfxProbe; + GfxProbe baselineGfxProbe; + bool gfxProbeActive; +}; + class TestRunnerMapObserver : public MapObserver { public: TestRunnerMapObserver() : mapLoadFailure(false), finishRenderingMap(false), idle(false) {} @@ -391,10 +408,91 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage, return false; } } + // Check gfx metrics + for (const auto& expected : metadata.expectedMetrics.gfx) { + auto actual = metadata.metrics.gfx.find(expected.first); + if (actual == metadata.metrics.gfx.end()) { + metadata.errorMessage = "Failed to find gfx probe: " + expected.first; + return false; + } + + const auto& probeName = expected.first; + const auto& expectedValue = expected.second; + const auto& actualValue = actual->second; + bool failed = false; + + if (expectedValue.numDrawCalls != actualValue.numDrawCalls) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Number of draw calls at probe\"" << probeName << "\" is " << actualValue.numDrawCalls + << ", expected is " << expectedValue.numDrawCalls; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.numTextures != actualValue.numTextures) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Number of textures at probe \"" << probeName << "\" is " << actualValue.numTextures + << ", expected is " << expectedValue.numTextures; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.numBuffers != actualValue.numBuffers) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Number of vertex and index buffers at probe \"" << probeName << "\" is " << actualValue.numBuffers + << ", expected is " << expectedValue.numBuffers; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.numFrameBuffers != actualValue.numFrameBuffers) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Number of frame buffers at probe \"" << probeName << "\" is " << actualValue.numFrameBuffers + << ", expected is " << expectedValue.numFrameBuffers; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.memTextures.peak != actualValue.memTextures.peak) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Allocated texture memory peak size at probe \"" << probeName << "\" is " + << actualValue.memTextures.peak << " bytes, expected is " << expectedValue.memTextures.peak << " bytes"; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.memIndexBuffers.peak != actualValue.memIndexBuffers.peak) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Allocated index buffer memory peak size at probe \"" << probeName << "\" is " + << actualValue.memIndexBuffers.peak << " bytes, expected is " << expectedValue.memIndexBuffers.peak + << " bytes"; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.memVertexBuffers.peak != actualValue.memVertexBuffers.peak) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Allocated vertex buffer memory peak size at probe \"" << probeName << "\" is " + << actualValue.memVertexBuffers.peak << " bytes, expected is " << expectedValue.memVertexBuffers.peak + << " bytes"; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (failed) return false; + } + return true; } -bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { +bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata, RunContext& ctx) { if (!metadata.document.HasMember("metadata") || !metadata.document["metadata"].HasMember("test") || !metadata.document["metadata"]["test"].HasMember("operations")) { return true; @@ -446,6 +544,9 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { static const std::string getFeatureStateOp("getFeatureState"); static const std::string removeFeatureStateOp("removeFeatureState"); static const std::string panGestureOp("panGesture"); + static const std::string gfxProbeOp("probeGFX"); + static const std::string gfxProbeStartOp("probeGFXStart"); + static const std::string gfxProbeEndOp("probeGFXEnd"); if (operationArray[0].GetString() == waitOp) { // wait @@ -955,13 +1056,47 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { metadata.metrics.fps.insert({std::move(mark), {averageFps, minOnePcFps, 0.0f}}); + } else if (operationArray[0].GetString() == gfxProbeStartOp) { + // probeGFXStart + assert(!ctx.gfxProbeActive); + ctx.gfxProbeActive = true; + ctx.baselineGfxProbe = ctx.activeGfxProbe; + } else if (operationArray[0].GetString() == gfxProbeEndOp) { + // probeGFXEnd + assert(ctx.gfxProbeActive); + ctx.gfxProbeActive = false; + } else if (operationArray[0].GetString() == gfxProbeOp) { + // probeGFX + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsString()); + + std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); + + // Render the map and fetch rendering stats + gfx::RenderingStats stats; + + try { + stats = frontend.render(map).stats; + } catch (const std::exception&) { + return false; + } + + ctx.activeGfxProbe = GfxProbe(stats, ctx.activeGfxProbe); + + // Compare memory allocations to the baseline probe + GfxProbe metricProbe = ctx.activeGfxProbe; + metricProbe.memIndexBuffers.peak -= ctx.baselineGfxProbe.memIndexBuffers.peak; + metricProbe.memVertexBuffers.peak -= ctx.baselineGfxProbe.memVertexBuffers.peak; + metricProbe.memTextures.peak -= ctx.baselineGfxProbe.memTextures.peak; + metadata.metrics.gfx.insert({mark, metricProbe}); + } else { metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString(); return false; } operationsArray.Erase(operationIt); - return runOperations(key, metadata); + return runOperations(key, metadata, ctx); } TestRunner::Impl::Impl(const TestMetadata& metadata) @@ -1002,13 +1137,15 @@ bool TestRunner::run(TestMetadata& metadata) { map.getStyle().loadJSON(serializeJsonValue(metadata.document)); map.jumpTo(map.getStyle().getDefaultCamera()); - if (!runOperations(key, metadata)) { + RunContext ctx{}; + + if (!runOperations(key, metadata, ctx)) { return false; } mbgl::PremultipliedImage image; try { - if (metadata.outputsImage) image = frontend.render(map); + if (metadata.outputsImage) image = frontend.render(map).image; } catch (const std::exception&) { return false; } diff --git a/render-test/runner.hpp b/render-test/runner.hpp index bc97f8300b..4f80286c0b 100644 --- a/render-test/runner.hpp +++ b/render-test/runner.hpp @@ -8,6 +8,7 @@ #include #include +struct RunContext; class TestRunnerMapObserver; struct TestMetadata; @@ -22,7 +23,7 @@ public: void doShuffle(uint32_t seed); private: - bool runOperations(const std::string& key, TestMetadata&); + bool runOperations(const std::string& key, TestMetadata&, RunContext&); bool checkQueryTestResults(mbgl::PremultipliedImage&& actualImage, std::vector&& features, TestMetadata&); diff --git a/test/api/annotations.test.cpp b/test/api/annotations.test.cpp index 03330dc4c6..4886d90a93 100644 --- a/test/api/annotations.test.cpp +++ b/test/api/annotations.test.cpp @@ -33,8 +33,7 @@ public: MapOptions().withMapMode(MapMode::Static).withSize(frontend.getSize())}; void checkRendering(const char * name) { - test::checkImage(std::string("test/fixtures/annotations/") + name, - frontend.render(map), 0.0002, 0.1); + test::checkImage(std::string("test/fixtures/annotations/") + name, frontend.render(map).image, 0.0002, 0.1); } }; diff --git a/test/api/custom_geometry_source.test.cpp b/test/api/custom_geometry_source.test.cpp index f35b4d335c..93cf6ba56e 100644 --- a/test/api/custom_geometry_source.test.cpp +++ b/test/api/custom_geometry_source.test.cpp @@ -64,5 +64,5 @@ TEST(CustomGeometrySource, Grid) { layer->setLineColor(Color{ 1.0, 1.0, 1.0, 1.0 }); map.getStyle().addLayer(std::move(layer)); - test::checkImage("test/fixtures/custom_geometry_source/grid", frontend.render(map), 0.0006, 0.1); + test::checkImage("test/fixtures/custom_geometry_source/grid", frontend.render(map).image, 0.0006, 0.1); } diff --git a/test/api/custom_layer.test.cpp b/test/api/custom_layer.test.cpp index 175d053f93..2222645147 100644 --- a/test/api/custom_layer.test.cpp +++ b/test/api/custom_layer.test.cpp @@ -108,5 +108,5 @@ TEST(CustomLayer, Basic) { layer->setFillColor(Color{ 1.0, 1.0, 0.0, 1.0 }); map.getStyle().addLayer(std::move(layer)); - test::checkImage("test/fixtures/custom_layer/basic", frontend.render(map), 0.0006, 0.1); + test::checkImage("test/fixtures/custom_layer/basic", frontend.render(map).image, 0.0006, 0.1); } diff --git a/test/api/recycle_map.cpp b/test/api/recycle_map.cpp index b3c573b1a2..be7bb77b4c 100644 --- a/test/api/recycle_map.cpp +++ b/test/api/recycle_map.cpp @@ -45,10 +45,10 @@ TEST(API, RecycleMapUpdateImages) { // default marker loadStyle("default_marker", "test/fixtures/sprites/default_marker.png"); - test::checkImage("test/fixtures/recycle_map/default_marker", frontend.render(*map), 0.0006, 0.1); + test::checkImage("test/fixtures/recycle_map/default_marker", frontend.render(*map).image, 0.0006, 0.1); // flipped marker loadStyle("flipped_marker", "test/fixtures/sprites/flipped_marker.png"); - test::checkImage("test/fixtures/recycle_map/flipped_marker", frontend.render(*map), 0.0006, 0.1); + test::checkImage("test/fixtures/recycle_map/flipped_marker", frontend.render(*map).image, 0.0006, 0.1); } diff --git a/test/gl/context.test.cpp b/test/gl/context.test.cpp index 5c42eb9344..9c709b7597 100644 --- a/test/gl/context.test.cpp +++ b/test/gl/context.test.cpp @@ -119,5 +119,5 @@ TEST(GLContextMode, Shared) { MBGL_CHECK_ERROR(glDrawArrays(GL_TRIANGLE_STRIP, 0, 3)); } - test::checkImage("test/fixtures/shared_context", frontend.render(map), 0.5, 0.1); + test::checkImage("test/fixtures/shared_context", frontend.render(map).image, 0.5, 0.1); } diff --git a/test/map/map.test.cpp b/test/map/map.test.cpp index 6f4b14a681..0eebc93f32 100644 --- a/test/map/map.test.cpp +++ b/test/map/map.test.cpp @@ -259,10 +259,7 @@ TEST(Map, Offline) { test.map.getStyle().loadURL(prefix + "style.json"); - test::checkImage("test/fixtures/map/offline", - test.frontend.render(test.map), - 0.0015, - 0.1); + test::checkImage("test/fixtures/map/offline", test.frontend.render(test.map).image, 0.0015, 0.1); NetworkStatus::Set(NetworkStatus::Status::Online); } @@ -608,7 +605,7 @@ TEST(Map, AddLayer) { layer->setBackgroundColor({ { 1, 0, 0, 1 } }); test.map.getStyle().addLayer(std::move(layer)); - test::checkImage("test/fixtures/map/add_layer", test.frontend.render(test.map)); + test::checkImage("test/fixtures/map/add_layer", test.frontend.render(test.map).image); } TEST(Map, WithoutVAOExtension) { @@ -623,7 +620,7 @@ TEST(Map, WithoutVAOExtension) { test.map.getStyle().loadJSON(util::read_file("test/fixtures/api/water.json")); - test::checkImage("test/fixtures/map/no_vao", test.frontend.render(test.map), 0.002); + test::checkImage("test/fixtures/map/no_vao", test.frontend.render(test.map).image, 0.002); } TEST(Map, RemoveLayer) { @@ -636,7 +633,7 @@ TEST(Map, RemoveLayer) { test.map.getStyle().addLayer(std::move(layer)); test.map.getStyle().removeLayer("background"); - test::checkImage("test/fixtures/map/remove_layer", test.frontend.render(test.map)); + test::checkImage("test/fixtures/map/remove_layer", test.frontend.render(test.map).image); } TEST(Map, DisabledSources) { @@ -694,9 +691,9 @@ TEST(Map, DisabledSources) { } )STYLE"); - test::checkImage("test/fixtures/map/disabled_layers/first", test.frontend.render(test.map)); + test::checkImage("test/fixtures/map/disabled_layers/first", test.frontend.render(test.map).image); test.map.jumpTo(CameraOptions().withZoom(0.5)); - test::checkImage("test/fixtures/map/disabled_layers/second", test.frontend.render(test.map)); + test::checkImage("test/fixtures/map/disabled_layers/second", test.frontend.render(test.map).image); } TEST(Map, DontLoadUnneededTiles) { @@ -815,10 +812,7 @@ TEST(Map, NoContentTiles) { }] })STYLE"); - test::checkImage("test/fixtures/map/nocontent", - test.frontend.render(test.map), - 0.0015, - 0.1); + test::checkImage("test/fixtures/map/nocontent", test.frontend.render(test.map).image, 0.0015, 0.1); } // https://github.com/mapbox/mapbox-gl-native/issues/12432 diff --git a/test/text/local_glyph_rasterizer.test.cpp b/test/text/local_glyph_rasterizer.test.cpp index 94c37b170b..d109b28f32 100644 --- a/test/text/local_glyph_rasterizer.test.cpp +++ b/test/text/local_glyph_rasterizer.test.cpp @@ -43,7 +43,9 @@ public: void checkRendering(const char * name, double imageMatchPixelsThreshold = 0.015, double pixelMatchThreshold = 0.1) { test::checkImage(std::string("test/fixtures/local_glyphs/") + name, - frontend.render(map), imageMatchPixelsThreshold, pixelMatchThreshold); + frontend.render(map).image, + imageMatchPixelsThreshold, + pixelMatchThreshold); } }; -- cgit v1.2.1