diff options
Diffstat (limited to 'render-test')
-rw-r--r-- | render-test/metadata.hpp | 31 | ||||
-rw-r--r-- | render-test/parser.cpp | 64 | ||||
-rw-r--r-- | render-test/runner.cpp | 145 | ||||
-rw-r--r-- | render-test/runner.hpp | 3 |
4 files changed, 237 insertions, 6 deletions
diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp index 996a2bc429..567c89e3fc 100644 --- a/render-test/metadata.hpp +++ b/render-test/metadata.hpp @@ -11,6 +11,12 @@ #include <map> +namespace mbgl { +namespace gfx { +struct RenderingStats; +} +} // namespace mbgl + struct TestStatistics { TestStatistics() = default; @@ -82,13 +88,36 @@ struct NetworkProbe { size_t transferred; }; +struct GfxProbe { + struct Memory { + Memory() = default; + Memory(int allocated_, int peak_) : allocated(allocated_), peak(peak_) {} + + int allocated; + int peak; + }; + + GfxProbe() = default; + GfxProbe(const mbgl::gfx::RenderingStats&, const GfxProbe&); + + int numDrawCalls; + int numTextures; + int numBuffers; + int numFrameBuffers; + + Memory memTextures; + Memory memIndexBuffers; + Memory memVertexBuffers; +}; + class TestMetrics { public: - bool isEmpty() const { return fileSize.empty() && memory.empty() && network.empty() && fps.empty(); } + bool isEmpty() const { return fileSize.empty() && memory.empty() && network.empty() && fps.empty() && gfx.empty(); } std::map<std::string, FileSizeProbe> fileSize; std::map<std::string, MemoryProbe> memory; std::map<std::string, NetworkProbe> network; std::map<std::string, FpsProbe> fps; + std::map<std::string, GfxProbe> gfx; }; struct TestMetadata { diff --git a/render-test/parser.cpp b/render-test/parser.cpp index 5a91fc7a58..b5d48d23a1 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -238,6 +238,36 @@ std::string serializeMetrics(const TestMetrics& metrics) { // End fps section } + if (!metrics.gfx.empty()) { + // Start gfx section + writer.Key("gfx"); + writer.StartArray(); + for (const auto& gfxProbe : metrics.gfx) { + assert(!gfxProbe.first.empty()); + writer.StartArray(); + writer.String(gfxProbe.first.c_str()); + writer.Int(gfxProbe.second.numDrawCalls); + writer.Int(gfxProbe.second.numTextures); + writer.Int(gfxProbe.second.numBuffers); + writer.Int(gfxProbe.second.numFrameBuffers); + writer.StartArray(); + writer.Int(gfxProbe.second.memTextures.allocated); + writer.Int(gfxProbe.second.memTextures.peak); + writer.EndArray(); + writer.StartArray(); + writer.Int(gfxProbe.second.memIndexBuffers.allocated); + writer.Int(gfxProbe.second.memIndexBuffers.peak); + writer.EndArray(); + writer.StartArray(); + writer.Int(gfxProbe.second.memVertexBuffers.allocated); + writer.Int(gfxProbe.second.memVertexBuffers.peak); + writer.EndArray(); + writer.EndArray(); + } + writer.EndArray(); + // End gfx section + } + writer.EndObject(); return s.GetString(); @@ -354,6 +384,40 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) { } } + if (document.HasMember("gfx")) { + const mbgl::JSValue& gfxValue = document["gfx"]; + assert(gfxValue.IsArray()); + for (auto& probeValue : gfxValue.GetArray()) { + assert(probeValue.IsArray()); + assert(probeValue.Size() >= 8u); + assert(probeValue[0].IsString()); + assert(probeValue[1].IsInt()); + assert(probeValue[2].IsInt()); + assert(probeValue[3].IsInt()); + assert(probeValue[4].IsInt()); + assert(probeValue[5].IsArray()); + assert(probeValue[6].IsArray()); + assert(probeValue[7].IsArray()); + + const std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()}; + assert(!mark.empty()); + + GfxProbe probe; + probe.numDrawCalls = probeValue[1].GetInt(); + probe.numTextures = probeValue[2].GetInt(); + probe.numBuffers = probeValue[3].GetInt(); + probe.numFrameBuffers = probeValue[4].GetInt(); + probe.memTextures.allocated = probeValue[5].GetArray()[0].GetInt(); + probe.memTextures.peak = probeValue[5].GetArray()[1].GetInt(); + probe.memIndexBuffers.allocated = probeValue[6].GetArray()[0].GetInt(); + probe.memIndexBuffers.peak = probeValue[6].GetArray()[1].GetInt(); + probe.memVertexBuffers.allocated = probeValue[7].GetArray()[0].GetInt(); + probe.memVertexBuffers.peak = probeValue[7].GetArray()[1].GetInt(); + + result.gfx.insert({mark, std::move(probe)}); + } + } + return result; } diff --git a/render-test/runner.cpp b/render-test/runner.cpp index e882e394be..8a4b0b3b0e 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -34,6 +34,23 @@ using namespace mbgl; +GfxProbe::GfxProbe(const mbgl::gfx::RenderingStats& stats, const GfxProbe& prev) + : numBuffers(stats.numBuffers), + numDrawCalls(stats.numDrawCalls), + numFrameBuffers(stats.numFrameBuffers), + numTextures(stats.numActiveTextures), + memIndexBuffers(stats.memIndexBuffers, std::max(stats.memIndexBuffers, prev.memIndexBuffers.peak)), + memVertexBuffers(stats.memVertexBuffers, std::max(stats.memVertexBuffers, prev.memVertexBuffers.peak)), + memTextures(stats.memTextures, std::max(stats.memTextures, prev.memTextures.peak)) {} + +struct RunContext { + RunContext() = default; + + GfxProbe activeGfxProbe; + GfxProbe baselineGfxProbe; + bool gfxProbeActive; +}; + class TestRunnerMapObserver : public MapObserver { public: TestRunnerMapObserver() : mapLoadFailure(false), finishRenderingMap(false), idle(false) {} @@ -391,10 +408,91 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage, return false; } } + // Check gfx metrics + for (const auto& expected : metadata.expectedMetrics.gfx) { + auto actual = metadata.metrics.gfx.find(expected.first); + if (actual == metadata.metrics.gfx.end()) { + metadata.errorMessage = "Failed to find gfx probe: " + expected.first; + return false; + } + + const auto& probeName = expected.first; + const auto& expectedValue = expected.second; + const auto& actualValue = actual->second; + bool failed = false; + + if (expectedValue.numDrawCalls != actualValue.numDrawCalls) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Number of draw calls at probe\"" << probeName << "\" is " << actualValue.numDrawCalls + << ", expected is " << expectedValue.numDrawCalls; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.numTextures != actualValue.numTextures) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Number of textures at probe \"" << probeName << "\" is " << actualValue.numTextures + << ", expected is " << expectedValue.numTextures; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.numBuffers != actualValue.numBuffers) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Number of vertex and index buffers at probe \"" << probeName << "\" is " << actualValue.numBuffers + << ", expected is " << expectedValue.numBuffers; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.numFrameBuffers != actualValue.numFrameBuffers) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Number of frame buffers at probe \"" << probeName << "\" is " << actualValue.numFrameBuffers + << ", expected is " << expectedValue.numFrameBuffers; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.memTextures.peak != actualValue.memTextures.peak) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Allocated texture memory peak size at probe \"" << probeName << "\" is " + << actualValue.memTextures.peak << " bytes, expected is " << expectedValue.memTextures.peak << " bytes"; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.memIndexBuffers.peak != actualValue.memIndexBuffers.peak) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Allocated index buffer memory peak size at probe \"" << probeName << "\" is " + << actualValue.memIndexBuffers.peak << " bytes, expected is " << expectedValue.memIndexBuffers.peak + << " bytes"; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (expectedValue.memVertexBuffers.peak != actualValue.memVertexBuffers.peak) { + std::stringstream ss; + if (!metadata.errorMessage.empty()) ss << std::endl; + ss << "Allocated vertex buffer memory peak size at probe \"" << probeName << "\" is " + << actualValue.memVertexBuffers.peak << " bytes, expected is " << expectedValue.memVertexBuffers.peak + << " bytes"; + metadata.errorMessage += ss.str(); + failed = true; + } + + if (failed) return false; + } + return true; } -bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { +bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata, RunContext& ctx) { if (!metadata.document.HasMember("metadata") || !metadata.document["metadata"].HasMember("test") || !metadata.document["metadata"]["test"].HasMember("operations")) { return true; @@ -446,6 +544,9 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { static const std::string getFeatureStateOp("getFeatureState"); static const std::string removeFeatureStateOp("removeFeatureState"); static const std::string panGestureOp("panGesture"); + static const std::string gfxProbeOp("probeGFX"); + static const std::string gfxProbeStartOp("probeGFXStart"); + static const std::string gfxProbeEndOp("probeGFXEnd"); if (operationArray[0].GetString() == waitOp) { // wait @@ -955,13 +1056,47 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { metadata.metrics.fps.insert({std::move(mark), {averageFps, minOnePcFps, 0.0f}}); + } else if (operationArray[0].GetString() == gfxProbeStartOp) { + // probeGFXStart + assert(!ctx.gfxProbeActive); + ctx.gfxProbeActive = true; + ctx.baselineGfxProbe = ctx.activeGfxProbe; + } else if (operationArray[0].GetString() == gfxProbeEndOp) { + // probeGFXEnd + assert(ctx.gfxProbeActive); + ctx.gfxProbeActive = false; + } else if (operationArray[0].GetString() == gfxProbeOp) { + // probeGFX + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsString()); + + std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); + + // Render the map and fetch rendering stats + gfx::RenderingStats stats; + + try { + stats = frontend.render(map).stats; + } catch (const std::exception&) { + return false; + } + + ctx.activeGfxProbe = GfxProbe(stats, ctx.activeGfxProbe); + + // Compare memory allocations to the baseline probe + GfxProbe metricProbe = ctx.activeGfxProbe; + metricProbe.memIndexBuffers.peak -= ctx.baselineGfxProbe.memIndexBuffers.peak; + metricProbe.memVertexBuffers.peak -= ctx.baselineGfxProbe.memVertexBuffers.peak; + metricProbe.memTextures.peak -= ctx.baselineGfxProbe.memTextures.peak; + metadata.metrics.gfx.insert({mark, metricProbe}); + } else { metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString(); return false; } operationsArray.Erase(operationIt); - return runOperations(key, metadata); + return runOperations(key, metadata, ctx); } TestRunner::Impl::Impl(const TestMetadata& metadata) @@ -1002,13 +1137,15 @@ bool TestRunner::run(TestMetadata& metadata) { map.getStyle().loadJSON(serializeJsonValue(metadata.document)); map.jumpTo(map.getStyle().getDefaultCamera()); - if (!runOperations(key, metadata)) { + RunContext ctx{}; + + if (!runOperations(key, metadata, ctx)) { return false; } mbgl::PremultipliedImage image; try { - if (metadata.outputsImage) image = frontend.render(map); + if (metadata.outputsImage) image = frontend.render(map).image; } catch (const std::exception&) { return false; } diff --git a/render-test/runner.hpp b/render-test/runner.hpp index bc97f8300b..4f80286c0b 100644 --- a/render-test/runner.hpp +++ b/render-test/runner.hpp @@ -8,6 +8,7 @@ #include <memory> #include <string> +struct RunContext; class TestRunnerMapObserver; struct TestMetadata; @@ -22,7 +23,7 @@ public: void doShuffle(uint32_t seed); private: - bool runOperations(const std::string& key, TestMetadata&); + bool runOperations(const std::string& key, TestMetadata&, RunContext&); bool checkQueryTestResults(mbgl::PremultipliedImage&& actualImage, std::vector<mbgl::Feature>&& features, TestMetadata&); |