summaryrefslogtreecommitdiff
path: root/render-test
diff options
context:
space:
mode:
Diffstat (limited to 'render-test')
-rw-r--r--render-test/metadata.hpp31
-rw-r--r--render-test/parser.cpp64
-rw-r--r--render-test/runner.cpp145
-rw-r--r--render-test/runner.hpp3
4 files changed, 237 insertions, 6 deletions
diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp
index 996a2bc429..567c89e3fc 100644
--- a/render-test/metadata.hpp
+++ b/render-test/metadata.hpp
@@ -11,6 +11,12 @@
#include <map>
+namespace mbgl {
+namespace gfx {
+struct RenderingStats;
+}
+} // namespace mbgl
+
struct TestStatistics {
TestStatistics() = default;
@@ -82,13 +88,36 @@ struct NetworkProbe {
size_t transferred;
};
+struct GfxProbe {
+ struct Memory {
+ Memory() = default;
+ Memory(int allocated_, int peak_) : allocated(allocated_), peak(peak_) {}
+
+ int allocated;
+ int peak;
+ };
+
+ GfxProbe() = default;
+ GfxProbe(const mbgl::gfx::RenderingStats&, const GfxProbe&);
+
+ int numDrawCalls;
+ int numTextures;
+ int numBuffers;
+ int numFrameBuffers;
+
+ Memory memTextures;
+ Memory memIndexBuffers;
+ Memory memVertexBuffers;
+};
+
class TestMetrics {
public:
- bool isEmpty() const { return fileSize.empty() && memory.empty() && network.empty() && fps.empty(); }
+ bool isEmpty() const { return fileSize.empty() && memory.empty() && network.empty() && fps.empty() && gfx.empty(); }
std::map<std::string, FileSizeProbe> fileSize;
std::map<std::string, MemoryProbe> memory;
std::map<std::string, NetworkProbe> network;
std::map<std::string, FpsProbe> fps;
+ std::map<std::string, GfxProbe> gfx;
};
struct TestMetadata {
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index 5a91fc7a58..b5d48d23a1 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -238,6 +238,36 @@ std::string serializeMetrics(const TestMetrics& metrics) {
// End fps section
}
+ if (!metrics.gfx.empty()) {
+ // Start gfx section
+ writer.Key("gfx");
+ writer.StartArray();
+ for (const auto& gfxProbe : metrics.gfx) {
+ assert(!gfxProbe.first.empty());
+ writer.StartArray();
+ writer.String(gfxProbe.first.c_str());
+ writer.Int(gfxProbe.second.numDrawCalls);
+ writer.Int(gfxProbe.second.numTextures);
+ writer.Int(gfxProbe.second.numBuffers);
+ writer.Int(gfxProbe.second.numFrameBuffers);
+ writer.StartArray();
+ writer.Int(gfxProbe.second.memTextures.allocated);
+ writer.Int(gfxProbe.second.memTextures.peak);
+ writer.EndArray();
+ writer.StartArray();
+ writer.Int(gfxProbe.second.memIndexBuffers.allocated);
+ writer.Int(gfxProbe.second.memIndexBuffers.peak);
+ writer.EndArray();
+ writer.StartArray();
+ writer.Int(gfxProbe.second.memVertexBuffers.allocated);
+ writer.Int(gfxProbe.second.memVertexBuffers.peak);
+ writer.EndArray();
+ writer.EndArray();
+ }
+ writer.EndArray();
+ // End gfx section
+ }
+
writer.EndObject();
return s.GetString();
@@ -354,6 +384,40 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
}
}
+ if (document.HasMember("gfx")) {
+ const mbgl::JSValue& gfxValue = document["gfx"];
+ assert(gfxValue.IsArray());
+ for (auto& probeValue : gfxValue.GetArray()) {
+ assert(probeValue.IsArray());
+ assert(probeValue.Size() >= 8u);
+ assert(probeValue[0].IsString());
+ assert(probeValue[1].IsInt());
+ assert(probeValue[2].IsInt());
+ assert(probeValue[3].IsInt());
+ assert(probeValue[4].IsInt());
+ assert(probeValue[5].IsArray());
+ assert(probeValue[6].IsArray());
+ assert(probeValue[7].IsArray());
+
+ const std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()};
+ assert(!mark.empty());
+
+ GfxProbe probe;
+ probe.numDrawCalls = probeValue[1].GetInt();
+ probe.numTextures = probeValue[2].GetInt();
+ probe.numBuffers = probeValue[3].GetInt();
+ probe.numFrameBuffers = probeValue[4].GetInt();
+ probe.memTextures.allocated = probeValue[5].GetArray()[0].GetInt();
+ probe.memTextures.peak = probeValue[5].GetArray()[1].GetInt();
+ probe.memIndexBuffers.allocated = probeValue[6].GetArray()[0].GetInt();
+ probe.memIndexBuffers.peak = probeValue[6].GetArray()[1].GetInt();
+ probe.memVertexBuffers.allocated = probeValue[7].GetArray()[0].GetInt();
+ probe.memVertexBuffers.peak = probeValue[7].GetArray()[1].GetInt();
+
+ result.gfx.insert({mark, std::move(probe)});
+ }
+ }
+
return result;
}
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index e882e394be..8a4b0b3b0e 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -34,6 +34,23 @@
using namespace mbgl;
+GfxProbe::GfxProbe(const mbgl::gfx::RenderingStats& stats, const GfxProbe& prev)
+ : numBuffers(stats.numBuffers),
+ numDrawCalls(stats.numDrawCalls),
+ numFrameBuffers(stats.numFrameBuffers),
+ numTextures(stats.numActiveTextures),
+ memIndexBuffers(stats.memIndexBuffers, std::max(stats.memIndexBuffers, prev.memIndexBuffers.peak)),
+ memVertexBuffers(stats.memVertexBuffers, std::max(stats.memVertexBuffers, prev.memVertexBuffers.peak)),
+ memTextures(stats.memTextures, std::max(stats.memTextures, prev.memTextures.peak)) {}
+
+struct RunContext {
+ RunContext() = default;
+
+ GfxProbe activeGfxProbe;
+ GfxProbe baselineGfxProbe;
+ bool gfxProbeActive;
+};
+
class TestRunnerMapObserver : public MapObserver {
public:
TestRunnerMapObserver() : mapLoadFailure(false), finishRenderingMap(false), idle(false) {}
@@ -391,10 +408,91 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage,
return false;
}
}
+ // Check gfx metrics
+ for (const auto& expected : metadata.expectedMetrics.gfx) {
+ auto actual = metadata.metrics.gfx.find(expected.first);
+ if (actual == metadata.metrics.gfx.end()) {
+ metadata.errorMessage = "Failed to find gfx probe: " + expected.first;
+ return false;
+ }
+
+ const auto& probeName = expected.first;
+ const auto& expectedValue = expected.second;
+ const auto& actualValue = actual->second;
+ bool failed = false;
+
+ if (expectedValue.numDrawCalls != actualValue.numDrawCalls) {
+ std::stringstream ss;
+ if (!metadata.errorMessage.empty()) ss << std::endl;
+ ss << "Number of draw calls at probe\"" << probeName << "\" is " << actualValue.numDrawCalls
+ << ", expected is " << expectedValue.numDrawCalls;
+ metadata.errorMessage += ss.str();
+ failed = true;
+ }
+
+ if (expectedValue.numTextures != actualValue.numTextures) {
+ std::stringstream ss;
+ if (!metadata.errorMessage.empty()) ss << std::endl;
+ ss << "Number of textures at probe \"" << probeName << "\" is " << actualValue.numTextures
+ << ", expected is " << expectedValue.numTextures;
+ metadata.errorMessage += ss.str();
+ failed = true;
+ }
+
+ if (expectedValue.numBuffers != actualValue.numBuffers) {
+ std::stringstream ss;
+ if (!metadata.errorMessage.empty()) ss << std::endl;
+ ss << "Number of vertex and index buffers at probe \"" << probeName << "\" is " << actualValue.numBuffers
+ << ", expected is " << expectedValue.numBuffers;
+ metadata.errorMessage += ss.str();
+ failed = true;
+ }
+
+ if (expectedValue.numFrameBuffers != actualValue.numFrameBuffers) {
+ std::stringstream ss;
+ if (!metadata.errorMessage.empty()) ss << std::endl;
+ ss << "Number of frame buffers at probe \"" << probeName << "\" is " << actualValue.numFrameBuffers
+ << ", expected is " << expectedValue.numFrameBuffers;
+ metadata.errorMessage += ss.str();
+ failed = true;
+ }
+
+ if (expectedValue.memTextures.peak != actualValue.memTextures.peak) {
+ std::stringstream ss;
+ if (!metadata.errorMessage.empty()) ss << std::endl;
+ ss << "Allocated texture memory peak size at probe \"" << probeName << "\" is "
+ << actualValue.memTextures.peak << " bytes, expected is " << expectedValue.memTextures.peak << " bytes";
+ metadata.errorMessage += ss.str();
+ failed = true;
+ }
+
+ if (expectedValue.memIndexBuffers.peak != actualValue.memIndexBuffers.peak) {
+ std::stringstream ss;
+ if (!metadata.errorMessage.empty()) ss << std::endl;
+ ss << "Allocated index buffer memory peak size at probe \"" << probeName << "\" is "
+ << actualValue.memIndexBuffers.peak << " bytes, expected is " << expectedValue.memIndexBuffers.peak
+ << " bytes";
+ metadata.errorMessage += ss.str();
+ failed = true;
+ }
+
+ if (expectedValue.memVertexBuffers.peak != actualValue.memVertexBuffers.peak) {
+ std::stringstream ss;
+ if (!metadata.errorMessage.empty()) ss << std::endl;
+ ss << "Allocated vertex buffer memory peak size at probe \"" << probeName << "\" is "
+ << actualValue.memVertexBuffers.peak << " bytes, expected is " << expectedValue.memVertexBuffers.peak
+ << " bytes";
+ metadata.errorMessage += ss.str();
+ failed = true;
+ }
+
+ if (failed) return false;
+ }
+
return true;
}
-bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
+bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata, RunContext& ctx) {
if (!metadata.document.HasMember("metadata") || !metadata.document["metadata"].HasMember("test") ||
!metadata.document["metadata"]["test"].HasMember("operations")) {
return true;
@@ -446,6 +544,9 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
static const std::string getFeatureStateOp("getFeatureState");
static const std::string removeFeatureStateOp("removeFeatureState");
static const std::string panGestureOp("panGesture");
+ static const std::string gfxProbeOp("probeGFX");
+ static const std::string gfxProbeStartOp("probeGFXStart");
+ static const std::string gfxProbeEndOp("probeGFXEnd");
if (operationArray[0].GetString() == waitOp) {
// wait
@@ -955,13 +1056,47 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) {
metadata.metrics.fps.insert({std::move(mark), {averageFps, minOnePcFps, 0.0f}});
+ } else if (operationArray[0].GetString() == gfxProbeStartOp) {
+ // probeGFXStart
+ assert(!ctx.gfxProbeActive);
+ ctx.gfxProbeActive = true;
+ ctx.baselineGfxProbe = ctx.activeGfxProbe;
+ } else if (operationArray[0].GetString() == gfxProbeEndOp) {
+ // probeGFXEnd
+ assert(ctx.gfxProbeActive);
+ ctx.gfxProbeActive = false;
+ } else if (operationArray[0].GetString() == gfxProbeOp) {
+ // probeGFX
+ assert(operationArray.Size() >= 2u);
+ assert(operationArray[1].IsString());
+
+ std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
+
+ // Render the map and fetch rendering stats
+ gfx::RenderingStats stats;
+
+ try {
+ stats = frontend.render(map).stats;
+ } catch (const std::exception&) {
+ return false;
+ }
+
+ ctx.activeGfxProbe = GfxProbe(stats, ctx.activeGfxProbe);
+
+ // Compare memory allocations to the baseline probe
+ GfxProbe metricProbe = ctx.activeGfxProbe;
+ metricProbe.memIndexBuffers.peak -= ctx.baselineGfxProbe.memIndexBuffers.peak;
+ metricProbe.memVertexBuffers.peak -= ctx.baselineGfxProbe.memVertexBuffers.peak;
+ metricProbe.memTextures.peak -= ctx.baselineGfxProbe.memTextures.peak;
+ metadata.metrics.gfx.insert({mark, metricProbe});
+
} else {
metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString();
return false;
}
operationsArray.Erase(operationIt);
- return runOperations(key, metadata);
+ return runOperations(key, metadata, ctx);
}
TestRunner::Impl::Impl(const TestMetadata& metadata)
@@ -1002,13 +1137,15 @@ bool TestRunner::run(TestMetadata& metadata) {
map.getStyle().loadJSON(serializeJsonValue(metadata.document));
map.jumpTo(map.getStyle().getDefaultCamera());
- if (!runOperations(key, metadata)) {
+ RunContext ctx{};
+
+ if (!runOperations(key, metadata, ctx)) {
return false;
}
mbgl::PremultipliedImage image;
try {
- if (metadata.outputsImage) image = frontend.render(map);
+ if (metadata.outputsImage) image = frontend.render(map).image;
} catch (const std::exception&) {
return false;
}
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
index bc97f8300b..4f80286c0b 100644
--- a/render-test/runner.hpp
+++ b/render-test/runner.hpp
@@ -8,6 +8,7 @@
#include <memory>
#include <string>
+struct RunContext;
class TestRunnerMapObserver;
struct TestMetadata;
@@ -22,7 +23,7 @@ public:
void doShuffle(uint32_t seed);
private:
- bool runOperations(const std::string& key, TestMetadata&);
+ bool runOperations(const std::string& key, TestMetadata&, RunContext&);
bool checkQueryTestResults(mbgl::PremultipliedImage&& actualImage,
std::vector<mbgl::Feature>&& features,
TestMetadata&);
View Lorry Controller Status