From 0ca96fd8a402ae530da72e3955196007a2ec365f Mon Sep 17 00:00:00 2001 From: Mikko Pulkki <55925868+mpulkki-mapbox@users.noreply.github.com> Date: Tue, 22 Oct 2019 12:13:55 +0300 Subject: [render-test] Implement fps benchmarking tests (#15803) --- include/mbgl/util/monotonic_timer.hpp | 24 ++ next/platform/android/android.cmake | 1 + next/platform/ios/ios.cmake | 1 + next/platform/linux/linux.cmake | 1 + next/platform/macos/macos.cmake | 1 + next/platform/qt/qt.cmake | 1 + platform/android/core-files.json | 1 + .../default/include/mbgl/gfx/headless_backend.hpp | 10 +- .../default/include/mbgl/gfx/headless_frontend.hpp | 8 +- .../default/include/mbgl/gl/headless_backend.hpp | 7 +- .../default/src/mbgl/gfx/headless_frontend.cpp | 61 +++-- platform/default/src/mbgl/gl/headless_backend.cpp | 32 ++- platform/default/src/mbgl/map/map_snapshotter.cpp | 4 +- platform/default/src/mbgl/util/monotonic_timer.cpp | 24 ++ platform/ios/core-files.json | 1 + platform/linux/config.cmake | 1 + platform/macos/core-files.json | 1 + render-test/metadata.hpp | 10 +- render-test/parser.cpp | 77 +++++-- render-test/render_test.cpp | 3 +- render-test/runner.cpp | 256 ++++++++++++++++----- render-test/runner.hpp | 3 + src/core-files.json | 1 + src/mbgl/gl/context.cpp | 4 + src/mbgl/gl/context.hpp | 2 + test/gl/context.test.cpp | 5 +- test/text/local_glyph_rasterizer.test.cpp | 4 +- 27 files changed, 422 insertions(+), 122 deletions(-) create mode 100644 include/mbgl/util/monotonic_timer.hpp create mode 100644 platform/default/src/mbgl/util/monotonic_timer.cpp diff --git a/include/mbgl/util/monotonic_timer.hpp b/include/mbgl/util/monotonic_timer.hpp new file mode 100644 index 0000000000..bdb167214b --- /dev/null +++ b/include/mbgl/util/monotonic_timer.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include +#include +#include +#include + +namespace mbgl { +namespace util { + +class MonotonicTimer { +public: + static std::chrono::duration now(); + + template + inline static std::chrono::duration duration(F&& func, Args&&... args) { + auto start = now(); + func(std::forward(args)...); + return now() - start; + } +}; + +} // namespace util +} // namespace mbgl \ No newline at end of file diff --git a/next/platform/android/android.cmake b/next/platform/android/android.cmake index 1bde72d11d..6032294ff7 100644 --- a/next/platform/android/android.cmake +++ b/next/platform/android/android.cmake @@ -213,6 +213,7 @@ target_sources( ${MBGL_ROOT}/platform/default/src/mbgl/storage/sqlite3.cpp ${MBGL_ROOT}/platform/default/src/mbgl/text/bidi.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/compression.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/util/monotonic_timer.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/png_writer.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/thread_local.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/utf.cpp diff --git a/next/platform/ios/ios.cmake b/next/platform/ios/ios.cmake index 135a637783..4558b47b68 100644 --- a/next/platform/ios/ios.cmake +++ b/next/platform/ios/ios.cmake @@ -39,6 +39,7 @@ target_sources( ${MBGL_ROOT}/platform/default/src/mbgl/storage/sqlite3.cpp ${MBGL_ROOT}/platform/default/src/mbgl/text/bidi.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/compression.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/util/monotonic_timer.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/png_writer.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/thread_local.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/utf.cpp diff --git a/next/platform/linux/linux.cmake b/next/platform/linux/linux.cmake index 7b77e134ea..6a68a1f2d2 100644 --- a/next/platform/linux/linux.cmake +++ b/next/platform/linux/linux.cmake @@ -37,6 +37,7 @@ target_sources( ${MBGL_ROOT}/platform/default/src/mbgl/util/image.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/jpeg_reader.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/logging_stderr.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/util/monotonic_timer.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/png_reader.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/png_writer.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/run_loop.cpp diff --git a/next/platform/macos/macos.cmake b/next/platform/macos/macos.cmake index d36b89ac6b..71e53c474a 100644 --- a/next/platform/macos/macos.cmake +++ b/next/platform/macos/macos.cmake @@ -105,6 +105,7 @@ target_sources( ${MBGL_ROOT}/platform/default/src/mbgl/storage/sqlite3.cpp ${MBGL_ROOT}/platform/default/src/mbgl/text/bidi.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/compression.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/util/monotonic_timer.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/png_writer.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/thread_local.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/utf.cpp diff --git a/next/platform/qt/qt.cmake b/next/platform/qt/qt.cmake index b902388565..4fb56176d0 100644 --- a/next/platform/qt/qt.cmake +++ b/next/platform/qt/qt.cmake @@ -43,6 +43,7 @@ target_sources( ${MBGL_ROOT}/platform/default/src/mbgl/storage/online_file_source.cpp ${MBGL_ROOT}/platform/default/src/mbgl/storage/sqlite3.cpp ${MBGL_ROOT}/platform/default/src/mbgl/util/compression.cpp + ${MBGL_ROOT}/platform/default/src/mbgl/util/monotonic_timer.cpp ${MBGL_ROOT}/platform/qt/src/async_task.cpp ${MBGL_ROOT}/platform/qt/src/async_task_impl.hpp ${MBGL_ROOT}/platform/qt/src/number_format.cpp diff --git a/platform/android/core-files.json b/platform/android/core-files.json index d536247154..e21a586e29 100644 --- a/platform/android/core-files.json +++ b/platform/android/core-files.json @@ -93,6 +93,7 @@ "platform/default/src/mbgl/map/map_snapshotter.cpp", "platform/default/src/mbgl/text/bidi.cpp", "platform/default/src/mbgl/util/compression.cpp", + "platform/default/src/mbgl/util/monotonic_timer.cpp", "platform/default/src/mbgl/util/png_writer.cpp", "platform/default/src/mbgl/util/thread_local.cpp", "platform/default/src/mbgl/util/utf.cpp", diff --git a/platform/default/include/mbgl/gfx/headless_backend.hpp b/platform/default/include/mbgl/gfx/headless_backend.hpp index 325422323a..5167e6a465 100644 --- a/platform/default/include/mbgl/gfx/headless_backend.hpp +++ b/platform/default/include/mbgl/gfx/headless_backend.hpp @@ -15,11 +15,13 @@ namespace gfx { // of readStillImage. class HeadlessBackend : public gfx::Renderable { public: + enum class SwapBehaviour { NoFlush, Flush }; + // Factory. - static std::unique_ptr - Create(const Size size = { 256, 256 }, - const gfx::ContextMode contextMode = gfx::ContextMode::Unique) { - return Backend::Create(size, contextMode); + static std::unique_ptr Create(const Size size = {256, 256}, + SwapBehaviour swapBehavior = SwapBehaviour::NoFlush, + const gfx::ContextMode contextMode = gfx::ContextMode::Unique) { + return Backend::Create(size, swapBehavior, contextMode); } virtual PremultipliedImage readStillImage() = 0; diff --git a/platform/default/include/mbgl/gfx/headless_frontend.hpp b/platform/default/include/mbgl/gfx/headless_frontend.hpp index 8f7a7bf202..353452123d 100644 --- a/platform/default/include/mbgl/gfx/headless_frontend.hpp +++ b/platform/default/include/mbgl/gfx/headless_frontend.hpp @@ -1,11 +1,12 @@ #pragma once +#include #include #include -#include #include #include +#include #include namespace mbgl { @@ -17,10 +18,12 @@ class TransformState; class HeadlessFrontend : public RendererFrontend { public: HeadlessFrontend(float pixelRatio_, + gfx::HeadlessBackend::SwapBehaviour swapBehviour = gfx::HeadlessBackend::SwapBehaviour::NoFlush, gfx::ContextMode mode = gfx::ContextMode::Unique, const optional localFontFamily = {}); HeadlessFrontend(Size, float pixelRatio_, + gfx::HeadlessBackend::SwapBehaviour swapBehviour = gfx::HeadlessBackend::SwapBehaviour::NoFlush, gfx::ContextMode mode = gfx::ContextMode::Unique, const optional localFontFamily = {}); ~HeadlessFrontend() override; @@ -29,6 +32,7 @@ public: void update(std::shared_ptr) override; void setObserver(RendererObserver&) override; + double getFrameTime() const; Size getSize() const; void setSize(Size); @@ -45,6 +49,7 @@ public: PremultipliedImage readStillImage(); PremultipliedImage render(Map&); + void renderOnce(Map&); optional getTransformState() const; @@ -52,6 +57,7 @@ private: Size size; float pixelRatio; + std::atomic frameTime; std::unique_ptr backend; util::AsyncTask asyncInvalidate; diff --git a/platform/default/include/mbgl/gl/headless_backend.hpp b/platform/default/include/mbgl/gl/headless_backend.hpp index 8aefb5ff6c..b77f1b756f 100644 --- a/platform/default/include/mbgl/gl/headless_backend.hpp +++ b/platform/default/include/mbgl/gl/headless_backend.hpp @@ -10,13 +10,17 @@ namespace gl { class HeadlessBackend final : public gl::RendererBackend, public gfx::HeadlessBackend { public: - HeadlessBackend(Size = { 256, 256 }, gfx::ContextMode = gfx::ContextMode::Unique); + HeadlessBackend(Size = {256, 256}, + SwapBehaviour = SwapBehaviour::NoFlush, + gfx::ContextMode = gfx::ContextMode::Unique); ~HeadlessBackend() override; void updateAssumedState() override; gfx::Renderable& getDefaultRenderable() override; PremultipliedImage readStillImage() override; RendererBackend* getRendererBackend() override; + void swap(); + class Impl { public: virtual ~Impl() = default; @@ -37,6 +41,7 @@ private: private: std::unique_ptr impl; bool active = false; + SwapBehaviour swapBehaviour = SwapBehaviour::NoFlush; }; } // namespace gl diff --git a/platform/default/src/mbgl/gfx/headless_frontend.cpp b/platform/default/src/mbgl/gfx/headless_frontend.cpp index 287567adbd..87d09911a2 100644 --- a/platform/default/src/mbgl/gfx/headless_frontend.cpp +++ b/platform/default/src/mbgl/gfx/headless_frontend.cpp @@ -1,43 +1,50 @@ -#include #include +#include +#include +#include #include #include #include -#include -#include +#include #include namespace mbgl { HeadlessFrontend::HeadlessFrontend(float pixelRatio_, + gfx::HeadlessBackend::SwapBehaviour swapBehavior, const gfx::ContextMode contextMode, const optional localFontFamily) - : HeadlessFrontend( - { 256, 256 }, pixelRatio_, contextMode, localFontFamily) { -} + : HeadlessFrontend({256, 256}, pixelRatio_, swapBehavior, contextMode, localFontFamily) {} HeadlessFrontend::HeadlessFrontend(Size size_, float pixelRatio_, + gfx::HeadlessBackend::SwapBehaviour swapBehavior, const gfx::ContextMode contextMode, const optional localFontFamily) : size(size_), pixelRatio(pixelRatio_), - backend(gfx::HeadlessBackend::Create({ static_cast(size.width * pixelRatio), - static_cast(size.height * pixelRatio) }, contextMode)), - asyncInvalidate([this] { - if (renderer && updateParameters) { - gfx::BackendScope guard { *getBackend() }; - - // onStyleImageMissing might be called during a render. The user implemented method - // could trigger a call to MGLRenderFrontend#update which overwrites `updateParameters`. - // Copy the shared pointer here so that the parameters aren't destroyed while `render(...)` is - // still using them. - auto updateParameters_ = updateParameters; - renderer->render(*updateParameters_); - } - }), - renderer(std::make_unique(*getBackend(), pixelRatio, localFontFamily)) { -} + frameTime(0), + backend(gfx::HeadlessBackend::Create( + {static_cast(size.width * pixelRatio), static_cast(size.height * pixelRatio)}, + swapBehavior, + contextMode)), + asyncInvalidate([this] { + if (renderer && updateParameters) { + auto startTime = mbgl::util::MonotonicTimer::now(); + gfx::BackendScope guard{*getBackend()}; + + // onStyleImageMissing might be called during a render. The user implemented method + // could trigger a call to MGLRenderFrontend#update which overwrites `updateParameters`. + // Copy the shared pointer here so that the parameters aren't destroyed while `render(...)` is + // still using them. + auto updateParameters_ = updateParameters; + renderer->render(*updateParameters_); + + auto endTime = mbgl::util::MonotonicTimer::now(); + frameTime = (endTime - startTime).count(); + } + }), + renderer(std::make_unique(*getBackend(), pixelRatio, localFontFamily)) {} HeadlessFrontend::~HeadlessFrontend() = default; @@ -56,6 +63,10 @@ void HeadlessFrontend::setObserver(RendererObserver& observer_) { renderer->setObserver(&observer_); } +double HeadlessFrontend::getFrameTime() const { + return frameTime; +} + Size HeadlessFrontend::getSize() const { return size; } @@ -147,10 +158,14 @@ PremultipliedImage HeadlessFrontend::render(Map& map) { if (error) { std::rethrow_exception(error); } - + return result; } +void HeadlessFrontend::renderOnce(Map&) { + util::RunLoop::Get()->runOnce(); +} + optional HeadlessFrontend::getTransformState() const { if (updateParameters) { return updateParameters->transformState; diff --git a/platform/default/src/mbgl/gl/headless_backend.cpp b/platform/default/src/mbgl/gl/headless_backend.cpp index 732e4babae..697c560f76 100644 --- a/platform/default/src/mbgl/gl/headless_backend.cpp +++ b/platform/default/src/mbgl/gl/headless_backend.cpp @@ -12,12 +12,12 @@ namespace gl { class HeadlessRenderableResource final : public gl::RenderableResource { public: - HeadlessRenderableResource(gl::Context& context_, Size size_) - : context(context_), + HeadlessRenderableResource(HeadlessBackend& backend_, gl::Context& context_, Size size_) + : backend(backend_), + context(context_), color(context.createRenderbuffer(size_)), depthStencil(context.createRenderbuffer(size_)), - framebuffer(context.createFramebuffer(color, depthStencil)) { - } + framebuffer(context.createFramebuffer(color, depthStencil)) {} void bind() override { context.bindFramebuffer = framebuffer.framebuffer; @@ -25,18 +25,22 @@ public: context.viewport = { 0, 0, framebuffer.size }; } + void swap() override { backend.swap(); } + + HeadlessBackend& backend; gl::Context& context; gfx::Renderbuffer color; gfx::Renderbuffer depthStencil; gl::Framebuffer framebuffer; }; -HeadlessBackend::HeadlessBackend(const Size size_, const gfx::ContextMode contextMode_) - : mbgl::gl::RendererBackend(contextMode_), mbgl::gfx::HeadlessBackend(size_) { -} +HeadlessBackend::HeadlessBackend(const Size size_, + gfx::HeadlessBackend::SwapBehaviour swapBehaviour_, + const gfx::ContextMode contextMode_) + : mbgl::gl::RendererBackend(contextMode_), mbgl::gfx::HeadlessBackend(size_), swapBehaviour(swapBehaviour_) {} HeadlessBackend::~HeadlessBackend() { - gfx::BackendScope guard { *this }; + gfx::BackendScope guard{*this}; resource.reset(); // Explicitly reset the context so that it is destructed and cleaned up before we destruct // the impl object. @@ -67,11 +71,15 @@ void HeadlessBackend::deactivate() { gfx::Renderable& HeadlessBackend::getDefaultRenderable() { if (!resource) { - resource = std::make_unique(static_cast(getContext()), size); + resource = std::make_unique(*this, static_cast(getContext()), size); } return *this; } +void HeadlessBackend::swap() { + if (swapBehaviour == SwapBehaviour::Flush) static_cast(getContext()).finish(); +} + void HeadlessBackend::updateAssumedState() { // no-op } @@ -89,9 +97,9 @@ RendererBackend* HeadlessBackend::getRendererBackend() { namespace gfx { template <> -std::unique_ptr -Backend::Create(const Size size, const gfx::ContextMode contextMode) { - return std::make_unique(size, contextMode); +std::unique_ptr Backend::Create( + const Size size, gfx::HeadlessBackend::SwapBehaviour swapBehavior, const gfx::ContextMode contextMode) { + return std::make_unique(size, swapBehavior, contextMode); } } // namespace gfx diff --git a/platform/default/src/mbgl/map/map_snapshotter.cpp b/platform/default/src/mbgl/map/map_snapshotter.cpp index 5f4060e3f0..705a791af9 100644 --- a/platform/default/src/mbgl/map/map_snapshotter.cpp +++ b/platform/default/src/mbgl/map/map_snapshotter.cpp @@ -51,8 +51,8 @@ MapSnapshotter::Impl::Impl(const std::pair style, const optional region, const optional localFontFamily, const ResourceOptions& resourceOptions) - : frontend( - size, pixelRatio, gfx::ContextMode::Unique, localFontFamily), + : frontend( + size, pixelRatio, gfx::HeadlessBackend::SwapBehaviour::NoFlush, gfx::ContextMode::Unique, localFontFamily), map(frontend, MapObserver::nullObserver(), MapOptions().withMapMode(MapMode::Static).withSize(size).withPixelRatio(pixelRatio), diff --git a/platform/default/src/mbgl/util/monotonic_timer.cpp b/platform/default/src/mbgl/util/monotonic_timer.cpp new file mode 100644 index 0000000000..43c2ce6717 --- /dev/null +++ b/platform/default/src/mbgl/util/monotonic_timer.cpp @@ -0,0 +1,24 @@ +#include +#include +#include + +namespace mbgl { +namespace util { + +// Prefer high resolution timer if it is monotonic +template * = nullptr> +static T sample() { + return std::chrono::duration_cast(std::chrono::high_resolution_clock::now().time_since_epoch()); +} + +template * = nullptr> +static T sample() { + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()); +} + +std::chrono::duration MonotonicTimer::now() { + return sample>(); +} + +} // namespace util +} // namespace mbgl \ No newline at end of file diff --git a/platform/ios/core-files.json b/platform/ios/core-files.json index 7c916b027b..08cf1b5946 100644 --- a/platform/ios/core-files.json +++ b/platform/ios/core-files.json @@ -17,6 +17,7 @@ "platform/default/src/mbgl/map/map_snapshotter.cpp", "platform/default/src/mbgl/text/bidi.cpp", "platform/default/src/mbgl/util/compression.cpp", + "platform/default/src/mbgl/util/monotonic_timer.cpp", "platform/default/src/mbgl/util/png_writer.cpp", "platform/default/src/mbgl/util/thread_local.cpp", "platform/default/src/mbgl/util/utf.cpp" diff --git a/platform/linux/config.cmake b/platform/linux/config.cmake index 26de2430ce..39ae7c6d52 100644 --- a/platform/linux/config.cmake +++ b/platform/linux/config.cmake @@ -52,6 +52,7 @@ macro(mbgl_platform_core) PRIVATE platform/default/src/mbgl/layermanager/layer_manager.cpp PRIVATE platform/default/src/mbgl/util/compression.cpp PRIVATE platform/default/src/mbgl/util/logging_stderr.cpp + PRIVATE platform/default/src/mbgl/util/monotonic_timer.cpp PRIVATE platform/default/src/mbgl/util/string_stdlib.cpp PRIVATE platform/default/src/mbgl/util/thread.cpp PRIVATE platform/default/src/mbgl/util/thread_local.cpp diff --git a/platform/macos/core-files.json b/platform/macos/core-files.json index b0536c4863..5fde52876a 100644 --- a/platform/macos/core-files.json +++ b/platform/macos/core-files.json @@ -16,6 +16,7 @@ "platform/default/src/mbgl/map/map_snapshotter.cpp", "platform/default/src/mbgl/text/bidi.cpp", "platform/default/src/mbgl/util/compression.cpp", + "platform/default/src/mbgl/util/monotonic_timer.cpp", "platform/default/src/mbgl/util/png_writer.cpp", "platform/default/src/mbgl/util/thread_local.cpp", "platform/default/src/mbgl/util/utf.cpp" diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp index 1d5a346f22..996a2bc429 100644 --- a/render-test/metadata.hpp +++ b/render-test/metadata.hpp @@ -68,6 +68,12 @@ struct MemoryProbe { } }; +struct FpsProbe { + float average = 0.0; + float minOnePc = 0.0; + float tolerance = 0.0f; +}; + struct NetworkProbe { NetworkProbe() = default; NetworkProbe(size_t requests_, size_t transferred_) : requests(requests_), transferred(transferred_) {} @@ -78,10 +84,11 @@ struct NetworkProbe { class TestMetrics { public: - bool isEmpty() const { return fileSize.empty() && memory.empty() && network.empty(); } + bool isEmpty() const { return fileSize.empty() && memory.empty() && network.empty() && fps.empty(); } std::map fileSize; std::map memory; std::map network; + std::map fps; }; struct TestMetadata { @@ -90,6 +97,7 @@ struct TestMetadata { TestPaths paths; mbgl::JSDocument document; bool renderTest = true; + bool outputsImage = true; mbgl::Size size{ 512u, 512u }; float pixelRatio = 1.0f; diff --git a/render-test/parser.cpp b/render-test/parser.cpp index da5e6bea81..69d6981c02 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -303,6 +303,22 @@ std::string serializeMetrics(const TestMetrics& metrics) { writer.EndArray(); } + if (!metrics.fps.empty()) { + // Start fps section + writer.Key("fps"); + writer.StartArray(); + for (const auto& fpsProbe : metrics.fps) { + assert(!fpsProbe.first.empty()); + writer.StartArray(); + writer.String(fpsProbe.first.c_str()); + writer.Double(fpsProbe.second.average); + writer.Double(fpsProbe.second.minOnePc); + writer.EndArray(); + } + writer.EndArray(); + // End fps section + } + writer.EndObject(); return s.GetString(); @@ -433,6 +449,7 @@ ArgumentsTuple parseArguments(int argc, char** argv) { if (testFilterValue && !std::regex_match(testPath.path().string(), args::get(testFilterValue))) { continue; } + if (testPath.path().filename() == "style.json") { testPaths.emplace_back(testPath, getTestExpectations(testPath, path, expectationsPaths)); } @@ -549,6 +566,23 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) { } } + if (document.HasMember("fps")) { + const mbgl::JSValue& fpsValue = document["fps"]; + assert(fpsValue.IsArray()); + for (auto& probeValue : fpsValue.GetArray()) { + assert(probeValue.IsArray()); + assert(probeValue.Size() >= 4u); + assert(probeValue[0].IsString()); + assert(probeValue[1].IsNumber()); // Average + assert(probeValue[2].IsNumber()); // Minimum + assert(probeValue[3].IsNumber()); // Tolerance + const std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()}; + assert(!mark.empty()); + result.fps.insert( + {std::move(mark), {probeValue[1].GetFloat(), probeValue[2].GetFloat(), probeValue[3].GetFloat()}}); + } + } + return result; } @@ -607,8 +641,21 @@ TestMetadata parseTestMetadata(const TestPaths& paths) { } if (testValue.HasMember("mapMode")) { + metadata.outputsImage = true; assert(testValue["mapMode"].IsString()); - metadata.mapMode = testValue["mapMode"].GetString() == std::string("tile") ? mbgl::MapMode::Tile : mbgl::MapMode::Static; + std::string mapModeStr = testValue["mapMode"].GetString(); + if (mapModeStr == "tile") + metadata.mapMode = mbgl::MapMode::Tile; + else if (mapModeStr == "continuous") { + metadata.mapMode = mbgl::MapMode::Continuous; + metadata.outputsImage = false; + } else if (mapModeStr == "static") + metadata.mapMode = mbgl::MapMode::Static; + else { + mbgl::Log::Warning( + mbgl::Event::ParseStyle, "Unknown map mode: %s. Falling back to static mode", mapModeStr.c_str()); + metadata.mapMode = mbgl::MapMode::Static; + } } // Test operations handled in runner.cpp. @@ -703,19 +750,21 @@ std::string createResultItem(const TestMetadata& metadata, bool hasFailedTests) html.append("
\n"); html.append(R"(

" + metadata.status + " " + metadata.id + "

\n"); if (metadata.status != "errored") { - if (metadata.renderTest) { - html.append("\n"); - - html.append("\n"); - } else { - html.append("\n"); + if (metadata.outputsImage) { + if (metadata.renderTest) { + html.append("\n"); + + html.append("\n"); + } else { + html.append("\n"); + } } } else { assert(!metadata.errorMessage.empty()); diff --git a/render-test/render_test.cpp b/render-test/render_test.cpp index c3a9c6dd77..df1658265a 100644 --- a/render-test/render_test.cpp +++ b/render-test/render_test.cpp @@ -100,7 +100,8 @@ int runRenderTests(int argc, char** argv) { errored = !runner.run(metadata) || !metadata.errorMessage.empty(); } - bool passed = !errored && !metadata.diff.empty() && metadata.difference <= metadata.allowed; + bool passed = + !errored && (!metadata.outputsImage || !metadata.diff.empty()) && metadata.difference <= metadata.allowed; if (shouldIgnore) { if (passed) { diff --git a/render-test/runner.cpp b/render-test/runner.cpp index 1204c72d71..46b890b76c 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -33,12 +34,41 @@ using namespace mbgl; -// static +class TestRunnerMapObserver : public MapObserver { +public: + TestRunnerMapObserver() : mapLoadFailure(false), finishRenderingMap(false), idle(false) {} + + void onDidFailLoadingMap(MapLoadError, const std::string&) override { mapLoadFailure = true; } + + void onDidFinishRenderingMap(RenderMode mode) override final { + if (!finishRenderingMap) finishRenderingMap = mode == RenderMode::Full; + } + + void onDidBecomeIdle() override final { idle = true; } + + void reset() { + mapLoadFailure = false; + finishRenderingMap = false; + idle = false; + } + + bool mapLoadFailure; + bool finishRenderingMap; + bool idle; +}; + +// static const std::string& TestRunner::getBasePath() { const static std::string result = std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration"); return result; } +// static +gfx::HeadlessBackend::SwapBehaviour swapBehavior(MapMode mode) { + return mode == MapMode::Continuous ? gfx::HeadlessBackend::SwapBehaviour::Flush + : gfx::HeadlessBackend::SwapBehaviour::NoFlush; +} + std::string simpleDiff(const Value& result, const Value& expected) { std::vector resultTokens{tokenize(toJSON(result, 2, false))}; std::vector expectedTokens{tokenize(toJSON(expected, 2, false))}; @@ -159,81 +189,97 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage, const std::string& base = metadata.paths.defaultExpectations(); const std::vector& expectations = metadata.paths.expectations; - metadata.actual = mbgl::encodePNG(actualImage); + if (metadata.outputsImage) { + metadata.actual = mbgl::encodePNG(actualImage); - if (actualImage.size.isEmpty()) { - metadata.errorMessage = "Invalid size for actual image"; - return false; - } + if (actualImage.size.isEmpty()) { + metadata.errorMessage = "Invalid size for actual image"; + return false; + } #if !TEST_READ_ONLY - if (getenv("UPDATE_PLATFORM")) { - mbgl::filesystem::create_directories(expectations.back()); - mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actualImage)); - return true; - } else if (getenv("UPDATE_DEFAULT")) { - mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actualImage)); - return true; - } else if (getenv("UPDATE_METRICS")) { - if (!metadata.metrics.isEmpty()) { + if (getenv("UPDATE_PLATFORM")) { mbgl::filesystem::create_directories(expectations.back()); - mbgl::util::write_file(expectations.back().string() + "/metrics.json", serializeMetrics(metadata.metrics)); + mbgl::util::write_file(expectations.back().string() + "/expected.png", mbgl::encodePNG(actualImage)); + return true; + } else if (getenv("UPDATE_DEFAULT")) { + mbgl::util::write_file(base + "/expected.png", mbgl::encodePNG(actualImage)); return true; } - } - mbgl::util::write_file(base + "/actual.png", metadata.actual); + mbgl::util::write_file(base + "/actual.png", metadata.actual); #endif - mbgl::PremultipliedImage expectedImage { actualImage.size }; - mbgl::PremultipliedImage imageDiff { actualImage.size }; + mbgl::PremultipliedImage expectedImage{actualImage.size}; + mbgl::PremultipliedImage imageDiff{actualImage.size}; - double pixels = 0.0; - std::vector expectedImagesPaths; - mbgl::filesystem::path expectedMetricsPath; - for (auto rit = expectations.rbegin(); rit!= expectations.rend(); ++rit) { - if (mbgl::filesystem::exists(*rit)) { - if (metadata.expectedMetrics.isEmpty()) { - mbgl::filesystem::path maybeExpectedMetricsPath{ *rit }; - maybeExpectedMetricsPath.replace_filename("metrics.json"); - metadata.expectedMetrics = readExpectedMetrics(maybeExpectedMetricsPath); + double pixels = 0.0; + std::vector expectedImagesPaths; + for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) { + if (mbgl::filesystem::exists(*rit)) { + expectedImagesPaths = readExpectedImageEntries(*rit); + if (!expectedImagesPaths.empty()) break; } - expectedImagesPaths = readExpectedImageEntries(*rit); - if (!expectedImagesPaths.empty()) break; } - } - if (expectedImagesPaths.empty()) { - metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string(); - return false; - } - - for (const auto& entry: expectedImagesPaths) { - mbgl::optional maybeExpectedImage = mbgl::util::readFile(entry); - if (!maybeExpectedImage) { - metadata.errorMessage = "Failed to load expected image " + entry; + if (expectedImagesPaths.empty()) { + metadata.errorMessage = "Failed to find expectations for: " + metadata.paths.stylePath.string(); return false; } - metadata.expected = *maybeExpectedImage; + for (const auto& entry : expectedImagesPaths) { + mbgl::optional maybeExpectedImage = mbgl::util::readFile(entry); + if (!maybeExpectedImage) { + metadata.errorMessage = "Failed to load expected image " + entry; + return false; + } + + metadata.expected = *maybeExpectedImage; - expectedImage = mbgl::decodeImage(*maybeExpectedImage); + expectedImage = mbgl::decodeImage(*maybeExpectedImage); - pixels = // implicitly converting from uint64_t - mapbox::pixelmatch(actualImage.data.get(), expectedImage.data.get(), expectedImage.size.width, - expectedImage.size.height, imageDiff.data.get(), 0.1285); // Defined in GL JS + pixels = // implicitly converting from uint64_t + mapbox::pixelmatch(actualImage.data.get(), + expectedImage.data.get(), + expectedImage.size.width, + expectedImage.size.height, + imageDiff.data.get(), + 0.1285); // Defined in GL JS - metadata.diff = mbgl::encodePNG(imageDiff); + metadata.diff = mbgl::encodePNG(imageDiff); #if !TEST_READ_ONLY - mbgl::util::write_file(base + "/diff.png", metadata.diff); + mbgl::util::write_file(base + "/diff.png", metadata.diff); #endif - metadata.difference = pixels / expectedImage.size.area(); - if (metadata.difference <= metadata.allowed) { - break; + metadata.difference = pixels / expectedImage.size.area(); + if (metadata.difference <= metadata.allowed) { + break; + } + } + } + +#if !TEST_READ_ONLY + if (getenv("UPDATE_METRICS")) { + if (!metadata.metrics.isEmpty()) { + mbgl::filesystem::create_directories(expectations.back()); + mbgl::util::write_file(expectations.back().string() + "/metrics.json", serializeMetrics(metadata.metrics)); + return true; + } + } +#endif + + mbgl::filesystem::path expectedMetricsPath; + for (auto rit = expectations.rbegin(); rit != expectations.rend(); ++rit) { + if (mbgl::filesystem::exists(*rit)) { + if (metadata.expectedMetrics.isEmpty()) { + mbgl::filesystem::path maybeExpectedMetricsPath{*rit}; + maybeExpectedMetricsPath.replace_filename("metrics.json"); + metadata.expectedMetrics = readExpectedMetrics(maybeExpectedMetricsPath); + } } } + // Check file size metrics. for (const auto& expected : metadata.expectedMetrics.fileSize) { auto actual = metadata.metrics.fileSize.find(expected.first); @@ -317,16 +363,38 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage, } } #endif // !defined(SANITIZE) + // Check fps metrics + for (const auto& expected : metadata.expectedMetrics.fps) { + auto actual = metadata.metrics.fps.find(expected.first); + if (actual == metadata.metrics.fps.end()) { + metadata.errorMessage = "Failed to find fps probe: " + expected.first; + return false; + } + auto result = checkValue(expected.second.average, actual->second.average, expected.second.tolerance); + if (!std::get(result)) { + std::stringstream ss; + ss << "Average fps at probe \"" << expected.first << "\" is " << actual->second.average + << ", expected to be " << expected.second.average << " with tolerance of " << expected.second.tolerance; + metadata.errorMessage = ss.str(); + return false; + } + result = checkValue(expected.second.minOnePc, actual->second.minOnePc, expected.second.tolerance); + if (!std::get(result)) { + std::stringstream ss; + ss << "Minimum(1%) fps at probe \"" << expected.first << "\" is " << actual->second.minOnePc + << ", expected to be " << expected.second.minOnePc << " with tolerance of " << expected.second.tolerance; + metadata.errorMessage = ss.str(); + return false; + } + } return true; } bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { - if (!metadata.document.HasMember("metadata") || - !metadata.document["metadata"].HasMember("test") || + if (!metadata.document.HasMember("metadata") || !metadata.document["metadata"].HasMember("test") || !metadata.document["metadata"]["test"].HasMember("operations")) { return true; } - assert(metadata.document["metadata"]["test"]["operations"].IsArray()); const auto& operationsArray = metadata.document["metadata"]["test"]["operations"].GetArray(); @@ -342,6 +410,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { auto& frontend = maps[key]->frontend; auto& map = maps[key]->map; + auto& observer = maps[key]->observer; static const std::string waitOp("wait"); static const std::string sleepOp("sleep"); @@ -372,6 +441,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { static const std::string setFeatureStateOp("setFeatureState"); static const std::string getFeatureStateOp("getFeatureState"); static const std::string removeFeatureStateOp("removeFeatureState"); + static const std::string panGestureOp("panGesture"); if (operationArray[0].GetString() == waitOp) { // wait @@ -813,9 +883,76 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { return false; } frontend.getRenderer()->removeFeatureState(sourceID, sourceLayer, featureID, stateKey); + } else if (operationArray[0].GetString() == panGestureOp) { + // benchmarkPanGesture + assert(operationArray.Size() >= 4u); + assert(operationArray[1].IsString()); // identifier + assert(operationArray[2].IsNumber()); // duration + assert(operationArray[3].IsArray()); // start [lat, lng, zoom] + assert(operationArray[4].IsArray()); // end [lat, lng, zoom] + + if (metadata.mapMode != mbgl::MapMode::Continuous) { + metadata.errorMessage = "Map mode must be Continous for " + panGestureOp + " operation"; + return false; + } + + std::string mark = operationArray[1].GetString(); + int duration = operationArray[2].GetFloat(); + LatLng startPos, endPos; + double startZoom, endZoom; + std::vector samples; + + auto parsePosition = [](auto arr) -> std::tuple { + assert(arr.Size() >= 3); + return {{arr[1].GetDouble(), arr[0].GetDouble()}, arr[2].GetDouble()}; + }; + + std::tie(startPos, startZoom) = parsePosition(operationArray[3].GetArray()); + std::tie(endPos, endZoom) = parsePosition(operationArray[4].GetArray()); + + // Jump to the starting point of the segment and make sure there's something to render + map.jumpTo(mbgl::CameraOptions().withCenter(startPos).withZoom(startZoom)); + + observer->reset(); + while (!observer->finishRenderingMap) { + frontend.renderOnce(map); + } + + if (observer->mapLoadFailure) return false; + + size_t frames = 0; + float totalTime = 0.0; + bool transitionFinished = false; + + mbgl::AnimationOptions animationOptions(mbgl::Milliseconds(duration * 1000)); + animationOptions.minZoom = util::min(startZoom, endZoom); + animationOptions.transitionFinishFn = [&]() { transitionFinished = true; }; + + map.flyTo(mbgl::CameraOptions().withCenter(endPos).withZoom(endZoom), animationOptions); + + for (; !transitionFinished; frames++) { + frontend.renderOnce(map); + float frameTime = (float)frontend.getFrameTime(); + totalTime += frameTime; + + samples.push_back(frameTime); + } + + float averageFps = totalTime > 0.0 ? frames / totalTime : 0.0; + float minFrameTime = 0.0; + + // Use 1% of the longest frames to compute the minimum fps + std::sort(samples.begin(), samples.end()); + + int sampleCount = util::max(1, (int)samples.size() / 100); + for (auto it = samples.rbegin(); it != samples.rbegin() + sampleCount; it++) minFrameTime += *it; + + float minOnePcFps = sampleCount / minFrameTime; + + metadata.metrics.fps.insert({std::move(mark), {averageFps, minOnePcFps, 0.0f}}); } else { - metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString(); + metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString(); return false; } @@ -824,9 +961,10 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { } TestRunner::Impl::Impl(const TestMetadata& metadata) - : frontend(metadata.size, metadata.pixelRatio), + : observer(std::make_unique()), + frontend(metadata.size, metadata.pixelRatio, swapBehavior(metadata.mapMode)), map(frontend, - mbgl::MapObserver::nullObserver(), + *observer.get(), mbgl::MapOptions() .withMapMode(metadata.mapMode) .withSize(metadata.size) @@ -834,6 +972,8 @@ TestRunner::Impl::Impl(const TestMetadata& metadata) .withCrossSourceCollisions(metadata.crossSourceCollisions), mbgl::ResourceOptions().withCacheOnlyRequestsSupport(false)) {} +TestRunner::Impl::~Impl() {} + bool TestRunner::run(TestMetadata& metadata) { AllocationIndex::setActive(false); AllocationIndex::reset(); @@ -864,7 +1004,7 @@ bool TestRunner::run(TestMetadata& metadata) { mbgl::PremultipliedImage image; try { - image = frontend.render(map); + if (metadata.outputsImage) image = frontend.render(map); } catch (const std::exception&) { return false; } diff --git a/render-test/runner.hpp b/render-test/runner.hpp index ea593bcc61..6aba2d2391 100644 --- a/render-test/runner.hpp +++ b/render-test/runner.hpp @@ -5,6 +5,7 @@ #include +class TestRunnerMapObserver; struct TestMetadata; class TestRunner { @@ -26,7 +27,9 @@ private: struct Impl { Impl(const TestMetadata&); + ~Impl(); + std::unique_ptr observer; mbgl::HeadlessFrontend frontend; mbgl::Map map; }; diff --git a/src/core-files.json b/src/core-files.json index e5fa493312..e8795fbdd3 100644 --- a/src/core-files.json +++ b/src/core-files.json @@ -483,6 +483,7 @@ "mbgl/util/indexed_tuple.hpp": "include/mbgl/util/indexed_tuple.hpp", "mbgl/util/interpolate.hpp": "include/mbgl/util/interpolate.hpp", "mbgl/util/logging.hpp": "include/mbgl/util/logging.hpp", + "mbgl/util/monotonic_timer.hpp": "include/mbgl/util/monotonic_timer.hpp", "mbgl/util/noncopyable.hpp": "include/mbgl/util/noncopyable.hpp", "mbgl/util/optional.hpp": "include/mbgl/util/optional.hpp", "mbgl/util/platform.hpp": "include/mbgl/util/platform.hpp", diff --git a/src/mbgl/gl/context.cpp b/src/mbgl/gl/context.cpp index 18b376e3dc..f81ac48ee5 100644 --- a/src/mbgl/gl/context.cpp +++ b/src/mbgl/gl/context.cpp @@ -583,6 +583,10 @@ std::unique_ptr Context::createCommandEncoder() { return std::make_unique(*this); } +void Context::finish() { + MBGL_CHECK_ERROR(glFinish()); +} + void Context::draw(const gfx::DrawMode& drawMode, std::size_t indexOffset, std::size_t indexLength) { diff --git a/src/mbgl/gl/context.hpp b/src/mbgl/gl/context.hpp index 70f12e5a8d..edcdde1ec6 100644 --- a/src/mbgl/gl/context.hpp +++ b/src/mbgl/gl/context.hpp @@ -92,6 +92,8 @@ public: std::size_t indexOffset, std::size_t indexLength); + void finish(); + // Actually remove the objects we marked as abandoned with the above methods. // Only call this while the OpenGL context is exclusive to this thread. void performCleanup() override; diff --git a/test/gl/context.test.cpp b/test/gl/context.test.cpp index 770434c5be..5c42eb9344 100644 --- a/test/gl/context.test.cpp +++ b/test/gl/context.test.cpp @@ -91,9 +91,10 @@ TEST(GLContextMode, Shared) { util::RunLoop loop; - HeadlessFrontend frontend { 1, gfx::ContextMode::Shared }; + HeadlessFrontend frontend{1, gfx::HeadlessBackend::SwapBehaviour::NoFlush, gfx::ContextMode::Shared}; - Map map(frontend, MapObserver::nullObserver(), + Map map(frontend, + MapObserver::nullObserver(), MapOptions().withMapMode(MapMode::Static).withSize(frontend.getSize()), ResourceOptions().withCachePath(":memory:").withAssetPath("test/fixtures/api/assets")); map.getStyle().loadJSON(util::read_file("test/fixtures/api/water.json")); diff --git a/test/text/local_glyph_rasterizer.test.cpp b/test/text/local_glyph_rasterizer.test.cpp index 2722ee5849..94c37b170b 100644 --- a/test/text/local_glyph_rasterizer.test.cpp +++ b/test/text/local_glyph_rasterizer.test.cpp @@ -33,9 +33,7 @@ namespace { class LocalGlyphRasterizerTest { public: LocalGlyphRasterizerTest(const optional fontFamily) - : frontend(1, gfx::ContextMode::Unique, fontFamily) - { - } + : frontend(1, gfx::HeadlessBackend::SwapBehaviour::NoFlush, gfx::ContextMode::Unique, fontFamily) {} util::RunLoop loop; std::shared_ptr fileSource = std::make_shared(); -- cgit v1.2.1