diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2018-01-29 16:35:13 +0100 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2018-02-01 15:33:35 +0000 |
commit | c8c2d1901aec01e934adf561a9fdf0cc776cdef8 (patch) | |
tree | 9157c3d9815e5870799e070b113813bec53e0535 /chromium/components/viz/service/frame_sinks | |
parent | abefd5095b41dac94ca451d784ab6e27372e981a (diff) | |
download | qtwebengine-chromium-c8c2d1901aec01e934adf561a9fdf0cc776cdef8.tar.gz |
BASELINE: Update Chromium to 64.0.3282.139
Change-Id: I1cae68fe9c94ff7608b26b8382fc19862cdb293a
Reviewed-by: Alexandru Croitor <alexandru.croitor@qt.io>
Diffstat (limited to 'chromium/components/viz/service/frame_sinks')
31 files changed, 3245 insertions, 116 deletions
diff --git a/chromium/components/viz/service/frame_sinks/compositor_frame_sink_impl.h b/chromium/components/viz/service/frame_sinks/compositor_frame_sink_impl.h index 21f8e2f0954..1d37626bb02 100644 --- a/chromium/components/viz/service/frame_sinks/compositor_frame_sink_impl.h +++ b/chromium/components/viz/service/frame_sinks/compositor_frame_sink_impl.h @@ -35,14 +35,17 @@ class CompositorFrameSinkImpl : public mojom::CompositorFrameSink { uint64_t submit_time) override; void DidNotProduceFrame(const BeginFrameAck& begin_frame_ack) override; + CompositorFrameSinkSupport* support() const { return support_.get(); } + private: void OnClientConnectionLost(); mojom::CompositorFrameSinkClientPtr compositor_frame_sink_client_; mojo::Binding<mojom::CompositorFrameSink> compositor_frame_sink_binding_; - // Must be destroyed before |compositor_frame_sink_client_|. - std::unique_ptr<CompositorFrameSinkSupport> support_; + // Must be destroyed before |compositor_frame_sink_client_|. This must never + // change for the lifetime of CompositorFrameSinkImpl. + const std::unique_ptr<CompositorFrameSinkSupport> support_; DISALLOW_COPY_AND_ASSIGN(CompositorFrameSinkImpl); }; diff --git a/chromium/components/viz/service/frame_sinks/compositor_frame_sink_support.cc b/chromium/components/viz/service/frame_sinks/compositor_frame_sink_support.cc index c414d7668e7..be6c3640072 100644 --- a/chromium/components/viz/service/frame_sinks/compositor_frame_sink_support.cc +++ b/chromium/components/viz/service/frame_sinks/compositor_frame_sink_support.cc @@ -32,6 +32,9 @@ std::unique_ptr<CompositorFrameSinkSupport> CompositorFrameSinkSupport::Create( } CompositorFrameSinkSupport::~CompositorFrameSinkSupport() { + // No video capture clients should remain at this point. + DCHECK(capture_clients_.empty()); + if (!destruction_callback_.is_null()) std::move(destruction_callback_).Run(); @@ -51,9 +54,9 @@ CompositorFrameSinkSupport::~CompositorFrameSinkSupport() { frame_sink_manager_->UnregisterFrameSinkManagerClient(frame_sink_id_); } -void CompositorFrameSinkSupport::SetWillDrawSurfaceCallback( - WillDrawCallback callback) { - will_draw_callback_ = std::move(callback); +void CompositorFrameSinkSupport::SetAggregatedDamageCallback( + AggregatedDamageCallback callback) { + aggregated_damage_callback_ = std::move(callback); } void CompositorFrameSinkSupport::SetDestructionCallback( @@ -67,6 +70,9 @@ void CompositorFrameSinkSupport::OnSurfaceActivated(Surface* surface) { DCHECK(surface->active_referenced_surfaces()); UpdateSurfaceReferences(surface->surface_id().local_surface_id(), *surface->active_referenced_surfaces()); + uint32_t frame_token = surface->GetActiveFrame().metadata.frame_token; + if (frame_token) + frame_sink_manager_->OnFrameTokenChanged(frame_sink_id_, frame_token); } void CompositorFrameSinkSupport::RefResources( @@ -110,9 +116,19 @@ void CompositorFrameSinkSupport::SetBeginFrameSource( void CompositorFrameSinkSupport::EvictCurrentSurface() { if (!current_surface_id_.is_valid()) return; + SurfaceId to_destroy_surface_id = current_surface_id_; current_surface_id_ = SurfaceId(); surface_manager_->DestroySurface(to_destroy_surface_id); + + // For display root surfaces the surface is no longer going to be visible. + // Make it unreachable from the top-level root. + if (referenced_local_surface_id_.has_value()) { + auto reference = MakeTopLevelRootReference( + SurfaceId(frame_sink_id_, referenced_local_surface_id_.value())); + surface_manager_->RemoveSurfaceReferences({reference}); + referenced_local_surface_id_.reset(); + } } void CompositorFrameSinkSupport::SetNeedsBeginFrame(bool needs_begin_frame) { @@ -121,7 +137,7 @@ void CompositorFrameSinkSupport::SetNeedsBeginFrame(bool needs_begin_frame) { } void CompositorFrameSinkSupport::DidNotProduceFrame(const BeginFrameAck& ack) { - TRACE_EVENT2("cc", "CompositorFrameSinkSupport::DidNotProduceFrame", + TRACE_EVENT2("viz", "CompositorFrameSinkSupport::DidNotProduceFrame", "ack.source_id", ack.source_id, "ack.sequence_number", ack.sequence_number); DCHECK_GE(ack.sequence_number, BeginFrameArgs::kStartingFrameNumber); @@ -149,7 +165,8 @@ bool CompositorFrameSinkSupport::SubmitCompositorFrame( const LocalSurfaceId& local_surface_id, CompositorFrame frame, mojom::HitTestRegionListPtr hit_test_region_list) { - TRACE_EVENT0("cc", "CompositorFrameSinkSupport::SubmitCompositorFrame"); + TRACE_EVENT1("viz", "CompositorFrameSinkSupport::SubmitCompositorFrame", + "FrameSinkId", frame_sink_id_.ToString()); DCHECK(local_surface_id.is_valid()); DCHECK(!frame.render_pass_list.empty()); @@ -183,14 +200,24 @@ bool CompositorFrameSinkSupport::SubmitCompositorFrame( SurfaceInfo surface_info(surface_id, frame.device_scale_factor(), frame.size_in_pixels()); - if (!surface_info.is_valid()) { - TRACE_EVENT_INSTANT0("cc", "Invalid SurfaceInfo", + // LocalSurfaceIds should be monotonically increasing. This ID is used + // to determine the freshness of a surface at aggregation time. + bool monotonically_increasing_id = + local_surface_id.parent_id() > + current_surface_id_.local_surface_id().parent_id(); + + if (!surface_info.is_valid() || !monotonically_increasing_id) { + TRACE_EVENT_INSTANT0("viz", "Surface Invariants Violation", TRACE_EVENT_SCOPE_THREAD); EvictCurrentSurface(); std::vector<ReturnedResource> resources = TransferableResource::ReturnResources(frame.resource_list); ReturnResources(resources); DidReceiveCompositorFrameAck(); + if (frame.metadata.presentation_token) { + DidPresentCompositorFrame(frame.metadata.presentation_token, + base::TimeTicks(), base::TimeDelta(), 0); + } return true; } @@ -199,13 +226,17 @@ bool CompositorFrameSinkSupport::SubmitCompositorFrame( surface_manager_->SurfaceDamageExpected(current_surface->surface_id(), last_begin_frame_args_); } - bool result = current_surface->QueueFrame( std::move(frame), frame_index, base::BindOnce(&CompositorFrameSinkSupport::DidReceiveCompositorFrameAck, weak_factory_.GetWeakPtr()), - will_draw_callback_); - + base::BindRepeating(&CompositorFrameSinkSupport::OnAggregatedDamage, + weak_factory_.GetWeakPtr()), + frame.metadata.presentation_token + ? base::BindOnce( + &CompositorFrameSinkSupport::DidPresentCompositorFrame, + weak_factory_.GetWeakPtr(), frame.metadata.presentation_token) + : Surface::PresentedCallback()); if (!result) { EvictCurrentSurface(); return false; @@ -283,6 +314,22 @@ void CompositorFrameSinkSupport::DidReceiveCompositorFrameAck() { surface_returned_resources_.clear(); } +void CompositorFrameSinkSupport::DidPresentCompositorFrame( + uint32_t presentation_token, + base::TimeTicks time, + base::TimeDelta refresh, + uint32_t flags) { + DCHECK(presentation_token); + if (client_) { + if (time != base::TimeTicks()) { + client_->DidPresentCompositorFrame(presentation_token, time, refresh, + flags); + } else { + client_->DidDiscardCompositorFrame(presentation_token); + } + } +} + CompositorFrameSinkSupport::CompositorFrameSinkSupport( mojom::CompositorFrameSinkClient* client, const FrameSinkId& frame_sink_id, @@ -309,6 +356,8 @@ void CompositorFrameSinkSupport::OnBeginFrame(const BeginFrameArgs& args) { last_begin_frame_args_ = args; if (client_) client_->OnBeginFrame(args); + for (CapturableFrameSink::Client* capture_client : capture_clients_) + capture_client->OnBeginFrame(args); } const BeginFrameArgs& CompositorFrameSinkSupport::LastUsedBeginFrameArgs() @@ -342,6 +391,31 @@ Surface* CompositorFrameSinkSupport::CreateSurface( frame_sink_manager_->GetPrimaryBeginFrameSource(), needs_sync_tokens_); } +void CompositorFrameSinkSupport::AttachCaptureClient( + CapturableFrameSink::Client* client) { + DCHECK(std::find(capture_clients_.begin(), capture_clients_.end(), client) == + capture_clients_.end()); + capture_clients_.push_back(client); +} + +void CompositorFrameSinkSupport::DetachCaptureClient( + CapturableFrameSink::Client* client) { + const auto it = + std::find(capture_clients_.begin(), capture_clients_.end(), client); + if (it != capture_clients_.end()) + capture_clients_.erase(it); +} + +gfx::Size CompositorFrameSinkSupport::GetSurfaceSize() { + if (current_surface_id_.is_valid()) { + Surface* current_surface = + surface_manager_->GetSurfaceForId(current_surface_id_); + if (current_surface) + return current_surface->size_in_pixels(); + } + return gfx::Size(); +} + void CompositorFrameSinkSupport::RequestCopyOfSurface( std::unique_ptr<CopyOutputRequest> copy_request) { if (!current_surface_id_.is_valid()) @@ -359,4 +433,19 @@ Surface* CompositorFrameSinkSupport::GetCurrentSurfaceForTesting() { return surface_manager_->GetSurfaceForId(current_surface_id_); } +void CompositorFrameSinkSupport::OnAggregatedDamage( + const LocalSurfaceId& local_surface_id, + const gfx::Rect& damage_rect, + const CompositorFrame& frame) const { + DCHECK(!damage_rect.IsEmpty()); + + if (aggregated_damage_callback_) + aggregated_damage_callback_.Run(local_surface_id, damage_rect); + + const BeginFrameAck& ack = frame.metadata.begin_frame_ack; + const gfx::Size& frame_size = frame.size_in_pixels(); + for (CapturableFrameSink::Client* client : capture_clients_) + client->OnFrameDamaged(ack, frame_size, damage_rect); +} + } // namespace viz diff --git a/chromium/components/viz/service/frame_sinks/compositor_frame_sink_support.h b/chromium/components/viz/service/frame_sinks/compositor_frame_sink_support.h index 28837796716..b8e3be55f18 100644 --- a/chromium/components/viz/service/frame_sinks/compositor_frame_sink_support.h +++ b/chromium/components/viz/service/frame_sinks/compositor_frame_sink_support.h @@ -19,17 +19,12 @@ #include "components/viz/service/frame_sinks/referenced_surface_tracker.h" #include "components/viz/service/frame_sinks/surface_resource_holder.h" #include "components/viz/service/frame_sinks/surface_resource_holder_client.h" +#include "components/viz/service/frame_sinks/video_capture/capturable_frame_sink.h" #include "components/viz/service/surfaces/surface_client.h" #include "components/viz/service/viz_service_export.h" #include "services/viz/public/interfaces/compositing/compositor_frame_sink.mojom.h" #include "services/viz/public/interfaces/hit_test/hit_test_region_list.mojom.h" -namespace { -// The frame index starts at 2 so that empty frames will be treated as -// completely damaged the first time they're drawn from. -constexpr int kFrameIndexStart = 2; -} // namespace - namespace viz { class FrameSinkManagerImpl; @@ -41,12 +36,15 @@ class VIZ_SERVICE_EXPORT CompositorFrameSinkSupport public SurfaceResourceHolderClient, public FrameSinkManagerClient, public SurfaceClient, + public CapturableFrameSink, public mojom::CompositorFrameSink { public: - using WillDrawCallback = + using AggregatedDamageCallback = base::RepeatingCallback<void(const LocalSurfaceId& local_surface_id, const gfx::Rect& damage_rect)>; + static const uint64_t kFrameIndexStart = 2; + static std::unique_ptr<CompositorFrameSinkSupport> Create( mojom::CompositorFrameSinkClient* client, FrameSinkManagerImpl* frame_sink_manager, @@ -58,14 +56,17 @@ class VIZ_SERVICE_EXPORT CompositorFrameSinkSupport const FrameSinkId& frame_sink_id() const { return frame_sink_id_; } + const SurfaceId& current_surface_id() const { return current_surface_id_; } + const LocalSurfaceId& local_surface_id() const { return current_surface_id_.local_surface_id(); } FrameSinkManagerImpl* frame_sink_manager() { return frame_sink_manager_; } - // Sets callback that will be provided to Surface::QueueFrame(). - void SetWillDrawSurfaceCallback(WillDrawCallback callback); + // The provided callback will be run every time a surface owned by this object + // or one of its descendents is determined to be damaged at aggregation time. + void SetAggregatedDamageCallback(AggregatedDamageCallback callback); // Sets callback called on destruction. void SetDestructionCallback(base::OnceCallback<void()> callback); @@ -100,7 +101,13 @@ class VIZ_SERVICE_EXPORT CompositorFrameSinkSupport const LocalSurfaceId& local_surface_id, CompositorFrame frame, mojom::HitTestRegionListPtr hit_test_region_list = nullptr); - void RequestCopyOfSurface(std::unique_ptr<CopyOutputRequest> request); + + // CapturableFrameSink implementation. + void AttachCaptureClient(CapturableFrameSink::Client* client) override; + void DetachCaptureClient(CapturableFrameSink::Client* client) override; + gfx::Size GetSurfaceSize() override; + void RequestCopyOfSurface( + std::unique_ptr<CopyOutputRequest> request) override; Surface* GetCurrentSurfaceForTesting(); @@ -124,6 +131,10 @@ class VIZ_SERVICE_EXPORT CompositorFrameSinkSupport SurfaceReference MakeTopLevelRootReference(const SurfaceId& surface_id); void DidReceiveCompositorFrameAck(); + void DidPresentCompositorFrame(uint32_t presentation_token, + base::TimeTicks time, + base::TimeDelta refresh, + uint32_t flags); // BeginFrameObserver implementation. void OnBeginFrame(const BeginFrameArgs& args) override; @@ -133,6 +144,10 @@ class VIZ_SERVICE_EXPORT CompositorFrameSinkSupport void UpdateNeedsBeginFramesInternal(); Surface* CreateSurface(const SurfaceInfo& surface_info); + void OnAggregatedDamage(const LocalSurfaceId& local_surface_id, + const gfx::Rect& damage_rect, + const CompositorFrame& frame) const; + mojom::CompositorFrameSinkClient* const client_; FrameSinkManagerImpl* frame_sink_manager_ = nullptr; @@ -171,11 +186,16 @@ class VIZ_SERVICE_EXPORT CompositorFrameSinkSupport // A callback that will be run at the start of the destructor if set. base::OnceCallback<void()> destruction_callback_; - // A callback that will be provided to Surface::QueueFrame(). - WillDrawCallback will_draw_callback_; + // TODO(crbug.com/754872): Remove once tab capture has moved into VIZ. + AggregatedDamageCallback aggregated_damage_callback_; uint64_t last_frame_index_ = kFrameIndexStart; + // The video capture clients hooking into this instance to observe frame + // begins and damage, and then make CopyOutputRequests on the appropriate + // frames. + std::vector<CapturableFrameSink::Client*> capture_clients_; + base::WeakPtrFactory<CompositorFrameSinkSupport> weak_factory_; DISALLOW_COPY_AND_ASSIGN(CompositorFrameSinkSupport); diff --git a/chromium/components/viz/service/frame_sinks/compositor_frame_sink_support_unittest.cc b/chromium/components/viz/service/frame_sinks/compositor_frame_sink_support_unittest.cc index b7604a22ad6..8dd1b6902e6 100644 --- a/chromium/components/viz/service/frame_sinks/compositor_frame_sink_support_unittest.cc +++ b/chromium/components/viz/service/frame_sinks/compositor_frame_sink_support_unittest.cc @@ -89,6 +89,8 @@ class FakeFrameSinkManagerClient : public mojom::FrameSinkManagerClient { void SwitchActiveAggregatedHitTestRegionList( const FrameSinkId& frame_sink_id, uint8_t active_handle_index) override {} + void OnFrameTokenChanged(const FrameSinkId& frame_sink_id, + uint32_t frame_token) override {} private: mojom::FrameSinkManager* const manager_; @@ -209,7 +211,7 @@ TEST_F(CompositorFrameSinkSupportTest, ResourceLifetimeSimple) { // The second frame references no resources of first frame and thus should // make all resources of first frame available to be returned. - SubmitCompositorFrameWithResources(NULL, 0); + SubmitCompositorFrameWithResources(nullptr, 0); ResourceId expected_returned_ids[] = {1, 2, 3}; int expected_returned_counts[] = {1, 1, 1}; @@ -261,7 +263,7 @@ TEST_F(CompositorFrameSinkSupportTest, // The second frame references no resources and thus should make all resources // available to be returned as soon as the resource provider releases them. - SubmitCompositorFrameWithResources(NULL, 0); + SubmitCompositorFrameWithResources(nullptr, 0); EXPECT_EQ(0u, fake_support_client_.returned_resources().size()); fake_support_client_.clear_returned_resources(); @@ -291,14 +293,14 @@ TEST_F(CompositorFrameSinkSupportTest, ResourceReusedBeforeReturn) { arraysize(first_frame_ids)); // This removes all references to resource id 7. - SubmitCompositorFrameWithResources(NULL, 0); + SubmitCompositorFrameWithResources(nullptr, 0); // This references id 7 again. SubmitCompositorFrameWithResources(first_frame_ids, arraysize(first_frame_ids)); // This removes it again. - SubmitCompositorFrameWithResources(NULL, 0); + SubmitCompositorFrameWithResources(nullptr, 0); // Now it should be returned. // We don't care how many entries are in the returned array for 7, so long as @@ -335,7 +337,7 @@ TEST_F(CompositorFrameSinkSupportTest, ResourceRefMultipleTimes) { // Submit a frame with no resources to remove all current frame refs from // submitted resources. - SubmitCompositorFrameWithResources(NULL, 0); + SubmitCompositorFrameWithResources(nullptr, 0); EXPECT_EQ(0u, fake_support_client_.returned_resources().size()); fake_support_client_.clear_returned_resources(); @@ -482,7 +484,7 @@ TEST_F(CompositorFrameSinkSupportTest, ResourceLifetime) { // If we submit an empty frame, however, they should become available. // Resources that were previously unref'd also return at this point. - SubmitCompositorFrameWithResources(NULL, 0u); + SubmitCompositorFrameWithResources(nullptr, 0u); { SCOPED_TRACE("fourth frame, second unref"); @@ -505,10 +507,13 @@ TEST_F(CompositorFrameSinkSupportTest, AddDuringEviction) { LocalSurfaceId local_surface_id(6, kArbitraryToken); support->SubmitCompositorFrame(local_surface_id, test::MakeCompositorFrame()); + SurfaceManager* surface_manager = manager_.surface_manager(); + EXPECT_CALL(mock_client, DidReceiveCompositorFrameAck(_)) - .WillOnce(testing::InvokeWithoutArgs([&support, &mock_client]() { + .WillOnce(testing::InvokeWithoutArgs([&]() { LocalSurfaceId new_id(7, base::UnguessableToken::Create()); support->SubmitCompositorFrame(new_id, test::MakeCompositorFrame()); + surface_manager->GarbageCollectSurfaces(); })) .WillRepeatedly(testing::Return()); support->EvictCurrentSurface(); @@ -543,6 +548,7 @@ TEST_F(CompositorFrameSinkSupportTest, EvictCurrentSurface) { EXPECT_CALL(mock_client, DidReceiveCompositorFrameAck(returned_resources)) .Times(1); support->EvictCurrentSurface(); + manager_.surface_manager()->GarbageCollectSurfaces(); EXPECT_FALSE(GetSurfaceForId(id)); manager_.InvalidateFrameSinkId(kAnotherArbitraryFrameSinkId); } @@ -624,6 +630,7 @@ TEST_F(CompositorFrameSinkSupportTest, DuplicateCopyRequest) { support_->EvictCurrentSurface(); local_surface_id_ = LocalSurfaceId(); + manager_.surface_manager()->GarbageCollectSurfaces(); EXPECT_TRUE(called1); EXPECT_TRUE(called2); EXPECT_TRUE(called3); @@ -696,6 +703,7 @@ TEST_F(CompositorFrameSinkSupportTest, FrameSizeMismatch) { frame.render_pass_list.push_back(std::move(pass)); EXPECT_FALSE( support_->SubmitCompositorFrame(local_surface_id_, std::move(frame))); + manager_.surface_manager()->GarbageCollectSurfaces(); EXPECT_FALSE(GetSurfaceForId(id)); } @@ -718,6 +726,7 @@ TEST_F(CompositorFrameSinkSupportTest, DeviceScaleFactorMismatch) { frame.metadata.device_scale_factor = 0.4f; EXPECT_FALSE( support_->SubmitCompositorFrame(local_surface_id_, std::move(frame))); + manager_.surface_manager()->GarbageCollectSurfaces(); EXPECT_FALSE(GetSurfaceForId(id)); } diff --git a/chromium/components/viz/service/frame_sinks/direct_layer_tree_frame_sink.cc b/chromium/components/viz/service/frame_sinks/direct_layer_tree_frame_sink.cc index b98c32c6b66..366a41d0c15 100644 --- a/chromium/components/viz/service/frame_sinks/direct_layer_tree_frame_sink.cc +++ b/chromium/components/viz/service/frame_sinks/direct_layer_tree_frame_sink.cc @@ -66,12 +66,6 @@ bool DirectLayerTreeFrameSink::BindToClient( if (!cc::LayerTreeFrameSink::BindToClient(client)) return false; - // We want the Display's output surface to hear about lost context, and since - // this shares a context with it, we should not be listening for lost context - // callbacks on the context here. - if (auto* cp = context_provider()) - cp->SetLostContextCallback(base::Closure()); - constexpr bool is_root = true; support_ = support_manager_->CreateCompositorFrameSinkSupport( this, frame_sink_id_, is_root, @@ -144,6 +138,19 @@ void DirectLayerTreeFrameSink::DidReceiveCompositorFrameAck( client_->DidReceiveCompositorFrameAck(); } +void DirectLayerTreeFrameSink::DidPresentCompositorFrame( + uint32_t presentation_token, + base::TimeTicks time, + base::TimeDelta refresh, + uint32_t flags) { + client_->DidPresentCompositorFrame(presentation_token, time, refresh, flags); +} + +void DirectLayerTreeFrameSink::DidDiscardCompositorFrame( + uint32_t presentation_token) { + client_->DidDiscardCompositorFrame(presentation_token); +} + void DirectLayerTreeFrameSink::OnBeginFrame(const BeginFrameArgs& args) { begin_frame_source_->OnBeginFrame(args); } @@ -161,4 +168,8 @@ void DirectLayerTreeFrameSink::OnNeedsBeginFrames(bool needs_begin_frame) { support_->SetNeedsBeginFrame(needs_begin_frame); } +void DirectLayerTreeFrameSink::OnContextLost() { + // The display will be listening for OnContextLost(). Do nothing here. +} + } // namespace viz diff --git a/chromium/components/viz/service/frame_sinks/direct_layer_tree_frame_sink.h b/chromium/components/viz/service/frame_sinks/direct_layer_tree_frame_sink.h index 58606db6d06..23346102ff1 100644 --- a/chromium/components/viz/service/frame_sinks/direct_layer_tree_frame_sink.h +++ b/chromium/components/viz/service/frame_sinks/direct_layer_tree_frame_sink.h @@ -70,6 +70,11 @@ class VIZ_SERVICE_EXPORT DirectLayerTreeFrameSink // mojom::CompositorFrameSinkClient implementation: void DidReceiveCompositorFrameAck( const std::vector<ReturnedResource>& resources) override; + void DidPresentCompositorFrame(uint32_t presentation_token, + base::TimeTicks time, + base::TimeDelta refresh, + uint32_t flags) override; + void DidDiscardCompositorFrame(uint32_t presentation_token) override; void OnBeginFrame(const BeginFrameArgs& args) override; void ReclaimResources( const std::vector<ReturnedResource>& resources) override; @@ -78,6 +83,9 @@ class VIZ_SERVICE_EXPORT DirectLayerTreeFrameSink // ExternalBeginFrameSourceClient implementation: void OnNeedsBeginFrames(bool needs_begin_frame) override; + // ContextLostObserver implementation: + void OnContextLost() override; + // This class is only meant to be used on a single thread. THREAD_CHECKER(thread_checker_); diff --git a/chromium/components/viz/service/frame_sinks/direct_layer_tree_frame_sink_unittest.cc b/chromium/components/viz/service/frame_sinks/direct_layer_tree_frame_sink_unittest.cc index f79f3464cce..f499652759e 100644 --- a/chromium/components/viz/service/frame_sinks/direct_layer_tree_frame_sink_unittest.cc +++ b/chromium/components/viz/service/frame_sinks/direct_layer_tree_frame_sink_unittest.cc @@ -6,7 +6,6 @@ #include <memory> -#include "base/memory/ptr_util.h" #include "cc/test/fake_layer_tree_frame_sink_client.h" #include "cc/test/fake_output_surface.h" #include "cc/test/test_context_provider.h" @@ -18,7 +17,6 @@ #include "components/viz/common/surfaces/local_surface_id_allocator.h" #include "components/viz/service/display/display.h" #include "components/viz/service/display/display_scheduler.h" -#include "components/viz/service/display/texture_mailbox_deleter.h" #include "components/viz/service/frame_sinks/compositor_frame_sink_support_manager.h" #include "components/viz/service/frame_sinks/frame_sink_manager_impl.h" #include "components/viz/test/begin_frame_args_test.h" @@ -74,19 +72,18 @@ class DirectLayerTreeFrameSinkTest : public testing::Test { auto display_output_surface = cc::FakeOutputSurface::Create3d(); display_output_surface_ = display_output_surface.get(); - begin_frame_source_ = base::MakeUnique<BackToBackBeginFrameSource>( - base::MakeUnique<DelayBasedTimeSource>(task_runner_.get())); + begin_frame_source_ = std::make_unique<BackToBackBeginFrameSource>( + std::make_unique<DelayBasedTimeSource>(task_runner_.get())); int max_frames_pending = 2; - std::unique_ptr<DisplayScheduler> scheduler(new DisplayScheduler( - begin_frame_source_.get(), task_runner_.get(), max_frames_pending)); + auto scheduler = std::make_unique<DisplayScheduler>( + begin_frame_source_.get(), task_runner_.get(), max_frames_pending); - display_.reset(new Display( + display_ = std::make_unique<Display>( &bitmap_manager_, &gpu_memory_buffer_manager_, RendererSettings(), kArbitraryFrameSinkId, std::move(display_output_surface), - std::move(scheduler), - base::MakeUnique<TextureMailboxDeleter>(task_runner_.get()))); - layer_tree_frame_sink_ = base::MakeUnique<TestDirectLayerTreeFrameSink>( + std::move(scheduler), task_runner_); + layer_tree_frame_sink_ = std::make_unique<TestDirectLayerTreeFrameSink>( kArbitraryFrameSinkId, &support_manager_, &frame_sink_manager_, display_.get(), context_provider_, nullptr, &gpu_memory_buffer_manager_, &bitmap_manager_); diff --git a/chromium/components/viz/service/frame_sinks/frame_sink_manager_impl.cc b/chromium/components/viz/service/frame_sinks/frame_sink_manager_impl.cc index 301324c125b..eb7ed9cce2e 100644 --- a/chromium/components/viz/service/frame_sinks/frame_sink_manager_impl.cc +++ b/chromium/components/viz/service/frame_sinks/frame_sink_manager_impl.cc @@ -11,9 +11,11 @@ #include "components/viz/service/display/display.h" #include "components/viz/service/display_embedder/display_provider.h" #include "components/viz/service/frame_sinks/compositor_frame_sink_impl.h" +#include "components/viz/service/frame_sinks/compositor_frame_sink_support.h" #include "components/viz/service/frame_sinks/frame_sink_manager_client.h" #include "components/viz/service/frame_sinks/primary_begin_frame_source.h" #include "components/viz/service/frame_sinks/root_compositor_frame_sink_impl.h" +#include "components/viz/service/frame_sinks/video_capture/capturable_frame_sink.h" #if DCHECK_IS_ON() #include <sstream> @@ -30,6 +32,10 @@ FrameSinkManagerImpl::FrameSinkSourceMapping::FrameSinkSourceMapping( FrameSinkManagerImpl::FrameSinkSourceMapping::~FrameSinkSourceMapping() = default; +FrameSinkManagerImpl::SinkAndSupport::SinkAndSupport() = default; + +FrameSinkManagerImpl::SinkAndSupport::~SinkAndSupport() = default; + FrameSinkManagerImpl::FrameSinkManagerImpl( SurfaceManager::LifetimeType lifetime_type, DisplayProvider* display_provider) @@ -75,6 +81,8 @@ void FrameSinkManagerImpl::RegisterFrameSinkId( const FrameSinkId& frame_sink_id) { DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); surface_manager_.RegisterFrameSinkId(frame_sink_id); + if (video_detector_) + video_detector_->OnFrameSinkIdRegistered(frame_sink_id); } void FrameSinkManagerImpl::InvalidateFrameSinkId( @@ -82,6 +90,8 @@ void FrameSinkManagerImpl::InvalidateFrameSinkId( DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); compositor_frame_sinks_.erase(frame_sink_id); surface_manager_.InvalidateFrameSinkId(frame_sink_id); + if (video_detector_) + video_detector_->OnFrameSinkIdInvalidated(frame_sink_id); } void FrameSinkManagerImpl::SetFrameSinkDebugLabel( @@ -102,15 +112,17 @@ void FrameSinkManagerImpl::CreateRootCompositorFrameSink( DCHECK_EQ(0u, compositor_frame_sinks_.count(frame_sink_id)); DCHECK(display_provider_); - std::unique_ptr<BeginFrameSource> begin_frame_source; + std::unique_ptr<SyntheticBeginFrameSource> begin_frame_source; auto display = display_provider_->CreateDisplay( frame_sink_id, surface_handle, renderer_settings, &begin_frame_source); - compositor_frame_sinks_[frame_sink_id] = - base::MakeUnique<RootCompositorFrameSinkImpl>( - this, frame_sink_id, std::move(display), - std::move(begin_frame_source), std::move(request), std::move(client), - std::move(display_private_request)); + auto frame_sink = std::make_unique<RootCompositorFrameSinkImpl>( + this, frame_sink_id, std::move(display), std::move(begin_frame_source), + std::move(request), std::move(client), + std::move(display_private_request)); + SinkAndSupport& entry = compositor_frame_sinks_[frame_sink_id]; + entry.support = frame_sink->support(); + entry.sink = std::move(frame_sink); } void FrameSinkManagerImpl::CreateCompositorFrameSink( @@ -120,9 +132,11 @@ void FrameSinkManagerImpl::CreateCompositorFrameSink( DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); DCHECK_EQ(0u, compositor_frame_sinks_.count(frame_sink_id)); - compositor_frame_sinks_[frame_sink_id] = - base::MakeUnique<CompositorFrameSinkImpl>( - this, frame_sink_id, std::move(request), std::move(client)); + auto frame_sink = std::make_unique<CompositorFrameSinkImpl>( + this, frame_sink_id, std::move(request), std::move(client)); + SinkAndSupport& entry = compositor_frame_sinks_[frame_sink_id]; + entry.support = frame_sink->support(); + entry.sink = std::move(frame_sink); } void FrameSinkManagerImpl::RegisterFrameSinkHierarchy( @@ -312,6 +326,14 @@ void FrameSinkManagerImpl::RecursivelyDetachBeginFrameSource( } } +CapturableFrameSink* FrameSinkManagerImpl::FindCapturableFrameSink( + const FrameSinkId& frame_sink_id) { + const auto it = compositor_frame_sinks_.find(frame_sink_id); + if (it == compositor_frame_sinks_.end()) + return nullptr; + return it->second.support; +} + bool FrameSinkManagerImpl::ChildContains( const FrameSinkId& child_frame_sink_id, const FrameSinkId& search_frame_sink_id) const { @@ -358,7 +380,8 @@ void FrameSinkManagerImpl::OnSurfaceDamageExpected(const SurfaceId& surface_id, const BeginFrameArgs& args) { } -void FrameSinkManagerImpl::OnSurfaceWillDraw(const SurfaceId& surface_id) {} +void FrameSinkManagerImpl::OnSurfaceSubtreeDamaged( + const SurfaceId& surface_id) {} void FrameSinkManagerImpl::OnClientConnectionLost( const FrameSinkId& frame_sink_id) { @@ -380,6 +403,12 @@ uint64_t FrameSinkManagerImpl::GetActiveFrameIndex( return surface_manager_.GetSurfaceForId(surface_id)->GetActiveFrameIndex(); } +void FrameSinkManagerImpl::OnFrameTokenChanged(const FrameSinkId& frame_sink_id, + uint32_t frame_token) { + if (client_) + client_->OnFrameTokenChanged(frame_sink_id, frame_token); +} + void FrameSinkManagerImpl::OnAggregatedHitTestRegionListUpdated( const FrameSinkId& frame_sink_id, mojo::ScopedSharedBufferHandle active_handle, @@ -404,4 +433,20 @@ void FrameSinkManagerImpl::SwitchActiveAggregatedHitTestRegionList( } } +void FrameSinkManagerImpl::AddVideoDetectorObserver( + mojom::VideoDetectorObserverPtr observer) { + if (!video_detector_) + video_detector_ = std::make_unique<VideoDetector>(&surface_manager_); + video_detector_->AddObserver(std::move(observer)); +} + +VideoDetector* FrameSinkManagerImpl::CreateVideoDetectorForTesting( + std::unique_ptr<base::TickClock> tick_clock, + scoped_refptr<base::SequencedTaskRunner> task_runner) { + DCHECK(!video_detector_); + video_detector_ = std::make_unique<VideoDetector>( + surface_manager(), std::move(tick_clock), task_runner); + return video_detector_.get(); +} + } // namespace viz diff --git a/chromium/components/viz/service/frame_sinks/frame_sink_manager_impl.h b/chromium/components/viz/service/frame_sinks/frame_sink_manager_impl.h index 63c9947a87a..5b3c43c1dd6 100644 --- a/chromium/components/viz/service/frame_sinks/frame_sink_manager_impl.h +++ b/chromium/components/viz/service/frame_sinks/frame_sink_manager_impl.h @@ -16,6 +16,7 @@ #include "base/threading/thread_checker.h" #include "components/viz/common/surfaces/frame_sink_id.h" #include "components/viz/service/frame_sinks/primary_begin_frame_source.h" +#include "components/viz/service/frame_sinks/video_detector.h" #include "components/viz/service/hit_test/hit_test_manager.h" #include "components/viz/service/surfaces/surface_manager.h" #include "components/viz/service/surfaces/surface_observer.h" @@ -23,6 +24,7 @@ #include "gpu/ipc/common/surface_handle.h" #include "mojo/public/cpp/bindings/binding.h" #include "services/viz/privileged/interfaces/compositing/frame_sink_manager.mojom.h" +#include "services/viz/public/interfaces/compositing/video_detector_observer.mojom.h" namespace cc { @@ -36,6 +38,8 @@ class SurfaceSynchronizationTest; namespace viz { +class CapturableFrameSink; +class CompositorFrameSinkSupport; class DisplayProvider; class FrameSinkManagerClient; @@ -84,6 +88,8 @@ class VIZ_SERVICE_EXPORT FrameSinkManagerImpl : public SurfaceObserver, void AssignTemporaryReference(const SurfaceId& surface_id, const FrameSinkId& owner) override; void DropTemporaryReference(const SurfaceId& surface_id) override; + void AddVideoDetectorObserver( + mojom::VideoDetectorObserverPtr observer) override; // CompositorFrameSinkSupport, hierarchy, and BeginFrameSource can be // registered and unregistered in any order with respect to each other. @@ -124,7 +130,7 @@ class VIZ_SERVICE_EXPORT FrameSinkManagerImpl : public SurfaceObserver, void OnSurfaceDestroyed(const SurfaceId& surface_id) override; void OnSurfaceDamageExpected(const SurfaceId& surface_id, const BeginFrameArgs& args) override; - void OnSurfaceWillDraw(const SurfaceId& surface_id) override; + void OnSurfaceSubtreeDamaged(const SurfaceId& surface_id) override; void OnClientConnectionLost(const FrameSinkId& frame_sink_id); @@ -152,6 +158,16 @@ class VIZ_SERVICE_EXPORT FrameSinkManagerImpl : public SurfaceObserver, // This method is virtual so the implementation can be modified in unit tests. virtual uint64_t GetActiveFrameIndex(const SurfaceId& surface_id); + // Instantiates |video_detector_| for tests where we simulate the passage of + // time. + VideoDetector* CreateVideoDetectorForTesting( + std::unique_ptr<base::TickClock> tick_clock, + scoped_refptr<base::SequencedTaskRunner> task_runner); + + // Called when |frame_token| is changed on a submitted CompositorFrame. + void OnFrameTokenChanged(const FrameSinkId& frame_sink_id, + uint32_t frame_token); + private: friend class cc::test::SurfaceSynchronizationTest; @@ -160,6 +176,11 @@ class VIZ_SERVICE_EXPORT FrameSinkManagerImpl : public SurfaceObserver, void RecursivelyDetachBeginFrameSource(const FrameSinkId& frame_sink_id, BeginFrameSource* source); + // TODO(crbug.com/754872): To be used by FrameSinkVideoCapturerImpl in + // an upcoming change. + CapturableFrameSink* FindCapturableFrameSink( + const FrameSinkId& frame_sink_id); + // Returns true if |child framesink| is or has |search_frame_sink_id| as a // child. bool ChildContains(const FrameSinkId& child_frame_sink_id, @@ -199,9 +220,13 @@ class VIZ_SERVICE_EXPORT FrameSinkManagerImpl : public SurfaceObserver, HitTestManager hit_test_manager_; - std::unordered_map<FrameSinkId, - std::unique_ptr<mojom::CompositorFrameSink>, - FrameSinkIdHash> + struct SinkAndSupport { + SinkAndSupport(); + ~SinkAndSupport(); + std::unique_ptr<mojom::CompositorFrameSink> sink; + CompositorFrameSinkSupport* support; // Owned by |sink|. + }; + std::unordered_map<FrameSinkId, SinkAndSupport, FrameSinkIdHash> compositor_frame_sinks_; THREAD_CHECKER(thread_checker_); @@ -210,6 +235,10 @@ class VIZ_SERVICE_EXPORT FrameSinkManagerImpl : public SurfaceObserver, // directly connected. Use this to make function calls. mojom::FrameSinkManagerClient* client_ = nullptr; + // |video_detector_| is instantiated lazily in order to avoid overhead on + // platforms that don't need video detection. + std::unique_ptr<VideoDetector> video_detector_; + mojom::FrameSinkManagerClientPtr client_ptr_; mojo::Binding<mojom::FrameSinkManager> binding_; diff --git a/chromium/components/viz/service/frame_sinks/primary_begin_frame_source.cc b/chromium/components/viz/service/frame_sinks/primary_begin_frame_source.cc index be6fa15ac19..a3120e41542 100644 --- a/chromium/components/viz/service/frame_sinks/primary_begin_frame_source.cc +++ b/chromium/components/viz/service/frame_sinks/primary_begin_frame_source.cc @@ -7,7 +7,7 @@ namespace viz { PrimaryBeginFrameSource::PrimaryBeginFrameSource() - : begin_frame_source_(this) {} + : BeginFrameSource(kNotRestartableId), begin_frame_source_(this) {} PrimaryBeginFrameSource::~PrimaryBeginFrameSource() = default; diff --git a/chromium/components/viz/service/frame_sinks/referenced_surface_tracker_unittest.cc b/chromium/components/viz/service/frame_sinks/referenced_surface_tracker_unittest.cc index e9816711e2f..7e7354ce88e 100644 --- a/chromium/components/viz/service/frame_sinks/referenced_surface_tracker_unittest.cc +++ b/chromium/components/viz/service/frame_sinks/referenced_surface_tracker_unittest.cc @@ -32,10 +32,10 @@ base::flat_set<SurfaceId> MakeReferenceSet( return base::flat_set<SurfaceId>(surface_ids, base::KEEP_FIRST_OF_DUPES); } -SurfaceId MakeSurfaceId(const FrameSinkId& frame_sink_id, uint32_t local_id) { +SurfaceId MakeSurfaceId(const FrameSinkId& frame_sink_id, uint32_t parent_id) { return SurfaceId( frame_sink_id, - LocalSurfaceId(local_id, base::UnguessableToken::Deserialize(0, 1u))); + LocalSurfaceId(parent_id, base::UnguessableToken::Deserialize(0, 1u))); } } // namespace diff --git a/chromium/components/viz/service/frame_sinks/root_compositor_frame_sink_impl.cc b/chromium/components/viz/service/frame_sinks/root_compositor_frame_sink_impl.cc index a8a85be383c..7072e33af98 100644 --- a/chromium/components/viz/service/frame_sinks/root_compositor_frame_sink_impl.cc +++ b/chromium/components/viz/service/frame_sinks/root_compositor_frame_sink_impl.cc @@ -16,7 +16,7 @@ RootCompositorFrameSinkImpl::RootCompositorFrameSinkImpl( FrameSinkManagerImpl* frame_sink_manager, const FrameSinkId& frame_sink_id, std::unique_ptr<Display> display, - std::unique_ptr<BeginFrameSource> begin_frame_source, + std::unique_ptr<SyntheticBeginFrameSource> synthetic_begin_frame_source, mojom::CompositorFrameSinkAssociatedRequest request, mojom::CompositorFrameSinkClientPtr client, mojom::DisplayPrivateAssociatedRequest display_private_request) @@ -29,23 +29,23 @@ RootCompositorFrameSinkImpl::RootCompositorFrameSinkImpl( frame_sink_id, true /* is_root */, true /* needs_sync_points */)), - display_begin_frame_source_(std::move(begin_frame_source)), + synthetic_begin_frame_source_(std::move(synthetic_begin_frame_source)), display_(std::move(display)), hit_test_aggregator_(frame_sink_manager->hit_test_manager(), this) { - DCHECK(display_begin_frame_source_); + DCHECK(synthetic_begin_frame_source_); DCHECK(display_); compositor_frame_sink_binding_.set_connection_error_handler( base::Bind(&RootCompositorFrameSinkImpl::OnClientConnectionLost, base::Unretained(this))); frame_sink_manager->RegisterBeginFrameSource( - display_begin_frame_source_.get(), frame_sink_id); + synthetic_begin_frame_source_.get(), frame_sink_id); display_->Initialize(this, frame_sink_manager->surface_manager()); } RootCompositorFrameSinkImpl::~RootCompositorFrameSinkImpl() { support_->frame_sink_manager()->UnregisterBeginFrameSource( - display_begin_frame_source_.get()); + synthetic_begin_frame_source_.get()); } void RootCompositorFrameSinkImpl::SetDisplayVisible(bool visible) { @@ -62,6 +62,12 @@ void RootCompositorFrameSinkImpl::SetOutputIsSecure(bool secure) { display_->SetOutputIsSecure(secure); } +void RootCompositorFrameSinkImpl::SetAuthoritativeVSyncInterval( + base::TimeDelta interval) { + if (synthetic_begin_frame_source_) + synthetic_begin_frame_source_->SetAuthoritativeVSyncInterval(interval); +} + void RootCompositorFrameSinkImpl::SetNeedsBeginFrame(bool needs_begin_frame) { support_->SetNeedsBeginFrame(needs_begin_frame); } @@ -71,7 +77,7 @@ void RootCompositorFrameSinkImpl::SubmitCompositorFrame( CompositorFrame frame, mojom::HitTestRegionListPtr hit_test_region_list, uint64_t submit_time) { - // Update |display_| when size or local surface id changes. + // Update display when size or local surface id changes. if (support_->local_surface_id() != local_surface_id) { display_->Resize(frame.size_in_pixels()); display_->SetLocalSurfaceId(local_surface_id, frame.device_scale_factor()); diff --git a/chromium/components/viz/service/frame_sinks/root_compositor_frame_sink_impl.h b/chromium/components/viz/service/frame_sinks/root_compositor_frame_sink_impl.h index 66598d1e1b6..14e0bfe9ee3 100644 --- a/chromium/components/viz/service/frame_sinks/root_compositor_frame_sink_impl.h +++ b/chromium/components/viz/service/frame_sinks/root_compositor_frame_sink_impl.h @@ -19,9 +19,9 @@ namespace viz { -class BeginFrameSource; class Display; class FrameSinkManagerImpl; +class SyntheticBeginFrameSource; // The viz portion of a root CompositorFrameSink. Holds the Binding/InterfacePtr // for the mojom::CompositorFrameSink interface and owns the Display. @@ -34,18 +34,21 @@ class RootCompositorFrameSinkImpl : public mojom::CompositorFrameSink, FrameSinkManagerImpl* frame_sink_manager, const FrameSinkId& frame_sink_id, std::unique_ptr<Display> display, - std::unique_ptr<BeginFrameSource> begin_frame_source, + std::unique_ptr<SyntheticBeginFrameSource> begin_frame_source, mojom::CompositorFrameSinkAssociatedRequest request, mojom::CompositorFrameSinkClientPtr client, mojom::DisplayPrivateAssociatedRequest display_private_request); ~RootCompositorFrameSinkImpl() override; + CompositorFrameSinkSupport* support() const { return support_.get(); } + // mojom::DisplayPrivate: void SetDisplayVisible(bool visible) override; void SetDisplayColorSpace(const gfx::ColorSpace& blending_color_space, const gfx::ColorSpace& device_color_space) override; void SetOutputIsSecure(bool secure) override; + void SetAuthoritativeVSyncInterval(base::TimeDelta interval) override; // mojom::CompositorFrameSink: void SetNeedsBeginFrame(bool needs_begin_frame) override; @@ -78,12 +81,13 @@ class RootCompositorFrameSinkImpl : public mojom::CompositorFrameSink, compositor_frame_sink_binding_; mojo::AssociatedBinding<mojom::DisplayPrivate> display_private_binding_; - // Must be destroyed before |compositor_frame_sink_client_|. - std::unique_ptr<CompositorFrameSinkSupport> support_; + // Must be destroyed before |compositor_frame_sink_client_|. This must never + // change for the lifetime of RootCompositorFrameSinkImpl. + const std::unique_ptr<CompositorFrameSinkSupport> support_; // RootCompositorFrameSinkImpl holds a Display and its BeginFrameSource if // it was created with a non-null gpu::SurfaceHandle. - std::unique_ptr<BeginFrameSource> display_begin_frame_source_; + std::unique_ptr<SyntheticBeginFrameSource> synthetic_begin_frame_source_; std::unique_ptr<Display> display_; HitTestAggregator hit_test_aggregator_; diff --git a/chromium/components/viz/service/frame_sinks/surface_references_unittest.cc b/chromium/components/viz/service/frame_sinks/surface_references_unittest.cc index 39bc90347aa..abbe38be422 100644 --- a/chromium/components/viz/service/frame_sinks/surface_references_unittest.cc +++ b/chromium/components/viz/service/frame_sinks/surface_references_unittest.cc @@ -46,8 +46,9 @@ class SurfaceReferencesTest : public testing::Test { // Creates a new Surface with the provided |frame_sink_id| and |local_id|. // Will first create a Surfacesupport for |frame_sink_id| if necessary. - SurfaceId CreateSurface(const FrameSinkId& frame_sink_id, uint32_t local_id) { - LocalSurfaceId local_surface_id(local_id, + SurfaceId CreateSurface(const FrameSinkId& frame_sink_id, + uint32_t parent_id) { + LocalSurfaceId local_surface_id(parent_id, base::UnguessableToken::Deserialize(0, 1u)); GetCompositorFrameSinkSupport(frame_sink_id) .SubmitCompositorFrame(local_surface_id, MakeCompositorFrame()); @@ -212,6 +213,7 @@ TEST_F(SurfaceReferencesTest, NewSurfaceFromFrameSink) { EXPECT_THAT(GetReferencesFor(id3), UnorderedElementsAre(id2, id2_next)); RemoveSurfaceReference(id1, id2); + GetSurfaceManager().GarbageCollectSurfaces(); EXPECT_THAT(GetReferencesFor(id2), IsEmpty()); EXPECT_THAT(GetReferencesFor(id2_next), UnorderedElementsAre(id1)); EXPECT_THAT(GetReferencesFor(id3), UnorderedElementsAre(id2_next)); @@ -240,6 +242,8 @@ TEST_F(SurfaceReferencesTest, ReferenceCycleGetsDeleted) { RemoveSurfaceReference(GetSurfaceManager().GetRootSurfaceId(), id1); + GetSurfaceManager().GarbageCollectSurfaces(); + // Removing the reference from the root to id1 should allow all three surfaces // to be deleted during GC even with a cycle between 2 and 3. EXPECT_EQ(nullptr, GetSurfaceManager().GetSurfaceForId(id1)); @@ -261,19 +265,23 @@ TEST_F(SurfaceReferencesTest, SurfacesAreDeletedDuringGarbageCollection) { // active reference on all surfaces. DestroySurface(id1); DestroySurface(id2); + GetSurfaceManager().GarbageCollectSurfaces(); + EXPECT_NE(nullptr, GetSurfaceManager().GetSurfaceForId(id1)); EXPECT_NE(nullptr, GetSurfaceManager().GetSurfaceForId(id2)); // Should delete |id2| when the only reference to it is removed. RemoveSurfaceReference(id1, id2); + GetSurfaceManager().GarbageCollectSurfaces(); EXPECT_EQ(nullptr, GetSurfaceManager().GetSurfaceForId(id2)); // Should delete |id1| when the only reference to it is removed. RemoveSurfaceReference(GetSurfaceManager().GetRootSurfaceId(), id1); + GetSurfaceManager().GarbageCollectSurfaces(); EXPECT_EQ(nullptr, GetSurfaceManager().GetSurfaceForId(id1)); } -TEST_F(SurfaceReferencesTest, GarbageCollectionWorksRecusively) { +TEST_F(SurfaceReferencesTest, GarbageCollectionWorksRecursively) { SurfaceId id1 = CreateSurface(kFrameSink1, 1); SurfaceId id2 = CreateSurface(kFrameSink2, 1); SurfaceId id3 = CreateSurface(kFrameSink3, 1); @@ -286,6 +294,8 @@ TEST_F(SurfaceReferencesTest, GarbageCollectionWorksRecusively) { DestroySurface(id2); DestroySurface(id1); + GetSurfaceManager().GarbageCollectSurfaces(); + // Destroying the surfaces shouldn't delete them yet, since there is still an // active reference on all surfaces. EXPECT_NE(nullptr, GetSurfaceManager().GetSurfaceForId(id3)); @@ -294,6 +304,8 @@ TEST_F(SurfaceReferencesTest, GarbageCollectionWorksRecusively) { RemoveSurfaceReference(GetSurfaceManager().GetRootSurfaceId(), id1); + GetSurfaceManager().GarbageCollectSurfaces(); + // Removing the reference from the root to id1 should allow all three surfaces // to be deleted during GC. EXPECT_EQ(nullptr, GetSurfaceManager().GetSurfaceForId(id1)); @@ -316,18 +328,21 @@ TEST_F(SurfaceReferencesTest, LiveSurfaceStillReachable) { // Marking |id3| for destruction shouldn't cause it be garbage collected // because |id2| is still reachable from the root. DestroySurface(id3); + GetSurfaceManager().GarbageCollectSurfaces(); EXPECT_NE(nullptr, GetSurfaceManager().GetSurfaceForId(id3)); // Removing the surface reference to |id2| makes it not reachable from the // root, however it's not marked as destroyed and is still live. Make sure we // also don't delete any dependencies, such as |id3|, as well. RemoveSurfaceReference(id1, id2); + GetSurfaceManager().GarbageCollectSurfaces(); EXPECT_NE(nullptr, GetSurfaceManager().GetSurfaceForId(id3)); EXPECT_NE(nullptr, GetSurfaceManager().GetSurfaceForId(id2)); // |id2| is unreachable and destroyed. Garbage collection should delete both // |id2| and |id3| now. DestroySurface(id2); + GetSurfaceManager().GarbageCollectSurfaces(); EXPECT_EQ(nullptr, GetSurfaceManager().GetSurfaceForId(id3)); EXPECT_EQ(nullptr, GetSurfaceManager().GetSurfaceForId(id2)); } @@ -437,8 +452,8 @@ TEST_F(SurfaceReferencesTest, AddSurfacesSkipReference) { // Add two surfaces that have the same FrameSinkId. This would happen // when a client submits two CompositorFrames before parent submits a new // CompositorFrame. - const SurfaceId surface_id1 = CreateSurface(kFrameSink2, 2); - const SurfaceId surface_id2 = CreateSurface(kFrameSink2, 1); + const SurfaceId surface_id1 = CreateSurface(kFrameSink2, 1); + const SurfaceId surface_id2 = CreateSurface(kFrameSink2, 2); // Temporary references should be added for both surfaces and they should be // stored in the order of creation. @@ -497,6 +512,8 @@ TEST_F(SurfaceReferencesTest, SurfaceWithTemporaryReferenceIsNotDeleted) { DestroySurface(id2); RemoveSurfaceReference(GetSurfaceManager().GetRootSurfaceId(), id1); + GetSurfaceManager().GarbageCollectSurfaces(); + // |id1| is destroyed and has no references, so it's deleted. EXPECT_EQ(nullptr, GetSurfaceManager().GetSurfaceForId(id1)); diff --git a/chromium/components/viz/service/frame_sinks/surface_synchronization_unittest.cc b/chromium/components/viz/service/frame_sinks/surface_synchronization_unittest.cc index 33f6a58b381..a3ea60c229c 100644 --- a/chromium/components/viz/service/frame_sinks/surface_synchronization_unittest.cc +++ b/chromium/components/viz/service/frame_sinks/surface_synchronization_unittest.cc @@ -37,10 +37,10 @@ std::vector<SurfaceId> empty_surface_ids() { return std::vector<SurfaceId>(); } -SurfaceId MakeSurfaceId(const FrameSinkId& frame_sink_id, uint32_t local_id) { +SurfaceId MakeSurfaceId(const FrameSinkId& frame_sink_id, uint32_t parent_id) { return SurfaceId( frame_sink_id, - LocalSurfaceId(local_id, base::UnguessableToken::Deserialize(0, 1u))); + LocalSurfaceId(parent_id, base::UnguessableToken::Deserialize(0, 1u))); } } // namespace @@ -123,6 +123,13 @@ class SurfaceSynchronizationTest : public testing::Test { surface_id); } + Surface* GetLatestInFlightSurface(const FrameSinkId& parent, + const SurfaceId& primary_surface_id, + const SurfaceId& fallback_surface_id) { + return frame_sink_manager().surface_manager()->GetLatestInFlightSurface( + parent, primary_surface_id, fallback_surface_id); + } + FakeExternalBeginFrameSource* begin_frame_source() { return begin_frame_source_.get(); } @@ -238,6 +245,8 @@ TEST_F(SurfaceSynchronizationTest, RootSurfaceReceivesReferences) { frame_sink_manager().surface_manager()->GetRootSurfaceId()), UnorderedElementsAre(display_id_second)); + frame_sink_manager().surface_manager()->GarbageCollectSurfaces(); + // Surface |display_id_first| is unreachable and should get deleted. EXPECT_EQ(nullptr, GetSurfaceForId(display_id_first)); } @@ -1028,6 +1037,8 @@ TEST_F(SurfaceSynchronizationTest, LocalSurfaceIdIsReusable) { // Destroy the surface. child_support1().EvictCurrentSurface(); + frame_sink_manager().surface_manager()->GarbageCollectSurfaces(); + EXPECT_EQ(nullptr, GetSurfaceForId(child_id)); // Submit another frame with the same local surface id. This should work fine @@ -1177,29 +1188,29 @@ TEST_F(SurfaceSynchronizationTest, OnlyBlockOnEmbeddedSurfaces) { // Submitting a CompositorFrame with |parent_id2| so that the display // CompositorFrame can hold a reference to it. - parent_support().SubmitCompositorFrame(parent_id2.local_surface_id(), + parent_support().SubmitCompositorFrame(parent_id1.local_surface_id(), MakeCompositorFrame()); display_support().SubmitCompositorFrame( display_id.local_surface_id(), - MakeCompositorFrame({parent_id1}, {parent_id2}, + MakeCompositorFrame({parent_id2}, {parent_id1}, std::vector<TransferableResource>())); EXPECT_TRUE(display_surface()->HasPendingFrame()); EXPECT_FALSE(display_surface()->HasActiveFrame()); EXPECT_TRUE(display_surface()->has_deadline()); - // Verify that the display CompositorFrame will only block on |parent_id1| - // but not |parent_id2|. + // Verify that the display CompositorFrame will only block on |parent_id2| + // but not |parent_id1|. EXPECT_THAT(display_surface()->activation_dependencies(), - UnorderedElementsAre(parent_id1)); + UnorderedElementsAre(parent_id2)); // Verify that the display surface holds no references while its // CompositorFrame is pending. EXPECT_THAT(GetChildReferences(display_id), IsEmpty()); - // Submitting a CompositorFrame with |parent_id1| should unblock the + // Submitting a CompositorFrame with |parent_id2| should unblock the // display CompositorFrame. - parent_support().SubmitCompositorFrame(parent_id1.local_surface_id(), + parent_support().SubmitCompositorFrame(parent_id2.local_surface_id(), MakeCompositorFrame()); EXPECT_FALSE(display_surface()->has_deadline()); @@ -1631,6 +1642,8 @@ TEST_F(SurfaceSynchronizationTest, FrameActivationAfterFrameSinkDestruction) { display_support().SubmitCompositorFrame(display_id.local_surface_id(), MakeCompositorFrame()); + frame_sink_manager().surface_manager()->GarbageCollectSurfaces(); + parent_surface = GetSurfaceForId(parent_id); EXPECT_EQ(nullptr, parent_surface); } @@ -1699,52 +1712,249 @@ TEST_F(SurfaceSynchronizationTest, FrameIndexWithPendingFrames) { parent_surface->GetActiveFrameIndex()); } +// This test verifies that a new surface with a pending CompositorFrame gets +// a temporary reference immediately, as opposed to when the surface activates. TEST_F(SurfaceSynchronizationTest, PendingSurfaceKeptAlive) { const SurfaceId display_id = MakeSurfaceId(kDisplayFrameSink, 1); const SurfaceId parent_id1 = MakeSurfaceId(kParentFrameSink, 1); - const SurfaceId parent_id2 = MakeSurfaceId(kParentFrameSink, 2); - const SurfaceId child_id = MakeSurfaceId(kChildFrameSink1, 1); // |display_id| depends on |parent_id1|. It shouldn't activate. display_support().SubmitCompositorFrame( display_id.local_surface_id(), MakeCompositorFrame({parent_id1}, empty_surface_ids(), std::vector<TransferableResource>())); - EXPECT_FALSE(GetSurfaceForId(display_id)->HasActiveFrame()); - EXPECT_TRUE(GetSurfaceForId(display_id)->HasPendingFrame()); + EXPECT_FALSE(display_surface()->HasActiveFrame()); + EXPECT_TRUE(display_surface()->HasPendingFrame()); + EXPECT_TRUE(HasTemporaryReference(display_id)); +} + +// Tests getting the correct active frame index. +TEST_F(SurfaceSynchronizationTest, ActiveFrameIndex) { + const SurfaceId parent_id = MakeSurfaceId(kParentFrameSink, 1); + const SurfaceId child_id1 = MakeSurfaceId(kChildFrameSink1, 1); + const SurfaceId child_id2 = MakeSurfaceId(kChildFrameSink2, 1); - // |parent_id1| is created but it depends on |child_id|. |display_id| and - // |parent_id1| must remain pending. parent_support().SubmitCompositorFrame( - parent_id1.local_surface_id(), - MakeCompositorFrame({child_id}, empty_surface_ids(), + parent_id.local_surface_id(), + MakeCompositorFrame({child_id1, child_id2}, empty_surface_ids(), std::vector<TransferableResource>())); - EXPECT_FALSE(GetSurfaceForId(display_id)->HasActiveFrame()); - EXPECT_TRUE(GetSurfaceForId(display_id)->HasPendingFrame()); - EXPECT_FALSE(GetSurfaceForId(parent_id1)->HasActiveFrame()); - EXPECT_TRUE(GetSurfaceForId(parent_id1)->HasPendingFrame()); - // Parent submits a new CompositorFrame to |parent_id2|. |display_id| and - // |parent_id1| should remain pending. + // parent_support is blocked on |child_id1| and |child_id2|. + EXPECT_FALSE(parent_surface()->HasActiveFrame()); + EXPECT_EQ(0u, parent_surface()->GetActiveFrameIndex()); + + child_support1().SubmitCompositorFrame(child_id1.local_surface_id(), + MakeCompositorFrame()); + child_support2().SubmitCompositorFrame(child_id2.local_surface_id(), + MakeCompositorFrame()); + EXPECT_TRUE(parent_surface()->HasActiveFrame()); + EXPECT_EQ(3u, parent_surface()->GetActiveFrameIndex()); +} + +// This test verifies that SurfaceManager::GetLatestInFlightSurface returns +// the latest child surface not yet set as a fallback by the parent. +// Alternatively, it returns the fallback surface specified, if no tempoary +// references to child surfaces are available. This mechanism is used by surface +// synchronization to present the freshest surfaces available at aggregation +// time. +TEST_F(SurfaceSynchronizationTest, LatestInFlightSurface) { + const SurfaceId parent_id = MakeSurfaceId(kParentFrameSink, 1); + const SurfaceId child_id1 = MakeSurfaceId(kChildFrameSink1, 1); + const SurfaceId child_id2 = MakeSurfaceId(kChildFrameSink1, 2); + + child_support1().SubmitCompositorFrame(child_id1.local_surface_id(), + MakeCompositorFrame()); + parent_support().SubmitCompositorFrame( - parent_id2.local_surface_id(), - MakeCompositorFrame(empty_surface_ids(), empty_surface_ids(), + parent_id.local_surface_id(), + MakeCompositorFrame({child_id1}, empty_surface_ids(), + std::vector<TransferableResource>())); + + // Verify that the child CompositorFrame activates immediately. + EXPECT_TRUE(child_surface1()->HasActiveFrame()); + EXPECT_FALSE(child_surface1()->HasPendingFrame()); + EXPECT_THAT(child_surface1()->activation_dependencies(), IsEmpty()); + + // Verify that the parent Surface has activated. + EXPECT_TRUE(parent_surface()->HasActiveFrame()); + EXPECT_FALSE(parent_surface()->HasPendingFrame()); + EXPECT_THAT(parent_surface()->activation_dependencies(), IsEmpty()); + + // Verify that there is a temporary reference for the child and there is + // no reference from the parent to the child yet. + EXPECT_TRUE(HasTemporaryReference(child_id1)); + EXPECT_THAT(GetChildReferences(parent_id), IsEmpty()); + EXPECT_EQ(GetSurfaceForId(child_id1), + GetLatestInFlightSurface(parent_id.frame_sink_id(), child_id2, + child_id1)); + + parent_support().SubmitCompositorFrame( + parent_id.local_surface_id(), + MakeCompositorFrame(empty_surface_ids(), {child_id1}, + std::vector<TransferableResource>())); + + // Verify that the parent Surface has activated. + EXPECT_TRUE(parent_surface()->HasActiveFrame()); + EXPECT_FALSE(parent_surface()->HasPendingFrame()); + EXPECT_THAT(parent_surface()->activation_dependencies(), IsEmpty()); + + // Verify that there is no temporary reference for the child and there is + // a reference from the parent to the child. + EXPECT_FALSE(HasTemporaryReference(child_id1)); + EXPECT_THAT(GetChildReferences(parent_id), UnorderedElementsAre(child_id1)); + EXPECT_EQ(GetSurfaceForId(child_id1), + GetLatestInFlightSurface(parent_id.frame_sink_id(), child_id2, + child_id1)); + + // Submit a child CompositorFrame to a new SurfaceId and verify that + // GetLatestInFlightSurface returns the right surface. + child_support1().SubmitCompositorFrame(child_id2.local_surface_id(), + MakeCompositorFrame()); + + // Verify that there is a temporary reference for child_id2 and there is + // a reference from the parent to child_id1. + EXPECT_TRUE(HasTemporaryReference(child_id2)); + EXPECT_THAT(GetChildReferences(parent_id), UnorderedElementsAre(child_id1)); + + // GetLatestInFlightSurface will not return child_id2's surface because it + // does not yet have an owner. + EXPECT_EQ(GetSurfaceForId(child_id1), + GetLatestInFlightSurface(parent_id.frame_sink_id(), child_id2, + child_id1)); + + // Now that the owner of |child_id2| is known, GetLatestInFlightSurface will + // return it as a possible fallback. + frame_sink_manager().surface_manager()->AssignTemporaryReference( + child_id2, parent_id.frame_sink_id()); + EXPECT_EQ(GetSurfaceForId(child_id2), + GetLatestInFlightSurface(parent_id.frame_sink_id(), child_id2, + child_id1)); + + // If the primary surface is old, then we shouldn't return an in-flight + // surface that is newer than the primary. + EXPECT_EQ(GetSurfaceForId(child_id1), + GetLatestInFlightSurface(parent_id.frame_sink_id(), child_id1, + child_id1)); +} + +// This test verifies that GetLatestInFlightSurface will return nullptr +// if it has a bogus fallback SurfaceID. +TEST_F(SurfaceSynchronizationTest, LatestInFlightSurfaceWithBogusFallback) { + const SurfaceId parent_id = MakeSurfaceId(kParentFrameSink, 1); + const SurfaceId child_id1 = MakeSurfaceId(kChildFrameSink1, 1); + + child_support1().SubmitCompositorFrame(child_id1.local_surface_id(), + MakeCompositorFrame()); + + parent_support().SubmitCompositorFrame( + parent_id.local_surface_id(), + MakeCompositorFrame({child_id1}, empty_surface_ids(), std::vector<TransferableResource>())); - EXPECT_FALSE(GetSurfaceForId(display_id)->HasActiveFrame()); - EXPECT_TRUE(GetSurfaceForId(display_id)->HasPendingFrame()); - EXPECT_FALSE(GetSurfaceForId(parent_id1)->HasActiveFrame()); - EXPECT_TRUE(GetSurfaceForId(parent_id1)->HasPendingFrame()); - // |child_id| becomes available. |display_id| and |parent_id1| should - // activate. + // Verify that the parent and child CompositorFrames are active. + EXPECT_TRUE(child_surface1()->HasActiveFrame()); + EXPECT_FALSE(child_surface1()->HasPendingFrame()); + EXPECT_THAT(child_surface1()->activation_dependencies(), IsEmpty()); + + EXPECT_TRUE(parent_surface()->HasActiveFrame()); + EXPECT_FALSE(parent_surface()->HasPendingFrame()); + EXPECT_THAT(parent_surface()->activation_dependencies(), IsEmpty()); + + // If the fallback surface doesn't exist, then GetLatestInFlightSurface should + // always return nullptr. + const SurfaceId bogus_child_id = MakeSurfaceId(kChildFrameSink1, 10); + EXPECT_EQ(nullptr, GetLatestInFlightSurface(parent_id.frame_sink_id(), + child_id1, bogus_child_id)); +} + +// This test verifies that GetLatestInFlightSurface will return the fallback +// surface if the primary and fallback SurfaceIds have different FrameSinkIds. +// This is important to preserve the property that that latest in-flight surface +// is no newer than the primary. If the FrameSinkId changes then we cannot be +// sure of that so we simply return the fallback surface. +TEST_F(SurfaceSynchronizationTest, LatestInFlightSurfaceDifferentFrameSinkIds) { + const SurfaceId parent_id = MakeSurfaceId(kParentFrameSink, 1); + const SurfaceId child_id1 = MakeSurfaceId(kChildFrameSink1, 1); + const SurfaceId child_id2 = MakeSurfaceId(kChildFrameSink2, 1); + + child_support1().SubmitCompositorFrame(child_id1.local_surface_id(), + MakeCompositorFrame()); + + parent_support().SubmitCompositorFrame( + parent_id.local_surface_id(), + MakeCompositorFrame({child_id2}, {child_id1}, + std::vector<TransferableResource>())); + + // Submit a child CompositorFrame without a different FrameSinkId and verify + // that if the fallback and primary differ in FrameSinkId then + // GetLatestInFlightSurface will always return the specified fallback. + child_support2().SubmitCompositorFrame(child_id2.local_surface_id(), + MakeCompositorFrame()); + + // Submit a child CompositorFrame without a different FrameSinkId and verify + // that if the fallback and primary differ in FrameSinkId then + // GetLatestInFlightSurface will always return the specified fallback. + child_support2().SubmitCompositorFrame(child_id2.local_surface_id(), + MakeCompositorFrame()); + EXPECT_EQ(GetSurfaceForId(child_id1), + GetLatestInFlightSurface(parent_id.frame_sink_id(), child_id2, + child_id1)); +} + +// This test verifies that if a child submits a LocalSurfaceId newer that the +// parent's dependency, then the parent will drop its dependency and activate +// if possible. +TEST_F(SurfaceSynchronizationTest, DropDependenciesThatWillNeverArrive) { + const SurfaceId parent_id = MakeSurfaceId(kParentFrameSink, 1); + const SurfaceId child_id11 = MakeSurfaceId(kChildFrameSink1, 1); + const SurfaceId child_id12 = MakeSurfaceId(kChildFrameSink1, 2); + const SurfaceId child_id21 = MakeSurfaceId(kChildFrameSink2, 1); + const SurfaceId arbitrary_id = MakeSurfaceId(kArbitraryFrameSink, 1); + + // |parent_id| depends on { child_id11, child_id12, child_id21 }. It + // shouldn't activate. + parent_support().SubmitCompositorFrame( + parent_id.local_surface_id(), + MakeCompositorFrame({child_id11, child_id12, child_id21}, + empty_surface_ids(), + std::vector<TransferableResource>())); + EXPECT_FALSE(parent_surface()->HasActiveFrame()); + EXPECT_TRUE(parent_surface()->HasPendingFrame()); + + // |child_id11| is created but it depends on |arbitrary_id|. |parent_id| and + // |child_id11| must remain pending. child_support1().SubmitCompositorFrame( - child_id.local_surface_id(), + child_id11.local_surface_id(), + MakeCompositorFrame({arbitrary_id}, empty_surface_ids(), + std::vector<TransferableResource>())); + EXPECT_FALSE(parent_surface()->HasActiveFrame()); + EXPECT_TRUE(parent_surface()->HasPendingFrame()); + EXPECT_THAT(parent_surface()->activation_dependencies(), + UnorderedElementsAre(child_id11, child_id12, child_id21)); + EXPECT_FALSE(GetSurfaceForId(child_id11)->HasActiveFrame()); + EXPECT_TRUE(GetSurfaceForId(child_id11)->HasPendingFrame()); + + // The first child submits a new CompositorFrame to |child_id12|. |parent_id| + // no longer depends on |child_id11| because it cannot expect it to arrive. + // However, the parent is still blocked on |child_id21|. + child_support1().SubmitCompositorFrame( + child_id12.local_surface_id(), MakeCompositorFrame(empty_surface_ids(), empty_surface_ids(), std::vector<TransferableResource>())); - EXPECT_TRUE(GetSurfaceForId(display_id)->HasActiveFrame()); - EXPECT_FALSE(GetSurfaceForId(display_id)->HasPendingFrame()); - EXPECT_TRUE(GetSurfaceForId(parent_id1)->HasActiveFrame()); - EXPECT_FALSE(GetSurfaceForId(parent_id1)->HasPendingFrame()); + EXPECT_FALSE(parent_surface()->HasActiveFrame()); + EXPECT_TRUE(parent_surface()->HasPendingFrame()); + EXPECT_THAT(parent_surface()->activation_dependencies(), + UnorderedElementsAre(child_id21)); + + // Finally, the second child submits a frame to the remaining dependency and + // the parent activates. + child_support2().SubmitCompositorFrame( + child_id21.local_surface_id(), + MakeCompositorFrame(empty_surface_ids(), empty_surface_ids(), + std::vector<TransferableResource>())); + EXPECT_TRUE(parent_surface()->HasActiveFrame()); + EXPECT_FALSE(parent_surface()->HasPendingFrame()); + EXPECT_THAT(parent_surface()->activation_dependencies(), IsEmpty()); } } // namespace test diff --git a/chromium/components/viz/service/frame_sinks/video_capture/DEPS b/chromium/components/viz/service/frame_sinks/video_capture/DEPS new file mode 100644 index 00000000000..6365bee3a0c --- /dev/null +++ b/chromium/components/viz/service/frame_sinks/video_capture/DEPS @@ -0,0 +1,5 @@ +include_rules = [ + "+media/base", + "+media/capture", + "+media/mojo", +] diff --git a/chromium/components/viz/service/frame_sinks/video_capture/OWNERS b/chromium/components/viz/service/frame_sinks/video_capture/OWNERS new file mode 100644 index 00000000000..02bdb39030f --- /dev/null +++ b/chromium/components/viz/service/frame_sinks/video_capture/OWNERS @@ -0,0 +1 @@ +miu@chromium.org diff --git a/chromium/components/viz/service/frame_sinks/video_capture/capturable_frame_sink.h b/chromium/components/viz/service/frame_sinks/video_capture/capturable_frame_sink.h new file mode 100644 index 00000000000..db4a2ae2b8f --- /dev/null +++ b/chromium/components/viz/service/frame_sinks/video_capture/capturable_frame_sink.h @@ -0,0 +1,62 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_CAPTURABLE_FRAME_SINK_H_ +#define COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_CAPTURABLE_FRAME_SINK_H_ + +#include <memory> + +#include "ui/gfx/geometry/size.h" + +namespace gfx { +class Rect; +} // namespace gfx + +namespace viz { + +struct BeginFrameAck; +struct BeginFrameArgs; +class CopyOutputRequest; + +// Interface for CompositorFrameSink implementations that support frame sink +// video capture. +class CapturableFrameSink { + public: + // Interface for a client that observes certain frame events and calls + // RequestCopyOfSurface() at the appropriate times. + class Client { + public: + virtual ~Client() = default; + + // Called to indicate compositing has started for a new frame. + virtual void OnBeginFrame(const BeginFrameArgs& args) = 0; + + // Called to indicate a frame's content has changed since the last + // frame. |ack| identifies the frame. |frame_size| is the output size of the + // frame, with |damage_rect| being the region within the frame that has + // changed. + virtual void OnFrameDamaged(const BeginFrameAck& ack, + const gfx::Size& frame_size, + const gfx::Rect& damage_rect) = 0; + }; + + virtual ~CapturableFrameSink() = default; + + // Attach/Detach a video capture client to the frame sink. The client will + // receive frame begin and draw events, and issue copy requests, when + // appropriate. + virtual void AttachCaptureClient(Client* client) = 0; + virtual void DetachCaptureClient(Client* client) = 0; + + // Returns the current surface size. + virtual gfx::Size GetSurfaceSize() = 0; + + // Issues a request for a copy of the next composited frame. + virtual void RequestCopyOfSurface( + std::unique_ptr<CopyOutputRequest> request) = 0; +}; + +} // namespace viz + +#endif // COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_CAPTURABLE_FRAME_SINK_H_ diff --git a/chromium/components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_impl.cc b/chromium/components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_impl.cc new file mode 100644 index 00000000000..cb2e2b57059 --- /dev/null +++ b/chromium/components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_impl.cc @@ -0,0 +1,487 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_impl.h" + +#include "base/bind.h" +#include "base/trace_event/trace_event.h" +#include "components/viz/common/frame_sinks/copy_output_request.h" +#include "components/viz/common/frame_sinks/copy_output_result.h" +#include "components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_manager.h" +#include "components/viz/service/frame_sinks/video_capture/frame_sink_video_consumer.h" +#include "media/base/limits.h" +#include "media/base/video_util.h" +#include "ui/gfx/color_space.h" +#include "ui/gfx/geometry/rect.h" +#include "ui/gfx/geometry/size.h" + +using media::VideoCaptureOracle; +using media::VideoFrame; +using media::VideoFrameMetadata; + +namespace viz { + +// static +constexpr media::VideoPixelFormat + FrameSinkVideoCapturerImpl::kDefaultPixelFormat; + +// static +constexpr media::ColorSpace FrameSinkVideoCapturerImpl::kDefaultColorSpace; + +FrameSinkVideoCapturerImpl::FrameSinkVideoCapturerImpl( + FrameSinkVideoCapturerManager* frame_sink_manager) + : frame_sink_manager_(frame_sink_manager), + copy_request_source_(base::UnguessableToken::Create()), + oracle_(true /* enable_auto_throttling */), + frame_pool_(kDesignLimitMaxFrames), + feedback_weak_factory_(&oracle_), + capture_weak_factory_(this) { + DCHECK(frame_sink_manager_); +} + +FrameSinkVideoCapturerImpl::~FrameSinkVideoCapturerImpl() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + Stop(); + SetResolvedTarget(nullptr); +} + +void FrameSinkVideoCapturerImpl::SetResolvedTarget( + CapturableFrameSink* target) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + if (resolved_target_ == target) { + return; + } + + if (resolved_target_) { + resolved_target_->DetachCaptureClient(this); + } + resolved_target_ = target; + if (resolved_target_) { + resolved_target_->AttachCaptureClient(this); + const gfx::Size& source_size = resolved_target_->GetSurfaceSize(); + if (source_size != oracle_.source_size()) { + oracle_.SetSourceSize(source_size); + } + MaybeCaptureFrame(VideoCaptureOracle::kActiveRefreshRequest, + gfx::Rect(source_size), clock_->NowTicks()); + } else { + // Not calling consumer_->OnTargetLost() because SetResolvedTarget() should + // be called by FrameSinkManagerImpl with a valid target very soon. + } +} + +void FrameSinkVideoCapturerImpl::OnTargetWillGoAway() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + if (!resolved_target_) { + return; + } + resolved_target_->DetachCaptureClient(this); + resolved_target_ = nullptr; + + if (requested_target_.is_valid()) { + if (consumer_) { + consumer_->OnTargetLost(requested_target_); + } + requested_target_ = FrameSinkId(); + } +} + +void FrameSinkVideoCapturerImpl::SetFormat(media::VideoPixelFormat format, + media::ColorSpace color_space) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + if (format != media::PIXEL_FORMAT_I420 && + format != media::PIXEL_FORMAT_YV12) { + LOG(DFATAL) << "Invalid pixel format: Only I420 or YV12 are supported."; + } else { + pixel_format_ = format; + } + + if (color_space == media::COLOR_SPACE_UNSPECIFIED) { + color_space = kDefaultColorSpace; + } + // TODO(crbug/758057): Plumb output color space through to the + // CopyOutputRequests. + if (color_space != media::COLOR_SPACE_HD_REC709) { + LOG(DFATAL) << "Unsupported color space: Only BT.709 is supported."; + } else { + color_space_ = color_space; + } +} + +void FrameSinkVideoCapturerImpl::SetMinCapturePeriod( + base::TimeDelta min_capture_period) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + constexpr base::TimeDelta kMinMinCapturePeriod = + base::TimeDelta::FromMicroseconds(base::Time::kMicrosecondsPerSecond / + media::limits::kMaxFramesPerSecond); + if (min_capture_period < kMinMinCapturePeriod) { + min_capture_period = kMinMinCapturePeriod; + } + + // On machines without high-resolution clocks, limit the maximum frame rate to + // 30 FPS. This avoids a potential issue where the system clock may not + // advance between two successive frames. + if (!base::TimeTicks::IsHighResolution()) { + constexpr base::TimeDelta kMinLowResCapturePeriod = + base::TimeDelta::FromMicroseconds(base::Time::kMicrosecondsPerSecond / + 30); + if (min_capture_period < kMinLowResCapturePeriod) { + min_capture_period = kMinLowResCapturePeriod; + } + } + + oracle_.SetMinCapturePeriod(min_capture_period); +} + +void FrameSinkVideoCapturerImpl::SetResolutionConstraints( + const gfx::Size& min_size, + const gfx::Size& max_size, + bool use_fixed_aspect_ratio) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + if (min_size.width() <= 0 || min_size.height() <= 0 || + max_size.width() > media::limits::kMaxDimension || + max_size.height() > media::limits::kMaxDimension || + min_size.width() > max_size.width() || + min_size.height() > max_size.height()) { + LOG(DFATAL) << "Invalid resolutions constraints: " << min_size.ToString() + << " must not be greater than " << max_size.ToString() + << "; and also within media::limits."; + return; + } + + oracle_.SetCaptureSizeConstraints(min_size, max_size, use_fixed_aspect_ratio); +} + +void FrameSinkVideoCapturerImpl::ChangeTarget( + const FrameSinkId& frame_sink_id) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + requested_target_ = frame_sink_id; + SetResolvedTarget( + frame_sink_manager_->FindCapturableFrameSink(frame_sink_id)); +} + +void FrameSinkVideoCapturerImpl::Start( + std::unique_ptr<FrameSinkVideoConsumer> consumer) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + Stop(); + consumer_ = std::move(consumer); + MaybeCaptureFrame(VideoCaptureOracle::kActiveRefreshRequest, + gfx::Rect(oracle_.source_size()), clock_->NowTicks()); +} + +void FrameSinkVideoCapturerImpl::Stop() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + // Cancel any captures in-flight and any captured frames pending delivery. + capture_weak_factory_.InvalidateWeakPtrs(); + oracle_.CancelAllCaptures(); + while (!delivery_queue_.empty()) { + delivery_queue_.pop(); + } + next_delivery_frame_number_ = next_capture_frame_number_; + + if (consumer_) { + consumer_->OnStopped(); + consumer_.reset(); + } +} + +void FrameSinkVideoCapturerImpl::RequestRefreshFrame() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + MaybeCaptureFrame(VideoCaptureOracle::kPassiveRefreshRequest, + gfx::Rect(oracle_.source_size()), clock_->NowTicks()); +} + +void FrameSinkVideoCapturerImpl::OnBeginFrame(const BeginFrameArgs& args) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(args.IsValid()); + DCHECK(resolved_target_); + + frame_display_times_[args.sequence_number % frame_display_times_.size()] = + args.frame_time + args.interval; + current_begin_frame_source_id_ = args.source_id; +} + +void FrameSinkVideoCapturerImpl::OnFrameDamaged(const BeginFrameAck& ack, + const gfx::Size& frame_size, + const gfx::Rect& damage_rect) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(!frame_size.IsEmpty()); + DCHECK(!damage_rect.IsEmpty()); + DCHECK(resolved_target_); + + if (ack.source_id != current_begin_frame_source_id_ || + ack.sequence_number < BeginFrameArgs::kStartingFrameNumber) { + return; + } + + if (frame_size != oracle_.source_size()) { + oracle_.SetSourceSize(frame_size); + } + + MaybeCaptureFrame( + VideoCaptureOracle::kCompositorUpdate, damage_rect, + frame_display_times_[ack.sequence_number % frame_display_times_.size()]); +} + +void FrameSinkVideoCapturerImpl::MaybeCaptureFrame( + VideoCaptureOracle::Event event, + const gfx::Rect& damage_rect, + base::TimeTicks event_time) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + // Consult the oracle to determine whether this frame should be captured. + if (!oracle_.ObserveEventAndDecideCapture(event, damage_rect, event_time)) { + TRACE_EVENT_INSTANT1("gpu.capture", "FpsRateLimited", + TRACE_EVENT_SCOPE_THREAD, "trigger", + VideoCaptureOracle::EventAsString(event)); + return; + } + + // If there is no |consumer_| present, punt. This check is being done after + // consulting the oracle because it helps to "prime" the oracle in the short + // period of time where the capture target is known but the |consumer_| has + // not yet been provided in the call to Start(). + if (!consumer_) { + TRACE_EVENT_INSTANT1("gpu.capture", "NoConsumer", TRACE_EVENT_SCOPE_THREAD, + "trigger", VideoCaptureOracle::EventAsString(event)); + return; + } + + // Reserve a buffer from the pool for the next frame. + const OracleFrameNumber oracle_frame_number = oracle_.next_frame_number(); + scoped_refptr<VideoFrame> frame; + if (event == VideoCaptureOracle::kPassiveRefreshRequest) { + frame = frame_pool_.ResurrectLastVideoFrame(oracle_.capture_size(), + pixel_format_); + // If the resurrection failed, promote the passive refresh request to an + // active refresh request and retry. + if (!frame) { + TRACE_EVENT_INSTANT0("gpu.capture", "ResurrectionFailed", + TRACE_EVENT_SCOPE_THREAD); + MaybeCaptureFrame(VideoCaptureOracle::kActiveRefreshRequest, damage_rect, + event_time); + return; + } + } else { + frame = + frame_pool_.ReserveVideoFrame(oracle_.capture_size(), pixel_format_); + } + + // Compute the current in-flight utilization and attenuate it: The utilization + // reported to the oracle is in terms of a maximum sustainable amount (not the + // absolute maximum). + const float utilization = + frame_pool_.GetUtilization() / kTargetPipelineUtilization; + + // Do not proceed if the pool did not provide a frame: This indicates the + // pipeline is full. + if (!frame) { + TRACE_EVENT_INSTANT2( + "gpu.capture", "PipelineLimited", TRACE_EVENT_SCOPE_THREAD, "trigger", + VideoCaptureOracle::EventAsString(event), "atten_util_percent", + base::saturated_cast<int>(utilization * 100.0f + 0.5f)); + oracle_.RecordWillNotCapture(utilization); + return; + } + + // Record a trace event if the capture pipeline is redlining, but capture will + // still proceed. + if (utilization >= 1.0) { + TRACE_EVENT_INSTANT2( + "gpu.capture", "NearlyPipelineLimited", TRACE_EVENT_SCOPE_THREAD, + "trigger", VideoCaptureOracle::EventAsString(event), + "atten_util_percent", + base::saturated_cast<int>(utilization * 100.0f + 0.5f)); + } + + // At this point, the capture is going to proceed. Populate the VideoFrame's + // metadata, and notify the oracle. + VideoFrameMetadata* const metadata = frame->metadata(); + metadata->SetTimeTicks(VideoFrameMetadata::CAPTURE_BEGIN_TIME, + clock_->NowTicks()); + // See TODO in SetFormat(). For now, always assume Rec. 709. + frame->set_color_space(gfx::ColorSpace::CreateREC709()); + metadata->SetInteger(VideoFrameMetadata::COLOR_SPACE, + media::COLOR_SPACE_HD_REC709); + metadata->SetTimeDelta(VideoFrameMetadata::FRAME_DURATION, + oracle_.estimated_frame_duration()); + metadata->SetDouble(VideoFrameMetadata::FRAME_RATE, + 1.0 / oracle_.min_capture_period().InSecondsF()); + metadata->SetTimeTicks(VideoFrameMetadata::REFERENCE_TIME, event_time); + + oracle_.RecordCapture(utilization); + const int64_t frame_number = next_capture_frame_number_++; + TRACE_EVENT_ASYNC_BEGIN2("gpu.capture", "Capture", frame.get(), + "frame_number", frame_number, "trigger", + VideoCaptureOracle::EventAsString(event)); + + // If there is currently no resolved target (or the target has zero area), + // deliver a blank black frame. + const gfx::Size& source_size = oracle_.source_size(); + if (!resolved_target_ || source_size.IsEmpty()) { + media::FillYUV(frame.get(), 0x00, 0x80, 0x80); + DidCaptureFrame(frame_number, oracle_frame_number, std::move(frame)); + return; + } + + // For passive refresh requests, just deliver the resurrected frame. + if (event == VideoCaptureOracle::kPassiveRefreshRequest) { + DidCaptureFrame(frame_number, oracle_frame_number, std::move(frame)); + return; + } + + // Request a copy of the next frame from the frame sink. + const gfx::Rect content_rect = + media::ComputeLetterboxRegionForI420(frame->visible_rect(), source_size); + std::unique_ptr<CopyOutputRequest> request(new CopyOutputRequest( + CopyOutputRequest::ResultFormat::I420_PLANES, + base::BindOnce(&FrameSinkVideoCapturerImpl::DidCopyFrame, + capture_weak_factory_.GetWeakPtr(), frame_number, + oracle_frame_number, std::move(frame), content_rect))); + request->set_source(copy_request_source_); + request->set_area(gfx::Rect(source_size)); + request->SetScaleRatio( + gfx::Vector2d(source_size.width(), source_size.height()), + gfx::Vector2d(content_rect.width(), content_rect.height())); + // TODO(crbug.com/775740): As an optimization, set the result selection to + // just the part of the result that would have changed due to aggregated + // damage over all the frames that weren't captured. + request->set_result_selection(gfx::Rect(content_rect.size())); + resolved_target_->RequestCopyOfSurface(std::move(request)); +} + +void FrameSinkVideoCapturerImpl::DidCopyFrame( + int64_t frame_number, + OracleFrameNumber oracle_frame_number, + scoped_refptr<VideoFrame> frame, + const gfx::Rect& content_rect, + std::unique_ptr<CopyOutputResult> result) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK_GE(frame_number, next_delivery_frame_number_); + DCHECK(frame); + DCHECK_EQ(content_rect.x() % 2, 0); + DCHECK_EQ(content_rect.y() % 2, 0); + DCHECK_EQ(content_rect.width() % 2, 0); + DCHECK_EQ(content_rect.height() % 2, 0); + DCHECK(result); + + // Stop() should have canceled any outstanding copy requests. So, by reaching + // this point, there should be a |consumer_| present. + DCHECK(consumer_); + + // Populate the VideoFrame from the CopyOutputResult. + const int y_stride = frame->stride(VideoFrame::kYPlane); + uint8_t* const y = frame->visible_data(VideoFrame::kYPlane) + + content_rect.y() * y_stride + content_rect.x(); + const int u_stride = frame->stride(VideoFrame::kUPlane); + uint8_t* const u = frame->visible_data(VideoFrame::kUPlane) + + (content_rect.y() / 2) * u_stride + (content_rect.x() / 2); + const int v_stride = frame->stride(VideoFrame::kVPlane); + uint8_t* const v = frame->visible_data(VideoFrame::kVPlane) + + (content_rect.y() / 2) * v_stride + (content_rect.x() / 2); + if (result->ReadI420Planes(y, y_stride, u, u_stride, v, v_stride)) { + media::LetterboxYUV(frame.get(), content_rect); + } else { + frame = nullptr; + } + + DidCaptureFrame(frame_number, oracle_frame_number, std::move(frame)); +} + +void FrameSinkVideoCapturerImpl::DidCaptureFrame( + int64_t frame_number, + OracleFrameNumber oracle_frame_number, + scoped_refptr<VideoFrame> frame) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK_GE(frame_number, next_delivery_frame_number_); + + if (frame) { + frame->metadata()->SetTimeTicks(VideoFrameMetadata::CAPTURE_END_TIME, + clock_->NowTicks()); + } + + // Ensure frames are delivered in-order by using a min-heap, and only + // deliver the next frame(s) in-sequence when they are found at the top. + delivery_queue_.emplace(frame_number, oracle_frame_number, std::move(frame)); + while (delivery_queue_.top().frame_number == next_delivery_frame_number_) { + MaybeDeliverFrame(delivery_queue_.top().oracle_frame_number, + std::move(delivery_queue_.top().frame)); + ++next_delivery_frame_number_; + delivery_queue_.pop(); + if (delivery_queue_.empty()) { + break; + } + } +} + +void FrameSinkVideoCapturerImpl::MaybeDeliverFrame( + OracleFrameNumber oracle_frame_number, + scoped_refptr<VideoFrame> frame) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + // The Oracle has the final say in whether frame delivery will proceed. It + // also rewrites the media timestamp in terms of the smooth flow of the + // original source content. + base::TimeTicks media_ticks; + const bool should_deliver_frame = + oracle_.CompleteCapture(oracle_frame_number, !!frame, &media_ticks); + DCHECK(frame || !should_deliver_frame); + + // The following is used by + // chrome/browser/extension/api/cast_streaming/performance_test.cc, in + // addition to the usual runtime tracing. + TRACE_EVENT_ASYNC_END2("gpu.capture", "Capture", frame.get(), "success", + should_deliver_frame, "timestamp", + (media_ticks - base::TimeTicks()).InMicroseconds()); + + if (!should_deliver_frame) { + return; + } + + // Set media timestamp in terms of the time offset since the first frame. + if (!first_frame_media_ticks_) { + first_frame_media_ticks_ = media_ticks; + } + frame->set_timestamp(media_ticks - *first_frame_media_ticks_); + + // Construct a new InFlightFrameDelivery instance, to provide the done signal + // for this frame and deliver the frame to the consumer. + const gfx::Rect update_rect = frame->visible_rect(); + auto delivery = std::make_unique<InFlightFrameDelivery>( + frame_pool_.HoldFrameForDelivery(frame.get()), + base::BindOnce(&VideoCaptureOracle::RecordConsumerFeedback, + feedback_weak_factory_.GetWeakPtr(), oracle_frame_number)); + consumer_->OnFrameCaptured(std::move(frame), update_rect, + std::move(delivery)); +} + +FrameSinkVideoCapturerImpl::CapturedFrame::CapturedFrame( + int64_t fn, + OracleFrameNumber ofn, + scoped_refptr<VideoFrame> fr) + : frame_number(fn), oracle_frame_number(ofn), frame(std::move(fr)) {} + +FrameSinkVideoCapturerImpl::CapturedFrame::CapturedFrame( + const CapturedFrame& other) = default; + +FrameSinkVideoCapturerImpl::CapturedFrame::~CapturedFrame() = default; + +bool FrameSinkVideoCapturerImpl::CapturedFrame::operator<( + const FrameSinkVideoCapturerImpl::CapturedFrame& other) const { + // Reverse the sort order; so std::priority_queue<CapturedFrame> becomes a + // min-heap instead of a max-heap. + return other.frame_number < frame_number; +} + +} // namespace viz diff --git a/chromium/components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_impl.h b/chromium/components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_impl.h new file mode 100644 index 00000000000..ee95cecb42f --- /dev/null +++ b/chromium/components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_impl.h @@ -0,0 +1,241 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_FRAME_SINK_VIDEO_CAPTURER_IMPL_H_ +#define COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_FRAME_SINK_VIDEO_CAPTURER_IMPL_H_ + +#include <stdint.h> + +#include <array> +#include <queue> + +#include "base/macros.h" +#include "base/memory/weak_ptr.h" +#include "base/optional.h" +#include "base/sequence_checker.h" +#include "base/time/default_tick_clock.h" +#include "base/time/time.h" +#include "base/unguessable_token.h" +#include "components/viz/common/frame_sinks/begin_frame_args.h" +#include "components/viz/common/surfaces/frame_sink_id.h" +#include "components/viz/service/frame_sinks/video_capture/capturable_frame_sink.h" +#include "components/viz/service/frame_sinks/video_capture/in_flight_frame_delivery.h" +#include "components/viz/service/frame_sinks/video_capture/interprocess_frame_pool.h" +#include "components/viz/service/viz_service_export.h" +#include "media/base/video_frame.h" +#include "media/capture/content/video_capture_oracle.h" + +namespace gfx { +class Rect; +class Size; +} // namespace gfx + +namespace viz { + +class FrameSinkVideoCapturerManager; +class FrameSinkVideoConsumer; +class CopyOutputResult; + +// Captures the frames of a CompositorFrameSink's surface as a video stream. +// +// The capturer works with FrameSinkVideoCapturerManager to resolve the capture +// target, a CapturableFrameSink, from a given FrameSinkId. Once the +// target is resolved, this capturer attaches to it to receive events of +// interest regarding the frame flow, display timiming, and changes to the frame +// sink's surface. For some subset of frames, decided by +// media::VideoCaptureOracle, this capturer will make a CopyOutputRequest on the +// surface. +class VIZ_SERVICE_EXPORT FrameSinkVideoCapturerImpl final + : public CapturableFrameSink::Client { + public: + // |frame_sink_manager| must outlive this instance. + explicit FrameSinkVideoCapturerImpl( + FrameSinkVideoCapturerManager* frame_sink_manager); + + ~FrameSinkVideoCapturerImpl() final; + + // The currently-requested frame sink for capture. The frame sink manager + // calls this when it learns of a new CapturableFrameSink to see if the target + // can be resolved. + const FrameSinkId& requested_target() const { return requested_target_; } + + // Sets the resolved target, detaching this capturer from the previous target + // (if any), and attaching to the new target. This is called by the frame sink + // manager. If |target| is null, the capturer goes idle and expects this + // method to be called again in the near future, once the target becomes known + // to the frame sink manager. + void SetResolvedTarget(CapturableFrameSink* target); + + // Notifies this capturer that the current target will be destroyed, and the + // FrameSinkId associated with it has become invalid. This is called by the + // frame sink manager. + void OnTargetWillGoAway(); + + // TODO(crbug.com/754872): mojom::FrameSinkVideoCapturer implementation: + void SetFormat(media::VideoPixelFormat format, media::ColorSpace color_space); + void SetMinCapturePeriod(base::TimeDelta min_capture_period); + void SetResolutionConstraints(const gfx::Size& min_size, + const gfx::Size& max_size, + bool use_fixed_aspect_ratio); + void ChangeTarget(const FrameSinkId& frame_sink_id); + void Start(std::unique_ptr<FrameSinkVideoConsumer> consumer); + void Stop(); + void RequestRefreshFrame(); + + // Default configuration. + static constexpr media::VideoPixelFormat kDefaultPixelFormat = + media::PIXEL_FORMAT_I420; + static constexpr media::ColorSpace kDefaultColorSpace = + media::COLOR_SPACE_HD_REC709; + + // The maximum number of frames in-flight in the capture pipeline, reflecting + // the storage capacity dedicated for this purpose. Example numbers, for a + // frame pool that is fully-allocated with 10 frames of size 1920x1080, using + // the I420 pixel format (12 bits per pixel). Then: + // + // storage_bytes_for_all_ten_frames = 10 * (1920 * 1080 * 12/8) + // --> ~29.7 MB + // + // In practice, only 0-3 frames will be in-flight at a time, depending on the + // content change rate and system performance. + static constexpr int kDesignLimitMaxFrames = 10; + + // A safe, sustainable maximum number of frames in-flight. In other words, + // exceeding 60% of the design limit is considered "red line" operation. + static constexpr float kTargetPipelineUtilization = 0.6f; + + private: + friend class FrameSinkVideoCapturerTest; + + using BeginFrameSourceId = decltype(BeginFrameArgs::source_id); + using OracleFrameNumber = + decltype(std::declval<media::VideoCaptureOracle>().next_frame_number()); + + // CapturableFrameSink::Client implementation: + void OnBeginFrame(const BeginFrameArgs& args) final; + void OnFrameDamaged(const BeginFrameAck& ack, + const gfx::Size& frame_size, + const gfx::Rect& damage_rect) final; + + // Consults the VideoCaptureOracle to decide whether to capture a frame, + // then ensures prerequisites are met before initiating the capture: that + // there is a consumer present and that the pipeline is not already full. + void MaybeCaptureFrame(media::VideoCaptureOracle::Event event, + const gfx::Rect& damage_rect, + base::TimeTicks event_time); + + // Extracts the image data from the copy output |result|, populating the + // |content_rect| region of a [possibly letterboxed] video |frame|. + void DidCopyFrame(int64_t frame_number, + OracleFrameNumber oracle_frame_number, + scoped_refptr<media::VideoFrame> frame, + const gfx::Rect& content_rect, + std::unique_ptr<CopyOutputResult> result); + + // Places the frame in the |delivery_queue_| and calls MaybeDeliverFrame(), + // one frame at a time, in-order. |frame| may be null to indicate a + // completed, but unsuccessful capture. + void DidCaptureFrame(int64_t frame_number, + OracleFrameNumber oracle_frame_number, + scoped_refptr<media::VideoFrame> frame); + + // Delivers a |frame| to the consumer, if the VideoCaptureOracle allows + // it. |frame| can be null to indicate a completed, but unsuccessful capture. + // In this case, some state will be updated, but nothing will be sent to the + // consumer. + void MaybeDeliverFrame(OracleFrameNumber oracle_frame_number, + scoped_refptr<media::VideoFrame> frame); + + // Owner/Manager of this instance. + FrameSinkVideoCapturerManager* const frame_sink_manager_; + + // Represents this instance as an issuer of CopyOutputRequests. The Surface + // uses this to auto-cancel stale requests (i.e., prior requests that did not + // execute). Also, the implementations that execute CopyOutputRequests use + // this as a hint to cache objects for a huge performance improvement. + const base::UnguessableToken copy_request_source_; + + // Use the default base::TimeTicks clock; but allow unit tests to provide a + // replacement. + base::DefaultTickClock default_tick_clock_; + base::TickClock* clock_ = &default_tick_clock_; + + // Current image format. + media::VideoPixelFormat pixel_format_ = kDefaultPixelFormat; + media::ColorSpace color_space_ = kDefaultColorSpace; + + // Models current content change/draw behavior and proposes when to capture + // frames, and at what size and frame rate. + media::VideoCaptureOracle oracle_; + + // The target requested by the client, as provided in the last call to + // ChangeTarget(). + FrameSinkId requested_target_; + + // The resolved target of video capture, or null if the requested target does + // not yet exist (or no longer exists). + CapturableFrameSink* resolved_target_ = nullptr; + + // The current video frame consumer. This is set when Start() is called and + // cleared when Stop() is called. + std::unique_ptr<FrameSinkVideoConsumer> consumer_; + + // A cache of recently-recorded future frame display times, according to the + // BeginFrameArgs passed to OnBeginFrame() calls. This array is a ring buffer + // mapping BeginFrameArgs::sequence_number to the expected display time. + std::array<base::TimeTicks, kDesignLimitMaxFrames> frame_display_times_; + + // The following is used to track when the BeginFrameSource changes, to + // determine whether the |frame_display_times_| are valid for the current + // source. + BeginFrameSourceId current_begin_frame_source_id_ = + BeginFrameArgs::kStartingSourceId; + + // These are sequence counters used to ensure that the frames are being + // delivered in the same order they are captured. + int64_t next_capture_frame_number_ = 0; + int64_t next_delivery_frame_number_ = 0; + + // Provides a pool of VideoFrames that can be efficiently delivered across + // processes. The size of this pool is used to limit the maximum number of + // frames in-flight at any one time. + InterprocessFramePool frame_pool_; + + // A queue of captured frames pending delivery. This queue is used to re-order + // frames, if they should happen to be captured out-of-order. + struct CapturedFrame { + int64_t frame_number; + OracleFrameNumber oracle_frame_number; + scoped_refptr<media::VideoFrame> frame; + CapturedFrame(int64_t frame_number, + OracleFrameNumber oracle_frame_number, + scoped_refptr<media::VideoFrame> frame); + CapturedFrame(const CapturedFrame& other); + ~CapturedFrame(); + bool operator<(const CapturedFrame& other) const; + }; + std::priority_queue<CapturedFrame> delivery_queue_; + + // The Oracle-provided media timestamp of the first frame. This is used to + // compute the relative media stream timestamps for each successive frame. + base::Optional<base::TimeTicks> first_frame_media_ticks_; + + // This class assumes its control operations and async callbacks won't execute + // simultaneously. + SEQUENCE_CHECKER(sequence_checker_); + + // A weak pointer factory used for cancelling consumer feedback from any + // in-flight frame deliveries. + base::WeakPtrFactory<media::VideoCaptureOracle> feedback_weak_factory_; + + // A weak pointer factory used for cancelling the results from any in-flight + // copy output requests. + base::WeakPtrFactory<FrameSinkVideoCapturerImpl> capture_weak_factory_; + + DISALLOW_COPY_AND_ASSIGN(FrameSinkVideoCapturerImpl); +}; + +} // namespace viz + +#endif // COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_FRAME_SINK_VIDEO_CAPTURER_IMPL_H_ diff --git a/chromium/components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_impl_unittest.cc b/chromium/components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_impl_unittest.cc new file mode 100644 index 00000000000..936497866d4 --- /dev/null +++ b/chromium/components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_impl_unittest.cc @@ -0,0 +1,730 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_impl.h" + +#include "base/bind.h" +#include "base/callback.h" +#include "base/optional.h" +#include "base/test/simple_test_tick_clock.h" +#include "base/time/time.h" +#include "components/viz/common/frame_sinks/begin_frame_args.h" +#include "components/viz/common/frame_sinks/copy_output_request.h" +#include "components/viz/common/frame_sinks/copy_output_result.h" +#include "components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_manager.h" +#include "components/viz/service/frame_sinks/video_capture/frame_sink_video_consumer.h" +#include "media/base/limits.h" +#include "media/base/video_util.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/skia/include/core/SkBitmap.h" +#include "ui/gfx/color_space.h" +#include "ui/gfx/geometry/point.h" +#include "ui/gfx/geometry/rect.h" +#include "ui/gfx/geometry/size.h" + +using media::VideoCaptureOracle; +using media::VideoFrame; +using media::VideoFrameMetadata; + +using testing::_; +using testing::InvokeWithoutArgs; +using testing::NiceMock; +using testing::Return; + +namespace viz { +namespace { + +// Dummy frame sink ID. +constexpr FrameSinkId kFrameSinkId = FrameSinkId(1, 1); + +// The compositor frame interval. +constexpr base::TimeDelta kVsyncInterval = + base::TimeDelta::FromSecondsD(1.0 / 60.0); + +// The size of the compositor frame sink's Surface. +constexpr gfx::Size kSourceSize = gfx::Size(100, 100); + +// The size of the VideoFrames produced by the capturer. +constexpr gfx::Size kCaptureSize = gfx::Size(32, 18); + +// The location of the letterboxed content within each VideoFrame. All pixels +// outside of this region should be black. +constexpr gfx::Rect kContentRect = gfx::Rect(6, 0, 18, 18); + +struct YUVColor { + uint8_t y; + uint8_t u; + uint8_t v; +}; + +class MockFrameSinkManager : public FrameSinkVideoCapturerManager { + public: + MOCK_METHOD1(FindCapturableFrameSink, + CapturableFrameSink*(const FrameSinkId& frame_sink_id)); + MOCK_METHOD1(OnCapturerConnectionLost, + void(FrameSinkVideoCapturerImpl* capturer)); +}; + +class MockConsumer : public FrameSinkVideoConsumer { + public: + MOCK_METHOD3(OnFrameCapturedMock, + void(scoped_refptr<VideoFrame> frame, + const gfx::Rect& update_rect, + InFlightFrameDelivery* delivery)); + MOCK_METHOD1(OnTargetLost, void(const FrameSinkId& frame_sink_id)); + MOCK_METHOD0(OnStopped, void()); + + int num_frames_received() const { return frames_.size(); } + + scoped_refptr<VideoFrame> TakeFrame(int i) { return std::move(frames_[i]); } + + void SendDoneNotification(int i) { std::move(done_callbacks_[i]).Run(); } + + private: + void OnFrameCaptured(scoped_refptr<VideoFrame> frame, + const gfx::Rect& update_rect, + std::unique_ptr<InFlightFrameDelivery> delivery) final { + ASSERT_TRUE(frame.get()); + EXPECT_EQ(gfx::Rect(kCaptureSize), update_rect); + ASSERT_TRUE(delivery.get()); + OnFrameCapturedMock(frame, update_rect, delivery.get()); + frames_.push_back(std::move(frame)); + done_callbacks_.push_back( + base::BindOnce(&InFlightFrameDelivery::Done, base::Passed(&delivery))); + } + + std::vector<scoped_refptr<VideoFrame>> frames_; + std::vector<base::OnceClosure> done_callbacks_; +}; + +class SolidColorI420Result : public CopyOutputResult { + public: + SolidColorI420Result(const gfx::Rect rect, YUVColor color) + : CopyOutputResult(CopyOutputResult::Format::I420_PLANES, rect), + color_(color) {} + + bool ReadI420Planes(uint8_t* y_out, + int y_out_stride, + uint8_t* u_out, + int u_out_stride, + uint8_t* v_out, + int v_out_stride) const final { + CHECK(y_out); + CHECK(y_out_stride >= size().width()); + CHECK(u_out); + const int chroma_width = (size().width() + 1) / 2; + CHECK(u_out_stride >= chroma_width); + CHECK(v_out); + CHECK(v_out_stride >= chroma_width); + for (int i = 0; i < size().height(); ++i, y_out += y_out_stride) { + memset(y_out, color_.y, size().width()); + } + const int chroma_height = (size().height() + 1) / 2; + for (int i = 0; i < chroma_height; ++i, u_out += u_out_stride) { + memset(u_out, color_.u, chroma_width); + } + for (int i = 0; i < chroma_height; ++i, v_out += v_out_stride) { + memset(v_out, color_.v, chroma_width); + } + return true; + } + + private: + const YUVColor color_; +}; + +class FakeCapturableFrameSink : public CapturableFrameSink { + public: + Client* attached_client() const { return client_; } + + void AttachCaptureClient(Client* client) override { + ASSERT_FALSE(client_); + ASSERT_TRUE(client); + client_ = client; + } + + void DetachCaptureClient(Client* client) override { + ASSERT_TRUE(client); + ASSERT_EQ(client, client_); + client_ = nullptr; + } + + gfx::Size GetSurfaceSize() override { return kSourceSize; } + + void RequestCopyOfSurface( + std::unique_ptr<CopyOutputRequest> request) override { + EXPECT_EQ(CopyOutputResult::Format::I420_PLANES, request->result_format()); + EXPECT_NE(base::UnguessableToken(), request->source()); + EXPECT_EQ(gfx::Rect(kSourceSize), request->area()); + EXPECT_EQ(gfx::Rect(kContentRect.size()), request->result_selection()); + + auto result = std::make_unique<SolidColorI420Result>( + request->result_selection(), color_); + results_.push_back(base::BindOnce( + [](std::unique_ptr<CopyOutputRequest> request, + std::unique_ptr<CopyOutputResult> result) { + request->SendResult(std::move(result)); + }, + base::Passed(&request), base::Passed(&result))); + } + + void SetCopyOutputColor(YUVColor color) { color_ = color; } + + int num_copy_results() const { return results_.size(); } + + void SendCopyOutputResult(int offset) { + auto it = results_.begin() + offset; + std::move(*it).Run(); + } + + private: + CapturableFrameSink::Client* client_ = nullptr; + YUVColor color_ = {0xde, 0xad, 0xbf}; + + std::vector<base::OnceClosure> results_; +}; + +// Matcher that returns true if the content region of a letterboxed VideoFrame +// is filled with the given color, and black everywhere else. +MATCHER_P(IsLetterboxedFrame, color, "") { + if (!arg) { + return false; + } + + const VideoFrame& frame = *arg; + const auto IsLetterboxedPlane = [&frame](int plane, uint8_t color) { + gfx::Rect content_rect = kContentRect; + if (plane != VideoFrame::kYPlane) { + content_rect = + gfx::Rect(content_rect.x() / 2, content_rect.y() / 2, + content_rect.width() / 2, content_rect.height() / 2); + } + for (int row = 0; row < frame.rows(plane); ++row) { + const uint8_t* p = frame.visible_data(plane) + row * frame.stride(plane); + for (int col = 0; col < frame.row_bytes(plane); ++col) { + if (content_rect.Contains(gfx::Point(col, row))) { + if (p[col] != color) { + return false; + } + } else { // Letterbox border around content. + if (plane == VideoFrame::kYPlane && p[col] != 0x00) { + return false; + } + } + } + } + return true; + }; + + return IsLetterboxedPlane(VideoFrame::kYPlane, color.y) && + IsLetterboxedPlane(VideoFrame::kUPlane, color.u) && + IsLetterboxedPlane(VideoFrame::kVPlane, color.v); +} + +} // namespace + +class FrameSinkVideoCapturerTest : public testing::Test { + public: + FrameSinkVideoCapturerTest() : capturer_(&frame_sink_manager_) {} + + void SetUp() override { + // Override the capturer's TickClock with the one controlled by the tests. + start_time_ = base::TimeTicks() + base::TimeDelta::FromSeconds(1); + clock_.SetNowTicks(start_time_); + capturer_.clock_ = &clock_; + + // Before setting the format, ensure the defaults are in-place. Then, for + // these tests, set a specific format and color space. + ASSERT_EQ(FrameSinkVideoCapturerImpl::kDefaultPixelFormat, + capturer_.pixel_format_); + ASSERT_EQ(FrameSinkVideoCapturerImpl::kDefaultColorSpace, + capturer_.color_space_); + capturer_.SetFormat(media::PIXEL_FORMAT_I420, media::COLOR_SPACE_HD_REC709); + ASSERT_EQ(media::PIXEL_FORMAT_I420, capturer_.pixel_format_); + ASSERT_EQ(media::COLOR_SPACE_HD_REC709, capturer_.color_space_); + + // Set min capture period as small as possible so that the + // media::VideoCapturerOracle used by the capturer will want to capture + // every composited frame. The capturer will override the too-small value of + // zero with a value based on media::limits::kMaxFramesPerSecond. + capturer_.SetMinCapturePeriod(base::TimeDelta()); + ASSERT_LT(base::TimeDelta(), capturer_.oracle_.min_capture_period()); + + capturer_.SetResolutionConstraints(kCaptureSize, kCaptureSize, false); + } + + void AdvanceClockToNextVsync() { + const auto num_vsyncs_elapsed = + (clock_.NowTicks() - start_time_) / kVsyncInterval; + const auto advance_to_time = + start_time_ + (num_vsyncs_elapsed + 1) * kVsyncInterval; + clock_.Advance(advance_to_time - clock_.NowTicks()); + } + + void NotifyBeginFrame(int frame_number) { + BeginFrameArgs args; + args.interval = kVsyncInterval; + args.frame_time = clock_.NowTicks(); + args.sequence_number = BeginFrameArgs::kStartingFrameNumber + frame_number; + args.source_id = 1; + capturer_.OnBeginFrame(args); + } + + void NotifyFrameDamaged(int frame_number) { + BeginFrameAck ack; + ack.sequence_number = BeginFrameArgs::kStartingFrameNumber + frame_number; + ack.source_id = 1; + capturer_.OnFrameDamaged(ack, kSourceSize, gfx::Rect(kSourceSize)); + } + + protected: + base::TimeTicks start_time_; + base::SimpleTestTickClock clock_; + MockFrameSinkManager frame_sink_manager_; + FakeCapturableFrameSink frame_sink_; + FrameSinkVideoCapturerImpl capturer_; +}; + +// Tests that the capturer attaches to a frame sink immediately, in the case +// where the frame sink was already known by the manger. +TEST_F(FrameSinkVideoCapturerTest, ResolvesTargetImmediately) { + EXPECT_CALL(frame_sink_manager_, FindCapturableFrameSink(kFrameSinkId)) + .WillRepeatedly(Return(&frame_sink_)); + + EXPECT_EQ(FrameSinkId(), capturer_.requested_target()); + capturer_.ChangeTarget(kFrameSinkId); + EXPECT_EQ(kFrameSinkId, capturer_.requested_target()); + EXPECT_EQ(&capturer_, frame_sink_.attached_client()); +} + +// Tests that the capturer attaches to a frame sink later, in the case where the +// frame sink becomes known to the manager at some later point. +TEST_F(FrameSinkVideoCapturerTest, ResolvesTargetLater) { + EXPECT_CALL(frame_sink_manager_, FindCapturableFrameSink(kFrameSinkId)) + .WillRepeatedly(Return(nullptr)); + + EXPECT_EQ(FrameSinkId(), capturer_.requested_target()); + capturer_.ChangeTarget(kFrameSinkId); + EXPECT_EQ(kFrameSinkId, capturer_.requested_target()); + EXPECT_EQ(nullptr, frame_sink_.attached_client()); + + capturer_.SetResolvedTarget(&frame_sink_); + EXPECT_EQ(&capturer_, frame_sink_.attached_client()); +} + +// Tests that the capturer reports a lost target to the consumer. The consumer +// may then change targets to capture something else. +TEST_F(FrameSinkVideoCapturerTest, ReportsTargetLost) { + FakeCapturableFrameSink prior_frame_sink; + constexpr FrameSinkId kPriorFrameSinkId = FrameSinkId(1, 2); + EXPECT_CALL(frame_sink_manager_, FindCapturableFrameSink(kPriorFrameSinkId)) + .WillOnce(Return(&prior_frame_sink)); + EXPECT_CALL(frame_sink_manager_, FindCapturableFrameSink(kFrameSinkId)) + .WillOnce(Return(&frame_sink_)); + + auto consumer = std::make_unique<NiceMock<MockConsumer>>(); + EXPECT_CALL(*consumer, OnTargetLost(kPriorFrameSinkId)).Times(1); + capturer_.Start(std::move(consumer)); + + capturer_.ChangeTarget(kPriorFrameSinkId); + EXPECT_EQ(kPriorFrameSinkId, capturer_.requested_target()); + EXPECT_EQ(&capturer_, prior_frame_sink.attached_client()); + EXPECT_EQ(nullptr, frame_sink_.attached_client()); + + capturer_.OnTargetWillGoAway(); + EXPECT_EQ(nullptr, prior_frame_sink.attached_client()); + EXPECT_EQ(nullptr, frame_sink_.attached_client()); + + capturer_.ChangeTarget(kFrameSinkId); + EXPECT_EQ(kFrameSinkId, capturer_.requested_target()); + EXPECT_EQ(nullptr, prior_frame_sink.attached_client()); + EXPECT_EQ(&capturer_, frame_sink_.attached_client()); + + capturer_.Stop(); +} + +// Tests that an initial black frame is sent, in the case where a target is not +// resolved at the time Start() is called. +TEST_F(FrameSinkVideoCapturerTest, SendsBlackFrameOnStartWithoutATarget) { + EXPECT_CALL(frame_sink_manager_, FindCapturableFrameSink(kFrameSinkId)) + .WillRepeatedly(Return(&frame_sink_)); + + auto consumer = std::make_unique<MockConsumer>(); + EXPECT_CALL( + *consumer, + OnFrameCapturedMock(IsLetterboxedFrame(YUVColor{0x00, 0x80, 0x80}), _, _)) + .Times(1); + EXPECT_CALL(*consumer, OnTargetLost(kFrameSinkId)).Times(0); + EXPECT_CALL(*consumer, OnStopped()).Times(1); + + capturer_.Start(std::move(consumer)); + // A copy request was not necessary. + EXPECT_EQ(0, frame_sink_.num_copy_results()); + capturer_.Stop(); +} + +// An end-to-end pipeline test where compositor updates trigger the capturer to +// make copy requests, and a stream of video frames is delivered to the +// consumer. +TEST_F(FrameSinkVideoCapturerTest, CapturesCompositedFrames) { + frame_sink_.SetCopyOutputColor(YUVColor{0x80, 0x80, 0x80}); + EXPECT_CALL(frame_sink_manager_, FindCapturableFrameSink(kFrameSinkId)) + .WillRepeatedly(Return(&frame_sink_)); + + capturer_.ChangeTarget(kFrameSinkId); + + MockConsumer* consumer; + { + auto mock_consumer = std::make_unique<MockConsumer>(); + consumer = mock_consumer.get(); + capturer_.Start(std::move(mock_consumer)); + } + const int num_refresh_frames = 1; + const int num_update_frames = + 3 * FrameSinkVideoCapturerImpl::kDesignLimitMaxFrames; + EXPECT_CALL(*consumer, OnFrameCapturedMock(_, _, _)) + .Times(num_refresh_frames + num_update_frames); + EXPECT_CALL(*consumer, OnTargetLost(_)).Times(0); + EXPECT_CALL(*consumer, OnStopped()).Times(1); + + // To start, the capturer will make a copy request for the initial refresh + // frame. Simulate a copy result and expect to see the refresh frame delivered + // to the consumer. + ASSERT_EQ(num_refresh_frames, frame_sink_.num_copy_results()); + frame_sink_.SendCopyOutputResult(0); + ASSERT_EQ(num_refresh_frames, consumer->num_frames_received()); + EXPECT_THAT(consumer->TakeFrame(0), + IsLetterboxedFrame(YUVColor{0x80, 0x80, 0x80})); + consumer->SendDoneNotification(0); + + // Drive the capturer pipeline for a series of frame composites. + base::TimeDelta last_timestamp; + for (int i = num_refresh_frames; i < num_refresh_frames + num_update_frames; + ++i) { + SCOPED_TRACE(testing::Message() << "frame #" << i); + + // Move time forward to the next display vsync and notify the capturer that + // compositing of the frame has begun. + AdvanceClockToNextVsync(); + const base::TimeTicks expected_reference_time = + clock_.NowTicks() + kVsyncInterval; + NotifyBeginFrame(i); + + // Change the content of the frame sink and notify the capturer of the + // damage. + const YUVColor color = {i << 4, (i << 4) + 0x10, (i << 4) + 0x20}; + frame_sink_.SetCopyOutputColor(color); + clock_.Advance(kVsyncInterval / 4); + const base::TimeTicks expected_capture_begin_time = clock_.NowTicks(); + NotifyFrameDamaged(i); + + // The frame sink should have received a CopyOutputRequest. Simulate a short + // pause before the result is sent back to the capturer, and the capturer + // should then deliver the frame. + ASSERT_EQ(i + 1, frame_sink_.num_copy_results()); + clock_.Advance(kVsyncInterval / 4); + const base::TimeTicks expected_capture_end_time = clock_.NowTicks(); + frame_sink_.SendCopyOutputResult(i); + ASSERT_EQ(i + 1, consumer->num_frames_received()); + + // Verify the frame is the right size, has the right content, and has + // required metadata set. + const scoped_refptr<VideoFrame> frame = consumer->TakeFrame(i); + EXPECT_THAT(frame, IsLetterboxedFrame(color)); + EXPECT_EQ(kCaptureSize, frame->coded_size()); + EXPECT_EQ(gfx::Rect(kCaptureSize), frame->visible_rect()); + EXPECT_EQ(gfx::ColorSpace::CreateREC709(), frame->ColorSpace()); + EXPECT_LT(last_timestamp, frame->timestamp()); + last_timestamp = frame->timestamp(); + const VideoFrameMetadata* metadata = frame->metadata(); + base::TimeTicks capture_begin_time; + EXPECT_TRUE(metadata->GetTimeTicks(VideoFrameMetadata::CAPTURE_BEGIN_TIME, + &capture_begin_time)); + EXPECT_EQ(expected_capture_begin_time, capture_begin_time); + base::TimeTicks capture_end_time; + EXPECT_TRUE(metadata->GetTimeTicks(VideoFrameMetadata::CAPTURE_END_TIME, + &capture_end_time)); + EXPECT_EQ(expected_capture_end_time, capture_end_time); + int color_space = media::COLOR_SPACE_UNSPECIFIED; + EXPECT_TRUE( + metadata->GetInteger(VideoFrameMetadata::COLOR_SPACE, &color_space)); + EXPECT_EQ(media::COLOR_SPACE_HD_REC709, color_space); + EXPECT_TRUE(metadata->HasKey(VideoFrameMetadata::FRAME_DURATION)); + // FRAME_DURATION is an estimate computed by the VideoCaptureOracle, so it + // its exact value is not being checked here. + double frame_rate = 0.0; + EXPECT_TRUE( + metadata->GetDouble(VideoFrameMetadata::FRAME_RATE, &frame_rate)); + EXPECT_NEAR(media::limits::kMaxFramesPerSecond, frame_rate, 0.001); + base::TimeTicks reference_time; + EXPECT_TRUE(metadata->GetTimeTicks(VideoFrameMetadata::REFERENCE_TIME, + &reference_time)); + EXPECT_EQ(expected_reference_time, reference_time); + + // Notify the capturer that the consumer is done with the frame. + consumer->SendDoneNotification(i); + + if (HasFailure()) { + break; + } + } + + capturer_.Stop(); +} + +// Tests that frame capturing halts when too many frames are in-flight, whether +// that is because there are too many copy requests in-flight or because the +// consumer has not finished consuming frames fast enough. +TEST_F(FrameSinkVideoCapturerTest, HaltsWhenPipelineIsFull) { + EXPECT_CALL(frame_sink_manager_, FindCapturableFrameSink(kFrameSinkId)) + .WillRepeatedly(Return(&frame_sink_)); + + capturer_.ChangeTarget(kFrameSinkId); + + MockConsumer* consumer; + { + auto mock_consumer = std::make_unique<NiceMock<MockConsumer>>(); + consumer = mock_consumer.get(); + capturer_.Start(std::move(mock_consumer)); + } + + // Saturate the pipeline with CopyOutputRequests that have not yet executed. + const int num_refresh_frames = 1; + int num_frames = FrameSinkVideoCapturerImpl::kDesignLimitMaxFrames; + for (int i = num_refresh_frames; i < num_frames; ++i) { + AdvanceClockToNextVsync(); + NotifyBeginFrame(i); + NotifyFrameDamaged(i); + } + ASSERT_EQ(num_frames, frame_sink_.num_copy_results()); + + // Notifying the capturer of new compositor updates should cause no new copy + // requests to be issued at this point. + const int first_uncaptured_frame = num_frames; + AdvanceClockToNextVsync(); + NotifyBeginFrame(first_uncaptured_frame); + NotifyFrameDamaged(first_uncaptured_frame); + ASSERT_EQ(num_frames, frame_sink_.num_copy_results()); + + // Complete the first copy request. When notifying the capturer of another + // compositor update, no new copy requests should be issued because the first + // frame is still in the middle of being delivered/consumed. + frame_sink_.SendCopyOutputResult(0); + ASSERT_EQ(1, consumer->num_frames_received()); + const int second_uncaptured_frame = num_frames; + AdvanceClockToNextVsync(); + NotifyBeginFrame(second_uncaptured_frame); + NotifyFrameDamaged(second_uncaptured_frame); + ASSERT_EQ(num_frames, frame_sink_.num_copy_results()); + + // Notify the capturer that the first frame has been consumed. Then, with + // another compositor update, the capturer should issue another new copy + // request. + EXPECT_TRUE(consumer->TakeFrame(0)); + consumer->SendDoneNotification(0); + const int first_capture_resumed_frame = second_uncaptured_frame + 1; + AdvanceClockToNextVsync(); + NotifyBeginFrame(first_capture_resumed_frame); + NotifyFrameDamaged(first_capture_resumed_frame); + ++num_frames; + ASSERT_EQ(num_frames, frame_sink_.num_copy_results()); + + // With yet another compositor update, no new copy requests should be issued + // because the pipeline became saturated again. + const int third_uncaptured_frame = first_capture_resumed_frame + 1; + AdvanceClockToNextVsync(); + NotifyBeginFrame(third_uncaptured_frame); + NotifyFrameDamaged(third_uncaptured_frame); + ASSERT_EQ(num_frames, frame_sink_.num_copy_results()); + + // Complete all pending copy requests. Another compositor update should not + // cause any new copy requests to be issued because all frames are being + // delivered/consumed. + for (int i = 1; i < frame_sink_.num_copy_results(); ++i) { + SCOPED_TRACE(testing::Message() << "frame #" << i); + frame_sink_.SendCopyOutputResult(i); + } + ASSERT_EQ(frame_sink_.num_copy_results(), consumer->num_frames_received()); + const int fourth_uncaptured_frame = third_uncaptured_frame + 1; + AdvanceClockToNextVsync(); + NotifyBeginFrame(fourth_uncaptured_frame); + NotifyFrameDamaged(fourth_uncaptured_frame); + ASSERT_EQ(num_frames, frame_sink_.num_copy_results()); + + // Notify the capturer that all frames have been consumed. Finally, with + // another compositor update, capture should resume. + for (int i = 1; i < consumer->num_frames_received(); ++i) { + SCOPED_TRACE(testing::Message() << "frame #" << i); + EXPECT_TRUE(consumer->TakeFrame(i)); + consumer->SendDoneNotification(i); + } + const int second_capture_resumed_frame = fourth_uncaptured_frame + 1; + AdvanceClockToNextVsync(); + NotifyBeginFrame(second_capture_resumed_frame); + NotifyFrameDamaged(second_capture_resumed_frame); + ++num_frames; + ASSERT_EQ(num_frames, frame_sink_.num_copy_results()); + frame_sink_.SendCopyOutputResult(frame_sink_.num_copy_results() - 1); + ASSERT_EQ(frame_sink_.num_copy_results(), consumer->num_frames_received()); + + capturer_.Stop(); +} + +// Tests that copy requests completed out-of-order are accounted for by the +// capturer, with results delivered to the consumer in-order. +TEST_F(FrameSinkVideoCapturerTest, DeliversFramesInOrder) { + std::vector<YUVColor> colors; + colors.push_back(YUVColor{0x00, 0x80, 0x80}); + frame_sink_.SetCopyOutputColor(colors.back()); + EXPECT_CALL(frame_sink_manager_, FindCapturableFrameSink(kFrameSinkId)) + .WillRepeatedly(Return(&frame_sink_)); + + capturer_.ChangeTarget(kFrameSinkId); + + MockConsumer* consumer; + { + auto mock_consumer = std::make_unique<NiceMock<MockConsumer>>(); + consumer = mock_consumer.get(); + capturer_.Start(std::move(mock_consumer)); + } + + // Issue five CopyOutputRequests (1 refresh frame plus 4 compositor + // updates). Each composited frame has its content region set to a different + // color to check that the video frames are being delivered in-order. + const int num_refresh_frames = 1; + int num_frames = 5; + ASSERT_EQ(num_refresh_frames, frame_sink_.num_copy_results()); + for (int i = num_refresh_frames; i < num_frames; ++i) { + colors.push_back(YUVColor{static_cast<uint8_t>(i << 4), + static_cast<uint8_t>((i << 4) + 0x10), + static_cast<uint8_t>((i << 4) + 0x20)}); + frame_sink_.SetCopyOutputColor(colors.back()); + AdvanceClockToNextVsync(); + NotifyBeginFrame(i); + NotifyFrameDamaged(i); + } + ASSERT_EQ(num_frames, frame_sink_.num_copy_results()); + + // Complete the copy requests out-of-order. Check that frames are not + // delivered until they can all be delivered in-order, and that the content of + // each video frame is correct. + frame_sink_.SendCopyOutputResult(0); + ASSERT_EQ(1, consumer->num_frames_received()); + EXPECT_THAT(consumer->TakeFrame(0), IsLetterboxedFrame(colors[0])); + frame_sink_.SendCopyOutputResult(2); + ASSERT_EQ(1, consumer->num_frames_received()); // Waiting for frame 1. + frame_sink_.SendCopyOutputResult(3); + ASSERT_EQ(1, consumer->num_frames_received()); // Still waiting for frame 1. + frame_sink_.SendCopyOutputResult(1); + ASSERT_EQ(4, consumer->num_frames_received()); // Sent frames 1, 2, and 3. + EXPECT_THAT(consumer->TakeFrame(1), IsLetterboxedFrame(colors[1])); + EXPECT_THAT(consumer->TakeFrame(2), IsLetterboxedFrame(colors[2])); + EXPECT_THAT(consumer->TakeFrame(3), IsLetterboxedFrame(colors[3])); + frame_sink_.SendCopyOutputResult(4); + ASSERT_EQ(5, consumer->num_frames_received()); + EXPECT_THAT(consumer->TakeFrame(4), IsLetterboxedFrame(colors[4])); + + capturer_.Stop(); +} + +// Tests that in-flight copy requests are canceled when the capturer is +// stopped. When it is started again with a new consumer, only the results from +// newer copy requests should appear in video frames delivered to the consumer. +TEST_F(FrameSinkVideoCapturerTest, CancelsInFlightCapturesOnStop) { + const YUVColor color1 = {0xaa, 0xbb, 0xcc}; + frame_sink_.SetCopyOutputColor(color1); + EXPECT_CALL(frame_sink_manager_, FindCapturableFrameSink(kFrameSinkId)) + .WillRepeatedly(Return(&frame_sink_)); + + capturer_.ChangeTarget(kFrameSinkId); + + // Start capturing to the first consumer. + MockConsumer* consumer; + { + auto mock_consumer = std::make_unique<MockConsumer>(); + consumer = mock_consumer.get(); + capturer_.Start(std::move(mock_consumer)); + } + EXPECT_CALL(*consumer, OnFrameCapturedMock(_, _, _)).Times(2); + EXPECT_CALL(*consumer, OnTargetLost(_)).Times(0); + EXPECT_CALL(*consumer, OnStopped()).Times(1); + + // Issue three additional CopyOutputRequests. With the initial refresh frame, + // the total should be four. + int num_refresh_frames = 1; + ASSERT_EQ(num_refresh_frames, frame_sink_.num_copy_results()); + int num_copy_requests = 4; + for (int i = num_refresh_frames; i < num_copy_requests; ++i) { + SCOPED_TRACE(testing::Message() << "frame #" << i); + AdvanceClockToNextVsync(); + NotifyBeginFrame(i); + NotifyFrameDamaged(i); + } + ASSERT_EQ(num_copy_requests, frame_sink_.num_copy_results()); + + // Complete the first two copy requests. + int num_completed_captures = 2; + for (int i = 0; i < num_completed_captures; ++i) { + SCOPED_TRACE(testing::Message() << "frame #" << i); + frame_sink_.SendCopyOutputResult(i); + ASSERT_EQ(i + 1, consumer->num_frames_received()); + EXPECT_THAT(consumer->TakeFrame(i), IsLetterboxedFrame(color1)); + } + + // Stopping capture should cancel the remaning copy requests. + capturer_.Stop(); + + // Change the content color and start capturing to the second consumer. + const YUVColor color2 = {0xdd, 0xee, 0xff}; + frame_sink_.SetCopyOutputColor(color2); + MockConsumer* consumer2; + { + auto mock_consumer2 = std::make_unique<MockConsumer>(); + consumer2 = mock_consumer2.get(); + capturer_.Start(std::move(mock_consumer2)); + } + const int num_captures_for_second_consumer = 3; + EXPECT_CALL(*consumer2, OnFrameCapturedMock(_, _, _)) + .Times(num_captures_for_second_consumer); + EXPECT_CALL(*consumer2, OnTargetLost(_)).Times(0); + EXPECT_CALL(*consumer2, OnStopped()).Times(1); + + // Complete the copy requests for the first consumer. Expect that they have no + // effect on the second consumer. + for (int i = num_completed_captures; i < num_copy_requests; ++i) { + frame_sink_.SendCopyOutputResult(i); + ASSERT_EQ(0, consumer2->num_frames_received()); + } + num_completed_captures = 0; + + // Note: Because the clock hasn't advanced while switching consumers, the + // capturer won't send a refresh frame. This is because the VideoCaptureOracle + // thinks the frame rate would be too fast. + // + // TODO(crbug.com/785072): Revisit this because the capturer should always + // guarantee an initial video frame is sent to the consumer within a + // reasonable time period. In practice, it does; but if things happen too + // quickly, it might not. + num_refresh_frames = 0; + + // From here, any new copy requests should be executed with video frames + // delivered to the consumer containing |color2|. + for (int i = num_refresh_frames; i < num_captures_for_second_consumer; ++i) { + AdvanceClockToNextVsync(); + NotifyBeginFrame(num_copy_requests); + NotifyFrameDamaged(num_copy_requests); + ++num_copy_requests; + ASSERT_EQ(num_copy_requests, frame_sink_.num_copy_results()); + frame_sink_.SendCopyOutputResult(frame_sink_.num_copy_results() - 1); + ++num_completed_captures; + ASSERT_EQ(num_completed_captures, consumer2->num_frames_received()); + EXPECT_THAT(consumer2->TakeFrame(consumer2->num_frames_received() - 1), + IsLetterboxedFrame(color2)); + } + + capturer_.Stop(); +} + +} // namespace viz diff --git a/chromium/components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_manager.h b/chromium/components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_manager.h new file mode 100644 index 00000000000..15a682ae9ad --- /dev/null +++ b/chromium/components/viz/service/frame_sinks/video_capture/frame_sink_video_capturer_manager.h @@ -0,0 +1,35 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_FRAME_SINK_VIDEO_CAPTURER_MANAGER_H_ +#define COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_FRAME_SINK_VIDEO_CAPTURER_MANAGER_H_ + +namespace viz { + +class CapturableFrameSink; +class FrameSinkId; +class FrameSinkVideoCapturerImpl; + +// Interface implemented by the owner/manager of FrameSinkVideoCapturerImpl +// instances. This allows the capturer to query for the existence of a frame +// sink target and also notify the owner when it should be destroyed. +class FrameSinkVideoCapturerManager { + public: + // Returns the CapturableFrameSink implementation associated with the given + // |frame_sink_id|, or nullptr if unknown. + virtual CapturableFrameSink* FindCapturableFrameSink( + const FrameSinkId& frame_sink_id) = 0; + + // Called once, when the mojo binding for the given |capturer| has been + // closed. At this point, the capturer is a zombie waiting to be destroyed. + virtual void OnCapturerConnectionLost( + FrameSinkVideoCapturerImpl* capturer) = 0; + + protected: + virtual ~FrameSinkVideoCapturerManager() = default; +}; + +} // namespace viz + +#endif // COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_FRAME_SINK_VIDEO_CAPTURER_MANAGER_H_ diff --git a/chromium/components/viz/service/frame_sinks/video_capture/frame_sink_video_consumer.h b/chromium/components/viz/service/frame_sinks/video_capture/frame_sink_video_consumer.h new file mode 100644 index 00000000000..180de6607ef --- /dev/null +++ b/chromium/components/viz/service/frame_sinks/video_capture/frame_sink_video_consumer.h @@ -0,0 +1,48 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_FRAME_SINK_VIDEO_CONSUMER_H_ +#define COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_FRAME_SINK_VIDEO_CONSUMER_H_ + +#include <memory> + +#include "base/memory/ref_counted.h" + +namespace gfx { +class Rect; +} // namespace gfx + +namespace media { +class VideoFrame; +} // namespace media + +namespace viz { + +class FrameSinkId; +class InFlightFrameDelivery; + +// TODO(crbug.com/754872): This is a temporary placeholder; to be replaced with +// a mojo-IDL-generated interface. +class FrameSinkVideoConsumer { + public: + virtual ~FrameSinkVideoConsumer() = default; + + // Called to deliver each frame. + virtual void OnFrameCaptured( + scoped_refptr<media::VideoFrame> frame, + const gfx::Rect& update_rect, + std::unique_ptr<InFlightFrameDelivery> delivery) = 0; + + // Indicates the video capture target (a frame sink) has gone away. A consumer + // should use this to determine whether to change to a different target or + // shutdown. + virtual void OnTargetLost(const FrameSinkId& frame_sink_id) = 0; + + // Indicates that OnFrameCaptured() will not be called again. + virtual void OnStopped() = 0; +}; + +} // namespace viz + +#endif // COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_FRAME_SINK_VIDEO_CONSUMER_H_ diff --git a/chromium/components/viz/service/frame_sinks/video_capture/in_flight_frame_delivery.cc b/chromium/components/viz/service/frame_sinks/video_capture/in_flight_frame_delivery.cc new file mode 100644 index 00000000000..511b6649433 --- /dev/null +++ b/chromium/components/viz/service/frame_sinks/video_capture/in_flight_frame_delivery.cc @@ -0,0 +1,31 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/viz/service/frame_sinks/video_capture/in_flight_frame_delivery.h" + +namespace viz { + +InFlightFrameDelivery::InFlightFrameDelivery( + base::OnceClosure post_delivery_callback, + base::OnceCallback<void(double)> feedback_callback) + : post_delivery_callback_(std::move(post_delivery_callback)), + feedback_callback_(std::move(feedback_callback)) {} + +InFlightFrameDelivery::~InFlightFrameDelivery() { + Done(); +} + +void InFlightFrameDelivery::Done() { + if (!post_delivery_callback_.is_null()) { + std::move(post_delivery_callback_).Run(); + } +} + +void InFlightFrameDelivery::ProvideFeedback(double utilization) { + if (!feedback_callback_.is_null()) { + std::move(feedback_callback_).Run(utilization); + } +} + +} // namespace viz diff --git a/chromium/components/viz/service/frame_sinks/video_capture/in_flight_frame_delivery.h b/chromium/components/viz/service/frame_sinks/video_capture/in_flight_frame_delivery.h new file mode 100644 index 00000000000..847d2a242be --- /dev/null +++ b/chromium/components/viz/service/frame_sinks/video_capture/in_flight_frame_delivery.h @@ -0,0 +1,40 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_IN_FLIGHT_FRAME_DELIVERY_H_ +#define COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_IN_FLIGHT_FRAME_DELIVERY_H_ + +#include "base/callback.h" +#include "base/macros.h" +#include "components/viz/service/viz_service_export.h" + +namespace viz { + +// Represents an in-flight frame delivery to the consumer. Its main purpose is +// to proxy callbacks from the consumer back to the relevant capturer +// components owned and operated by FrameSinkVideoCapturerImpl. +// +// TODO(crbug.com/754872): This will extend a mojo-generated interface in an +// upcoming change. +class VIZ_SERVICE_EXPORT InFlightFrameDelivery { + public: + InFlightFrameDelivery(base::OnceClosure post_delivery_callback, + base::OnceCallback<void(double)> feedback_callback); + + virtual ~InFlightFrameDelivery(); + + // TODO(crbug.com/754872): mojom::FrameSinkVideoFrameCallbacks implementation: + void Done(); + void ProvideFeedback(double utilization); + + private: + base::OnceClosure post_delivery_callback_; + base::OnceCallback<void(double)> feedback_callback_; + + DISALLOW_COPY_AND_ASSIGN(InFlightFrameDelivery); +}; + +} // namespace viz + +#endif // COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_IN_FLIGHT_FRAME_DELIVERY_H_ diff --git a/chromium/components/viz/service/frame_sinks/video_capture/interprocess_frame_pool.cc b/chromium/components/viz/service/frame_sinks/video_capture/interprocess_frame_pool.cc new file mode 100644 index 00000000000..6effe72c449 --- /dev/null +++ b/chromium/components/viz/service/frame_sinks/video_capture/interprocess_frame_pool.cc @@ -0,0 +1,142 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/viz/service/frame_sinks/video_capture/interprocess_frame_pool.h" + +#include "base/bind.h" +#include "base/bind_helpers.h" +#include "media/base/video_frame.h" +#include "media/capture/video/video_capture_buffer_tracker_factory_impl.h" +#include "media/mojo/common/mojo_shared_buffer_video_frame.h" +#include "ui/gfx/geometry/size.h" + +using media::VideoCaptureBufferPool; +using media::VideoCaptureBufferPoolImpl; +using media::VideoFrame; +using media::VideoPixelFormat; + +namespace viz { + +InterprocessFramePool::InterprocessFramePool(int capacity) + : buffer_pool_(new VideoCaptureBufferPoolImpl( + std::make_unique<media::VideoCaptureBufferTrackerFactoryImpl>(), + capacity)), + weak_factory_(this) {} + +InterprocessFramePool::~InterprocessFramePool() = default; + +scoped_refptr<VideoFrame> InterprocessFramePool::ReserveVideoFrame( + const gfx::Size& size, + VideoPixelFormat format) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + BufferId ignored; // Not applicable to this buffer pool use case. + const BufferId buffer_id = buffer_pool_->ReserveForProducer( + size, format, media::PIXEL_STORAGE_CPU, 0 /* unused: frame_feedback_id */, + &ignored); + if (buffer_id == VideoCaptureBufferPool::kInvalidId) { + return nullptr; + } + resurrectable_buffer_id_ = VideoCaptureBufferPool::kInvalidId; + return WrapBuffer(buffer_id, size, format); +} + +scoped_refptr<VideoFrame> InterprocessFramePool::ResurrectLastVideoFrame( + const gfx::Size& expected_size, + VideoPixelFormat expected_format) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + const BufferId buffer_id = buffer_pool_->ResurrectLastForProducer( + expected_size, expected_format, media::PIXEL_STORAGE_CPU); + if (buffer_id != resurrectable_buffer_id_ || + buffer_id == VideoCaptureBufferPool::kInvalidId) { + return nullptr; + } + return WrapBuffer(buffer_id, expected_size, expected_format); +} + +base::OnceClosure InterprocessFramePool::HoldFrameForDelivery( + const VideoFrame* frame) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + const auto it = buffer_map_.find(frame); + DCHECK(it != buffer_map_.end()); + const BufferId buffer_id = it->second; + resurrectable_buffer_id_ = buffer_id; + buffer_pool_->HoldForConsumers(buffer_id, 1); + return base::BindOnce(&VideoCaptureBufferPoolImpl::RelinquishConsumerHold, + buffer_pool_, buffer_id, 1); +} + +float InterprocessFramePool::GetUtilization() const { + // Note: Technically, the |buffer_pool_| is completely thread-safe; so no + // sequence check is needed here. + return buffer_pool_->GetBufferPoolUtilization(); +} + +scoped_refptr<VideoFrame> InterprocessFramePool::WrapBuffer( + BufferId buffer_id, + const gfx::Size& size, + VideoPixelFormat format) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + // Mojo takes ownership of shared memory handles. Here, the pool has created + // and owns the original shared memory handle, but will duplicate it for Mojo. + mojo::ScopedSharedBufferHandle buffer = + buffer_pool_->GetHandleForInterProcessTransit(buffer_id, + false /* read-write */); + if (!buffer.is_valid()) { + return nullptr; + } + + const gfx::Size& y_plane_size = + VideoFrame::PlaneSize(format, VideoFrame::kYPlane, size); + const int y_plane_bytes = y_plane_size.GetArea(); + const gfx::Size& u_plane_size = + VideoFrame::PlaneSize(format, VideoFrame::kUPlane, size); + const int u_plane_bytes = u_plane_size.GetArea(); + const gfx::Size& v_plane_size = + VideoFrame::PlaneSize(format, VideoFrame::kVPlane, size); + const int v_plane_bytes = v_plane_size.GetArea(); + // TODO(miu): This could all be made much simpler if we had our own ctor in + // MojoSharedBufferVideoFrame. + const scoped_refptr<VideoFrame> frame = + media::MojoSharedBufferVideoFrame::Create( + format, size, gfx::Rect(size), size, std::move(buffer), + y_plane_bytes + u_plane_bytes + v_plane_bytes, 0 /* y_offset */, + y_plane_bytes /* u_offset */, + y_plane_bytes + u_plane_bytes /* v_offset */, y_plane_size.width(), + u_plane_size.width(), v_plane_size.width(), base::TimeDelta()); + DCHECK(frame); + + // Add an entry so that HoldFrameForDelivery() can identify the buffer backing + // this VideoFrame. + buffer_map_[frame.get()] = buffer_id; + + // Add a destruction observer to update |buffer_map_|. A WeakPtr is used for + // safety, just in case the VideoFrame's scope extends beyond that of this + // InterprocessFramePool. + frame->AddDestructionObserver(base::BindOnce( + &InterprocessFramePool::OnFrameWrapperDestroyed, + weak_factory_.GetWeakPtr(), base::Unretained(frame.get()))); + + // The second destruction observer is a callback to the + // VideoCaptureBufferPoolImpl directly. Since |buffer_pool_| is ref-counted, + // this means InterprocessFramePool can be safely destroyed even if there are + // still outstanding VideoFrames. + frame->AddDestructionObserver( + base::BindOnce(&VideoCaptureBufferPoolImpl::RelinquishProducerReservation, + buffer_pool_, buffer_id)); + return frame; +} + +void InterprocessFramePool::OnFrameWrapperDestroyed(const VideoFrame* frame) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + const auto it = buffer_map_.find(frame); + DCHECK(it != buffer_map_.end()); + buffer_map_.erase(it); +} + +} // namespace viz diff --git a/chromium/components/viz/service/frame_sinks/video_capture/interprocess_frame_pool.h b/chromium/components/viz/service/frame_sinks/video_capture/interprocess_frame_pool.h new file mode 100644 index 00000000000..59c7ec07e08 --- /dev/null +++ b/chromium/components/viz/service/frame_sinks/video_capture/interprocess_frame_pool.h @@ -0,0 +1,94 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_INTERPROCESS_FRAME_POOL_H_ +#define COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_INTERPROCESS_FRAME_POOL_H_ + +#include "base/callback.h" +#include "base/containers/flat_map.h" +#include "base/macros.h" +#include "base/memory/scoped_refptr.h" +#include "base/memory/weak_ptr.h" +#include "base/sequence_checker.h" +#include "components/viz/service/viz_service_export.h" +#include "media/base/video_types.h" +#include "media/capture/video/video_capture_buffer_pool_impl.h" + +namespace gfx { +class Size; +} + +namespace media { +class VideoFrame; +} + +namespace viz { + +// A pool of VideoFrames backed by shared memory that can be transported +// efficiently across mojo service boundaries. +class VIZ_SERVICE_EXPORT InterprocessFramePool { + public: + explicit InterprocessFramePool(int capacity); + ~InterprocessFramePool(); + + // Reserves a buffer from the pool and creates a I420 VideoFrame to wrap its + // shared memory. When the ref-count of the returned VideoFrame goes to zero, + // the reservation is released. Returns null if the pool is fully utilized. + scoped_refptr<media::VideoFrame> ReserveVideoFrame( + const gfx::Size& size, + media::VideoPixelFormat format); + + // Finds the last VideoFrame delivered, and attempts to re-materialize it. If + // the attempt fails, null is returned. This is used when the client knows the + // content of the video frame has not changed and is trying to avoid having to + // re-populate a new VideoFrame with the same content. + scoped_refptr<media::VideoFrame> ResurrectLastVideoFrame( + const gfx::Size& expected_size, + media::VideoPixelFormat expected_format); + + // Pins the memory buffer backing the given |frame| while it is being + // delivered to, and read by, downstream consumers. This prevents the memory + // buffer from being reserved until after the returned closure is run. The + // caller must guarantee the returned closure is run to prevent memory leaks. + base::OnceClosure HoldFrameForDelivery(const media::VideoFrame* frame); + + // Returns the current pool utilization, consisting of all reserved frames + // plus those being held for delivery. + float GetUtilization() const; + + private: + using BufferId = decltype(media::VideoCaptureBufferPool::kInvalidId + 0); + + // Helper to build a media::MojoSharedBufferVideoFrame instance backed by a + // specific buffer from the |buffer_pool_|. + scoped_refptr<media::VideoFrame> WrapBuffer(BufferId buffer_id, + const gfx::Size& size, + media::VideoPixelFormat format); + + // Called when a reserved/resurrected VideoFrame goes out of scope, to remove + // the entry from |buffer_map_|. + void OnFrameWrapperDestroyed(const media::VideoFrame* frame); + + // Maintains a pool of shared memory buffers for inter-process delivery of + // video frame data. + const scoped_refptr<media::VideoCaptureBufferPoolImpl> buffer_pool_; + + // The VideoFrames that are currently wrapping a specific buffer in the pool. + base::flat_map<const media::VideoFrame*, BufferId> buffer_map_; + + // The ID of the buffer that was last delivered. This is used to determine + // whether the previous VideoFrame was actually populated, and should be + // resurrected by ResurrectLastVideoFrame(). + BufferId resurrectable_buffer_id_ = media::VideoCaptureBufferPool::kInvalidId; + + SEQUENCE_CHECKER(sequence_checker_); + + base::WeakPtrFactory<InterprocessFramePool> weak_factory_; + + DISALLOW_COPY_AND_ASSIGN(InterprocessFramePool); +}; + +} // namespace viz + +#endif // COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_CAPTURE_INTERPROCESS_FRAME_POOL_H_ diff --git a/chromium/components/viz/service/frame_sinks/video_capture/interprocess_frame_pool_unittest.cc b/chromium/components/viz/service/frame_sinks/video_capture/interprocess_frame_pool_unittest.cc new file mode 100644 index 00000000000..22b50d42aeb --- /dev/null +++ b/chromium/components/viz/service/frame_sinks/video_capture/interprocess_frame_pool_unittest.cc @@ -0,0 +1,174 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/viz/service/frame_sinks/video_capture/interprocess_frame_pool.h" + +#include "media/base/video_frame.h" +#include "media/base/video_util.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "ui/gfx/geometry/rect.h" +#include "ui/gfx/geometry/size.h" + +using media::VideoFrame; + +namespace viz { +namespace { + +constexpr gfx::Size kSize = gfx::Size(32, 18); +constexpr media::VideoPixelFormat kFormat = media::PIXEL_FORMAT_I420; + +TEST(InterprocessFramePoolTest, FramesConfiguredCorrectly) { + InterprocessFramePool pool(1); + const scoped_refptr<media::VideoFrame> frame = + pool.ReserveVideoFrame(kSize, kFormat); + ASSERT_TRUE(frame); + ASSERT_EQ(kSize, frame->coded_size()); + ASSERT_EQ(gfx::Rect(kSize), frame->visible_rect()); + ASSERT_EQ(kSize, frame->natural_size()); + ASSERT_EQ(frame->storage_type(), VideoFrame::STORAGE_MOJO_SHARED_BUFFER); + ASSERT_TRUE(frame->IsMappable()); +} + +TEST(InterprocessFramePoolTest, ReachesCapacityLimit) { + InterprocessFramePool pool(2); + scoped_refptr<media::VideoFrame> frames[5]; + + // Reserve two frames from a pool of capacity 2. + frames[0] = pool.ReserveVideoFrame(kSize, kFormat); + ASSERT_TRUE(frames[0]); + frames[1] = pool.ReserveVideoFrame(kSize, kFormat); + ASSERT_TRUE(frames[1]); + + // Now, try to reserve a third frame. This should fail (return null). + frames[2] = pool.ReserveVideoFrame(kSize, kFormat); + ASSERT_FALSE(frames[2]); + + // Release the first frame. Then, retry reserving a frame. This should + // succeed. + frames[0] = nullptr; + frames[3] = pool.ReserveVideoFrame(kSize, kFormat); + ASSERT_TRUE(frames[3]); + + // Finally, try to reserve yet another frame. This should fail. + frames[4] = pool.ReserveVideoFrame(kSize, kFormat); + ASSERT_FALSE(frames[4]); +} + +// Returns true iff each plane of the given |frame| is filled with +// |values[plane]|. +bool PlanesAreFilledWithValues(const VideoFrame& frame, const uint8_t* values) { + static_assert(VideoFrame::kUPlane == (VideoFrame::kYPlane + 1) && + VideoFrame::kVPlane == (VideoFrame::kUPlane + 1), + "enum values changed, will break code below"); + for (int plane = VideoFrame::kYPlane; plane <= VideoFrame::kVPlane; ++plane) { + const uint8_t expected_value = values[plane - VideoFrame::kYPlane]; + for (int y = 0; y < frame.rows(plane); ++y) { + const uint8_t* row = frame.visible_data(plane) + y * frame.stride(plane); + for (int x = 0; x < frame.row_bytes(plane); ++x) { + EXPECT_EQ(expected_value, row[x]) + << "at row " << y << " in plane " << plane; + if (expected_value != row[x]) + return false; + } + } + } + return true; +} + +TEST(InterprocessFramePoolTest, ResurrectsDeliveredFramesOnly) { + InterprocessFramePool pool(2); + + // Reserve a frame, populate it, but release it before delivery. + scoped_refptr<media::VideoFrame> frame = + pool.ReserveVideoFrame(kSize, kFormat); + ASSERT_TRUE(frame); + media::FillYUV(frame.get(), 0x11, 0x22, 0x33); + frame = nullptr; + + // The pool should fail to resurrect the last frame because it was never + // delivered. + frame = pool.ResurrectLastVideoFrame(kSize, kFormat); + ASSERT_FALSE(frame); + + // Reserve a frame and populate it with different color values; only this + // time, deliver it before releasing it. + frame = pool.ReserveVideoFrame(kSize, kFormat); + ASSERT_TRUE(frame); + const uint8_t kValues[3] = {0x44, 0x55, 0x66}; + media::FillYUV(frame.get(), kValues[0], kValues[1], kValues[2]); + base::OnceClosure post_delivery_callback = + pool.HoldFrameForDelivery(frame.get()); + frame = nullptr; + std::move(post_delivery_callback).Run(); + + // Confirm that the last frame can be resurrected repeatedly. + for (int i = 0; i < 3; ++i) { + frame = pool.ResurrectLastVideoFrame(kSize, kFormat); + ASSERT_TRUE(frame); + ASSERT_TRUE(PlanesAreFilledWithValues(*frame, kValues)); + frame = nullptr; + } + + // A frame that is being delivered cannot be resurrected. + for (int i = 0; i < 2; ++i) { + if (i == 0) { // Test this for a resurrected frame. + frame = pool.ResurrectLastVideoFrame(kSize, kFormat); + ASSERT_TRUE(frame); + ASSERT_TRUE(PlanesAreFilledWithValues(*frame, kValues)); + } else { // Test this for a normal frame. + frame = pool.ReserveVideoFrame(kSize, kFormat); + ASSERT_TRUE(frame); + media::FillYUV(frame.get(), 0x77, 0x88, 0x99); + } + post_delivery_callback = pool.HoldFrameForDelivery(frame.get()); + frame = nullptr; + scoped_refptr<media::VideoFrame> should_be_null = + pool.ResurrectLastVideoFrame(kSize, kFormat); + ASSERT_FALSE(should_be_null); + std::move(post_delivery_callback).Run(); + } + + // Finally, reserve a frame, populate it, and don't deliver it. Expect that, + // still, undelivered frames cannot be resurrected. + frame = pool.ReserveVideoFrame(kSize, kFormat); + ASSERT_TRUE(frame); + media::FillYUV(frame.get(), 0xaa, 0xbb, 0xcc); + frame = nullptr; + frame = pool.ResurrectLastVideoFrame(kSize, kFormat); + ASSERT_FALSE(frame); +} + +TEST(InterprocessFramePoolTest, ReportsCorrectUtilization) { + InterprocessFramePool pool(2); + ASSERT_EQ(0.0f, pool.GetUtilization()); + + // Run through a typical sequence twice: Once for normal frame reservation, + // and the second time for a resurrected frame. + for (int i = 0; i < 2; ++i) { + // Reserve the frame and expect 1/2 the pool to be utilized. + scoped_refptr<media::VideoFrame> frame = + (i == 0) ? pool.ReserveVideoFrame(kSize, kFormat) + : pool.ResurrectLastVideoFrame(kSize, kFormat); + ASSERT_TRUE(frame); + ASSERT_EQ(0.5f, pool.GetUtilization()); + + // Hold the frame for delivery. This should not change the utilization. + base::OnceClosure post_delivery_callback = + pool.HoldFrameForDelivery(frame.get()); + ASSERT_EQ(0.5f, pool.GetUtilization()); + + // Release the frame. Since it is being held for delivery, this should not + // change the utilization. + frame = nullptr; + ASSERT_EQ(0.5f, pool.GetUtilization()); + + // Finally, run the callback to indicate the frame has been delivered. This + // should cause the utilization to go back down to zero. + std::move(post_delivery_callback).Run(); + ASSERT_EQ(0.0f, pool.GetUtilization()); + } +} + +} // namespace +} // namespace viz diff --git a/chromium/components/viz/service/frame_sinks/video_detector.cc b/chromium/components/viz/service/frame_sinks/video_detector.cc new file mode 100644 index 00000000000..6bf93c22b3b --- /dev/null +++ b/chromium/components/viz/service/frame_sinks/video_detector.cc @@ -0,0 +1,167 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "components/viz/service/frame_sinks/video_detector.h" + +#include "base/time/time.h" +#include "components/viz/common/quads/compositor_frame.h" +#include "components/viz/service/surfaces/surface_manager.h" +#include "ui/gfx/geometry/dip_util.h" +#include "ui/gfx/geometry/rect.h" + +namespace viz { + +constexpr base::TimeDelta VideoDetector::kVideoTimeout; +constexpr base::TimeDelta VideoDetector::kMinVideoDuration; + +// Stores information about updates to a client and determines whether it's +// likely that a video is playing in it. +class VideoDetector::ClientInfo { + public: + ClientInfo() = default; + + // Called when a Surface belonging to this client is drawn. Returns true if we + // determine that video is playing in this client. + bool ReportDrawnAndCheckForVideo(Surface* surface, base::TimeTicks now) { + uint64_t frame_index = surface->GetActiveFrameIndex(); + + // If |frame_index| hasn't increased, then no new frame was submitted since + // the last draw. + if (frame_index <= last_drawn_frame_index_) + return false; + + last_drawn_frame_index_ = frame_index; + + const CompositorFrame& frame = surface->GetActiveFrame(); + + gfx::Rect damage = + gfx::ConvertRectToDIP(frame.device_scale_factor(), + frame.render_pass_list.back()->damage_rect); + + if (damage.width() < kMinDamageWidth || damage.height() < kMinDamageHeight) + return false; + + // If the buffer is full, drop the first timestamp. + if (buffer_size_ == kMinFramesPerSecond) { + buffer_start_ = (buffer_start_ + 1) % kMinFramesPerSecond; + buffer_size_--; + } + + update_times_[(buffer_start_ + buffer_size_) % kMinFramesPerSecond] = now; + buffer_size_++; + + const bool in_video = + (buffer_size_ == kMinFramesPerSecond) && + (now - update_times_[buffer_start_] <= base::TimeDelta::FromSeconds(1)); + + if (in_video && video_start_time_.is_null()) + video_start_time_ = update_times_[buffer_start_]; + else if (!in_video && !video_start_time_.is_null()) + video_start_time_ = base::TimeTicks(); + + const base::TimeDelta elapsed = now - video_start_time_; + return in_video && elapsed >= kMinVideoDuration; + } + + private: + // Circular buffer containing update times of the last (up to + // |kMinFramesPerSecond|) video-sized updates to this client. + base::TimeTicks update_times_[kMinFramesPerSecond]; + + // Time at which the current sequence of updates that looks like video + // started. Empty if video isn't currently playing. + base::TimeTicks video_start_time_; + + // Index into |update_times_| of the oldest update. + uint32_t buffer_start_ = 0; + + // Number of updates stored in |update_times_|. + uint32_t buffer_size_ = 0; + + // Frame index of the last drawn Surface. We use this number to determine + // whether a new frame was submitted since the last time the Surface was + // drawn. + uint64_t last_drawn_frame_index_ = 0; + + DISALLOW_COPY_AND_ASSIGN(ClientInfo); +}; + +VideoDetector::VideoDetector( + SurfaceManager* surface_manager, + std::unique_ptr<base::TickClock> tick_clock, + scoped_refptr<base::SequencedTaskRunner> task_runner) + : tick_clock_(std::move(tick_clock)), + video_inactive_timer_(tick_clock_.get()), + surface_manager_(surface_manager) { + surface_manager_->AddObserver(this); + for (auto it : surface_manager_->valid_frame_sink_labels()) + client_infos_[it.first] = std::make_unique<ClientInfo>(); + if (task_runner) + video_inactive_timer_.SetTaskRunner(task_runner); +} + +VideoDetector::~VideoDetector() { + surface_manager_->RemoveObserver(this); +} + +void VideoDetector::OnVideoActivityEnded() { + DCHECK(video_is_playing_); + video_is_playing_ = false; + observers_.ForAllPtrs([](mojom::VideoDetectorObserver* observer) { + observer->OnVideoActivityEnded(); + }); +} + +void VideoDetector::AddObserver(mojom::VideoDetectorObserverPtr observer) { + if (video_is_playing_) + observer->OnVideoActivityStarted(); + observers_.AddPtr(std::move(observer)); +} + +void VideoDetector::OnFrameSinkIdRegistered(const FrameSinkId& frame_sink_id) { + DCHECK(!client_infos_.count(frame_sink_id)); + client_infos_[frame_sink_id] = std::make_unique<ClientInfo>(); +} + +void VideoDetector::OnFrameSinkIdInvalidated(const FrameSinkId& frame_sink_id) { + client_infos_.erase(frame_sink_id); +} + +bool VideoDetector::OnSurfaceDamaged(const SurfaceId& surface_id, + const BeginFrameAck& ack) { + return false; +} + +// |surface| is scheduled to be drawn. See if it has a new frame since the +// last time it was drawn and record the damage. +void VideoDetector::OnSurfaceWillBeDrawn(Surface* surface) { + // If there is no observer, don't waste cycles detecting video activity. + if (observers_.empty()) + return; + + const FrameSinkId& frame_sink_id = surface->surface_id().frame_sink_id(); + + auto it = client_infos_.find(frame_sink_id); + + // If the corresponding entry in |client_infos_| does not exist, it means the + // FrameSinkId has been invalidated and the client's CompositorFrameSink is + // destroyed so it cannot send new frames and video activity is impossible. + if (it == client_infos_.end()) + return; + + base::TimeTicks now = tick_clock_->NowTicks(); + + if (it->second->ReportDrawnAndCheckForVideo(surface, now)) { + video_inactive_timer_.Start(FROM_HERE, kVideoTimeout, this, + &VideoDetector::OnVideoActivityEnded); + if (!video_is_playing_) { + video_is_playing_ = true; + observers_.ForAllPtrs([](mojom::VideoDetectorObserver* observer) { + observer->OnVideoActivityStarted(); + }); + } + } +} + +} // namespace viz diff --git a/chromium/components/viz/service/frame_sinks/video_detector.h b/chromium/components/viz/service/frame_sinks/video_detector.h new file mode 100644 index 00000000000..192f579cbf9 --- /dev/null +++ b/chromium/components/viz/service/frame_sinks/video_detector.h @@ -0,0 +1,109 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_DETECTOR_H_ +#define COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_DETECTOR_H_ + +#include <unordered_map> + +#include "base/time/default_tick_clock.h" +#include "base/timer/timer.h" +#include "components/viz/common/surfaces/frame_sink_id.h" +#include "components/viz/service/surfaces/surface_observer.h" +#include "components/viz/service/viz_service_export.h" +#include "mojo/public/cpp/bindings/interface_ptr_set.h" +#include "services/viz/public/interfaces/compositing/video_detector_observer.mojom.h" + +namespace viz { + +class SurfaceManager; +class VideoDetectorTest; + +// Watches for updates to clients and tries to detect when a video is playing. +// If a client sends CompositorFrames with damages of size at least +// |kMinDamageWidth| x |kMinDamageHeight| at the rate of at least +// |kMinFramesPerSecond| for the duration of at least |kMinVideoDuration| then +// we say it is playing video. We err on the side of false positives and can be +// fooled by things like continuous scrolling of a page. +class VIZ_SERVICE_EXPORT VideoDetector : public SurfaceObserver { + public: + VideoDetector(SurfaceManager* surface_manager, + std::unique_ptr<base::TickClock> tick_clock = + std::make_unique<base::DefaultTickClock>(), + scoped_refptr<base::SequencedTaskRunner> task_runner = nullptr); + virtual ~VideoDetector(); + + // Adds an observer. The observer can be removed by closing the mojo + // connection. + void AddObserver(mojom::VideoDetectorObserverPtr observer); + + // When a FrameSinkId is registered/invalidated, we need to insert/delete the + // corresponding entry in client_infos_. + void OnFrameSinkIdRegistered(const FrameSinkId& frame_sink_id); + void OnFrameSinkIdInvalidated(const FrameSinkId& frame_sink_id); + + private: + friend class VideoDetectorTest; + + class ClientInfo; + + // Minimum dimensions in pixels that damages must have to be considered a + // potential video frame. + static constexpr int kMinDamageWidth = 333; + static constexpr int kMinDamageHeight = 250; + + // Number of video-sized updates that we must see within a second in a client + // before we assume that a video is playing. + static constexpr int kMinFramesPerSecond = 15; + + // Timeout after which video is no longer considered to be playing. + static constexpr base::TimeDelta kVideoTimeout = + base::TimeDelta::FromMilliseconds(1000); + + // Duration video must be playing in a client before it is reported to + // observers. + static constexpr base::TimeDelta kMinVideoDuration = + base::TimeDelta::FromMilliseconds(3000); + + // If no video activity is detected for |kVideoTimeout|, this + // method will be called by |video_inactive_timer_|; + void OnVideoActivityEnded(); + + // SurfaceObserver implementation. + void OnFirstSurfaceActivation(const SurfaceInfo& surface_info) override {} + void OnSurfaceActivated(const SurfaceId& surface_id) override {} + void OnSurfaceDestroyed(const SurfaceId& surface_id) override {} + bool OnSurfaceDamaged(const SurfaceId& surface_id, + const BeginFrameAck& ack) override; + void OnSurfaceDiscarded(const SurfaceId& surface_id) override {} + void OnSurfaceDamageExpected(const SurfaceId& surface_id, + const BeginFrameArgs& args) override {} + void OnSurfaceSubtreeDamaged(const SurfaceId& surface_id) override {} + void OnSurfaceWillBeDrawn(Surface* surface) override; + + // True if video has been observed in the last |kVideoTimeout|. + bool video_is_playing_ = false; + + // Provides the current time. + std::unique_ptr<base::TickClock> tick_clock_; + + // Calls OnVideoActivityEnded() after |kVideoTimeout|. Uses |tick_clock_| to + // measure time. + base::OneShotTimer video_inactive_timer_; + + // Contains information used for determining video activity in each client. + base::flat_map<FrameSinkId, std::unique_ptr<ClientInfo>> client_infos_; + + // Observers that are interested to know about video activity. We only detect + // video activity if there is at least one client. + mojo::InterfacePtrSet<mojom::VideoDetectorObserver> observers_; + + SurfaceManager* const surface_manager_; + + DISALLOW_COPY_AND_ASSIGN(VideoDetector); +}; + +} // namespace viz + +#endif // COMPONENTS_VIZ_SERVICE_FRAME_SINKS_VIDEO_DETECTOR_H_ diff --git a/chromium/components/viz/service/frame_sinks/video_detector_unittest.cc b/chromium/components/viz/service/frame_sinks/video_detector_unittest.cc new file mode 100644 index 00000000000..6562eba19ca --- /dev/null +++ b/chromium/components/viz/service/frame_sinks/video_detector_unittest.cc @@ -0,0 +1,315 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <memory> +#include <set> + +#include "base/compiler_specific.h" +#include "base/containers/circular_deque.h" +#include "base/test/test_mock_time_task_runner.h" +#include "base/time/tick_clock.h" +#include "base/time/time.h" +#include "components/viz/common/quads/surface_draw_quad.h" +#include "components/viz/common/surfaces/local_surface_id_allocator.h" +#include "components/viz/service/display/surface_aggregator.h" +#include "components/viz/service/frame_sinks/compositor_frame_sink_support.h" +#include "components/viz/service/frame_sinks/frame_sink_manager_impl.h" +#include "components/viz/service/frame_sinks/video_detector.h" +#include "components/viz/test/compositor_frame_helpers.h" +#include "components/viz/test/fake_compositor_frame_sink_client.h" +#include "mojo/public/cpp/bindings/binding.h" +#include "services/viz/public/interfaces/compositing/video_detector_observer.mojom.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/skia/include/core/SkColor.h" +#include "ui/gfx/geometry/rect.h" + +namespace viz { + +namespace { + +// Implementation that just records video state changes. +class TestObserver : public mojom::VideoDetectorObserver { + public: + TestObserver() : binding_(this) {} + + void Bind(mojom::VideoDetectorObserverRequest request) { + binding_.Bind(std::move(request)); + } + + bool IsEmpty() { + binding_.FlushForTesting(); + return states_.empty(); + } + + void Reset() { + binding_.FlushForTesting(); + states_.clear(); + } + + // Pops and returns the earliest-received state. + bool PopState() { + binding_.FlushForTesting(); + CHECK(!states_.empty()); + uint8_t first_state = states_.front(); + states_.pop_front(); + return first_state; + } + + // mojom::VideoDetectorObserver implementation. + void OnVideoActivityStarted() override { states_.push_back(true); } + void OnVideoActivityEnded() override { states_.push_back(false); } + + private: + // States in the order they were received. + base::circular_deque<bool> states_; + + mojo::Binding<mojom::VideoDetectorObserver> binding_; + + DISALLOW_COPY_AND_ASSIGN(TestObserver); +}; + +} // namespace + +class VideoDetectorTest : public testing::Test { + public: + VideoDetectorTest() + : surface_aggregator_(frame_sink_manager_.surface_manager(), + nullptr, + false) {} + + ~VideoDetectorTest() override {} + + void SetUp() override { + mock_task_runner_ = base::MakeRefCounted<base::TestMockTimeTaskRunner>(); + + detector_ = frame_sink_manager_.CreateVideoDetectorForTesting( + mock_task_runner_->GetMockTickClock(), mock_task_runner_); + + mojom::VideoDetectorObserverPtr video_detector_observer; + observer_.Bind(mojo::MakeRequest(&video_detector_observer)); + detector_->AddObserver(std::move(video_detector_observer)); + + root_frame_sink_ = CreateFrameSink(); + root_frame_sink_->SubmitCompositorFrame( + local_surface_id_allocator_.GenerateId(), test::MakeCompositorFrame()); + } + + protected: + // Constants placed here for convenience. + static constexpr int kMinFps = VideoDetector::kMinFramesPerSecond; + static constexpr gfx::Rect kMinRect = + gfx::Rect(VideoDetector::kMinDamageWidth, + VideoDetector::kMinDamageHeight); + static constexpr base::TimeDelta kMinDuration = + VideoDetector::kMinVideoDuration; + static constexpr base::TimeDelta kTimeout = VideoDetector::kVideoTimeout; + + // Move |detector_|'s idea of the current time forward by |delta|. + void AdvanceTime(base::TimeDelta delta) { + mock_task_runner_->FastForwardBy(delta); + } + + void CreateDisplayFrame() { + surface_aggregator_.Aggregate(root_frame_sink_->current_surface_id()); + } + + void EmbedClient(CompositorFrameSinkSupport* frame_sink) { + embedded_clients_.insert(frame_sink); + SubmitRootFrame(); + } + + void SubmitRootFrame() { + CompositorFrame frame = test::MakeCompositorFrame(); + RenderPass* render_pass = frame.render_pass_list.back().get(); + SharedQuadState* shared_quad_state = + render_pass->CreateAndAppendSharedQuadState(); + for (CompositorFrameSinkSupport* frame_sink : embedded_clients_) { + SurfaceDrawQuad* quad = + render_pass->CreateAndAppendDrawQuad<SurfaceDrawQuad>(); + quad->SetNew(shared_quad_state, gfx::Rect(0, 0, 10, 10), + gfx::Rect(0, 0, 5, 5), frame_sink->current_surface_id(), + base::nullopt, SK_ColorMAGENTA, false); + } + root_frame_sink_->SubmitCompositorFrame( + root_frame_sink_->local_surface_id(), std::move(frame)); + } + + void SendUpdate(CompositorFrameSinkSupport* frame_sink, + const gfx::Rect& damage) { + LocalSurfaceId local_surface_id = + frame_sink->local_surface_id().is_valid() + ? frame_sink->local_surface_id() + : local_surface_id_allocator_.GenerateId(); + frame_sink->SubmitCompositorFrame(local_surface_id, + MakeDamagedCompositorFrame(damage)); + } + + // Report updates to |client| of area |damage| at a rate of + // |updates_per_second| over |duration|. The first update will be sent + // immediately and time will have advanced by |duration| upon returning. + void SendUpdates(CompositorFrameSinkSupport* frame_sink, + const gfx::Rect& damage, + int updates_per_second, + base::TimeDelta duration) { + const base::TimeDelta time_between_updates = + base::TimeDelta::FromSecondsD(1.0 / updates_per_second); + for (base::TimeDelta d; d < duration; d += time_between_updates) { + SendUpdate(frame_sink, damage); + CreateDisplayFrame(); + AdvanceTime(std::min(time_between_updates, duration - d)); + } + } + + std::unique_ptr<CompositorFrameSinkSupport> CreateFrameSink() { + constexpr bool is_root = false; + constexpr bool needs_sync_points = true; + static uint32_t client_id = 1; + FrameSinkId frame_sink_id(client_id++, 0); + frame_sink_manager_.RegisterFrameSinkId(frame_sink_id); + std::unique_ptr<CompositorFrameSinkSupport> frame_sink = + CompositorFrameSinkSupport::Create(&frame_sink_client_, + &frame_sink_manager_, frame_sink_id, + is_root, needs_sync_points); + SendUpdate(frame_sink.get(), gfx::Rect()); + return frame_sink; + } + + VideoDetector* detector_; + TestObserver observer_; + + scoped_refptr<base::TestMockTimeTaskRunner> mock_task_runner_; + + private: + CompositorFrame MakeDamagedCompositorFrame(const gfx::Rect& damage) { + constexpr gfx::Size kFrameSinkSize = gfx::Size(10000, 10000); + CompositorFrame frame = test::MakeCompositorFrame(kFrameSinkSize); + frame.render_pass_list.back()->damage_rect = damage; + return frame; + } + + FrameSinkManagerImpl frame_sink_manager_; + FakeCompositorFrameSinkClient frame_sink_client_; + LocalSurfaceIdAllocator local_surface_id_allocator_; + SurfaceAggregator surface_aggregator_; + std::unique_ptr<CompositorFrameSinkSupport> root_frame_sink_; + std::set<CompositorFrameSinkSupport*> embedded_clients_; + + DISALLOW_COPY_AND_ASSIGN(VideoDetectorTest); +}; + +constexpr gfx::Rect VideoDetectorTest::kMinRect; +constexpr base::TimeDelta VideoDetectorTest::kMinDuration; +constexpr base::TimeDelta VideoDetectorTest::kTimeout; + +// Verify that VideoDetector does not report clients with small damage rects. +TEST_F(VideoDetectorTest, DontReportWhenDamageTooSmall) { + std::unique_ptr<CompositorFrameSinkSupport> frame_sink = CreateFrameSink(); + EmbedClient(frame_sink.get()); + + { + // Send damages with a smaller width than |kMinRect|. Make sure video + // activity isn't detected. + gfx::Rect rect = kMinRect; + rect.Inset(0, 0, 1, 0); + SendUpdates(frame_sink.get(), rect, 2 * kMinFps, 2 * kMinDuration); + EXPECT_TRUE(observer_.IsEmpty()); + } + + { + // Send damages with a smaller height than |kMinRect|. Make sure video + // activity isn't detected. + gfx::Rect rect = kMinRect; + rect.Inset(0, 0, 1, 0); + SendUpdates(frame_sink.get(), rect, 2 * kMinFps, 2 * kMinDuration); + EXPECT_TRUE(observer_.IsEmpty()); + } +} + +// Verify that VideoDetector does not report clients with a low frame rate. +TEST_F(VideoDetectorTest, DontReportWhenFramerateTooLow) { + std::unique_ptr<CompositorFrameSinkSupport> frame_sink = CreateFrameSink(); + EmbedClient(frame_sink.get()); + SendUpdates(frame_sink.get(), kMinRect, kMinFps - 5, 2 * kMinDuration); + EXPECT_TRUE(observer_.IsEmpty()); +} + +// Verify that VideoDetector does not report clients until they have played for +// the minimum necessary duration. +TEST_F(VideoDetectorTest, DontReportWhenNotPlayingLongEnough) { + std::unique_ptr<CompositorFrameSinkSupport> frame_sink = CreateFrameSink(); + EmbedClient(frame_sink.get()); + SendUpdates(frame_sink.get(), kMinRect, 2 * kMinFps, 0.5 * kMinDuration); + EXPECT_TRUE(observer_.IsEmpty()); + + SendUpdates(frame_sink.get(), kMinRect, 2 * kMinFps, 0.6 * kMinDuration); + EXPECT_TRUE(observer_.PopState()); + EXPECT_TRUE(observer_.IsEmpty()); +} + +// Verify that VideoDetector does not report clients that are not visible +// on screen. +TEST_F(VideoDetectorTest, DontReportWhenClientHidden) { + std::unique_ptr<CompositorFrameSinkSupport> frame_sink = CreateFrameSink(); + + SendUpdates(frame_sink.get(), kMinRect, kMinFps + 5, 2 * kMinDuration); + EXPECT_TRUE(observer_.IsEmpty()); + + // Make the client visible. + observer_.Reset(); + AdvanceTime(kTimeout); + EmbedClient(frame_sink.get()); + SendUpdates(frame_sink.get(), kMinRect, kMinFps + 5, 2 * kMinDuration); + EXPECT_TRUE(observer_.PopState()); + EXPECT_TRUE(observer_.IsEmpty()); +} + +// Turn video activity on and off. Make sure the observers are notified +// properly. +TEST_F(VideoDetectorTest, ReportStartAndStop) { + const base::TimeDelta kDuration = + kMinDuration + base::TimeDelta::FromMilliseconds(100); + std::unique_ptr<CompositorFrameSinkSupport> frame_sink = CreateFrameSink(); + EmbedClient(frame_sink.get()); + SendUpdates(frame_sink.get(), kMinRect, kMinFps + 5, kDuration); + EXPECT_TRUE(observer_.PopState()); + EXPECT_TRUE(observer_.IsEmpty()); + + AdvanceTime(kTimeout); + EXPECT_FALSE(observer_.PopState()); + EXPECT_TRUE(observer_.IsEmpty()); + + // Start playing again. + SendUpdates(frame_sink.get(), kMinRect, kMinFps + 5, kDuration); + EXPECT_TRUE(observer_.PopState()); + EXPECT_TRUE(observer_.IsEmpty()); + + AdvanceTime(kTimeout); + EXPECT_FALSE(observer_.PopState()); + EXPECT_TRUE(observer_.IsEmpty()); +} + +// If there are multiple clients playing video, make sure that observers only +// receive a single notification. +TEST_F(VideoDetectorTest, ReportOnceForMultipleClients) { + std::unique_ptr<CompositorFrameSinkSupport> frame_sink1 = CreateFrameSink(); + std::unique_ptr<CompositorFrameSinkSupport> frame_sink2 = CreateFrameSink(); + EmbedClient(frame_sink1.get()); + EmbedClient(frame_sink2.get()); + + // Even if there's video playing in both clients, the observer should only + // receive a single notification. + constexpr int fps = 2 * kMinFps; + constexpr base::TimeDelta time_between_updates = + base::TimeDelta::FromSecondsD(1.0 / fps); + for (base::TimeDelta d; d < 2 * kMinDuration; d += time_between_updates) { + SendUpdate(frame_sink1.get(), kMinRect); + SendUpdate(frame_sink2.get(), kMinRect); + AdvanceTime(time_between_updates); + CreateDisplayFrame(); + } + EXPECT_TRUE(observer_.PopState()); + EXPECT_TRUE(observer_.IsEmpty()); +} + +} // namespace viz |