diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2018-01-29 16:35:13 +0100 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2018-02-01 15:33:35 +0000 |
commit | c8c2d1901aec01e934adf561a9fdf0cc776cdef8 (patch) | |
tree | 9157c3d9815e5870799e070b113813bec53e0535 /chromium/media/gpu/android | |
parent | abefd5095b41dac94ca451d784ab6e27372e981a (diff) | |
download | qtwebengine-chromium-c8c2d1901aec01e934adf561a9fdf0cc776cdef8.tar.gz |
BASELINE: Update Chromium to 64.0.3282.139
Change-Id: I1cae68fe9c94ff7608b26b8382fc19862cdb293a
Reviewed-by: Alexandru Croitor <alexandru.croitor@qt.io>
Diffstat (limited to 'chromium/media/gpu/android')
34 files changed, 1624 insertions, 324 deletions
diff --git a/chromium/media/gpu/android/android_video_decode_accelerator.cc b/chromium/media/gpu/android/android_video_decode_accelerator.cc index 4aa7deab85a..d6b717fda94 100644 --- a/chromium/media/gpu/android/android_video_decode_accelerator.cc +++ b/chromium/media/gpu/android/android_video_decode_accelerator.cc @@ -69,11 +69,6 @@ enum { kNumPictureBuffers = limits::kMaxVideoFrames + 1 }; // NotifyEndOfBitstreamBuffer() before getting output from the bitstream. enum { kMaxBitstreamsNotifiedInAdvance = 32 }; -// Number of frames to defer overlays for when entering fullscreen. This lets -// blink relayout settle down a bit. If overlay positions were synchronous, -// then we wouldn't need this. -enum { kFrameDelayForFullscreenLayout = 15 }; - // MediaCodec is only guaranteed to support baseline, but some devices may // support others. Advertise support for all H264 profiles and let the // MediaCodec fail when decoding if it's not actually supported. It's assumed @@ -118,13 +113,6 @@ constexpr base::TimeDelta NoWaitTimeOut = base::TimeDelta::FromMicroseconds(0); constexpr base::TimeDelta IdleTimerTimeOut = base::TimeDelta::FromSeconds(1); -// How often do we let the surface chooser try for an overlay? While we'll -// retry if some relevant state changes on our side (e.g., fullscreen state), -// there's plenty of state that we don't know about (e.g., power efficiency, -// memory pressure => cancelling an old overlay, etc.). We just let the chooser -// retry every once in a while for those things. -constexpr base::TimeDelta RetryChooserTimeout = base::TimeDelta::FromSeconds(5); - // On low end devices (< KitKat is always low-end due to buggy MediaCodec), // defer the surface creation until the codec is actually used if we know no // software fallback exists. @@ -261,12 +249,14 @@ AndroidVideoDecodeAccelerator::AndroidVideoDecodeAccelerator( deferred_initialization_pending_(false), codec_needs_reset_(false), defer_surface_creation_(false), - surface_chooser_(std::move(surface_chooser)), + surface_chooser_helper_( + std::move(surface_chooser), + base::CommandLine::ForCurrentProcess()->HasSwitch( + switches::kForceVideoOverlays), + base::FeatureList::IsEnabled(media::kUseAndroidOverlayAggressively)), device_info_(device_info), force_defer_surface_creation_for_testing_(false), overlay_factory_cb_(overlay_factory_cb), - promotion_hint_aggregator_( - base::MakeUnique<PromotionHintAggregatorImpl>()), weak_this_factory_(this) {} AndroidVideoDecodeAccelerator::~AndroidVideoDecodeAccelerator() { @@ -368,17 +358,6 @@ bool AndroidVideoDecodeAccelerator::Initialize(const Config& config, codec_allocator_->StartThread(this); - // If we're supposed to use overlays all the time, then they should always - // be marked as required. - if (base::CommandLine::ForCurrentProcess()->HasSwitch( - switches::kForceVideoOverlays)) { - surface_chooser_state_.is_required = is_overlay_required_ = true; - } - - // If we're trying for fullscreen-div cases, then we should promote more. - surface_chooser_state_.promote_aggressively = - base::FeatureList::IsEnabled(media::kUseAndroidOverlayAggressively); - // For encrypted media, start by initializing the CDM. Otherwise, start with // the surface. if (config_.is_encrypted()) { @@ -407,9 +386,9 @@ void AndroidVideoDecodeAccelerator::StartSurfaceChooser() { return; } - surface_chooser_state_.is_fullscreen = config_.overlay_info.is_fullscreen; + surface_chooser_helper_.SetIsFullscreen(config_.overlay_info.is_fullscreen); - surface_chooser_->SetClientCallbacks( + surface_chooser_helper_.chooser()->SetClientCallbacks( base::Bind(&AndroidVideoDecodeAccelerator::OnSurfaceTransition, weak_this_factory_.GetWeakPtr()), base::Bind(&AndroidVideoDecodeAccelerator::OnSurfaceTransition, @@ -457,7 +436,7 @@ void AndroidVideoDecodeAccelerator::StartSurfaceChooser() { // the synchronous case. It will be soon, though. For pre-M, we rely on the // fact that |surface_chooser_| won't tell us to use a SurfaceTexture while // waiting for an overlay to become ready, for example. - surface_chooser_->UpdateState(std::move(factory), surface_chooser_state_); + surface_chooser_helper_.UpdateChooserState(std::move(factory)); } void AndroidVideoDecodeAccelerator::OnSurfaceTransition( @@ -931,7 +910,9 @@ void AndroidVideoDecodeAccelerator::SendDecodedFrameToClient( // Record the frame type that we're sending and some information about why. UMA_HISTOGRAM_ENUMERATION( "Media.AVDA.FrameInformation", cached_frame_information_, - static_cast<int>(FrameInformation::FRAME_INFORMATION_MAX) + 1); + static_cast<int>( + SurfaceChooserHelper::FrameInformation::FRAME_INFORMATION_MAX) + + 1); // PRESUBMIT_IGNORE_UMA_MAX // We unconditionally mark the picture as overlayable, even if // |!allow_overlay|, if we want to get hints. It's required, else we won't @@ -1310,27 +1291,8 @@ void AndroidVideoDecodeAccelerator::SetOverlayInfo( if (state_ == BEFORE_OVERLAY_INIT) return; - // Release any overlay immediately when hiding a frame. Otherwise, it will - // stick around as long as the VideoFrame does, which can be a long time. - if (overlay_info.is_frame_hidden) - picture_buffer_manager_.ImmediatelyForgetOverlay(output_picture_buffers_); - - surface_chooser_state_.is_frame_hidden = overlay_info.is_frame_hidden; - - if (overlay_info.is_fullscreen && !surface_chooser_state_.is_fullscreen) { - // It would be nice if we could just delay until we get a hint from an - // overlay that's "in fullscreen" in the sense that the CompositorFrame it - // came from had some flag set to indicate that the renderer was in - // fullscreen mode when it was generated. However, even that's hard, since - // there's no real connection between "renderer finds out about fullscreen" - // and "blink has completed layouts for it". The latter is what we really - // want to know. - surface_chooser_state_.is_expecting_relayout = true; - hints_until_clear_relayout_flag_ = kFrameDelayForFullscreenLayout; - } - // Notify the chooser about the fullscreen state. - surface_chooser_state_.is_fullscreen = overlay_info.is_fullscreen; + surface_chooser_helper_.SetIsFullscreen(overlay_info.is_fullscreen); // Note that these might be kNoSurfaceID / empty. In that case, we will // revoke the factory. @@ -1349,7 +1311,7 @@ void AndroidVideoDecodeAccelerator::SetOverlayInfo( new_factory = base::Bind(&ContentVideoViewOverlay::Create, surface_id); } - surface_chooser_->UpdateState(new_factory, surface_chooser_state_); + surface_chooser_helper_.UpdateChooserState(new_factory); } void AndroidVideoDecodeAccelerator::Destroy() { @@ -1541,9 +1503,10 @@ void AndroidVideoDecodeAccelerator::OnMediaCryptoReady( // Request a secure surface in all cases. For L3, it's okay if we fall back // to SurfaceTexture rather than fail composition. For L1, it's required. // It's also required if the command line says so. - surface_chooser_state_.is_secure = true; - surface_chooser_state_.is_required = - requires_secure_video_codec || is_overlay_required_; + surface_chooser_helper_.SetSecureSurfaceMode( + requires_secure_video_codec + ? SurfaceChooserHelper::SecureSurfaceMode::kRequired + : SurfaceChooserHelper::SecureSurfaceMode::kRequested); // After receiving |media_crypto_| we can start with surface creation. StartSurfaceChooser(); @@ -1618,47 +1581,10 @@ AndroidVideoDecodeAccelerator::GetPromotionHintCB() { void AndroidVideoDecodeAccelerator::NotifyPromotionHint( PromotionHintAggregator::Hint hint) { - bool update_state = false; - - promotion_hint_aggregator_->NotifyPromotionHint(hint); - - // If we're expecting a full screen relayout, then also use this hint as a - // notification that another frame has happened. - if (hints_until_clear_relayout_flag_ > 0) { - hints_until_clear_relayout_flag_--; - if (hints_until_clear_relayout_flag_ == 0) { - surface_chooser_state_.is_expecting_relayout = false; - update_state = true; - } - } - - surface_chooser_state_.initial_position = hint.screen_rect; - bool promotable = promotion_hint_aggregator_->IsSafeToPromote(); - if (promotable != surface_chooser_state_.is_compositor_promotable) { - surface_chooser_state_.is_compositor_promotable = promotable; - update_state = true; - } - - // If we've been provided with enough new frames, then update the state even - // if it hasn't changed. This lets |surface_chooser_| retry for an overlay. - // It's especially helpful for power-efficient overlays, since we don't know - // when an overlay becomes power efficient. It also helps retry any failure - // that's not accompanied by a state change, such as if android destroys the - // overlay asynchronously for a transient reason. - // - // If we're already using an overlay, then there's no need to do this. - base::TimeTicks now = base::TimeTicks::Now(); - if (codec_config_->surface_bundle && - !codec_config_->surface_bundle->overlay && - now - most_recent_chooser_retry_ >= RetryChooserTimeout) { - update_state = true; - } - - if (update_state) { - most_recent_chooser_retry_ = now; - surface_chooser_->UpdateState(base::Optional<AndroidOverlayFactoryCB>(), - surface_chooser_state_); - } + bool is_using_overlay = + codec_config_->surface_bundle && codec_config_->surface_bundle->overlay; + surface_chooser_helper_.NotifyPromotionHintAndUpdateChooser(hint, + is_using_overlay); } void AndroidVideoDecodeAccelerator::ManageTimer(bool did_work) { @@ -1833,27 +1759,11 @@ void AndroidVideoDecodeAccelerator::ReleaseCodecAndBundle() { } void AndroidVideoDecodeAccelerator::CacheFrameInformation() { - if (!codec_config_->surface_bundle || - !codec_config_->surface_bundle->overlay) { - // Not an overlay. - cached_frame_information_ = surface_chooser_state_.is_secure - ? FrameInformation::SURFACETEXTURE_L3 - : FrameInformation::SURFACETEXTURE_INSECURE; - return; - } - - // Overlay. - if (surface_chooser_state_.is_secure) { - cached_frame_information_ = surface_chooser_state_.is_required - ? FrameInformation::OVERLAY_L1 - : FrameInformation::OVERLAY_L3; - return; - } + bool is_using_overlay = + codec_config_->surface_bundle && codec_config_->surface_bundle->overlay; cached_frame_information_ = - surface_chooser_state_.is_fullscreen - ? FrameInformation::OVERLAY_INSECURE_PLAYER_ELEMENT_FULLSCREEN - : FrameInformation::OVERLAY_INSECURE_NON_PLAYER_ELEMENT_FULLSCREEN; + surface_chooser_helper_.ComputeFrameInformation(is_using_overlay); } } // namespace media diff --git a/chromium/media/gpu/android/android_video_decode_accelerator.h b/chromium/media/gpu/android/android_video_decode_accelerator.h index f6b2c2db29d..5cbb6a308df 100644 --- a/chromium/media/gpu/android/android_video_decode_accelerator.h +++ b/chromium/media/gpu/android/android_video_decode_accelerator.h @@ -22,11 +22,11 @@ #include "media/base/android/media_drm_bridge_cdm_context.h" #include "media/base/android_overlay_mojo_factory.h" #include "media/base/content_decryption_module.h" -#include "media/gpu/android/android_video_surface_chooser.h" #include "media/gpu/android/avda_codec_allocator.h" #include "media/gpu/android/avda_picture_buffer_manager.h" #include "media/gpu/android/avda_state_provider.h" #include "media/gpu/android/device_info.h" +#include "media/gpu/android/surface_chooser_helper.h" #include "media/gpu/gpu_video_decode_accelerator_helpers.h" #include "media/gpu/media_gpu_export.h" #include "media/video/video_decode_accelerator.h" @@ -34,6 +34,7 @@ #include "ui/gl/android/surface_texture.h" namespace media { +class AndroidVideoSurfaceChooser; class SharedMemoryRegion; class PromotionHintAggregator; @@ -387,53 +388,24 @@ class MEDIA_GPU_EXPORT AndroidVideoDecodeAccelerator // told to move to SurfaceTexture, then this will be value() == nullptr. base::Optional<std::unique_ptr<AndroidOverlay>> incoming_overlay_; - std::unique_ptr<AndroidVideoSurfaceChooser> surface_chooser_; + SurfaceChooserHelper surface_chooser_helper_; DeviceInfo* device_info_; bool force_defer_surface_creation_for_testing_; - AndroidVideoSurfaceChooser::State surface_chooser_state_; - - // Number of promotion hints that we need to receive before clearing the - // "delay overlay promotion" flag in |surface_chooser_state_|. We do this so - // that the transition looks better, since it gives blink time to stabilize. - // Since overlay positioning isn't synchronous, it's good to make sure that - // blink isn't moving the quad around too. - int hints_until_clear_relayout_flag_ = 0; - // Optional factory to produce mojo AndroidOverlay instances. AndroidOverlayMojoFactoryCB overlay_factory_cb_; std::unique_ptr<PromotionHintAggregator> promotion_hint_aggregator_; - // Are overlays required by command-line options? - bool is_overlay_required_ = false; - - // Must match AVDAFrameInformation UMA enum. Please do not remove or re-order - // values, only append new ones. - enum class FrameInformation { - SURFACETEXTURE_INSECURE = 0, - SURFACETEXTURE_L3 = 1, - OVERLAY_L3 = 2, - OVERLAY_L1 = 3, - OVERLAY_INSECURE_PLAYER_ELEMENT_FULLSCREEN = 4, - OVERLAY_INSECURE_NON_PLAYER_ELEMENT_FULLSCREEN = 5, - - // Max enum value. - FRAME_INFORMATION_MAX = OVERLAY_INSECURE_NON_PLAYER_ELEMENT_FULLSCREEN - }; - // Update |cached_frame_information_|. void CacheFrameInformation(); // Most recently cached frame information, so that we can dispatch it without // recomputing it on every frame. It changes very rarely. - FrameInformation cached_frame_information_ = - FrameInformation::SURFACETEXTURE_INSECURE; - - // Time since we last updated the chooser state. - base::TimeTicks most_recent_chooser_retry_; + SurfaceChooserHelper::FrameInformation cached_frame_information_ = + SurfaceChooserHelper::FrameInformation::SURFACETEXTURE_INSECURE; // WeakPtrFactory for posting tasks back to |this|. base::WeakPtrFactory<AndroidVideoDecodeAccelerator> weak_this_factory_; diff --git a/chromium/media/gpu/android/android_video_decode_accelerator_unittest.cc b/chromium/media/gpu/android/android_video_decode_accelerator_unittest.cc index 4fcfb62eb29..7de5cfd083b 100644 --- a/chromium/media/gpu/android/android_video_decode_accelerator_unittest.cc +++ b/chromium/media/gpu/android/android_video_decode_accelerator_unittest.cc @@ -26,8 +26,8 @@ #include "media/gpu/android/android_video_decode_accelerator.h" #include "media/gpu/android/android_video_surface_chooser.h" #include "media/gpu/android/avda_codec_allocator.h" -#include "media/gpu/android/fake_android_video_surface_chooser.h" #include "media/gpu/android/fake_codec_allocator.h" +#include "media/gpu/android/mock_android_video_surface_chooser.h" #include "media/gpu/android/mock_device_info.h" #include "media/video/picture.h" #include "media/video/video_decode_accelerator.h" @@ -101,7 +101,7 @@ class AndroidVideoDecodeAcceleratorTest : public testing::Test { device_info_ = base::MakeUnique<NiceMock<MockDeviceInfo>>(); chooser_that_is_usually_null_ = - base::MakeUnique<NiceMock<FakeSurfaceChooser>>(); + base::MakeUnique<NiceMock<MockAndroidVideoSurfaceChooser>>(); chooser_ = chooser_that_is_usually_null_.get(); // By default, allow deferred init. @@ -115,7 +115,7 @@ class AndroidVideoDecodeAcceleratorTest : public testing::Test { codec_allocator_ = nullptr; context_ = nullptr; surface_ = nullptr; - gl::init::ShutdownGL(); + gl::init::ShutdownGL(false); } // Create and initialize AVDA with |config_|, and return the result. @@ -206,8 +206,8 @@ class AndroidVideoDecodeAcceleratorTest : public testing::Test { std::unique_ptr<FakeCodecAllocator> codec_allocator_; // Only set until InitializeAVDA() is called. - std::unique_ptr<FakeSurfaceChooser> chooser_that_is_usually_null_; - FakeSurfaceChooser* chooser_; + std::unique_ptr<MockAndroidVideoSurfaceChooser> chooser_that_is_usually_null_; + MockAndroidVideoSurfaceChooser* chooser_; VideoDecodeAccelerator::Config config_; std::unique_ptr<MockDeviceInfo> device_info_; diff --git a/chromium/media/gpu/android/android_video_encode_accelerator.cc b/chromium/media/gpu/android/android_video_encode_accelerator.cc index a3341698027..684ffea6d2d 100644 --- a/chromium/media/gpu/android/android_video_encode_accelerator.cc +++ b/chromium/media/gpu/android/android_video_encode_accelerator.cc @@ -356,16 +356,30 @@ void AndroidVideoEncodeAccelerator::QueueInput() { frame->coded_size().height()); RETURN_ON_FAILURE(converted, "Failed to I420ToNV12!", kPlatformFailureError); - input_timestamp_ += base::TimeDelta::FromMicroseconds( + // MediaCodec encoder assumes the presentation timestamps to be monotonically + // increasing at initialized framerate. But in Chromium, the video capture + // may be paused for a while or drop some frames, so the timestamp in input + // frames won't be continious. Here we cache the timestamps of input frames, + // mapping to the generated |presentation_timestamp_|, and will read them out + // after encoding. Then encoder can work happily always and we can preserve + // the timestamps in captured frames for other purpose. + presentation_timestamp_ += base::TimeDelta::FromMicroseconds( base::Time::kMicrosecondsPerSecond / INITIAL_FRAMERATE); + DCHECK(frame_timestamp_map_.find(presentation_timestamp_) == + frame_timestamp_map_.end()); + frame_timestamp_map_[presentation_timestamp_] = frame->timestamp(); + status = media_codec_->QueueInputBuffer(input_buf_index, nullptr, queued_size, - input_timestamp_); + presentation_timestamp_); UMA_HISTOGRAM_TIMES("Media.AVDA.InputQueueTime", base::Time::Now() - std::get<2>(input)); RETURN_ON_FAILURE(status == MEDIA_CODEC_OK, "Failed to QueueInputBuffer: " << status, kPlatformFailureError); ++num_buffers_at_codec_; + DCHECK(static_cast<int32_t>(frame_timestamp_map_.size()) == + num_buffers_at_codec_); + pending_frames_.pop(); } @@ -380,9 +394,10 @@ void AndroidVideoEncodeAccelerator::DequeueOutput() { size_t size = 0; bool key_frame = false; - MediaCodecStatus status = - media_codec_->DequeueOutputBuffer(NoWaitTimeOut(), &buf_index, &offset, - &size, nullptr, nullptr, &key_frame); + base::TimeDelta presentaion_timestamp; + MediaCodecStatus status = media_codec_->DequeueOutputBuffer( + NoWaitTimeOut(), &buf_index, &offset, &size, &presentaion_timestamp, + nullptr, &key_frame); switch (status) { case MEDIA_CODEC_TRY_AGAIN_LATER: return; @@ -407,6 +422,11 @@ void AndroidVideoEncodeAccelerator::DequeueOutput() { break; } + const auto it = frame_timestamp_map_.find(presentaion_timestamp); + DCHECK(it != frame_timestamp_map_.end()); + const base::TimeDelta frame_timestamp = it->second; + frame_timestamp_map_.erase(it); + BitstreamBuffer bitstream_buffer = available_bitstream_buffers_.back(); available_bitstream_buffers_.pop_back(); std::unique_ptr<SharedMemoryRegion> shm( @@ -427,7 +447,7 @@ void AndroidVideoEncodeAccelerator::DequeueOutput() { FROM_HERE, base::Bind(&VideoEncodeAccelerator::Client::BitstreamBufferReady, client_ptr_factory_->GetWeakPtr(), bitstream_buffer.id(), size, - key_frame, base::TimeDelta())); + key_frame, frame_timestamp)); } } // namespace media diff --git a/chromium/media/gpu/android/android_video_encode_accelerator.h b/chromium/media/gpu/android/android_video_encode_accelerator.h index b9624217312..91ad14b6bf5 100644 --- a/chromium/media/gpu/android/android_video_encode_accelerator.h +++ b/chromium/media/gpu/android/android_video_encode_accelerator.h @@ -9,6 +9,7 @@ #include <stdint.h> #include <list> +#include <map> #include <memory> #include <tuple> #include <vector> @@ -96,7 +97,11 @@ class MEDIA_GPU_EXPORT AndroidVideoEncodeAccelerator int32_t num_buffers_at_codec_; // A monotonically-growing value. - base::TimeDelta input_timestamp_; + base::TimeDelta presentation_timestamp_; + + std::map<base::TimeDelta /* presentation_timestamp */, + base::TimeDelta /* frame_timestamp */> + frame_timestamp_map_; // Resolution of input stream. Set once in initialization and not allowed to // change after. diff --git a/chromium/media/gpu/android/android_video_surface_chooser_impl.cc b/chromium/media/gpu/android/android_video_surface_chooser_impl.cc index d1882075e72..135ff519301 100644 --- a/chromium/media/gpu/android/android_video_surface_chooser_impl.cc +++ b/chromium/media/gpu/android/android_video_surface_chooser_impl.cc @@ -53,6 +53,8 @@ void AndroidVideoSurfaceChooserImpl::UpdateState( if (!initial_state_received_) { initial_state_received_ = true; // Choose here so that Choose() doesn't have to handle non-dynamic. + // Note that we ignore |is_expecting_relayout| here, since it's transient. + // We don't want to pick SurfaceTexture permanently for that. if (overlay_factory_ && (current_state_.is_fullscreen || current_state_.is_secure || current_state_.is_required)) { @@ -131,10 +133,6 @@ void AndroidVideoSurfaceChooserImpl::Choose() { new_overlay_state = kUsingSurfaceTexture; } - // If our frame is hidden, then don't use overlays. - if (current_state_.is_frame_hidden) - new_overlay_state = kUsingSurfaceTexture; - // If an overlay is required, then choose one. The only way we won't is if we // don't have a factory or our request fails. if (current_state_.is_required) { diff --git a/chromium/media/gpu/android/android_video_surface_chooser_impl_unittest.cc b/chromium/media/gpu/android/android_video_surface_chooser_impl_unittest.cc index a7aeaeae781..24045f10288 100644 --- a/chromium/media/gpu/android/android_video_surface_chooser_impl_unittest.cc +++ b/chromium/media/gpu/android/android_video_surface_chooser_impl_unittest.cc @@ -65,7 +65,6 @@ enum class AllowDynamic { No, Yes }; enum class IsFullscreen { No, Yes }; enum class IsRequired { No, Yes }; enum class IsSecure { No, Yes }; -enum class IsFrameHidden { No, Yes }; enum class IsCCPromotable { No, Yes }; enum class IsExpectingRelayout { No, Yes }; enum class PromoteAggressively { No, Yes }; @@ -76,7 +75,6 @@ using TestParams = std::tuple<ShouldUseOverlay, IsRequired, IsFullscreen, IsSecure, - IsFrameHidden, IsCCPromotable, IsExpectingRelayout, PromoteAggressively>; @@ -385,10 +383,9 @@ TEST_P(AndroidVideoSurfaceChooserImplTest, OverlayIsUsedOrNotBasedOnState) { chooser_state_.is_required = IsYes(IsRequired, 3); chooser_state_.is_fullscreen = IsYes(IsFullscreen, 4); chooser_state_.is_secure = IsYes(IsSecure, 5); - chooser_state_.is_frame_hidden = IsYes(IsFrameHidden, 6); - chooser_state_.is_compositor_promotable = IsYes(IsCCPromotable, 7); - chooser_state_.is_expecting_relayout = IsYes(IsExpectingRelayout, 8); - chooser_state_.promote_aggressively = IsYes(PromoteAggressively, 9); + chooser_state_.is_compositor_promotable = IsYes(IsCCPromotable, 6); + chooser_state_.is_expecting_relayout = IsYes(IsExpectingRelayout, 7); + chooser_state_.promote_aggressively = IsYes(PromoteAggressively, 8); MockAndroidOverlay* overlay = overlay_.get(); @@ -422,7 +419,6 @@ INSTANTIATE_TEST_CASE_P(NoFullscreenUsesSurfaceTexture, Values(IsRequired::No), Values(IsFullscreen::No), Values(IsSecure::No), - Either(IsFrameHidden), Either(IsCCPromotable), Either(IsExpectingRelayout), Values(PromoteAggressively::No))); @@ -435,7 +431,6 @@ INSTANTIATE_TEST_CASE_P(FullscreenUsesOverlay, Either(IsRequired), Values(IsFullscreen::Yes), Values(IsSecure::No), - Values(IsFrameHidden::No), Values(IsCCPromotable::Yes), Values(IsExpectingRelayout::No), Either(PromoteAggressively))); @@ -448,7 +443,6 @@ INSTANTIATE_TEST_CASE_P(RequiredUsesOverlay, Values(IsRequired::Yes), Either(IsFullscreen), Either(IsSecure), - Either(IsFrameHidden), Either(IsCCPromotable), Either(IsExpectingRelayout), Either(PromoteAggressively))); @@ -464,24 +458,10 @@ INSTANTIATE_TEST_CASE_P(SecureUsesOverlayIfPromotable, Either(IsRequired), Either(IsFullscreen), Values(IsSecure::Yes), - Values(IsFrameHidden::No), Values(IsCCPromotable::Yes), Values(IsExpectingRelayout::No), Either(PromoteAggressively))); -INSTANTIATE_TEST_CASE_P(HiddenFramesUseSurfaceTexture, - AndroidVideoSurfaceChooserImplTest, - Combine(Values(ShouldUseOverlay::No), - Values(ShouldBePowerEfficient::No), - Values(AllowDynamic::Yes), - Values(IsRequired::No), - Either(IsFullscreen), - Either(IsSecure), - Values(IsFrameHidden::Yes), - Either(IsCCPromotable), - Either(IsExpectingRelayout), - Either(PromoteAggressively))); - // For all dynamic cases, we shouldn't use an overlay if the compositor won't // promote it, unless it's marked as required. This includes secure surfaces, // so that L3 will fall back to SurfaceTexture. Non-dynamic is excluded, since @@ -495,7 +475,6 @@ INSTANTIATE_TEST_CASE_P(NotCCPromotableNotRequiredUsesSurfaceTexture, Values(IsRequired::No), Either(IsFullscreen), Either(IsSecure), - Values(IsFrameHidden::No), Values(IsCCPromotable::No), Either(IsExpectingRelayout), Either(PromoteAggressively))); @@ -510,7 +489,6 @@ INSTANTIATE_TEST_CASE_P(InsecureExpectingRelayoutUsesSurfaceTexture, Values(IsRequired::No), Either(IsFullscreen), Either(IsSecure), - Either(IsFrameHidden), Either(IsCCPromotable), Values(IsExpectingRelayout::Yes), Either(PromoteAggressively))); @@ -524,7 +502,6 @@ INSTANTIATE_TEST_CASE_P(NotDynamicInFullscreenUsesOverlay, Either(IsRequired), Values(IsFullscreen::Yes), Either(IsSecure), - Either(IsFrameHidden), Either(IsCCPromotable), Either(IsExpectingRelayout), Either(PromoteAggressively))); @@ -538,7 +515,6 @@ INSTANTIATE_TEST_CASE_P(NotDynamicSecureUsesOverlay, Either(IsRequired), Either(IsFullscreen), Values(IsSecure::Yes), - Either(IsFrameHidden), Either(IsCCPromotable), Either(IsExpectingRelayout), Either(PromoteAggressively))); @@ -552,7 +528,6 @@ INSTANTIATE_TEST_CASE_P(NotDynamicRequiredUsesOverlay, Values(IsRequired::Yes), Either(IsFullscreen), Either(IsSecure), - Either(IsFrameHidden), Either(IsCCPromotable), Either(IsExpectingRelayout), Either(PromoteAggressively))); @@ -566,7 +541,6 @@ INSTANTIATE_TEST_CASE_P(AggressiveOverlayIsPowerEfficient, Values(IsRequired::No), Values(IsFullscreen::No), Values(IsSecure::No), - Values(IsFrameHidden::No), Values(IsCCPromotable::Yes), Values(IsExpectingRelayout::No), Values(PromoteAggressively::Yes))); diff --git a/chromium/media/gpu/android/avda_codec_image.h b/chromium/media/gpu/android/avda_codec_image.h index cff49ad2086..849ec60c65f 100644 --- a/chromium/media/gpu/android/avda_codec_image.h +++ b/chromium/media/gpu/android/avda_codec_image.h @@ -42,6 +42,7 @@ class AVDACodecImage : public gpu::gles2::GLStreamTextureImage { gfx::OverlayTransform transform, const gfx::Rect& bounds_rect, const gfx::RectF& crop_rect) override; + void SetColorSpace(const gfx::ColorSpace& color_space) override {} void Flush() override {} void OnMemoryDump(base::trace_event::ProcessMemoryDump* pmd, uint64_t process_tracing_id, diff --git a/chromium/media/gpu/android/avda_picture_buffer_manager.cc b/chromium/media/gpu/android/avda_picture_buffer_manager.cc index a452cfacede..8efc505f324 100644 --- a/chromium/media/gpu/android/avda_picture_buffer_manager.cc +++ b/chromium/media/gpu/android/avda_picture_buffer_manager.cc @@ -266,13 +266,4 @@ bool AVDAPictureBufferManager::HasUnrenderedPictures() const { return false; } -void AVDAPictureBufferManager::ImmediatelyForgetOverlay( - const PictureBufferMap& buffers) { - if (!shared_state_ || !shared_state_->overlay()) - return; - - ReleaseCodecBuffers(buffers); - shared_state_->ClearOverlay(shared_state_->overlay()); -} - } // namespace media diff --git a/chromium/media/gpu/android/avda_picture_buffer_manager.h b/chromium/media/gpu/android/avda_picture_buffer_manager.h index 8d71af1eebf..f2bcc0ca0a5 100644 --- a/chromium/media/gpu/android/avda_picture_buffer_manager.h +++ b/chromium/media/gpu/android/avda_picture_buffer_manager.h @@ -89,11 +89,6 @@ class MEDIA_GPU_EXPORT AVDAPictureBufferManager { // Are there any unrendered picture buffers oustanding? bool HasUnrenderedPictures() const; - // If we're using an overlay, then drop all codec buffers for it, and also - // drop any reference to the surface bundle. If we're not using an overlay, - // then do nothing. - void ImmediatelyForgetOverlay(const PictureBufferMap& buffers); - // Returns the GL texture target that the PictureBuffer textures use. // Always use OES textures even though this will cause flickering in dev tools // when inspecting a fullscreen video. See http://crbug.com/592798 diff --git a/chromium/media/gpu/android/avda_surface_bundle.cc b/chromium/media/gpu/android/avda_surface_bundle.cc index d8db0cfaa60..1c09e8fcdbc 100644 --- a/chromium/media/gpu/android/avda_surface_bundle.cc +++ b/chromium/media/gpu/android/avda_surface_bundle.cc @@ -4,19 +4,29 @@ #include "media/gpu/android/avda_surface_bundle.h" +#include "base/threading/sequenced_task_runner_handle.h" #include "media/base/android/android_overlay.h" namespace media { -AVDASurfaceBundle::AVDASurfaceBundle() = default; +AVDASurfaceBundle::AVDASurfaceBundle() + : RefCountedDeleteOnSequence<AVDASurfaceBundle>( + base::SequencedTaskRunnerHandle::Get()), + weak_factory_(this) {} AVDASurfaceBundle::AVDASurfaceBundle(std::unique_ptr<AndroidOverlay> overlay) - : overlay(std::move(overlay)) {} + : RefCountedDeleteOnSequence<AVDASurfaceBundle>( + base::SequencedTaskRunnerHandle::Get()), + overlay(std::move(overlay)), + weak_factory_(this) {} AVDASurfaceBundle::AVDASurfaceBundle( scoped_refptr<SurfaceTextureGLOwner> surface_texture_owner) - : surface_texture(std::move(surface_texture_owner)), - surface_texture_surface(surface_texture->CreateJavaSurface()) {} + : RefCountedDeleteOnSequence<AVDASurfaceBundle>( + base::SequencedTaskRunnerHandle::Get()), + surface_texture(std::move(surface_texture_owner)), + surface_texture_surface(surface_texture->CreateJavaSurface()), + weak_factory_(this) {} AVDASurfaceBundle::~AVDASurfaceBundle() { // Explicitly free the surface first, just to be sure that it's deleted before @@ -44,4 +54,14 @@ const base::android::JavaRef<jobject>& AVDASurfaceBundle::GetJavaSurface() return surface_texture_surface.j_surface(); } +AVDASurfaceBundle::ScheduleLayoutCB AVDASurfaceBundle::GetScheduleLayoutCB() { + return base::BindRepeating(&AVDASurfaceBundle::ScheduleLayout, + weak_factory_.GetWeakPtr()); +} + +void AVDASurfaceBundle::ScheduleLayout(gfx::Rect rect) { + if (overlay) + overlay->ScheduleLayout(rect); +} + } // namespace media diff --git a/chromium/media/gpu/android/avda_surface_bundle.h b/chromium/media/gpu/android/avda_surface_bundle.h index b2236726727..e0dc749df37 100644 --- a/chromium/media/gpu/android/avda_surface_bundle.h +++ b/chromium/media/gpu/android/avda_surface_bundle.h @@ -5,7 +5,7 @@ #ifndef MEDIA_GPU_ANDROID_AVDA_SURFACE_BUNDLE_H_ #define MEDIA_GPU_ANDROID_AVDA_SURFACE_BUNDLE_H_ -#include "base/memory/ref_counted.h" +#include "base/memory/ref_counted_delete_on_sequence.h" #include "media/base/android/android_overlay.h" #include "media/base/surface_manager.h" #include "media/gpu/android/surface_texture_gl_owner.h" @@ -22,8 +22,10 @@ namespace media { // crashes due to the codec losing its output surface. // TODO(watk): Remove AVDA from the name. struct MEDIA_GPU_EXPORT AVDASurfaceBundle - : public base::RefCountedThreadSafe<AVDASurfaceBundle> { + : public base::RefCountedDeleteOnSequence<AVDASurfaceBundle> { public: + using ScheduleLayoutCB = base::RepeatingCallback<void(gfx::Rect)>; + // Create an empty bundle to be manually populated. explicit AVDASurfaceBundle(); explicit AVDASurfaceBundle(std::unique_ptr<AndroidOverlay> overlay); @@ -32,6 +34,11 @@ struct MEDIA_GPU_EXPORT AVDASurfaceBundle const base::android::JavaRef<jobject>& GetJavaSurface() const; + // Returns a callback that can be used to position this overlay. It must be + // called on the correct thread for the overlay. It will not keep a ref to + // |this|; the cb will do nothing if |this| is destroyed. + ScheduleLayoutCB GetScheduleLayoutCB(); + // The Overlay or SurfaceTexture. std::unique_ptr<AndroidOverlay> overlay; scoped_refptr<SurfaceTextureGLOwner> surface_texture; @@ -41,7 +48,12 @@ struct MEDIA_GPU_EXPORT AVDASurfaceBundle private: ~AVDASurfaceBundle(); - friend class base::RefCountedThreadSafe<AVDASurfaceBundle>; + friend class base::RefCountedDeleteOnSequence<AVDASurfaceBundle>; + friend class base::DeleteHelper<AVDASurfaceBundle>; + + void ScheduleLayout(gfx::Rect rect); + + base::WeakPtrFactory<AVDASurfaceBundle> weak_factory_; DISALLOW_COPY_AND_ASSIGN(AVDASurfaceBundle); }; diff --git a/chromium/media/gpu/android/codec_image.cc b/chromium/media/gpu/android/codec_image.cc index e230f276664..9887d553a19 100644 --- a/chromium/media/gpu/android/codec_image.cc +++ b/chromium/media/gpu/android/codec_image.cc @@ -35,16 +35,19 @@ std::unique_ptr<ui::ScopedMakeCurrent> MakeCurrentIfNeeded( CodecImage::CodecImage( std::unique_ptr<CodecOutputBuffer> output_buffer, scoped_refptr<SurfaceTextureGLOwner> surface_texture, - PromotionHintAggregator::NotifyPromotionHintCB promotion_hint_cb, - DestructionCb destruction_cb) + PromotionHintAggregator::NotifyPromotionHintCB promotion_hint_cb) : phase_(Phase::kInCodec), output_buffer_(std::move(output_buffer)), surface_texture_(std::move(surface_texture)), - promotion_hint_cb_(std::move(promotion_hint_cb)), - destruction_cb_(std::move(destruction_cb)) {} + promotion_hint_cb_(std::move(promotion_hint_cb)) {} CodecImage::~CodecImage() { - destruction_cb_.Run(this); + if (destruction_cb_) + std::move(destruction_cb_).Run(this); +} + +void CodecImage::SetDestructionCb(DestructionCb destruction_cb) { + destruction_cb_ = std::move(destruction_cb); } gfx::Size CodecImage::GetSize() { @@ -95,9 +98,11 @@ bool CodecImage::ScheduleOverlayPlane(gfx::AcceleratedWidget widget, // Move the overlay if needed. if (most_recent_bounds_ != bounds_rect) { most_recent_bounds_ = bounds_rect; - // TODO(liberato): When we start getting promotion hints, then we should - // not send a hint from NotifyPromotionHint() if it's promotable and we - // don't have a surface texture. We'll handle it here. + // Note that, if we're actually promoted to overlay, that this is where the + // hint is sent to the callback. NotifyPromotionHint detects this case and + // lets us do it. If we knew that we were going to get promotion hints, + // then we could always let NotifyPromotionHint do it. Unfortunately, we + // don't know that. promotion_hint_cb_.Run(PromotionHintAggregator::Hint(bounds_rect, true)); } @@ -129,6 +134,21 @@ void CodecImage::GetTextureMatrix(float matrix[16]) { YInvertMatrix(matrix); } +void CodecImage::NotifyPromotionHint(bool promotion_hint, + int display_x, + int display_y, + int display_width, + int display_height) { + // If this is promotable, and we're using an overlay, then skip sending this + // hint. ScheduleOverlayPlane will do it. + if (promotion_hint && !surface_texture_) + return; + + promotion_hint_cb_.Run(PromotionHintAggregator::Hint( + gfx::Rect(display_x, display_y, display_width, display_height), + promotion_hint)); +} + bool CodecImage::RenderToFrontBuffer() { return surface_texture_ ? RenderToSurfaceTextureFrontBuffer(BindingsMode::kRestore) @@ -202,4 +222,9 @@ bool CodecImage::RenderToOverlay() { return true; } +void CodecImage::SurfaceDestroyed() { + output_buffer_ = nullptr; + phase_ = Phase::kInvalidated; +} + } // namespace media diff --git a/chromium/media/gpu/android/codec_image.h b/chromium/media/gpu/android/codec_image.h index 3b9b7afb01e..d2dfde625a7 100644 --- a/chromium/media/gpu/android/codec_image.h +++ b/chromium/media/gpu/android/codec_image.h @@ -23,13 +23,15 @@ namespace media { // as needed in order to draw them. class MEDIA_GPU_EXPORT CodecImage : public gpu::gles2::GLStreamTextureImage { public: - // A callback for observing CodecImage destruction. - using DestructionCb = base::Callback<void(CodecImage*)>; + // A callback for observing CodecImage destruction. This is a repeating cb + // since CodecImageGroup calls the same cb for multiple images. + using DestructionCb = base::RepeatingCallback<void(CodecImage*)>; CodecImage(std::unique_ptr<CodecOutputBuffer> output_buffer, scoped_refptr<SurfaceTextureGLOwner> surface_texture, - PromotionHintAggregator::NotifyPromotionHintCB promotion_hint_cb, - DestructionCb destruction_cb); + PromotionHintAggregator::NotifyPromotionHintCB promotion_hint_cb); + + void SetDestructionCb(DestructionCb destruction_cb); // gl::GLImage implementation gfx::Size GetSize() override; @@ -45,12 +47,18 @@ class MEDIA_GPU_EXPORT CodecImage : public gpu::gles2::GLStreamTextureImage { gfx::OverlayTransform transform, const gfx::Rect& bounds_rect, const gfx::RectF& crop_rect) override; + void SetColorSpace(const gfx::ColorSpace& color_space) override {} void Flush() override {} void OnMemoryDump(base::trace_event::ProcessMemoryDump* pmd, uint64_t process_tracing_id, const std::string& dump_name) override; // gpu::gles2::GLStreamTextureMatrix implementation void GetTextureMatrix(float xform[16]) override; + void NotifyPromotionHint(bool promotion_hint, + int display_x, + int display_y, + int display_width, + int display_height) override; // Whether the codec buffer has been rendered to the front buffer. bool was_rendered_to_front_buffer() const { @@ -75,14 +83,19 @@ class MEDIA_GPU_EXPORT CodecImage : public gpu::gles2::GLStreamTextureImage { // buffer. Returns false if the buffer was invalidated. bool RenderToSurfaceTextureBackBuffer(); + // Called when we're no longer renderable because our surface is gone. We'll + // discard any codec buffer, and generally do nothing. + virtual void SurfaceDestroyed(); + + protected: + ~CodecImage() override; + private: // The lifecycle phases of an image. // The only possible transitions are from left to right. Both // kInFrontBuffer and kInvalidated are terminal. enum class Phase { kInCodec, kInBackBuffer, kInFrontBuffer, kInvalidated }; - ~CodecImage() override; - // Renders this image to the surface texture front buffer by first rendering // it to the back buffer if it's not already there, and then waiting for the // frame available event before calling UpdateTexImage(). Passing diff --git a/chromium/media/gpu/android/codec_image_group.cc b/chromium/media/gpu/android/codec_image_group.cc new file mode 100644 index 00000000000..c88aab0e3ab --- /dev/null +++ b/chromium/media/gpu/android/codec_image_group.cc @@ -0,0 +1,77 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/gpu/android/codec_image_group.h" + +#include "base/sequenced_task_runner.h" +#include "media/gpu/android/avda_surface_bundle.h" + +namespace media { + +CodecImageGroup::CodecImageGroup( + scoped_refptr<base::SequencedTaskRunner> task_runner, + scoped_refptr<AVDASurfaceBundle> surface_bundle) + : surface_bundle_(std::move(surface_bundle)), weak_this_factory_(this) { + // If the surface bundle has an overlay, then register for destruction + // callbacks. We thread-hop to the right thread, which means that we might + // find out about destruction asynchronously. Remember that the wp will be + // cleared on |task_runner|. + if (surface_bundle_->overlay) { + surface_bundle_->overlay->AddSurfaceDestroyedCallback(base::BindOnce( + [](scoped_refptr<base::SequencedTaskRunner> task_runner, + base::OnceCallback<void(AndroidOverlay*)> cb, + AndroidOverlay* overlay) -> void { + task_runner->PostTask(FROM_HERE, + base::BindOnce(std::move(cb), overlay)); + }, + std::move(task_runner), + base::BindOnce(&CodecImageGroup::OnSurfaceDestroyed, + weak_this_factory_.GetWeakPtr()))); + } + + // TODO(liberato): if there's no overlay, should we clear |surface_bundle_|? + // be sure not to call SurfaceDestroyed if !surface_bundle_ in that case when + // adding a new image. +} + +CodecImageGroup::~CodecImageGroup() {} + +void CodecImageGroup::SetDestructionCb( + CodecImage::DestructionCb destruction_cb) { + destruction_cb_ = std::move(destruction_cb); +} + +void CodecImageGroup::AddCodecImage(CodecImage* image) { + // If somebody adds an image after the surface has been destroyed, fail the + // image immediately. This can happen due to thread hopping. + if (!surface_bundle_) { + image->SurfaceDestroyed(); + return; + } + + images_.insert(image); + + // Bind a strong ref to |this| so that the callback will prevent us from being + // destroyed until the CodecImage is destroyed. + image->SetDestructionCb( + base::BindRepeating(&CodecImageGroup::OnCodecImageDestroyed, + scoped_refptr<CodecImageGroup>(this))); +} + +void CodecImageGroup::OnCodecImageDestroyed(CodecImage* image) { + images_.erase(image); + if (destruction_cb_) + destruction_cb_.Run(image); +} + +void CodecImageGroup::OnSurfaceDestroyed(AndroidOverlay* overlay) { + for (CodecImage* image : images_) + image->SurfaceDestroyed(); + + // While this might cause |surface_bundle_| to be deleted, it's okay because + // it's a RefCountedDeleteOnSequence. + surface_bundle_ = nullptr; +} + +} // namespace media diff --git a/chromium/media/gpu/android/codec_image_group.h b/chromium/media/gpu/android/codec_image_group.h new file mode 100644 index 00000000000..e463c51b7e5 --- /dev/null +++ b/chromium/media/gpu/android/codec_image_group.h @@ -0,0 +1,79 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_GPU_ANDROID_CODEC_IMAGE_GROUP_H_ +#define MEDIA_GPU_ANDROID_CODEC_IMAGE_GROUP_H_ + +#include <unordered_set> + +#include "base/memory/ref_counted.h" +#include "base/memory/weak_ptr.h" +#include "media/gpu/android/codec_image.h" +#include "media/gpu/android/promotion_hint_aggregator.h" +#include "media/gpu/media_gpu_export.h" + +namespace base { +class SequencedTaskRunner; +} + +namespace media { + +class AndroidOverlay; +struct AVDASurfaceBundle; +class CodecImage; + +// Object that lives on the GPU thread that knows about all CodecImages that +// share the same bundle. We are responsible for keeping the surface bundle +// around while any image is using it. If the overlay is destroyed, then we +// unback the images. +// +// We're held by the codec images that use us, so that we last at least as long +// as each of them. We might also be held by the VideoFrameFactory, if it's +// going to add new images to the group. +// +// Note that this class must be constructed on the thread on which the surface +// bundle (and overlay) may be accessed. All other methods will run on the +// provided task runner. +class MEDIA_GPU_EXPORT CodecImageGroup + : public base::RefCountedThreadSafe<CodecImageGroup> { + public: + // NOTE: Construction happens on the correct thread to access |bundle| and + // any overlay it contains. All other access to this class will happen on + // |task_runner|, including destruction. + CodecImageGroup(scoped_refptr<base::SequencedTaskRunner> task_runner, + scoped_refptr<AVDASurfaceBundle> bundle); + + // Set the callback that we'll notify when any image is destroyed. + void SetDestructionCb(CodecImage::DestructionCb destruction_cb); + + // Notify us that |image| uses |surface_bundle_|. + void AddCodecImage(CodecImage* image); + + protected: + virtual ~CodecImageGroup(); + friend class base::RefCountedThreadSafe<CodecImageGroup>; + friend class base::DeleteHelper<CodecImageGroup>; + + // Notify us that |image| has been destroyed. + void OnCodecImageDestroyed(CodecImage* image); + + // Notify us that our overlay surface has been destroyed. + void OnSurfaceDestroyed(AndroidOverlay*); + + private: + // Remember that this lives on some other thread. Do not actually use it. + scoped_refptr<AVDASurfaceBundle> surface_bundle_; + + // All the images that use |surface_bundle_|. + std::unordered_set<CodecImage*> images_; + + // We'll forward CodecImage destructions to |destruction_cb_|. + CodecImage::DestructionCb destruction_cb_; + + base::WeakPtrFactory<CodecImageGroup> weak_this_factory_; +}; + +} // namespace media + +#endif // MEDIA_GPU_ANDROID_CODEC_IMAGE_H_ diff --git a/chromium/media/gpu/android/codec_image_group_unittest.cc b/chromium/media/gpu/android/codec_image_group_unittest.cc new file mode 100644 index 00000000000..e0f632deda9 --- /dev/null +++ b/chromium/media/gpu/android/codec_image_group_unittest.cc @@ -0,0 +1,200 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/gpu/android/codec_image_group.h" + +#include "base/bind.h" +#include "base/callback.h" +#include "base/memory/ref_counted.h" +#include "base/test/scoped_task_environment.h" +#include "base/test/test_simple_task_runner.h" +#include "base/threading/thread.h" +#include "media/base/android/mock_android_overlay.h" +#include "media/gpu/android/avda_surface_bundle.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace media { + +namespace { +// Subclass of CodecImageGroup which will notify us when it's destroyed. +class CodecImageGroupWithDestructionHook : public CodecImageGroup { + public: + CodecImageGroupWithDestructionHook( + scoped_refptr<base::SequencedTaskRunner> task_runner, + scoped_refptr<AVDASurfaceBundle> surface_bundle) + : CodecImageGroup(std::move(task_runner), std::move(surface_bundle)) {} + + void SetDestructionCallback(base::OnceClosure cb) { + destruction_cb_ = std::move(cb); + } + + private: + ~CodecImageGroupWithDestructionHook() override { + if (destruction_cb_) + std::move(destruction_cb_).Run(); + } + + base::OnceClosure destruction_cb_; +}; + +// CodecImage with a mocked SurfaceDestroyed. +class MockCodecImage : public CodecImage { + public: + MockCodecImage() + : CodecImage(nullptr, + nullptr, + PromotionHintAggregator::NotifyPromotionHintCB()) {} + + MOCK_METHOD0(SurfaceDestroyed, void()); + + protected: + ~MockCodecImage() override {} +}; + +} // namespace + +class CodecImageGroupTest : public testing::Test { + public: + CodecImageGroupTest() = default; + + void SetUp() override { + gpu_task_runner_ = base::MakeRefCounted<base::TestSimpleTaskRunner>(); + } + + void TearDown() override {} + + struct Record { + scoped_refptr<AVDASurfaceBundle> surface_bundle; + scoped_refptr<CodecImageGroupWithDestructionHook> image_group; + + MockAndroidOverlay* overlay() const { + return static_cast<MockAndroidOverlay*>(surface_bundle->overlay.get()); + } + }; + + // Create an image group for a surface bundle with an overlay. + Record CreateImageGroup() { + std::unique_ptr<MockAndroidOverlay> overlay = + base::MakeUnique<MockAndroidOverlay>(); + EXPECT_CALL(*overlay.get(), MockAddSurfaceDestroyedCallback()); + Record rec; + rec.surface_bundle = + base::MakeRefCounted<AVDASurfaceBundle>(std::move(overlay)); + rec.image_group = base::MakeRefCounted<CodecImageGroupWithDestructionHook>( + gpu_task_runner_, rec.surface_bundle); + + return rec; + } + + // Handy method to check that CodecImage destruction is relayed properly. + MOCK_METHOD1(OnCodecImageDestroyed, void(CodecImage*)); + + base::test::ScopedTaskEnvironment env_; + + // Our thread is the mcvd thread. This is the task runner for the gpu thread. + scoped_refptr<base::TestSimpleTaskRunner> gpu_task_runner_; +}; + +TEST_F(CodecImageGroupTest, GroupRegistersForOverlayDestruction) { + // When we provide an image group with an overlay, it should register for + // destruction on that overlay. + Record rec = CreateImageGroup(); + // Note that we don't run any thread loop to completion -- it should assign + // the callback on our thread, since that's where the overlay is used. + // We verify expectations now, so that it doesn't matter if any task runners + // run during teardown. I'm not sure if the expectations would be checked + // before or after that. If after, then posting would still pass, which we + // don't want. + testing::Mock::VerifyAndClearExpectations(this); + + // There should not be just one ref to the surface bundle; the CodecImageGroup + // should have one too. + ASSERT_FALSE(rec.surface_bundle->HasOneRef()); +} + +TEST_F(CodecImageGroupTest, SurfaceBundleWithoutOverlayDoesntCrash) { + // Make sure that it's okay not to have an overlay. CodecImageGroup should + // handle ST surface bundles without crashing. + scoped_refptr<AVDASurfaceBundle> surface_bundle = + base::MakeRefCounted<AVDASurfaceBundle>(); + scoped_refptr<CodecImageGroup> image_group = + base::MakeRefCounted<CodecImageGroup>(gpu_task_runner_, surface_bundle); + // TODO(liberato): we should also make sure that adding an image doesn't call + // SurfaceDestroyed when it's added. +} + +TEST_F(CodecImageGroupTest, ImagesRetainRefToGroup) { + // Make sure that keeping an image around is sufficient to keep the group. + Record rec = CreateImageGroup(); + bool was_destroyed = false; + rec.image_group->SetDestructionCallback( + base::BindOnce([](bool* flag) -> void { *flag = true; }, &was_destroyed)); + scoped_refptr<CodecImage> image = new MockCodecImage(); + // We're supposed to call this from |gpu_task_runner_|, but all + // CodecImageGroup really cares about is being single sequence. + rec.image_group->AddCodecImage(image.get()); + + // The image should be sufficient to prevent destruction. + rec.image_group = nullptr; + ASSERT_FALSE(was_destroyed); + + // The image should be the last ref to the image group. + image = nullptr; + ASSERT_TRUE(was_destroyed); +} + +TEST_F(CodecImageGroupTest, DestroyedImagesForwardsImageDestruction) { + // Make sure that CodecImageGroup relays CodecImage destruction callbacks. + Record rec = CreateImageGroup(); + scoped_refptr<CodecImage> image_1 = new MockCodecImage(); + scoped_refptr<CodecImage> image_2 = new MockCodecImage(); + rec.image_group->SetDestructionCb(base::Bind( + &CodecImageGroupTest::OnCodecImageDestroyed, base::Unretained(this))); + rec.image_group->AddCodecImage(image_1.get()); + rec.image_group->AddCodecImage(image_2.get()); + + // Destroying |image_1| should call us back. + EXPECT_CALL(*this, OnCodecImageDestroyed(image_1.get())); + image_1 = nullptr; + testing::Mock::VerifyAndClearExpectations(this); + + // Same for |image_2|. + EXPECT_CALL(*this, OnCodecImageDestroyed(image_2.get())); + image_2 = nullptr; + testing::Mock::VerifyAndClearExpectations(this); +} + +TEST_F(CodecImageGroupTest, ImageGroupDropsForwardsSurfaceDestruction) { + // CodecImageGroup should notify all images when the surface is destroyed. We + // also verify that the image group drops its ref to the surface bundle, so + // that it doesn't prevent destruction of the overlay that provided it. + Record rec = CreateImageGroup(); + scoped_refptr<MockCodecImage> image_1 = new MockCodecImage(); + scoped_refptr<MockCodecImage> image_2 = new MockCodecImage(); + rec.image_group->AddCodecImage(image_1.get()); + rec.image_group->AddCodecImage(image_2.get()); + + // Destroy the surface. All destruction messages should be posted to the + // gpu thread. + EXPECT_CALL(*image_1.get(), SurfaceDestroyed()).Times(0); + EXPECT_CALL(*image_2.get(), SurfaceDestroyed()).Times(0); + // Note that we're calling this on the wrong thread, but that's okay. + rec.overlay()->OnSurfaceDestroyed(); + env_.RunUntilIdle(); + // Run the main loop and guarantee that nothing has run. It should be posted + // to |gpu_task_runner_|. + testing::Mock::VerifyAndClearExpectations(this); + + // Now run |gpu_task_runner_| and verify that the callbacks run. + EXPECT_CALL(*image_1.get(), SurfaceDestroyed()).Times(1); + EXPECT_CALL(*image_2.get(), SurfaceDestroyed()).Times(1); + gpu_task_runner_->RunUntilIdle(); + testing::Mock::VerifyAndClearExpectations(this); + + // The image group should drop its ref to the surface bundle. + ASSERT_TRUE(rec.surface_bundle->HasOneRef()); +} + +} // namespace media diff --git a/chromium/media/gpu/android/codec_image_unittest.cc b/chromium/media/gpu/android/codec_image_unittest.cc index a9711bdb997..b9553981f5c 100644 --- a/chromium/media/gpu/android/codec_image_unittest.cc +++ b/chromium/media/gpu/android/codec_image_unittest.cc @@ -67,7 +67,7 @@ class CodecImageTest : public testing::Test { context_ = nullptr; share_group_ = nullptr; surface_ = nullptr; - gl::init::ShutdownGL(); + gl::init::ShutdownGL(false); wrapper_->TakeCodecSurfacePair(); } @@ -77,11 +77,13 @@ class CodecImageTest : public testing::Test { CodecImage::DestructionCb destruction_cb = kNoop) { std::unique_ptr<CodecOutputBuffer> buffer; wrapper_->DequeueOutputBuffer(nullptr, nullptr, &buffer); - return new CodecImage( + scoped_refptr<CodecImage> image = new CodecImage( std::move(buffer), kind == kSurfaceTexture ? surface_texture_ : nullptr, base::BindRepeating(&PromotionHintReceiver::OnPromotionHint, - base::Unretained(&promotion_hint_receiver_)), - std::move(destruction_cb)); + base::Unretained(&promotion_hint_receiver_))); + + image->SetDestructionCb(std::move(destruction_cb)); + return image; } base::test::ScopedTaskEnvironment scoped_task_environment_; @@ -269,4 +271,22 @@ TEST_F(CodecImageTest, RenderToFrontBufferRestoresGLContext) { surface = nullptr; } +TEST_F(CodecImageTest, ScheduleOverlayPlaneDoesntSendDuplicateHints) { + // SOP should send only one promotion hint unless the position changes. + auto i = NewImage(kOverlay); + // Also verify that it sends the appropriate promotion hint so that the + // overlay is positioned properly. + PromotionHintAggregator::Hint hint1(gfx::Rect(1, 2, 3, 4), true); + PromotionHintAggregator::Hint hint2(gfx::Rect(5, 6, 7, 8), true); + EXPECT_CALL(promotion_hint_receiver_, OnPromotionHint(hint1)).Times(1); + EXPECT_CALL(promotion_hint_receiver_, OnPromotionHint(hint2)).Times(1); + i->ScheduleOverlayPlane(gfx::AcceleratedWidget(), 0, gfx::OverlayTransform(), + hint1.screen_rect, gfx::RectF()); + i->ScheduleOverlayPlane(gfx::AcceleratedWidget(), 0, gfx::OverlayTransform(), + hint1.screen_rect, gfx::RectF()); + // Sending a different rectangle should send another hint. + i->ScheduleOverlayPlane(gfx::AcceleratedWidget(), 0, gfx::OverlayTransform(), + hint2.screen_rect, gfx::RectF()); +} + } // namespace media diff --git a/chromium/media/gpu/android/fake_codec_allocator.cc b/chromium/media/gpu/android/fake_codec_allocator.cc index 500957d9026..3cdf0c3975e 100644 --- a/chromium/media/gpu/android/fake_codec_allocator.cc +++ b/chromium/media/gpu/android/fake_codec_allocator.cc @@ -19,7 +19,8 @@ FakeCodecAllocator::FakeCodecAllocator( scoped_refptr<base::SequencedTaskRunner> task_runner) : testing::NiceMock<AVDACodecAllocator>( base::BindRepeating(&MockMediaCodecBridge::CreateVideoDecoder), - task_runner) {} + task_runner), + most_recent_config(new CodecConfig()) {} FakeCodecAllocator::~FakeCodecAllocator() = default; @@ -29,8 +30,7 @@ void FakeCodecAllocator::StopThread(AVDACodecAllocatorClient* client) {} std::unique_ptr<MediaCodecBridge> FakeCodecAllocator::CreateMediaCodecSync( scoped_refptr<CodecConfig> config) { - most_recent_overlay = config->surface_bundle->overlay.get(); - most_recent_surface_texture = config->surface_bundle->surface_texture.get(); + CopyCodecConfig(config); MockCreateMediaCodecSync(most_recent_overlay, most_recent_surface_texture); std::unique_ptr<MockMediaCodecBridge> codec; @@ -53,8 +53,7 @@ void FakeCodecAllocator::CreateMediaCodecAsync( // Clear |most_recent_codec| until somebody calls Provide*CodecAsync(). most_recent_codec = nullptr; most_recent_codec_destruction_observer = nullptr; - most_recent_overlay = config->surface_bundle->overlay.get(); - most_recent_surface_texture = config->surface_bundle->surface_texture.get(); + CopyCodecConfig(config); pending_surface_bundle_ = config->surface_bundle; client_ = client; codec_creation_pending_ = true; @@ -96,4 +95,22 @@ void FakeCodecAllocator::ProvideNullCodecAsync() { client_->OnCodecConfigured(nullptr, std::move(pending_surface_bundle_)); } +void FakeCodecAllocator::CopyCodecConfig(scoped_refptr<CodecConfig> config) { + // CodecConfig isn't copyable, since it has unique_ptrs and such. + most_recent_overlay = config->surface_bundle->overlay.get(); + most_recent_surface_texture = config->surface_bundle->surface_texture.get(); + most_recent_config->media_crypto = + config->media_crypto + ? base::MakeUnique<base::android::ScopedJavaGlobalRef<jobject>>( + *config->media_crypto) + : nullptr; + most_recent_config->requires_secure_codec = config->requires_secure_codec; + most_recent_config->initial_expected_coded_size = + config->initial_expected_coded_size; + most_recent_config->software_codec_forbidden = + config->software_codec_forbidden; + most_recent_config->csd0 = config->csd0; + most_recent_config->csd1 = config->csd1; +} + } // namespace media diff --git a/chromium/media/gpu/android/fake_codec_allocator.h b/chromium/media/gpu/android/fake_codec_allocator.h index 8ff6bf29b06..61a3e95b0c7 100644 --- a/chromium/media/gpu/android/fake_codec_allocator.h +++ b/chromium/media/gpu/android/fake_codec_allocator.h @@ -72,7 +72,13 @@ class FakeCodecAllocator : public testing::NiceMock<AVDACodecAllocator> { // Whether CreateMediaCodecSync() is allowed to succeed. bool allow_sync_creation = true; + // Copy of most of the fields in the most recent config, except for the ptrs. + scoped_refptr<CodecConfig> most_recent_config; + private: + // Copy |config| to |most_recent_config| etc. + void CopyCodecConfig(scoped_refptr<CodecConfig> config); + // Whether CreateMediaCodecAsync() has been called but a codec hasn't been // provided yet. bool codec_creation_pending_ = false; diff --git a/chromium/media/gpu/android/media_codec_video_decoder.cc b/chromium/media/gpu/android/media_codec_video_decoder.cc index efea07ab824..55d9a9e8c9b 100644 --- a/chromium/media/gpu/android/media_codec_video_decoder.cc +++ b/chromium/media/gpu/android/media_codec_video_decoder.cc @@ -6,13 +6,16 @@ #include "base/callback.h" #include "base/callback_helpers.h" +#include "base/command_line.h" #include "base/logging.h" #include "base/memory/ptr_util.h" #include "base/memory/weak_ptr.h" +#include "base/metrics/histogram_macros.h" #include "media/base/android/media_codec_bridge_impl.h" #include "media/base/android/media_codec_util.h" #include "media/base/bind_to_current_loop.h" #include "media/base/decoder_buffer.h" +#include "media/base/media_switches.h" #include "media/base/video_codecs.h" #include "media/base/video_decoder_config.h" #include "media/gpu/android/android_video_surface_chooser.h" @@ -106,7 +109,11 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder( : output_cb_(output_cb), codec_allocator_(codec_allocator), request_overlay_info_cb_(std::move(request_overlay_info_cb)), - surface_chooser_(std::move(surface_chooser)), + surface_chooser_helper_( + std::move(surface_chooser), + base::CommandLine::ForCurrentProcess()->HasSwitch( + switches::kForceVideoOverlays), + base::FeatureList::IsEnabled(media::kUseAndroidOverlayAggressively)), video_frame_factory_(std::move(video_frame_factory)), overlay_factory_cb_(std::move(overlay_factory_cb)), device_info_(device_info), @@ -116,7 +123,7 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder( weak_factory_(this), codec_allocator_weak_factory_(this) { DVLOG(2) << __func__; - surface_chooser_->SetClientCallbacks( + surface_chooser_helper_.chooser()->SetClientCallbacks( base::Bind(&MediaCodecVideoDecoder::OnSurfaceChosen, weak_factory_.GetWeakPtr()), base::Bind(&MediaCodecVideoDecoder::OnSurfaceChosen, @@ -127,6 +134,17 @@ MediaCodecVideoDecoder::~MediaCodecVideoDecoder() { DVLOG(2) << __func__; ReleaseCodec(); codec_allocator_->StopThread(this); + + if (!media_drm_bridge_cdm_context_) + return; + + DCHECK(cdm_registration_id_); + + // Cancel previously registered callback (if any). + media_drm_bridge_cdm_context_->SetMediaCryptoReadyCB( + MediaDrmBridgeCdmContext::MediaCryptoReadyCB()); + + media_drm_bridge_cdm_context_->UnregisterPlayer(cdm_registration_id_); } void MediaCodecVideoDecoder::Destroy() { @@ -168,9 +186,73 @@ void MediaCodecVideoDecoder::Initialize(const VideoDecoderConfig& config, ExtractSpsAndPps(config.extra_data(), &csd0_, &csd1_); #endif + // For encrypted content, defer signalling success until the Cdm is ready. + if (config.is_encrypted()) { + SetCdm(cdm_context, init_cb); + return; + } + // Do the rest of the initialization lazily on the first decode. - // TODO(watk): Add CDM Support. - DCHECK(!cdm_context); + init_cb.Run(true); +} + +void MediaCodecVideoDecoder::SetCdm(CdmContext* cdm_context, + const InitCB& init_cb) { + if (!cdm_context) { + LOG(ERROR) << "No CDM provided"; + EnterTerminalState(State::kError); + init_cb.Run(false); + return; + } + + // On Android platform the CdmContext must be a MediaDrmBridgeCdmContext. + media_drm_bridge_cdm_context_ = + static_cast<media::MediaDrmBridgeCdmContext*>(cdm_context); + + // Register CDM callbacks. The callbacks registered will be posted back to + // this thread via BindToCurrentLoop. + + // Since |this| holds a reference to the |cdm_|, by the time the CDM is + // destructed, UnregisterPlayer() must have been called and |this| has been + // destructed as well. So the |cdm_unset_cb| will never have a chance to be + // called. + // TODO(xhwang): Remove |cdm_unset_cb| after it's not used on all platforms. + cdm_registration_id_ = media_drm_bridge_cdm_context_->RegisterPlayer( + media::BindToCurrentLoop(base::Bind(&MediaCodecVideoDecoder::OnKeyAdded, + weak_factory_.GetWeakPtr())), + base::Bind(&base::DoNothing)); + + media_drm_bridge_cdm_context_->SetMediaCryptoReadyCB(media::BindToCurrentLoop( + base::Bind(&MediaCodecVideoDecoder::OnMediaCryptoReady, + weak_factory_.GetWeakPtr(), init_cb))); +} + +void MediaCodecVideoDecoder::OnMediaCryptoReady( + const InitCB& init_cb, + JavaObjectPtr media_crypto, + bool requires_secure_video_codec) { + DVLOG(1) << __func__; + + DCHECK(state_ == State::kInitializing); + + if (!media_crypto || media_crypto->is_null()) { + LOG(ERROR) << "MediaCrypto is not available"; + EnterTerminalState(State::kError); + init_cb.Run(false); + return; + } + + media_crypto_ = *media_crypto; + requires_secure_codec_ = requires_secure_video_codec; + + // Request a secure surface in all cases. For L3, it's okay if we fall back + // to SurfaceTexture rather than fail composition. For L1, it's required. + surface_chooser_helper_.SetSecureSurfaceMode( + requires_secure_video_codec + ? SurfaceChooserHelper::SecureSurfaceMode::kRequired + : SurfaceChooserHelper::SecureSurfaceMode::kRequested); + + // Signal success, and create the codec lazily on the first decode. init_cb.Run(true); } @@ -184,7 +266,14 @@ void MediaCodecVideoDecoder::StartLazyInit() { DVLOG(2) << __func__; lazy_init_pending_ = false; codec_allocator_->StartThread(this); + // Only ask for promotion hints if we can actually switch surfaces, since we + // wouldn't be able to do anything with them. Also, if threaded texture + // mailboxes are enabled, then we turn off overlays anyway. + const bool want_promotion_hints = + device_info_->IsSetOutputSurfaceSupported() && + !enable_threaded_texture_mailboxes_; video_frame_factory_->Initialize( + want_promotion_hints, base::Bind(&MediaCodecVideoDecoder::OnVideoFrameFactoryInitialized, weak_factory_.GetWeakPtr())); } @@ -222,18 +311,12 @@ void MediaCodecVideoDecoder::OnOverlayInfoChanged( if (InTerminalState()) return; - // TODO(watk): Handle frame_hidden like AVDA. Maybe even if in a terminal - // state. - // TODO(watk): Incorporate the other chooser_state_ signals. - bool overlay_changed = !overlay_info_.RefersToSameOverlayAs(overlay_info); overlay_info_ = overlay_info; - chooser_state_.is_fullscreen = overlay_info_.is_fullscreen; - chooser_state_.is_frame_hidden = overlay_info_.is_frame_hidden; - surface_chooser_->UpdateState( + surface_chooser_helper_.SetIsFullscreen(overlay_info_.is_fullscreen); + surface_chooser_helper_.UpdateChooserState( overlay_changed ? base::make_optional(CreateOverlayFactoryCb()) - : base::nullopt, - chooser_state_); + : base::nullopt); } void MediaCodecVideoDecoder::OnSurfaceChosen( @@ -267,7 +350,8 @@ void MediaCodecVideoDecoder::OnSurfaceDestroyed(AndroidOverlay* overlay) { // a single overlay so this must be the one we're using. In this case it's // the responsibility of our consumer to destroy us for surface transitions. // TODO(liberato): This might not be true for L1 / L3, since our caller has - // no idea that this has happened. We should unback the frames here. + // no idea that this has happened. We should unback the frames here. This + // might work now that we have CodecImageGroup -- verify this. if (!device_info_->IsSetOutputSurfaceSupported()) { EnterTerminalState(State::kSurfaceDestroyed); return; @@ -291,8 +375,14 @@ void MediaCodecVideoDecoder::TransitionToTargetSurface() { DCHECK(SurfaceTransitionPending()); DCHECK(device_info_->IsSetOutputSurfaceSupported()); - if (!codec_->SetSurface(target_surface_bundle_)) + if (!codec_->SetSurface(target_surface_bundle_)) { + video_frame_factory_->SetSurfaceBundle(nullptr); EnterTerminalState(State::kError); + return; + } + + video_frame_factory_->SetSurfaceBundle(target_surface_bundle_); + CacheFrameInformation(); } void MediaCodecVideoDecoder::CreateCodec() { @@ -302,11 +392,16 @@ void MediaCodecVideoDecoder::CreateCodec() { scoped_refptr<CodecConfig> config = new CodecConfig(); config->codec = decoder_config_.codec(); - // TODO(watk): Set |requires_secure_codec| correctly using - // MediaDrmBridgeCdmContext::MediaCryptoReadyCB. - config->requires_secure_codec = decoder_config_.is_encrypted(); + config->requires_secure_codec = requires_secure_codec_; + // TODO(liberato): per android_util.h, remove JavaObjectPtr. + config->media_crypto = + base::MakeUnique<base::android::ScopedJavaGlobalRef<jobject>>( + media_crypto_); config->initial_expected_coded_size = decoder_config_.coded_size(); config->surface_bundle = target_surface_bundle_; + // Note that this might be the same surface bundle that we've been using, if + // we're reinitializing the codec without changing surfaces. That's fine. + video_frame_factory_->SetSurfaceBundle(target_surface_bundle_); codec_allocator_->CreateMediaCodecAsync( codec_allocator_weak_factory_.GetWeakPtr(), std::move(config)); } @@ -334,6 +429,9 @@ void MediaCodecVideoDecoder::OnCodecConfigured( if (SurfaceTransitionPending()) TransitionToTargetSurface(); + // Cache the frame information that goes with this codec. + CacheFrameInformation(); + StartTimer(); } @@ -535,13 +633,16 @@ bool MediaCodecVideoDecoder::DequeueOutput() { if (drain_type_) return true; + // Record the frame type that we're sending and some information about why. + UMA_HISTOGRAM_ENUMERATION( + "Media.AVDA.FrameInformation", cached_frame_information_, + static_cast<int>( + SurfaceChooserHelper::FrameInformation::FRAME_INFORMATION_MAX) + + 1); // PRESUBMIT_IGNORE_UMA_MAX + video_frame_factory_->CreateVideoFrame( - std::move(output_buffer), - codec_->SurfaceBundle()->overlay - ? nullptr - : surface_texture_bundle_->surface_texture, - presentation_time, decoder_config_.natural_size(), - CreatePromotionHintCB(), + std::move(output_buffer), presentation_time, + decoder_config_.natural_size(), CreatePromotionHintCB(), base::Bind(&MediaCodecVideoDecoder::ForwardVideoFrame, weak_factory_.GetWeakPtr(), reset_generation_)); return true; @@ -707,27 +808,47 @@ int MediaCodecVideoDecoder::GetMaxDecodeRequests() const { } PromotionHintAggregator::NotifyPromotionHintCB -MediaCodecVideoDecoder::CreatePromotionHintCB() const { +MediaCodecVideoDecoder::CreatePromotionHintCB() { // Right now, we don't request promotion hints. This is only used by SOP. // While we could simplify it a bit, this is the general form that we'll use // when handling promotion hints. - // TODO(liberato): Keeping the surface bundle around as long as the images - // doesn't work so well if the surface is destroyed. In that case, the right - // thing to do is (a) wait for any codec to quit using the surface, and (b) - // clear |overlay| out of the surface bundle. - // Having the surface bundle register for destruction callbacks, instead of - // us, makes sense. + // Note that this keeps only a wp to the surface bundle via |layout_cb|. It + // also continues to work even if |this| is destroyed; images might want to + // move an overlay around even after MCVD has been torn down. For example + // inline L1 content will fall into this case. return BindToCurrentLoop(base::BindRepeating( - [](scoped_refptr<AVDASurfaceBundle> surface_bundle, + [](base::WeakPtr<MediaCodecVideoDecoder> mcvd, + AVDASurfaceBundle::ScheduleLayoutCB layout_cb, PromotionHintAggregator::Hint hint) { // If we're promotable, and we have a surface bundle, then also // position the overlay. We could do this even if the overlay is // not promotable, but it wouldn't have any visible effect. - if (hint.is_promotable && surface_bundle) - surface_bundle->overlay->ScheduleLayout(hint.screen_rect); + if (hint.is_promotable) + layout_cb.Run(hint.screen_rect); + + // Notify MCVD about the promotion hint, so that it can decide if it + // wants to switch to / from an overlay. + if (mcvd) + mcvd->NotifyPromotionHint(hint); }, - codec_->SurfaceBundle())); + weak_factory_.GetWeakPtr(), + codec_->SurfaceBundle()->GetScheduleLayoutCB())); +} + +bool MediaCodecVideoDecoder::IsUsingOverlay() const { + return codec_ && codec_->SurfaceBundle() && codec_->SurfaceBundle()->overlay; +} + +void MediaCodecVideoDecoder::NotifyPromotionHint( + PromotionHintAggregator::Hint hint) { + surface_chooser_helper_.NotifyPromotionHintAndUpdateChooser(hint, + IsUsingOverlay()); +} + +void MediaCodecVideoDecoder::CacheFrameInformation() { + cached_frame_information_ = + surface_chooser_helper_.ComputeFrameInformation(IsUsingOverlay()); } } // namespace media diff --git a/chromium/media/gpu/android/media_codec_video_decoder.h b/chromium/media/gpu/android/media_codec_video_decoder.h index 2175c953fb4..4f1d2c644da 100644 --- a/chromium/media/gpu/android/media_codec_video_decoder.h +++ b/chromium/media/gpu/android/media_codec_video_decoder.h @@ -18,6 +18,7 @@ #include "media/gpu/android/avda_codec_allocator.h" #include "media/gpu/android/codec_wrapper.h" #include "media/gpu/android/device_info.h" +#include "media/gpu/android/surface_chooser_helper.h" #include "media/gpu/android/video_frame_factory.h" #include "media/gpu/media_gpu_export.h" #include "services/service_manager/public/cpp/service_context_ref.h" @@ -82,6 +83,17 @@ class MEDIA_GPU_EXPORT MediaCodecVideoDecoder // Protected for testing. ~MediaCodecVideoDecoder() override; + // Set up |cdm_context| as part of initialization. Guarantees that |init_cb| + // will be called depending on the outcome, though not necessarily before this + // function returns. + void SetCdm(CdmContext* cdm_context, const InitCB& init_cb); + + // Called when the Cdm provides |media_crypto|. Will signal |init_cb| based + // on the result, and set the codec config properly. + void OnMediaCryptoReady(const InitCB& init_cb, + JavaObjectPtr media_crypto, + bool requires_secure_video_codec); + private: // The test has access for PumpCodec(). friend class MediaCodecVideoDecoderTest; @@ -174,12 +186,21 @@ class MEDIA_GPU_EXPORT MediaCodecVideoDecoder // Releases |codec_| if it's not null. void ReleaseCodec(); + // Return true if we have a codec that's outputting to an overlay. + bool IsUsingOverlay() const; + + // Notify us about a promotion hint. + void NotifyPromotionHint(PromotionHintAggregator::Hint hint); + + // Update |cached_frame_information_|. + void CacheFrameInformation(); + // Creates an overlay factory cb based on the value of overlay_info_. AndroidOverlayFactoryCB CreateOverlayFactoryCb(); // Create a callback that will handle promotion hints, and set the overlay // position if required. - PromotionHintAggregator::NotifyPromotionHintCB CreatePromotionHintCB() const; + PromotionHintAggregator::NotifyPromotionHintCB CreatePromotionHintCB(); State state_ = State::kInitializing; @@ -234,12 +255,8 @@ class MEDIA_GPU_EXPORT MediaCodecVideoDecoder // The current overlay info, which possibly specifies an overlay to render to. OverlayInfo overlay_info_; - // The surface chooser we use to decide which kind of surface to configure the - // codec with. - std::unique_ptr<AndroidVideoSurfaceChooser> surface_chooser_; - - // Current state for the chooser. - AndroidVideoSurfaceChooser::State chooser_state_; + // The helper which manages our surface chooser for us. + SurfaceChooserHelper surface_chooser_helper_; // The factory for creating VideoFrames from CodecOutputBuffers. std::unique_ptr<VideoFrameFactory> video_frame_factory_; @@ -250,6 +267,27 @@ class MEDIA_GPU_EXPORT MediaCodecVideoDecoder DeviceInfo* device_info_; bool enable_threaded_texture_mailboxes_; + // Most recently cached frame information, so that we can dispatch it without + // recomputing it on every frame. It changes very rarely. + SurfaceChooserHelper::FrameInformation cached_frame_information_ = + SurfaceChooserHelper::FrameInformation::SURFACETEXTURE_INSECURE; + + // CDM related stuff. + + // CDM context that knowns about MediaCrypto. Owned by CDM which is external + // to this decoder. + MediaDrmBridgeCdmContext* media_drm_bridge_cdm_context_ = nullptr; + + // MediaDrmBridge requires registration/unregistration of the player, this + // registration id is used for this. + int cdm_registration_id_ = 0; + + // Do we need a hw-secure codec? + bool requires_secure_codec_ = false; + + // Optional crypto object from the Cdm. + base::android::ScopedJavaGlobalRef<jobject> media_crypto_; + // If we're running in a service context this ref lets us keep the service // thread alive until destruction. std::unique_ptr<service_manager::ServiceContextRef> context_ref_; diff --git a/chromium/media/gpu/android/media_codec_video_decoder_unittest.cc b/chromium/media/gpu/android/media_codec_video_decoder_unittest.cc index 8f88da07a02..676b56beb39 100644 --- a/chromium/media/gpu/android/media_codec_video_decoder_unittest.cc +++ b/chromium/media/gpu/android/media_codec_video_decoder_unittest.cc @@ -3,6 +3,7 @@ // found in the LICENSE file. #include "media/gpu/android/media_codec_video_decoder.h" +#include "base/android/jni_android.h" #include "base/bind.h" #include "base/run_loop.h" #include "base/test/mock_callback.h" @@ -11,12 +12,13 @@ #include "gpu/command_buffer/service/gpu_preferences.h" #include "media/base/android/media_codec_util.h" #include "media/base/android/mock_android_overlay.h" +#include "media/base/android/mock_media_drm_bridge_cdm_context.h" #include "media/base/decoder_buffer.h" #include "media/base/gmock_callback_support.h" #include "media/base/test_helpers.h" #include "media/gpu/android/android_video_surface_chooser_impl.h" -#include "media/gpu/android/fake_android_video_surface_chooser.h" #include "media/gpu/android/fake_codec_allocator.h" +#include "media/gpu/android/mock_android_video_surface_chooser.h" #include "media/gpu/android/mock_device_info.h" #include "media/gpu/android/mock_surface_texture_gl_owner.h" #include "media/gpu/android/video_frame_factory.h" @@ -61,7 +63,8 @@ class MockServiceContextRef : public service_manager::ServiceContextRef { class MockVideoFrameFactory : public VideoFrameFactory { public: - MOCK_METHOD1(Initialize, void(InitCb init_cb)); + MOCK_METHOD2(Initialize, void(bool wants_promotion_hint, InitCb init_cb)); + MOCK_METHOD1(MockSetSurfaceBundle, void(scoped_refptr<AVDASurfaceBundle>)); MOCK_METHOD6( MockCreateVideoFrame, void(CodecOutputBuffer* raw_output_buffer, @@ -74,14 +77,24 @@ class MockVideoFrameFactory : public VideoFrameFactory { void(base::OnceClosure* closure)); MOCK_METHOD0(CancelPendingCallbacks, void()); + void SetSurfaceBundle( + scoped_refptr<AVDASurfaceBundle> surface_bundle) override { + MockSetSurfaceBundle(surface_bundle); + if (!surface_bundle) { + surface_texture_ = nullptr; + } else { + surface_texture_ = + surface_bundle->overlay ? nullptr : surface_bundle->surface_texture; + } + } + void CreateVideoFrame( std::unique_ptr<CodecOutputBuffer> output_buffer, - scoped_refptr<SurfaceTextureGLOwner> surface_texture, base::TimeDelta timestamp, gfx::Size natural_size, PromotionHintAggregator::NotifyPromotionHintCB promotion_hint_cb, OutputWithReleaseMailboxCB output_cb) override { - MockCreateVideoFrame(output_buffer.get(), surface_texture, timestamp, + MockCreateVideoFrame(output_buffer.get(), surface_texture_, timestamp, natural_size, promotion_hint_cb, output_cb); last_output_buffer_ = std::move(output_buffer); } @@ -92,6 +105,7 @@ class MockVideoFrameFactory : public VideoFrameFactory { } std::unique_ptr<CodecOutputBuffer> last_output_buffer_; + scoped_refptr<SurfaceTextureGLOwner> surface_texture_; base::OnceClosure last_closure_; }; @@ -114,7 +128,8 @@ class MediaCodecVideoDecoderTest : public testing::Test { } void CreateMcvd() { - auto surface_chooser = base::MakeUnique<NiceMock<FakeSurfaceChooser>>(); + auto surface_chooser = + base::MakeUnique<NiceMock<MockAndroidVideoSurfaceChooser>>(); surface_chooser_ = surface_chooser.get(); auto surface_texture = @@ -126,8 +141,10 @@ class MediaCodecVideoDecoderTest : public testing::Test { base::MakeUnique<NiceMock<MockVideoFrameFactory>>(); video_frame_factory_ = video_frame_factory.get(); // Set up VFF to pass |surface_texture_| via its InitCb. - ON_CALL(*video_frame_factory_, Initialize(_)) - .WillByDefault(RunCallback<0>(surface_texture)); + const bool want_promotion_hint = + device_info_->IsSetOutputSurfaceSupported(); + ON_CALL(*video_frame_factory_, Initialize(want_promotion_hint, _)) + .WillByDefault(RunCallback<1>(surface_texture)); auto* observable_mcvd = new DestructionObservableMCVD( gpu_preferences_, base::Bind(&OutputWithReleaseMailboxCb), @@ -144,6 +161,17 @@ class MediaCodecVideoDecoderTest : public testing::Test { destruction_observer_->ExpectDestruction(); } + void CreateCdm(bool require_secure_video_decoder) { + cdm_ = base::MakeUnique<MockMediaDrmBridgeCdmContext>(cdm_id_); + require_secure_video_decoder_ = require_secure_video_decoder; + + // We need to send an object as the media crypto, but MCVD shouldn't + // use it for anything. Just send in some random java object, so that + // it's not null. + media_crypto_ = base::android::ScopedJavaGlobalRef<jobject>( + gl::SurfaceTexture::Create(0)->j_surface_texture()); + } + // Just call Initialize(). MCVD will be waiting for a call to Decode() before // continuining initialization. bool Initialize( @@ -152,9 +180,21 @@ class MediaCodecVideoDecoderTest : public testing::Test { CreateMcvd(); bool result = false; auto init_cb = [](bool* result_out, bool result) { *result_out = result; }; - mcvd_->Initialize(config, false, nullptr, base::Bind(init_cb, &result), + mcvd_->Initialize(config, false, cdm_.get(), base::Bind(init_cb, &result), base::Bind(&OutputCb)); base::RunLoop().RunUntilIdle(); + + if (config.is_encrypted() && cdm_) { + // If the output is encrypted, then we expect that MCVD will be waiting + // for the media crypto object. + // TODO(liberato): why does CreateJavaObjectPtr() not link? + cdm_->media_crypto_ready_cb.Run( + base::MakeUnique<base::android::ScopedJavaGlobalRef<jobject>>( + media_crypto_), + require_secure_video_decoder_); + base::RunLoop().RunUntilIdle(); + } + return result; } @@ -208,7 +248,7 @@ class MediaCodecVideoDecoderTest : public testing::Test { scoped_refptr<DecoderBuffer> fake_decoder_buffer_; std::unique_ptr<MockDeviceInfo> device_info_; std::unique_ptr<FakeCodecAllocator> codec_allocator_; - FakeSurfaceChooser* surface_chooser_; + MockAndroidVideoSurfaceChooser* surface_chooser_; MockSurfaceTextureGLOwner* surface_texture_; MockVideoFrameFactory* video_frame_factory_; NiceMock<base::MockCallback<VideoDecoder::DecodeCB>> decode_cb_; @@ -217,10 +257,17 @@ class MediaCodecVideoDecoderTest : public testing::Test { bool restart_for_transitions_; gpu::GpuPreferences gpu_preferences_; + const int cdm_id_ = 123; + // This is not an actual media crypto object. + base::android::ScopedJavaGlobalRef<jobject> media_crypto_; + bool require_secure_video_decoder_ = false; + // |mcvd_raw_| lets us call PumpCodec() even after |mcvd_| is dropped, for // testing the teardown path. MediaCodecVideoDecoder* mcvd_raw_; std::unique_ptr<MediaCodecVideoDecoder> mcvd_; + // This must outlive |mcvd_| . + std::unique_ptr<MockMediaDrmBridgeCdmContext> cdm_; }; TEST_F(MediaCodecVideoDecoderTest, UnknownCodecIsRejected) { @@ -238,7 +285,7 @@ TEST_F(MediaCodecVideoDecoderTest, SmallVp8IsRejected) { TEST_F(MediaCodecVideoDecoderTest, InitializeDoesntInitSurfaceOrCodec) { CreateMcvd(); - EXPECT_CALL(*video_frame_factory_, Initialize(_)).Times(0); + EXPECT_CALL(*video_frame_factory_, Initialize(_, _)).Times(0); EXPECT_CALL(*surface_chooser_, MockUpdateState()).Times(0); EXPECT_CALL(*codec_allocator_, MockCreateMediaCodecAsync(_, _)).Times(0); Initialize(); @@ -246,7 +293,7 @@ TEST_F(MediaCodecVideoDecoderTest, InitializeDoesntInitSurfaceOrCodec) { TEST_F(MediaCodecVideoDecoderTest, FirstDecodeTriggersFrameFactoryInit) { Initialize(); - EXPECT_CALL(*video_frame_factory_, Initialize(_)); + EXPECT_CALL(*video_frame_factory_, Initialize(_, _)); mcvd_->Decode(fake_decoder_buffer_, decode_cb_.Get()); } @@ -269,9 +316,9 @@ TEST_F(MediaCodecVideoDecoderTest, } TEST_F(MediaCodecVideoDecoderTest, RestartForOverlayTransitionsFlagIsCorrect) { - Initialize(); ON_CALL(*device_info_, IsSetOutputSurfaceSupported()) .WillByDefault(Return(true)); + Initialize(); mcvd_->Decode(fake_decoder_buffer_, decode_cb_.Get()); ASSERT_FALSE(restart_for_transitions_); } @@ -300,8 +347,8 @@ TEST_F(MediaCodecVideoDecoderTest, CodecIsCreatedAfterSurfaceChosen) { TEST_F(MediaCodecVideoDecoderTest, FrameFactoryInitFailureIsAnError) { Initialize(); - ON_CALL(*video_frame_factory_, Initialize(_)) - .WillByDefault(RunCallback<0>(nullptr)); + ON_CALL(*video_frame_factory_, Initialize(_, _)) + .WillByDefault(RunCallback<1>(nullptr)); EXPECT_CALL(decode_cb_, Run(DecodeStatus::DECODE_ERROR)).Times(1); EXPECT_CALL(*surface_chooser_, MockUpdateState()).Times(0); mcvd_->Decode(fake_decoder_buffer_, decode_cb_.Get()); @@ -386,9 +433,9 @@ TEST_F(MediaCodecVideoDecoderTest, CodecIsCreatedWithChosenOverlay) { TEST_F(MediaCodecVideoDecoderTest, CodecCreationWeakPtrIsInvalidatedBySurfaceDestroyed) { - auto* overlay = InitializeWithOverlay_OneDecodePending(); ON_CALL(*device_info_, IsSetOutputSurfaceSupported()) .WillByDefault(Return(false)); + auto* overlay = InitializeWithOverlay_OneDecodePending(); overlay->OnSurfaceDestroyed(); // MCVD should invalidate its CodecAllocatorClient WeakPtr so that it doesn't @@ -422,12 +469,12 @@ TEST_F(MediaCodecVideoDecoderTest, SurfaceDestroyedDoesSyncSurfaceTransition) { TEST_F(MediaCodecVideoDecoderTest, SurfaceDestroyedReleasesCodecIfSetSurfaceIsNotSupported) { + ON_CALL(*device_info_, IsSetOutputSurfaceSupported()) + .WillByDefault(Return(false)); auto* overlay = InitializeWithOverlay_OneDecodePending(); auto* codec = codec_allocator_->ProvideMockCodecAsync(); // MCVD must synchronously release the codec. - ON_CALL(*device_info_, IsSetOutputSurfaceSupported()) - .WillByDefault(Return(false)); EXPECT_CALL(*codec, SetSurface(_)).Times(0); EXPECT_CALL(*codec_allocator_, MockReleaseMediaCodec(codec, NotNull(), _)); overlay->OnSurfaceDestroyed(); @@ -684,4 +731,73 @@ TEST_F(MediaCodecVideoDecoderTest, TeardownDrainsVp8CodecsBeforeDestruction) { base::RunLoop().RunUntilIdle(); } +TEST_F(MediaCodecVideoDecoderTest, CdmInitializationWorksForL3) { + // Make sure that MCVD uses the cdm, and sends it along to the codec. + CreateCdm(false); + EXPECT_CALL(*cdm_, RegisterPlayer(_, _)); + InitializeWithOverlay_OneDecodePending( + TestVideoConfig::NormalEncrypted(kCodecH264)); + ASSERT_TRUE(!!cdm_->new_key_cb); + ASSERT_TRUE(!!cdm_->cdm_unset_cb); + ASSERT_TRUE(!!cdm_->media_crypto_ready_cb); + ASSERT_EQ(surface_chooser_->current_state_.is_secure, true); + ASSERT_EQ(surface_chooser_->current_state_.is_required, false); + ASSERT_FALSE(codec_allocator_->most_recent_config->requires_secure_codec); + // We can't check for equality safely, but verify that something was provided. + ASSERT_TRUE(codec_allocator_->most_recent_config->media_crypto->obj()); + + // When |mcvd_| is destroyed, expect that it will unregister itself. + EXPECT_CALL(*cdm_, + UnregisterPlayer(MockMediaDrmBridgeCdmContext::kRegistrationId)); +} + +TEST_F(MediaCodecVideoDecoderTest, CdmInitializationWorksForL1) { + // Make sure that MCVD uses the cdm, and sends it along to the codec. + CreateCdm(true); + EXPECT_CALL(*cdm_, RegisterPlayer(_, _)); + InitializeWithOverlay_OneDecodePending( + TestVideoConfig::NormalEncrypted(kCodecH264)); + ASSERT_TRUE(!!cdm_->new_key_cb); + ASSERT_TRUE(!!cdm_->cdm_unset_cb); + ASSERT_TRUE(!!cdm_->media_crypto_ready_cb); + ASSERT_EQ(surface_chooser_->current_state_.is_secure, true); + ASSERT_EQ(surface_chooser_->current_state_.is_required, true); + ASSERT_TRUE(codec_allocator_->most_recent_config->requires_secure_codec); + ASSERT_TRUE(codec_allocator_->most_recent_config->media_crypto->obj()); + + // When |mcvd_| is destroyed, expect that it will unregister itself. + EXPECT_CALL(*cdm_, + UnregisterPlayer(MockMediaDrmBridgeCdmContext::kRegistrationId)); +} + +TEST_F(MediaCodecVideoDecoderTest, CdmIsIgnoredIfNotEncrypted) { + CreateCdm(true); + // It should not register or unregister. + EXPECT_CALL(*cdm_, RegisterPlayer(_, _)).Times(0); + EXPECT_CALL(*cdm_, + UnregisterPlayer(MockMediaDrmBridgeCdmContext::kRegistrationId)) + .Times(0); + ASSERT_TRUE(Initialize(TestVideoConfig::NormalH264())); + ASSERT_TRUE(!cdm_->new_key_cb); + ASSERT_TRUE(!cdm_->cdm_unset_cb); + ASSERT_TRUE(!cdm_->media_crypto_ready_cb); + ASSERT_EQ(surface_chooser_->current_state_.is_secure, false); + ASSERT_EQ(surface_chooser_->current_state_.is_required, false); +} + +TEST_F(MediaCodecVideoDecoderTest, MissingMediaCryptoFailsInit) { + // Encrypted media that doesn't get a mediacrypto should fail to init. + CreateCdm(true); + media_crypto_ = nullptr; + EXPECT_CALL(*cdm_, RegisterPlayer(_, _)); + ASSERT_FALSE(Initialize(TestVideoConfig::NormalEncrypted(kCodecH264))); + EXPECT_CALL(*cdm_, + UnregisterPlayer(MockMediaDrmBridgeCdmContext::kRegistrationId)); +} + +TEST_F(MediaCodecVideoDecoderTest, MissingCdmFailsInit) { + // MCVD should fail init if we don't provide a cdm with an encrypted config. + ASSERT_FALSE(Initialize(TestVideoConfig::NormalEncrypted(kCodecH264))); +} + } // namespace media diff --git a/chromium/media/gpu/android/fake_android_video_surface_chooser.cc b/chromium/media/gpu/android/mock_android_video_surface_chooser.cc index ec005ffbb50..21641c838f8 100644 --- a/chromium/media/gpu/android/fake_android_video_surface_chooser.cc +++ b/chromium/media/gpu/android/mock_android_video_surface_chooser.cc @@ -2,14 +2,14 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "media/gpu/android/fake_android_video_surface_chooser.h" +#include "media/gpu/android/mock_android_video_surface_chooser.h" namespace media { -FakeSurfaceChooser::FakeSurfaceChooser() = default; -FakeSurfaceChooser::~FakeSurfaceChooser() = default; +MockAndroidVideoSurfaceChooser::MockAndroidVideoSurfaceChooser() = default; +MockAndroidVideoSurfaceChooser::~MockAndroidVideoSurfaceChooser() = default; -void FakeSurfaceChooser::SetClientCallbacks( +void MockAndroidVideoSurfaceChooser::SetClientCallbacks( UseOverlayCB use_overlay_cb, UseSurfaceTextureCB use_surface_texture_cb) { MockSetClientCallbacks(); @@ -17,7 +17,7 @@ void FakeSurfaceChooser::SetClientCallbacks( use_surface_texture_cb_ = std::move(use_surface_texture_cb); } -void FakeSurfaceChooser::UpdateState( +void MockAndroidVideoSurfaceChooser::UpdateState( base::Optional<AndroidOverlayFactoryCB> factory, const State& new_state) { MockUpdateState(); @@ -28,11 +28,11 @@ void FakeSurfaceChooser::UpdateState( current_state_ = new_state; } -void FakeSurfaceChooser::ProvideSurfaceTexture() { +void MockAndroidVideoSurfaceChooser::ProvideSurfaceTexture() { use_surface_texture_cb_.Run(); } -void FakeSurfaceChooser::ProvideOverlay( +void MockAndroidVideoSurfaceChooser::ProvideOverlay( std::unique_ptr<AndroidOverlay> overlay) { use_overlay_cb_.Run(std::move(overlay)); } diff --git a/chromium/media/gpu/android/fake_android_video_surface_chooser.h b/chromium/media/gpu/android/mock_android_video_surface_chooser.h index a7000ca5f57..9f3fdc64758 100644 --- a/chromium/media/gpu/android/fake_android_video_surface_chooser.h +++ b/chromium/media/gpu/android/mock_android_video_surface_chooser.h @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#ifndef MEDIA_GPU_ANDROID_FAKE_ANDROID_VIDEO_SURFACE_CHOOSER_H_ -#define MEDIA_GPU_ANDROID_FAKE_ANDROID_VIDEO_SURFACE_CHOOSER_H_ +#ifndef MEDIA_GPU_ANDROID_MOCK_ANDROID_VIDEO_SURFACE_CHOOSER_H_ +#define MEDIA_GPU_ANDROID_MOCK_ANDROID_VIDEO_SURFACE_CHOOSER_H_ #include "media/gpu/android/android_video_surface_chooser.h" #include "testing/gmock/include/gmock/gmock.h" @@ -11,12 +11,12 @@ namespace media { -// A fake surface chooser that lets tests choose the surface with +// A mock surface chooser that lets tests choose the surface with // ProvideOverlay() and ProvideSurfaceTexture(). -class FakeSurfaceChooser : public AndroidVideoSurfaceChooser { +class MockAndroidVideoSurfaceChooser : public AndroidVideoSurfaceChooser { public: - FakeSurfaceChooser(); - ~FakeSurfaceChooser() override; + MockAndroidVideoSurfaceChooser(); + ~MockAndroidVideoSurfaceChooser() override; // Mocks that are called by the fakes below. MOCK_METHOD0(MockSetClientCallbacks, void()); @@ -41,9 +41,9 @@ class FakeSurfaceChooser : public AndroidVideoSurfaceChooser { State current_state_; private: - DISALLOW_COPY_AND_ASSIGN(FakeSurfaceChooser); + DISALLOW_COPY_AND_ASSIGN(MockAndroidVideoSurfaceChooser); }; } // namespace media -#endif // MEDIA_GPU_ANDROID_FAKE_ANDROID_VIDEO_SURFACE_CHOOSER_H_ +#endif // MEDIA_GPU_ANDROID_MOCK_ANDROID_VIDEO_SURFACE_CHOOSER_H_ diff --git a/chromium/media/gpu/android/mock_promotion_hint_aggregator.cc b/chromium/media/gpu/android/mock_promotion_hint_aggregator.cc new file mode 100644 index 00000000000..c97c9344bfa --- /dev/null +++ b/chromium/media/gpu/android/mock_promotion_hint_aggregator.cc @@ -0,0 +1,22 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/gpu/android/mock_promotion_hint_aggregator.h" + +using testing::_; +using testing::Return; + +namespace media { + +MockPromotionHintAggregator::MockPromotionHintAggregator() { + SetIsSafeToPromote(false); +} + +MockPromotionHintAggregator::~MockPromotionHintAggregator() {} + +void MockPromotionHintAggregator::SetIsSafeToPromote(bool is_safe) { + ON_CALL(*this, IsSafeToPromote()).WillByDefault(Return(is_safe)); +} + +} // namespace media diff --git a/chromium/media/gpu/android/mock_promotion_hint_aggregator.h b/chromium/media/gpu/android/mock_promotion_hint_aggregator.h new file mode 100644 index 00000000000..7470d929035 --- /dev/null +++ b/chromium/media/gpu/android/mock_promotion_hint_aggregator.h @@ -0,0 +1,29 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_GPU_ANDROID_MOCK_PROMOTION_HINT_AGGREGATOR_H_ +#define MEDIA_GPU_ANDROID_MOCK_PROMOTION_HINT_AGGREGATOR_H_ + +#include "media/gpu/android/promotion_hint_aggregator.h" + +#include "testing/gmock/include/gmock/gmock.h" + +namespace media { + +class MockPromotionHintAggregator + : public testing::NiceMock<PromotionHintAggregator> { + public: + MockPromotionHintAggregator(); + ~MockPromotionHintAggregator(); + + MOCK_METHOD1(NotifyPromotionHint, void(const Hint& hint)); + MOCK_METHOD0(IsSafeToPromote, bool()); + + // Convenience function to change the return of IsSafeToPromote. + void SetIsSafeToPromote(bool is_safe); +}; + +} // namespace media + +#endif // MEDIA_GPU_ANDROID_MOCK_PROMOTION_HINT_AGGREGATOR_H_ diff --git a/chromium/media/gpu/android/surface_chooser_helper.cc b/chromium/media/gpu/android/surface_chooser_helper.cc new file mode 100644 index 00000000000..489c547930b --- /dev/null +++ b/chromium/media/gpu/android/surface_chooser_helper.cc @@ -0,0 +1,160 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/gpu/android/surface_chooser_helper.h" + +#include "base/memory/ptr_util.h" +#include "base/time/default_tick_clock.h" +#include "base/time/tick_clock.h" +#include "media/gpu/android/android_video_surface_chooser.h" +#include "media/gpu/android/promotion_hint_aggregator_impl.h" + +namespace media { + +namespace { + +// Number of frames to defer overlays for when entering fullscreen. This lets +// blink relayout settle down a bit. If overlay positions were synchronous, +// then we wouldn't need this. +enum { kFrameDelayForFullscreenLayout = 15 }; + +// How often do we let the surface chooser try for an overlay? While we'll +// retry if some relevant state changes on our side (e.g., fullscreen state), +// there's plenty of state that we don't know about (e.g., power efficiency, +// memory pressure => cancelling an old overlay, etc.). We just let the chooser +// retry every once in a while for those things. +constexpr base::TimeDelta RetryChooserTimeout = base::TimeDelta::FromSeconds(5); + +} // namespace + +SurfaceChooserHelper::SurfaceChooserHelper( + std::unique_ptr<AndroidVideoSurfaceChooser> surface_chooser, + bool is_overlay_required, + bool promote_aggressively, + std::unique_ptr<PromotionHintAggregator> promotion_hint_aggregator, + std::unique_ptr<base::TickClock> tick_clock) + : surface_chooser_(std::move(surface_chooser)), + is_overlay_required_(is_overlay_required), + promotion_hint_aggregator_( + promotion_hint_aggregator + ? std::move(promotion_hint_aggregator) + : base::MakeUnique<PromotionHintAggregatorImpl>()), + tick_clock_(tick_clock ? std::move(tick_clock) + : base::MakeUnique<base::DefaultTickClock>()) { + surface_chooser_state_.is_required = is_overlay_required_; + surface_chooser_state_.promote_aggressively = promote_aggressively; +} + +SurfaceChooserHelper::~SurfaceChooserHelper() {} + +void SurfaceChooserHelper::SetSecureSurfaceMode(SecureSurfaceMode mode) { + bool is_secure = false; + requires_secure_video_surface_ = false; + + switch (mode) { + case SecureSurfaceMode::kInsecure: + break; + case SecureSurfaceMode::kRequested: + is_secure = true; + break; + case SecureSurfaceMode::kRequired: + is_secure = true; + requires_secure_video_surface_ = true; + break; + } + + surface_chooser_state_.is_secure = is_secure; + surface_chooser_state_.is_required = + requires_secure_video_surface_ || is_overlay_required_; +} + +void SurfaceChooserHelper::SetIsFullscreen(bool is_fullscreen) { + // TODO(liberato): AVDA previously only set is_expecting_relayout when + // getting overlay info, not when checking fullscreen for the first time. + // This might affect pre-M devices. I think the pre-M path doesn't care. + if (is_fullscreen && !surface_chooser_state_.is_fullscreen) { + // It would be nice if we could just delay until we get a hint from an + // overlay that's "in fullscreen" in the sense that the CompositorFrame it + // came from had some flag set to indicate that the renderer was in + // fullscreen mode when it was generated. However, even that's hard, since + // there's no real connection between "renderer finds out about fullscreen" + // and "blink has completed layouts for it". The latter is what we really + // want to know. + surface_chooser_state_.is_expecting_relayout = true; + hints_until_clear_relayout_flag_ = kFrameDelayForFullscreenLayout; + } + + surface_chooser_state_.is_fullscreen = is_fullscreen; +} + +void SurfaceChooserHelper::UpdateChooserState( + base::Optional<AndroidOverlayFactoryCB> new_factory) { + surface_chooser_->UpdateState(std::move(new_factory), surface_chooser_state_); +} + +void SurfaceChooserHelper::NotifyPromotionHintAndUpdateChooser( + const PromotionHintAggregator::Hint& hint, + bool is_using_overlay) { + bool update_state = false; + + promotion_hint_aggregator_->NotifyPromotionHint(hint); + + // If we're expecting a full screen relayout, then also use this hint as a + // notification that another frame has happened. + if (hints_until_clear_relayout_flag_ > 0) { + hints_until_clear_relayout_flag_--; + if (hints_until_clear_relayout_flag_ == 0) { + surface_chooser_state_.is_expecting_relayout = false; + update_state = true; + } + } + + surface_chooser_state_.initial_position = hint.screen_rect; + bool promotable = promotion_hint_aggregator_->IsSafeToPromote(); + if (promotable != surface_chooser_state_.is_compositor_promotable) { + surface_chooser_state_.is_compositor_promotable = promotable; + update_state = true; + } + + // If we've been provided with enough new frames, then update the state even + // if it hasn't changed. This lets |surface_chooser_| retry for an overlay. + // It's especially helpful for power-efficient overlays, since we don't know + // when an overlay becomes power efficient. It also helps retry any failure + // that's not accompanied by a state change, such as if android destroys the + // overlay asynchronously for a transient reason. + // + // If we're already using an overlay, then there's no need to do this. + base::TimeTicks now = tick_clock_->NowTicks(); + if (!is_using_overlay && + now - most_recent_chooser_retry_ >= RetryChooserTimeout) { + update_state = true; + } + + if (update_state) { + most_recent_chooser_retry_ = now; + UpdateChooserState(base::Optional<AndroidOverlayFactoryCB>()); + } +} + +SurfaceChooserHelper::FrameInformation +SurfaceChooserHelper::ComputeFrameInformation(bool is_using_overlay) { + if (!is_using_overlay) { + // Not an overlay. + return surface_chooser_state_.is_secure + ? FrameInformation::SURFACETEXTURE_L3 + : FrameInformation::SURFACETEXTURE_INSECURE; + } + + // Overlay. + if (surface_chooser_state_.is_secure) { + return surface_chooser_state_.is_required ? FrameInformation::OVERLAY_L1 + : FrameInformation::OVERLAY_L3; + } + + return surface_chooser_state_.is_fullscreen + ? FrameInformation::OVERLAY_INSECURE_PLAYER_ELEMENT_FULLSCREEN + : FrameInformation::OVERLAY_INSECURE_NON_PLAYER_ELEMENT_FULLSCREEN; +} + +} // namespace media diff --git a/chromium/media/gpu/android/surface_chooser_helper.h b/chromium/media/gpu/android/surface_chooser_helper.h new file mode 100644 index 00000000000..148df6e743e --- /dev/null +++ b/chromium/media/gpu/android/surface_chooser_helper.h @@ -0,0 +1,122 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_GPU_ANDROID_SURFACE_CHOOSER_HELPER_H_ +#define MEDIA_GPU_ANDROID_SURFACE_CHOOSER_HELPER_H_ + +#include <memory> + +#include "base/macros.h" +#include "base/time/time.h" +#include "media/gpu/android/android_video_surface_chooser.h" +#include "media/gpu/android/promotion_hint_aggregator.h" +#include "media/gpu/media_gpu_export.h" + +namespace base { +class TickClock; +} + +namespace media { + +// Helper class to manage state transitions for SurfaceChooser::State. It's +// complicated and standalone enough not to be part of SurfaceChooser itself. +class MEDIA_GPU_EXPORT SurfaceChooserHelper { + public: + // |promotion_hint_aggregator| and |tick_clock| are for tests. Normally, we + // create the correct default implementations ourself. + // |is_overlay_required| tells us to require overlays(!). + // |promote_aggressively| causes us to use overlays whenever they're power- + // efficient, which lets us catch fullscreen-div cases. + SurfaceChooserHelper( + std::unique_ptr<AndroidVideoSurfaceChooser> surface_chooser, + bool is_overlay_required, + bool promote_aggressively, + std::unique_ptr<PromotionHintAggregator> promotion_hint_aggregator = + nullptr, + std::unique_ptr<base::TickClock> tick_clock = nullptr); + ~SurfaceChooserHelper(); + + enum class SecureSurfaceMode { + // The surface should not be secure. This allows both overlays and + // SurfaceTexture surfaces. + kInsecure, + + // It is preferable to have a secure surface, but insecure + // (SurfaceTexture) is better than failing. + kRequested, + + // The surface must be a secure surface, and should fail otherwise. + kRequired, + }; + + // Must match AVDAFrameInformation UMA enum. Please do not remove or re-order + // values, only append new ones. + enum class FrameInformation { + SURFACETEXTURE_INSECURE = 0, + SURFACETEXTURE_L3 = 1, + OVERLAY_L3 = 2, + OVERLAY_L1 = 3, + OVERLAY_INSECURE_PLAYER_ELEMENT_FULLSCREEN = 4, + OVERLAY_INSECURE_NON_PLAYER_ELEMENT_FULLSCREEN = 5, + + // Max enum value. + FRAME_INFORMATION_MAX = OVERLAY_INSECURE_NON_PLAYER_ELEMENT_FULLSCREEN + }; + + // The setters do not update the chooser state, since pre-M requires us to be + // careful about the first update, since we can't change it later. + + // Notify us about the desired surface security. Does not update the chooser + // state. + void SetSecureSurfaceMode(SecureSurfaceMode mode); + + // Notify us about the fullscreen state. Does not update the chooser state. + void SetIsFullscreen(bool is_fullscreen); + + // Update the chooser state using the given factory. + void UpdateChooserState(base::Optional<AndroidOverlayFactoryCB> new_factory); + + // Notify us about a promotion hint. This will update the chooser state + // if needed. + void NotifyPromotionHintAndUpdateChooser( + const PromotionHintAggregator::Hint& hint, + bool is_using_overlay); + + AndroidVideoSurfaceChooser* chooser() const { return surface_chooser_.get(); } + + // Return the FrameInformation bucket number that the config reflects, given + // that |is_using_overlay| reflects whether we're currently using an overlay + // or not. + FrameInformation ComputeFrameInformation(bool is_using_overlay); + + private: + AndroidVideoSurfaceChooser::State surface_chooser_state_; + std::unique_ptr<AndroidVideoSurfaceChooser> surface_chooser_; + + // Are overlays required by command-line options? + bool is_overlay_required_ = false; + + // Do we require an overlay due to the surface mode? + bool requires_secure_video_surface_ = false; + + std::unique_ptr<PromotionHintAggregator> promotion_hint_aggregator_; + + // Time since we last updated the chooser state. + base::TimeTicks most_recent_chooser_retry_; + + std::unique_ptr<base::TickClock> tick_clock_; + + // Number of promotion hints that we need to receive before clearing the + // "delay overlay promotion" flag in |surface_chooser_state_|. We do this so + // that the transition looks better, since it gives blink time to stabilize. + // Since overlay positioning isn't synchronous, it's good to make sure that + // blink isn't moving the quad around too. + int hints_until_clear_relayout_flag_ = 0; + + DISALLOW_COPY_AND_ASSIGN(SurfaceChooserHelper); +}; + +} // namespace media + +#endif // MEDIA_GPU_ANDROID_SURFACE_CHOOSER_HELPER_H_ diff --git a/chromium/media/gpu/android/surface_chooser_helper_unittest.cc b/chromium/media/gpu/android/surface_chooser_helper_unittest.cc new file mode 100644 index 00000000000..5c69b8c28d2 --- /dev/null +++ b/chromium/media/gpu/android/surface_chooser_helper_unittest.cc @@ -0,0 +1,274 @@ +// Copyright 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/gpu/android/surface_chooser_helper.h" + +#include <stdint.h> + +#include <memory> + +#include "base/bind.h" +#include "base/logging.h" +#include "base/memory/ptr_util.h" +#include "base/test/simple_test_tick_clock.h" +#include "media/gpu/android/mock_android_video_surface_chooser.h" +#include "media/gpu/android/mock_promotion_hint_aggregator.h" +#include "testing/gtest/include/gtest/gtest.h" + +using base::TimeDelta; +using testing::_; +using testing::AtLeast; + +namespace media { + +// Unit tests for PromotionHintAggregatorImplTest +class SurfaceChooserHelperTest : public testing::Test { + public: + ~SurfaceChooserHelperTest() override {} + + void SetUp() override { + // Create a default helper. + ReplaceHelper(false, false); + } + + void TearDown() override {} + + void ReplaceHelper(bool is_overlay_required, bool promote_aggressively) { + // Advance the clock so that time 0 isn't recent. + std::unique_ptr<base::SimpleTestTickClock> tick_clock = + base::MakeUnique<base::SimpleTestTickClock>(); + tick_clock_ = tick_clock.get(); + tick_clock_->Advance(TimeDelta::FromSeconds(10000)); + + std::unique_ptr<MockAndroidVideoSurfaceChooser> chooser = + base::MakeUnique<MockAndroidVideoSurfaceChooser>(); + chooser_ = chooser.get(); + std::unique_ptr<MockPromotionHintAggregator> aggregator = + base::MakeUnique<MockPromotionHintAggregator>(); + aggregator_ = aggregator.get(); + helper_ = base::MakeUnique<SurfaceChooserHelper>( + std::move(chooser), is_overlay_required, promote_aggressively, + std::move(aggregator), std::move(tick_clock)); + } + + // Convenience function. + void UpdateChooserState() { + EXPECT_CALL(*chooser_, MockUpdateState()); + helper_->UpdateChooserState(base::Optional<AndroidOverlayFactoryCB>()); + } + + base::SimpleTestTickClock* tick_clock_ = nullptr; + + MockPromotionHintAggregator* aggregator_ = nullptr; + + MockAndroidVideoSurfaceChooser* chooser_ = nullptr; + + std::unique_ptr<SurfaceChooserHelper> helper_; +}; + +TEST_F(SurfaceChooserHelperTest, SetIsFullscreen) { + // Entering fullscreen should expect relayout. + helper_->SetIsFullscreen(true); + UpdateChooserState(); + ASSERT_TRUE(chooser_->current_state_.is_fullscreen); + ASSERT_TRUE(chooser_->current_state_.is_expecting_relayout); + + // Exiting fullscreen should not reset the expecting layout flag. + helper_->SetIsFullscreen(false); + UpdateChooserState(); + ASSERT_FALSE(chooser_->current_state_.is_fullscreen); + // We don't really care if it sets expecting_relayout, clears it, or not. +} + +TEST_F(SurfaceChooserHelperTest, SetIsOverlayRequired) { + // The default helper was created without |is_required|, so verify that. + UpdateChooserState(); + ASSERT_FALSE(chooser_->current_state_.is_required); + + ReplaceHelper(true, false); + UpdateChooserState(); + ASSERT_TRUE(chooser_->current_state_.is_required); +} + +TEST_F(SurfaceChooserHelperTest, SetInsecureSurface) { + helper_->SetSecureSurfaceMode( + SurfaceChooserHelper::SecureSurfaceMode::kInsecure); + UpdateChooserState(); + ASSERT_FALSE(chooser_->current_state_.is_secure); + ASSERT_FALSE(chooser_->current_state_.is_required); +} + +TEST_F(SurfaceChooserHelperTest, SetRequestedSecureSurface) { + helper_->SetSecureSurfaceMode( + SurfaceChooserHelper::SecureSurfaceMode::kRequested); + UpdateChooserState(); + ASSERT_TRUE(chooser_->current_state_.is_secure); + ASSERT_FALSE(chooser_->current_state_.is_required); +} + +TEST_F(SurfaceChooserHelperTest, SetRequiredSecureSurface) { + helper_->SetSecureSurfaceMode( + SurfaceChooserHelper::SecureSurfaceMode::kRequired); + UpdateChooserState(); + ASSERT_TRUE(chooser_->current_state_.is_secure); + ASSERT_TRUE(chooser_->current_state_.is_required); + + // Also check that removing kRequired puts |is_required| back, since that has + // special processing for "always required". + helper_->SetSecureSurfaceMode( + SurfaceChooserHelper::SecureSurfaceMode::kInsecure); + UpdateChooserState(); + ASSERT_FALSE(chooser_->current_state_.is_required); +} + +TEST_F(SurfaceChooserHelperTest, StillRequiredAfterClearingSecure) { + // Verify that setting then clearing kRequired doesn't make |is_required| + // false if overlays were required during construction. + ReplaceHelper(true, false); + helper_->SetSecureSurfaceMode( + SurfaceChooserHelper::SecureSurfaceMode::kRequired); + UpdateChooserState(); + ASSERT_TRUE(chooser_->current_state_.is_required); + + helper_->SetSecureSurfaceMode( + SurfaceChooserHelper::SecureSurfaceMode::kInsecure); + UpdateChooserState(); + // Should still be true. + ASSERT_TRUE(chooser_->current_state_.is_required); +} + +TEST_F(SurfaceChooserHelperTest, SetPromoteAggressively) { + UpdateChooserState(); + ASSERT_FALSE(chooser_->current_state_.promote_aggressively); + + ReplaceHelper(false, true); + UpdateChooserState(); + ASSERT_TRUE(chooser_->current_state_.promote_aggressively); +} + +TEST_F(SurfaceChooserHelperTest, PromotionHintsForwardsHint) { + // Make sure that NotifyPromotionHint relays the hint to the aggregator. + PromotionHintAggregator::Hint hint(gfx::Rect(1, 2, 3, 4), false); + EXPECT_CALL(*aggregator_, NotifyPromotionHint(hint)); + helper_->NotifyPromotionHintAndUpdateChooser(hint, false); +} + +TEST_F(SurfaceChooserHelperTest, PromotionHintsRelayPosition) { + // Make sure that the overlay position is sent to the chooser. + gfx::Rect rect(0, 1, 2, 3); + + // Send an unpromotable hint and verify that the state reflects it. We set + // it to be promotable so that it notifies the chooser. + helper_->NotifyPromotionHintAndUpdateChooser( + PromotionHintAggregator::Hint(rect, true), false); + ASSERT_EQ(chooser_->current_state_.initial_position, rect); +} + +TEST_F(SurfaceChooserHelperTest, PromotionHintsRelayPromotable) { + // Make sure that the promotability state is forwarded to the chooser. + EXPECT_CALL(*chooser_, MockUpdateState()).Times(AtLeast(1)); + PromotionHintAggregator::Hint hint(gfx::Rect(), false); + + // Send a hint while the aggregator says it's unpromotable, and verify that + // the state reflects it. + helper_->NotifyPromotionHintAndUpdateChooser(hint, false); + ASSERT_FALSE(chooser_->current_state_.is_compositor_promotable); + + // Send a promotable hint and check the state. Note that the hint has nothing + // to do with it; it's the aggregator's state. + aggregator_->SetIsSafeToPromote(true); + helper_->NotifyPromotionHintAndUpdateChooser(hint, false); + ASSERT_TRUE(chooser_->current_state_.is_compositor_promotable); +} + +TEST_F(SurfaceChooserHelperTest, PromotionHintsClearRelayoutFlag) { + // Set fullscreen to enable relayout. + helper_->SetIsFullscreen(true); + UpdateChooserState(); + ASSERT_TRUE(chooser_->current_state_.is_expecting_relayout); + + // Send a bunch of hints. + EXPECT_CALL(*chooser_, MockUpdateState()).Times(AtLeast(1)); + for (int i = 0; i < 15; i++) { + PromotionHintAggregator::Hint hint(gfx::Rect(), false); + helper_->NotifyPromotionHintAndUpdateChooser(hint, false); + } + + // It should no longer be expecting fs. + ASSERT_FALSE(chooser_->current_state_.is_expecting_relayout); +} + +TEST_F(SurfaceChooserHelperTest, PromotionHintsUpdateChooserStatePeriodically) { + // Verify that, if enough time passes, we'll get chooser updates if we want + // and overlay but don't have one. + PromotionHintAggregator::Hint hint(gfx::Rect(), false); + + // Sending the first hint should update the chooser, since we're becoming + // safe to promote. + aggregator_->SetIsSafeToPromote(true); + EXPECT_CALL(*chooser_, MockUpdateState()).Times(1); + helper_->NotifyPromotionHintAndUpdateChooser(hint, false); + + // Sending an additional hint should not, whether or not we're using an + // overlay currently. + EXPECT_CALL(*chooser_, MockUpdateState()).Times(0); + helper_->NotifyPromotionHintAndUpdateChooser(hint, true); + EXPECT_CALL(*chooser_, MockUpdateState()).Times(0); + helper_->NotifyPromotionHintAndUpdateChooser(hint, false); + + // Advancing the time and using an overlay should not send a hint. + tick_clock_->Advance(base::TimeDelta::FromSeconds(10)); + EXPECT_CALL(*chooser_, MockUpdateState()).Times(0); + helper_->NotifyPromotionHintAndUpdateChooser(hint, true); + + // If we're not using an overlay, then it should update the chooser to see + // if it's willing to try for one now. + EXPECT_CALL(*chooser_, MockUpdateState()).Times(1); + helper_->NotifyPromotionHintAndUpdateChooser(hint, false); +} + +TEST_F(SurfaceChooserHelperTest, FrameInformationIsCorrectForL1) { + // Verify L1 cases. + helper_->SetSecureSurfaceMode( + SurfaceChooserHelper::SecureSurfaceMode::kRequired); + + ASSERT_EQ(SurfaceChooserHelper::FrameInformation::OVERLAY_L1, + helper_->ComputeFrameInformation(true)); + // We don't check the "not using overlay" case; it's unclear what we should be + // doing in this case anyway. +} + +TEST_F(SurfaceChooserHelperTest, FrameInformationIsCorrectForL3) { + // Verify L3 cases. + helper_->SetSecureSurfaceMode( + SurfaceChooserHelper::SecureSurfaceMode::kRequested); + + ASSERT_EQ(SurfaceChooserHelper::FrameInformation::OVERLAY_L3, + helper_->ComputeFrameInformation(true)); + ASSERT_EQ(SurfaceChooserHelper::FrameInformation::SURFACETEXTURE_L3, + helper_->ComputeFrameInformation(false)); +} + +TEST_F(SurfaceChooserHelperTest, FrameInformationIsCorrectForInsecure) { + // Verify insecure cases. + helper_->SetSecureSurfaceMode( + SurfaceChooserHelper::SecureSurfaceMode::kInsecure); + + // Not using an overlay should be SURFACETEXTURE_INSECURE + ASSERT_EQ(SurfaceChooserHelper::FrameInformation::SURFACETEXTURE_INSECURE, + helper_->ComputeFrameInformation(false)); + + // Fullscreen state should affect the result, so that we can tell the + // difference between player-element-fs and div-fs (custom controls). + helper_->SetIsFullscreen(true); + ASSERT_EQ(SurfaceChooserHelper::FrameInformation:: + OVERLAY_INSECURE_PLAYER_ELEMENT_FULLSCREEN, + helper_->ComputeFrameInformation(true)); + helper_->SetIsFullscreen(false); + ASSERT_EQ(SurfaceChooserHelper::FrameInformation:: + OVERLAY_INSECURE_NON_PLAYER_ELEMENT_FULLSCREEN, + helper_->ComputeFrameInformation(true)); +} + +} // namespace media diff --git a/chromium/media/gpu/android/surface_texture_gl_owner_unittest.cc b/chromium/media/gpu/android/surface_texture_gl_owner_unittest.cc index b29941a2dee..1d395cb78a8 100644 --- a/chromium/media/gpu/android/surface_texture_gl_owner_unittest.cc +++ b/chromium/media/gpu/android/surface_texture_gl_owner_unittest.cc @@ -57,7 +57,7 @@ class SurfaceTextureGLOwnerTest : public testing::Test { context_ = nullptr; share_group_ = nullptr; surface_ = nullptr; - gl::init::ShutdownGL(); + gl::init::ShutdownGL(false); } scoped_refptr<SurfaceTextureGLOwner> surface_texture_; diff --git a/chromium/media/gpu/android/video_frame_factory.h b/chromium/media/gpu/android/video_frame_factory.h index 09075dc44d9..6a84042ccf9 100644 --- a/chromium/media/gpu/android/video_frame_factory.h +++ b/chromium/media/gpu/android/video_frame_factory.h @@ -21,6 +21,7 @@ struct SyncToken; namespace media { +struct AVDASurfaceBundle; class CodecOutputBuffer; class SurfaceTextureGLOwner; class VideoFrame; @@ -42,15 +43,21 @@ class MEDIA_GPU_EXPORT VideoFrameFactory { // Initializes the factory and runs |init_cb| on the current thread when it's // complete. If initialization fails, the returned surface texture will be - // null. - virtual void Initialize(InitCb init_cb) = 0; + // null. |wants_promotion_hint| tells us whether to mark VideoFrames for + // compositor overlay promotion hints or not. + virtual void Initialize(bool wants_promotion_hint, InitCb init_cb) = 0; + + // Notify us about the current surface bundle that subsequent video frames + // should use. + virtual void SetSurfaceBundle( + scoped_refptr<AVDASurfaceBundle> surface_bundle) = 0; // Creates a new VideoFrame backed by |output_buffer| and |surface_texture|. // |surface_texture| may be null if the buffer is backed by an overlay // instead. Runs |output_cb| on the calling sequence to return the frame. + // TODO(liberato): update the comment. virtual void CreateVideoFrame( std::unique_ptr<CodecOutputBuffer> output_buffer, - scoped_refptr<SurfaceTextureGLOwner> surface_texture, base::TimeDelta timestamp, gfx::Size natural_size, PromotionHintAggregator::NotifyPromotionHintCB promotion_hint_cb, diff --git a/chromium/media/gpu/android/video_frame_factory_impl.cc b/chromium/media/gpu/android/video_frame_factory_impl.cc index ec84d6b2fb2..d0019319412 100644 --- a/chromium/media/gpu/android/video_frame_factory_impl.cc +++ b/chromium/media/gpu/android/video_frame_factory_impl.cc @@ -20,6 +20,7 @@ #include "media/base/scoped_callback_runner.h" #include "media/base/video_frame.h" #include "media/gpu//android/codec_image.h" +#include "media/gpu//android/codec_image_group.h" #include "media/gpu/android/codec_wrapper.h" #include "ui/gl/android/surface_texture.h" #include "ui/gl/gl_bindings.h" @@ -45,7 +46,8 @@ VideoFrameFactoryImpl::~VideoFrameFactoryImpl() { gpu_task_runner_->DeleteSoon(FROM_HERE, gpu_video_frame_factory_.release()); } -void VideoFrameFactoryImpl::Initialize(InitCb init_cb) { +void VideoFrameFactoryImpl::Initialize(bool wants_promotion_hint, + InitCb init_cb) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DCHECK(!gpu_video_frame_factory_); gpu_video_frame_factory_ = base::MakeUnique<GpuVideoFrameFactory>(); @@ -53,13 +55,42 @@ void VideoFrameFactoryImpl::Initialize(InitCb init_cb) { gpu_task_runner_.get(), FROM_HERE, base::Bind(&GpuVideoFrameFactory::Initialize, base::Unretained(gpu_video_frame_factory_.get()), - get_stub_cb_), + wants_promotion_hint, get_stub_cb_), std::move(init_cb)); } +void VideoFrameFactoryImpl::SetSurfaceBundle( + scoped_refptr<AVDASurfaceBundle> surface_bundle) { + scoped_refptr<CodecImageGroup> image_group; + if (!surface_bundle) { + // Clear everything, just so we're not holding a reference. + surface_texture_ = nullptr; + } else { + // If |surface_bundle| is using a SurfaceTexture, then get it. + surface_texture_ = + surface_bundle->overlay ? nullptr : surface_bundle->surface_texture; + + // Start a new image group. Note that there's no reason that we can't have + // more than one group per surface bundle; it's okay if we're called + // mulitiple times with the same surface bundle. It just helps to combine + // the callbacks if we don't, especially since AndroidOverlay doesn't know + // how to remove destruction callbacks. That's one reason why we don't just + // make the CodecImage register itself. The other is that the threading is + // easier if we do it this way, since the image group is constructed on the + // proper thread to talk to the overlay. + image_group = + base::MakeRefCounted<CodecImageGroup>(gpu_task_runner_, surface_bundle); + } + + gpu_task_runner_->PostTask( + FROM_HERE, + base::BindOnce(&GpuVideoFrameFactory::SetImageGroup, + base::Unretained(gpu_video_frame_factory_.get()), + std::move(image_group))); +} + void VideoFrameFactoryImpl::CreateVideoFrame( std::unique_ptr<CodecOutputBuffer> output_buffer, - scoped_refptr<SurfaceTextureGLOwner> surface_texture, base::TimeDelta timestamp, gfx::Size natural_size, PromotionHintAggregator::NotifyPromotionHintCB promotion_hint_cb, @@ -69,7 +100,7 @@ void VideoFrameFactoryImpl::CreateVideoFrame( FROM_HERE, base::Bind(&GpuVideoFrameFactory::CreateVideoFrame, base::Unretained(gpu_video_frame_factory_.get()), - base::Passed(&output_buffer), surface_texture, timestamp, + base::Passed(&output_buffer), surface_texture_, timestamp, natural_size, std::move(promotion_hint_cb), std::move(output_cb), base::ThreadTaskRunnerHandle::Get())); } @@ -94,8 +125,10 @@ GpuVideoFrameFactory::~GpuVideoFrameFactory() { } scoped_refptr<SurfaceTextureGLOwner> GpuVideoFrameFactory::Initialize( + bool wants_promotion_hint, VideoFrameFactoryImpl::GetStubCb get_stub_cb) { DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + wants_promotion_hint_ = wants_promotion_hint; stub_ = get_stub_cb.Run(); if (!MakeContextCurrent(stub_)) return nullptr; @@ -182,11 +215,13 @@ void GpuVideoFrameFactory::CreateVideoFrameInternal( size.width(), size.height(), GL_RGBA, GL_UNSIGNED_BYTE); auto image = base::MakeRefCounted<CodecImage>( - std::move(output_buffer), surface_texture, std::move(promotion_hint_cb), - base::Bind(&GpuVideoFrameFactory::OnImageDestructed, - weak_factory_.GetWeakPtr())); + std::move(output_buffer), surface_texture, std::move(promotion_hint_cb)); images_.push_back(image.get()); + // Add |image| to our current image group. This makes suer that any overlay + // lasts as long as the images. For SurfaceTexture, it doesn't do much. + image_group_->AddCodecImage(image.get()); + // Attach the image to the texture. // If we're attaching a SurfaceTexture backed image, we set the state to // UNBOUND. This ensures that the implementation will call CopyTexImage() @@ -218,8 +253,17 @@ void GpuVideoFrameFactory::CreateVideoFrameInternal( if (stub_->GetGpuPreferences().enable_threaded_texture_mailboxes) frame->metadata()->SetBoolean(VideoFrameMetadata::COPY_REQUIRED, true); + // We unconditionally mark the picture as overlayable, even if + // |!surface_texture|, if we want to get hints. It's required, else we won't + // get hints. + const bool allow_overlay = !surface_texture || wants_promotion_hint_; + frame->metadata()->SetBoolean(VideoFrameMetadata::ALLOW_OVERLAY, - !surface_texture); + allow_overlay); + frame->metadata()->SetBoolean(VideoFrameMetadata::WANTS_PROMOTION_HINT, + wants_promotion_hint_); + frame->metadata()->SetBoolean(VideoFrameMetadata::SURFACE_TEXTURE, + !!surface_texture); *video_frame_out = std::move(frame); *texture_ref_out = std::move(texture_ref); @@ -264,4 +308,15 @@ void GpuVideoFrameFactory::OnImageDestructed(CodecImage* image) { internal::MaybeRenderEarly(&images_); } +void GpuVideoFrameFactory::SetImageGroup( + scoped_refptr<CodecImageGroup> image_group) { + image_group_ = std::move(image_group); + + if (!image_group_) + return; + + image_group_->SetDestructionCb(base::BindRepeating( + &GpuVideoFrameFactory::OnImageDestructed, weak_factory_.GetWeakPtr())); +} + } // namespace media diff --git a/chromium/media/gpu/android/video_frame_factory_impl.h b/chromium/media/gpu/android/video_frame_factory_impl.h index 7042166ae76..f072b5a2513 100644 --- a/chromium/media/gpu/android/video_frame_factory_impl.h +++ b/chromium/media/gpu/android/video_frame_factory_impl.h @@ -19,6 +19,7 @@ #include "ui/gl/gl_bindings.h" namespace media { +class CodecImageGroup; class GpuVideoFrameFactory; // VideoFrameFactoryImpl creates CodecOutputBuffer backed VideoFrames and tries @@ -34,10 +35,11 @@ class MEDIA_GPU_EXPORT VideoFrameFactoryImpl : public VideoFrameFactory { GetStubCb get_stub_cb); ~VideoFrameFactoryImpl() override; - void Initialize(InitCb init_cb) override; + void Initialize(bool wants_promotion_hint, InitCb init_cb) override; + void SetSurfaceBundle( + scoped_refptr<AVDASurfaceBundle> surface_bundle) override; void CreateVideoFrame( std::unique_ptr<CodecOutputBuffer> output_buffer, - scoped_refptr<SurfaceTextureGLOwner> surface_texture, base::TimeDelta timestamp, gfx::Size natural_size, PromotionHintAggregator::NotifyPromotionHintCB promotion_hint_cb, @@ -50,6 +52,9 @@ class MEDIA_GPU_EXPORT VideoFrameFactoryImpl : public VideoFrameFactory { scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner_; GetStubCb get_stub_cb_; + // The surface texture that video frames should use, or nullptr. + scoped_refptr<SurfaceTextureGLOwner> surface_texture_; + SEQUENCE_CHECKER(sequence_checker_); DISALLOW_COPY_AND_ASSIGN(VideoFrameFactoryImpl); }; @@ -63,6 +68,7 @@ class GpuVideoFrameFactory ~GpuVideoFrameFactory() override; scoped_refptr<SurfaceTextureGLOwner> Initialize( + bool wants_promotion_hint, VideoFrameFactory::GetStubCb get_stub_cb); // Creates and returns a VideoFrame with its ReleaseMailboxCB. @@ -75,6 +81,10 @@ class GpuVideoFrameFactory VideoFrameFactory::OutputWithReleaseMailboxCB output_cb, scoped_refptr<base::SingleThreadTaskRunner> task_runner); + // Set our image group. Must be called before the first call to + // CreateVideoFrame occurs. + void SetImageGroup(scoped_refptr<CodecImageGroup> image_group); + private: // Creates a TextureRef and VideoFrame. void CreateVideoFrameInternal( @@ -109,8 +119,19 @@ class GpuVideoFrameFactory texture_refs_; gpu::GpuCommandBufferStub* stub_; + // Callback to notify us that an image has been destroyed. + CodecImage::DestructionCb destruction_cb_; + + // Do we want promotion hints from the compositor? + bool wants_promotion_hint_ = false; + // A helper for creating textures. Only valid while |stub_| is valid. std::unique_ptr<GLES2DecoderHelper> decoder_helper_; + + // Current image group to which new images (frames) will be added. We'll + // replace this when SetImageGroup() is called. + scoped_refptr<CodecImageGroup> image_group_; + THREAD_CHECKER(thread_checker_); base::WeakPtrFactory<GpuVideoFrameFactory> weak_factory_; DISALLOW_COPY_AND_ASSIGN(GpuVideoFrameFactory); |