diff options
| author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-10-06 12:48:11 +0200 |
|---|---|---|
| committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-10-13 09:33:43 +0000 |
| commit | 7b5b123ac58f58ffde0f4f6e488bcd09aa4decd3 (patch) | |
| tree | fa14ba0ca8d2683ba2efdabd246dc9b18a1229c6 /chromium/media | |
| parent | 79b4f909db1049fca459c07cca55af56a9b54fe3 (diff) | |
| download | qtwebengine-chromium-7b5b123ac58f58ffde0f4f6e488bcd09aa4decd3.tar.gz | |
BASELINE: Update Chromium to 84.0.4147.141
Change-Id: Ib85eb4cfa1cbe2b2b81e5022c8cad5c493969535
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>
Diffstat (limited to 'chromium/media')
572 files changed, 11593 insertions, 9118 deletions
diff --git a/chromium/media/BUILD.gn b/chromium/media/BUILD.gn index 8ba5ec2ca12..13ec8a79c74 100644 --- a/chromium/media/BUILD.gn +++ b/chromium/media/BUILD.gn @@ -22,7 +22,6 @@ buildflag_header("media_buildflags") { "CDM_PLATFORM_SPECIFIC_PATH=\"$cdm_platform_specific_path\"", "ENABLE_PLATFORM_AC3_EAC3_AUDIO=$enable_platform_ac3_eac3_audio", "ENABLE_CDM_HOST_VERIFICATION=$enable_cdm_host_verification", - "ENABLE_CDM_PROXY=$enable_cdm_proxy", "ENABLE_CDM_STORAGE_ID=$enable_cdm_storage_id", "ENABLE_DAV1D_DECODER=$enable_dav1d_decoder", "ENABLE_AV1_DECODER=$enable_av1_decoder", @@ -158,6 +157,7 @@ test("media_unittests") { "//media/test:pipeline_integration_tests", "//media/test:run_all_unittests", "//media/video:unit_tests", + "//media/webcodecs:unit_tests", "//media/webrtc:unit_tests", ] @@ -182,7 +182,10 @@ test("media_unittests") { } if (is_fuchsia) { - deps += [ "//media/fuchsia/audio:unittests" ] + deps += [ + "//media/fuchsia/audio:unittests", + "//media/fuchsia/metrics:unittests", + ] } if (enable_media_remoting) { diff --git a/chromium/media/README.md b/chromium/media/README.md index 01b21adc2e9..71f8fb1d91a 100644 --- a/chromium/media/README.md +++ b/chromium/media/README.md @@ -98,6 +98,11 @@ components required for HTML media elements and extensions: * [Media Source Extensions](https://www.w3.org/TR/media-source/) * [Encrypted Media Extensions](https://www.w3.org/TR/encrypted-media/) +The following diagram provides a simplified overview of the media playback +pipeline. + + + As a case study we'll consider the playback of a video through the `<video>` tag. `<video>` (and `<audio>`) starts in `blink::HTMLMediaElement` in diff --git a/chromium/media/audio/android/opensles_util.cc b/chromium/media/audio/android/opensles_util.cc index cf5ea3b98d8..128a3bd20d2 100644 --- a/chromium/media/audio/android/opensles_util.cc +++ b/chromium/media/audio/android/opensles_util.cc @@ -4,6 +4,8 @@ #include "media/audio/android/opensles_util.h" +#include "base/logging.h" + namespace media { #define SL_ANDROID_SPEAKER_QUAD \ diff --git a/chromium/media/audio/audio_device_description.cc b/chromium/media/audio/audio_device_description.cc index c72b61478a7..bd3f61100e8 100644 --- a/chromium/media/audio/audio_device_description.cc +++ b/chromium/media/audio/audio_device_description.cc @@ -7,7 +7,7 @@ #include <utility> #include "base/bind.h" -#include "base/logging.h" +#include "base/notreached.h" #include "build/chromecast_buildflags.h" #include "media/base/localized_strings.h" diff --git a/chromium/media/audio/audio_device_thread.cc b/chromium/media/audio/audio_device_thread.cc index 2acfb8ad6e9..b813314b867 100644 --- a/chromium/media/audio/audio_device_thread.cc +++ b/chromium/media/audio/audio_device_thread.cc @@ -6,7 +6,7 @@ #include <limits> -#include "base/logging.h" +#include "base/check_op.h" #include "base/system/sys_info.h" namespace media { diff --git a/chromium/media/audio/audio_input_device.cc b/chromium/media/audio/audio_input_device.cc index 26367015351..aad6984307d 100644 --- a/chromium/media/audio/audio_input_device.cc +++ b/chromium/media/audio/audio_input_device.cc @@ -81,7 +81,6 @@ class AudioInputDevice::AudioThreadCallback base::ReadOnlySharedMemoryRegion shared_memory_region_; base::ReadOnlySharedMemoryMapping shared_memory_mapping_; const base::TimeTicks start_time_; - bool no_callbacks_received_; size_t current_segment_id_; uint32_t last_buffer_id_; std::vector<std::unique_ptr<const media::AudioBus>> audio_buses_; @@ -357,7 +356,6 @@ AudioInputDevice::AudioThreadCallback::AudioThreadCallback( enable_uma_(enable_uma), shared_memory_region_(std::move(shared_memory_region)), start_time_(base::TimeTicks::Now()), - no_callbacks_received_(true), current_segment_id_(0u), last_buffer_id_(UINT32_MAX), capture_callback_(capture_callback), @@ -400,15 +398,6 @@ void AudioInputDevice::AudioThreadCallback::MapSharedMemory() { void AudioInputDevice::AudioThreadCallback::Process(uint32_t pending_data) { TRACE_EVENT_BEGIN0("audio", "AudioInputDevice::AudioThreadCallback::Process"); - - if (no_callbacks_received_) { - if (enable_uma_) { - UMA_HISTOGRAM_TIMES("Media.Audio.Render.InputDeviceStartTime", - base::TimeTicks::Now() - start_time_); - } - no_callbacks_received_ = false; - } - // The shared memory represents parameters, size of the data buffer and the // actual data buffer containing audio data. Map the memory into this // structure and parse out parameters and the data area. diff --git a/chromium/media/audio/audio_output_device.cc b/chromium/media/audio/audio_output_device.cc index 86239ef1bc8..92448e7469f 100644 --- a/chromium/media/audio/audio_output_device.cc +++ b/chromium/media/audio/audio_output_device.cc @@ -402,8 +402,7 @@ void AudioOutputDevice::OnStreamCreated( DCHECK(!audio_callback_); audio_callback_.reset(new AudioOutputDeviceThreadCallback( - audio_parameters_, std::move(shared_memory_region), callback_, - std::make_unique<AudioOutputDeviceThreadCallback::Metrics>())); + audio_parameters_, std::move(shared_memory_region), callback_)); if (playing_automatically) audio_callback_->InitializePlayStartTime(); audio_thread_.reset(new AudioDeviceThread( diff --git a/chromium/media/audio/audio_output_device_thread_callback.cc b/chromium/media/audio/audio_output_device_thread_callback.cc index c19252afbb9..c834b0d77d8 100644 --- a/chromium/media/audio/audio_output_device_thread_callback.cc +++ b/chromium/media/audio/audio_output_device_thread_callback.cc @@ -11,57 +11,23 @@ namespace media { -AudioOutputDeviceThreadCallback::Metrics::Metrics() - : first_play_start_time_(base::nullopt) {} - -AudioOutputDeviceThreadCallback::Metrics::~Metrics() = default; - -void AudioOutputDeviceThreadCallback::Metrics::OnCreated() { - start_time_ = base::TimeTicks::Now(); -} - -void AudioOutputDeviceThreadCallback::Metrics::OnProcess() { - if (first_play_start_time_) { - UMA_HISTOGRAM_TIMES("Media.Audio.Render.OutputDeviceStartTime", - base::TimeTicks::Now() - *first_play_start_time_); - } -} - -void AudioOutputDeviceThreadCallback::Metrics::OnInitializePlayStartTime() { - if (!first_play_start_time_.has_value()) - first_play_start_time_ = base::TimeTicks::Now(); -} - -void AudioOutputDeviceThreadCallback::Metrics::OnDestroyed() { - DCHECK(!start_time_.is_null()); - UMA_HISTOGRAM_LONG_TIMES("Media.Audio.Render.OutputStreamDuration", - base::TimeTicks::Now() - start_time_); -} - AudioOutputDeviceThreadCallback::AudioOutputDeviceThreadCallback( const media::AudioParameters& audio_parameters, base::UnsafeSharedMemoryRegion shared_memory_region, - media::AudioRendererSink::RenderCallback* render_callback, - std::unique_ptr<Metrics> metrics) + media::AudioRendererSink::RenderCallback* render_callback) : media::AudioDeviceThread::Callback( audio_parameters, ComputeAudioOutputBufferSize(audio_parameters), /*segment count*/ 1), shared_memory_region_(std::move(shared_memory_region)), render_callback_(render_callback), - callback_num_(0), - metrics_(std::move(metrics)) { + callback_num_(0) { // CHECK that the shared memory is large enough. The memory allocated must be // at least as large as expected. CHECK(memory_length_ <= shared_memory_region_.GetSize()); - if (metrics_) - metrics_->OnCreated(); } -AudioOutputDeviceThreadCallback::~AudioOutputDeviceThreadCallback() { - if (metrics_) - metrics_->OnDestroyed(); -} +AudioOutputDeviceThreadCallback::~AudioOutputDeviceThreadCallback() = default; void AudioOutputDeviceThreadCallback::MapSharedMemory() { CHECK_EQ(total_segments_, 1u); @@ -103,11 +69,8 @@ void AudioOutputDeviceThreadCallback::Process(uint32_t control_signal) { // When playback starts, we get an immediate callback to Process to make sure // that we have some data, we'll get another one after the device is awake and // ingesting data, which is what we want to track with this trace. - if (callback_num_ == 2) { - if (metrics_) - metrics_->OnProcess(); + if (callback_num_ == 2) TRACE_EVENT_ASYNC_END0("audio", "StartingPlayback", this); - } // Update the audio-delay measurement, inform about the number of skipped // frames, and ask client to render audio. Since |output_bus_| is wrapping @@ -131,9 +94,6 @@ bool AudioOutputDeviceThreadCallback::CurrentThreadIsAudioDeviceThread() { return thread_checker_.CalledOnValidThread(); } -void AudioOutputDeviceThreadCallback::InitializePlayStartTime() { - if (metrics_) - metrics_->OnInitializePlayStartTime(); -} +void AudioOutputDeviceThreadCallback::InitializePlayStartTime() {} } // namespace media diff --git a/chromium/media/audio/audio_output_device_thread_callback.h b/chromium/media/audio/audio_output_device_thread_callback.h index 2a529bf9082..84b80d9af78 100644 --- a/chromium/media/audio/audio_output_device_thread_callback.h +++ b/chromium/media/audio/audio_output_device_thread_callback.h @@ -20,27 +20,10 @@ namespace media { class MEDIA_EXPORT AudioOutputDeviceThreadCallback : public media::AudioDeviceThread::Callback { public: - class Metrics { - public: - Metrics(); - ~Metrics(); - - void OnCreated(); - void OnProcess(); - void OnInitializePlayStartTime(); - void OnDestroyed(); - - private: - base::TimeTicks start_time_; - // If set, this is used to record the startup duration UMA stat. - base::Optional<base::TimeTicks> first_play_start_time_; - }; - AudioOutputDeviceThreadCallback( const media::AudioParameters& audio_parameters, base::UnsafeSharedMemoryRegion shared_memory_region, - media::AudioRendererSink::RenderCallback* render_callback, - std::unique_ptr<Metrics> metrics = nullptr); + media::AudioRendererSink::RenderCallback* render_callback); ~AudioOutputDeviceThreadCallback() override; void MapSharedMemory() override; @@ -64,7 +47,6 @@ class MEDIA_EXPORT AudioOutputDeviceThreadCallback media::AudioRendererSink::RenderCallback* render_callback_; std::unique_ptr<media::AudioBus> output_bus_; uint64_t callback_num_; - std::unique_ptr<Metrics> metrics_; DISALLOW_COPY_AND_ASSIGN(AudioOutputDeviceThreadCallback); }; diff --git a/chromium/media/audio/audio_output_proxy.cc b/chromium/media/audio/audio_output_proxy.cc index ec6657a5f67..d205d67091e 100644 --- a/chromium/media/audio/audio_output_proxy.cc +++ b/chromium/media/audio/audio_output_proxy.cc @@ -4,7 +4,7 @@ #include "media/audio/audio_output_proxy.h" -#include "base/logging.h" +#include "base/check_op.h" #include "media/audio/audio_manager.h" #include "media/audio/audio_output_dispatcher.h" diff --git a/chromium/media/audio/cras/audio_manager_cras.cc b/chromium/media/audio/cras/audio_manager_cras.cc index 376989bfb50..fb4f792d5d0 100644 --- a/chromium/media/audio/cras/audio_manager_cras.cc +++ b/chromium/media/audio/cras/audio_manager_cras.cc @@ -11,9 +11,9 @@ #include <utility> #include "base/bind.h" +#include "base/check_op.h" #include "base/command_line.h" #include "base/environment.h" -#include "base/logging.h" #include "base/metrics/field_trial_params.h" #include "base/nix/xdg_util.h" #include "base/stl_util.h" diff --git a/chromium/media/audio/fake_audio_output_stream.cc b/chromium/media/audio/fake_audio_output_stream.cc index b39e13454d0..04a624e321f 100644 --- a/chromium/media/audio/fake_audio_output_stream.cc +++ b/chromium/media/audio/fake_audio_output_stream.cc @@ -6,7 +6,7 @@ #include "base/bind.h" #include "base/bind_helpers.h" -#include "base/logging.h" +#include "base/check.h" #include "base/single_thread_task_runner.h" #include "base/time/time.h" #include "media/audio/audio_manager_base.h" diff --git a/chromium/media/audio/mock_audio_manager.cc b/chromium/media/audio/mock_audio_manager.cc index 16ab5c719ad..a4056b2edf7 100644 --- a/chromium/media/audio/mock_audio_manager.cc +++ b/chromium/media/audio/mock_audio_manager.cc @@ -8,7 +8,7 @@ #include "base/bind.h" #include "base/callback.h" -#include "base/logging.h" +#include "base/check.h" #include "media/audio/mock_audio_debug_recording_manager.h" #include "media/base/audio_parameters.h" diff --git a/chromium/media/audio/pulse/pulse_input.cc b/chromium/media/audio/pulse/pulse_input.cc index 3f0c9d7a1d4..10bbd2d2af6 100644 --- a/chromium/media/audio/pulse/pulse_input.cc +++ b/chromium/media/audio/pulse/pulse_input.cc @@ -6,7 +6,7 @@ #include <stdint.h> -#include "base/logging.h" +#include "base/check.h" #include "base/strings/stringprintf.h" #include "media/audio/audio_device_description.h" #include "media/audio/pulse/audio_manager_pulse.h" diff --git a/chromium/media/audio/simple_sources_unittest.cc b/chromium/media/audio/simple_sources_unittest.cc index 44009d4b87c..263d639242f 100644 --- a/chromium/media/audio/simple_sources_unittest.cc +++ b/chromium/media/audio/simple_sources_unittest.cc @@ -11,7 +11,6 @@ #include <memory> #include "base/files/file_util.h" -#include "base/logging.h" #include "base/time/time.h" #include "media/audio/audio_io.h" #include "media/audio/test_data.h" diff --git a/chromium/media/audio/wav_audio_handler_unittest.cc b/chromium/media/audio/wav_audio_handler_unittest.cc index 9969c116c43..89521245a61 100644 --- a/chromium/media/audio/wav_audio_handler_unittest.cc +++ b/chromium/media/audio/wav_audio_handler_unittest.cc @@ -9,7 +9,6 @@ #include <memory> #include <string> -#include "base/logging.h" #include "base/stl_util.h" #include "base/strings/string_piece.h" #include "media/audio/test_data.h" diff --git a/chromium/media/audio/win/audio_output_win_unittest.cc b/chromium/media/audio/win/audio_output_win_unittest.cc index f207cf2b0fb..7a4fc8648c4 100644 --- a/chromium/media/audio/win/audio_output_win_unittest.cc +++ b/chromium/media/audio/win/audio_output_win_unittest.cc @@ -472,8 +472,12 @@ TEST_F(WinAudioTest, PCMWaveStreamPendingBytes) { // from a potentially remote thread. class SyncSocketSource : public AudioOutputStream::AudioSourceCallback { public: - SyncSocketSource(base::SyncSocket* socket, const AudioParameters& params) - : socket_(socket), params_(params) { + SyncSocketSource(base::SyncSocket* socket, + const AudioParameters& params, + int expected_packet_count) + : socket_(socket), + params_(params), + expected_packet_count_(expected_packet_count) { // Setup AudioBus wrapping data we'll receive over the sync socket. packet_size_ = AudioBus::CalculateMemorySize(params); data_.reset(static_cast<float*>( @@ -488,18 +492,28 @@ class SyncSocketSource : public AudioOutputStream::AudioSourceCallback { base::TimeTicks delay_timestamp, int /* prior_frames_skipped */, AudioBus* dest) override { - uint32_t control_signal = 0; - socket_->Send(&control_signal, sizeof(control_signal)); - output_buffer()->params.delay_us = delay.InMicroseconds(); - output_buffer()->params.delay_timestamp_us = - (delay_timestamp - base::TimeTicks()).InMicroseconds(); - uint32_t size = socket_->Receive(data_.get(), packet_size_); - - DCHECK_EQ(static_cast<size_t>(size) % sizeof(*audio_bus_->channel(0)), 0U); - audio_bus_->CopyTo(dest); - return audio_bus_->frames(); + // If we ask for more data once the producer has shutdown, we will hang + // on |socket_->Receive()|. + if (current_packet_count_ < expected_packet_count_) { + uint32_t control_signal = 0; + socket_->Send(&control_signal, sizeof(control_signal)); + output_buffer()->params.delay_us = delay.InMicroseconds(); + output_buffer()->params.delay_timestamp_us = + (delay_timestamp - base::TimeTicks()).InMicroseconds(); + uint32_t size = socket_->Receive(data_.get(), packet_size_); + ++current_packet_count_; + + DCHECK_EQ(static_cast<size_t>(size) % sizeof(*audio_bus_->channel(0)), + 0U); + audio_bus_->CopyTo(dest); + return audio_bus_->frames(); + } + + return 0; } + int packet_size() const { return packet_size_; } + AudioOutputBuffer* output_buffer() const { return reinterpret_cast<AudioOutputBuffer*>(data_.get()); } @@ -513,6 +527,11 @@ class SyncSocketSource : public AudioOutputStream::AudioSourceCallback { int packet_size_; std::unique_ptr<float, base::AlignedFreeDeleter> data_; std::unique_ptr<AudioBus> audio_bus_; + + // This test produces a fixed number of packets, we need these so we know + // when to stop listening. + const int expected_packet_count_; + int current_packet_count_ = 0; }; struct SyncThreadContext { @@ -523,6 +542,7 @@ struct SyncThreadContext { double sine_freq; uint32_t packet_size_bytes; AudioOutputBuffer* buffer; + int total_packets; }; // This thread provides the data that the SyncSocketSource above needs @@ -541,10 +561,11 @@ DWORD __stdcall SyncSocketThread(void* context) { AudioBus::WrapMemory(ctx.channels, ctx.frames, data.get()); SineWaveAudioSource sine(1, ctx.sine_freq, ctx.sample_rate); - const int kTwoSecFrames = ctx.sample_rate * 2; uint32_t control_signal = 0; - for (int ix = 0; ix < kTwoSecFrames; ix += ctx.frames) { + for (int ix = 0; ix < ctx.total_packets; ++ix) { + // Listen for a signal from the Audio Stream that it wants data. This is a + // blocking call and will not proceed until we receive the signal. if (ctx.socket->Receive(&control_signal, sizeof(control_signal)) == 0) break; base::TimeDelta delay = @@ -553,6 +574,8 @@ DWORD __stdcall SyncSocketThread(void* context) { base::TimeTicks() + base::TimeDelta::FromMicroseconds( ctx.buffer->params.delay_timestamp_us); sine.OnMoreData(delay, delay_timestamp, 0, audio_bus.get()); + + // Send the audio data to the Audio Stream. ctx.socket->Send(data.get(), ctx.packet_size_bytes); } @@ -572,6 +595,9 @@ TEST_F(WinAudioTest, SyncSocketBasic) { static const int sample_rate = AudioParameters::kAudioCDSampleRate; static const uint32_t kSamples20ms = sample_rate / 50; + // We want 2 seconds of audio, which means we need 100 packets as each packet + // contains 20ms worth of audio samples. + static const int kPackets2s = 100; AudioParameters params(AudioParameters::AUDIO_PCM_LINEAR, CHANNEL_LAYOUT_MONO, sample_rate, kSamples20ms); @@ -581,11 +607,20 @@ TEST_F(WinAudioTest, SyncSocketBasic) { ASSERT_TRUE(oas->Open()); + // Create two sockets and connect them with a named pipe. base::SyncSocket sockets[2]; ASSERT_TRUE(base::SyncSocket::CreatePair(&sockets[0], &sockets[1])); - SyncSocketSource source(&sockets[0], params); + // Give one socket to the source, which receives requests from the + // AudioOutputStream for more data. On such a request, it will send a control + // signal to the SyncThreadContext, then it will receive + // an audio packet back which it will give to the AudioOutputStream. + SyncSocketSource source(&sockets[0], params, kPackets2s); + // Give the other socket to the thread. This thread runs a loop that will + // generate enough audio packets for 2 seconds worth of audio. It will listen + // for a control signal and when it gets one it will write one audio packet + // to the pipe that connects the two sockets. SyncThreadContext thread_context; thread_context.sample_rate = params.sample_rate(); thread_context.sine_freq = 200.0; @@ -594,15 +629,22 @@ TEST_F(WinAudioTest, SyncSocketBasic) { thread_context.channels = params.channels(); thread_context.socket = &sockets[1]; thread_context.buffer = source.output_buffer(); + thread_context.total_packets = kPackets2s; HANDLE thread = ::CreateThread(NULL, 0, SyncSocketThread, &thread_context, 0, NULL); + // Start the AudioOutputStream, which will request data via + // SyncSocketSource::OnMoreData until the SyncThreadContext has run out of + // data to give. oas->Start(&source); + // Wait for the SyncThreadContext to finish its loop, should take 2 seconds. + // During this time it is providing audio data as described above. ::WaitForSingleObject(thread, INFINITE); ::CloseHandle(thread); + // Once no more data is being sent, we can stop and close the stream. oas->Stop(); oas->Close(); } diff --git a/chromium/media/audio/win/avrt_wrapper_win.cc b/chromium/media/audio/win/avrt_wrapper_win.cc index 2fb716fcb59..6e3c1f75112 100644 --- a/chromium/media/audio/win/avrt_wrapper_win.cc +++ b/chromium/media/audio/win/avrt_wrapper_win.cc @@ -4,7 +4,7 @@ #include "media/audio/win/avrt_wrapper_win.h" -#include "base/logging.h" +#include "base/check.h" #include "base/stl_util.h" namespace avrt { @@ -23,7 +23,7 @@ bool Initialize() { if (!g_set_mm_thread_priority) { // The avrt.dll is available on Windows Vista and later. wchar_t path[MAX_PATH] = {0}; - ExpandEnvironmentStrings(L"%WINDIR%\\system32\\avrt.dll", path, + ExpandEnvironmentStrings(L"%SystemRoot%\\system32\\avrt.dll", path, base::size(path)); g_avrt = LoadLibraryExW(path, NULL, LOAD_WITH_ALTERED_SEARCH_PATH); if (!g_avrt) diff --git a/chromium/media/base/BUILD.gn b/chromium/media/base/BUILD.gn index 80cc2864762..e5f3cb49ffd 100644 --- a/chromium/media/base/BUILD.gn +++ b/chromium/media/base/BUILD.gn @@ -269,7 +269,6 @@ jumbo_source_set("base") { "simple_watch_timer.h", "sinc_resampler.cc", "sinc_resampler.h", - "speech_recognition_client.h", "status.cc", "status.h", "status_codes.cc", @@ -309,6 +308,8 @@ jumbo_source_set("base") { "video_decoder.h", "video_decoder_config.cc", "video_decoder_config.h", + "video_encoder.cc", + "video_encoder.h", "video_frame.cc", "video_frame.h", "video_frame_layout.cc", @@ -376,10 +377,6 @@ jumbo_source_set("base") { } if (use_x11) { - configs += [ - "//build/config/linux:x11", - "//build/config/linux:xext", - ] sources += [ "user_input_monitor_linux.cc" ] deps += [ "//ui/events:events_base", @@ -425,6 +422,10 @@ jumbo_source_set("base") { sources += [ "demuxer_memory_limit_default.cc" ] } + if (!is_android) { + sources += [ "speech_recognition_client.h" ] + } + if (enable_media_drm_storage) { sources += [ "media_drm_key_type.h", @@ -441,6 +442,7 @@ source_set("video_facing") { if (is_android) { java_cpp_enum("java_enums") { sources = [ + "container_names.h", "encryption_scheme.h", "video_codecs.h", ] diff --git a/chromium/media/base/android/BUILD.gn b/chromium/media/base/android/BUILD.gn index 480089fa540..8b37cff0072 100644 --- a/chromium/media/base/android/BUILD.gn +++ b/chromium/media/base/android/BUILD.gn @@ -160,6 +160,7 @@ if (is_android) { android_library("media_java") { deps = [ + ":display_java", ":media_java_resources", "//base:base_java", "//base:jni_java", @@ -177,6 +178,7 @@ if (is_android) { "java/src/org/chromium/media/BitrateAdjuster.java", "java/src/org/chromium/media/CodecProfileLevelList.java", "java/src/org/chromium/media/HdrMetadata.java", + "java/src/org/chromium/media/MaxAnticipatedResolutionEstimator.java", "java/src/org/chromium/media/MediaCodecBridge.java", "java/src/org/chromium/media/MediaCodecBridgeBuilder.java", "java/src/org/chromium/media/MediaCodecEncoder.java", @@ -188,9 +190,17 @@ if (is_android) { "java/src/org/chromium/media/MediaPlayerBridge.java", "java/src/org/chromium/media/MediaPlayerListener.java", "java/src/org/chromium/media/MediaServerCrashListener.java", + "java/src/org/chromium/media/ScreenResolutionUtil.java", ] } + # TODO (b/146418831): Replace with androidx version + android_library("display_java") { + sources = [ "java/src/org/chromium/media/DisplayCompat.java" ] + + deps = [ "//third_party/android_deps:androidx_annotation_annotation_java" ] + } + junit_binary("media_base_junit_tests") { sources = [ "java/src/test/org/chromium/media/AudioTrackOutputStreamTest.java", diff --git a/chromium/media/base/android/media_codec_util.cc b/chromium/media/base/android/media_codec_util.cc index d9af96da9e8..e70b11484a2 100644 --- a/chromium/media/base/android/media_codec_util.cc +++ b/chromium/media/base/android/media_codec_util.cc @@ -239,19 +239,6 @@ std::set<int> MediaCodecUtil::GetEncoderColorFormats( } // static -bool MediaCodecUtil::IsHLSPath(const GURL& url) { - return (url.SchemeIsHTTPOrHTTPS() || url.SchemeIsFile()) && - base::EndsWith(url.path(), ".m3u8", - base::CompareCase::INSENSITIVE_ASCII); -} - -// static -bool MediaCodecUtil::IsHLSURL(const GURL& url) { - return (url.SchemeIsHTTPOrHTTPS() || url.SchemeIsFile()) && - url.spec().find("m3u8") != std::string::npos; -} - -// static bool MediaCodecUtil::IsVp8DecoderAvailable() { return IsMediaCodecAvailable() && IsDecoderSupportedByDevice(kVp8MimeType); } @@ -303,16 +290,16 @@ bool MediaCodecUtil::IsSurfaceViewOutputSupported() { // Notably this is codec agnostic at present, so any devices added to // the blacklist will avoid trying to play any codecs on SurfaceView. If // needed in the future this can be expanded to be codec specific. - const char* model_prefixes[] = {// Exynos 4 (Mali-400) - "GT-I9300", "GT-I9305", "SHV-E210", - // Snapdragon S4 (Adreno-225) - "SCH-I535", "SCH-J201", "SCH-R530", - "SCH-I960", "SCH-S968", "SGH-T999", - "SGH-I747", "SGH-N064", 0}; + constexpr const char* kDisabledModels[] = {// Exynos 4 (Mali-400) + "GT-I9300", "GT-I9305", "SHV-E210", + // Snapdragon S4 (Adreno-225) + "SCH-I535", "SCH-J201", "SCH-R530", + "SCH-I960", "SCH-S968", "SGH-T999", + "SGH-I747", "SGH-N064"}; std::string model(base::android::BuildInfo::GetInstance()->model()); - for (int i = 0; model_prefixes[i]; ++i) { - if (base::StartsWith(model, model_prefixes[i], + for (auto* disabled_model : kDisabledModels) { + if (base::StartsWith(model, disabled_model, base::CompareCase::INSENSITIVE_ASCII)) { return false; } @@ -343,8 +330,25 @@ bool MediaCodecUtil::CanDecode(AudioCodec codec) { } // static -bool MediaCodecUtil::IsH264EncoderAvailable() { - return IsMediaCodecAvailable() && IsEncoderSupportedByDevice(kAvcMimeType); +bool MediaCodecUtil::IsH264EncoderAvailable(bool use_codec_list) { + if (!IsMediaCodecAvailable()) + return false; + + constexpr const char* kDisabledModels[] = {"SAMSUNG-SGH-I337", "Nexus 7", + "Nexus 4"}; + const std::string model(base::android::BuildInfo::GetInstance()->model()); + for (auto* disabled_model : kDisabledModels) { + if (base::StartsWith(model, disabled_model, + base::CompareCase::INSENSITIVE_ASCII)) { + return false; + } + } + + if (use_codec_list) + return IsEncoderSupportedByDevice(kAvcMimeType); + + // Assume support since Chrome only supports Lollipop+. + return true; } // static diff --git a/chromium/media/base/android/media_codec_util.h b/chromium/media/base/android/media_codec_util.h index e077453f637..5fd273de2ed 100644 --- a/chromium/media/base/android/media_codec_util.h +++ b/chromium/media/base/android/media_codec_util.h @@ -17,8 +17,6 @@ #include "media/base/media_export.h" #include "media/base/video_codecs.h" -class GURL; - namespace media { class MediaCodecBridge; @@ -51,12 +49,6 @@ class MEDIA_EXPORT MediaCodecUtil { // Returns true if MediaCodec supports CBCS Encryption. static bool PlatformSupportsCbcsEncryption(int sdk); - // Test whether a URL contains "m3u8". - static bool IsHLSURL(const GURL& url); - - // Test whether the path of a URL ends with ".m3u8". - static bool IsHLSPath(const GURL& url); - // Indicates if the vp8 decoder or encoder is available on this device. static bool IsVp8DecoderAvailable(); static bool IsVp8EncoderAvailable(); @@ -103,9 +95,10 @@ class MEDIA_EXPORT MediaCodecUtil { // Indicates if the h264 encoder is available on this device. // - // WARNING: This can't be used from the renderer process since it attempts to - // access MediaCodecList (which requires permissions). - static bool IsH264EncoderAvailable(); + // WARNING: If |use_codec_list| is true, this can't be used from the renderer + // process since it attempts to access MediaCodecList (which requires + // permissions). + static bool IsH264EncoderAvailable(bool use_codec_list = true); // Returns a vector of supported codecs profiles and levels. // diff --git a/chromium/media/base/android/media_drm_bridge_client.cc b/chromium/media/base/android/media_drm_bridge_client.cc index 8b4b35b59b8..cad887c4e14 100644 --- a/chromium/media/base/android/media_drm_bridge_client.cc +++ b/chromium/media/base/android/media_drm_bridge_client.cc @@ -4,7 +4,7 @@ #include "media/base/android/media_drm_bridge_client.h" -#include "base/logging.h" +#include "base/check.h" #include "base/stl_util.h" namespace media { diff --git a/chromium/media/base/android/media_drm_bridge_delegate.cc b/chromium/media/base/android/media_drm_bridge_delegate.cc index 5630d4af0da..8bd43ca1120 100644 --- a/chromium/media/base/android/media_drm_bridge_delegate.cc +++ b/chromium/media/base/android/media_drm_bridge_delegate.cc @@ -4,7 +4,7 @@ #include "media/base/android/media_drm_bridge_delegate.h" -#include "base/logging.h" +#include "base/check.h" namespace media { diff --git a/chromium/media/base/android/media_player_bridge.cc b/chromium/media/base/android/media_player_bridge.cc index c6e7846be91..c0a10cef065 100644 --- a/chromium/media/base/android/media_player_bridge.cc +++ b/chromium/media/base/android/media_player_bridge.cc @@ -11,8 +11,9 @@ #include "base/android/jni_string.h" #include "base/android/scoped_java_ref.h" #include "base/bind.h" -#include "base/logging.h" +#include "base/check_op.h" #include "base/metrics/histogram_macros.h" +#include "base/notreached.h" #include "base/numerics/ranges.h" #include "base/strings/string_util.h" #include "base/threading/thread_task_runner_handle.h" diff --git a/chromium/media/base/android/media_player_listener.cc b/chromium/media/base/android/media_player_listener.cc index 11a5dc87f30..5b7f206dceb 100644 --- a/chromium/media/base/android/media_player_listener.cc +++ b/chromium/media/base/android/media_player_listener.cc @@ -7,8 +7,8 @@ #include "base/android/jni_android.h" #include "base/android/scoped_java_ref.h" #include "base/bind.h" +#include "base/check.h" #include "base/location.h" -#include "base/logging.h" #include "base/single_thread_task_runner.h" #include "media/base/android/media_jni_headers/MediaPlayerListener_jni.h" #include "media/base/android/media_player_bridge.h" diff --git a/chromium/media/base/android/stream_texture_wrapper.h b/chromium/media/base/android/stream_texture_wrapper.h index 69a388e5cef..4900adf3805 100644 --- a/chromium/media/base/android/stream_texture_wrapper.h +++ b/chromium/media/base/android/stream_texture_wrapper.h @@ -23,7 +23,6 @@ class MEDIA_EXPORT StreamTextureWrapper { // See StreamTextureWrapperImpl. virtual void Initialize( const base::RepeatingClosure& received_frame_cb, - const gfx::Size& natural_size, scoped_refptr<base::SingleThreadTaskRunner> compositor_task_runner, StreamTextureWrapperInitCB init_cb) = 0; diff --git a/chromium/media/base/audio_block_fifo.cc b/chromium/media/base/audio_block_fifo.cc index fe33ce5ef22..85fa6d27b93 100644 --- a/chromium/media/base/audio_block_fifo.cc +++ b/chromium/media/base/audio_block_fifo.cc @@ -8,7 +8,7 @@ #include "media/base/audio_block_fifo.h" -#include "base/logging.h" +#include "base/check_op.h" #include "base/trace_event/trace_event.h" namespace media { diff --git a/chromium/media/base/audio_buffer_converter.cc b/chromium/media/base/audio_buffer_converter.cc index 17779fc6f92..3e5c0fa114f 100644 --- a/chromium/media/base/audio_buffer_converter.cc +++ b/chromium/media/base/audio_buffer_converter.cc @@ -7,7 +7,7 @@ #include <algorithm> #include <cmath> -#include "base/logging.h" +#include "base/check_op.h" #include "media/base/audio_bus.h" #include "media/base/audio_decoder_config.h" #include "media/base/audio_timestamp_helper.h" diff --git a/chromium/media/base/audio_buffer_queue.cc b/chromium/media/base/audio_buffer_queue.cc index d77946578d1..ca5be72da5c 100644 --- a/chromium/media/base/audio_buffer_queue.cc +++ b/chromium/media/base/audio_buffer_queue.cc @@ -6,7 +6,7 @@ #include <algorithm> -#include "base/logging.h" +#include "base/check_op.h" #include "media/base/audio_bus.h" namespace media { diff --git a/chromium/media/base/audio_buffer_queue_unittest.cc b/chromium/media/base/audio_buffer_queue_unittest.cc index 8fbeb98de08..3cf5e120c56 100644 --- a/chromium/media/base/audio_buffer_queue_unittest.cc +++ b/chromium/media/base/audio_buffer_queue_unittest.cc @@ -9,7 +9,6 @@ #include <limits> #include <memory> -#include "base/logging.h" #include "base/time/time.h" #include "media/base/audio_buffer.h" #include "media/base/audio_bus.h" diff --git a/chromium/media/base/audio_bus.cc b/chromium/media/base/audio_bus.cc index 991e73e11e0..694b53e8c24 100644 --- a/chromium/media/base/audio_bus.cc +++ b/chromium/media/base/audio_bus.cc @@ -7,11 +7,13 @@ #include <stddef.h> #include <stdint.h> +#include <cstring> #include <limits> #include <utility> -#include "base/logging.h" +#include "base/check_op.h" #include "base/memory/ptr_util.h" +#include "base/notreached.h" #include "base/numerics/safe_conversions.h" #include "media/base/audio_parameters.h" #include "media/base/limits.h" diff --git a/chromium/media/base/audio_codecs.cc b/chromium/media/base/audio_codecs.cc index ba5dbc66d12..06a9e73c295 100644 --- a/chromium/media/base/audio_codecs.cc +++ b/chromium/media/base/audio_codecs.cc @@ -4,7 +4,6 @@ #include "media/base/audio_codecs.h" -#include "base/logging.h" #include "base/strings/string_util.h" namespace media { diff --git a/chromium/media/base/audio_fifo.cc b/chromium/media/base/audio_fifo.cc index 507edd133f9..75408b335fc 100644 --- a/chromium/media/base/audio_fifo.cc +++ b/chromium/media/base/audio_fifo.cc @@ -4,7 +4,9 @@ #include "media/base/audio_fifo.h" -#include "base/logging.h" +#include <cstring> + +#include "base/check_op.h" namespace media { diff --git a/chromium/media/base/audio_hash_unittest.cc b/chromium/media/base/audio_hash_unittest.cc index 8e04bf0d5c2..03383cfb3f8 100644 --- a/chromium/media/base/audio_hash_unittest.cc +++ b/chromium/media/base/audio_hash_unittest.cc @@ -4,7 +4,6 @@ #include <memory> -#include "base/logging.h" #include "base/macros.h" #include "media/base/audio_bus.h" #include "media/base/audio_hash.h" diff --git a/chromium/media/base/audio_latency_unittest.cc b/chromium/media/base/audio_latency_unittest.cc index b4c3d9496f1..dcf378defbe 100644 --- a/chromium/media/base/audio_latency_unittest.cc +++ b/chromium/media/base/audio_latency_unittest.cc @@ -6,7 +6,6 @@ #include <stdint.h> -#include "base/logging.h" #include "base/time/time.h" #include "build/build_config.h" #include "media/base/limits.h" diff --git a/chromium/media/base/audio_parameters.cc b/chromium/media/base/audio_parameters.cc index 8bf16c27f67..b3b07180107 100644 --- a/chromium/media/base/audio_parameters.cc +++ b/chromium/media/base/audio_parameters.cc @@ -4,7 +4,8 @@ #include "media/base/audio_parameters.h" -#include "base/logging.h" +#include <sstream> + #include "media/base/limits.h" namespace media { diff --git a/chromium/media/base/audio_power_monitor.cc b/chromium/media/base/audio_power_monitor.cc index 70d9afd4172..3b3c6b5556c 100644 --- a/chromium/media/base/audio_power_monitor.cc +++ b/chromium/media/base/audio_power_monitor.cc @@ -7,7 +7,7 @@ #include <algorithm> #include <cmath> -#include "base/logging.h" +#include "base/check_op.h" #include "base/numerics/ranges.h" #include "base/time/time.h" #include "media/base/audio_bus.h" diff --git a/chromium/media/base/audio_pull_fifo.cc b/chromium/media/base/audio_pull_fifo.cc index 65a637d11e4..ed9e9efcc17 100644 --- a/chromium/media/base/audio_pull_fifo.cc +++ b/chromium/media/base/audio_pull_fifo.cc @@ -6,7 +6,7 @@ #include <algorithm> -#include "base/logging.h" +#include "base/check_op.h" #include "media/base/audio_bus.h" namespace media { diff --git a/chromium/media/base/audio_push_fifo.cc b/chromium/media/base/audio_push_fifo.cc index 5922b8a5844..afb2b0fbeba 100644 --- a/chromium/media/base/audio_push_fifo.cc +++ b/chromium/media/base/audio_push_fifo.cc @@ -7,6 +7,7 @@ #include <algorithm> #include "base/logging.h" +#include "base/trace_event/trace_event.h" namespace media { @@ -27,6 +28,8 @@ void AudioPushFifo::Reset(int frames_per_buffer) { } void AudioPushFifo::Push(const AudioBus& input_bus) { + TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("audio"), "AudioPushFifo::Push"); + DCHECK_GT(frames_per_buffer_, 0); // Fast path: No buffering required. diff --git a/chromium/media/base/audio_renderer_mixer.cc b/chromium/media/base/audio_renderer_mixer.cc index d8a61da696c..d3c1bd8c7db 100644 --- a/chromium/media/base/audio_renderer_mixer.cc +++ b/chromium/media/base/audio_renderer_mixer.cc @@ -8,13 +8,17 @@ #include "base/bind.h" #include "base/bind_helpers.h" -#include "base/logging.h" +#include "base/check_op.h" #include "base/memory/ptr_util.h" #include "base/metrics/histogram_macros.h" +#include "base/task/task_traits.h" +#include "base/task/thread_pool.h" #include "base/time/time.h" #include "base/trace_event/trace_event.h" #include "media/base/audio_renderer_mixer_input.h" #include "media/base/audio_timestamp_helper.h" +#include "media/base/media_switches.h" +#include "media/base/silent_sink_suspender.h" namespace media { @@ -64,13 +68,28 @@ AudioRendererMixer::AudioRendererMixer(const AudioParameters& output_params, playing_(true), input_count_tracker_(new UMAMaxValueTracker(std::move(log_callback))) { DCHECK(audio_sink_); - audio_sink_->Initialize(output_params, this); + + // If enabled we will disable the real audio output stream for muted/silent + // playbacks after some time elapses. + RenderCallback* callback = this; + if (base::FeatureList::IsEnabled(media::kSuspendMutedAudio)) { + // We use slightly more than |pause_delay_| time before suspending the sink + // to ensure that we just Pause() entirely instead of using a fake sink when + // possible. + muted_suspender_.reset(new SilentSinkSuspender( + this, pause_delay_ + base::TimeDelta::FromMilliseconds(500), + output_params, audio_sink_, GetSuspenderTaskRunner())); + callback = muted_suspender_.get(); + } + + audio_sink_->Initialize(output_params, callback); audio_sink_->Start(); } AudioRendererMixer::~AudioRendererMixer() { // AudioRendererSink must be stopped before mixer is destructed. audio_sink_->Stop(); + muted_suspender_.reset(); // Ensure that all mixer inputs have removed themselves prior to destruction. DCHECK(master_converter_.empty()); @@ -167,6 +186,8 @@ int AudioRendererMixer::Render(base::TimeDelta delay, last_play_time_ = now; } else if (now - last_play_time_ >= pause_delay_ && playing_) { audio_sink_->Pause(); + if (muted_suspender_) + muted_suspender_->OnPaused(); playing_ = false; } @@ -188,4 +209,14 @@ void AudioRendererMixer::OnRenderError() { input->OnRenderError(); } +scoped_refptr<base::SingleThreadTaskRunner> +AudioRendererMixer::GetSuspenderTaskRunner() { + if (!suspender_task_runner_) { + suspender_task_runner_ = base::ThreadPool::CreateSingleThreadTaskRunner( + {base::TaskPriority::USER_VISIBLE, + base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN}); + } + return suspender_task_runner_; +} + } // namespace media diff --git a/chromium/media/base/audio_renderer_mixer.h b/chromium/media/base/audio_renderer_mixer.h index 9c9325a2281..b8917dcf532 100644 --- a/chromium/media/base/audio_renderer_mixer.h +++ b/chromium/media/base/audio_renderer_mixer.h @@ -21,8 +21,13 @@ #include "media/base/audio_renderer_sink.h" #include "media/base/loopback_audio_converter.h" +namespace base { +class SingleThreadTaskRunner; +} + namespace media { class AudioRendererMixerInput; +class SilentSinkSuspender; // Mixes a set of AudioConverter::InputCallbacks into a single output stream // which is funneled into a single shared AudioRendererSink; saving a bundle @@ -66,6 +71,8 @@ class MEDIA_EXPORT AudioRendererMixer AudioBus* audio_bus) override; void OnRenderError() override; + scoped_refptr<base::SingleThreadTaskRunner> GetSuspenderTaskRunner(); + bool is_master_sample_rate(int sample_rate) const { return sample_rate == output_params_.sample_rate(); } @@ -76,6 +83,13 @@ class MEDIA_EXPORT AudioRendererMixer // Output sink for this mixer. const scoped_refptr<AudioRendererSink> audio_sink_; + // Optional utility class which disables audio output when only silent audio + // is being delivered to the physical audio output stream. + std::unique_ptr<SilentSinkSuspender> muted_suspender_; + + // Task Runner used by |muted_suspender_|. + scoped_refptr<base::SingleThreadTaskRunner> suspender_task_runner_; + // ---------------[ All variables below protected by |lock_| ]--------------- base::Lock lock_; diff --git a/chromium/media/base/audio_timestamp_helper.cc b/chromium/media/base/audio_timestamp_helper.cc index 0d62c125b75..5a65a240281 100644 --- a/chromium/media/base/audio_timestamp_helper.cc +++ b/chromium/media/base/audio_timestamp_helper.cc @@ -4,7 +4,7 @@ #include "media/base/audio_timestamp_helper.h" -#include "base/logging.h" +#include "base/check_op.h" #include "media/base/timestamp_constants.h" namespace media { diff --git a/chromium/media/base/bit_reader_core.cc b/chromium/media/base/bit_reader_core.cc index cd3e896cc05..715a25b3e36 100644 --- a/chromium/media/base/bit_reader_core.cc +++ b/chromium/media/base/bit_reader_core.cc @@ -5,6 +5,7 @@ #include "media/base/bit_reader_core.h" #include <stdint.h> +#include <cstring> #include "base/sys_byteorder.h" diff --git a/chromium/media/base/buffering_state.cc b/chromium/media/base/buffering_state.cc index 4cf3af511fd..eabc940bf52 100644 --- a/chromium/media/base/buffering_state.cc +++ b/chromium/media/base/buffering_state.cc @@ -3,8 +3,11 @@ // found in the LICENSE file. #include "media/base/buffering_state.h" + #include <string> -#include "base/logging.h" +#include <vector> + +#include "base/check.h" namespace media { diff --git a/chromium/media/base/byte_queue.cc b/chromium/media/base/byte_queue.cc index c54ac79bdfb..ed3d67ebca3 100644 --- a/chromium/media/base/byte_queue.cc +++ b/chromium/media/base/byte_queue.cc @@ -5,8 +5,9 @@ #include "media/base/byte_queue.h" #include <algorithm> +#include <cstring> -#include "base/logging.h" +#include "base/check_op.h" #include "base/numerics/checked_math.h" namespace media { diff --git a/chromium/media/base/cdm_callback_promise.cc b/chromium/media/base/cdm_callback_promise.cc index 103507f72fb..26d6f7aa839 100644 --- a/chromium/media/base/cdm_callback_promise.cc +++ b/chromium/media/base/cdm_callback_promise.cc @@ -5,7 +5,7 @@ #include "media/base/cdm_callback_promise.h" #include "base/callback_helpers.h" -#include "base/logging.h" +#include "base/check.h" namespace media { diff --git a/chromium/media/base/cdm_context.cc b/chromium/media/base/cdm_context.cc index a09fb064c80..dac22fbd70f 100644 --- a/chromium/media/base/cdm_context.cc +++ b/chromium/media/base/cdm_context.cc @@ -29,11 +29,12 @@ bool CdmContext::RequiresMediaFoundationRenderer() { return false; } -#if BUILDFLAG(ENABLE_LIBRARY_CDMS) -CdmProxyContext* CdmContext::GetCdmProxyContext() { - return nullptr; +#if defined(OS_WIN) +bool CdmContext::GetMediaFoundationCdmProxy( + GetMediaFoundationCdmProxyCB get_mf_cdm_proxy_cb) { + return false; } -#endif // BUILDFLAG(ENABLE_LIBRARY_CDMS) +#endif #if defined(OS_ANDROID) MediaCryptoContext* CdmContext::GetMediaCryptoContext() { diff --git a/chromium/media/base/cdm_context.h b/chromium/media/base/cdm_context.h index 53986b9fb03..e346698746e 100644 --- a/chromium/media/base/cdm_context.h +++ b/chromium/media/base/cdm_context.h @@ -11,10 +11,13 @@ #include "media/base/media_export.h" #include "media/media_buildflags.h" +#if defined(OS_WIN) +struct IMFCdmProxy; +#endif + namespace media { class CallbackRegistration; -class CdmProxyContext; class Decryptor; class MediaCryptoContext; @@ -86,12 +89,18 @@ class MEDIA_EXPORT CdmContext { // TODO(crbug.com/804397): Use base::UnguessableToken for CDM ID. virtual int GetCdmId() const; -#if BUILDFLAG(ENABLE_LIBRARY_CDMS) - // Returns a CdmProxyContext that can be used by hardware decoders/decryptors. - // Returns nullptr if CdmProxyContext is not supported, e.g. |this| is not - // hosted by a CdmProxy. - virtual CdmProxyContext* GetCdmProxyContext(); -#endif // BUILDFLAG(ENABLE_LIBRARY_CDMS) +#if defined(OS_WIN) + using GetMediaFoundationCdmProxyCB = base::OnceCallback<void(IMFCdmProxy*)>; + // This allows a CdmContext to expose an IMFTrustedInput instance for use in + // a Media Foundation rendering pipeline. This method is asynchronous because + // the underlying MF-based CDM might not have a native session created yet. + // When the return value is true, the callback might also not be invoked + // if the application has never caused the MF-based CDM to create its + // native session. + // NOTE: the callback should always be fired asynchronously. + virtual bool GetMediaFoundationCdmProxy( + GetMediaFoundationCdmProxyCB get_mf_cdm_proxy_cb); +#endif #if defined(OS_ANDROID) // Returns a MediaCryptoContext that can be used by MediaCodec based decoders. @@ -120,7 +129,7 @@ MEDIA_EXPORT void IgnoreCdmAttached(bool success); // A reference holder to make sure the CdmContext is always valid as long as // |this| is alive. Typically |this| will hold a reference (directly or -// indirectly) to the host, e.g. a ContentDecryptionModule or a CdmProxy. +// indirectly) to the host, e.g. a ContentDecryptionModule. // This class must be held on the same thread where the host lives. The raw // CdmContext pointer returned by GetCdmContext() may be used on other threads // if it's supported by the CdmContext implementation. diff --git a/chromium/media/base/cdm_key_information.cc b/chromium/media/base/cdm_key_information.cc index 66a5290a218..616732cc393 100644 --- a/chromium/media/base/cdm_key_information.cc +++ b/chromium/media/base/cdm_key_information.cc @@ -4,6 +4,7 @@ #include "media/base/cdm_key_information.h" +#include "base/notreached.h" #include "base/stl_util.h" #include "base/strings/string_number_conversions.h" diff --git a/chromium/media/base/channel_mixer.cc b/chromium/media/base/channel_mixer.cc index 841933842a6..87e6c4590db 100644 --- a/chromium/media/base/channel_mixer.cc +++ b/chromium/media/base/channel_mixer.cc @@ -5,8 +5,9 @@ #include "media/base/channel_mixer.h" #include <stddef.h> +#include <string.h> -#include "base/logging.h" +#include "base/check_op.h" #include "media/base/audio_bus.h" #include "media/base/audio_parameters.h" #include "media/base/channel_mixing_matrix.h" diff --git a/chromium/media/base/channel_mixing_matrix.cc b/chromium/media/base/channel_mixing_matrix.cc index 51a556ad04b..3ad627e1694 100644 --- a/chromium/media/base/channel_mixing_matrix.cc +++ b/chromium/media/base/channel_mixing_matrix.cc @@ -8,7 +8,7 @@ #include <algorithm> -#include "base/logging.h" +#include "base/check_op.h" #include "media/base/channel_mixer.h" namespace media { diff --git a/chromium/media/base/container_names.cc b/chromium/media/base/container_names.cc index 64fb021861d..496fdbef244 100644 --- a/chromium/media/base/container_names.cc +++ b/chromium/media/base/container_names.cc @@ -5,11 +5,12 @@ #include "media/base/container_names.h" #include <stddef.h> +#include <string.h> #include <cctype> #include <limits> -#include "base/logging.h" +#include "base/check_op.h" #include "base/numerics/safe_conversions.h" #include "base/stl_util.h" #include "media/base/bit_reader.h" diff --git a/chromium/media/base/container_names.h b/chromium/media/base/container_names.h index 4e464859054..99cb7bf7ccd 100644 --- a/chromium/media/base/container_names.h +++ b/chromium/media/base/container_names.h @@ -18,6 +18,9 @@ namespace container_names { // done at the end of the list (before CONTAINER_MAX). This list must be kept in // sync with the enum definition "MediaContainers" in // tools/metrics/histograms/histograms.xml. +// +// GENERATED_JAVA_ENUM_PACKAGE: org.chromium.media +// GENERATED_JAVA_PREFIX_TO_STRIP: CONTAINER_ enum MediaContainerName { CONTAINER_UNKNOWN, // Unknown CONTAINER_AAC, // AAC (Advanced Audio Coding) @@ -59,7 +62,7 @@ enum MediaContainerName { CONTAINER_WTV, // WTV (Windows Television) CONTAINER_DASH, // DASH (MPEG-DASH) CONTAINER_SMOOTHSTREAM, // SmoothStreaming - CONTAINER_MAX = CONTAINER_SMOOTHSTREAM // Must be last + CONTAINER_MAX = CONTAINER_SMOOTHSTREAM, // Must be last }; // Minimum size considered for processing. diff --git a/chromium/media/base/data_source.cc b/chromium/media/base/data_source.cc index 0ded6d30d37..48f76091f31 100644 --- a/chromium/media/base/data_source.cc +++ b/chromium/media/base/data_source.cc @@ -4,7 +4,6 @@ #include "media/base/data_source.h" -#include "base/logging.h" namespace media { diff --git a/chromium/media/base/decode_status.cc b/chromium/media/base/decode_status.cc index 110696fd941..97fda7276d0 100644 --- a/chromium/media/base/decode_status.cc +++ b/chromium/media/base/decode_status.cc @@ -19,6 +19,9 @@ const char* GetDecodeStatusString(DecodeStatus status) { case DecodeStatus::DECODE_ERROR: return "DecodeStatus::DECODE_ERROR"; } + + NOTREACHED(); + return ""; } std::ostream& operator<<(std::ostream& os, const DecodeStatus& status) { diff --git a/chromium/media/base/decoder_buffer.cc b/chromium/media/base/decoder_buffer.cc index 5a07221ef8a..0a593b8d6ef 100644 --- a/chromium/media/base/decoder_buffer.cc +++ b/chromium/media/base/decoder_buffer.cc @@ -4,6 +4,8 @@ #include "media/base/decoder_buffer.h" +#include <sstream> + #include "base/debug/alias.h" namespace media { diff --git a/chromium/media/base/decrypt_config.cc b/chromium/media/base/decrypt_config.cc index ce994949a5a..5b7c02c8f5e 100644 --- a/chromium/media/base/decrypt_config.cc +++ b/chromium/media/base/decrypt_config.cc @@ -6,7 +6,7 @@ #include <stddef.h> -#include "base/logging.h" +#include "base/check_op.h" #include "base/memory/ptr_util.h" #include "base/strings/string_number_conversions.h" #include "media/media_buildflags.h" diff --git a/chromium/media/base/demuxer.h b/chromium/media/base/demuxer.h index c41493a781d..a06793ab07a 100644 --- a/chromium/media/base/demuxer.h +++ b/chromium/media/base/demuxer.h @@ -13,6 +13,7 @@ #include "base/macros.h" #include "base/optional.h" #include "base/time/time.h" +#include "media/base/container_names.h" #include "media/base/data_source.h" #include "media/base/demuxer_stream.h" #include "media/base/eme_constants.h" @@ -138,6 +139,13 @@ class MEDIA_EXPORT Demuxer : public MediaResource { // Returns the memory usage in bytes for the demuxer. virtual int64_t GetMemoryUsage() const = 0; + // Returns the container name to use for metrics. + // Implementations where this is not meaningful will return an empty value. + // Implementations that do provide values should always provide a value, + // returning CONTAINER_UNKNOWN in cases where the container is not known. + virtual base::Optional<container_names::MediaContainerName> + GetContainerForMetrics() const = 0; + // The |track_ids| vector has either 1 track, or is empty, indicating that // all tracks should be disabled. |change_completed_cb| is fired after the // demuxer streams are disabled, however this callback should then notify diff --git a/chromium/media/base/encryption_scheme.cc b/chromium/media/base/encryption_scheme.cc index 40de184e5df..0fd36097378 100644 --- a/chromium/media/base/encryption_scheme.cc +++ b/chromium/media/base/encryption_scheme.cc @@ -6,7 +6,6 @@ #include <ostream> -#include "base/logging.h" namespace media { diff --git a/chromium/media/base/fake_audio_worker.cc b/chromium/media/base/fake_audio_worker.cc index c32f8503d22..bcaf73ebef5 100644 --- a/chromium/media/base/fake_audio_worker.cc +++ b/chromium/media/base/fake_audio_worker.cc @@ -9,8 +9,8 @@ #include "base/bind.h" #include "base/bind_helpers.h" #include "base/cancelable_callback.h" +#include "base/check_op.h" #include "base/location.h" -#include "base/logging.h" #include "base/macros.h" #include "base/single_thread_task_runner.h" #include "base/synchronization/lock.h" diff --git a/chromium/media/base/fake_demuxer_stream.cc b/chromium/media/base/fake_demuxer_stream.cc index dac08196da9..61b463adc80 100644 --- a/chromium/media/base/fake_demuxer_stream.cc +++ b/chromium/media/base/fake_demuxer_stream.cc @@ -11,8 +11,9 @@ #include "base/bind.h" #include "base/callback_helpers.h" +#include "base/check_op.h" #include "base/location.h" -#include "base/logging.h" +#include "base/notreached.h" #include "base/single_thread_task_runner.h" #include "base/stl_util.h" #include "base/threading/thread_task_runner_handle.h" diff --git a/chromium/media/base/hdr_metadata.h b/chromium/media/base/hdr_metadata.h index 4846025e3bf..715642d32cd 100644 --- a/chromium/media/base/hdr_metadata.h +++ b/chromium/media/base/hdr_metadata.h @@ -52,6 +52,15 @@ struct MEDIA_EXPORT HDRMetadata { } }; +// HDR metadata types as described in +// https://w3c.github.io/media-capabilities/#enumdef-hdrmetadatatype +enum class HdrMetadataType { + kNone, + kSmpteSt2086, + kSmpteSt2094_10, + kSmpteSt2094_40, +}; + } // namespace media #endif // MEDIA_BASE_HDR_METADATA_H_ diff --git a/chromium/media/base/ipc/media_param_traits_macros.h b/chromium/media/base/ipc/media_param_traits_macros.h index 3e09669bce0..1eca0edb227 100644 --- a/chromium/media/base/ipc/media_param_traits_macros.h +++ b/chromium/media/base/ipc/media_param_traits_macros.h @@ -37,9 +37,6 @@ #include "media/base/video_types.h" #include "media/base/waiting.h" #include "media/base/watch_time_keys.h" -// TODO(crbug.com/676224): When EnabledIf attribute is supported in mojom files, -// move CdmProxy related code into #if BUILDFLAG(ENABLE_LIBRARY_CDMS). -#include "media/cdm/cdm_proxy.h" #include "media/media_buildflags.h" #include "media/video/supported_video_decoder_config.h" #include "ui/gfx/ipc/color/gfx_param_traits_macros.h" @@ -73,18 +70,6 @@ IPC_ENUM_TRAITS_MAX_VALUE(media::CdmMessageType, IPC_ENUM_TRAITS_MAX_VALUE(media::CdmPromise::Exception, media::CdmPromise::Exception::EXCEPTION_MAX) -IPC_ENUM_TRAITS_MAX_VALUE(media::CdmProxy::Function, - media::CdmProxy::Function::kMaxValue) - -IPC_ENUM_TRAITS_MAX_VALUE(media::CdmProxy::KeyType, - media::CdmProxy::KeyType::kMaxValue) - -IPC_ENUM_TRAITS_MAX_VALUE(media::CdmProxy::Protocol, - media::CdmProxy::Protocol::kMaxValue) - -IPC_ENUM_TRAITS_MAX_VALUE(media::CdmProxy::Status, - media::CdmProxy::Status::kMaxValue) - IPC_ENUM_TRAITS_MAX_VALUE(media::CdmSessionType, media::CdmSessionType::kMaxValue) diff --git a/chromium/media/base/key_systems_unittest.cc b/chromium/media/base/key_systems_unittest.cc index 29c042a4c77..64a7a4b4336 100644 --- a/chromium/media/base/key_systems_unittest.cc +++ b/chromium/media/base/key_systems_unittest.cc @@ -12,7 +12,8 @@ #include <string> #include <vector> -#include "base/logging.h" +#include "base/check.h" +#include "base/notreached.h" #include "media/base/audio_parameters.h" #include "media/base/decrypt_config.h" #include "media/base/eme_constants.h" diff --git a/chromium/media/base/keyboard_event_counter.cc b/chromium/media/base/keyboard_event_counter.cc index 46c2a2d7a73..6c17f676e8c 100644 --- a/chromium/media/base/keyboard_event_counter.cc +++ b/chromium/media/base/keyboard_event_counter.cc @@ -4,7 +4,7 @@ #include "media/base/keyboard_event_counter.h" -#include "base/logging.h" +#include "base/check_op.h" namespace media { diff --git a/chromium/media/base/keyboard_event_counter_unittest.cc b/chromium/media/base/keyboard_event_counter_unittest.cc index ec3707c7ad5..7d104dc76c8 100644 --- a/chromium/media/base/keyboard_event_counter_unittest.cc +++ b/chromium/media/base/keyboard_event_counter_unittest.cc @@ -6,7 +6,6 @@ #include <memory> -#include "base/logging.h" #include "base/run_loop.h" #include "build/build_config.h" #include "testing/gmock/include/gmock/gmock.h" diff --git a/chromium/media/base/localized_strings.cc b/chromium/media/base/localized_strings.cc index 5c1f24bdd7f..613accca08a 100644 --- a/chromium/media/base/localized_strings.cc +++ b/chromium/media/base/localized_strings.cc @@ -4,7 +4,6 @@ #include "media/base/localized_strings.h" -#include "base/logging.h" #include "base/strings/utf_string_conversions.h" #include "build/build_config.h" diff --git a/chromium/media/base/mac/audio_latency_mac.cc b/chromium/media/base/mac/audio_latency_mac.cc index a4e9bd59f03..3d1aa2fd638 100644 --- a/chromium/media/base/mac/audio_latency_mac.cc +++ b/chromium/media/base/mac/audio_latency_mac.cc @@ -3,7 +3,7 @@ // found in the LICENSE file. #include "media/base/mac/audio_latency_mac.h" -#include "base/logging.h" +#include "base/check_op.h" #include "media/base/limits.h" namespace media { diff --git a/chromium/media/base/media_client.cc b/chromium/media/base/media_client.cc index 584ccc185fd..011f3b74137 100644 --- a/chromium/media/base/media_client.cc +++ b/chromium/media/base/media_client.cc @@ -4,7 +4,6 @@ #include "media/base/media_client.h" -#include "base/logging.h" namespace media { diff --git a/chromium/media/base/media_log.cc b/chromium/media/base/media_log.cc index babef2baa8a..6cb08ed64cb 100644 --- a/chromium/media/base/media_log.cc +++ b/chromium/media/base/media_log.cc @@ -24,8 +24,7 @@ static base::AtomicSequenceNumber g_media_log_count; MediaLog::MediaLog() : MediaLog(new ParentLogRecord(this)) {} MediaLog::MediaLog(scoped_refptr<ParentLogRecord> parent_log_record) - : parent_log_record_(std::move(parent_log_record)), - id_(g_media_log_count.GetNext()) {} + : parent_log_record_(std::move(parent_log_record)) {} MediaLog::~MediaLog() { // If we are the underlying log, then somebody should have called @@ -39,8 +38,7 @@ MediaLog::~MediaLog() { // We could get around this if we introduce a new NullMediaLog that handles // log invalidation, so we could dcheck here. However, that seems like a lot // of boilerplate. - if (parent_log_record_->media_log == this) - InvalidateLog(); + InvalidateLog(); } // Default *Locked implementations @@ -65,6 +63,13 @@ void MediaLog::NotifyError(PipelineStatus status) { AddLogRecord(std::move(record)); } +void MediaLog::NotifyError(Status status) { + DCHECK(!status.is_ok()); + std::string output_str; + base::JSONWriter::Write(MediaSerialize(status), &output_str); + AddMessage(MediaLogMessageLevel::kERROR, output_str); +} + void MediaLog::OnWebMediaPlayerDestroyedLocked() {} void MediaLog::OnWebMediaPlayerDestroyed() { AddEvent<MediaLogEvent::kWebMediaPlayerDestroyed>(); @@ -98,7 +103,7 @@ void MediaLog::AddLogRecord(std::unique_ptr<MediaLogRecord> record) { std::unique_ptr<MediaLogRecord> MediaLog::CreateRecord( MediaLogRecord::Type type) { auto record = std::make_unique<MediaLogRecord>(); - record->id = id_; + record->id = id(); record->type = type; record->time = base::TimeTicks::Now(); return record; @@ -106,15 +111,15 @@ std::unique_ptr<MediaLogRecord> MediaLog::CreateRecord( void MediaLog::InvalidateLog() { base::AutoLock auto_lock(parent_log_record_->lock); - // It's almost certainly unintentional to invalidate a parent log. - DCHECK(parent_log_record_->media_log == nullptr || - parent_log_record_->media_log == this); - - parent_log_record_->media_log = nullptr; + // Do nothing if this log didn't create the record, i.e. + // it's not the parent log. The parent log should invalidate itself. + if (parent_log_record_->media_log == this) + parent_log_record_->media_log = nullptr; // Keep |parent_log_record_| around, since the lock must keep working. } -MediaLog::ParentLogRecord::ParentLogRecord(MediaLog* log) : media_log(log) {} +MediaLog::ParentLogRecord::ParentLogRecord(MediaLog* log) + : id(g_media_log_count.GetNext()), media_log(log) {} MediaLog::ParentLogRecord::~ParentLogRecord() = default; LogHelper::LogHelper(MediaLogMessageLevel level, MediaLog* media_log) diff --git a/chromium/media/base/media_log.h b/chromium/media/base/media_log.h index 957181676fd..4812b87b119 100644 --- a/chromium/media/base/media_log.h +++ b/chromium/media/base/media_log.h @@ -23,7 +23,6 @@ #include "media/base/media_log_message_levels.h" #include "media/base/media_log_properties.h" #include "media/base/media_log_record.h" -#include "media/base/pipeline_impl.h" #include "media/base/pipeline_status.h" #include "url/gurl.h" @@ -81,6 +80,9 @@ class MEDIA_EXPORT MediaLog { // TODO(tmathmeyer) replace with Status when that's ready. void NotifyError(PipelineStatus status); + // Notify a non-ok Status. This method Should _not_ be given an OK status. + void NotifyError(Status status); + // Notify the media log that the player is destroyed. Some implementations // will want to change event handling based on this. void OnWebMediaPlayerDestroyed(); @@ -97,16 +99,17 @@ class MEDIA_EXPORT MediaLog { // Getter for |id_|. Used by MojoMediaLogService to construct MediaLogRecords // to log into this MediaLog. Also used in trace events to associate each // event with a specific media playback. - int32_t id() const { return id_; } + int32_t id() const { return parent_log_record_->id; } // Add a record to this log. Inheritors should override AddLogRecordLocked to // do something. This needs to be public for MojoMediaLogService to use it. void AddLogRecord(std::unique_ptr<MediaLogRecord> event); // Provide a MediaLog which can have a separate lifetime from this one, but - // still write to the same log. It is not guaranteed that this will log - // forever; it might start silently discarding log messages if the original - // log is closed by whoever owns it. + // still write to the same player's log. It is not guaranteed that this will + // log forever; it might start silently discarding log messages if the + // original log is closed by whoever owns it. However, it's safe to use it + // even if this occurs, in the "won't crash" sense. virtual std::unique_ptr<MediaLog> Clone(); protected: @@ -147,7 +150,10 @@ class MEDIA_EXPORT MediaLog { void InvalidateLog(); struct ParentLogRecord : base::RefCountedThreadSafe<ParentLogRecord> { - ParentLogRecord(MediaLog* log); + explicit ParentLogRecord(MediaLog* log); + + // A unique (to this process) id for this MediaLog. + int32_t id; // |lock_| protects the rest of this structure. base::Lock lock; @@ -162,15 +168,15 @@ class MEDIA_EXPORT MediaLog { DISALLOW_COPY_AND_ASSIGN(ParentLogRecord); }; - // Use |parent_log_record| instead of making a new one. - MediaLog(scoped_refptr<ParentLogRecord> parent_log_record); - private: // Allows MediaLogTest to construct MediaLog directly for testing. friend class MediaLogTest; FRIEND_TEST_ALL_PREFIXES(MediaLogTest, EventsAreForwarded); FRIEND_TEST_ALL_PREFIXES(MediaLogTest, EventsAreNotForwardedAfterInvalidate); + // Use |parent_log_record| instead of making a new one. + explicit MediaLog(scoped_refptr<ParentLogRecord> parent_log_record); + // Helper methods to create events and their parameters. std::unique_ptr<MediaLogRecord> CreateRecord(MediaLogRecord::Type type); @@ -187,8 +193,6 @@ class MEDIA_EXPORT MediaLog { // The underlying media log. scoped_refptr<ParentLogRecord> parent_log_record_; - // A unique (to this process) id for this MediaLog. - int32_t id_; DISALLOW_COPY_AND_ASSIGN(MediaLog); }; diff --git a/chromium/media/base/media_log_events.cc b/chromium/media/base/media_log_events.cc index 187ddcc9b5e..7ce6c22375c 100644 --- a/chromium/media/base/media_log_events.cc +++ b/chromium/media/base/media_log_events.cc @@ -6,7 +6,7 @@ #include <string> -#include "base/logging.h" +#include "base/notreached.h" namespace media { @@ -41,4 +41,13 @@ std::string MediaLogEventToString(MediaLogEvent level) { return ""; } +std::string TruncateUrlString(std::string url) { + constexpr size_t kMaxUrlLength = 1000; + if (url.length() > kMaxUrlLength) { + url.resize(kMaxUrlLength); + url.replace(url.end() - 3, url.end(), "..."); + } + return url; +} + } // namespace media diff --git a/chromium/media/base/media_log_events.h b/chromium/media/base/media_log_events.h index eed7a29725b..9cd26e5b128 100644 --- a/chromium/media/base/media_log_events.h +++ b/chromium/media/base/media_log_events.h @@ -67,6 +67,9 @@ enum class MediaLogEvent { // instead of using macro stringification. MEDIA_EXPORT std::string MediaLogEventToString(MediaLogEvent level); +// Sometimes URLs can have encoded data that can be exteremly large. +MEDIA_EXPORT std::string TruncateUrlString(std::string url); + // These events can be triggered with no extra associated data. MEDIA_LOG_EVENT_TYPELESS(kPlay); MEDIA_LOG_EVENT_TYPELESS(kPause); @@ -77,12 +80,15 @@ MEDIA_LOG_EVENT_TYPELESS(kWebMediaPlayerCreated); // These events can be triggered with the extra data / names as defined here. // Note that some events can be defined multiple times. -MEDIA_LOG_EVENT_NAMED_DATA(kLoad, std::string, "url"); MEDIA_LOG_EVENT_NAMED_DATA(kSeek, double, "seek_target"); MEDIA_LOG_EVENT_NAMED_DATA(kVideoSizeChanged, gfx::Size, "dimensions"); MEDIA_LOG_EVENT_NAMED_DATA(kDurationChanged, base::TimeDelta, "duration"); -MEDIA_LOG_EVENT_NAMED_DATA(kWebMediaPlayerCreated, std::string, "origin_url"); MEDIA_LOG_EVENT_NAMED_DATA(kPipelineStateChange, std::string, "pipeline_state"); +MEDIA_LOG_EVENT_NAMED_DATA_OP(kLoad, std::string, "url", TruncateUrlString); +MEDIA_LOG_EVENT_NAMED_DATA_OP(kWebMediaPlayerCreated, + std::string, + "origin_url", + TruncateUrlString); // Each type of buffering state gets a different name. MEDIA_LOG_EVENT_NAMED_DATA( diff --git a/chromium/media/base/media_log_message_levels.cc b/chromium/media/base/media_log_message_levels.cc index bea3f563008..c39257c8a31 100644 --- a/chromium/media/base/media_log_message_levels.cc +++ b/chromium/media/base/media_log_message_levels.cc @@ -6,7 +6,7 @@ #include <string> -#include "base/logging.h" +#include "base/notreached.h" namespace media { diff --git a/chromium/media/base/media_log_properties.cc b/chromium/media/base/media_log_properties.cc index d9ced3b410f..80d6a8e5c73 100644 --- a/chromium/media/base/media_log_properties.cc +++ b/chromium/media/base/media_log_properties.cc @@ -32,6 +32,8 @@ std::string MediaLogPropertyKeyToString(MediaLogProperty property) { STRINGIFY(kIsPlatformAudioDecoder); STRINGIFY(kAudioTracks); STRINGIFY(kVideoTracks); + STRINGIFY(kFramerate); + STRINGIFY(kVideoPlaybackRoughness); } #undef STRINGIFY } diff --git a/chromium/media/base/media_log_properties.h b/chromium/media/base/media_log_properties.h index f5283195094..d14b702c658 100644 --- a/chromium/media/base/media_log_properties.h +++ b/chromium/media/base/media_log_properties.h @@ -75,6 +75,13 @@ enum class MediaLogProperty { // Track metadata. kAudioTracks, kVideoTracks, + + // Effective video playback frame rate adjusted for the playback speed. + // Updated along with kVideoPlaybackRoughness (i.e. not very often) + kFramerate, + + // A playback quality metric calculated by VideoPlaybackRoughnessReporter + kVideoPlaybackRoughness, }; MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kResolution, gfx::Size); @@ -97,6 +104,8 @@ MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kIsPlatformAudioDecoder, bool); MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kIsAudioDecryptingDemuxerStream, bool); MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kAudioTracks, std::vector<AudioDecoderConfig>); MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kVideoTracks, std::vector<VideoDecoderConfig>); +MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kFramerate, double); +MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kVideoPlaybackRoughness, double); // Convert the enum to a string (used for the front-end enum matching). MEDIA_EXPORT std::string MediaLogPropertyKeyToString(MediaLogProperty property); diff --git a/chromium/media/base/media_log_record.h b/chromium/media/base/media_log_record.h index 76822114c9e..36125cc2cd0 100644 --- a/chromium/media/base/media_log_record.h +++ b/chromium/media/base/media_log_record.h @@ -13,6 +13,8 @@ namespace media { +// TODO(tmathmeyer) refactor this so that we aren't packing type-erased data +// with different meanings into "params". struct MediaLogRecord { MediaLogRecord() {} diff --git a/chromium/media/base/media_log_type_enforcement.h b/chromium/media/base/media_log_type_enforcement.h index 8404e00a23f..7624ce34490 100644 --- a/chromium/media/base/media_log_type_enforcement.h +++ b/chromium/media/base/media_log_type_enforcement.h @@ -46,6 +46,16 @@ struct MediaLogEventTypeSupport {}; static std::string TypeName() { return #EVENT; } \ } +#define MEDIA_LOG_EVENT_NAMED_DATA_OP(EVENT, TYPE, DISPLAY, OP) \ + template <> \ + struct MediaLogEventTypeSupport<MediaLogEvent::EVENT, TYPE> { \ + static void AddExtraData(base::Value* params, const TYPE& t) { \ + DCHECK(params); \ + params->SetKey(DISPLAY, MediaSerialize<TYPE>(OP(t))); \ + } \ + static std::string TypeName() { return #EVENT; } \ + } + // Specifically do not create the Convert or DisplayName methods #define MEDIA_LOG_EVENT_TYPELESS(EVENT) \ template <> \ diff --git a/chromium/media/base/media_log_unittest.cc b/chromium/media/base/media_log_unittest.cc index 09a5126bca3..2a126bada33 100644 --- a/chromium/media/base/media_log_unittest.cc +++ b/chromium/media/base/media_log_unittest.cc @@ -16,15 +16,11 @@ namespace media { // Friend class of MediaLog for access to internal constants. class MediaLogTest : public testing::Test { - public: - static constexpr size_t kMaxUrlLength = MediaLog::kMaxUrlLength; - protected: - MediaLog media_log; -}; - -constexpr size_t MediaLogTest::kMaxUrlLength; + std::unique_ptr<MockMediaLog> root_log_; + void CreateLog() { root_log_ = std::make_unique<MockMediaLog>(); } +}; TEST_F(MediaLogTest, EventsAreForwarded) { // Make sure that |root_log_| receives events. @@ -44,4 +40,64 @@ TEST_F(MediaLogTest, EventsAreNotForwardedAfterInvalidate) { child_media_log->AddMessage(MediaLogMessageLevel::kERROR, "test"); } +TEST_F(MediaLogTest, ClonedLogsInhertParentPlayerId) { + std::unique_ptr<MockMediaLog> root_log(std::make_unique<MockMediaLog>()); + std::unique_ptr<MediaLog> child_media_log(root_log->Clone()); + EXPECT_CALL(*root_log, DoAddLogRecordLogString(_)).Times(1); + child_media_log->AddMessage(MediaLogMessageLevel::kERROR, "test"); + auto event = root_log->take_most_recent_event(); + EXPECT_NE(event, nullptr); + EXPECT_EQ(event->id, root_log->id()); +} + +TEST_F(MediaLogTest, DontTruncateShortUrlString) { + CreateLog(); + EXPECT_CALL(*root_log_, DoAddLogRecordLogString(_)).Times(2); + const std::string short_url("chromium.org"); + + // Verify that LoadEvent does not truncate the short URL. + root_log_->AddEvent<MediaLogEvent::kLoad>(short_url); + auto event = root_log_->take_most_recent_event(); + EXPECT_NE(event, nullptr); + EXPECT_EQ(*event->params.FindStringPath("url"), "chromium.org"); + + // Verify that CreatedEvent does not truncate the short URL. + root_log_->AddEvent<MediaLogEvent::kWebMediaPlayerCreated>(short_url); + event = root_log_->take_most_recent_event(); + EXPECT_NE(event, nullptr); + EXPECT_EQ(*event->params.FindStringPath("origin_url"), "chromium.org"); +} + +TEST_F(MediaLogTest, TruncateLongUrlStrings) { + CreateLog(); + EXPECT_CALL(*root_log_, DoAddLogRecordLogString(_)).Times(2); + // Build a long string that exceeds the URL length limit. + std::stringstream string_builder; + constexpr size_t kLongStringLength = 1010; + for (size_t i = 0; i < kLongStringLength; i++) { + string_builder << "c"; + } + const std::string long_url = string_builder.str(); + + std::stringstream expected_string_builder; + constexpr size_t kMaxLength = 1000; + for (size_t i = 0; i < kMaxLength - 3; i++) { + expected_string_builder << "c"; + } + expected_string_builder << "..."; + const std::string expected_url = expected_string_builder.str(); + + // Verify that LoadEvent does not truncate the short URL. + root_log_->AddEvent<MediaLogEvent::kLoad>(long_url); + auto event = root_log_->take_most_recent_event(); + EXPECT_NE(event, nullptr); + EXPECT_EQ(*event->params.FindStringPath("url"), expected_url); + + // Verify that CreatedEvent does not truncate the short URL. + root_log_->AddEvent<MediaLogEvent::kWebMediaPlayerCreated>(long_url); + event = root_log_->take_most_recent_event(); + EXPECT_NE(event, nullptr); + EXPECT_EQ(*event->params.FindStringPath("origin_url"), expected_url); +} + } // namespace media diff --git a/chromium/media/base/media_observer.h b/chromium/media/base/media_observer.h index 225cc8d7985..eb0e3969c58 100644 --- a/chromium/media/base/media_observer.h +++ b/chromium/media/base/media_observer.h @@ -67,6 +67,9 @@ class MEDIA_EXPORT MediaObserver { // Remote Playback API spec: https://w3c.github.io/remote-playback virtual void OnRemotePlaybackDisabled(bool disabled) = 0; + // Called on Android, whenever we detect that we are playing back HLS. + virtual void OnHlsManifestDetected() = 0; + // Called when the media is playing/paused. virtual void OnPlaying() = 0; virtual void OnPaused() = 0; diff --git a/chromium/media/base/media_serializers.h b/chromium/media/base/media_serializers.h index 6333c44170f..832625da3ff 100644 --- a/chromium/media/base/media_serializers.h +++ b/chromium/media/base/media_serializers.h @@ -5,6 +5,7 @@ #ifndef MEDIA_BASE_MEDIA_SERIALIZERS_H_ #define MEDIA_BASE_MEDIA_SERIALIZERS_H_ +#include <sstream> #include <vector> #include "base/location.h" diff --git a/chromium/media/base/media_switches.cc b/chromium/media/base/media_switches.cc index 636dec1c063..e0d62061b43 100644 --- a/chromium/media/base/media_switches.cc +++ b/chromium/media/base/media_switches.cc @@ -29,9 +29,9 @@ const char kFailAudioStreamCreation[] = "fail-audio-stream-creation"; // Set number of threads to use for video decoding. const char kVideoThreads[] = "video-threads"; -// Suspend media pipeline on background tabs. -const char kEnableMediaSuspend[] = "enable-media-suspend"; -const char kDisableMediaSuspend[] = "disable-media-suspend"; +// Do not immediately suspend media in background tabs. +const char kDisableBackgroundMediaSuspend[] = + "disable-background-media-suspend"; // Force to report VP9 as an unsupported MIME type. const char kReportVp9AsAnUnsupportedMimeType[] = @@ -76,9 +76,6 @@ const char kEnableProtectedVideoBuffers[] = "enable-protected-video-buffers"; const char kForceProtectedVideoOutputBuffers[] = "force-protected-video-output-buffers"; -// Enables fuchsia.media.AudioConsumer to be used to render audio streams. -const char kEnableFuchsiaAudioConsumer[] = "enable-fuchsia-audio-consumer"; - #endif // defined(OS_FUCHSIA) #if defined(USE_CRAS) @@ -369,7 +366,14 @@ const base::Feature kGlobalMediaControlsOverlayControls{ // Show picture-in-picture button in Global Media Controls. const base::Feature kGlobalMediaControlsPictureInPicture{ - "GlobalMediaControlsPictureInPicture", base::FEATURE_DISABLED_BY_DEFAULT}; + "GlobalMediaControlsPictureInPicture", +#if defined(OS_WIN) || defined(OS_MACOSX) || \ + (defined(OS_LINUX) && !defined(OS_CHROMEOS)) + base::FEATURE_ENABLED_BY_DEFAULT +#else + base::FEATURE_DISABLED_BY_DEFAULT +#endif +}; // Enable new cpu load estimator. Intended for evaluation in local // testing and origin-trial. @@ -382,6 +386,11 @@ const base::Feature kNewEncodeCpuLoadEstimator{ const base::Feature kSpecCompliantCanPlayThrough{ "SpecCompliantCanPlayThrough", base::FEATURE_ENABLED_BY_DEFAULT}; +// Disables the real audio output stream after silent audio has been delivered +// for too long. Should save quite a bit of power in the muted video case. +const base::Feature kSuspendMutedAudio{"SuspendMutedAudio", + base::FEATURE_DISABLED_BY_DEFAULT}; + // Use shared block-based buffering for media. const base::Feature kUseNewMediaCache{"use-new-media-cache", base::FEATURE_ENABLED_BY_DEFAULT}; @@ -390,12 +399,6 @@ const base::Feature kUseNewMediaCache{"use-new-media-cache", const base::Feature kUseMediaHistoryStore{"UseMediaHistoryStore", base::FEATURE_DISABLED_BY_DEFAULT}; -// Causes video.requestAniationFrame to use a microtask instead of running with -// the rendering steps. TODO(crbug.com/1012063): Remove this once we figure out -// which implementation to use. -const base::Feature kUseMicrotaskForVideoRAF{"UseMicrotaskForVideoRAF", - base::FEATURE_DISABLED_BY_DEFAULT}; - // Use R16 texture for 9-16 bit channel instead of half-float conversion by CPU. const base::Feature kUseR16Texture{"use-r16-texture", base::FEATURE_DISABLED_BY_DEFAULT}; @@ -455,6 +458,9 @@ const base::Feature kFailUrlProvisionFetcherForTesting{ const base::Feature kHardwareSecureDecryption{ "HardwareSecureDecryption", base::FEATURE_DISABLED_BY_DEFAULT}; +const base::Feature kWakeLockOptimisationHiddenMuted{ + "kWakeLockOptimisationHiddenMuted", base::FEATURE_ENABLED_BY_DEFAULT}; + // Enables encrypted AV1 support in EME requestMediaKeySystemAccess() query by // Widevine key system if it is also supported by the underlying Widevine CDM. // This feature does not affect the actual playback of encrypted AV1 if it's @@ -565,9 +571,20 @@ const base::Feature kUsePooledSharedImageVideoProvider{ const base::Feature kDelayCopyNV12Textures{"DelayCopyNV12Textures", base::FEATURE_ENABLED_BY_DEFAULT}; -// Enables H264 HW encode acceleration using Media Foundation for Windows. -const base::Feature kMediaFoundationH264Encoding{ - "MediaFoundationH264Encoding", base::FEATURE_ENABLED_BY_DEFAULT}; +// Enables DirectShow GetPhotoState implementation +// Created to act as a kill switch by disabling it, in the case of the +// resurgence of https://crbug.com/722038 +const base::Feature kDirectShowGetPhotoState{"DirectShowGetPhotoState", + base::FEATURE_ENABLED_BY_DEFAULT}; + +// Enables asynchronous H264 HW encode acceleration using Media Foundation for +// Windows. +const base::Feature kMediaFoundationAsyncH264Encoding{ + "MediaFoundationAsyncH264Encoding", base::FEATURE_DISABLED_BY_DEFAULT}; + +// Enables AV1 decode acceleration for Windows. +const base::Feature MEDIA_EXPORT kMediaFoundationAV1Decoding{ + "MediaFoundationAV1Decoding", base::FEATURE_DISABLED_BY_DEFAULT}; // Enables MediaFoundation based video capture const base::Feature kMediaFoundationVideoCapture{ @@ -577,12 +594,6 @@ const base::Feature kMediaFoundationVideoCapture{ const base::Feature MEDIA_EXPORT kMediaFoundationVP8Decoding{ "MediaFoundationVP8Decoding", base::FEATURE_DISABLED_BY_DEFAULT}; -// Enables DirectShow GetPhotoState implementation -// Created to act as a kill switch by disabling it, in the case of the -// resurgence of https://crbug.com/722038 -const base::Feature kDirectShowGetPhotoState{"DirectShowGetPhotoState", - base::FEATURE_ENABLED_BY_DEFAULT}; - #endif // defined(OS_WIN) std::string GetEffectiveAutoplayPolicy(const base::CommandLine& command_line) { @@ -644,11 +655,11 @@ const base::Feature kMediaFeeds{"MediaFeeds", // Enables checking Media Feeds against safe search to prevent adult content. const base::Feature kMediaFeedsSafeSearch{"MediaFeedsSafeSearch", - base::FEATURE_DISABLED_BY_DEFAULT}; + base::FEATURE_ENABLED_BY_DEFAULT}; // Send events to devtools rather than to chrome://media-internals const base::Feature kMediaInspectorLogging{"MediaInspectorLogging", - base::FEATURE_DISABLED_BY_DEFAULT}; + base::FEATURE_ENABLED_BY_DEFAULT}; // Enables experimental local learning for media. Used in the context of media // capabilities only. Adds reporting only; does not change media behavior. diff --git a/chromium/media/base/media_switches.h b/chromium/media/base/media_switches.h index dfe622f670c..5add050c996 100644 --- a/chromium/media/base/media_switches.h +++ b/chromium/media/base/media_switches.h @@ -31,9 +31,7 @@ MEDIA_EXPORT extern const char kFailAudioStreamCreation[]; MEDIA_EXPORT extern const char kVideoThreads[]; -// TODO(crbug.com/867146): remove these switches. -MEDIA_EXPORT extern const char kEnableMediaSuspend[]; -MEDIA_EXPORT extern const char kDisableMediaSuspend[]; +MEDIA_EXPORT extern const char kDisableBackgroundMediaSuspend[]; MEDIA_EXPORT extern const char kReportVp9AsAnUnsupportedMimeType[]; @@ -52,7 +50,6 @@ MEDIA_EXPORT extern const char kWaveOutBuffers[]; #if defined(OS_FUCHSIA) MEDIA_EXPORT extern const char kEnableProtectedVideoBuffers[]; MEDIA_EXPORT extern const char kForceProtectedVideoOutputBuffers[]; -MEDIA_EXPORT extern const char kEnableFuchsiaAudioConsumer[]; #endif #if BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION) @@ -159,12 +156,12 @@ MEDIA_EXPORT extern const base::Feature kRecordMediaEngagementScores; MEDIA_EXPORT extern const base::Feature kRecordWebAudioEngagement; MEDIA_EXPORT extern const base::Feature kResumeBackgroundVideo; MEDIA_EXPORT extern const base::Feature kRevokeMediaSourceObjectURLOnAttach; +MEDIA_EXPORT extern const base::Feature kSuspendMutedAudio; MEDIA_EXPORT extern const base::Feature kSpecCompliantCanPlayThrough; MEDIA_EXPORT extern const base::Feature kUnifiedAutoplay; MEDIA_EXPORT extern const base::Feature kUseAndroidOverlayAggressively; MEDIA_EXPORT extern const base::Feature kUseFakeDeviceForMediaStream; MEDIA_EXPORT extern const base::Feature kUseMediaHistoryStore; -MEDIA_EXPORT extern const base::Feature kUseMicrotaskForVideoRAF; MEDIA_EXPORT extern const base::Feature kUseNewMediaCache; MEDIA_EXPORT extern const base::Feature kUseR16Texture; MEDIA_EXPORT extern const base::Feature kVaapiH264AMDEncoder; @@ -172,6 +169,7 @@ MEDIA_EXPORT extern const base::Feature kVaapiLowPowerEncoder; MEDIA_EXPORT extern const base::Feature kVaapiVP8Encoder; MEDIA_EXPORT extern const base::Feature kVaapiVP9Encoder; MEDIA_EXPORT extern const base::Feature kVideoBlitColorAccuracy; +MEDIA_EXPORT extern const base::Feature kWakeLockOptimisationHiddenMuted; MEDIA_EXPORT extern const base::Feature kWidevineAv1; MEDIA_EXPORT extern const base::Feature kWidevineAv1ForceSupportForTesting; @@ -195,10 +193,11 @@ MEDIA_EXPORT extern const base::Feature kUsePooledSharedImageVideoProvider; #if defined(OS_WIN) MEDIA_EXPORT extern const base::Feature kDelayCopyNV12Textures; -MEDIA_EXPORT extern const base::Feature kMediaFoundationH264Encoding; +MEDIA_EXPORT extern const base::Feature kDirectShowGetPhotoState; +MEDIA_EXPORT extern const base::Feature kMediaFoundationAsyncH264Encoding; +MEDIA_EXPORT extern const base::Feature kMediaFoundationAV1Decoding; MEDIA_EXPORT extern const base::Feature kMediaFoundationVideoCapture; MEDIA_EXPORT extern const base::Feature kMediaFoundationVP8Decoding; -MEDIA_EXPORT extern const base::Feature kDirectShowGetPhotoState; #endif // defined(OS_WIN) // Based on a |command_line| and the current platform, returns the effective diff --git a/chromium/media/base/media_types.h b/chromium/media/base/media_types.h index c7f6f2a5d99..acffbce4486 100644 --- a/chromium/media/base/media_types.h +++ b/chromium/media/base/media_types.h @@ -33,6 +33,7 @@ struct MEDIA_EXPORT VideoType { VideoCodecProfile profile; int level; VideoColorSpace color_space; + HdrMetadataType hdr_metadata_type; }; MEDIA_EXPORT bool operator==(const AudioType& x, const AudioType& y); diff --git a/chromium/media/base/media_url_demuxer.cc b/chromium/media/base/media_url_demuxer.cc index 71e2764ac2c..ce7af50c5a8 100644 --- a/chromium/media/base/media_url_demuxer.cc +++ b/chromium/media/base/media_url_demuxer.cc @@ -82,6 +82,11 @@ int64_t MediaUrlDemuxer::GetMemoryUsage() const { return 0; } +base::Optional<container_names::MediaContainerName> +MediaUrlDemuxer::GetContainerForMetrics() const { + return base::nullopt; +} + void MediaUrlDemuxer::OnEnabledAudioTracksChanged( const std::vector<MediaTrack::Id>& track_ids, base::TimeDelta curr_time, diff --git a/chromium/media/base/media_url_demuxer.h b/chromium/media/base/media_url_demuxer.h index ff0fe02a477..638cc9b14a7 100644 --- a/chromium/media/base/media_url_demuxer.h +++ b/chromium/media/base/media_url_demuxer.h @@ -58,6 +58,8 @@ class MEDIA_EXPORT MediaUrlDemuxer : public Demuxer { base::TimeDelta GetStartTime() const override; base::Time GetTimelineOffset() const override; int64_t GetMemoryUsage() const override; + base::Optional<container_names::MediaContainerName> GetContainerForMetrics() + const override; void OnEnabledAudioTracksChanged(const std::vector<MediaTrack::Id>& track_ids, base::TimeDelta curr_time, TrackChangeCB change_completed_cb) override; diff --git a/chromium/media/base/mime_util_internal.cc b/chromium/media/base/mime_util_internal.cc index 000325d0dd4..30347811de0 100644 --- a/chromium/media/base/mime_util_internal.cc +++ b/chromium/media/base/mime_util_internal.cc @@ -6,6 +6,7 @@ #include "base/command_line.h" #include "base/feature_list.h" +#include "base/logging.h" #include "base/no_destructor.h" #include "base/strings/string_number_conversions.h" #include "base/strings/string_split.h" diff --git a/chromium/media/base/mock_filters.cc b/chromium/media/base/mock_filters.cc index 85c3e6dd11f..e16cd6a65a9 100644 --- a/chromium/media/base/mock_filters.cc +++ b/chromium/media/base/mock_filters.cc @@ -4,7 +4,7 @@ #include "media/base/mock_filters.h" -#include "base/logging.h" +#include "base/check_op.h" using ::testing::_; using ::testing::NiceMock; diff --git a/chromium/media/base/mock_filters.h b/chromium/media/base/mock_filters.h index 36a270f7101..1d900919416 100644 --- a/chromium/media/base/mock_filters.h +++ b/chromium/media/base/mock_filters.h @@ -13,6 +13,7 @@ #include "base/callback.h" #include "base/macros.h" +#include "build/build_config.h" #include "media/base/audio_decoder.h" #include "media/base/audio_decoder_config.h" #include "media/base/audio_parameters.h" @@ -168,6 +169,8 @@ class MockDemuxer : public Demuxer { MOCK_CONST_METHOD0(GetStartTime, base::TimeDelta()); MOCK_CONST_METHOD0(GetTimelineOffset, base::Time()); MOCK_CONST_METHOD0(GetMemoryUsage, int64_t()); + MOCK_CONST_METHOD0(GetContainerForMetrics, + base::Optional<container_names::MediaContainerName>()); MOCK_METHOD3(OnEnabledAudioTracksChanged, void(const std::vector<MediaTrack::Id>&, base::TimeDelta, @@ -523,6 +526,11 @@ class MockCdmContext : public CdmContext { MOCK_METHOD0(GetDecryptor, Decryptor*()); MOCK_METHOD0(RequiresMediaFoundationRenderer, bool()); +#if defined(OS_WIN) + MOCK_METHOD1(GetMediaFoundationCdmProxy, + bool(GetMediaFoundationCdmProxyCB get_mf_cdm_proxy_cb)); +#endif + int GetCdmId() const override; void set_cdm_id(int cdm_id); diff --git a/chromium/media/base/mock_media_log.cc b/chromium/media/base/mock_media_log.cc index 3ae29f64829..0d5729a4e38 100644 --- a/chromium/media/base/mock_media_log.cc +++ b/chromium/media/base/mock_media_log.cc @@ -15,6 +15,7 @@ void MockMediaLog::AddLogRecordLocked(std::unique_ptr<MediaLogRecord> event) { const auto log_string = MediaEventToLogString(*event); VLOG(2) << "MediaLog: " << log_string; DoAddLogRecordLogString(log_string); + most_recent_event_ = std::move(event); } std::string MockMediaLog::MediaEventToLogString(const MediaLogRecord& event) { diff --git a/chromium/media/base/mock_media_log.h b/chromium/media/base/mock_media_log.h index 828960f0171..b568f7e1080 100644 --- a/chromium/media/base/mock_media_log.h +++ b/chromium/media/base/mock_media_log.h @@ -97,7 +97,15 @@ class MockMediaLog : public MediaLog { static std::string MediaEventToLogString(const MediaLogRecord& event); + // Return whatever the last AddLogRecordLocked() was given. + // TODO(tmathmeyer): Remove this in favor of a mock, to allow EXPECT_CALL. + std::unique_ptr<MediaLogRecord> take_most_recent_event() { + return std::move(most_recent_event_); + } + private: + std::unique_ptr<MediaLogRecord> most_recent_event_; + DISALLOW_COPY_AND_ASSIGN(MockMediaLog); }; diff --git a/chromium/media/base/multi_channel_resampler.cc b/chromium/media/base/multi_channel_resampler.cc index ecc1e0b9574..f7e52a70278 100644 --- a/chromium/media/base/multi_channel_resampler.cc +++ b/chromium/media/base/multi_channel_resampler.cc @@ -9,7 +9,7 @@ #include "base/bind.h" #include "base/bind_helpers.h" -#include "base/logging.h" +#include "base/check_op.h" #include "media/base/audio_bus.h" namespace media { diff --git a/chromium/media/base/multi_channel_resampler_unittest.cc b/chromium/media/base/multi_channel_resampler_unittest.cc index df0a8447e93..dae0c72a805 100644 --- a/chromium/media/base/multi_channel_resampler_unittest.cc +++ b/chromium/media/base/multi_channel_resampler_unittest.cc @@ -9,7 +9,6 @@ #include "base/bind.h" #include "base/bind_helpers.h" -#include "base/logging.h" #include "base/macros.h" #include "media/base/audio_bus.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/chromium/media/base/reentrancy_checker_unittest.cc b/chromium/media/base/reentrancy_checker_unittest.cc index 7eff2e7fa65..0f0ae67882f 100644 --- a/chromium/media/base/reentrancy_checker_unittest.cc +++ b/chromium/media/base/reentrancy_checker_unittest.cc @@ -4,7 +4,7 @@ #include "media/base/reentrancy_checker.h" -#include "base/logging.h" +#include "base/check_op.h" #include "base/test/gtest_util.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/chromium/media/base/sample_format.cc b/chromium/media/base/sample_format.cc index 97492ccf418..591572eda5f 100644 --- a/chromium/media/base/sample_format.cc +++ b/chromium/media/base/sample_format.cc @@ -4,7 +4,9 @@ #include "media/base/sample_format.h" -#include "base/logging.h" +#include <ostream> + +#include "base/notreached.h" namespace media { diff --git a/chromium/media/base/sample_rates.cc b/chromium/media/base/sample_rates.cc index 8c2fb931a8a..6af08622e9f 100644 --- a/chromium/media/base/sample_rates.cc +++ b/chromium/media/base/sample_rates.cc @@ -4,7 +4,7 @@ #include "media/base/sample_rates.h" -#include "base/logging.h" +#include "base/check.h" namespace media { diff --git a/chromium/media/base/scopedfd_helper.cc b/chromium/media/base/scopedfd_helper.cc index e506cccc8e1..8dcf3dc1f16 100644 --- a/chromium/media/base/scopedfd_helper.cc +++ b/chromium/media/base/scopedfd_helper.cc @@ -6,7 +6,7 @@ #include <vector> -#include "base/logging.h" +#include "base/check.h" #include "base/posix/eintr_wrapper.h" #include "media/base/scopedfd_helper.h" diff --git a/chromium/media/base/seekable_buffer.cc b/chromium/media/base/seekable_buffer.cc index 1828920a62a..9e8faf6f1e6 100644 --- a/chromium/media/base/seekable_buffer.cc +++ b/chromium/media/base/seekable_buffer.cc @@ -6,7 +6,7 @@ #include <algorithm> -#include "base/logging.h" +#include "base/check_op.h" #include "media/base/data_buffer.h" #include "media/base/timestamp_constants.h" diff --git a/chromium/media/base/seekable_buffer_unittest.cc b/chromium/media/base/seekable_buffer_unittest.cc index e3895620650..9c3a60b585a 100644 --- a/chromium/media/base/seekable_buffer_unittest.cc +++ b/chromium/media/base/seekable_buffer_unittest.cc @@ -9,7 +9,6 @@ #include <cstdlib> -#include "base/logging.h" #include "base/stl_util.h" #include "base/time/time.h" #include "media/base/data_buffer.h" diff --git a/chromium/media/base/silent_sink_suspender.cc b/chromium/media/base/silent_sink_suspender.cc index abcb48cb726..42de9b7cd83 100644 --- a/chromium/media/base/silent_sink_suspender.cc +++ b/chromium/media/base/silent_sink_suspender.cc @@ -14,14 +14,14 @@ SilentSinkSuspender::SilentSinkSuspender( AudioRendererSink::RenderCallback* callback, base::TimeDelta silence_timeout, const AudioParameters& params, - const scoped_refptr<AudioRendererSink>& sink, - const scoped_refptr<base::SingleThreadTaskRunner>& worker) + scoped_refptr<AudioRendererSink> sink, + scoped_refptr<base::SingleThreadTaskRunner> worker) : callback_(callback), params_(params), - sink_(sink), + sink_(std::move(sink)), task_runner_(base::ThreadTaskRunnerHandle::Get()), silence_timeout_(silence_timeout), - fake_sink_(worker, params_), + fake_sink_(std::move(worker), params_), sink_transition_callback_( base::BindRepeating(&SilentSinkSuspender::TransitionSinks, base::Unretained(this))) { @@ -58,11 +58,11 @@ int SilentSinkSuspender::Render(base::TimeDelta delay, // |delay_timestamp| contains the value cached at // |latest_output_delay_timestamp_| // so we simulate the real sink output, promoting |delay_timestamp| with - // |elapsedTime|. + // |elapsed_time|. DCHECK_EQ(delay_timestamp, latest_output_delay_timestamp_); - base::TimeDelta elapsedTime = + base::TimeDelta elapsed_time = base::TimeTicks::Now() - fake_sink_transition_time_; - delay_timestamp += elapsedTime; + delay_timestamp += elapsed_time; // If we have no buffers or a transition is pending, one or more extra // Render() calls have occurred in before TransitionSinks() can run, so we @@ -84,7 +84,7 @@ int SilentSinkSuspender::Render(base::TimeDelta delay, callback_->Render(delay, delay_timestamp, prior_frames_skipped, dest); // Check for silence or real audio data and transition if necessary. - if (!dest->AreFramesZero()) { + if (!dest->AreFramesZero() || !detect_silence_) { first_silence_time_ = base::TimeTicks(); if (is_using_fake_sink_) { is_transition_pending_ = true; @@ -115,6 +115,52 @@ void SilentSinkSuspender::OnRenderError() { callback_->OnRenderError(); } +void SilentSinkSuspender::OnPaused() { + DCHECK(task_runner_->BelongsToCurrentThread()); + + base::AutoLock al(transition_lock_); + + // Nothing to do if we haven't touched the sink. + if (!is_using_fake_sink_ && !is_transition_pending_) { + first_silence_time_ = base::TimeTicks(); + return; + } + + // If we've moved over to the fake sink, we just need to stop it and cancel + // any pending transitions. + if (is_using_fake_sink_) { + is_using_fake_sink_ = false; + fake_sink_.Stop(); + } + + // Cancel any pending transitions. + is_transition_pending_ = false; + first_silence_time_ = base::TimeTicks(); + sink_transition_callback_.Reset(base::BindRepeating( + &SilentSinkSuspender::TransitionSinks, base::Unretained(this))); +} + +void SilentSinkSuspender::SetDetectSilence(bool detect_silence) { + DCHECK(task_runner_->BelongsToCurrentThread()); + + bool should_transition_to_real_sink = false; + { + base::AutoLock lock(transition_lock_); + detect_silence_ = detect_silence; + should_transition_to_real_sink = !detect_silence_ && is_using_fake_sink_; + + // If we haven't started the transition abort it. + if (is_transition_pending_) { + is_transition_pending_ = false; + sink_transition_callback_.Reset(base::BindRepeating( + &SilentSinkSuspender::TransitionSinks, base::Unretained(this))); + } + } + + if (should_transition_to_real_sink) + TransitionSinks(/*use_fake_sink=*/false); +} + bool SilentSinkSuspender::IsUsingFakeSinkForTesting() { base::AutoLock al(transition_lock_); return is_using_fake_sink_; @@ -170,4 +216,4 @@ void SilentSinkSuspender::TransitionSinks(bool use_fake_sink) { } } -} // namespace content +} // namespace media diff --git a/chromium/media/base/silent_sink_suspender.h b/chromium/media/base/silent_sink_suspender.h index d9638651159..52fffadbfe4 100644 --- a/chromium/media/base/silent_sink_suspender.h +++ b/chromium/media/base/silent_sink_suspender.h @@ -29,7 +29,7 @@ namespace media { // Helper class for suspending AudioRenderSink instances after silence has been // detected for some time. When this is detected, the provided |sink_| is paused // and a fake sink is injected to black hole the silent audio data and avoid -// physical hardwasre usage. Note: The transition from real to fake audio output +// physical hardware usage. Note: The transition from real to fake audio output // and vice versa may result in some irregular Render() callbacks. class MEDIA_EXPORT SilentSinkSuspender : public AudioRendererSink::RenderCallback { @@ -38,12 +38,11 @@ class MEDIA_EXPORT SilentSinkSuspender // used to initialize |sink|, |sink| is the sink to monitor for idle, and // |worker| is the task runner to run the fake Render() callbacks on. The // amount of silence to allow before suspension is |silence_timeout|. - SilentSinkSuspender( - AudioRendererSink::RenderCallback* callback, - base::TimeDelta silence_timeout, - const AudioParameters& params, - const scoped_refptr<AudioRendererSink>& sink, - const scoped_refptr<base::SingleThreadTaskRunner>& worker); + SilentSinkSuspender(AudioRendererSink::RenderCallback* callback, + base::TimeDelta silence_timeout, + const AudioParameters& params, + scoped_refptr<AudioRendererSink> sink, + scoped_refptr<base::SingleThreadTaskRunner> worker); ~SilentSinkSuspender() override; // AudioRendererSink::RenderCallback implementation. @@ -53,6 +52,16 @@ class MEDIA_EXPORT SilentSinkSuspender AudioBus* dest) override; void OnRenderError() override; + // Cancels any outstanding callbacks and transitions. Subsequent playback will + // be through the real sink until the suspend conditions are met again. + void OnPaused(); + + // Enables or disables silence detection. If disabled, |silence_timeout| will + // be ignored and we will never transition to the fake sink. If we're already + // on the fake sink when SetDetectSilence(false) is called, we'll transition + // back to the real sink. + void SetDetectSilence(bool detect_silence); + bool IsUsingFakeSinkForTesting(); private: @@ -95,6 +104,10 @@ class MEDIA_EXPORT SilentSinkSuspender // only be used when |transition_lock_| is held or both sinks are stopped. bool is_transition_pending_ GUARDED_BY(transition_lock_) = false; + // Whether we should detect silence and transition to the fake sink or not if + // |silence_timeout_| elapses. + bool detect_silence_ GUARDED_BY(transition_lock_) = true; + // Buffers accumulated during the transition from |fake_sink_| to |sink_|. base::circular_deque<std::unique_ptr<AudioBus>> buffers_after_silence_; @@ -105,15 +118,17 @@ class MEDIA_EXPORT SilentSinkSuspender // Audio output delay at the moment when transition to |fake_sink_| starts. base::TimeDelta latest_output_delay_; + // Audio output delay timestamp at the moment when transition to |fake_sink_| // starts. base::TimeTicks latest_output_delay_timestamp_; + // Time when transition to |fake_sink_| starts. base::TimeTicks fake_sink_transition_time_; DISALLOW_COPY_AND_ASSIGN(SilentSinkSuspender); }; -} // namespace content +} // namespace media #endif // MEDIA_BASE_SILENT_SINK_SUSPENDER_H_ diff --git a/chromium/media/base/silent_sink_suspender_unittest.cc b/chromium/media/base/silent_sink_suspender_unittest.cc index 3d8316bedfc..5ac51cc74a2 100644 --- a/chromium/media/base/silent_sink_suspender_unittest.cc +++ b/chromium/media/base/silent_sink_suspender_unittest.cc @@ -184,4 +184,120 @@ TEST_F(SilentSinkSuspenderTest, MultipleResume) { 0); } +TEST_F(SilentSinkSuspenderTest, SetDetectSilence) { + // Verify a normal Render() doesn't invoke suspend. + EXPECT_FALSE(suspender_.IsUsingFakeSinkForTesting()); + temp_bus_->Zero(); + EXPECT_EQ(temp_bus_->frames(), + suspender_.Render(base::TimeDelta(), base::TimeTicks(), 0, + temp_bus_.get())); + EXPECT_FALSE(temp_bus_->AreFramesZero()); + base::RunLoop().RunUntilIdle(); + EXPECT_FALSE(suspender_.IsUsingFakeSinkForTesting()); + + // Mute all audio generated by the callback, this should suspend immediately. + fake_callback_.set_volume(0); + temp_bus_->Zero(); + EXPECT_EQ(temp_bus_->frames(), + suspender_.Render(base::TimeDelta(), base::TimeTicks(), 0, + temp_bus_.get())); + EXPECT_TRUE(temp_bus_->AreFramesZero()); + { + base::RunLoop run_loop; + EXPECT_CALL(*mock_sink_, Pause()) + .WillOnce(RunClosure(run_loop.QuitClosure())); + run_loop.Run(); + EXPECT_TRUE(suspender_.IsUsingFakeSinkForTesting()); + } + + // Disable silence detection, but don't unmute the audio, the FakeWorker + // inside |suspender_| should be running now, so we don't need to manually + // invoke Render(). + { + base::RunLoop run_loop; + EXPECT_CALL(*mock_sink_, Play()) + .WillOnce(RunClosure(run_loop.QuitClosure())); + + // Disable silence detection which should put us back on the real sink. + suspender_.SetDetectSilence(false); + + run_loop.Run(); + EXPECT_FALSE(suspender_.IsUsingFakeSinkForTesting()); + } + + // Run one more Render() to ensure we stay on the real sink even w/ silent + // audio. + temp_bus_->Zero(); + EXPECT_EQ(temp_bus_->frames(), + suspender_.Render(base::TimeDelta(), base::TimeTicks(), 0, + temp_bus_.get())); + + base::RunLoop().RunUntilIdle(); + EXPECT_FALSE(suspender_.IsUsingFakeSinkForTesting()); + + // Enable silence detection which should transition to the fake sink on the + // next Render(). + suspender_.SetDetectSilence(true); + temp_bus_->Zero(); + EXPECT_EQ(temp_bus_->frames(), + suspender_.Render(base::TimeDelta(), base::TimeTicks(), 0, + temp_bus_.get())); + EXPECT_TRUE(temp_bus_->AreFramesZero()); + { + base::RunLoop run_loop; + EXPECT_CALL(*mock_sink_, Pause()) + .WillOnce(RunClosure(run_loop.QuitClosure())); + run_loop.Run(); + EXPECT_TRUE(suspender_.IsUsingFakeSinkForTesting()); + } +} + +TEST_F(SilentSinkSuspenderTest, OnPaused) { + // Verify a normal Render() doesn't invoke suspend. + EXPECT_FALSE(suspender_.IsUsingFakeSinkForTesting()); + temp_bus_->Zero(); + EXPECT_EQ(temp_bus_->frames(), + suspender_.Render(base::TimeDelta(), base::TimeTicks(), 0, + temp_bus_.get())); + EXPECT_FALSE(temp_bus_->AreFramesZero()); + base::RunLoop().RunUntilIdle(); + EXPECT_FALSE(suspender_.IsUsingFakeSinkForTesting()); + + // OnPaused() at this point should do nothing, since we're on the real sink. + suspender_.OnPaused(); + + // Mute all audio generated by the callback, this should suspend immediately. + fake_callback_.set_volume(0); + temp_bus_->Zero(); + EXPECT_EQ(temp_bus_->frames(), + suspender_.Render(base::TimeDelta(), base::TimeTicks(), 0, + temp_bus_.get())); + EXPECT_TRUE(temp_bus_->AreFramesZero()); + { + base::RunLoop run_loop; + EXPECT_CALL(*mock_sink_, Pause()) + .WillOnce(RunClosure(run_loop.QuitClosure())); + run_loop.Run(); + EXPECT_TRUE(suspender_.IsUsingFakeSinkForTesting()); + } + + // Call OnPaused(), which should disable the fake sink, but won't call Play(). + suspender_.OnPaused(); + base::RunLoop().RunUntilIdle(); + EXPECT_FALSE(suspender_.IsUsingFakeSinkForTesting()); + + // Render silence again, which should attempt to transition to the fake sink. + temp_bus_->Zero(); + EXPECT_EQ(temp_bus_->frames(), + suspender_.Render(base::TimeDelta(), base::TimeTicks(), 0, + temp_bus_.get())); + + // OnPaused() should cancel any pending transitions. + suspender_.OnPaused(); + + // We should not transition to the fake sink now. + base::RunLoop().RunUntilIdle(); + EXPECT_FALSE(suspender_.IsUsingFakeSinkForTesting()); +} + } // namespace media diff --git a/chromium/media/base/sinc_resampler.cc b/chromium/media/base/sinc_resampler.cc index d9965ca9ec2..79b5e2fbe12 100644 --- a/chromium/media/base/sinc_resampler.cc +++ b/chromium/media/base/sinc_resampler.cc @@ -77,7 +77,7 @@ #include <limits> -#include "base/logging.h" +#include "base/check_op.h" #include "base/numerics/math_constants.h" #include "build/build_config.h" #include "cc/base/math_util.h" diff --git a/chromium/media/base/status_codes.h b/chromium/media/base/status_codes.h index 3f161c20316..468ff813946 100644 --- a/chromium/media/base/status_codes.h +++ b/chromium/media/base/status_codes.h @@ -44,11 +44,15 @@ enum class StatusCode : StatusCodeType { // Windows Errors: 0x02 kWindowsWrappedHresult = 0x00000201, kWindowsApiNotAvailible = 0x00000202, + kWindowsD3D11Error = 0x00000203, // D3D11VideoDecoder Errors: 0x03 kCantMakeContextCurrent = 0x00000301, kCantPostTexture = 0x00000302, kCantPostAcquireStream = 0x00000303, + kCantCreateEglStream = 0x00000304, + kCantCreateEglStreamConsumer = 0x00000305, + kCantCreateEglStreamProducer = 0x00000306, // MojoDecoder Errors: 0x04 kMojoDecoderNoWrappedDecoder = 0x00000401, @@ -68,6 +72,15 @@ enum class StatusCode : StatusCodeType { kVaapiReinitializedDuringDecode = 0x00000508, kVaapiFailedAcceleratorCreation = 0x00000509, + // Encoder Error: 0x06 + kEncoderInitializeNeverCompleted = 0x00000601, + kEncoderInitializeTwice = 0x00000602, + kEncoderFailedEncode = 0x00000603, + kEncoderUnsupportedProfile = 0x00000604, + kEncoderUnsupportedCodec = 0x00000605, + kEncoderUnsupportedConfig = 0x00000606, + kEncoderInitializationError = 0x00000607, + // Special codes kGenericErrorPleaseRemove = 0x79999999, kCodeOnlyForTesting = std::numeric_limits<StatusCodeType>::max(), diff --git a/chromium/media/base/stream_parser_buffer.cc b/chromium/media/base/stream_parser_buffer.cc index 5bff35f2ede..4ff451afb8d 100644 --- a/chromium/media/base/stream_parser_buffer.cc +++ b/chromium/media/base/stream_parser_buffer.cc @@ -6,7 +6,7 @@ #include <algorithm> -#include "base/logging.h" +#include "base/check_op.h" #include "base/memory/ptr_util.h" #include "media/base/timestamp_constants.h" diff --git a/chromium/media/base/supported_types.cc b/chromium/media/base/supported_types.cc index 19266cf96d8..c98ce24e557 100644 --- a/chromium/media/base/supported_types.cc +++ b/chromium/media/base/supported_types.cc @@ -5,6 +5,7 @@ #include "media/base/supported_types.h" #include "base/feature_list.h" +#include "base/logging.h" #include "base/no_destructor.h" #include "build/build_config.h" #include "media/base/media.h" @@ -31,6 +32,25 @@ namespace media { +namespace { + +bool IsSupportedHdrMetadata(const HdrMetadataType& hdr_metadata_type) { + switch (hdr_metadata_type) { + case HdrMetadataType::kNone: + return true; + + case HdrMetadataType::kSmpteSt2086: + case HdrMetadataType::kSmpteSt2094_10: + case HdrMetadataType::kSmpteSt2094_40: + return false; + } + + NOTREACHED(); + return false; +} + +} // namespace + bool IsSupportedAudioType(const AudioType& type) { MediaClient* media_client = GetMediaClient(); if (media_client) @@ -182,6 +202,9 @@ bool IsAudioCodecProprietary(AudioCodec codec) { case kUnknownAudioCodec: return false; } + + NOTREACHED(); + return false; } bool IsDefaultSupportedAudioType(const AudioType& type) { @@ -252,11 +275,17 @@ bool IsVideoCodecProprietary(VideoCodec codec) { case kCodecAV1: return false; } + + NOTREACHED(); + return false; } // TODO(chcunningham): Add platform specific logic for Android (move from // MimeUtilIntenral). bool IsDefaultSupportedVideoType(const VideoType& type) { + if (!IsSupportedHdrMetadata(type.hdr_metadata_type)) + return false; + #if !BUILDFLAG(USE_PROPRIETARY_CODECS) if (IsVideoCodecProprietary(type.codec)) return false; diff --git a/chromium/media/base/supported_types_unittest.cc b/chromium/media/base/supported_types_unittest.cc index 764561440d6..48013e18e31 100644 --- a/chromium/media/base/supported_types_unittest.cc +++ b/chromium/media/base/supported_types_unittest.cc @@ -240,4 +240,58 @@ TEST(SupportedTypesTest, XHE_AACSupportedOnAndroidOnly) { #endif } +TEST(SupportedTypesTest, IsSupportedVideoTypeWithHdrMetadataBasics) { + // Default to common 709. + media::VideoColorSpace color_space = media::VideoColorSpace::REC709(); + + // Some codecs do not have a notion of level. + const int kUnspecifiedLevel = 0; + + // Expect support for baseline configuration of known codecs. + EXPECT_TRUE(IsSupportedVideoType({media::kCodecVP8, media::VP8PROFILE_ANY, + kUnspecifiedLevel, color_space})); + EXPECT_TRUE( + IsSupportedVideoType({media::kCodecVP9, media::VP9PROFILE_PROFILE0, + kUnspecifiedLevel, color_space})); + EXPECT_TRUE(IsSupportedVideoType({media::kCodecTheora, + media::VIDEO_CODEC_PROFILE_UNKNOWN, + kUnspecifiedLevel, color_space})); + + // All combinations of combinations of color gamuts and transfer functions + // should be supported. + color_space.primaries = media::VideoColorSpace::PrimaryID::SMPTEST431_2; + color_space.transfer = media::VideoColorSpace::TransferID::SMPTEST2084; + EXPECT_TRUE(IsSupportedVideoType({media::kCodecVP8, media::VP8PROFILE_ANY, + kUnspecifiedLevel, color_space})); + EXPECT_TRUE( + IsSupportedVideoType({media::kCodecVP9, media::VP9PROFILE_PROFILE0, + kUnspecifiedLevel, color_space})); + EXPECT_TRUE(IsSupportedVideoType({media::kCodecTheora, + media::VIDEO_CODEC_PROFILE_UNKNOWN, + kUnspecifiedLevel, color_space})); + + color_space.primaries = media::VideoColorSpace::PrimaryID::BT2020; + color_space.transfer = media::VideoColorSpace::TransferID::ARIB_STD_B67; + EXPECT_TRUE(IsSupportedVideoType({media::kCodecVP8, media::VP8PROFILE_ANY, + kUnspecifiedLevel, color_space})); + EXPECT_TRUE( + IsSupportedVideoType({media::kCodecVP9, media::VP9PROFILE_PROFILE0, + kUnspecifiedLevel, color_space})); + EXPECT_TRUE(IsSupportedVideoType({media::kCodecTheora, + media::VIDEO_CODEC_PROFILE_UNKNOWN, + kUnspecifiedLevel, color_space})); + + // No HDR metadata types are supported. + EXPECT_FALSE(IsSupportedVideoType({media::kCodecVP8, media::VP8PROFILE_ANY, + kUnspecifiedLevel, color_space, + media::HdrMetadataType::kSmpteSt2086})); + + EXPECT_FALSE(IsSupportedVideoType({media::kCodecVP8, media::VP8PROFILE_ANY, + kUnspecifiedLevel, color_space, + media::HdrMetadataType::kSmpteSt2094_10})); + + EXPECT_FALSE(IsSupportedVideoType({media::kCodecVP8, media::VP8PROFILE_ANY, + kUnspecifiedLevel, color_space, + media::HdrMetadataType::kSmpteSt2094_40})); +} } // namespace media diff --git a/chromium/media/base/test_data_util.cc b/chromium/media/base/test_data_util.cc index d849db56af1..fe6691abb84 100644 --- a/chromium/media/base/test_data_util.cc +++ b/chromium/media/base/test_data_util.cc @@ -6,9 +6,9 @@ #include <stdint.h> +#include "base/check_op.h" #include "base/containers/flat_map.h" #include "base/files/file_util.h" -#include "base/logging.h" #include "base/no_destructor.h" #include "base/numerics/safe_conversions.h" #include "base/path_service.h" diff --git a/chromium/media/base/test_helpers.cc b/chromium/media/base/test_helpers.cc index c815328ee39..140bcba4fdf 100644 --- a/chromium/media/base/test_helpers.cc +++ b/chromium/media/base/test_helpers.cc @@ -7,8 +7,9 @@ #include <stdint.h> #include "base/bind.h" -#include "base/logging.h" +#include "base/check_op.h" #include "base/macros.h" +#include "base/notreached.h" #include "base/pickle.h" #include "base/run_loop.h" #include "base/test/test_timeouts.h" diff --git a/chromium/media/base/test_helpers.h b/chromium/media/base/test_helpers.h index 8e97cdc778d..bffe5960363 100644 --- a/chromium/media/base/test_helpers.h +++ b/chromium/media/base/test_helpers.h @@ -16,6 +16,7 @@ #include "base/strings/stringprintf.h" #include "media/base/audio_parameters.h" #include "media/base/channel_layout.h" +#include "media/base/demuxer_stream.h" #include "media/base/media_log.h" #include "media/base/pipeline_status.h" #include "media/base/sample_format.h" diff --git a/chromium/media/base/text_ranges.cc b/chromium/media/base/text_ranges.cc index a88cc90c076..c57dc65529e 100644 --- a/chromium/media/base/text_ranges.cc +++ b/chromium/media/base/text_ranges.cc @@ -4,7 +4,7 @@ #include "media/base/text_ranges.h" -#include "base/logging.h" +#include "base/check_op.h" namespace media { diff --git a/chromium/media/base/text_renderer.cc b/chromium/media/base/text_renderer.cc index aa1e5a34d1e..06c18f6a9d6 100644 --- a/chromium/media/base/text_renderer.cc +++ b/chromium/media/base/text_renderer.cc @@ -10,7 +10,8 @@ #include "base/bind.h" #include "base/callback_helpers.h" -#include "base/logging.h" +#include "base/check_op.h" +#include "base/notreached.h" #include "base/single_thread_task_runner.h" #include "media/base/bind_to_current_loop.h" #include "media/base/decoder_buffer.h" diff --git a/chromium/media/base/time_delta_interpolator.cc b/chromium/media/base/time_delta_interpolator.cc index cbfe38d8765..33e06a6f0be 100644 --- a/chromium/media/base/time_delta_interpolator.cc +++ b/chromium/media/base/time_delta_interpolator.cc @@ -8,7 +8,7 @@ #include <algorithm> -#include "base/logging.h" +#include "base/check.h" #include "base/time/tick_clock.h" #include "media/base/timestamp_constants.h" diff --git a/chromium/media/base/time_delta_interpolator_unittest.cc b/chromium/media/base/time_delta_interpolator_unittest.cc index 8f7122b46e0..540768ee45b 100644 --- a/chromium/media/base/time_delta_interpolator_unittest.cc +++ b/chromium/media/base/time_delta_interpolator_unittest.cc @@ -2,7 +2,6 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "base/logging.h" #include "base/test/simple_test_tick_clock.h" #include "media/base/time_delta_interpolator.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/chromium/media/base/unaligned_shared_memory_unittest.cc b/chromium/media/base/unaligned_shared_memory_unittest.cc index d9891a89509..8af5cf69715 100644 --- a/chromium/media/base/unaligned_shared_memory_unittest.cc +++ b/chromium/media/base/unaligned_shared_memory_unittest.cc @@ -9,7 +9,6 @@ #include <limits> -#include "base/logging.h" #include "base/stl_util.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/chromium/media/base/user_input_monitor_linux.cc b/chromium/media/base/user_input_monitor_linux.cc index b52df992d79..00d818f1f68 100644 --- a/chromium/media/base/user_input_monitor_linux.cc +++ b/chromium/media/base/user_input_monitor_linux.cc @@ -120,21 +120,21 @@ uint32_t UserInputMonitorLinuxCore::GetKeyPressCount() const { void UserInputMonitorLinuxCore::StartMonitor() { DCHECK(io_task_runner_->BelongsToCurrentThread()); - // TODO(jamiewalch): We should pass the display in. At that point, since - // XRecord needs a private connection to the X Server for its data channel - // and both channels are used from a separate thread, we'll need to duplicate - // them with something like the following: - // XOpenDisplay(DisplayString(display)); - if (!x_control_display_) - x_control_display_ = gfx::OpenNewXDisplay(); - - if (!x_record_display_) - x_record_display_ = gfx::OpenNewXDisplay(); - if (!x_control_display_ || !x_record_display_) { - LOG(ERROR) << "Couldn't open X display"; - StopMonitor(); - return; + // TODO(jamiewalch): We should pass the display in. + if (auto* x_display = gfx::GetXDisplay()) { + if (!x_control_display_) + x_control_display_ = gfx::CloneXDisplay(x_display); + + if (!x_record_display_) + x_record_display_ = gfx::CloneXDisplay(x_display); + } + + if (!x_control_display_ || !x_record_display_) { + LOG(ERROR) << "Couldn't open X display"; + StopMonitor(); + return; + } } int xr_opcode, xr_event, xr_error; diff --git a/chromium/media/base/vector_math.cc b/chromium/media/base/vector_math.cc index e6e26922249..fabb796d324 100644 --- a/chromium/media/base/vector_math.cc +++ b/chromium/media/base/vector_math.cc @@ -7,7 +7,7 @@ #include <algorithm> -#include "base/logging.h" +#include "base/check_op.h" #include "build/build_config.h" // NaCl does not allow intrinsics. diff --git a/chromium/media/base/video_bitrate_allocation.cc b/chromium/media/base/video_bitrate_allocation.cc index 8d270b866cb..d9a286dd649 100644 --- a/chromium/media/base/video_bitrate_allocation.cc +++ b/chromium/media/base/video_bitrate_allocation.cc @@ -4,10 +4,11 @@ #include "video_bitrate_allocation.h" +#include <cstring> #include <limits> #include <numeric> -#include "base/logging.h" +#include "base/check_op.h" #include "base/numerics/checked_math.h" namespace media { diff --git a/chromium/media/base/video_bitrate_allocation_unittest.cc b/chromium/media/base/video_bitrate_allocation_unittest.cc index df8b3ae43ea..32dffd3cc2d 100644 --- a/chromium/media/base/video_bitrate_allocation_unittest.cc +++ b/chromium/media/base/video_bitrate_allocation_unittest.cc @@ -4,7 +4,6 @@ #include <set> -#include "base/logging.h" #include "media/base/video_bitrate_allocation.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/chromium/media/base/video_codecs_unittest.cc b/chromium/media/base/video_codecs_unittest.cc index c6d9484e3c8..c5ac4c592ae 100644 --- a/chromium/media/base/video_codecs_unittest.cc +++ b/chromium/media/base/video_codecs_unittest.cc @@ -4,7 +4,6 @@ #include <set> -#include "base/logging.h" #include "base/strings/stringprintf.h" #include "media/base/video_codecs.h" #include "media/base/video_color_space.h" diff --git a/chromium/media/base/video_color_space_unittest.cc b/chromium/media/base/video_color_space_unittest.cc index c49d08bccb6..580fe15f5ed 100644 --- a/chromium/media/base/video_color_space_unittest.cc +++ b/chromium/media/base/video_color_space_unittest.cc @@ -3,7 +3,6 @@ // found in the LICENSE file. #include "media/base/video_color_space.h" -#include "base/logging.h" #include "testing/gtest/include/gtest/gtest.h" #include "ui/gfx/color_space.h" #include "ui/gfx/color_transform.h" diff --git a/chromium/media/base/video_decoder_config.cc b/chromium/media/base/video_decoder_config.cc index e9c54664dfc..f59b7394a03 100644 --- a/chromium/media/base/video_decoder_config.cc +++ b/chromium/media/base/video_decoder_config.cc @@ -7,7 +7,8 @@ #include <iomanip> #include <vector> -#include "base/logging.h" +#include "base/check_op.h" +#include "base/notreached.h" #include "base/strings/string_number_conversions.h" #include "media/base/limits.h" #include "media/base/media_util.h" diff --git a/chromium/media/base/video_encoder.cc b/chromium/media/base/video_encoder.cc new file mode 100644 index 00000000000..2be81f8e9b2 --- /dev/null +++ b/chromium/media/base/video_encoder.cc @@ -0,0 +1,21 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/base/video_encoder.h" + +#include "media/base/video_frame.h" + +namespace media { + +VideoEncoderOutput::VideoEncoderOutput() = default; +VideoEncoderOutput::VideoEncoderOutput(VideoEncoderOutput&&) = default; +VideoEncoderOutput::~VideoEncoderOutput() = default; + +VideoEncoder::VideoEncoder() = default; +VideoEncoder::~VideoEncoder() = default; + +VideoEncoder::Options::Options() = default; +VideoEncoder::Options::~Options() = default; + +} // namespace media
\ No newline at end of file diff --git a/chromium/media/base/video_encoder.h b/chromium/media/base/video_encoder.h new file mode 100644 index 00000000000..90a15478122 --- /dev/null +++ b/chromium/media/base/video_encoder.h @@ -0,0 +1,107 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_BASE_VIDEO_ENCODER_H_ +#define MEDIA_BASE_VIDEO_ENCODER_H_ + +#include "base/callback.h" +#include "base/macros.h" +#include "base/optional.h" +#include "media/base/media_export.h" +#include "media/base/status.h" +#include "media/base/video_codecs.h" +#include "ui/gfx/geometry/size.h" + +namespace media { + +class VideoFrame; + +// Encoded video frame, its data and metadata. +struct MEDIA_EXPORT VideoEncoderOutput { + VideoEncoderOutput(); + VideoEncoderOutput(VideoEncoderOutput&&); + ~VideoEncoderOutput(); + + // Feel free take this buffer out and use underlying memory as is without + // copying. + std::unique_ptr<uint8_t[]> data; + size_t size; + + base::TimeDelta timestamp; + bool key_frame; +}; + +class MEDIA_EXPORT VideoEncoder { + public: + struct MEDIA_EXPORT Options { + Options(); + ~Options(); + base::Optional<uint64_t> bitrate; + double framerate = 30.0; + + int width = 0; + int height = 0; + + base::Optional<int> threads; + base::Optional<int> keyframe_interval = 10000; + }; + + // Callback for VideoEncoder to report an encoded video frame whenever it + // becomes available. + using OutputCB = base::RepeatingCallback<void(VideoEncoderOutput output)>; + + // Callback to report success and errors in encoder calls. + using StatusCB = base::OnceCallback<void(Status error)>; + + VideoEncoder(); + virtual ~VideoEncoder(); + + // Initializes a VideoEncoder with the given |options|, executing the + // |done_cb| upon completion. |output_cb| is called for each encoded frame + // produced by the coder. + // + // Note: + // 1) Can't be called more than once for the same instance of the encoder. + // 2) No VideoEncoder calls should be made before |done_cb| is executed. + virtual void Initialize(VideoCodecProfile profile, + const Options& options, + OutputCB output_cb, + StatusCB done_cb) = 0; + + // Requests a |frame| to be encoded. The status of the encoder and the frame + // are returned via the provided callback |done_cb|. + // + // |done_cb| will not be called from within this method, and that it will be + // called even if Encode() is never called again. + + // After the frame, or several frames, are encoded the encoder calls + // |output_cb| specified in Initialize() for available VideoEncoderOutput. + // |output_cb| may be called before or after |done_cb|, + // including before Encode() returns. + // Encode() does not expect EOS frames, use Flush() to finalize the stream + // and harvest the outputs. + virtual void Encode(scoped_refptr<const VideoFrame> frame, + bool key_frame, + StatusCB done_cb) = 0; + + // Adjust encoder options for future frames, executing the + // |done_cb| upon completion. + // + // Note: + // 1. Not all options can be changed on the fly. + // 2. ChangeOptions() should be called after calling Flush() and waiting + // for it to finish. + virtual void ChangeOptions(const Options& options, StatusCB done_cb) = 0; + + // Requests all outputs for already encoded frames to be + // produced via |output_cb| and calls |dene_cb| after that. + virtual void Flush(StatusCB done_cb) = 0; + + protected: + DISALLOW_COPY_AND_ASSIGN(VideoEncoder); +}; + +} // namespace media + +#endif // MEDIA_BASE_VIDEO_ENCODER_H_ diff --git a/chromium/media/base/video_frame.h b/chromium/media/base/video_frame.h index 6b1a5b455a7..271fcdf19fd 100644 --- a/chromium/media/base/video_frame.h +++ b/chromium/media/base/video_frame.h @@ -27,6 +27,7 @@ #include "build/build_config.h" #include "gpu/command_buffer/common/mailbox_holder.h" #include "gpu/ipc/common/vulkan_ycbcr_info.h" +#include "media/base/hdr_metadata.h" #include "media/base/video_frame_layout.h" #include "media/base/video_frame_metadata.h" #include "media/base/video_types.h" @@ -417,6 +418,14 @@ class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> { color_space_ = color_space; } + const base::Optional<HDRMetadata>& hdr_metadata() const { + return hdr_metadata_; + } + + void set_hdr_metadata(const base::Optional<HDRMetadata>& hdr_metadata) { + hdr_metadata_ = hdr_metadata; + } + const VideoFrameLayout& layout() const { return layout_; } VideoPixelFormat format() const { return layout_.format(); } @@ -461,7 +470,7 @@ class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> { } const base::Optional<gpu::VulkanYCbCrInfo>& ycbcr_info() const { - return ycbcr_info_; + return wrapped_frame_ ? wrapped_frame_->ycbcr_info() : ycbcr_info_; } // Returns pointer to the data in the visible region of the frame, for @@ -686,6 +695,7 @@ class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> { const int unique_id_; gfx::ColorSpace color_space_; + base::Optional<HDRMetadata> hdr_metadata_; // Sampler conversion information which is used in vulkan context for android. base::Optional<gpu::VulkanYCbCrInfo> ycbcr_info_; diff --git a/chromium/media/base/video_frame_layout.cc b/chromium/media/base/video_frame_layout.cc index b7210ce1126..9ded967aecc 100644 --- a/chromium/media/base/video_frame_layout.cc +++ b/chromium/media/base/video_frame_layout.cc @@ -8,7 +8,7 @@ #include <numeric> #include <sstream> -#include "base/logging.h" +#include "base/notreached.h" namespace media { diff --git a/chromium/media/base/video_frame_metadata.cc b/chromium/media/base/video_frame_metadata.cc index 1b403379bd0..ad45ea010c9 100644 --- a/chromium/media/base/video_frame_metadata.cc +++ b/chromium/media/base/video_frame_metadata.cc @@ -6,8 +6,10 @@ #include <stdint.h> #include <utility> +#include <vector> -#include "base/logging.h" +#include "base/check_op.h" +#include "base/no_destructor.h" #include "base/strings/string_number_conversions.h" #include "base/value_conversions.h" #include "ui/gfx/geometry/rect.h" @@ -16,10 +18,19 @@ namespace media { namespace { -// Map enum key to internal std::string key used by base::DictionaryValue. -inline std::string ToInternalKey(VideoFrameMetadata::Key key) { +std::vector<std::string> CreateInternalKeys() { + std::vector<std::string> result(VideoFrameMetadata::NUM_KEYS); + for (size_t i = 0; i < result.size(); i++) + result[i] = base::NumberToString(i); + return result; +} + +// Map enum key to internal StringPiece key used by base::DictionaryValue. +inline base::StringPiece ToInternalKey(VideoFrameMetadata::Key key) { DCHECK_LT(key, VideoFrameMetadata::NUM_KEYS); - return base::NumberToString(static_cast<int>(key)); + static const base::NoDestructor<std::vector<std::string>> internal_keys( + CreateInternalKeys()); + return (*internal_keys)[int{key}]; } } // namespace diff --git a/chromium/media/base/video_transformation.cc b/chromium/media/base/video_transformation.cc index 4ba3658ffae..8b107b940db 100644 --- a/chromium/media/base/video_transformation.cc +++ b/chromium/media/base/video_transformation.cc @@ -7,7 +7,7 @@ #include <math.h> #include <stddef.h> -#include "base/logging.h" +#include "base/notreached.h" namespace media { namespace { diff --git a/chromium/media/base/video_types.cc b/chromium/media/base/video_types.cc index 6f23ea771a0..6814a364db1 100644 --- a/chromium/media/base/video_types.cc +++ b/chromium/media/base/video_types.cc @@ -4,7 +4,9 @@ #include "media/base/video_types.h" -#include "base/logging.h" +#include <ostream> + +#include "base/notreached.h" #include "base/strings/stringprintf.h" namespace media { diff --git a/chromium/media/base/video_util.cc b/chromium/media/base/video_util.cc index bc4c55619d3..3daa1cb27d5 100644 --- a/chromium/media/base/video_util.cc +++ b/chromium/media/base/video_util.cc @@ -7,7 +7,8 @@ #include <cmath> #include "base/bind.h" -#include "base/logging.h" +#include "base/check_op.h" +#include "base/notreached.h" #include "base/numerics/safe_conversions.h" #include "base/numerics/safe_math.h" #include "media/base/video_frame.h" diff --git a/chromium/media/base/watch_time_keys.cc b/chromium/media/base/watch_time_keys.cc index 90f301cb55e..c0480b51303 100644 --- a/chromium/media/base/watch_time_keys.cc +++ b/chromium/media/base/watch_time_keys.cc @@ -4,6 +4,8 @@ #include "media/base/watch_time_keys.h" +#include "base/notreached.h" + namespace media { // TODO(dalecurtis): Key strings aren't really necessary anymore, so instead diff --git a/chromium/media/base/win/BUILD.gn b/chromium/media/base/win/BUILD.gn index c4d20d2fed4..b59888ffc6c 100644 --- a/chromium/media/base/win/BUILD.gn +++ b/chromium/media/base/win/BUILD.gn @@ -49,6 +49,14 @@ source_set("d3d11") { deps = [ "//base" ] } +source_set("hresult_status_helper") { + sources = [ + "hresult_status_helper.cc", + "hresult_status_helper.h", + ] + deps = [ "//media" ] +} + source_set("d3d11_test_support") { testonly = true sources = [ diff --git a/chromium/media/base/win/hresult_status_helper.cc b/chromium/media/base/win/hresult_status_helper.cc new file mode 100644 index 00000000000..e3d6a43ebc0 --- /dev/null +++ b/chromium/media/base/win/hresult_status_helper.cc @@ -0,0 +1,22 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/base/win/hresult_status_helper.h" + +#include "base/logging.h" + +namespace media { + +Status HresultToStatus(HRESULT hresult, + const char* message, + StatusCode code, + const base::Location& location) { + if (SUCCEEDED(hresult)) + return OkStatus(); + + return Status(code, message == nullptr ? "HRESULT" : message, location) + .WithData("value", logging::SystemErrorCodeToString(hresult)); +} + +} // namespace media diff --git a/chromium/media/base/win/hresult_status_helper.h b/chromium/media/base/win/hresult_status_helper.h new file mode 100644 index 00000000000..67ffe3f8943 --- /dev/null +++ b/chromium/media/base/win/hresult_status_helper.h @@ -0,0 +1,25 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_BASE_WIN_HRESULT_STATUS_HELPER_H_ +#define MEDIA_BASE_WIN_HRESULT_STATUS_HELPER_H_ + +#include <wrl/client.h> + +#include "media/base/status.h" + +namespace media { + +// Generate a status from an HRESULT. If message is provided, it will add the +// HRESULT hex value as a data value to the status. Otherwise, the hex value +// will be included in the error message itself. +Status HresultToStatus( + HRESULT hresult, + const char* message = nullptr, + StatusCode code = StatusCode::kWindowsWrappedHresult, + const base::Location& location = base::Location::Current()); + +} // namespace media + +#endif // MEDIA_BASE_WIN_HRESULT_STATUS_HELPER_H_ diff --git a/chromium/media/base/win/mf_helpers.h b/chromium/media/base/win/mf_helpers.h index ac163084247..11c21881392 100644 --- a/chromium/media/base/win/mf_helpers.h +++ b/chromium/media/base/win/mf_helpers.h @@ -15,6 +15,12 @@ namespace media { +// Helper function to print HRESULT to std::string. +const auto PrintHr = logging::SystemErrorCodeToString; + +// Helper macro for DVLOG with function name and this pointer. +#define DVLOG_FUNC(level) DVLOG(level) << __func__ << ": (" << this << ") " + // Macros that contain return statements can make code harder to read. Only use // these when necessary, e.g. in places where we deal with a lot of Windows API // calls, for each of which we have to check the returned HRESULT. @@ -25,8 +31,8 @@ namespace media { do { \ HRESULT hresult = (expr); \ if (FAILED(hresult)) { \ - DLOG(ERROR) << __func__ << ": failed with \"" \ - << logging::SystemErrorCodeToString(hresult) << "\""; \ + DLOG(ERROR) << __func__ << ": failed with \"" << PrintHr(hresult) \ + << "\""; \ return hresult; \ } \ } while (0) @@ -39,10 +45,8 @@ namespace media { } \ } while (0) -#define RETURN_ON_HR_FAILURE(hresult, log, ret) \ - RETURN_ON_FAILURE(SUCCEEDED(hresult), \ - log << ", " << logging::SystemErrorCodeToString(hresult), \ - ret); +#define RETURN_ON_HR_FAILURE(hresult, log, ret) \ + RETURN_ON_FAILURE(SUCCEEDED(hresult), log << ", " << PrintHr(hresult), ret); // Creates a Media Foundation sample with one buffer of length |buffer_length| // on a |align|-byte boundary. Alignment must be a perfect power of 2 or 0. @@ -53,7 +57,7 @@ CreateEmptySampleWithBuffer(uint32_t buffer_length, int align); // instance. class MF_INITIALIZER_EXPORT MediaBufferScopedPointer { public: - MediaBufferScopedPointer(IMFMediaBuffer* media_buffer); + explicit MediaBufferScopedPointer(IMFMediaBuffer* media_buffer); ~MediaBufferScopedPointer(); uint8_t* get() { return buffer_; } @@ -72,7 +76,6 @@ class MF_INITIALIZER_EXPORT MediaBufferScopedPointer { class MF_INITIALIZER_EXPORT DXGIDeviceScopedHandle { public: explicit DXGIDeviceScopedHandle(IMFDXGIDeviceManager* device_manager); - ~DXGIDeviceScopedHandle(); HRESULT LockDevice(REFIID riid, void** device_out); diff --git a/chromium/media/base/win/mf_initializer.h b/chromium/media/base/win/mf_initializer.h index f1e1bd06a6c..2f8431b65bc 100644 --- a/chromium/media/base/win/mf_initializer.h +++ b/chromium/media/base/win/mf_initializer.h @@ -7,6 +7,8 @@ #include <mfapi.h> +#include <memory> + #include "base/logging.h" #include "media/base/win/mf_initializer_export.h" diff --git a/chromium/media/blink/cdm_result_promise_helper.cc b/chromium/media/blink/cdm_result_promise_helper.cc index a7999788c98..4e00f2bda04 100644 --- a/chromium/media/blink/cdm_result_promise_helper.cc +++ b/chromium/media/blink/cdm_result_promise_helper.cc @@ -4,8 +4,8 @@ #include "media/blink/cdm_result_promise_helper.h" -#include "base/logging.h" #include "base/metrics/histogram_functions.h" +#include "base/notreached.h" namespace media { diff --git a/chromium/media/blink/interval_map_unittest.cc b/chromium/media/blink/interval_map_unittest.cc index 2d10e2cdbce..fc53afd8ca5 100644 --- a/chromium/media/blink/interval_map_unittest.cc +++ b/chromium/media/blink/interval_map_unittest.cc @@ -6,6 +6,7 @@ #include <string> +#include "base/logging.h" #include "base/strings/stringprintf.h" #include "media/base/test_random.h" #include "media/blink/interval_map.h" diff --git a/chromium/media/blink/run_all_unittests.cc b/chromium/media/blink/run_all_unittests.cc index fddb3fed104..fa3e4ed6084 100644 --- a/chromium/media/blink/run_all_unittests.cc +++ b/chromium/media/blink/run_all_unittests.cc @@ -10,6 +10,7 @@ #include "media/base/media.h" #include "media/blink/blink_platform_with_task_environment.h" #include "mojo/public/cpp/bindings/binder_map.h" +#include "third_party/blink/public/platform/platform.h" #include "third_party/blink/public/platform/scheduler/test/renderer_scheduler_test_support.h" #include "third_party/blink/public/web/blink.h" @@ -60,6 +61,7 @@ class MediaBlinkTestSuite : public base::TestSuite { mojo::core::Init(); #endif + blink::Platform::InitializeBlink(); platform_ = std::make_unique<BlinkPlatformWithTaskEnvironment>(); mojo::BinderMap binders; diff --git a/chromium/media/blink/url_index_unittest.cc b/chromium/media/blink/url_index_unittest.cc index 1dc131a2348..0e4992c808d 100644 --- a/chromium/media/blink/url_index_unittest.cc +++ b/chromium/media/blink/url_index_unittest.cc @@ -7,7 +7,6 @@ #include <list> #include <string> -#include "base/logging.h" #include "base/run_loop.h" #include "base/strings/string_number_conversions.h" #include "base/strings/stringprintf.h" diff --git a/chromium/media/blink/video_frame_compositor.cc b/chromium/media/blink/video_frame_compositor.cc index 43a4c98bf17..2b3816c8c42 100644 --- a/chromium/media/blink/video_frame_compositor.cc +++ b/chromium/media/blink/video_frame_compositor.cc @@ -20,6 +20,7 @@ namespace media { // Amount of time to wait between UpdateCurrentFrame() callbacks before starting // background rendering to keep the Render() callbacks moving. const int kBackgroundRenderingTimeoutMs = 250; +const int kForceBeginFramesTimeoutMs = 1000; // static constexpr const char VideoFrameCompositor::kTracingCategory[]; @@ -34,6 +35,11 @@ VideoFrameCompositor::VideoFrameCompositor( base::TimeDelta::FromMilliseconds(kBackgroundRenderingTimeoutMs), base::Bind(&VideoFrameCompositor::BackgroundRender, base::Unretained(this))), + force_begin_frames_timer_( + FROM_HERE, + base::TimeDelta::FromMilliseconds(kForceBeginFramesTimeoutMs), + base::Bind(&VideoFrameCompositor::StopForceBeginFrames, + base::Unretained(this))), submitter_(std::move(submitter)) { if (submitter_) { task_runner_->PostTask( @@ -58,7 +64,7 @@ void VideoFrameCompositor::SetIsSurfaceVisible(bool is_visible) { void VideoFrameCompositor::InitializeSubmitter() { DCHECK(task_runner_->BelongsToCurrentThread()); - submitter_->Initialize(this); + submitter_->Initialize(this, /* is_media_stream = */ false); } VideoFrameCompositor::~VideoFrameCompositor() { @@ -276,6 +282,24 @@ void VideoFrameCompositor::SetOnFramePresentedCallback( OnNewFramePresentedCB present_cb) { base::AutoLock lock(current_frame_lock_); new_presented_frame_cb_ = std::move(present_cb); + + task_runner_->PostTask( + FROM_HERE, base::BindOnce(&VideoFrameCompositor::StartForceBeginFrames, + weak_ptr_factory_.GetWeakPtr())); +} + +void VideoFrameCompositor::StartForceBeginFrames() { + DCHECK(task_runner_->BelongsToCurrentThread()); + if (!submitter_) + return; + + submitter_->SetForceBeginFrames(true); + force_begin_frames_timer_.Reset(); +} + +void VideoFrameCompositor::StopForceBeginFrames() { + DCHECK(task_runner_->BelongsToCurrentThread()); + submitter_->SetForceBeginFrames(false); } std::unique_ptr<blink::WebMediaPlayer::VideoFramePresentationMetadata> diff --git a/chromium/media/blink/video_frame_compositor.h b/chromium/media/blink/video_frame_compositor.h index 5f2aabe1e74..fc8456053bd 100644 --- a/chromium/media/blink/video_frame_compositor.h +++ b/chromium/media/blink/video_frame_compositor.h @@ -189,6 +189,17 @@ class MEDIA_BLINK_EXPORT VideoFrameCompositor : public VideoRendererSink, void SetCurrentFrame_Locked(scoped_refptr<VideoFrame> frame, base::TimeTicks expected_display_time); + // Sets the ForceBeginFrames flag on |submitter_|, and resets + // |force_begin_frames_timer_|. + // + // The flag is used to keep receiving BeginFrame()/UpdateCurrentFrame() calls + // even if the video element is not visible, so websites can still use the + // requestVideoFrameCallback() API when the video is offscreen. + void StartForceBeginFrames(); + + // Called from |force_begin_frames_timer_| to unset the flag on |submitter_|. + void StopForceBeginFrames(); + // Called by |background_rendering_timer_| when enough time elapses where we // haven't seen a Render() call. void BackgroundRender(); @@ -220,6 +231,10 @@ class MEDIA_BLINK_EXPORT VideoFrameCompositor : public VideoRendererSink, // after each successful UpdateCurrentFrame() call. base::RetainingOneShotTimer background_rendering_timer_; + // Calls StopForceBeginFrames() once we stop receiving calls to + // requestVideoFrameCallback() (or SetOnFramePresentedCallback() in our case). + base::RetainingOneShotTimer force_begin_frames_timer_; + // These values are only set and read on the compositor thread. cc::VideoFrameProvider::Client* client_ = nullptr; bool rendering_ = false; diff --git a/chromium/media/blink/video_frame_compositor_unittest.cc b/chromium/media/blink/video_frame_compositor_unittest.cc index 3b147d1bfde..70dff67751a 100644 --- a/chromium/media/blink/video_frame_compositor_unittest.cc +++ b/chromium/media/blink/video_frame_compositor_unittest.cc @@ -34,11 +34,12 @@ class MockWebVideoFrameSubmitter : public blink::WebVideoFrameSubmitter { MOCK_METHOD0(StartRendering, void()); MOCK_METHOD0(StopRendering, void()); MOCK_CONST_METHOD0(IsDrivingFrameUpdates, bool(void)); - MOCK_METHOD1(Initialize, void(cc::VideoFrameProvider*)); + MOCK_METHOD2(Initialize, void(cc::VideoFrameProvider*, bool)); MOCK_METHOD1(SetRotation, void(media::VideoRotation)); MOCK_METHOD1(SetIsSurfaceVisible, void(bool)); MOCK_METHOD1(SetIsPageVisible, void(bool)); MOCK_METHOD1(SetForceSubmit, void(bool)); + MOCK_METHOD1(SetForceBeginFrames, void(bool)); void DidReceiveFrame() override { ++did_receive_frame_count_; } int did_receive_frame_count() { return did_receive_frame_count_; } @@ -69,7 +70,7 @@ class VideoFrameCompositorTest : public VideoRendererSink::RenderCallback, base::ThreadTaskRunnerHandle::Get(), nullptr); compositor_->SetVideoFrameProviderClient(client_.get()); } else { - EXPECT_CALL(*submitter_, Initialize(_)); + EXPECT_CALL(*submitter_, Initialize(_, _)); compositor_ = std::make_unique<VideoFrameCompositor>( base::ThreadTaskRunnerHandle::Get(), std::move(client_)); base::RunLoop().RunUntilIdle(); @@ -200,7 +201,7 @@ TEST_P(VideoFrameCompositorTest, PaintSingleFrame) { EXPECT_EQ(1, submitter_->did_receive_frame_count()); } -TEST_P(VideoFrameCompositorTest, RenderFiresPrensentationCallback) { +TEST_P(VideoFrameCompositorTest, RenderFiresPresentationCallback) { // Advance the clock so we can differentiate between base::TimeTicks::Now() // and base::TimeTicks(). tick_clock_.Advance(base::TimeDelta::FromSeconds(1)); @@ -208,6 +209,7 @@ TEST_P(VideoFrameCompositorTest, RenderFiresPrensentationCallback) { scoped_refptr<VideoFrame> opaque_frame = CreateOpaqueFrame(); EXPECT_CALL(*this, Render(_, _, true)).WillRepeatedly(Return(opaque_frame)); EXPECT_CALL(*this, OnNewFramePresented()); + EXPECT_CALL(*submitter_, SetForceBeginFrames(true)).Times(AnyNumber()); compositor()->SetOnFramePresentedCallback(GetNewFramePresentedCB()); StartVideoRendererSink(); StopVideoRendererSink(true); @@ -217,6 +219,26 @@ TEST_P(VideoFrameCompositorTest, RenderFiresPrensentationCallback) { EXPECT_NE(base::TimeTicks(), metadata->expected_display_time); } +TEST_P(VideoFrameCompositorTest, PresentationCallbackForcesBeginFrames) { + if (!IsSurfaceLayerForVideoEnabled()) + return; + + // A call to the requestVideoFrameCallback() API should set ForceBeginFrames. + EXPECT_CALL(*submitter_, SetForceBeginFrames(true)); + compositor()->SetOnFramePresentedCallback(GetNewFramePresentedCB()); + base::RunLoop().RunUntilIdle(); + + testing::Mock::VerifyAndClear(submitter_); + + // The flag should be un-set when stop receiving callbacks. + base::RunLoop run_loop; + EXPECT_CALL(*submitter_, SetForceBeginFrames(false)) + .WillOnce(RunClosure(run_loop.QuitClosure())); + run_loop.Run(); + + testing::Mock::VerifyAndClear(submitter_); +} + TEST_P(VideoFrameCompositorTest, MultiplePresentationCallbacks) { // Advance the clock so we can differentiate between base::TimeTicks::Now() // and base::TimeTicks(). @@ -231,6 +253,7 @@ TEST_P(VideoFrameCompositorTest, MultiplePresentationCallbacks) { scoped_refptr<VideoFrame> opaque_frame_3 = CreateOpaqueFrame(kSize3, kSize3); EXPECT_CALL(*this, OnNewFramePresented()).Times(1); + EXPECT_CALL(*submitter_, SetForceBeginFrames(_)).Times(AnyNumber()); compositor()->SetOnFramePresentedCallback(GetNewFramePresentedCB()); compositor()->PaintSingleFrame(opaque_frame_1); diff --git a/chromium/media/blink/watch_time_reporter_unittest.cc b/chromium/media/blink/watch_time_reporter_unittest.cc index 591629ea0b2..7db8151cfc9 100644 --- a/chromium/media/blink/watch_time_reporter_unittest.cc +++ b/chromium/media/blink/watch_time_reporter_unittest.cc @@ -261,6 +261,9 @@ class WatchTimeReporterTest const std::string& taskName, mojo::PendingReceiver<media::learning::mojom::LearningTaskController> receiver) override {} + void AcquirePlaybackEventsRecorder( + mojo::PendingReceiver<mojom::PlaybackEventsRecorder> receiver) + override {} void Initialize(bool is_mse, mojom::MediaURLScheme url_scheme) override {} void OnError(PipelineStatus status) override {} void SetIsEME() override {} diff --git a/chromium/media/blink/webcontentdecryptionmodule_impl.cc b/chromium/media/blink/webcontentdecryptionmodule_impl.cc index 1e86a444b62..9e4ade56fd7 100644 --- a/chromium/media/blink/webcontentdecryptionmodule_impl.cc +++ b/chromium/media/blink/webcontentdecryptionmodule_impl.cc @@ -7,7 +7,8 @@ #include <utility> #include "base/bind.h" -#include "base/logging.h" +#include "base/check.h" +#include "base/notreached.h" #include "base/numerics/safe_conversions.h" #include "base/strings/string_util.h" #include "base/strings/utf_string_conversions.h" diff --git a/chromium/media/blink/webcontentdecryptionmodulesession_impl.cc b/chromium/media/blink/webcontentdecryptionmodulesession_impl.cc index a62902e873a..464b8fa30a6 100644 --- a/chromium/media/blink/webcontentdecryptionmodulesession_impl.cc +++ b/chromium/media/blink/webcontentdecryptionmodulesession_impl.cc @@ -8,8 +8,9 @@ #include "base/bind.h" #include "base/callback_helpers.h" -#include "base/logging.h" +#include "base/check_op.h" #include "base/metrics/histogram_functions.h" +#include "base/notreached.h" #include "base/numerics/safe_conversions.h" #include "base/strings/string_util.h" #include "base/strings/utf_string_conversions.h" diff --git a/chromium/media/blink/webinbandtexttrack_impl.cc b/chromium/media/blink/webinbandtexttrack_impl.cc index 3b66718d924..df182a7b906 100644 --- a/chromium/media/blink/webinbandtexttrack_impl.cc +++ b/chromium/media/blink/webinbandtexttrack_impl.cc @@ -4,7 +4,7 @@ #include "media/blink/webinbandtexttrack_impl.h" -#include "base/logging.h" +#include "base/check.h" namespace media { diff --git a/chromium/media/blink/webmediaplayer_impl.cc b/chromium/media/blink/webmediaplayer_impl.cc index bf58056d782..254902da241 100644 --- a/chromium/media/blink/webmediaplayer_impl.cc +++ b/chromium/media/blink/webmediaplayer_impl.cc @@ -110,14 +110,10 @@ void SetSinkIdOnMediaThread( } bool IsBackgroundSuspendEnabled(const WebMediaPlayerImpl* wmpi) { - // TODO(crbug.com/867146): remove these switches. if (base::CommandLine::ForCurrentProcess()->HasSwitch( - switches::kDisableMediaSuspend)) + switches::kDisableBackgroundMediaSuspend)) { return false; - if (base::CommandLine::ForCurrentProcess()->HasSwitch( - switches::kEnableMediaSuspend)) - return true; - + } return wmpi->IsBackgroundMediaSuspendEnabled(); } @@ -280,7 +276,8 @@ void CreateAllocation(base::trace_event::ProcessMemoryDump* pmd, auto* std_allocator = base::trace_event::MemoryDumpManager::GetInstance() ->system_allocator_pool_name(); - pmd->AddSuballocation(dump->guid(), std_allocator); + if (std_allocator) + pmd->AddSuballocation(dump->guid(), std_allocator); } } // namespace @@ -335,13 +332,13 @@ WebMediaPlayerImpl::WebMediaPlayerImpl( is_background_video_track_optimization_supported_( params->IsBackgroundVideoTrackOptimizationSupported()), is_remoting_renderer_enabled_(params->IsRemotingRendererEnabled()), - reported_renderer_type_(RendererFactoryType::kDefault), simple_watch_timer_( base::BindRepeating(&WebMediaPlayerImpl::OnSimpleWatchTimerTick, base::Unretained(this)), base::BindRepeating(&WebMediaPlayerImpl::GetCurrentTimeInternal, base::Unretained(this))), will_play_helper_(nullptr), + demuxer_override_(params->TakeDemuxerOverride()), power_status_helper_(params->TakePowerStatusHelper()) { DVLOG(1) << __func__; DCHECK(adjust_allocated_memory_cb_); @@ -416,6 +413,13 @@ WebMediaPlayerImpl::WebMediaPlayerImpl( base::BindRepeating(&WebMediaPlayerImpl::OnMainThreadMemoryDump, weak_this_, media_log_->id())); + media_metrics_provider_->AcquirePlaybackEventsRecorder( + playback_events_recorder_.BindNewPipeAndPassReceiver()); + + // MediaMetricsProvider may drop the request for PlaybackEventsRecorder if + // it's not interested in recording these events. + playback_events_recorder_.reset_on_disconnect(); + #if defined(OS_ANDROID) renderer_factory_selector_->SetRemotePlayStateChangeCB( BindToCurrentLoop(base::BindRepeating( @@ -728,8 +732,9 @@ void WebMediaPlayerImpl::DoLoad(LoadType load_type, load_type == kLoadTypeURL ? blink::GetMediaURLScheme(loaded_url_) : mojom::MediaURLScheme::kUnknown); - // Media source pipelines can start immediately. - if (load_type == kLoadTypeMediaSource) { + if (demuxer_override_ || load_type == kLoadTypeMediaSource) { + // If a demuxer override was specified or a Media Source pipeline will be + // used, the pipeline can start immediately. StartPipeline(); } else { // Short circuit the more complex loading path for data:// URLs. Sending @@ -791,6 +796,9 @@ void WebMediaPlayerImpl::Play() { // Try to create the smoothness helper, in case we were paused before. UpdateSmoothnessHelper(); + if (playback_events_recorder_) + playback_events_recorder_->OnPlaying(); + watch_time_reporter_->SetAutoplayInitiated(client_->WasAutoplayInitiated()); // If we're seeking we'll trigger the watch time reporter upon seek completed; @@ -811,6 +819,9 @@ void WebMediaPlayerImpl::Play() { MaybeUpdateBufferSizesForPlayback(); UpdatePlayState(); + // Paused changed so we should update media position state. + UpdateMediaPositionState(); + // Notify the learning task, if needed. will_play_helper_.CompleteObservationIfNeeded(learning::TargetValue(true)); } @@ -842,6 +853,9 @@ void WebMediaPlayerImpl::Pause() { if (observer_) observer_->OnPaused(); + if (playback_events_recorder_) + playback_events_recorder_->OnPaused(); + DCHECK(watch_time_reporter_); watch_time_reporter_->OnPaused(); @@ -897,6 +911,9 @@ void WebMediaPlayerImpl::DoSeek(base::TimeDelta time, bool time_updated) { return; } + if (playback_events_recorder_) + playback_events_recorder_->OnSeeking(); + // Call this before setting |seeking_| so that the current media time can be // recorded by the reporter. if (watch_time_reporter_) @@ -936,6 +953,10 @@ void WebMediaPlayerImpl::SetRate(double rate) { pipeline_controller_->SetPlaybackRate(rate); MaybeUpdateBufferSizesForPlayback(); + + // The playback rate has changed so we should rebuild the media position + // state. + UpdateMediaPositionState(); } void WebMediaPlayerImpl::SetVolume(double volume) { @@ -1593,6 +1614,8 @@ void WebMediaPlayerImpl::OnPipelineSeeked(bool time_updated) { } else { DCHECK(watch_time_reporter_); watch_time_reporter_->OnPlaying(); + if (playback_events_recorder_) + playback_events_recorder_->OnPlaying(); } if (time_updated) should_notify_time_changed_ = true; @@ -1766,6 +1789,9 @@ void WebMediaPlayerImpl::OnError(PipelineStatus status) { if (found_hls && mb_data_source_) { demuxer_found_hls_ = true; + if (observer_) + observer_->OnHlsManifestDetected(); + UMA_HISTOGRAM_BOOLEAN("Media.WebMediaPlayerImpl.HLS.IsCorsCrossOrigin", mb_data_source_->IsCorsCrossOrigin()); if (mb_data_source_->IsCorsCrossOrigin()) { @@ -1823,10 +1849,12 @@ void WebMediaPlayerImpl::OnError(PipelineStatus status) { status = PIPELINE_ERROR_EXTERNAL_RENDERER_FAILED; #endif - MaybeSetContainerName(); + MaybeSetContainerNameForMetrics(); simple_watch_timer_.Stop(); media_log_->NotifyError(status); media_metrics_provider_->OnError(status); + if (playback_events_recorder_) + playback_events_recorder_->OnError(status); if (watch_time_reporter_) watch_time_reporter_->OnError(status); @@ -1857,6 +1885,9 @@ void WebMediaPlayerImpl::OnEnded() { ended_ = true; client_->TimeChanged(); + if (playback_events_recorder_) + playback_events_recorder_->OnEnded(); + // We don't actually want this to run until |client_| calls seek() or pause(), // but that should have already happened in timeChanged() and so this is // expected to be a no-op. @@ -1873,7 +1904,7 @@ void WebMediaPlayerImpl::OnMetadata(const PipelineMetadata& metadata) { media_metrics_provider_->SetTimeToMetadata(time_to_metadata_); RecordTimingUMA("Media.TimeToMetadata", time_to_metadata_); - MaybeSetContainerName(); + MaybeSetContainerNameForMetrics(); pipeline_metadata_ = metadata; if (power_status_helper_) @@ -2142,6 +2173,9 @@ void WebMediaPlayerImpl::OnBufferingStateChangeInternal( watch_time_reporter_->OnUnderflowComplete(elapsed); underflow_timer_.reset(); } + + if (playback_events_recorder_) + playback_events_recorder_->OnBufferingComplete(); } else { // Buffering has underflowed. DCHECK_EQ(state, BUFFERING_HAVE_NOTHING); @@ -2153,6 +2187,9 @@ void WebMediaPlayerImpl::OnBufferingStateChangeInternal( !seeking_) { underflow_timer_ = std::make_unique<base::ElapsedTimer>(); watch_time_reporter_->OnUnderflow(); + + if (playback_events_recorder_) + playback_events_recorder_->OnBuffering(); } // It shouldn't be possible to underflow if we've not advanced past @@ -2746,7 +2783,13 @@ void WebMediaPlayerImpl::StartPipeline() { #endif // defined(OS_ANDROID) // Figure out which demuxer to use. - if (load_type_ != kLoadTypeMediaSource) { + if (demuxer_override_) { + DCHECK(!chunk_demuxer_); + + SetDemuxer(std::move(demuxer_override_)); + // TODO(https://crbug.com/1076267): Should everything else after this block + // run in the demuxer override case? + } else if (load_type_ != kLoadTypeMediaSource) { DCHECK(!chunk_demuxer_); DCHECK(data_source_); @@ -2782,12 +2825,6 @@ void WebMediaPlayerImpl::StartPipeline() { } } - // TODO(sandersd): FileSystem objects may also be non-static, but due to our - // caching layer such situations are broken already. http://crbug.com/593159 - bool is_static = !chunk_demuxer_; - bool is_streaming = IsStreaming(); - UMA_HISTOGRAM_BOOLEAN("Media.IsStreaming", is_streaming); - // If possible attempt to avoid decoder spool up until playback starts. Pipeline::StartType start_type = Pipeline::StartType::kNormal; if (!chunk_demuxer_ && preload_ == MultibufferDataSource::METADATA && @@ -2799,10 +2836,14 @@ void WebMediaPlayerImpl::StartPipeline() { attempting_suspended_start_ = true; } + // TODO(sandersd): FileSystem objects may also be non-static, but due to our + // caching layer such situations are broken already. http://crbug.com/593159 + const bool is_static = !chunk_demuxer_; + // ... and we're ready to go! // TODO(sandersd): On Android, defer Start() if the tab is not visible. seeking_ = true; - pipeline_controller_->Start(start_type, demuxer_.get(), this, is_streaming, + pipeline_controller_->Start(start_type, demuxer_.get(), this, IsStreaming(), is_static); } @@ -2829,6 +2870,10 @@ void WebMediaPlayerImpl::SetReadyState(WebMediaPlayer::ReadyState state) { // Always notify to ensure client has the latest value. client_->ReadyStateChanged(); + + // The ready state affects the effective playback rate so we should update + // the media position state. + UpdateMediaPositionState(); } scoped_refptr<blink::WebAudioSourceProviderImpl> @@ -2898,8 +2943,12 @@ void WebMediaPlayerImpl::UpdateMediaPositionState() { if (current_time > duration) current_time = duration; - media_session::MediaPosition new_position(paused_ ? 0.0 : playback_rate_, - duration, current_time); + const double effective_playback_rate = + paused_ || ready_state_ < kReadyStateHaveFutureData ? 0.0 + : playback_rate_; + + media_session::MediaPosition new_position(effective_playback_rate, duration, + current_time); if (media_position_state_ == new_position) return; @@ -3191,7 +3240,7 @@ void WebMediaPlayerImpl::FinishMemoryUsageReport(int64_t demuxer_memory_usage) { stats.audio_memory_usage + video_memory_usage + data_source_memory_usage + demuxer_memory_usage; - DVLOG(2) << "Memory Usage -- Total: " << current_memory_usage + DVLOG(3) << "Memory Usage -- Total: " << current_memory_usage << " Audio: " << stats.audio_memory_usage << ", Video: " << video_memory_usage << ", DataSource: " << data_source_memory_usage @@ -3653,6 +3702,9 @@ void WebMediaPlayerImpl::RecordVideoNaturalSize(const gfx::Size& natural_size) { UMA_HISTOGRAM_VIDEO_HEIGHT("Media.VideoHeight.Initial.EME", height); UMA_HISTOGRAM_VIDEO_HEIGHT("Media.VideoHeight.Initial.All", height); + + if (playback_events_recorder_) + playback_events_recorder_->OnNaturalSizeChanged(natural_size); } #undef UMA_HISTOGRAM_VIDEO_HEIGHT @@ -3712,11 +3764,7 @@ void WebMediaPlayerImpl::OnPictureInPictureAvailabilityChanged(bool available) { delegate_->DidPictureInPictureAvailabilityChange(delegate_id_, available); } -void WebMediaPlayerImpl::MaybeSetContainerName() { - // MSE nor MediaPlayerRenderer provide container information. - if (chunk_demuxer_ || using_media_player_renderer_) - return; - +void WebMediaPlayerImpl::MaybeSetContainerNameForMetrics() { // Pipeline startup failed before even getting a demuxer setup. if (!demuxer_) return; @@ -3725,11 +3773,10 @@ void WebMediaPlayerImpl::MaybeSetContainerName() { if (highest_ready_state_ >= WebMediaPlayer::kReadyStateHaveMetadata) return; -// If ffmpeg isn't enabled, we can't get the container name. -#if BUILDFLAG(ENABLE_FFMPEG) - media_metrics_provider_->SetContainerName( - static_cast<FFmpegDemuxer*>(demuxer_.get())->container()); -#endif + // Only report metrics for demuxers that provide container information. + auto container = demuxer_->GetContainerForMetrics(); + if (container.has_value()) + media_metrics_provider_->SetContainerName(container.value()); } void WebMediaPlayerImpl::MaybeUpdateBufferSizesForPlayback() { @@ -3742,14 +3789,13 @@ void WebMediaPlayerImpl::MaybeUpdateBufferSizesForPlayback() { mb_data_source_->MediaPlaybackRateChanged(playback_rate_); if (!paused_) mb_data_source_->MediaIsPlaying(); - - // The playback rate has changed so we should rebuild the media position - // state. - UpdateMediaPositionState(); } void WebMediaPlayerImpl::OnSimpleWatchTimerTick() { RecordSimpleWatchTimeUMA(reported_renderer_type_); + + if (playback_events_recorder_) + playback_events_recorder_->OnPipelineStatistics(GetPipelineStatistics()); } GURL WebMediaPlayerImpl::GetSrcAfterRedirects() { diff --git a/chromium/media/blink/webmediaplayer_impl.h b/chromium/media/blink/webmediaplayer_impl.h index 69e853bb93e..b62f8cf93b9 100644 --- a/chromium/media/blink/webmediaplayer_impl.h +++ b/chromium/media/blink/webmediaplayer_impl.h @@ -45,6 +45,7 @@ #include "media/blink/webmediaplayer_params.h" #include "media/filters/pipeline_controller.h" #include "media/learning/common/media_learning_tasks.h" +#include "media/mojo/mojom/playback_events_recorder.mojom.h" #include "media/renderers/paint_canvas_video_renderer.h" #include "mojo/public/cpp/bindings/remote.h" #include "services/media_session/public/cpp/media_position.h" @@ -630,7 +631,7 @@ class MEDIA_BLINK_EXPORT WebMediaPlayerImpl bool IsInPictureInPicture() const; // Sets the UKM container name if needed. - void MaybeSetContainerName(); + void MaybeSetContainerNameForMetrics(); // Switch to SurfaceLayer, either initially or from VideoLayer. void ActivateSurfaceLayerForVideo(); @@ -867,6 +868,8 @@ class MEDIA_BLINK_EXPORT WebMediaPlayerImpl // unimportant. bool suppress_destruction_errors_ = false; + // TODO(dalecurtis): The following comment is inaccurate as this value is also + // used for, for example, data URLs. // Used for HLS playback and in certain fallback paths (e.g. on older devices // that can't support the unified media pipeline). GURL loaded_url_; @@ -1004,6 +1007,7 @@ class MEDIA_BLINK_EXPORT WebMediaPlayerImpl base::CancelableClosure update_background_status_cb_; mojo::Remote<mojom::MediaMetricsProvider> media_metrics_provider_; + mojo::Remote<mojom::PlaybackEventsRecorder> playback_events_recorder_; base::Optional<ReadyState> stale_state_override_for_testing_; @@ -1044,11 +1048,14 @@ class MEDIA_BLINK_EXPORT WebMediaPlayerImpl base::CancelableOnceClosure have_enough_after_lazy_load_cb_; // State for simplified watch time reporting. - RendererFactoryType reported_renderer_type_; + RendererFactoryType reported_renderer_type_ = RendererFactoryType::kDefault; SimpleWatchTimer simple_watch_timer_; LearningExperimentHelper will_play_helper_; + // Stores the optional override Demuxer until it is used in DoLoad(). + std::unique_ptr<Demuxer> demuxer_override_; + std::unique_ptr<PowerStatusHelper> power_status_helper_; // Created while playing, deleted otherwise. diff --git a/chromium/media/blink/webmediaplayer_impl_unittest.cc b/chromium/media/blink/webmediaplayer_impl_unittest.cc index f778620cf6e..77dc3344993 100644 --- a/chromium/media/blink/webmediaplayer_impl_unittest.cc +++ b/chromium/media/blink/webmediaplayer_impl_unittest.cc @@ -49,6 +49,7 @@ #include "media/mojo/services/watch_time_recorder.h" #include "media/renderers/default_decoder_factory.h" #include "media/renderers/default_renderer_factory.h" +#include "mojo/public/cpp/bindings/pending_associated_receiver.h" #include "mojo/public/cpp/bindings/pending_remote.h" #include "mojo/public/cpp/bindings/remote.h" #include "mojo/public/cpp/bindings/strong_binding.h" @@ -318,17 +319,17 @@ class WebMediaPlayerImplTest public: WebMediaPlayerImplTest() : media_thread_("MediaThreadForTest"), - web_view_( - blink::WebView::Create(/*client=*/nullptr, - /*is_hidden=*/false, - /*compositing_enabled=*/false, - nullptr, - mojo::ScopedInterfaceEndpointHandle())), - web_local_frame_( - blink::WebLocalFrame::CreateMainFrame(web_view_, - &web_frame_client_, - nullptr, - nullptr)), + web_view_(blink::WebView::Create(/*client=*/nullptr, + /*is_hidden=*/false, + /*compositing_enabled=*/false, + nullptr, + mojo::NullAssociatedReceiver())), + web_local_frame_(blink::WebLocalFrame::CreateMainFrame( + web_view_, + &web_frame_client_, + nullptr, + base::UnguessableToken::Create(), + nullptr)), context_provider_(viz::TestContextProvider::Create()), audio_parameters_(TestAudioParameters::Normal()), memory_dump_manager_( @@ -349,6 +350,30 @@ class WebMediaPlayerImplTest } void InitializeWebMediaPlayerImpl() { + InitializeWebMediaPlayerImplInternal(nullptr); + } + + ~WebMediaPlayerImplTest() override { + if (!wmpi_) + return; + EXPECT_CALL(client_, SetCcLayer(nullptr)); + EXPECT_CALL(client_, MediaRemotingStopped(_)); + + // Destruct WebMediaPlayerImpl and pump the message loop to ensure that + // objects passed to the message loop for destruction are released. + // + // NOTE: This should be done before any other member variables are + // destructed since WMPI may reference them during destruction. + wmpi_.reset(); + + CycleThreads(); + + web_view_->Close(); + } + + protected: + void InitializeWebMediaPlayerImplInternal( + std::unique_ptr<media::Demuxer> demuxer_override) { auto media_log = std::make_unique<NiceMock<MockMediaLog>>(); InitializeSurfaceLayerBridge(); @@ -361,13 +386,19 @@ class WebMediaPlayerImplTest auto factory_selector = std::make_unique<RendererFactorySelector>(); renderer_factory_selector_ = factory_selector.get(); decoder_factory_.reset(new media::DefaultDecoderFactory(nullptr)); +#if defined(OS_ANDROID) factory_selector->AddBaseFactory( RendererFactoryType::kDefault, std::make_unique<DefaultRendererFactory>( media_log.get(), decoder_factory_.get(), - DefaultRendererFactory::GetGpuFactoriesCB(), nullptr)); -#if defined(OS_ANDROID) + DefaultRendererFactory::GetGpuFactoriesCB())); factory_selector->StartRequestRemotePlayStateCB(base::DoNothing()); +#else + factory_selector->AddBaseFactory( + RendererFactoryType::kDefault, + std::make_unique<DefaultRendererFactory>( + media_log.get(), decoder_factory_.get(), + DefaultRendererFactory::GetGpuFactoriesCB(), nullptr)); #endif mojo::Remote<mojom::MediaMetricsProvider> provider; @@ -405,7 +436,7 @@ class WebMediaPlayerImplTest viz::TestContextProvider::Create(), blink::WebMediaPlayer::SurfaceLayerMode::kAlways, is_background_suspend_enabled_, is_background_video_playback_enabled_, - true, false, nullptr); + true, false, std::move(demuxer_override), nullptr); auto compositor = std::make_unique<NiceMock<MockVideoFrameCompositor>>( params->video_frame_compositor_task_runner()); @@ -417,25 +448,6 @@ class WebMediaPlayerImplTest std::move(params)); } - ~WebMediaPlayerImplTest() override { - if (!wmpi_) - return; - EXPECT_CALL(client_, SetCcLayer(nullptr)); - EXPECT_CALL(client_, MediaRemotingStopped(_)); - - // Destruct WebMediaPlayerImpl and pump the message loop to ensure that - // objects passed to the message loop for destruction are released. - // - // NOTE: This should be done before any other member variables are - // destructed since WMPI may reference them during destruction. - wmpi_.reset(); - - CycleThreads(); - - web_view_->Close(); - } - - protected: std::unique_ptr<blink::WebSurfaceLayerBridge> CreateMockSurfaceLayerBridge( blink::WebSurfaceLayerBridgeObserver*, cc::UpdateSubmissionStateCB) { @@ -1606,6 +1618,48 @@ TEST_F(WebMediaPlayerImplTest, MediaPositionState_PlayPauseSetRate) { Play(); } +TEST_F(WebMediaPlayerImplTest, MediaPositionState_Underflow) { + InitializeWebMediaPlayerImpl(); + + testing::Sequence sequence; + EXPECT_CALL(delegate_, + DidPlayerMediaPositionStateChange(delegate_.player_id(), _)) + .InSequence(sequence) + .WillOnce([](auto id, auto position) { + EXPECT_EQ(0.0, position.playback_rate()); + }); + EXPECT_CALL(delegate_, + DidPlayerMediaPositionStateChange(delegate_.player_id(), _)) + .InSequence(sequence) + .WillOnce([](auto id, auto position) { + EXPECT_EQ(1.0, position.playback_rate()); + }); + EXPECT_CALL(delegate_, + DidPlayerMediaPositionStateChange(delegate_.player_id(), _)) + .InSequence(sequence) + .WillOnce([](auto id, auto position) { + EXPECT_EQ(0.0, position.playback_rate()); + }); + EXPECT_CALL(delegate_, + DidPlayerMediaPositionStateChange(delegate_.player_id(), _)) + .InSequence(sequence) + .WillOnce([](auto id, auto position) { + EXPECT_EQ(1.0, position.playback_rate()); + }); + + wmpi_->SetRate(1.0); + LoadAndWaitForReadyState(kAudioOnlyTestFile, + blink::WebMediaPlayer::kReadyStateHaveCurrentData); + // Play will set the playback rate to 1.0. + Play(); + + // Underflow will set the playback rate to 0.0. + SetReadyState(blink::WebMediaPlayer::kReadyStateHaveCurrentData); + + // Leaving the underflow state will restore the playback rate of 1.0. + SetReadyState(blink::WebMediaPlayer::kReadyStateHaveFutureData); +} + TEST_F(WebMediaPlayerImplTest, MediaPositionState_Seeking) { InitializeWebMediaPlayerImpl(); @@ -1629,9 +1683,9 @@ TEST_F(WebMediaPlayerImplTest, MediaPositionState_Seeking) { DidPlayerMediaPositionStateChange(delegate_.player_id(), _)) .InSequence(s) .WillOnce([](auto id, auto position) { - EXPECT_EQ(1.0, position.playback_rate()); + EXPECT_EQ(0.0, position.playback_rate()); EXPECT_EQ(kAudioOnlyTestFileDuration, position.duration()); - EXPECT_EQ(base::TimeDelta::FromMilliseconds(100), + EXPECT_EQ(base::TimeDelta(), position.GetPositionAtTime(position.last_updated_time())); }); EXPECT_CALL(delegate_, DidPlayerMediaPositionStateChange( @@ -2139,6 +2193,33 @@ TEST_F(WebMediaPlayerImplTest, MemDumpReporting) { EXPECT_EQ(dump_count, 3); } +// Verify that a demuxer override is used when specified. +TEST_F(WebMediaPlayerImplTest, DemuxerOverride) { + std::unique_ptr<MockDemuxer> demuxer = + std::make_unique<NiceMock<MockDemuxer>>(); + StrictMock<MockDemuxerStream> stream(DemuxerStream::AUDIO); + stream.set_audio_decoder_config(TestAudioConfig::Normal()); + std::vector<DemuxerStream*> streams; + streams.push_back(&stream); + + EXPECT_CALL(stream, SupportsConfigChanges()).WillRepeatedly(Return(false)); + + EXPECT_CALL(*demuxer.get(), OnInitialize(_, _)) + .WillOnce(RunOnceCallback<1>(PIPELINE_OK)); + EXPECT_CALL(*demuxer.get(), GetAllStreams()).WillRepeatedly(Return(streams)); + // Called when WebMediaPlayerImpl is destroyed. + EXPECT_CALL(*demuxer.get(), Stop()); + + InitializeWebMediaPlayerImplInternal(std::move(demuxer)); + + EXPECT_FALSE(IsSuspended()); + wmpi_->Load(blink::WebMediaPlayer::kLoadTypeURL, + blink::WebMediaPlayerSource(blink::WebURL(GURL("data://test"))), + blink::WebMediaPlayer::kCorsModeUnspecified); + base::RunLoop().RunUntilIdle(); + EXPECT_TRUE(IsSuspended()); +} + class WebMediaPlayerImplBackgroundBehaviorTest : public WebMediaPlayerImplTest, public ::testing::WithParamInterface< diff --git a/chromium/media/blink/webmediaplayer_params.cc b/chromium/media/blink/webmediaplayer_params.cc index 24fda578b1a..a9dede964a4 100644 --- a/chromium/media/blink/webmediaplayer_params.cc +++ b/chromium/media/blink/webmediaplayer_params.cc @@ -7,6 +7,7 @@ #include "base/single_thread_task_runner.h" #include "base/task_runner.h" #include "media/base/audio_renderer_sink.h" +#include "media/base/demuxer.h" namespace media { @@ -33,6 +34,7 @@ WebMediaPlayerParams::WebMediaPlayerParams( bool is_background_video_playback_enabled, bool is_background_video_track_optimization_supported, bool is_remoting_renderer_enabled, + std::unique_ptr<Demuxer> demuxer_override, std::unique_ptr<PowerStatusHelper> power_status_helper) : defer_load_cb_(defer_load_cb), audio_renderer_sink_(audio_renderer_sink), @@ -57,8 +59,13 @@ WebMediaPlayerParams::WebMediaPlayerParams( is_background_video_track_optimization_supported_( is_background_video_track_optimization_supported), is_remoting_renderer_enabled_(is_remoting_renderer_enabled), + demuxer_override_(std::move(demuxer_override)), power_status_helper_(std::move(power_status_helper)) {} WebMediaPlayerParams::~WebMediaPlayerParams() = default; +std::unique_ptr<Demuxer> WebMediaPlayerParams::TakeDemuxerOverride() { + return std::move(demuxer_override_); +} + } // namespace media diff --git a/chromium/media/blink/webmediaplayer_params.h b/chromium/media/blink/webmediaplayer_params.h index 0a324a32db0..4cc8756d4e2 100644 --- a/chromium/media/blink/webmediaplayer_params.h +++ b/chromium/media/blink/webmediaplayer_params.h @@ -44,6 +44,7 @@ using CreateSurfaceLayerBridgeCB = blink::WebSurfaceLayerBridgeObserver*, cc::UpdateSubmissionStateCB)>; +class Demuxer; class SwitchableAudioRendererSink; // Holds parameters for constructing WebMediaPlayerImpl without having @@ -85,6 +86,7 @@ class MEDIA_BLINK_EXPORT WebMediaPlayerParams { bool is_background_video_play_enabled, bool is_background_video_track_optimization_supported, bool is_remoting_renderer_enabled, + std::unique_ptr<Demuxer> demuxer_override, std::unique_ptr<PowerStatusHelper> power_status_helper); ~WebMediaPlayerParams(); @@ -172,6 +174,8 @@ class MEDIA_BLINK_EXPORT WebMediaPlayerParams { return is_remoting_renderer_enabled_; } + std::unique_ptr<Demuxer> TakeDemuxerOverride(); + std::unique_ptr<PowerStatusHelper> TakePowerStatusHelper() { return std::move(power_status_helper_); } @@ -207,6 +211,9 @@ class MEDIA_BLINK_EXPORT WebMediaPlayerParams { // Whether the media in this frame is a remoting media. bool is_remoting_renderer_enabled_ = false; + // Optional custom demuxer to use instead of the standard demuxers. + std::unique_ptr<Demuxer> demuxer_override_; + std::unique_ptr<PowerStatusHelper> power_status_helper_; DISALLOW_IMPLICIT_CONSTRUCTORS(WebMediaPlayerParams); diff --git a/chromium/media/capabilities/in_memory_video_decode_stats_db_unittest.cc b/chromium/media/capabilities/in_memory_video_decode_stats_db_unittest.cc index d64077c95fe..2d69342f7f9 100644 --- a/chromium/media/capabilities/in_memory_video_decode_stats_db_unittest.cc +++ b/chromium/media/capabilities/in_memory_video_decode_stats_db_unittest.cc @@ -6,7 +6,7 @@ #include "base/bind.h" #include "base/bind_helpers.h" -#include "base/logging.h" +#include "base/check_op.h" #include "base/memory/ptr_util.h" #include "base/test/gtest_util.h" #include "base/test/task_environment.h" diff --git a/chromium/media/capabilities/video_decode_stats_db.cc b/chromium/media/capabilities/video_decode_stats_db.cc index d5b4d93b949..74c54417118 100644 --- a/chromium/media/capabilities/video_decode_stats_db.cc +++ b/chromium/media/capabilities/video_decode_stats_db.cc @@ -4,6 +4,7 @@ #include "media/capabilities/video_decode_stats_db.h" +#include "base/check_op.h" #include "base/format_macros.h" #include "base/strings/stringprintf.h" #include "media/capabilities/bucket_utility.h" diff --git a/chromium/media/capabilities/video_decode_stats_db_impl.cc b/chromium/media/capabilities/video_decode_stats_db_impl.cc index 359c2d6c941..ca4c73c9432 100644 --- a/chromium/media/capabilities/video_decode_stats_db_impl.cc +++ b/chromium/media/capabilities/video_decode_stats_db_impl.cc @@ -5,6 +5,7 @@ #include "media/capabilities/video_decode_stats_db_impl.h" #include <memory> +#include <string> #include <tuple> #include "base/bind.h" @@ -13,10 +14,12 @@ #include "base/logging.h" #include "base/memory/ptr_util.h" #include "base/metrics/field_trial_params.h" +#include "base/metrics/histogram_functions.h" #include "base/metrics/histogram_macros.h" #include "base/sequence_checker.h" #include "base/task/post_task.h" #include "base/task/thread_pool.h" +#include "base/threading/thread_task_runner_handle.h" #include "base/time/default_clock.h" #include "components/leveldb_proto/public/proto_database_provider.h" #include "media/base/media_switches.h" @@ -28,12 +31,23 @@ using ProtoDecodeStatsEntry = leveldb_proto::ProtoDatabase<DecodeStatsProto>; namespace { +// Timeout threshold for DB operations. See OnOperationTimeout(). +// NOTE: Used by UmaHistogramOpTime. Change the name if you change the time. +static constexpr base::TimeDelta kPendingOpTimeout = + base::TimeDelta::FromSeconds(30); + const int kMaxFramesPerBufferDefault = 2500; const int kMaxDaysToKeepStatsDefault = 30; const bool kEnableUnweightedEntriesDefault = false; +void UmaHistogramOpTime(const std::string& op_name, base::TimeDelta duration) { + base::UmaHistogramCustomMicrosecondsTimes( + "Media.VideoDecodeStatsDB.OpTiming." + op_name, duration, + base::TimeDelta::FromMilliseconds(1), kPendingOpTimeout, 50); +} + } // namespace const char VideoDecodeStatsDBImpl::kMaxFramesPerBufferParamName[] = @@ -45,6 +59,41 @@ const char VideoDecodeStatsDBImpl::kMaxDaysToKeepStatsParamName[] = const char VideoDecodeStatsDBImpl::kEnableUnweightedEntriesParamName[] = "db_enable_unweighted_entries"; +VideoDecodeStatsDBImpl::PendingOperation::PendingOperation( + std::string uma_str, + std::unique_ptr<base::CancelableOnceClosure> timeout_closure) + : uma_str_(uma_str), + timeout_closure_(std::move(timeout_closure)), + start_ticks_(base::TimeTicks::Now()) { + DVLOG(3) << __func__ << " Started " << uma_str_; +} + +VideoDecodeStatsDBImpl::PendingOperation::~PendingOperation() { + // Destroying a pending operation that hasn't timed out yet implies the + // operation has completed. + if (timeout_closure_ && !timeout_closure_->IsCancelled()) { + base::TimeDelta op_duration = base::TimeTicks::Now() - start_ticks_; + UmaHistogramOpTime(uma_str_, op_duration); + DVLOG(3) << __func__ << " Completed " << uma_str_ << " (" + << op_duration.InMilliseconds() << ")"; + + // Ensure the timeout doesn't fire. Destruction should cancel the callback + // implicitly, but that's not a documented contract, so just taking the safe + // route. + timeout_closure_->Cancel(); + } +} + +void VideoDecodeStatsDBImpl::PendingOperation::OnTimeout() { + UmaHistogramOpTime(uma_str_, kPendingOpTimeout); + LOG(WARNING) << " Timeout performing " << uma_str_ + << " operation on VideoDecodeStatsDB"; + + // Cancel the closure to ensure we don't double report the task as completed + // in ~PendingOperation(). + timeout_closure_->Cancel(); +} + // static int VideoDecodeStatsDBImpl::GetMaxFramesPerBuffer() { return base::GetFieldTrialParamByFeatureAsDouble( @@ -97,6 +146,43 @@ VideoDecodeStatsDBImpl::~VideoDecodeStatsDBImpl() { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); } +VideoDecodeStatsDBImpl::PendingOpId VideoDecodeStatsDBImpl::StartPendingOp( + std::string uma_str) { + PendingOpId op_id = next_op_id_++; + + auto timeout_closure = std::make_unique<base::CancelableOnceClosure>( + base::BindOnce(&VideoDecodeStatsDBImpl::OnPendingOpTimeout, + weak_ptr_factory_.GetWeakPtr(), op_id)); + + base::ThreadTaskRunnerHandle::Get()->PostDelayedTask( + FROM_HERE, timeout_closure->callback(), kPendingOpTimeout); + + pending_ops_.emplace(op_id, std::make_unique<PendingOperation>( + uma_str, std::move(timeout_closure))); + + return op_id; +} + +void VideoDecodeStatsDBImpl::CompletePendingOp(PendingOpId op_id) { + // Destructing the PendingOperation will trigger UMA for completion timing. + int count = pending_ops_.erase(op_id); + + // No big deal, but very unusual. Timeout is very generous, so tasks that + // timeout are generally assumed to be permanently hung. + if (!count) + DVLOG(2) << __func__ << " DB operation completed after timeout."; +} + +void VideoDecodeStatsDBImpl::OnPendingOpTimeout(PendingOpId op_id) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + auto it = pending_ops_.find(op_id); + DCHECK(it != pending_ops_.end()); + + it->second->OnTimeout(); + pending_ops_.erase(it); +} + void VideoDecodeStatsDBImpl::Initialize(InitializeCB init_cb) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DCHECK(init_cb); @@ -107,15 +193,18 @@ void VideoDecodeStatsDBImpl::Initialize(InitializeCB init_cb) { // spamming the cache. // TODO(chcunningham): Keep an eye on the size as the table evolves. db_->Init(base::BindOnce(&VideoDecodeStatsDBImpl::OnInit, - weak_ptr_factory_.GetWeakPtr(), std::move(init_cb))); + weak_ptr_factory_.GetWeakPtr(), + StartPendingOp("Initialize"), std::move(init_cb))); } -void VideoDecodeStatsDBImpl::OnInit(InitializeCB init_cb, +void VideoDecodeStatsDBImpl::OnInit(PendingOpId op_id, + InitializeCB init_cb, leveldb_proto::Enums::InitStatus status) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DCHECK_NE(status, leveldb_proto::Enums::InitStatus::kInvalidOperation); bool success = status == leveldb_proto::Enums::InitStatus::kOK; DVLOG(2) << __func__ << (success ? " succeeded" : " FAILED!"); + CompletePendingOp(op_id); UMA_HISTOGRAM_BOOLEAN("Media.VideoDecodeStatsDB.OpSuccess.Initialize", success); @@ -143,10 +232,11 @@ void VideoDecodeStatsDBImpl::AppendDecodeStats( DVLOG(3) << __func__ << " Reading key " << key.ToLogString() << " from DB with intent to update with " << entry.ToLogString(); - db_->GetEntry(key.Serialize(), - base::BindOnce(&VideoDecodeStatsDBImpl::WriteUpdatedEntry, - weak_ptr_factory_.GetWeakPtr(), key, entry, - std::move(append_done_cb))); + db_->GetEntry( + key.Serialize(), + base::BindOnce(&VideoDecodeStatsDBImpl::WriteUpdatedEntry, + weak_ptr_factory_.GetWeakPtr(), StartPendingOp("Read"), + key, entry, std::move(append_done_cb))); } void VideoDecodeStatsDBImpl::GetDecodeStats(const VideoDescKey& key, @@ -159,7 +249,8 @@ void VideoDecodeStatsDBImpl::GetDecodeStats(const VideoDescKey& key, db_->GetEntry( key.Serialize(), base::BindOnce(&VideoDecodeStatsDBImpl::OnGotDecodeStats, - weak_ptr_factory_.GetWeakPtr(), std::move(get_stats_cb))); + weak_ptr_factory_.GetWeakPtr(), StartPendingOp("Read"), + std::move(get_stats_cb))); } bool VideoDecodeStatsDBImpl::AreStatsUsable( @@ -211,6 +302,7 @@ bool VideoDecodeStatsDBImpl::AreStatsUsable( } void VideoDecodeStatsDBImpl::WriteUpdatedEntry( + PendingOpId op_id, const VideoDescKey& key, const DecodeStatsEntry& new_entry, AppendDecodeStatsCB append_done_cb, @@ -218,6 +310,7 @@ void VideoDecodeStatsDBImpl::WriteUpdatedEntry( std::unique_ptr<DecodeStatsProto> stats_proto) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DCHECK(IsInitialized()); + CompletePendingOp(op_id); // Note: outcome of "Write" operation logged in OnEntryUpdated(). UMA_HISTOGRAM_BOOLEAN("Media.VideoDecodeStatsDB.OpSuccess.Read", @@ -355,26 +448,31 @@ void VideoDecodeStatsDBImpl::WriteUpdatedEntry( std::unique_ptr<DBType::KeyEntryVector> entries = std::make_unique<DBType::KeyEntryVector>(); entries->emplace_back(key.Serialize(), *stats_proto); - db_->UpdateEntries(std::move(entries), - std::make_unique<leveldb_proto::KeyVector>(), - base::BindOnce(&VideoDecodeStatsDBImpl::OnEntryUpdated, - weak_ptr_factory_.GetWeakPtr(), - std::move(append_done_cb))); + db_->UpdateEntries( + std::move(entries), std::make_unique<leveldb_proto::KeyVector>(), + base::BindOnce(&VideoDecodeStatsDBImpl::OnEntryUpdated, + weak_ptr_factory_.GetWeakPtr(), StartPendingOp("Write"), + std::move(append_done_cb))); } -void VideoDecodeStatsDBImpl::OnEntryUpdated(AppendDecodeStatsCB append_done_cb, +void VideoDecodeStatsDBImpl::OnEntryUpdated(PendingOpId op_id, + AppendDecodeStatsCB append_done_cb, bool success) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - UMA_HISTOGRAM_BOOLEAN("Media.VideoDecodeStatsDB.OpSuccess.Write", success); DVLOG(3) << __func__ << " update " << (success ? "succeeded" : "FAILED!"); + CompletePendingOp(op_id); + UMA_HISTOGRAM_BOOLEAN("Media.VideoDecodeStatsDB.OpSuccess.Write", success); std::move(append_done_cb).Run(success); } void VideoDecodeStatsDBImpl::OnGotDecodeStats( + PendingOpId op_id, GetDecodeStatsCB get_stats_cb, bool success, std::unique_ptr<DecodeStatsProto> stats_proto) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DVLOG(3) << __func__ << " get " << (success ? "succeeded" : "FAILED!"); + CompletePendingOp(op_id); UMA_HISTOGRAM_BOOLEAN("Media.VideoDecodeStatsDB.OpSuccess.Read", success); std::unique_ptr<DecodeStatsEntry> entry; @@ -435,40 +533,23 @@ void VideoDecodeStatsDBImpl::ClearStats(base::OnceClosure clear_done_cb) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DVLOG(2) << __func__; - db_->LoadKeys( - base::BindOnce(&VideoDecodeStatsDBImpl::OnLoadAllKeysForClearing, - weak_ptr_factory_.GetWeakPtr(), std::move(clear_done_cb))); + db_->UpdateEntriesWithRemoveFilter( + std::make_unique<ProtoDecodeStatsEntry::KeyEntryVector>(), + base::BindRepeating([](const std::string& key) { return true; }), + base::BindOnce(&VideoDecodeStatsDBImpl::OnStatsCleared, + weak_ptr_factory_.GetWeakPtr(), StartPendingOp("Clear"), + std::move(clear_done_cb))); } -void VideoDecodeStatsDBImpl::OnLoadAllKeysForClearing( - base::OnceClosure clear_done_cb, - bool success, - std::unique_ptr<std::vector<std::string>> keys) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DVLOG(2) << __func__ << (success ? " succeeded" : " FAILED!"); - - UMA_HISTOGRAM_BOOLEAN("Media.VideoDecodeStatsDB.OpSuccess.LoadKeys", success); - - if (success) { - // Remove all keys. - db_->UpdateEntries( - std::make_unique<ProtoDecodeStatsEntry::KeyEntryVector>(), - std::move(keys) /* keys_to_remove */, - base::BindOnce(&VideoDecodeStatsDBImpl::OnStatsCleared, - weak_ptr_factory_.GetWeakPtr(), - std::move(clear_done_cb))); - } else { - // Fail silently. See comment in OnStatsCleared(). - std::move(clear_done_cb).Run(); - } -} - -void VideoDecodeStatsDBImpl::OnStatsCleared(base::OnceClosure clear_done_cb, +void VideoDecodeStatsDBImpl::OnStatsCleared(PendingOpId op_id, + base::OnceClosure clear_done_cb, bool success) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DVLOG(2) << __func__ << (success ? " succeeded" : " FAILED!"); - UMA_HISTOGRAM_BOOLEAN("Media.VideoDecodeStatsDB.OpSuccess.Destroy", success); + CompletePendingOp(op_id); + + UMA_HISTOGRAM_BOOLEAN("Media.VideoDecodeStatsDB.OpSuccess.Clear", success); // We don't pass success to |clear_done_cb|. Clearing is best effort and // there is no additional action for callers to take in case of failure. diff --git a/chromium/media/capabilities/video_decode_stats_db_impl.h b/chromium/media/capabilities/video_decode_stats_db_impl.h index f2c1cae047c..c9826ee9367 100644 --- a/chromium/media/capabilities/video_decode_stats_db_impl.h +++ b/chromium/media/capabilities/video_decode_stats_db_impl.h @@ -7,6 +7,8 @@ #include <memory> +#include "base/cancelable_callback.h" +#include "base/containers/flat_map.h" #include "base/files/file_path.h" #include "base/memory/weak_ptr.h" #include "components/leveldb_proto/public/proto_database.h" @@ -58,6 +60,8 @@ class MEDIA_EXPORT VideoDecodeStatsDBImpl : public VideoDecodeStatsDB { private: friend class VideoDecodeStatsDBImplTest; + using PendingOpId = int; + // Private constructor only called by tests (friends). Production code // should always use the static Create() method. VideoDecodeStatsDBImpl( @@ -82,16 +86,63 @@ class MEDIA_EXPORT VideoDecodeStatsDBImpl : public VideoDecodeStatsDB { // regardless of how many frames were decoded. static bool GetEnableUnweightedEntries(); + // Creates a PendingOperation using |uma_str| and adds it to |pending_ops_| + // map. Returns PendingOpId for newly started operation. Callers must later + // call CompletePendingOp() with this id to destroy the PendingOperation and + // finalize timing UMA. + PendingOpId StartPendingOp(std::string uma_str); + + // Removes PendingOperation from |pending_ops_| using |op_id_| as a key. This + // destroys the object and triggers timing UMA. + void CompletePendingOp(PendingOpId op_id); + + // Unified handler for timeouts of pending DB operations. PendingOperation + // will be notified that it timed out (to trigger timing UMA) and removed from + // |penidng_ops_|. + void OnPendingOpTimeout(PendingOpId id); + + // Helper to report timing information for DB operations, including when they + // hang indefinitely. + class PendingOperation { + public: + PendingOperation( + std::string uma_str, + std::unique_ptr<base::CancelableOnceClosure> timeout_closure); + // Records task timing UMA if it hasn't already timed out. + virtual ~PendingOperation(); + + // Copies disallowed. Incompatible with move-only members and UMA logging in + // the destructor. + PendingOperation(const PendingOperation&) = delete; + PendingOperation& operator=(const PendingOperation&) = delete; + + // Trigger UMA recording for timeout. + void OnTimeout(); + + private: + friend class VideoDecodeStatsDBImplTest; + + std::string uma_str_; + std::unique_ptr<base::CancelableOnceClosure> timeout_closure_; + base::TimeTicks start_ticks_; + }; + + // Map of operation id -> outstanding PendingOperations. + base::flat_map<PendingOpId, std::unique_ptr<PendingOperation>> pending_ops_; + // Called when the database has been initialized. Will immediately call // |init_cb| to forward |success|. - void OnInit(InitializeCB init_cb, leveldb_proto::Enums::InitStatus status); + void OnInit(PendingOpId id, + InitializeCB init_cb, + leveldb_proto::Enums::InitStatus status); // Returns true if the DB is successfully initialized. bool IsInitialized(); // Passed as the callback for |OnGotDecodeStats| by |AppendDecodeStats| to // update the database once we've read the existing stats entry. - void WriteUpdatedEntry(const VideoDescKey& key, + void WriteUpdatedEntry(PendingOpId op_id, + const VideoDescKey& key, const DecodeStatsEntry& entry, AppendDecodeStatsCB append_done_cb, bool read_success, @@ -99,24 +150,23 @@ class MEDIA_EXPORT VideoDecodeStatsDBImpl : public VideoDecodeStatsDB { // Called when the database has been modified after a call to // |WriteUpdatedEntry|. Will run |append_done_cb| when done. - void OnEntryUpdated(AppendDecodeStatsCB append_done_cb, bool success); + void OnEntryUpdated(PendingOpId op_id, + AppendDecodeStatsCB append_done_cb, + bool success); // Called when GetDecodeStats() operation was performed. |get_stats_cb| // will be run with |success| and a |DecodeStatsEntry| created from // |stats_proto| or nullptr if no entry was found for the requested key. - void OnGotDecodeStats(GetDecodeStatsCB get_stats_cb, + void OnGotDecodeStats(PendingOpId op_id, + GetDecodeStatsCB get_stats_cb, bool success, std::unique_ptr<DecodeStatsProto> stats_proto); - // Internal callback for first step of ClearStats(). Will clear all stats If - // |keys| fetched successfully. - void OnLoadAllKeysForClearing(base::OnceClosure clear_done_cb, - bool success, - std::unique_ptr<std::vector<std::string>> keys); - // Internal callback for OnLoadAllKeysForClearing(), initially triggered by // ClearStats(). Method simply logs |success| and runs |clear_done_cb|. - void OnStatsCleared(base::OnceClosure clear_done_cb, bool success); + void OnStatsCleared(PendingOpId op_id, + base::OnceClosure clear_done_cb, + bool success); // Return true if: // values aren't corrupted nonsense (e.g. way more frames dropped than @@ -130,6 +180,9 @@ class MEDIA_EXPORT VideoDecodeStatsDBImpl : public VideoDecodeStatsDB { wall_clock_ = tick_clock; } + // Next PendingOpId for use in |pending_ops_| map. See StartPendingOp(). + PendingOpId next_op_id_ = 0; + // Indicates whether initialization is completed. Does not indicate whether it // was successful. Will be reset upon calling DestroyStats(). Failed // initialization is signaled by setting |db_| to null. diff --git a/chromium/media/capabilities/video_decode_stats_db_impl_unittest.cc b/chromium/media/capabilities/video_decode_stats_db_impl_unittest.cc index e3a8c742a0a..2c504964dbf 100644 --- a/chromium/media/capabilities/video_decode_stats_db_impl_unittest.cc +++ b/chromium/media/capabilities/video_decode_stats_db_impl_unittest.cc @@ -8,8 +8,8 @@ #include "base/bind.h" #include "base/bind_helpers.h" #include "base/callback.h" +#include "base/check.h" #include "base/files/file_path.h" -#include "base/logging.h" #include "base/memory/ptr_util.h" #include "base/run_loop.h" #include "base/strings/string_number_conversions.h" @@ -65,6 +65,20 @@ class VideoDecodeStatsDBImplTest : public ::testing::Test { std::unique_ptr<FakeDB<DecodeStatsProto>>(fake_db_))); } + ~VideoDecodeStatsDBImplTest() override { + // Tests should always complete any pending operations + VerifyNoPendingOps(); + } + + void VerifyOnePendingOp(std::string op_name) { + EXPECT_EQ(stats_db_->pending_ops_.size(), 1u); + VideoDecodeStatsDBImpl::PendingOperation* pending_op = + stats_db_->pending_ops_.begin()->second.get(); + EXPECT_EQ(pending_op->uma_str_, op_name); + } + + void VerifyNoPendingOps() { EXPECT_TRUE(stats_db_->pending_ops_.empty()); } + int GetMaxFramesPerBuffer() { return VideoDecodeStatsDBImpl::GetMaxFramesPerBuffer(); } @@ -95,7 +109,9 @@ class VideoDecodeStatsDBImplTest : public ::testing::Test { key, entry, base::BindOnce(&VideoDecodeStatsDBImplTest::MockAppendDecodeStatsCb, base::Unretained(this))); + VerifyOnePendingOp("Read"); fake_db_->GetCallback(true); + VerifyOnePendingOp("Write"); fake_db_->UpdateCallback(true); testing::Mock::VerifyAndClearExpectations(this); } @@ -106,6 +122,7 @@ class VideoDecodeStatsDBImplTest : public ::testing::Test { stats_db_->GetDecodeStats( key, base::BindOnce(&VideoDecodeStatsDBImplTest::GetDecodeStatsCb, base::Unretained(this))); + VerifyOnePendingOp("Read"); fake_db_->GetCallback(true); testing::Mock::VerifyAndClearExpectations(this); } @@ -115,6 +132,7 @@ class VideoDecodeStatsDBImplTest : public ::testing::Test { stats_db_->GetDecodeStats( key, base::BindOnce(&VideoDecodeStatsDBImplTest::GetDecodeStatsCb, base::Unretained(this))); + VerifyOnePendingOp("Read"); fake_db_->GetCallback(true); testing::Mock::VerifyAndClearExpectations(this); } @@ -157,7 +175,8 @@ class VideoDecodeStatsDBImplTest : public ::testing::Test { MOCK_METHOD0(MockClearStatsCb, void()); protected: - base::test::TaskEnvironment task_environment_; + base::test::TaskEnvironment task_environment_{ + base::test::TaskEnvironment::TimeSource::MOCK_TIME}; const VideoDescKey kStatsKeyVp9; const VideoDescKey kStatsKeyAvc; @@ -175,9 +194,30 @@ class VideoDecodeStatsDBImplTest : public ::testing::Test { DISALLOW_COPY_AND_ASSIGN(VideoDecodeStatsDBImplTest); }; -TEST_F(VideoDecodeStatsDBImplTest, FailedInitialize) { +TEST_F(VideoDecodeStatsDBImplTest, InitializeFailed) { + stats_db_->Initialize(base::BindOnce( + &VideoDecodeStatsDBImplTest::OnInitialize, base::Unretained(this))); + EXPECT_CALL(*this, OnInitialize(false)); + fake_db_->InitStatusCallback(leveldb_proto::Enums::InitStatus::kError); +} + +TEST_F(VideoDecodeStatsDBImplTest, InitializeTimedOut) { + // Queue up an Initialize. stats_db_->Initialize(base::BindOnce( &VideoDecodeStatsDBImplTest::OnInitialize, base::Unretained(this))); + VerifyOnePendingOp("Initialize"); + + // Move time forward enough to trigger timeout. + EXPECT_CALL(*this, OnInitialize(_)).Times(0); + task_environment_.FastForwardBy(base::TimeDelta::FromSeconds(100)); + task_environment_.RunUntilIdle(); + + // Verify we didn't get an init callback and task is no longer considered + // pending (because it timed out). + testing::Mock::VerifyAndClearExpectations(this); + VerifyNoPendingOps(); + + // Verify callback still works if init completes very late. EXPECT_CALL(*this, OnInitialize(false)); fake_db_->InitStatusCallback(leveldb_proto::Enums::InitStatus::kError); } @@ -204,9 +244,10 @@ TEST_F(VideoDecodeStatsDBImplTest, WriteReadAndClear) { VerifyReadStats(kStatsKeyVp9, aggregate_entry); // Clear all stats from the DB. + EXPECT_CALL(*this, MockClearStatsCb); stats_db_->ClearStats(base::BindOnce( &VideoDecodeStatsDBImplTest::MockClearStatsCb, base::Unretained(this))); - fake_db_->LoadKeysCallback(true); + VerifyOnePendingOp("Clear"); fake_db_->UpdateCallback(true); // Database is now empty. Expect null entry. diff --git a/chromium/media/capture/BUILD.gn b/chromium/media/capture/BUILD.gn index 0f253c9e858..06e45ca2886 100644 --- a/chromium/media/capture/BUILD.gn +++ b/chromium/media/capture/BUILD.gn @@ -309,6 +309,14 @@ jumbo_component("capture_lib") { sources += [ "video/fuchsia/video_capture_device_factory_fuchsia.cc", "video/fuchsia/video_capture_device_factory_fuchsia.h", + "video/fuchsia/video_capture_device_fuchsia.cc", + "video/fuchsia/video_capture_device_fuchsia.h", + ] + deps += [ + "//media/fuchsia/common", + "//third_party/fuchsia-sdk/sdk/fidl/fuchsia.camera3", + "//third_party/fuchsia-sdk/sdk/pkg/sys_cpp", + "//third_party/libyuv", ] } } @@ -385,6 +393,16 @@ test("capture_unittests") { ] } + if (is_fuchsia) { + deps += [ + "//media/fuchsia/camera:test_support", + "//third_party/fuchsia-sdk/sdk/fidl/fuchsia.camera3", + "//third_party/fuchsia-sdk/sdk/fidl/fuchsia.sysmem", + "//third_party/fuchsia-sdk/sdk/pkg/sys_cpp", + ] + sources += [ "video/fuchsia/video_capture_device_fuchsia_test.cc" ] + } + if (is_win) { sources += [ "video/win/video_capture_device_factory_win_unittest.cc", diff --git a/chromium/media/capture/capture_switches.cc b/chromium/media/capture/capture_switches.cc index 6f88b435bd4..6348e859089 100644 --- a/chromium/media/capture/capture_switches.cc +++ b/chromium/media/capture/capture_switches.cc @@ -10,4 +10,14 @@ namespace switches { const char kVideoCaptureUseGpuMemoryBuffer[] = "video-capture-use-gpu-memory-buffer"; +// This is for the same feature controlled by kVideoCaptureUseGpuMemoryBuffer. +// kVideoCaptureUseGpuMemoryBuffer is settled by chromeos overlays. This flag is +// necessary to overwrite the settings via chrome:// flag. The behavior of +// chrome://flag#zero-copy-video-capture is as follows; +// Default : Respect chromeos overlays settings. +// Enabled : Force to enable kVideoCaptureUseGpuMemoryBuffer. +// Disabled : Force to disable kVideoCaptureUseGpuMemoryBuffer. +const char kDisableVideoCaptureUseGpuMemoryBuffer[] = + "disable-video-capture-use-gpu-memory-buffer"; + } // namespace switches diff --git a/chromium/media/capture/capture_switches.h b/chromium/media/capture/capture_switches.h index cccc92ae5a4..dfc352ab54d 100644 --- a/chromium/media/capture/capture_switches.h +++ b/chromium/media/capture/capture_switches.h @@ -11,6 +11,8 @@ namespace switches { CAPTURE_EXPORT extern const char kVideoCaptureUseGpuMemoryBuffer[]; +CAPTURE_EXPORT extern const char kDisableVideoCaptureUseGpuMemoryBuffer[]; + } // namespace switches #endif // MEDIA_CAPTURE_CAPTURE_SWITCHES_H_ diff --git a/chromium/media/capture/content/capture_resolution_chooser.cc b/chromium/media/capture/content/capture_resolution_chooser.cc index f665bf30dc5..496ae311fa9 100644 --- a/chromium/media/capture/content/capture_resolution_chooser.cc +++ b/chromium/media/capture/content/capture_resolution_chooser.cc @@ -7,6 +7,7 @@ #include <algorithm> #include <limits> +#include "base/logging.h" #include "base/strings/string_util.h" #include "media/base/video_util.h" diff --git a/chromium/media/capture/mojom/video_capture_types.mojom b/chromium/media/capture/mojom/video_capture_types.mojom index 5b09aac62ad..eaeff60009f 100644 --- a/chromium/media/capture/mojom/video_capture_types.mojom +++ b/chromium/media/capture/mojom/video_capture_types.mojom @@ -74,6 +74,7 @@ enum VideoCaptureApi { ANDROID_API2_LEGACY, ANDROID_API2_FULL, ANDROID_API2_LIMITED, + FUCHSIA_CAMERA3, VIRTUAL_DEVICE, UNKNOWN }; @@ -213,7 +214,14 @@ enum VideoCaptureError { kMacDeckLinkUnsupportedPixelFormat, kMacAvFoundationReceivedAVCaptureSessionRuntimeErrorNotification, kAndroidApi2ErrorConfiguringCamera, - kCrosHalV3DeviceDelegateFailedToFlush + kCrosHalV3DeviceDelegateFailedToFlush, + kFuchsiaCameraDeviceDisconnected, + kFuchsiaCameraStreamDisconnected, + kFuchsiaSysmemDidNotSetImageFormat, + kFuchsiaSysmemInvalidBufferIndex, + kFuchsiaSysmemInvalidBufferSize, + kFuchsiaUnsupportedPixelFormat, + kFuchsiaFailedToMapSysmemBuffer, }; enum VideoCaptureFrameDropReason { diff --git a/chromium/media/capture/mojom/video_capture_types_mojom_traits.cc b/chromium/media/capture/mojom/video_capture_types_mojom_traits.cc index 32dc4185a8c..a910706f1ac 100644 --- a/chromium/media/capture/mojom/video_capture_types_mojom_traits.cc +++ b/chromium/media/capture/mojom/video_capture_types_mojom_traits.cc @@ -686,6 +686,21 @@ EnumTraits<media::mojom::VideoCaptureError, media::VideoCaptureError>::ToMojom( case media::VideoCaptureError::kCrosHalV3DeviceDelegateFailedToFlush: return media::mojom::VideoCaptureError:: kCrosHalV3DeviceDelegateFailedToFlush; + case media::VideoCaptureError::kFuchsiaCameraDeviceDisconnected: + return media::mojom::VideoCaptureError::kFuchsiaCameraDeviceDisconnected; + case media::VideoCaptureError::kFuchsiaCameraStreamDisconnected: + return media::mojom::VideoCaptureError::kFuchsiaCameraStreamDisconnected; + case media::VideoCaptureError::kFuchsiaSysmemDidNotSetImageFormat: + return media::mojom::VideoCaptureError:: + kFuchsiaSysmemDidNotSetImageFormat; + case media::VideoCaptureError::kFuchsiaSysmemInvalidBufferIndex: + return media::mojom::VideoCaptureError::kFuchsiaSysmemInvalidBufferIndex; + case media::VideoCaptureError::kFuchsiaSysmemInvalidBufferSize: + return media::mojom::VideoCaptureError::kFuchsiaSysmemInvalidBufferSize; + case media::VideoCaptureError::kFuchsiaUnsupportedPixelFormat: + return media::mojom::VideoCaptureError::kFuchsiaUnsupportedPixelFormat; + case media::VideoCaptureError::kFuchsiaFailedToMapSysmemBuffer: + return media::mojom::VideoCaptureError::kFuchsiaFailedToMapSysmemBuffer; } NOTREACHED(); return media::mojom::VideoCaptureError::kNone; @@ -1207,6 +1222,27 @@ bool EnumTraits<media::mojom::VideoCaptureError, media::VideoCaptureError>:: case media::mojom::VideoCaptureError::kCrosHalV3DeviceDelegateFailedToFlush: *output = media::VideoCaptureError::kCrosHalV3DeviceDelegateFailedToFlush; return true; + case media::mojom::VideoCaptureError::kFuchsiaCameraDeviceDisconnected: + *output = media::VideoCaptureError::kFuchsiaCameraDeviceDisconnected; + return true; + case media::mojom::VideoCaptureError::kFuchsiaCameraStreamDisconnected: + *output = media::VideoCaptureError::kFuchsiaCameraStreamDisconnected; + return true; + case media::mojom::VideoCaptureError::kFuchsiaSysmemDidNotSetImageFormat: + *output = media::VideoCaptureError::kFuchsiaSysmemDidNotSetImageFormat; + return true; + case media::mojom::VideoCaptureError::kFuchsiaSysmemInvalidBufferIndex: + *output = media::VideoCaptureError::kFuchsiaSysmemInvalidBufferIndex; + return true; + case media::mojom::VideoCaptureError::kFuchsiaSysmemInvalidBufferSize: + *output = media::VideoCaptureError::kFuchsiaSysmemInvalidBufferSize; + return true; + case media::mojom::VideoCaptureError::kFuchsiaUnsupportedPixelFormat: + *output = media::VideoCaptureError::kFuchsiaUnsupportedPixelFormat; + return true; + case media::mojom::VideoCaptureError::kFuchsiaFailedToMapSysmemBuffer: + *output = media::VideoCaptureError::kFuchsiaFailedToMapSysmemBuffer; + return true; } NOTREACHED(); return false; @@ -1510,6 +1546,8 @@ EnumTraits<media::mojom::VideoCaptureApi, media::VideoCaptureApi>::ToMojom( return media::mojom::VideoCaptureApi::ANDROID_API2_FULL; case media::VideoCaptureApi::ANDROID_API2_LIMITED: return media::mojom::VideoCaptureApi::ANDROID_API2_LIMITED; + case media::VideoCaptureApi::FUCHSIA_CAMERA3: + return media::mojom::VideoCaptureApi::FUCHSIA_CAMERA3; case media::VideoCaptureApi::VIRTUAL_DEVICE: return media::mojom::VideoCaptureApi::VIRTUAL_DEVICE; case media::VideoCaptureApi::UNKNOWN: @@ -1554,6 +1592,9 @@ bool EnumTraits<media::mojom::VideoCaptureApi, media::VideoCaptureApi>:: case media::mojom::VideoCaptureApi::ANDROID_API2_LIMITED: *output = media::VideoCaptureApi::ANDROID_API2_LIMITED; return true; + case media::mojom::VideoCaptureApi::FUCHSIA_CAMERA3: + *output = media::VideoCaptureApi::FUCHSIA_CAMERA3; + return true; case media::mojom::VideoCaptureApi::VIRTUAL_DEVICE: *output = media::VideoCaptureApi::VIRTUAL_DEVICE; return true; diff --git a/chromium/media/capture/video/android/java/src/org/chromium/media/VideoCaptureCamera2.java b/chromium/media/capture/video/android/java/src/org/chromium/media/VideoCaptureCamera2.java index 7a1ad8db0e1..f849089585b 100644 --- a/chromium/media/capture/video/android/java/src/org/chromium/media/VideoCaptureCamera2.java +++ b/chromium/media/capture/video/android/java/src/org/chromium/media/VideoCaptureCamera2.java @@ -575,7 +575,9 @@ public class VideoCaptureCamera2 extends VideoCapture { } } try { - if (cameraCharacteristics.get(CameraCharacteristics.CONTROL_AE_LOCK_AVAILABLE)) { + Boolean ae_lock_available = + cameraCharacteristics.get(CameraCharacteristics.CONTROL_AE_LOCK_AVAILABLE); + if (ae_lock_available != null && ae_lock_available.booleanValue()) { exposureModes.add(Integer.valueOf(AndroidMeteringMode.FIXED)); } } catch (NoSuchFieldError e) { @@ -622,7 +624,9 @@ public class VideoCaptureCamera2 extends VideoCapture { } } try { - if (cameraCharacteristics.get(CameraCharacteristics.CONTROL_AWB_LOCK_AVAILABLE)) { + Boolean awb_lock_available = + cameraCharacteristics.get(CameraCharacteristics.CONTROL_AWB_LOCK_AVAILABLE); + if (awb_lock_available != null && awb_lock_available.booleanValue()) { whiteBalanceModes.add(Integer.valueOf(AndroidMeteringMode.FIXED)); } } catch (NoSuchFieldError e) { diff --git a/chromium/media/capture/video/chromeos/camera_app_device_impl.cc b/chromium/media/capture/video/chromeos/camera_app_device_impl.cc index 3120a85871d..d81afe5b3e5 100644 --- a/chromium/media/capture/video/chromeos/camera_app_device_impl.cc +++ b/chromium/media/capture/video/chromeos/camera_app_device_impl.cc @@ -28,13 +28,6 @@ ReprocessTask::ReprocessTask(ReprocessTask&& other) ReprocessTask::~ReprocessTask() = default; -bool CameraAppDeviceImpl::SizeComparator::operator()( - const gfx::Size& size_1, - const gfx::Size& size_2) const { - return size_1.width() < size_2.width() || (size_1.width() == size_2.width() && - size_1.height() < size_2.height()); -} - // static int CameraAppDeviceImpl::GetReprocessReturnCode( cros::mojom::Effect effect, @@ -107,16 +100,16 @@ void CameraAppDeviceImpl::ConsumeReprocessOptions( std::move(consumption_callback).Run(std::move(result_task_queue)); } -void CameraAppDeviceImpl::GetFpsRange(const gfx::Size& resolution, - GetFpsRangeCallback callback) { +base::Optional<gfx::Range> CameraAppDeviceImpl::GetFpsRange() { base::AutoLock lock(fps_ranges_lock_); - auto it = resolution_fps_range_map_.find(resolution); - if (it == resolution_fps_range_map_.end()) { - std::move(callback).Run({}); - return; - } - std::move(callback).Run(it->second); + return specified_fps_range_; +} + +gfx::Size CameraAppDeviceImpl::GetStillCaptureResolution() { + base::AutoLock lock(still_capture_resolution_lock_); + + return still_capture_resolution_; } cros::mojom::CaptureIntent CameraAppDeviceImpl::GetCaptureIntent() { @@ -144,19 +137,6 @@ void CameraAppDeviceImpl::OnShutterDone() { } } -void CameraAppDeviceImpl::SetReprocessResult( - SetReprocessOptionCallback callback, - const int32_t status, - media::mojom::BlobPtr blob) { - auto callback_on_mojo_thread = base::BindOnce( - [](const int32_t status, media::mojom::BlobPtr blob, - SetReprocessOptionCallback callback) { - std::move(callback).Run(status, std::move(blob)); - }, - status, std::move(blob), std::move(callback)); - task_runner_->PostTask(FROM_HERE, std::move(callback_on_mojo_thread)); -} - void CameraAppDeviceImpl::GetCameraInfo(GetCameraInfoCallback callback) { DCHECK(camera_info_); std::move(callback).Run(camera_info_.Clone()); @@ -183,8 +163,7 @@ void CameraAppDeviceImpl::SetReprocessOption( reprocess_task_queue_.push(std::move(task)); } -void CameraAppDeviceImpl::SetFpsRange(const gfx::Size& resolution, - const gfx::Range& fps_range, +void CameraAppDeviceImpl::SetFpsRange(const gfx::Range& fps_range, SetFpsRangeCallback callback) { const int entry_length = 2; @@ -208,19 +187,20 @@ void CameraAppDeviceImpl::SetFpsRange(const gfx::Size& resolution, base::AutoLock lock(fps_ranges_lock_); - if (!is_valid) { - // If the input range is invalid, we should still clear the cache range so - // that it will fallback to use default fps range rather than the cache one. - auto it = resolution_fps_range_map_.find(resolution); - if (it != resolution_fps_range_map_.end()) { - resolution_fps_range_map_.erase(it); - } - std::move(callback).Run(false); - return; + if (is_valid) { + specified_fps_range_ = fps_range; + } else { + specified_fps_range_ = {}; } + std::move(callback).Run(is_valid); +} - resolution_fps_range_map_[resolution] = fps_range; - std::move(callback).Run(true); +void CameraAppDeviceImpl::SetStillCaptureResolution( + const gfx::Size& resolution, + SetStillCaptureResolutionCallback callback) { + base::AutoLock lock(still_capture_resolution_lock_); + still_capture_resolution_ = resolution; + std::move(callback).Run(); } void CameraAppDeviceImpl::SetCaptureIntent( @@ -294,4 +274,17 @@ void CameraAppDeviceImpl::DisableEeNr(ReprocessTask* task) { task->extra_metadata.push_back(std::move(nr_entry)); } +void CameraAppDeviceImpl::SetReprocessResult( + SetReprocessOptionCallback callback, + const int32_t status, + media::mojom::BlobPtr blob) { + auto callback_on_mojo_thread = base::BindOnce( + [](const int32_t status, media::mojom::BlobPtr blob, + SetReprocessOptionCallback callback) { + std::move(callback).Run(status, std::move(blob)); + }, + status, std::move(blob), std::move(callback)); + task_runner_->PostTask(FROM_HERE, std::move(callback_on_mojo_thread)); +} + } // namespace media diff --git a/chromium/media/capture/video/chromeos/camera_app_device_impl.h b/chromium/media/capture/video/chromeos/camera_app_device_impl.h index 2f4ae3a20c5..a0853f0ac2e 100644 --- a/chromium/media/capture/video/chromeos/camera_app_device_impl.h +++ b/chromium/media/capture/video/chromeos/camera_app_device_impl.h @@ -42,22 +42,19 @@ constexpr uint32_t kPortraitModeVendorKey = 0x80000000; constexpr uint32_t kPortraitModeSegmentationResultVendorKey = 0x80000001; constexpr int32_t kReprocessSuccess = 0; +// Implementation of CameraAppDevice that is used as the ommunication bridge +// between Chrome Camera App (CCA) and the ChromeOS Video Capture Device. By +// using this, we can do more complicated operations on cameras which is not +// supported by Chrome API. class CAPTURE_EXPORT CameraAppDeviceImpl : public cros::mojom::CameraAppDevice { public: - struct SizeComparator { - bool operator()(const gfx::Size& size_1, const gfx::Size& size_2) const; - }; - - using GetFpsRangeCallback = - base::OnceCallback<void(base::Optional<gfx::Range>)>; - - using ResolutionFpsRangeMap = - base::flat_map<gfx::Size, gfx::Range, SizeComparator>; - + // Retrieve the return code for reprocess |effect| from the |metadata|. static int GetReprocessReturnCode( cros::mojom::Effect effect, const cros::mojom::CameraMetadataPtr* metadata); + // Construct a ReprocessTaskQueue for regular capture with + // |take_photo_callback|. static ReprocessTaskQueue GetSingleShotReprocessOptions( media::mojom::ImageCapture::TakePhotoCallback take_photo_callback); @@ -65,52 +62,58 @@ class CAPTURE_EXPORT CameraAppDeviceImpl : public cros::mojom::CameraAppDevice { cros::mojom::CameraInfoPtr camera_info); ~CameraAppDeviceImpl() override; + // Binds the mojo receiver to this implementation. void BindReceiver( mojo::PendingReceiver<cros::mojom::CameraAppDevice> receiver); + // Consumes all the pending reprocess tasks if there is any and eventually + // generates a ReprocessTaskQueue which contains: + // 1. A regular capture task with |take_photo_callback|. + // 2. One or more reprocess tasks if there is any. + // And passes the generated ReprocessTaskQueue through |consumption_callback|. void ConsumeReprocessOptions( media::mojom::ImageCapture::TakePhotoCallback take_photo_callback, base::OnceCallback<void(ReprocessTaskQueue)> consumption_callback); - void GetFpsRange(const gfx::Size& resolution, GetFpsRangeCallback callback); + // Retrieves the fps range if it is specified by the app. + base::Optional<gfx::Range> GetFpsRange(); + + // Retrieves the corresponding capture resolution which is specified by the + // app. + gfx::Size GetStillCaptureResolution(); + // Gets the capture intent which is specified by the app. cros::mojom::CaptureIntent GetCaptureIntent(); + // Delivers the result |metadata| with its |stream_type| to the metadata + // observers. void OnResultMetadataAvailable(const cros::mojom::CameraMetadataPtr& metadata, const cros::mojom::StreamType stream_type); + // Notifies the camera event observers that the shutter is finished. void OnShutterDone(); - void SetReprocessResult(SetReprocessOptionCallback callback, - const int32_t status, - media::mojom::BlobPtr blob); - // cros::mojom::CameraAppDevice implementations. void GetCameraInfo(GetCameraInfoCallback callback) override; - void SetReprocessOption(cros::mojom::Effect effect, SetReprocessOptionCallback callback) override; - - void SetFpsRange(const gfx::Size& resolution, - const gfx::Range& fps_range, + void SetFpsRange(const gfx::Range& fps_range, SetFpsRangeCallback callback) override; - + void SetStillCaptureResolution( + const gfx::Size& resolution, + SetStillCaptureResolutionCallback callback) override; void SetCaptureIntent(cros::mojom::CaptureIntent capture_intent, SetCaptureIntentCallback callback) override; - void AddResultMetadataObserver( mojo::PendingRemote<cros::mojom::ResultMetadataObserver> observer, cros::mojom::StreamType streamType, AddResultMetadataObserverCallback callback) override; - void RemoveResultMetadataObserver( uint32_t id, RemoveResultMetadataObserverCallback callback) override; - void AddCameraEventObserver( mojo::PendingRemote<cros::mojom::CameraEventObserver> observer, AddCameraEventObserverCallback callback) override; - void RemoveCameraEventObserver( uint32_t id, RemoveCameraEventObserverCallback callback) override; @@ -118,6 +121,10 @@ class CAPTURE_EXPORT CameraAppDeviceImpl : public cros::mojom::CameraAppDevice { private: static void DisableEeNr(ReprocessTask* task); + void SetReprocessResult(SetReprocessOptionCallback callback, + const int32_t status, + media::mojom::BlobPtr blob); + std::string device_id_; mojo::ReceiverSet<cros::mojom::CameraAppDevice> receivers_; @@ -126,34 +133,38 @@ class CAPTURE_EXPORT CameraAppDeviceImpl : public cros::mojom::CameraAppDevice { const scoped_refptr<base::SingleThreadTaskRunner> task_runner_; + // The queue will be enqueued and dequeued from different threads. base::Lock reprocess_tasks_lock_; + base::queue<ReprocessTask> reprocess_task_queue_ + GUARDED_BY(reprocess_tasks_lock_); - base::queue<ReprocessTask> reprocess_task_queue_; - + // It will be inserted and read from different threads. base::Lock fps_ranges_lock_; + base::Optional<gfx::Range> specified_fps_range_ GUARDED_BY(fps_ranges_lock_); - ResolutionFpsRangeMap resolution_fps_range_map_; + // It will be inserted and read from different threads. + base::Lock still_capture_resolution_lock_; + gfx::Size still_capture_resolution_ + GUARDED_BY(still_capture_resolution_lock_); + // It will be modified and read from different threads. base::Lock capture_intent_lock_; + cros::mojom::CaptureIntent capture_intent_ GUARDED_BY(capture_intent_lock_); - cros::mojom::CaptureIntent capture_intent_; - + // Those maps will be changed and used from different threads. base::Lock metadata_observers_lock_; - - base::Lock camera_event_observers_lock_; - - uint32_t next_metadata_observer_id_; - - uint32_t next_camera_event_observer_id_; - + uint32_t next_metadata_observer_id_ GUARDED_BY(metadata_observers_lock_); base::flat_map<uint32_t, mojo::Remote<cros::mojom::ResultMetadataObserver>> - metadata_observers_; - + metadata_observers_ GUARDED_BY(metadata_observers_lock_); base::flat_map<cros::mojom::StreamType, base::flat_set<uint32_t>> - stream_metadata_observer_ids_; + stream_metadata_observer_ids_ GUARDED_BY(metadata_observers_lock_); + // Those maps will be changed and used from different threads. + base::Lock camera_event_observers_lock_; + uint32_t next_camera_event_observer_id_ + GUARDED_BY(camera_event_observers_lock_); base::flat_map<uint32_t, mojo::Remote<cros::mojom::CameraEventObserver>> - camera_event_observers_; + camera_event_observers_ GUARDED_BY(camera_event_observers_lock_); std::unique_ptr<base::WeakPtrFactory<CameraAppDeviceImpl>> weak_ptr_factory_; diff --git a/chromium/media/capture/video/chromeos/camera_device_delegate.cc b/chromium/media/capture/video/chromeos/camera_device_delegate.cc index 8d5fae0d635..c361ae56ad9 100644 --- a/chromium/media/capture/video/chromeos/camera_device_delegate.cc +++ b/chromium/media/capture/video/chromeos/camera_device_delegate.cc @@ -621,25 +621,15 @@ void CameraDeviceDelegate::ConfigureStreams( int32_t blob_width = 0; int32_t blob_height = 0; if (require_photo) { - std::vector<gfx::Size> blob_resolutions; - GetStreamResolutions( - static_metadata_, cros::mojom::Camera3StreamType::CAMERA3_STREAM_OUTPUT, - cros::mojom::HalPixelFormat::HAL_PIXEL_FORMAT_BLOB, &blob_resolutions); - if (blob_resolutions.empty()) { + auto blob_resolution = GetBlobResolution(new_blob_resolution); + if (blob_resolution.IsEmpty()) { LOG(ERROR) << "Failed to configure streans: No BLOB resolution found."; return; } - if (new_blob_resolution.has_value() && - std::find(blob_resolutions.begin(), blob_resolutions.end(), - *new_blob_resolution) != blob_resolutions.end()) { - blob_width = new_blob_resolution->width(); - blob_height = new_blob_resolution->height(); - } else { - // Use the largest resolution as default. - blob_width = blob_resolutions.back().width(); - blob_height = blob_resolutions.back().height(); - } + blob_width = blob_resolution.width(); + blob_height = blob_resolution.height(); + cros::mojom::Camera3StreamPtr still_capture_stream = cros::mojom::Camera3Stream::New(); still_capture_stream->id = static_cast<uint64_t>(StreamType::kJpegOutput); @@ -875,11 +865,7 @@ void CameraDeviceDelegate::OnConstructedDefaultPreviewRequestSettings( } if (camera_app_device_) { - camera_app_device_->GetFpsRange( - chrome_capture_params_.requested_format.frame_size, - media::BindToCurrentLoop( - base::BindOnce(&CameraDeviceDelegate::OnGotFpsRange, GetWeakPtr(), - std::move(settings)))); + OnGotFpsRange(std::move(settings), camera_app_device_->GetFpsRange()); } else { OnGotFpsRange(std::move(settings), {}); } @@ -947,6 +933,35 @@ void CameraDeviceDelegate::OnGotFpsRange( } } +gfx::Size CameraDeviceDelegate::GetBlobResolution( + base::Optional<gfx::Size> new_blob_resolution) { + std::vector<gfx::Size> blob_resolutions; + GetStreamResolutions( + static_metadata_, cros::mojom::Camera3StreamType::CAMERA3_STREAM_OUTPUT, + cros::mojom::HalPixelFormat::HAL_PIXEL_FORMAT_BLOB, &blob_resolutions); + if (blob_resolutions.empty()) { + return {}; + } + + // Try the given blob resolution first. If it is invalid, try the + // resolution specified through Mojo API as a fallback. If it fails too, + // use the largest resolution as default. + if (new_blob_resolution.has_value() && + base::Contains(blob_resolutions, *new_blob_resolution)) { + return *new_blob_resolution; + } + + if (camera_app_device_) { + auto specified_capture_resolution = + camera_app_device_->GetStillCaptureResolution(); + if (!specified_capture_resolution.IsEmpty() && + base::Contains(blob_resolutions, specified_capture_resolution)) { + return specified_capture_resolution; + } + } + return blob_resolutions.back(); +} + void CameraDeviceDelegate::ProcessCaptureRequest( cros::mojom::Camera3CaptureRequestPtr request, base::OnceCallback<void(int32_t)> callback) { diff --git a/chromium/media/capture/video/chromeos/camera_device_delegate.h b/chromium/media/capture/video/chromeos/camera_device_delegate.h index 7440e2b5910..a8aceeea4c0 100644 --- a/chromium/media/capture/video/chromeos/camera_device_delegate.h +++ b/chromium/media/capture/video/chromeos/camera_device_delegate.h @@ -164,6 +164,8 @@ class CAPTURE_EXPORT CameraDeviceDelegate final { void OnGotFpsRange(cros::mojom::CameraMetadataPtr settings, base::Optional<gfx::Range> specified_fps_range); + gfx::Size GetBlobResolution(base::Optional<gfx::Size> new_blob_resolution); + // StreamCaptureInterface implementations. These methods are called by // |stream_buffer_manager_| on |ipc_task_runner_|. void ProcessCaptureRequest(cros::mojom::Camera3CaptureRequestPtr request, diff --git a/chromium/media/capture/video/chromeos/gpu_memory_buffer_tracker.cc b/chromium/media/capture/video/chromeos/gpu_memory_buffer_tracker.cc index dc3b45df656..a1be18153d4 100644 --- a/chromium/media/capture/video/chromeos/gpu_memory_buffer_tracker.cc +++ b/chromium/media/capture/video/chromeos/gpu_memory_buffer_tracker.cc @@ -4,7 +4,8 @@ #include "media/capture/video/chromeos/gpu_memory_buffer_tracker.h" -#include "base/logging.h" +#include "base/check.h" +#include "base/notreached.h" #include "media/capture/video/chromeos/pixel_format_utils.h" #include "media/capture/video/video_capture_buffer_handle.h" #include "ui/gfx/geometry/size.h" diff --git a/chromium/media/capture/video/chromeos/mock_vendor_tag_ops.cc b/chromium/media/capture/video/chromeos/mock_vendor_tag_ops.cc index cb828d2de33..07529a63bbe 100644 --- a/chromium/media/capture/video/chromeos/mock_vendor_tag_ops.cc +++ b/chromium/media/capture/video/chromeos/mock_vendor_tag_ops.cc @@ -5,7 +5,7 @@ #include "media/capture/video/chromeos/mock_vendor_tag_ops.h" #include "base/bind.h" -#include "base/logging.h" +#include "base/check.h" #include "base/synchronization/waitable_event.h" namespace media { diff --git a/chromium/media/capture/video/chromeos/mojom/camera_app.mojom b/chromium/media/capture/video/chromeos/mojom/camera_app.mojom index 9f4d63d0ec0..7bc0da4e710 100644 --- a/chromium/media/capture/video/chromeos/mojom/camera_app.mojom +++ b/chromium/media/capture/video/chromeos/mojom/camera_app.mojom @@ -86,12 +86,15 @@ interface CameraAppDevice { => (int32 status, media.mojom.Blob? blob); // Sets the fps range for upcoming configured camera stream. - // The caller sets the |fps_range| for target |resolution|. + // The caller sets the |fps_range|. // If the given fps range is valid and set successfully, |is_success| returns // true. If the given fps range is invalid, the fps range which is cached // previously will be cleared and |is_success| will return false. - SetFpsRange(gfx.mojom.Size resolution, gfx.mojom.Range fps_range) - => (bool is_success); + SetFpsRange(gfx.mojom.Range fps_range) => (bool is_success); + + // Sets the |resolution| for still capture stream which should be + // configured later so that we can configure the most suitable properties. + SetStillCaptureResolution(gfx.mojom.Size resolution) => (); // Sets the intent for the upcoming capture session. The underlying video // capture device should configure the streams accordingly. Returns an empty diff --git a/chromium/media/capture/video/chromeos/request_manager.cc b/chromium/media/capture/video/chromeos/request_manager.cc index 4908f3d8c34..9b93346ad4f 100644 --- a/chromium/media/capture/video/chromeos/request_manager.cc +++ b/chromium/media/capture/video/chromeos/request_manager.cc @@ -227,11 +227,10 @@ void RequestManager::UnsetRepeatingCaptureMetadata( } void RequestManager::SetJpegOrientation( - cros::mojom::CameraMetadataPtr* settings) { + cros::mojom::CameraMetadataPtr* settings, + int32_t orientation) { auto e = BuildMetadataEntry( - cros::mojom::CameraMetadataTag::ANDROID_JPEG_ORIENTATION, - base::checked_cast<int32_t>( - device_context_->GetCameraFrameOrientation())); + cros::mojom::CameraMetadataTag::ANDROID_JPEG_ORIENTATION, orientation); AddOrUpdateMetadataEntry(settings, std::move(e)); } @@ -319,6 +318,7 @@ void RequestManager::PrepareCaptureRequest() { pending_result.input_buffer_id = input_buffer_id; pending_result.reprocess_effect = reprocess_effect; pending_result.still_capture_callback = std::move(callback); + pending_result.orientation = device_context_->GetCameraFrameOrientation(); // For reprocess supported devices, bind the ReprocessTaskQueue with this // frame number. Once the shot result is returned, we will rebind the @@ -377,7 +377,7 @@ bool RequestManager::TryPrepareReprocessRequest( // Prepare metadata by adding extra metadata. *settings = reprocess_job_info->metadata.Clone(); SetSensorTimestamp(settings, reprocess_job_info->shutter_timestamp); - SetJpegOrientation(settings); + SetJpegOrientation(settings, reprocess_job_info->orientation); for (auto& metadata : task.extra_metadata) { AddOrUpdateMetadataEntry(settings, std::move(metadata)); } @@ -440,7 +440,7 @@ bool RequestManager::TryPrepareOneShotRequest( take_photo_callback_queue_.pop(); *settings = std::move(take_photo_settings_queue_.front()); - SetJpegOrientation(settings); + SetJpegOrientation(settings, device_context_->GetCameraFrameOrientation()); } SetZeroShutterLag(settings, true); take_photo_settings_queue_.pop(); @@ -795,7 +795,8 @@ void RequestManager::SubmitCaptureResult( DCHECK_GT(pending_result.shutter_timestamp, 0UL); ReprocessJobInfo reprocess_job_info( std::move(frame_number_reprocess_tasks_map_[frame_number]), - std::move(pending_result.metadata), pending_result.shutter_timestamp); + std::move(pending_result.metadata), pending_result.shutter_timestamp, + pending_result.orientation); buffer_id_reprocess_job_info_map_.emplace(buffer_ipc_id, std::move(reprocess_job_info)); frame_number_reprocess_tasks_map_.erase(frame_number); @@ -947,15 +948,18 @@ RequestManager::CaptureResult::~CaptureResult() = default; RequestManager::ReprocessJobInfo::ReprocessJobInfo( ReprocessTaskQueue queue, cros::mojom::CameraMetadataPtr metadata, - uint64_t timestamp) + uint64_t timestamp, + int32_t orientation) : task_queue(std::move(queue)), metadata(std::move(metadata)), - shutter_timestamp(timestamp) {} + shutter_timestamp(timestamp), + orientation(orientation) {} RequestManager::ReprocessJobInfo::ReprocessJobInfo(ReprocessJobInfo&& info) : task_queue(std::move(info.task_queue)), metadata(std::move(info.metadata)), - shutter_timestamp(info.shutter_timestamp) {} + shutter_timestamp(info.shutter_timestamp), + orientation(info.orientation) {} RequestManager::ReprocessJobInfo::~ReprocessJobInfo() = default; diff --git a/chromium/media/capture/video/chromeos/request_manager.h b/chromium/media/capture/video/chromeos/request_manager.h index 4d3afac35f6..0f2c89e1601 100644 --- a/chromium/media/capture/video/chromeos/request_manager.h +++ b/chromium/media/capture/video/chromeos/request_manager.h @@ -119,6 +119,9 @@ class CAPTURE_EXPORT RequestManager final cros::mojom::Effect reprocess_effect; // The input buffer id for this capture request. base::Optional<uint64_t> input_buffer_id; + // The orientation which is stored at the time the request is prepared. It + // can be used to construct the reprocess job info when the result is back. + int32_t orientation; }; RequestManager(mojo::PendingReceiver<cros::mojom::Camera3CallbackOps> @@ -207,17 +210,20 @@ class CAPTURE_EXPORT RequestManager final struct ReprocessJobInfo { ReprocessJobInfo(ReprocessTaskQueue queue, cros::mojom::CameraMetadataPtr metadata, - uint64_t timestamp); + uint64_t timestamp, + int32_t orientation); ReprocessJobInfo(ReprocessJobInfo&& info); ~ReprocessJobInfo(); ReprocessTaskQueue task_queue; cros::mojom::CameraMetadataPtr metadata; uint64_t shutter_timestamp; + int32_t orientation; }; // Puts Jpeg orientation information into the metadata. - void SetJpegOrientation(cros::mojom::CameraMetadataPtr* settings); + void SetJpegOrientation(cros::mojom::CameraMetadataPtr* settings, + int32_t orientation); // Puts sensor timestamp into the metadata for reprocess request. void SetSensorTimestamp(cros::mojom::CameraMetadataPtr* settings, diff --git a/chromium/media/capture/video/chromeos/video_capture_jpeg_decoder_impl.cc b/chromium/media/capture/video/chromeos/video_capture_jpeg_decoder_impl.cc index d1a11c243a3..665ac4eb39e 100644 --- a/chromium/media/capture/video/chromeos/video_capture_jpeg_decoder_impl.cc +++ b/chromium/media/capture/video/chromeos/video_capture_jpeg_decoder_impl.cc @@ -215,9 +215,8 @@ void VideoCaptureJpegDecoderImpl::FinishInitialization() { decoder_task_runner_, std::move(remote_decoder)); decoder_->InitializeAsync( - this, - base::BindRepeating(&VideoCaptureJpegDecoderImpl::OnInitializationDone, - weak_ptr_factory_.GetWeakPtr())); + this, base::BindOnce(&VideoCaptureJpegDecoderImpl::OnInitializationDone, + weak_ptr_factory_.GetWeakPtr())); } void VideoCaptureJpegDecoderImpl::OnInitializationDone(bool success) { diff --git a/chromium/media/capture/video/fuchsia/video_capture_device_factory_fuchsia.cc b/chromium/media/capture/video/fuchsia/video_capture_device_factory_fuchsia.cc index 42c2e28885d..32cf755b561 100644 --- a/chromium/media/capture/video/fuchsia/video_capture_device_factory_fuchsia.cc +++ b/chromium/media/capture/video/fuchsia/video_capture_device_factory_fuchsia.cc @@ -4,32 +4,257 @@ #include "media/capture/video/fuchsia/video_capture_device_factory_fuchsia.h" +#include <lib/sys/cpp/component_context.h> + +#include "base/check_op.h" +#include "base/fuchsia/default_context.h" +#include "base/fuchsia/fuchsia_logging.h" +#include "base/location.h" #include "base/logging.h" +#include "base/strings/string_number_conversions.h" +#include "base/threading/thread_task_runner_handle.h" +#include "base/time/time.h" +#include "media/capture/video/fuchsia/video_capture_device_fuchsia.h" namespace media { -VideoCaptureDeviceFactoryFuchsia::VideoCaptureDeviceFactoryFuchsia() = default; -VideoCaptureDeviceFactoryFuchsia::~VideoCaptureDeviceFactoryFuchsia() = default; +class VideoCaptureDeviceFactoryFuchsia::DeviceInfoFetcher { + public: + DeviceInfoFetcher(uint64_t device_id, fuchsia::camera3::DevicePtr device) + : device_id_(device_id), device_(std::move(device)) { + device_.set_error_handler( + fit::bind_member(this, &DeviceInfoFetcher::OnError)); + device_->GetIdentifier( + fit::bind_member(this, &DeviceInfoFetcher::OnDescription)); + device_->GetConfigurations( + fit::bind_member(this, &DeviceInfoFetcher::OnConfigurations)); + } + + ~DeviceInfoFetcher() = default; + + DeviceInfoFetcher(const DeviceInfoFetcher&) = delete; + const DeviceInfoFetcher& operator=(const DeviceInfoFetcher&) = delete; + + void WaitResults() { + DCHECK(!wait_results_run_loop_); + + if (have_results()) + return; + + if (!device_) + return; + + wait_results_run_loop_.emplace(); + wait_results_run_loop_->Run(); + wait_results_run_loop_.reset(); + } + + bool have_results() const { return description_ && formats_; } + bool is_usable() const { return device_ || have_results(); } + + uint64_t device_id() const { return device_id_; } + const std::string& description() const { return description_.value(); } + const VideoCaptureFormats& formats() const { return formats_.value(); } + + private: + void OnError(zx_status_t status) { + ZX_LOG(ERROR, status) << "fuchsia.camera3.Device disconnected"; + + if (wait_results_run_loop_) + wait_results_run_loop_->Quit(); + } + + void OnDescription(fidl::StringPtr identifier) { + description_ = identifier.value_or(""); + MaybeSignalDone(); + } + + void OnConfigurations(std::vector<fuchsia::camera3::Configuration> configs) { + VideoCaptureFormats formats; + for (auto& config : configs) { + for (auto& props : config.streams) { + VideoCaptureFormat format; + format.frame_size = gfx::Size(props.image_format.display_width, + props.image_format.display_height); + format.frame_rate = static_cast<float>(props.frame_rate.numerator) / + props.frame_rate.denominator; + format.pixel_format = + VideoCaptureDeviceFuchsia::GetConvertedPixelFormat( + props.image_format.pixel_format.type); + if (format.pixel_format == PIXEL_FORMAT_UNKNOWN) + continue; + formats.push_back(format); + } + } + + formats_ = std::move(formats); + MaybeSignalDone(); + } + + void MaybeSignalDone() { + if (!have_results()) + return; + + device_.Unbind(); + + if (wait_results_run_loop_) + wait_results_run_loop_->Quit(); + } + + uint64_t device_id_; + fuchsia::camera3::DevicePtr device_; + base::Optional<std::string> description_; + base::Optional<VideoCaptureFormats> formats_; + base::Optional<base::RunLoop> wait_results_run_loop_; +}; + +VideoCaptureDeviceFactoryFuchsia::VideoCaptureDeviceFactoryFuchsia() { + DETACH_FROM_THREAD(thread_checker_); +} + +VideoCaptureDeviceFactoryFuchsia::~VideoCaptureDeviceFactoryFuchsia() { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); +} std::unique_ptr<VideoCaptureDevice> VideoCaptureDeviceFactoryFuchsia::CreateDevice( const VideoCaptureDeviceDescriptor& device_descriptor) { DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); - NOTIMPLEMENTED(); - return nullptr; + uint64_t device_id; + bool converted = + base::StringToUint64(device_descriptor.device_id, &device_id); + + // Test may call CreateDevice() with an invalid |device_id|. + if (!converted) + return nullptr; + + fidl::InterfaceHandle<fuchsia::camera3::Device> device; + device_watcher_->ConnectToDevice(device_id, device.NewRequest()); + return std::make_unique<VideoCaptureDeviceFuchsia>(std::move(device)); } void VideoCaptureDeviceFactoryFuchsia::GetDeviceDescriptors( VideoCaptureDeviceDescriptors* device_descriptors) { DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + device_descriptors->clear(); + + if (!device_watcher_) { + DCHECK(!first_update_run_loop_); + DCHECK(devices_.empty()); + + Initialize(); + + // The RunLoop will quit when either we've received the first WatchDevices() + // response or DeviceWatcher fails. |devices_| will be empty in case of a + // failure. + first_update_run_loop_.emplace(); + first_update_run_loop_->Run(); + first_update_run_loop_.reset(); + } + + for (auto& d : devices_) { + d.second->WaitResults(); + if (d.second->is_usable()) { + device_descriptors->push_back(VideoCaptureDeviceDescriptor( + d.second->description(), base::NumberToString(d.first), + VideoCaptureApi::FUCHSIA_CAMERA3)); + } + } } void VideoCaptureDeviceFactoryFuchsia::GetSupportedFormats( const VideoCaptureDeviceDescriptor& device, VideoCaptureFormats* capture_formats) { DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); - capture_formats->clear(); + + uint64_t device_id; + bool converted = base::StringToUint64(device.device_id, &device_id); + DCHECK(converted); + + auto it = devices_.find(device_id); + if (it == devices_.end()) { + capture_formats->clear(); + } else { + it->second->WaitResults(); + *capture_formats = it->second->formats(); + } +} + +void VideoCaptureDeviceFactoryFuchsia::Initialize() { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + DCHECK(!device_watcher_); + DCHECK(devices_.empty()); + + base::fuchsia::ComponentContextForCurrentProcess()->svc()->Connect( + device_watcher_.NewRequest()); + + device_watcher_.set_error_handler(fit::bind_member( + this, &VideoCaptureDeviceFactoryFuchsia::OnDeviceWatcherDisconnected)); + + WatchDevices(); +} + +void VideoCaptureDeviceFactoryFuchsia::OnDeviceWatcherDisconnected( + zx_status_t status) { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + ZX_LOG(ERROR, status) << "fuchsia.camera3.DeviceWatcher disconnected."; + devices_.clear(); + + if (first_update_run_loop_) + first_update_run_loop_->Quit(); +} + +void VideoCaptureDeviceFactoryFuchsia::WatchDevices() { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + device_watcher_->WatchDevices(fit::bind_member( + this, &VideoCaptureDeviceFactoryFuchsia::OnWatchDevicesResult)); +} + +void VideoCaptureDeviceFactoryFuchsia::OnWatchDevicesResult( + std::vector<fuchsia::camera3::WatchDevicesEvent> events) { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + for (auto& e : events) { + if (e.is_removed()) { + int erased = devices_.erase(e.removed()); + if (!erased) { + LOG(WARNING) << "Received device removed event for a device that " + "wasn't previously registered."; + } + continue; + } + + uint64_t id; + if (e.is_added()) { + id = e.added(); + if (devices_.find(id) != devices_.end()) { + LOG(WARNING) << "Received device added event for a device that was " + "previously registered."; + continue; + } + } else { + id = e.existing(); + if (devices_.find(id) != devices_.end()) { + continue; + } + LOG(WARNING) << "Received device exists event for a device that wasn't " + "previously registered."; + } + + fuchsia::camera3::DevicePtr device; + device_watcher_->ConnectToDevice(id, device.NewRequest()); + devices_.emplace( + id, std::make_unique<DeviceInfoFetcher>(id, std::move(device))); + } + + if (first_update_run_loop_) + first_update_run_loop_->Quit(); + + // Watch for further updates. + WatchDevices(); } } // namespace media diff --git a/chromium/media/capture/video/fuchsia/video_capture_device_factory_fuchsia.h b/chromium/media/capture/video/fuchsia/video_capture_device_factory_fuchsia.h index 7a5a6507284..86f3c6b8268 100644 --- a/chromium/media/capture/video/fuchsia/video_capture_device_factory_fuchsia.h +++ b/chromium/media/capture/video/fuchsia/video_capture_device_factory_fuchsia.h @@ -5,6 +5,13 @@ #ifndef MEDIA_CAPTURE_VIDEO_FUCHSIA_VIDEO_CAPTURE_DEVICE_FACTORY_FUCHSIA_H_ #define MEDIA_CAPTURE_VIDEO_FUCHSIA_VIDEO_CAPTURE_DEVICE_FACTORY_FUCHSIA_H_ +#include <fuchsia/camera3/cpp/fidl.h> + +#include <map> + +#include "base/containers/small_map.h" +#include "base/optional.h" +#include "base/run_loop.h" #include "media/capture/video/video_capture_device_factory.h" namespace media { @@ -15,6 +22,12 @@ class CAPTURE_EXPORT VideoCaptureDeviceFactoryFuchsia VideoCaptureDeviceFactoryFuchsia(); ~VideoCaptureDeviceFactoryFuchsia() override; + VideoCaptureDeviceFactoryFuchsia(const VideoCaptureDeviceFactoryFuchsia&) = + delete; + VideoCaptureDeviceFactoryFuchsia& operator=( + const VideoCaptureDeviceFactoryFuchsia&) = delete; + + // VideoCaptureDeviceFactory implementation. std::unique_ptr<VideoCaptureDevice> CreateDevice( const VideoCaptureDeviceDescriptor& device_descriptor) override; void GetDeviceDescriptors( @@ -23,7 +36,26 @@ class CAPTURE_EXPORT VideoCaptureDeviceFactoryFuchsia VideoCaptureFormats* supported_formats) override; private: - DISALLOW_COPY_AND_ASSIGN(VideoCaptureDeviceFactoryFuchsia); + // Helper class used to fetch per-device information. + class DeviceInfoFetcher; + + void Initialize(); + + void OnDeviceWatcherDisconnected(zx_status_t status); + + void WatchDevices(); + void OnWatchDevicesResult( + std::vector<fuchsia::camera3::WatchDevicesEvent> events); + + fuchsia::camera3::DeviceWatcherPtr device_watcher_; + base::small_map<std::map<uint64_t, std::unique_ptr<DeviceInfoFetcher>>> + devices_; + + // RunLoop used to wait for the first WatchDevices() response. Currently + // required because GetDeviceDescriptors() is synchronous. + // TODO(crbug.com/1072932) Refactor interface to allow asynchronous + // enumeration and remove this hack. + base::Optional<base::RunLoop> first_update_run_loop_; }; } // namespace media diff --git a/chromium/media/capture/video/fuchsia/video_capture_device_fuchsia.cc b/chromium/media/capture/video/fuchsia/video_capture_device_fuchsia.cc new file mode 100644 index 00000000000..946ad151a4e --- /dev/null +++ b/chromium/media/capture/video/fuchsia/video_capture_device_fuchsia.cc @@ -0,0 +1,437 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/capture/video/fuchsia/video_capture_device_fuchsia.h" + +#include <zircon/status.h> + +#include "base/fuchsia/fuchsia_logging.h" +#include "base/strings/stringprintf.h" +#include "base/time/time.h" +#include "media/base/video_types.h" +#include "third_party/libyuv/include/libyuv/convert.h" +#include "third_party/libyuv/include/libyuv/video_common.h" +#include "ui/gfx/buffer_format_util.h" + +namespace media { + +namespace { + +size_t RoundUp(size_t value, size_t alignment) { + return ((value + alignment - 1) / alignment) * alignment; +} + +libyuv::FourCC GetFourccForPixelFormat( + fuchsia::sysmem::PixelFormatType src_pixel_format) { + switch (src_pixel_format) { + case fuchsia::sysmem::PixelFormatType::I420: + return libyuv::FourCC::FOURCC_I420; + case fuchsia::sysmem::PixelFormatType::YV12: + return libyuv::FourCC::FOURCC_YV12; + case fuchsia::sysmem::PixelFormatType::NV12: + return libyuv::FourCC::FOURCC_NV12; + default: + NOTREACHED(); + return libyuv::FourCC::FOURCC_I420; + } +} + +libyuv::RotationMode CameraOrientationToLibyuvRotation( + fuchsia::camera3::Orientation orientation, + bool* flip_y) { + switch (orientation) { + case fuchsia::camera3::Orientation::UP: + *flip_y = false; + return libyuv::RotationMode::kRotate0; + + case fuchsia::camera3::Orientation::DOWN: + *flip_y = false; + return libyuv::RotationMode::kRotate180; + + case fuchsia::camera3::Orientation::LEFT: + *flip_y = false; + return libyuv::RotationMode::kRotate270; + + case fuchsia::camera3::Orientation::RIGHT: + *flip_y = false; + return libyuv::RotationMode::kRotate90; + + case fuchsia::camera3::Orientation::UP_FLIPPED: + *flip_y = true; + return libyuv::RotationMode::kRotate180; + + case fuchsia::camera3::Orientation::DOWN_FLIPPED: + *flip_y = true; + return libyuv::RotationMode::kRotate0; + + case fuchsia::camera3::Orientation::LEFT_FLIPPED: + *flip_y = true; + return libyuv::RotationMode::kRotate90; + + case fuchsia::camera3::Orientation::RIGHT_FLIPPED: + *flip_y = true; + return libyuv::RotationMode::kRotate270; + } +} + +gfx::Size RotateSize(gfx::Size size, libyuv::RotationMode rotation) { + switch (rotation) { + case libyuv::RotationMode::kRotate0: + case libyuv::RotationMode::kRotate180: + return size; + + case libyuv::RotationMode::kRotate90: + case libyuv::RotationMode::kRotate270: + return gfx::Size(size.height(), size.width()); + } +} + +} // namespace + +// static +VideoPixelFormat VideoCaptureDeviceFuchsia::GetConvertedPixelFormat( + fuchsia::sysmem::PixelFormatType format) { + switch (format) { + case fuchsia::sysmem::PixelFormatType::I420: + case fuchsia::sysmem::PixelFormatType::YV12: + case fuchsia::sysmem::PixelFormatType::NV12: + // Convert all YUV formats to I420 since consumers currently don't support + // NV12 or YV12. + return PIXEL_FORMAT_I420; + + default: + LOG(ERROR) << "Camera uses unsupported pixel format " + << static_cast<int>(format); + return PIXEL_FORMAT_UNKNOWN; + } +} + +bool VideoCaptureDeviceFuchsia::IsSupportedPixelFormat( + fuchsia::sysmem::PixelFormatType format) { + return GetConvertedPixelFormat(format) != PIXEL_FORMAT_UNKNOWN; +} + +VideoCaptureDeviceFuchsia::VideoCaptureDeviceFuchsia( + fidl::InterfaceHandle<fuchsia::camera3::Device> device) { + device_.Bind(std::move(device)); + device_.set_error_handler( + fit::bind_member(this, &VideoCaptureDeviceFuchsia::OnDeviceError)); +} + +VideoCaptureDeviceFuchsia::~VideoCaptureDeviceFuchsia() { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); +} + +void VideoCaptureDeviceFuchsia::AllocateAndStart( + const VideoCaptureParams& params, + std::unique_ptr<Client> client) { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + DCHECK_EQ(params.requested_format.pixel_format, PIXEL_FORMAT_I420); + DCHECK(!client_); + DCHECK(!stream_); + + client_ = std::move(client); + + if (!device_) { + OnError(FROM_HERE, VideoCaptureError::kFuchsiaCameraDeviceDisconnected, + "fuchsia.camera3.Device disconnected"); + return; + } + + start_time_ = base::TimeTicks::Now(); + frames_received_ = 0; + + // TODO(crbug.com/1075839) Select stream_id based on requested resolution. + device_->ConnectToStream(/*stream_id=*/0, stream_.NewRequest()); + stream_.set_error_handler( + fit::bind_member(this, &VideoCaptureDeviceFuchsia::OnStreamError)); + + WatchResolution(); + WatchOrientation(); + + // Call SetBufferCollection() with a new buffer collection token to indicate + // that we are interested in buffer collection negotiation. The collection + // token will be returned back from WatchBufferCollection(). After that it + // will be initialized in InitializeBufferCollection(). + stream_->SetBufferCollection(sysmem_allocator_.CreateNewToken()); + WatchBufferCollection(); +} + +void VideoCaptureDeviceFuchsia::StopAndDeAllocate() { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + DisconnectStream(); + client_.reset(); +} + +void VideoCaptureDeviceFuchsia::OnDeviceError(zx_status_t status) { + OnError(FROM_HERE, VideoCaptureError::kFuchsiaCameraDeviceDisconnected, + base::StringPrintf("fuchsia.camera3.Device disconnected: %s (%d)", + zx_status_get_string(status), status)); +} + +void VideoCaptureDeviceFuchsia::OnStreamError(zx_status_t status) { + OnError(FROM_HERE, VideoCaptureError::kFuchsiaCameraStreamDisconnected, + base::StringPrintf("fuchsia.camera3.Stream disconnected: %s (%d)", + zx_status_get_string(status), status)); +} + +void VideoCaptureDeviceFuchsia::DisconnectStream() { + stream_.Unbind(); + buffer_collection_creator_.reset(); + buffer_collection_.reset(); + buffer_reader_.reset(); + frame_size_.reset(); +} + +void VideoCaptureDeviceFuchsia::OnError(base::Location location, + VideoCaptureError error, + const std::string& reason) { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + DisconnectStream(); + + if (client_) { + client_->OnError(error, location, reason); + } +} + +void VideoCaptureDeviceFuchsia::WatchResolution() { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + stream_->WatchResolution(fit::bind_member( + this, &VideoCaptureDeviceFuchsia::OnWatchResolutionResult)); +} + +void VideoCaptureDeviceFuchsia::OnWatchResolutionResult( + fuchsia::math::Size frame_size) { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + frame_size_ = gfx::Size(frame_size.width, frame_size.height); + + WatchResolution(); +} + +void VideoCaptureDeviceFuchsia::WatchOrientation() { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + stream_->WatchOrientation(fit::bind_member( + this, &VideoCaptureDeviceFuchsia::OnWatchOrientationResult)); +} + +void VideoCaptureDeviceFuchsia::OnWatchOrientationResult( + fuchsia::camera3::Orientation orientation) { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + orientation_ = orientation; + WatchOrientation(); +} + +void VideoCaptureDeviceFuchsia::WatchBufferCollection() { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + stream_->WatchBufferCollection( + [this](fidl::InterfaceHandle<fuchsia::sysmem::BufferCollectionToken> + token_handle) { + InitializeBufferCollection(std::move(token_handle)); + WatchBufferCollection(); + }); +} + +void VideoCaptureDeviceFuchsia::InitializeBufferCollection( + fidl::InterfaceHandle<fuchsia::sysmem::BufferCollectionToken> + token_handle) { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + // Drop old buffers. + buffer_collection_.reset(); + buffer_reader_.reset(); + + // Initialize the new collection. + fuchsia::sysmem::BufferCollectionTokenPtr token; + token.Bind(std::move(token_handle)); + buffer_collection_creator_ = + sysmem_allocator_.MakeBufferPoolCreatorFromToken(std::move(token)); + + // Request just one buffer in collection constraints: each frame is copied as + // soon as it's received. + const size_t kMaxUsedOutputFrames = 1; + fuchsia::sysmem::BufferCollectionConstraints constraints = + SysmemBufferReader::GetRecommendedConstraints(kMaxUsedOutputFrames); + buffer_collection_creator_->Create( + std::move(constraints), + base::BindOnce(&VideoCaptureDeviceFuchsia::OnBufferCollectionCreated, + base::Unretained(this))); +} + +void VideoCaptureDeviceFuchsia::OnBufferCollectionCreated( + std::unique_ptr<SysmemBufferPool> collection) { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + buffer_collection_ = std::move(collection); + buffer_collection_->CreateReader( + base::BindOnce(&VideoCaptureDeviceFuchsia::OnBufferReaderCreated, + base::Unretained(this))); +} + +void VideoCaptureDeviceFuchsia::OnBufferReaderCreated( + std::unique_ptr<SysmemBufferReader> reader) { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + buffer_reader_ = std::move(reader); + if (!buffer_reader_->buffer_settings().has_image_format_constraints) { + OnError(FROM_HERE, VideoCaptureError::kFuchsiaSysmemDidNotSetImageFormat, + "Sysmem created buffer without image format constraints"); + return; + } + + auto pixel_format = buffer_reader_->buffer_settings() + .image_format_constraints.pixel_format.type; + if (!IsSupportedPixelFormat(pixel_format)) { + OnError(FROM_HERE, VideoCaptureError::kFuchsiaUnsupportedPixelFormat, + base::StringPrintf("Unsupported video frame format: %d", + static_cast<int>(pixel_format))); + return; + } + + if (!started_) { + started_ = true; + client_->OnStarted(); + ReceiveNextFrame(); + } +} + +void VideoCaptureDeviceFuchsia::ReceiveNextFrame() { + stream_->GetNextFrame([this](fuchsia::camera3::FrameInfo frame_info) { + ProcessNewFrame(std::move(frame_info)); + ReceiveNextFrame(); + }); +} + +void VideoCaptureDeviceFuchsia::ProcessNewFrame( + fuchsia::camera3::FrameInfo frame_info) { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + DCHECK(client_); + + if (!buffer_reader_) { + DLOG(WARNING) << "Dropping frame received before sysmem collection has " + "been initialized."; + return; + } + + size_t index = frame_info.buffer_index; + if (index >= buffer_reader_->num_buffers()) { + OnError(FROM_HERE, VideoCaptureError::kFuchsiaSysmemInvalidBufferIndex, + base::StringPrintf("Received frame with invalid buffer_index=%zu", + index)); + return; + } + + const fuchsia::sysmem::ImageFormatConstraints& sysmem_buffer_format = + buffer_reader_->buffer_settings().image_format_constraints; + + // Calculate coded frame dimensions for the buffer collection based on the + // sysmem collection constraints. This logic should match + // LogicalBufferCollection::Allocate() in sysmem. + size_t src_coded_width = + RoundUp(std::max(sysmem_buffer_format.min_coded_width, + sysmem_buffer_format.required_max_coded_width), + sysmem_buffer_format.coded_width_divisor); + size_t src_coded_height = + RoundUp(std::max(sysmem_buffer_format.min_coded_height, + sysmem_buffer_format.required_max_coded_height), + sysmem_buffer_format.coded_height_divisor); + size_t src_stride = RoundUp( + std::max(static_cast<size_t>(sysmem_buffer_format.min_bytes_per_row), + src_coded_width), + sysmem_buffer_format.bytes_per_row_divisor); + gfx::Size visible_size = + frame_size_.value_or(gfx::Size(src_coded_width, src_coded_height)); + gfx::Size nonrotated_output_size((visible_size.width() + 1) & ~1, + (visible_size.height() + 1) & ~1); + + bool flip_y; + libyuv::RotationMode rotation = + CameraOrientationToLibyuvRotation(orientation_, &flip_y); + + gfx::Size output_size = RotateSize(nonrotated_output_size, rotation); + visible_size = RotateSize(visible_size, rotation); + + base::TimeTicks reference_time = + base::TimeTicks::FromZxTime(frame_info.timestamp); + base::TimeDelta timestamp = + std::max(reference_time - start_time_, base::TimeDelta()); + + float frame_rate = + (timestamp > base::TimeDelta()) + ? static_cast<float>(frames_received_) / timestamp.InSecondsF() + : 0.0; + VideoCaptureFormat capture_format(output_size, frame_rate, PIXEL_FORMAT_I420); + capture_format.pixel_format = PIXEL_FORMAT_I420; + + Client::Buffer buffer; + Client::ReserveResult result = client_->ReserveOutputBuffer( + capture_format.frame_size, capture_format.pixel_format, + /*frame_feedback_id=*/0, &buffer); + if (result != Client::ReserveResult::kSucceeded) { + DLOG(WARNING) << "Failed to allocate output buffer for a video frame"; + return; + } + + auto src_span = buffer_reader_->GetMappingForBuffer(index); + if (src_span.empty()) { + OnError(FROM_HERE, VideoCaptureError::kFuchsiaFailedToMapSysmemBuffer, + "Failed to map buffers allocated by sysmem"); + return; + } + + // For all supported formats (I420, NV12 and YV12) the U and V channels are + // subsampled at 2x in both directions, so together they occupy half of the + // space needed for the Y plane and the total buffer size is 3/2 of the Y + // plane size. + size_t src_buffer_size = src_coded_height * src_stride * 3 / 2; + if (src_span.size() < src_buffer_size) { + OnError(FROM_HERE, VideoCaptureError::kFuchsiaSysmemInvalidBufferSize, + "Sysmem allocated buffer that's smaller than expected"); + return; + } + + std::unique_ptr<VideoCaptureBufferHandle> output_handle = + buffer.handle_provider->GetHandleForInProcessAccess(); + + // Calculate offsets and strides for the output buffer. + uint8_t* dst_y = output_handle->data(); + int dst_stride_y = output_size.width(); + size_t dst_y_plane_size = output_size.width() * output_size.height(); + uint8_t* dst_u = dst_y + dst_y_plane_size; + int dst_stride_u = output_size.width() / 2; + uint8_t* dst_v = dst_u + dst_y_plane_size / 4; + int dst_stride_v = output_size.width() / 2; + + // Check that the output fits in the buffer. + const uint8_t* dst_end = dst_v + dst_y_plane_size / 4; + CHECK_LE(dst_end, output_handle->data() + output_handle->mapped_size()); + + // Vertical flip is indicated to ConvertToI420() by negating src_height. + int flipped_src_height = static_cast<int>(src_coded_height); + if (flip_y) + flipped_src_height = -flipped_src_height; + + auto four_cc = + GetFourccForPixelFormat(buffer_reader_->buffer_settings() + .image_format_constraints.pixel_format.type); + + libyuv::ConvertToI420(src_span.data(), src_span.size(), dst_y, dst_stride_y, + dst_u, dst_stride_u, dst_v, dst_stride_v, + /*crop_x=*/0, /*crop_y=*/0, src_stride, + flipped_src_height, nonrotated_output_size.width(), + nonrotated_output_size.height(), rotation, four_cc); + + client_->OnIncomingCapturedBufferExt( + std::move(buffer), capture_format, gfx::ColorSpace(), reference_time, + timestamp, gfx::Rect(visible_size), VideoFrameMetadata()); + + // Frame buffer is returned to the device by dropping the |frame_info|. +} + +} // namespace media
\ No newline at end of file diff --git a/chromium/media/capture/video/fuchsia/video_capture_device_fuchsia.h b/chromium/media/capture/video/fuchsia/video_capture_device_fuchsia.h new file mode 100644 index 00000000000..1d2e40ce55a --- /dev/null +++ b/chromium/media/capture/video/fuchsia/video_capture_device_fuchsia.h @@ -0,0 +1,118 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_CAPTURE_VIDEO_FUCHSIA_VIDEO_CAPTURE_DEVICE_FUCHSIA_H_ +#define MEDIA_CAPTURE_VIDEO_FUCHSIA_VIDEO_CAPTURE_DEVICE_FUCHSIA_H_ + +#include <fuchsia/camera3/cpp/fidl.h> + +#include <memory> + +#include "base/optional.h" +#include "base/threading/thread_checker.h" +#include "base/time/time.h" +#include "media/capture/video/video_capture_device.h" +#include "media/fuchsia/common/sysmem_buffer_pool.h" +#include "media/fuchsia/common/sysmem_buffer_reader.h" + +namespace media { + +class CAPTURE_EXPORT VideoCaptureDeviceFuchsia : public VideoCaptureDevice { + public: + // Returns pixel format to which video frames in the specified sysmem pixel + // |format| will be converted. PIXEL_FORMAT_UNKNOWN is returned for + // unsupported formats. + static VideoPixelFormat GetConvertedPixelFormat( + fuchsia::sysmem::PixelFormatType format); + + static bool IsSupportedPixelFormat(fuchsia::sysmem::PixelFormatType format); + + explicit VideoCaptureDeviceFuchsia( + fidl::InterfaceHandle<fuchsia::camera3::Device> device); + ~VideoCaptureDeviceFuchsia() final; + + VideoCaptureDeviceFuchsia(const VideoCaptureDeviceFuchsia&) = delete; + VideoCaptureDeviceFuchsia& operator=(const VideoCaptureDeviceFuchsia&) = + delete; + + // VideoCaptureDevice implementation. + void AllocateAndStart(const VideoCaptureParams& params, + std::unique_ptr<Client> client) final; + void StopAndDeAllocate() final; + + private: + // Disconnects the |stream_| and resets related state. + void DisconnectStream(); + + // Reports the specified |error| to the client. + void OnError(base::Location location, + VideoCaptureError error, + const std::string& reason); + + // Error handlers for the |device_| and |stream_|. + void OnDeviceError(zx_status_t status); + void OnStreamError(zx_status_t status); + + // Watches for resolution updates and updates |frame_size_| accordingly. + void WatchResolution(); + + // Callback for WatchResolution(). + void OnWatchResolutionResult(fuchsia::math::Size frame_size); + + // Watches for orientation updates and updates |orientation_| accordingly. + void WatchOrientation(); + + // Callback for WatchOrientation(). + void OnWatchOrientationResult(fuchsia::camera3::Orientation orientation); + + // Watches for sysmem buffer collection updates from the camera. + void WatchBufferCollection(); + + // Initializes buffer collection using the specified token. Initialization is + // asynchronous. |buffer_reader_| will be set once the initialization is + // complete. Old buffer collection are dropped synchronously (whether they + // have finished initialization or not). + void InitializeBufferCollection( + fidl::InterfaceHandle<fuchsia::sysmem::BufferCollectionToken> + token_handle); + + // Callback for SysmemBufferPool::Creator. + void OnBufferCollectionCreated(std::unique_ptr<SysmemBufferPool> collection); + + // Callback for SysmemBufferPool::CreateReader(). + void OnBufferReaderCreated(std::unique_ptr<SysmemBufferReader> reader); + + // Calls Stream::GetNextFrame() in a loop to receive incoming frames. + void ReceiveNextFrame(); + + // Processes new frames received by ReceiveNextFrame() and passes it to the + // |client_|. + void ProcessNewFrame(fuchsia::camera3::FrameInfo frame_info); + + fuchsia::camera3::DevicePtr device_; + fuchsia::camera3::StreamPtr stream_; + + std::unique_ptr<Client> client_; + + media::BufferAllocator sysmem_allocator_; + std::unique_ptr<SysmemBufferPool::Creator> buffer_collection_creator_; + std::unique_ptr<SysmemBufferPool> buffer_collection_; + std::unique_ptr<SysmemBufferReader> buffer_reader_; + + base::Optional<gfx::Size> frame_size_; + fuchsia::camera3::Orientation orientation_ = + fuchsia::camera3::Orientation::UP; + + base::TimeTicks start_time_; + + bool started_ = false; + + size_t frames_received_ = 0; + + THREAD_CHECKER(thread_checker_); +}; + +} // namespace media + +#endif // MEDIA_CAPTURE_VIDEO_FUCHSIA_VIDEO_CAPTURE_DEVICE_FUCHSIA_H_
\ No newline at end of file diff --git a/chromium/media/capture/video/fuchsia/video_capture_device_fuchsia_test.cc b/chromium/media/capture/video/fuchsia/video_capture_device_fuchsia_test.cc new file mode 100644 index 00000000000..63627c460a7 --- /dev/null +++ b/chromium/media/capture/video/fuchsia/video_capture_device_fuchsia_test.cc @@ -0,0 +1,324 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/capture/video/fuchsia/video_capture_device_fuchsia.h" + +#include "base/test/task_environment.h" +#include "media/fuchsia/camera/fake_fuchsia_camera.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "ui/gfx/geometry/rect.h" +#include "ui/gfx/geometry/size.h" + +namespace media { + +namespace { + +struct ReceivedFrame { + VideoCaptureDevice::Client::Buffer buffer; + VideoCaptureFormat format; + gfx::ColorSpace color_space; + base::TimeTicks reference_time; + base::TimeDelta timestamp; + gfx::Rect visible_rect; +}; + +void ValidateReceivedFrame(const ReceivedFrame& frame, + gfx::Size expected_size, + uint8_t salt) { + gfx::Size coded_size((expected_size.width() + 1) & ~1, + (expected_size.height() + 1) & ~1); + ASSERT_EQ(frame.format.frame_size, coded_size); + EXPECT_EQ(frame.format.pixel_format, PIXEL_FORMAT_I420); + EXPECT_EQ(frame.visible_rect, gfx::Rect(expected_size)); + EXPECT_EQ(frame.color_space, gfx::ColorSpace()); + + auto handle = frame.buffer.handle_provider->GetHandleForInProcessAccess(); + + FakeCameraStream::ValidateFrameData(handle->data(), coded_size, salt); +} + +// VideoCaptureBufferHandle implementation that references memory allocated on +// the heap. +class HeapBufferHandle : public VideoCaptureBufferHandle { + public: + HeapBufferHandle(size_t size, uint8_t* data) : size_(size), data_(data) {} + + size_t mapped_size() const final { return size_; } + uint8_t* data() const final { return data_; } + const uint8_t* const_data() const final { return data_; } + + private: + const size_t size_; + uint8_t* const data_; +}; + +// VideoCaptureDevice::Client::Buffer::HandleProvider implementation that +// allocates memory on the heap. +class HeapBufferHandleProvider + : public VideoCaptureDevice::Client::Buffer::HandleProvider { + public: + HeapBufferHandleProvider(size_t size) : data_(size) {} + ~HeapBufferHandleProvider() final = default; + + base::UnsafeSharedMemoryRegion DuplicateAsUnsafeRegion() final { + NOTREACHED(); + return {}; + } + + mojo::ScopedSharedBufferHandle DuplicateAsMojoBuffer() final { + NOTREACHED(); + return {}; + } + + std::unique_ptr<VideoCaptureBufferHandle> GetHandleForInProcessAccess() + override { + return std::make_unique<HeapBufferHandle>(data_.size(), data_.data()); + } + + gfx::GpuMemoryBufferHandle GetGpuMemoryBufferHandle() override { + return gfx::GpuMemoryBufferHandle(); + } + + private: + std::vector<uint8_t> data_; +}; + +class TestVideoCaptureClient : public VideoCaptureDevice::Client { + public: + ~TestVideoCaptureClient() final = default; + + void WaitFrame() { + EXPECT_FALSE(wait_frame_run_loop_); + + wait_frame_run_loop_.emplace(); + wait_frame_run_loop_->Run(); + wait_frame_run_loop_.reset(); + } + + const std::vector<ReceivedFrame>& received_frames() { + return received_frames_; + } + + private: + // VideoCaptureDevice::Client implementation. + void OnStarted() final { + EXPECT_FALSE(started_); + started_ = true; + } + + ReserveResult ReserveOutputBuffer(const gfx::Size& dimensions, + VideoPixelFormat format, + int frame_feedback_id, + Buffer* buffer) final { + EXPECT_TRUE(started_); + EXPECT_EQ(format, PIXEL_FORMAT_I420); + EXPECT_EQ(dimensions.width() % 2, 0); + EXPECT_EQ(dimensions.height() % 2, 0); + + size_t size = dimensions.width() * dimensions.height() * 3 / 2; + buffer->handle_provider = std::make_unique<HeapBufferHandleProvider>(size); + return VideoCaptureDevice::Client::ReserveResult::kSucceeded; + } + + void OnIncomingCapturedBufferExt( + Buffer buffer, + const VideoCaptureFormat& format, + const gfx::ColorSpace& color_space, + base::TimeTicks reference_time, + base::TimeDelta timestamp, + gfx::Rect visible_rect, + const VideoFrameMetadata& additional_metadata) final { + EXPECT_TRUE(started_); + + received_frames_.push_back(ReceivedFrame{std::move(buffer), format, + color_space, reference_time, + timestamp, visible_rect}); + + if (wait_frame_run_loop_) + wait_frame_run_loop_->Quit(); + } + + void OnIncomingCapturedData(const uint8_t* data, + int length, + const VideoCaptureFormat& frame_format, + const gfx::ColorSpace& color_space, + int clockwise_rotation, + bool flip_y, + base::TimeTicks reference_time, + base::TimeDelta timestamp, + int frame_feedback_id) final { + NOTREACHED(); + } + void OnIncomingCapturedGfxBuffer(gfx::GpuMemoryBuffer* buffer, + const VideoCaptureFormat& frame_format, + int clockwise_rotation, + base::TimeTicks reference_time, + base::TimeDelta timestamp, + int frame_feedback_id) final { + NOTREACHED(); + } + void OnIncomingCapturedBuffer(Buffer buffer, + const VideoCaptureFormat& format, + base::TimeTicks reference_time, + base::TimeDelta timestamp) final { + NOTREACHED(); + } + void OnError(VideoCaptureError error, + const base::Location& from_here, + const std::string& reason) final { + NOTREACHED(); + } + void OnFrameDropped(VideoCaptureFrameDropReason reason) final { + NOTREACHED(); + } + void OnLog(const std::string& message) final { NOTREACHED(); } + double GetBufferPoolUtilization() const final { + NOTREACHED(); + return 0; + } + + bool started_ = false; + std::vector<ReceivedFrame> received_frames_; + base::Optional<base::RunLoop> wait_frame_run_loop_; +}; + +} // namespace + +class VideoCaptureDeviceFuchsiaTest : public testing::Test { + public: + VideoCaptureDeviceFuchsiaTest() { + fidl::InterfaceHandle<fuchsia::camera3::Device> device_handle; + fake_device_.Bind(device_handle.NewRequest()); + device_ = + std::make_unique<VideoCaptureDeviceFuchsia>(std::move(device_handle)); + } + + ~VideoCaptureDeviceFuchsiaTest() override { device_->StopAndDeAllocate(); } + + void StartCapturer() { + VideoCaptureParams params; + params.requested_format.frame_size = FakeCameraStream::kDefaultFrameSize; + params.requested_format.frame_rate = 30.0; + params.requested_format.pixel_format = PIXEL_FORMAT_I420; + + auto client = std::make_unique<TestVideoCaptureClient>(); + client_ = client.get(); + device_->AllocateAndStart(params, std::move(client)); + + EXPECT_TRUE(fake_stream_.WaitBuffersAllocated()); + } + + protected: + base::test::SingleThreadTaskEnvironment task_environment_{ + base::test::SingleThreadTaskEnvironment::MainThreadType::IO}; + + FakeCameraStream fake_stream_; + FakeCameraDevice fake_device_{&fake_stream_}; + std::unique_ptr<VideoCaptureDeviceFuchsia> device_; + TestVideoCaptureClient* client_ = nullptr; +}; + +TEST_F(VideoCaptureDeviceFuchsiaTest, Initialize) { + StartCapturer(); +} + +TEST_F(VideoCaptureDeviceFuchsiaTest, SendFrame) { + StartCapturer(); + + auto frame_timestamp = base::TimeTicks::Now(); + fake_stream_.ProduceFrame(frame_timestamp, 1); + client_->WaitFrame(); + + ASSERT_EQ(client_->received_frames().size(), 1U); + EXPECT_EQ(client_->received_frames()[0].reference_time, frame_timestamp); + ValidateReceivedFrame(client_->received_frames()[0], + FakeCameraStream::kDefaultFrameSize, 1); +} + +TEST_F(VideoCaptureDeviceFuchsiaTest, MultipleFrames) { + StartCapturer(); + + EXPECT_TRUE(fake_stream_.WaitBuffersAllocated()); + + for (size_t i = 0; i < 10; ++i) { + ASSERT_TRUE(fake_stream_.WaitFreeBuffer()); + + auto frame_timestamp = + base::TimeTicks() + base::TimeDelta::FromMilliseconds(i * 16); + fake_stream_.ProduceFrame(frame_timestamp, i); + client_->WaitFrame(); + + ASSERT_EQ(client_->received_frames().size(), i + 1); + EXPECT_EQ(client_->received_frames()[i].reference_time, frame_timestamp); + ValidateReceivedFrame(client_->received_frames()[i], + FakeCameraStream::kDefaultFrameSize, i); + } +} + +TEST_F(VideoCaptureDeviceFuchsiaTest, FrameRotation) { + const gfx::Size kResolution(4, 2); + fake_stream_.SetFakeResolution(kResolution); + + StartCapturer(); + + EXPECT_TRUE(fake_stream_.WaitBuffersAllocated()); + + for (int i = static_cast<int>(fuchsia::camera3::Orientation::UP); + i <= static_cast<int>(fuchsia::camera3::Orientation::RIGHT_FLIPPED); + ++i) { + SCOPED_TRACE(testing::Message() << "Orientation " << i); + + auto orientation = static_cast<fuchsia::camera3::Orientation>(i); + + ASSERT_TRUE(fake_stream_.WaitFreeBuffer()); + fake_stream_.SetFakeOrientation(orientation); + fake_stream_.ProduceFrame(base::TimeTicks::Now(), i); + client_->WaitFrame(); + + gfx::Size expected_size = kResolution; + if (orientation == fuchsia::camera3::Orientation::LEFT || + orientation == fuchsia::camera3::Orientation::LEFT_FLIPPED || + orientation == fuchsia::camera3::Orientation::RIGHT || + orientation == fuchsia::camera3::Orientation::RIGHT_FLIPPED) { + expected_size = gfx::Size(expected_size.height(), expected_size.width()); + } + ValidateReceivedFrame(client_->received_frames().back(), expected_size, i); + } +} + +TEST_F(VideoCaptureDeviceFuchsiaTest, FrameDimensionsNotDivisibleBy2) { + const gfx::Size kOddResolution(21, 7); + fake_stream_.SetFakeResolution(kOddResolution); + + StartCapturer(); + + fake_stream_.ProduceFrame(base::TimeTicks::Now(), 1); + client_->WaitFrame(); + + ASSERT_EQ(client_->received_frames().size(), 1U); + ValidateReceivedFrame(client_->received_frames()[0], kOddResolution, 1); +} + +TEST_F(VideoCaptureDeviceFuchsiaTest, MidStreamResolutionChange) { + StartCapturer(); + + // Capture the first frame at the default resolution. + fake_stream_.ProduceFrame(base::TimeTicks::Now(), 1); + client_->WaitFrame(); + ASSERT_TRUE(fake_stream_.WaitFreeBuffer()); + + // Update resolution and produce another frames. + const gfx::Size kUpdatedResolution(3, 14); + fake_stream_.SetFakeResolution(kUpdatedResolution); + fake_stream_.ProduceFrame(base::TimeTicks::Now(), 1); + client_->WaitFrame(); + + // Verify that we get captured frames with correct resolution. + ASSERT_EQ(client_->received_frames().size(), 2U); + ValidateReceivedFrame(client_->received_frames()[0], + FakeCameraStream::kDefaultFrameSize, 1); + ValidateReceivedFrame(client_->received_frames()[1], kUpdatedResolution, 1); +} + +} // namespace media diff --git a/chromium/media/capture/video/mac/video_capture_device_avfoundation_mac.mm b/chromium/media/capture/video/mac/video_capture_device_avfoundation_mac.mm index f803ae0efc8..d19f9e68b86 100644 --- a/chromium/media/capture/video/mac/video_capture_device_avfoundation_mac.mm +++ b/chromium/media/capture/video/mac/video_capture_device_avfoundation_mac.mm @@ -24,10 +24,6 @@ #include "services/video_capture/public/uma/video_capture_service_event.h" #include "ui/gfx/geometry/size.h" -// Prefer MJPEG if frame width or height is larger than this. -static const int kMjpegWidthThreshold = 640; -static const int kMjpegHeightThreshold = 480; - namespace { enum MacBookVersions { @@ -351,16 +347,9 @@ void ExtractBaseAddressAndLength(char** base_address, _frameRate = frameRate; FourCharCode best_fourcc = kCMPixelFormat_422YpCbCr8; - const bool prefer_mjpeg = - width > kMjpegWidthThreshold || height > kMjpegHeightThreshold; for (AVCaptureDeviceFormat* format in [_captureDevice formats]) { const FourCharCode fourcc = CMFormatDescriptionGetMediaSubType([format formatDescription]); - if (prefer_mjpeg && fourcc == kCMVideoCodecType_JPEG_OpenDML) { - best_fourcc = fourcc; - break; - } - // Compare according to Chromium preference. if (media::VideoCaptureFormat::ComparePixelFormatPreference( FourCCToChromiumPixelFormat(fourcc), diff --git a/chromium/media/capture/video/shared_memory_buffer_tracker.cc b/chromium/media/capture/video/shared_memory_buffer_tracker.cc index 6cd0b698736..7865039b34a 100644 --- a/chromium/media/capture/video/shared_memory_buffer_tracker.cc +++ b/chromium/media/capture/video/shared_memory_buffer_tracker.cc @@ -4,7 +4,8 @@ #include "media/capture/video/shared_memory_buffer_tracker.h" -#include "base/logging.h" +#include "base/check.h" +#include "base/notreached.h" #include "media/base/video_frame.h" #include "mojo/public/cpp/system/platform_handle.h" #include "ui/gfx/geometry/size.h" diff --git a/chromium/media/capture/video/video_capture_buffer_handle.cc b/chromium/media/capture/video/video_capture_buffer_handle.cc index 693c3d18fd2..40f8c92c331 100644 --- a/chromium/media/capture/video/video_capture_buffer_handle.cc +++ b/chromium/media/capture/video/video_capture_buffer_handle.cc @@ -4,7 +4,9 @@ #include "media/capture/video/video_capture_buffer_handle.h" -#include "base/logging.h" +#include <ostream> + +#include "base/notreached.h" namespace media { diff --git a/chromium/media/capture/video/video_capture_buffer_tracker.h b/chromium/media/capture/video/video_capture_buffer_tracker.h index 19811b064b5..67576418379 100644 --- a/chromium/media/capture/video/video_capture_buffer_tracker.h +++ b/chromium/media/capture/video/video_capture_buffer_tracker.h @@ -62,6 +62,6 @@ class CAPTURE_EXPORT VideoCaptureBufferTracker { int frame_feedback_id_; }; -} // namespace content +} // namespace media #endif // MEDIA_CAPTURE_VIDEO_VIDEO_CAPTURE_BUFFER_TRACKER_H_ diff --git a/chromium/media/capture/video/video_capture_device_client_unittest.cc b/chromium/media/capture/video/video_capture_device_client_unittest.cc index 0e69a632fa4..3175ba31e69 100644 --- a/chromium/media/capture/video/video_capture_device_client_unittest.cc +++ b/chromium/media/capture/video/video_capture_device_client_unittest.cc @@ -9,7 +9,7 @@ #include <memory> #include "base/bind.h" -#include "base/logging.h" +#include "base/check.h" #include "base/macros.h" #include "build/build_config.h" #include "media/base/limits.h" diff --git a/chromium/media/capture/video/video_capture_device_descriptor.cc b/chromium/media/capture/video/video_capture_device_descriptor.cc index be6c87ec4ce..81ee65a45b8 100644 --- a/chromium/media/capture/video/video_capture_device_descriptor.cc +++ b/chromium/media/capture/video/video_capture_device_descriptor.cc @@ -4,7 +4,6 @@ #include "media/capture/video/video_capture_device_descriptor.h" -#include "base/logging.h" #include "base/strings/string_util.h" namespace media { @@ -90,6 +89,8 @@ const char* VideoCaptureDeviceDescriptor::GetCaptureApiTypeString() const { return "Camera API2 Full"; case VideoCaptureApi::ANDROID_API2_LIMITED: return "Camera API2 Limited"; + case VideoCaptureApi::FUCHSIA_CAMERA3: + return "fuchsia.camera3 API"; case VideoCaptureApi::VIRTUAL_DEVICE: return "Virtual Device"; case VideoCaptureApi::UNKNOWN: diff --git a/chromium/media/capture/video/video_capture_device_descriptor.h b/chromium/media/capture/video/video_capture_device_descriptor.h index 6d690a4e192..1c3da3f99dc 100644 --- a/chromium/media/capture/video/video_capture_device_descriptor.h +++ b/chromium/media/capture/video/video_capture_device_descriptor.h @@ -26,6 +26,7 @@ enum class VideoCaptureApi { ANDROID_API2_LEGACY, ANDROID_API2_FULL, ANDROID_API2_LIMITED, + FUCHSIA_CAMERA3, VIRTUAL_DEVICE, UNKNOWN }; diff --git a/chromium/media/capture/video/video_capture_device_unittest.cc b/chromium/media/capture/video/video_capture_device_unittest.cc index 7e690783c24..d7d800d6a08 100644 --- a/chromium/media/capture/video/video_capture_device_unittest.cc +++ b/chromium/media/capture/video/video_capture_device_unittest.cc @@ -69,8 +69,9 @@ #define MAYBE_UsingRealWebcam_CaptureWithSize UsingRealWebcam_CaptureWithSize #define MAYBE_UsingRealWebcam_CheckPhotoCallbackRelease \ UsingRealWebcam_CheckPhotoCallbackRelease -#elif defined(OS_WIN) -// TODO(crbug.com/893494): Fails on win: error: Value of: device_descriptor. +#elif defined(OS_WIN) || defined(OS_FUCHSIA) +// Windows test bots don't have camera. +// On Fuchsia the tests run under emulator that doesn't support camera. #define MAYBE_UsingRealWebcam_AllocateBadSize \ DISABLED_UsingRealWebcam_AllocateBadSize #define MAYBE_UsingRealWebcam_CaptureMjpeg DISABLED_UsingRealWebcam_CaptureMjpeg @@ -97,7 +98,9 @@ #define MAYBE_UsingRealWebcam_CaptureMjpeg UsingRealWebcam_CaptureMjpeg #define MAYBE_UsingRealWebcam_TakePhoto UsingRealWebcam_TakePhoto #define MAYBE_UsingRealWebcam_GetPhotoState UsingRealWebcam_GetPhotoState -#define MAYBE_UsingRealWebcam_CaptureWithSize UsingRealWebcam_CaptureWithSize +// Flaky crash: https://crbug.com/1069608 +#define MAYBE_UsingRealWebcam_CaptureWithSize \ + DISABLED_UsingRealWebcam_CaptureWithSize #define MAYBE_UsingRealWebcam_CheckPhotoCallbackRelease \ UsingRealWebcam_CheckPhotoCallbackRelease #elif defined(OS_LINUX) @@ -219,6 +222,18 @@ class MockImageCaptureClient mojom::PhotoStatePtr state_; }; +base::test::SingleThreadTaskEnvironment::MainThreadType kMainThreadType = +#if defined(OS_MACOSX) + // Video capture code on MacOSX must run on a CFRunLoop enabled thread + // for interaction with AVFoundation. + base::test::SingleThreadTaskEnvironment::MainThreadType::UI; +#elif defined(OS_FUCHSIA) + // FIDL APIs on Fuchsia requires IO thread. + base::test::SingleThreadTaskEnvironment::MainThreadType::IO; +#else + base::test::SingleThreadTaskEnvironment::MainThreadType::DEFAULT; +#endif + } // namespace class VideoCaptureDeviceTest @@ -247,13 +262,7 @@ class VideoCaptureDeviceTest typedef VideoCaptureDevice::Client Client; VideoCaptureDeviceTest() - : -#if defined(OS_MACOSX) - // Video capture code on MacOSX must run on a CFRunLoop enabled thread - // for interaction with AVFoundation. - task_environment_( - base::test::SingleThreadTaskEnvironment::MainThreadType::UI), -#endif + : task_environment_(kMainThreadType), device_descriptors_(new VideoCaptureDeviceDescriptors()), main_thread_task_runner_(base::ThreadTaskRunnerHandle::Get()), video_capture_client_(CreateDeviceClient()), @@ -466,8 +475,9 @@ class VideoCaptureDeviceTest std::unique_ptr<VideoCaptureDeviceFactory> video_capture_device_factory_; }; +// Causes a flaky crash on Chrome OS. https://crbug.com/1069608 // Cause hangs on Windows Debug. http://crbug.com/417824 -#if defined(OS_WIN) && !defined(NDEBUG) +#if defined(OS_CHROMEOS) || (defined(OS_WIN) && !defined(NDEBUG)) #define MAYBE_OpenInvalidDevice DISABLED_OpenInvalidDevice #else #define MAYBE_OpenInvalidDevice OpenInvalidDevice diff --git a/chromium/media/capture/video/win/capability_list_win.cc b/chromium/media/capture/video/win/capability_list_win.cc index c16708dc315..1e4a3425309 100644 --- a/chromium/media/capture/video/win/capability_list_win.cc +++ b/chromium/media/capture/video/win/capability_list_win.cc @@ -7,7 +7,7 @@ #include <algorithm> #include <functional> -#include "base/logging.h" +#include "base/check.h" #include "media/capture/video_capture_types.h" namespace media { diff --git a/chromium/media/capture/video/win/filter_base_win.cc b/chromium/media/capture/video/win/filter_base_win.cc index 5ad3b757ad2..f014afff6b0 100644 --- a/chromium/media/capture/video/win/filter_base_win.cc +++ b/chromium/media/capture/video/win/filter_base_win.cc @@ -4,6 +4,8 @@ #include "media/capture/video/win/filter_base_win.h" +#include "base/notreached.h" + #pragma comment(lib, "strmiids.lib") namespace media { diff --git a/chromium/media/capture/video/win/metrics.cc b/chromium/media/capture/video/win/metrics.cc index f128ee18e0b..812030168d5 100644 --- a/chromium/media/capture/video/win/metrics.cc +++ b/chromium/media/capture/video/win/metrics.cc @@ -6,6 +6,7 @@ #include "base/metrics/histogram_functions.h" #include "base/metrics/histogram_macros.h" +#include "base/notreached.h" namespace media { diff --git a/chromium/media/capture/video/win/pin_base_win.cc b/chromium/media/capture/video/win/pin_base_win.cc index a5d1c10998b..a2e840ff98f 100644 --- a/chromium/media/capture/video/win/pin_base_win.cc +++ b/chromium/media/capture/video/win/pin_base_win.cc @@ -4,7 +4,8 @@ #include "media/capture/video/win/pin_base_win.h" -#include "base/logging.h" +#include "base/check.h" +#include "base/notreached.h" namespace media { diff --git a/chromium/media/capture/video/win/sink_filter_win.cc b/chromium/media/capture/video/win/sink_filter_win.cc index 0b6c8e76fd3..90ad4ac4486 100644 --- a/chromium/media/capture/video/win/sink_filter_win.cc +++ b/chromium/media/capture/video/win/sink_filter_win.cc @@ -4,7 +4,6 @@ #include "media/capture/video/win/sink_filter_win.h" -#include "base/logging.h" #include "media/capture/video/win/sink_input_pin_win.h" namespace media { diff --git a/chromium/media/capture/video/win/video_capture_device_mf_win.cc b/chromium/media/capture/video/win/video_capture_device_mf_win.cc index 5d71ad2721b..c432b210efd 100644 --- a/chromium/media/capture/video/win/video_capture_device_mf_win.cc +++ b/chromium/media/capture/video/win/video_capture_device_mf_win.cc @@ -32,6 +32,17 @@ using Microsoft::WRL::ComPtr; namespace media { +#if DCHECK_IS_ON() +#define DLOG_IF_FAILED_WITH_HRESULT(message, hr) \ + { \ + DLOG_IF(ERROR, FAILED(hr)) \ + << (message) << ": " << logging::SystemErrorCodeToString(hr); \ + } +#else +#define DLOG_IF_FAILED_WITH_HRESULT(message, hr) \ + {} +#endif + namespace { class MFPhotoCallback final @@ -345,6 +356,26 @@ HRESULT CreateCaptureEngine(IMFCaptureEngine** engine) { return capture_engine_class_factory->CreateInstance(CLSID_MFCaptureEngine, IID_PPV_ARGS(engine)); } + +// Retrieves the control range and value, and +// optionally returns the associated supported and current mode. +template <typename ControlInterface, typename ControlProperty> +mojom::RangePtr RetrieveControlRangeAndCurrent( + ComPtr<ControlInterface>& control_interface, + ControlProperty control_property, + std::vector<mojom::MeteringMode>* supported_modes = nullptr, + mojom::MeteringMode* current_mode = nullptr, + double (*value_converter)(long) = PlatformToCaptureValue, + double (*step_converter)(long) = PlatformToCaptureValue) { + return media::RetrieveControlRangeAndCurrent( + [&control_interface, control_property](auto... args) { + return control_interface->GetRange(control_property, args...); + }, + [&control_interface, control_property](auto... args) { + return control_interface->Get(control_property, args...); + }, + supported_modes, current_mode, value_converter, step_converter); +} } // namespace class MFVideoCallback final @@ -407,9 +438,46 @@ class MFVideoCallback final ComPtr<IMFMediaBuffer> buffer; sample->GetBufferByIndex(i, &buffer); if (buffer) { - DWORD length = 0, max_length = 0; - BYTE* data = NULL; - buffer->Lock(&data, &max_length, &length); + // Lock the buffer using the fastest method that it supports. The + // Lock2DSize() method is faster than Lock2D(), which is faster than + // Lock(). + DWORD length = 0; + BYTE* data = nullptr; + ComPtr<IMF2DBuffer> buffer_2d; + if (SUCCEEDED(buffer.As(&buffer_2d))) { + HRESULT lock_result; + BYTE* scanline_0 = nullptr; + LONG pitch = 0; + ComPtr<IMF2DBuffer2> buffer_2d_2; + if (SUCCEEDED(buffer.As(&buffer_2d_2))) { + BYTE* data_start; + lock_result = + buffer_2d_2->Lock2DSize(MF2DBuffer_LockFlags_Read, &scanline_0, + &pitch, &data_start, &length); + } else { + lock_result = buffer_2d->Lock2D(&scanline_0, &pitch); + } + if (SUCCEEDED(lock_result)) { + // Use |buffer_2d| only if it is contiguous and has positive pitch. + BOOL is_contiguous; + if (pitch > 0 && + SUCCEEDED(buffer_2d->IsContiguousFormat(&is_contiguous)) && + is_contiguous && + (length || + SUCCEEDED(buffer_2d->GetContiguousLength(&length)))) { + data = scanline_0; + } else { + buffer_2d->Unlock2D(); + } + } + } + if (!data) { + // If the faster methods fail, fall back to Lock to lock the buffer. + buffer_2d = nullptr; + DWORD max_length = 0; + buffer->Lock(&data, &max_length, &length); + } + if (data) { observer_->OnIncomingCapturedData(data, length, reference_time, timestamp); @@ -418,7 +486,12 @@ class MFVideoCallback final VideoCaptureFrameDropReason:: kWinMediaFoundationLockingBufferDelieveredNullptr); } - buffer->Unlock(); + + if (buffer_2d) + buffer_2d->Unlock2D(); + else + buffer->Unlock(); + } else { observer_->OnFrameDropped( VideoCaptureFrameDropReason:: @@ -581,7 +654,10 @@ VideoCaptureDeviceMFWin::VideoCaptureDeviceMFWin( source_(source), engine_(engine), is_started_(false), - has_sent_on_started_to_client_(false) { + has_sent_on_started_to_client_(false), + exposure_mode_manual_(false), + focus_mode_manual_(false), + white_balance_mode_manual_(false) { DETACH_FROM_SEQUENCE(sequence_checker_); } @@ -602,14 +678,20 @@ VideoCaptureDeviceMFWin::~VideoCaptureDeviceMFWin() { bool VideoCaptureDeviceMFWin::Init() { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DCHECK(!is_initialized_); + HRESULT hr; - HRESULT hr = S_OK; - if (!engine_) - hr = CreateCaptureEngine(&engine_); + hr = source_.As(&camera_control_); + DLOG_IF_FAILED_WITH_HRESULT("Failed to retrieve IAMCameraControl", hr); - if (FAILED(hr)) { - LogError(FROM_HERE, hr); - return false; + hr = source_.As(&video_control_); + DLOG_IF_FAILED_WITH_HRESULT("Failed to retrieve IAMVideoProcAmp", hr); + + if (!engine_) { + hr = CreateCaptureEngine(&engine_); + if (FAILED(hr)) { + LogError(FROM_HERE, hr); + return false; + } } ComPtr<IMFAttributes> attributes; @@ -931,6 +1013,48 @@ void VideoCaptureDeviceMFWin::GetPhotoState(GetPhotoStateCallback callback) { photo_capabilities->width = mojom::Range::New( max_size.width(), min_size.width(), current_size.width(), 1); + if (camera_control_ && video_control_) { + photo_capabilities->color_temperature = RetrieveControlRangeAndCurrent( + video_control_, VideoProcAmp_WhiteBalance, + &photo_capabilities->supported_white_balance_modes, + &photo_capabilities->current_white_balance_mode); + + photo_capabilities->exposure_time = RetrieveControlRangeAndCurrent( + camera_control_, CameraControl_Exposure, + &photo_capabilities->supported_exposure_modes, + &photo_capabilities->current_exposure_mode, + PlatformExposureTimeToCaptureValue, PlatformExposureTimeToCaptureStep); + + photo_capabilities->focus_distance = RetrieveControlRangeAndCurrent( + camera_control_, CameraControl_Focus, + &photo_capabilities->supported_focus_modes, + &photo_capabilities->current_focus_mode); + + photo_capabilities->brightness = + RetrieveControlRangeAndCurrent(video_control_, VideoProcAmp_Brightness); + photo_capabilities->contrast = + RetrieveControlRangeAndCurrent(video_control_, VideoProcAmp_Contrast); + photo_capabilities->exposure_compensation = + RetrieveControlRangeAndCurrent(video_control_, VideoProcAmp_Gain); + // There is no ISO control property in IAMCameraControl or IAMVideoProcAmp + // interfaces nor any other control property with direct mapping to ISO. + photo_capabilities->iso = mojom::Range::New(); + photo_capabilities->red_eye_reduction = mojom::RedEyeReduction::NEVER; + photo_capabilities->saturation = + RetrieveControlRangeAndCurrent(video_control_, VideoProcAmp_Saturation); + photo_capabilities->sharpness = + RetrieveControlRangeAndCurrent(video_control_, VideoProcAmp_Sharpness); + photo_capabilities->torch = false; + photo_capabilities->pan = RetrieveControlRangeAndCurrent( + camera_control_, CameraControl_Pan, nullptr, nullptr, + PlatformAngleToCaptureValue, PlatformAngleToCaptureStep); + photo_capabilities->tilt = RetrieveControlRangeAndCurrent( + camera_control_, CameraControl_Tilt, nullptr, nullptr, + PlatformAngleToCaptureValue, PlatformAngleToCaptureStep); + photo_capabilities->zoom = + RetrieveControlRangeAndCurrent(camera_control_, CameraControl_Zoom); + } + std::move(callback).Run(std::move(photo_capabilities)); } @@ -979,6 +1103,131 @@ void VideoCaptureDeviceMFWin::SetPhotoOptions( selected_photo_capability_.reset(new CapabilityWin(best_match)); } + if (camera_control_ && video_control_) { + if (settings->has_white_balance_mode) { + if (settings->white_balance_mode == mojom::MeteringMode::CONTINUOUS) { + hr = video_control_->Set(VideoProcAmp_WhiteBalance, 0L, + VideoProcAmp_Flags_Auto); + DLOG_IF_FAILED_WITH_HRESULT("Auto white balance config failed", hr); + if (FAILED(hr)) + return; + white_balance_mode_manual_ = false; + } else { + white_balance_mode_manual_ = true; + } + } + if (white_balance_mode_manual_ && settings->has_color_temperature) { + hr = video_control_->Set(VideoProcAmp_WhiteBalance, + settings->color_temperature, + VideoProcAmp_Flags_Manual); + DLOG_IF_FAILED_WITH_HRESULT("Color temperature config failed", hr); + if (FAILED(hr)) + return; + } + + if (settings->has_exposure_mode) { + if (settings->exposure_mode == mojom::MeteringMode::CONTINUOUS) { + hr = camera_control_->Set(CameraControl_Exposure, 0L, + CameraControl_Flags_Auto); + DLOG_IF_FAILED_WITH_HRESULT("Auto exposure config failed", hr); + if (FAILED(hr)) + return; + exposure_mode_manual_ = false; + } else { + exposure_mode_manual_ = true; + } + } + if (exposure_mode_manual_ && settings->has_exposure_time) { + hr = camera_control_->Set( + CameraControl_Exposure, + CaptureExposureTimeToPlatformValue(settings->exposure_time), + CameraControl_Flags_Manual); + DLOG_IF_FAILED_WITH_HRESULT("Exposure Time config failed", hr); + if (FAILED(hr)) + return; + } + + if (settings->has_focus_mode) { + if (settings->focus_mode == mojom::MeteringMode::CONTINUOUS) { + hr = camera_control_->Set(CameraControl_Focus, 0L, + CameraControl_Flags_Auto); + DLOG_IF_FAILED_WITH_HRESULT("Auto focus config failed", hr); + if (FAILED(hr)) + return; + focus_mode_manual_ = false; + } else { + focus_mode_manual_ = true; + } + } + if (focus_mode_manual_ && settings->has_focus_distance) { + hr = camera_control_->Set(CameraControl_Focus, settings->focus_distance, + CameraControl_Flags_Manual); + DLOG_IF_FAILED_WITH_HRESULT("Focus Distance config failed", hr); + if (FAILED(hr)) + return; + } + + if (settings->has_brightness) { + hr = video_control_->Set(VideoProcAmp_Brightness, settings->brightness, + VideoProcAmp_Flags_Manual); + DLOG_IF_FAILED_WITH_HRESULT("Brightness config failed", hr); + if (FAILED(hr)) + return; + } + if (settings->has_contrast) { + hr = video_control_->Set(VideoProcAmp_Contrast, settings->contrast, + VideoProcAmp_Flags_Manual); + DLOG_IF_FAILED_WITH_HRESULT("Contrast config failed", hr); + if (FAILED(hr)) + return; + } + if (settings->has_exposure_compensation) { + hr = video_control_->Set(VideoProcAmp_Gain, + settings->exposure_compensation, + VideoProcAmp_Flags_Manual); + DLOG_IF_FAILED_WITH_HRESULT("Exposure Compensation config failed", hr); + if (FAILED(hr)) + return; + } + if (settings->has_saturation) { + hr = video_control_->Set(VideoProcAmp_Saturation, settings->saturation, + VideoProcAmp_Flags_Manual); + DLOG_IF_FAILED_WITH_HRESULT("Saturation config failed", hr); + if (FAILED(hr)) + return; + } + if (settings->has_sharpness) { + hr = video_control_->Set(VideoProcAmp_Sharpness, settings->sharpness, + VideoProcAmp_Flags_Manual); + DLOG_IF_FAILED_WITH_HRESULT("Sharpness config failed", hr); + if (FAILED(hr)) + return; + } + if (settings->has_pan) { + hr = camera_control_->Set(CameraControl_Pan, + CaptureAngleToPlatformValue(settings->pan), + CameraControl_Flags_Manual); + DLOG_IF_FAILED_WITH_HRESULT("Pan config failed", hr); + if (FAILED(hr)) + return; + } + if (settings->has_tilt) { + hr = camera_control_->Set(CameraControl_Tilt, + CaptureAngleToPlatformValue(settings->tilt), + CameraControl_Flags_Manual); + DLOG_IF_FAILED_WITH_HRESULT("Tilt config failed", hr); + if (FAILED(hr)) + return; + } + if (settings->has_zoom) { + hr = camera_control_->Set(CameraControl_Zoom, settings->zoom, + CameraControl_Flags_Manual); + DLOG_IF_FAILED_WITH_HRESULT("Zoom config failed", hr); + if (FAILED(hr)) + return; + } + } + std::move(callback).Run(true); } diff --git a/chromium/media/capture/video/win/video_capture_device_mf_win.h b/chromium/media/capture/video/win/video_capture_device_mf_win.h index 36111a184b2..e5ecd1250ba 100644 --- a/chromium/media/capture/video/win/video_capture_device_mf_win.h +++ b/chromium/media/capture/video/win/video_capture_device_mf_win.h @@ -13,6 +13,7 @@ #include <mfidl.h> #include <mfreadwrite.h> #include <stdint.h> +#include <strmif.h> #include <wrl/client.h> #include <vector> @@ -131,12 +132,18 @@ class CAPTURE_EXPORT VideoCaptureDeviceMFWin : public VideoCaptureDevice { std::unique_ptr<VideoCaptureDevice::Client> client_; const Microsoft::WRL::ComPtr<IMFMediaSource> source_; + Microsoft::WRL::ComPtr<IAMCameraControl> camera_control_; + Microsoft::WRL::ComPtr<IAMVideoProcAmp> video_control_; Microsoft::WRL::ComPtr<IMFCaptureEngine> engine_; std::unique_ptr<CapabilityWin> selected_video_capability_; CapabilityList photo_capabilities_; std::unique_ptr<CapabilityWin> selected_photo_capability_; bool is_started_; bool has_sent_on_started_to_client_; + // These flags keep the manual/auto mode between cycles of SetPhotoOptions(). + bool exposure_mode_manual_; + bool focus_mode_manual_; + bool white_balance_mode_manual_; base::queue<TakePhotoCallback> video_stream_take_photo_callbacks_; SEQUENCE_CHECKER(sequence_checker_); diff --git a/chromium/media/capture/video/win/video_capture_device_utils_win.cc b/chromium/media/capture/video/win/video_capture_device_utils_win.cc index 868ae1990b4..d6bf4584c77 100644 --- a/chromium/media/capture/video/win/video_capture_device_utils_win.cc +++ b/chromium/media/capture/video/win/video_capture_device_utils_win.cc @@ -4,6 +4,7 @@ #include "media/capture/video/win/video_capture_device_utils_win.h" +#include <cmath> #include <iostream> #include "base/win/win_util.h" @@ -11,6 +12,40 @@ namespace media { +namespace { +const int kDegreesToArcSeconds = 3600; +const int kSecondsTo100MicroSeconds = 10000; +} // namespace + +// Windows platform stores pan and tilt (min, max, step and current) in +// degrees. Spec expects them in arc seconds. +// https://docs.microsoft.com/en-us/windows/win32/api/strmif/ne-strmif-cameracontrolproperty +// spec: https://w3c.github.io/mediacapture-image/#pan +long CaptureAngleToPlatformValue(double arc_seconds) { + return std::round(arc_seconds / kDegreesToArcSeconds); +} + +double PlatformAngleToCaptureValue(long degrees) { + return 1.0 * degrees * kDegreesToArcSeconds; +} + +// Windows platform stores exposure time (min, max and current) in log base 2 +// seconds. If value is n, exposure time is 2^n seconds. Spec expects exposure +// times in 100 micro seconds. +// https://docs.microsoft.com/en-us/windows/win32/api/strmif/ne-strmif-cameracontrolproperty +// spec: https://w3c.github.io/mediacapture-image/#exposure-time +long CaptureExposureTimeToPlatformValue(double hundreds_of_microseconds) { + return std::log2(hundreds_of_microseconds / kSecondsTo100MicroSeconds); +} + +double PlatformExposureTimeToCaptureValue(long log_seconds) { + return std::exp2(log_seconds) * kSecondsTo100MicroSeconds; +} + +double PlatformExposureTimeToCaptureStep(long log_step) { + return std::exp2(log_step); +} + // Note: Because we can't find a solid way to detect camera location (front/back // or external USB camera) with Win32 APIs, assume it's always front camera when // auto rotation is enabled for now. diff --git a/chromium/media/capture/video/win/video_capture_device_utils_win.h b/chromium/media/capture/video/win/video_capture_device_utils_win.h index 84c6b1a8239..cff4bfc1b99 100644 --- a/chromium/media/capture/video/win/video_capture_device_utils_win.h +++ b/chromium/media/capture/video/win/video_capture_device_utils_win.h @@ -5,12 +5,33 @@ #ifndef MEDIA_CAPTURE_VIDEO_WIN_VIDEO_CAPTURE_DEVICE_UTILS_WIN_H_ #define MEDIA_CAPTURE_VIDEO_WIN_VIDEO_CAPTURE_DEVICE_UTILS_WIN_H_ +// Avoid including strsafe.h via dshow as it will cause build warnings. +#define NO_DSHOW_STRSAFE +#include <dshow.h> #include <windows.h> #include "media/base/video_facing.h" +#include "media/capture/mojom/image_capture_types.h" namespace media { +// Windows platform stores pan and tilt (min, max, step and current) in +// degrees. Spec expects them in arc seconds. +// https://docs.microsoft.com/en-us/windows/win32/api/strmif/ne-strmif-cameracontrolproperty +// spec: https://w3c.github.io/mediacapture-image/#pan +long CaptureAngleToPlatformValue(double arc_seconds); +double PlatformAngleToCaptureValue(long degrees); +constexpr auto PlatformAngleToCaptureStep = PlatformAngleToCaptureValue; + +// Windows platform stores exposure time (min, max and current) in log base 2 +// seconds. If value is n, exposure time is 2^n seconds. Spec expects exposure +// times in 100 micro seconds. +// https://docs.microsoft.com/en-us/windows/win32/api/strmif/ne-strmif-cameracontrolproperty +// spec: https://w3c.github.io/mediacapture-image/#exposure-time +long CaptureExposureTimeToPlatformValue(double hundreds_of_microseconds); +double PlatformExposureTimeToCaptureValue(long log_seconds); +double PlatformExposureTimeToCaptureStep(long log_step); + // Returns the rotation of the camera. Returns 0 if it's not a built-in camera, // or auto-rotation is not enabled, or only displays on external monitors. int GetCameraRotation(VideoFacingMode facing); @@ -30,6 +51,57 @@ HRESULT CheckPathInfoForInternal(const PCWSTR device_name); // Returns true if this is an integrated display panel. bool IsInternalVideoOutput( const DISPLAYCONFIG_VIDEO_OUTPUT_TECHNOLOGY video_output_tech_type); + +static double PlatformToCaptureValue(long value) { + return value; +} +constexpr auto PlatformToCaptureStep = PlatformToCaptureValue; + +// Retrieves the control range and value using the provided getters, and +// optionally returns the associated supported and current mode. +template <typename RangeGetter, typename CurrentValueGetter> +static mojom::RangePtr RetrieveControlRangeAndCurrent( + RangeGetter range_getter, + CurrentValueGetter current_value_getter, + std::vector<mojom::MeteringMode>* supported_modes = nullptr, + mojom::MeteringMode* current_mode = nullptr, + double (*value_converter)(long) = PlatformToCaptureValue, + double (*step_converter)(long) = PlatformToCaptureStep) { + auto control_range = mojom::Range::New(); + + long min, max, step, default_value, flags; + HRESULT hr = range_getter(&min, &max, &step, &default_value, &flags); + DLOG_IF(ERROR, FAILED(hr)) << "Control range reading failed: " + << logging::SystemErrorCodeToString(hr); + if (SUCCEEDED(hr)) { + control_range->min = value_converter(min); + control_range->max = value_converter(max); + control_range->step = step_converter(step); + if (supported_modes != nullptr) { + if (flags & CameraControl_Flags_Auto) + supported_modes->push_back(mojom::MeteringMode::CONTINUOUS); + if (flags & CameraControl_Flags_Manual) + supported_modes->push_back(mojom::MeteringMode::MANUAL); + } + } + + long current; + hr = current_value_getter(¤t, &flags); + DLOG_IF(ERROR, FAILED(hr)) << "Control value reading failed: " + << logging::SystemErrorCodeToString(hr); + if (SUCCEEDED(hr)) { + control_range->current = value_converter(current); + if (current_mode != nullptr) { + if (flags & CameraControl_Flags_Auto) + *current_mode = mojom::MeteringMode::CONTINUOUS; + else if (flags & CameraControl_Flags_Manual) + *current_mode = mojom::MeteringMode::MANUAL; + } + } + + return control_range; +} + } // namespace media -#endif // MEDIA_CAPTURE_VIDEO_WIN_VIDEO_CAPTURE_DEVICE_UTILS_WIN_H_
\ No newline at end of file +#endif // MEDIA_CAPTURE_VIDEO_WIN_VIDEO_CAPTURE_DEVICE_UTILS_WIN_H_ diff --git a/chromium/media/capture/video/win/video_capture_device_win.cc b/chromium/media/capture/video/win/video_capture_device_win.cc index 96421b15d91..ff7d0f13cad 100644 --- a/chromium/media/capture/video/win/video_capture_device_win.cc +++ b/chromium/media/capture/video/win/video_capture_device_win.cc @@ -29,19 +29,6 @@ using base::win::ScopedCoMem; using base::win::ScopedVariant; using Microsoft::WRL::ComPtr; -namespace { -const int kSecondsTo100MicroSeconds = 10000; - -// Windows platform stores exposure time (min, max and current) in log base 2 -// seconds. If value is n, exposure time is 2^n seconds. Spec expects exposure -// times in 100 micro seconds. -// https://docs.microsoft.com/en-us/previous-versions/ms784800(v%3Dvs.85) -// spec: https://w3c.github.io/mediacapture-image/#exposure-time -long ConvertWindowsTimeToSpec(long seconds) { - return (std::exp2(seconds) * kSecondsTo100MicroSeconds); -} -} // namespace - namespace media { #if DCHECK_IS_ON() @@ -81,45 +68,6 @@ bool PinMatchesMajorType(IPin* pin, REFGUID major_type) { return SUCCEEDED(hr) && connection_media_type.majortype == major_type; } -// Retrieves the control range and value using the provided getters, and -// optionally returns the associated supported and current mode. -template <typename RangeGetter, typename ValueGetter> -mojom::RangePtr RetrieveControlRangeAndCurrent( - RangeGetter range_getter, - ValueGetter value_getter, - std::vector<mojom::MeteringMode>* supported_modes = nullptr, - mojom::MeteringMode* current_mode = nullptr) { - auto control_range = mojom::Range::New(); - long min, max, step, default_value, flags; - HRESULT hr = range_getter(&min, &max, &step, &default_value, &flags); - DLOG_IF_FAILED_WITH_HRESULT("Control range reading failed", hr); - if (SUCCEEDED(hr)) { - control_range->min = min; - control_range->max = max; - control_range->step = step; - if (supported_modes != nullptr) { - if (flags & CameraControl_Flags_Auto) - supported_modes->push_back(mojom::MeteringMode::CONTINUOUS); - if (flags & CameraControl_Flags_Manual) - supported_modes->push_back(mojom::MeteringMode::MANUAL); - } - } - long current; - hr = value_getter(¤t, &flags); - DLOG_IF_FAILED_WITH_HRESULT("Control value reading failed", hr); - if (SUCCEEDED(hr)) { - control_range->current = current; - if (current_mode != nullptr) { - if (flags & CameraControl_Flags_Auto) - *current_mode = mojom::MeteringMode::CONTINUOUS; - else if (flags & CameraControl_Flags_Manual) - *current_mode = mojom::MeteringMode::MANUAL; - } - } - - return control_range; -} - // static void VideoCaptureDeviceWin::GetDeviceCapabilityList( const std::string& device_id, @@ -674,18 +622,8 @@ void VideoCaptureDeviceWin::GetPhotoState(GetPhotoStateCallback callback) { return this->camera_control_->get_Exposure(args...); }, &photo_capabilities->supported_exposure_modes, - &photo_capabilities->current_exposure_mode); - - // Windows returns the exposure time in log base 2 seconds. - // If value is n, exposure time is 2^n seconds. - photo_capabilities->exposure_time->min = - ConvertWindowsTimeToSpec(photo_capabilities->exposure_time->min); - photo_capabilities->exposure_time->max = - ConvertWindowsTimeToSpec(photo_capabilities->exposure_time->max); - photo_capabilities->exposure_time->step = - std::exp2(photo_capabilities->exposure_time->step); - photo_capabilities->exposure_time->current = - ConvertWindowsTimeToSpec(photo_capabilities->exposure_time->current); + &photo_capabilities->current_exposure_mode, + PlatformExposureTimeToCaptureValue, PlatformExposureTimeToCaptureStep); photo_capabilities->color_temperature = RetrieveControlRangeAndCurrent( [this](auto... args) { @@ -737,7 +675,20 @@ void VideoCaptureDeviceWin::GetPhotoState(GetPhotoStateCallback callback) { [this](auto... args) { return this->video_control_->get_Sharpness(args...); }); - + photo_capabilities->pan = RetrieveControlRangeAndCurrent( + [this](auto... args) { + return this->camera_control_->getRange_Pan(args...); + }, + [this](auto... args) { return this->camera_control_->get_Pan(args...); }, + nullptr, nullptr, PlatformAngleToCaptureValue, + PlatformAngleToCaptureStep); + photo_capabilities->tilt = RetrieveControlRangeAndCurrent( + [this](auto... args) { + return this->camera_control_->getRange_Tilt(args...); + }, + [this](auto... args) { return this->camera_control_->get_Tilt(args...); }, + nullptr, nullptr, PlatformAngleToCaptureValue, + PlatformAngleToCaptureStep); photo_capabilities->zoom = RetrieveControlRangeAndCurrent( [this](auto... args) { return this->camera_control_->getRange_Zoom(args...); @@ -770,13 +721,6 @@ void VideoCaptureDeviceWin::SetPhotoOptions( HRESULT hr; - if (settings->has_zoom) { - hr = camera_control_->put_Zoom(settings->zoom, CameraControl_Flags_Manual); - DLOG_IF_FAILED_WITH_HRESULT("Zoom config failed", hr); - if (FAILED(hr)) - return; - } - if (settings->has_white_balance_mode) { if (settings->white_balance_mode == mojom::MeteringMode::CONTINUOUS) { hr = video_control_->put_WhiteBalance(0L, VideoProcAmp_Flags_Auto); @@ -791,7 +735,7 @@ void VideoCaptureDeviceWin::SetPhotoOptions( } if (white_balance_mode_manual_ && settings->has_color_temperature) { hr = video_control_->put_WhiteBalance(settings->color_temperature, - CameraControl_Flags_Manual); + VideoProcAmp_Flags_Manual); DLOG_IF_FAILED_WITH_HRESULT("Color temperature config failed", hr); if (FAILED(hr)) return; @@ -799,7 +743,7 @@ void VideoCaptureDeviceWin::SetPhotoOptions( if (settings->has_focus_mode) { if (settings->focus_mode == mojom::MeteringMode::CONTINUOUS) { - hr = camera_control_->put_Focus(0L, VideoProcAmp_Flags_Auto); + hr = camera_control_->put_Focus(0L, CameraControl_Flags_Auto); DLOG_IF_FAILED_WITH_HRESULT("Auto focus config failed", hr); if (FAILED(hr)) return; @@ -819,7 +763,7 @@ void VideoCaptureDeviceWin::SetPhotoOptions( if (settings->has_exposure_mode) { if (settings->exposure_mode == mojom::MeteringMode::CONTINUOUS) { - hr = camera_control_->put_Exposure(0L, VideoProcAmp_Flags_Auto); + hr = camera_control_->put_Exposure(0L, CameraControl_Flags_Auto); DLOG_IF_FAILED_WITH_HRESULT("Auto exposure config failed", hr); if (FAILED(hr)) return; @@ -832,7 +776,7 @@ void VideoCaptureDeviceWin::SetPhotoOptions( if (exposure_mode_manual_ && settings->has_exposure_time) { // Windows expects the exposure time in log base 2 seconds. hr = camera_control_->put_Exposure( - std::log2(settings->exposure_time / kSecondsTo100MicroSeconds), + CaptureExposureTimeToPlatformValue(settings->exposure_time), CameraControl_Flags_Manual); DLOG_IF_FAILED_WITH_HRESULT("Exposure Time config failed", hr); if (FAILED(hr)) @@ -840,32 +784,52 @@ void VideoCaptureDeviceWin::SetPhotoOptions( } if (settings->has_brightness) { hr = video_control_->put_Brightness(settings->brightness, - CameraControl_Flags_Manual); + VideoProcAmp_Flags_Manual); DLOG_IF_FAILED_WITH_HRESULT("Brightness config failed", hr); if (FAILED(hr)) return; } if (settings->has_contrast) { hr = video_control_->put_Contrast(settings->contrast, - CameraControl_Flags_Manual); + VideoProcAmp_Flags_Manual); DLOG_IF_FAILED_WITH_HRESULT("Contrast config failed", hr); if (FAILED(hr)) return; } if (settings->has_saturation) { hr = video_control_->put_Saturation(settings->saturation, - CameraControl_Flags_Manual); + VideoProcAmp_Flags_Manual); DLOG_IF_FAILED_WITH_HRESULT("Saturation config failed", hr); if (FAILED(hr)) return; } if (settings->has_sharpness) { hr = video_control_->put_Sharpness(settings->sharpness, - CameraControl_Flags_Manual); + VideoProcAmp_Flags_Manual); DLOG_IF_FAILED_WITH_HRESULT("Sharpness config failed", hr); if (FAILED(hr)) return; } + if (settings->has_pan) { + hr = camera_control_->put_Pan(CaptureAngleToPlatformValue(settings->pan), + CameraControl_Flags_Manual); + DLOG_IF_FAILED_WITH_HRESULT("Pan config failed", hr); + if (FAILED(hr)) + return; + } + if (settings->has_tilt) { + hr = camera_control_->put_Tilt(CaptureAngleToPlatformValue(settings->tilt), + CameraControl_Flags_Manual); + DLOG_IF_FAILED_WITH_HRESULT("Tilt config failed", hr); + if (FAILED(hr)) + return; + } + if (settings->has_zoom) { + hr = camera_control_->put_Zoom(settings->zoom, CameraControl_Flags_Manual); + DLOG_IF_FAILED_WITH_HRESULT("Zoom config failed", hr); + if (FAILED(hr)) + return; + } std::move(callback).Run(true); } diff --git a/chromium/media/capture/video_capture_types.cc b/chromium/media/capture/video_capture_types.cc index 585dafba21b..bbee389d99f 100644 --- a/chromium/media/capture/video_capture_types.cc +++ b/chromium/media/capture/video_capture_types.cc @@ -4,7 +4,7 @@ #include "media/capture/video_capture_types.h" -#include "base/logging.h" +#include "base/check.h" #include "base/stl_util.h" #include "base/strings/stringprintf.h" #include "media/base/limits.h" diff --git a/chromium/media/capture/video_capture_types.h b/chromium/media/capture/video_capture_types.h index 98fca4a9bd0..4fab0d0a07f 100644 --- a/chromium/media/capture/video_capture_types.h +++ b/chromium/media/capture/video_capture_types.h @@ -181,7 +181,14 @@ enum class VideoCaptureError { kMacAvFoundationReceivedAVCaptureSessionRuntimeErrorNotification = 113, kAndroidApi2ErrorConfiguringCamera = 114, kCrosHalV3DeviceDelegateFailedToFlush = 115, - kMaxValue = 115 + kFuchsiaCameraDeviceDisconnected = 116, + kFuchsiaCameraStreamDisconnected = 117, + kFuchsiaSysmemDidNotSetImageFormat = 118, + kFuchsiaSysmemInvalidBufferIndex = 119, + kFuchsiaSysmemInvalidBufferSize = 120, + kFuchsiaUnsupportedPixelFormat = 121, + kFuchsiaFailedToMapSysmemBuffer = 122, + kMaxValue = 122 }; // WARNING: Do not change the values assigned to the entries. They are used for diff --git a/chromium/media/cast/BUILD.gn b/chromium/media/cast/BUILD.gn index a915bacefa2..ccd2acdb8ba 100644 --- a/chromium/media/cast/BUILD.gn +++ b/chromium/media/cast/BUILD.gn @@ -406,6 +406,7 @@ if (is_win || is_mac || (is_linux && !is_chromeos)) { executable("cast_receiver_app") { testonly = true sources = [ "test/receiver.cc" ] + public_deps = [] deps = [ ":common", ":net", @@ -423,10 +424,7 @@ if (is_win || is_mac || (is_linux && !is_chromeos)) { "test/linux_output_window.cc", "test/linux_output_window.h", ] - configs += [ - "//build/config/linux:x11", - "//build/config/linux:xext", - ] + public_deps += [ "//ui/gfx/x" ] deps += [ "//third_party/libyuv" ] } } diff --git a/chromium/media/cast/cast_config.cc b/chromium/media/cast/cast_config.cc index 3a27dc0db88..30e6c41c74c 100644 --- a/chromium/media/cast/cast_config.cc +++ b/chromium/media/cast/cast_config.cc @@ -23,10 +23,15 @@ VideoCodecParams::~VideoCodecParams() = default; FrameSenderConfig::FrameSenderConfig() : sender_ssrc(0), receiver_ssrc(0), + // In production, these values are overridden by the mirror settings + // and potentially the mirroring session parameters, however we provide + // a reasonable default here for some use cases, such as tests. + // All three delays are set to the same value due to adaptive latency + // being disabled in Chrome. This will be fixed as part of the migration + // to libcast. min_playout_delay( base::TimeDelta::FromMilliseconds(kDefaultRtpMaxDelayMs)), - max_playout_delay( - base::TimeDelta::FromMilliseconds(kDefaultRtpMaxDelayMs)), + max_playout_delay(min_playout_delay), animated_playout_delay(min_playout_delay), rtp_payload_type(RtpPayloadType::UNKNOWN), use_external_encoder(false), diff --git a/chromium/media/cast/cast_environment.cc b/chromium/media/cast/cast_environment.cc index 281b904903f..0f9b4ca896f 100644 --- a/chromium/media/cast/cast_environment.cc +++ b/chromium/media/cast/cast_environment.cc @@ -8,7 +8,7 @@ #include "base/bind.h" #include "base/location.h" -#include "base/logging.h" +#include "base/notreached.h" using base::SingleThreadTaskRunner; diff --git a/chromium/media/cast/cast_sender.h b/chromium/media/cast/cast_sender.h index b7bd819efc5..48b2d3d33d6 100644 --- a/chromium/media/cast/cast_sender.h +++ b/chromium/media/cast/cast_sender.h @@ -82,6 +82,9 @@ class AudioFrameInput : public base::RefCountedThreadSafe<AudioFrameInput> { // have halted the session. using StatusChangeCallback = base::RepeatingCallback<void(OperationalStatus)>; +// The equivalent of StatusChangeCallback when only one change is expected. +using StatusChangeOnceCallback = base::OnceCallback<void(OperationalStatus)>; + // All methods of CastSender must be called on the main thread. // Provided CastTransport will also be called on the main thread. class CastSender { @@ -100,9 +103,8 @@ class CastSender { // Initialize the audio stack. Must be called in order to send audio frames. // |status_change_cb| will be run as operational status changes. - virtual void InitializeAudio( - const FrameSenderConfig& audio_config, - const StatusChangeCallback& status_change_cb) = 0; + virtual void InitializeAudio(const FrameSenderConfig& audio_config, + StatusChangeOnceCallback status_change_cb) = 0; // Initialize the video stack. Must be called in order to send video frames. // |status_change_cb| will be run as operational status changes. diff --git a/chromium/media/cast/cast_sender_impl.cc b/chromium/media/cast/cast_sender_impl.cc index 7d9392b5981..bf75331db7e 100644 --- a/chromium/media/cast/cast_sender_impl.cc +++ b/chromium/media/cast/cast_sender_impl.cc @@ -104,7 +104,7 @@ CastSenderImpl::CastSenderImpl(scoped_refptr<CastEnvironment> cast_environment, void CastSenderImpl::InitializeAudio( const FrameSenderConfig& audio_config, - const StatusChangeCallback& status_change_cb) { + StatusChangeOnceCallback status_change_cb) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); CHECK(audio_config.use_external_encoder || cast_environment_->HasAudioThread()); @@ -113,8 +113,8 @@ void CastSenderImpl::InitializeAudio( audio_sender_ = std::make_unique<AudioSender>( cast_environment_, audio_config, - base::BindRepeating(&CastSenderImpl::OnAudioStatusChange, - weak_factory_.GetWeakPtr(), status_change_cb), + base::BindOnce(&CastSenderImpl::OnAudioStatusChange, + weak_factory_.GetWeakPtr(), std::move(status_change_cb)), transport_sender_); if (video_sender_) { DCHECK(audio_sender_->GetTargetPlayoutDelay() == @@ -169,14 +169,14 @@ void CastSenderImpl::SetTargetPlayoutDelay( } void CastSenderImpl::OnAudioStatusChange( - const StatusChangeCallback& status_change_cb, + StatusChangeOnceCallback status_change_cb, OperationalStatus status) { DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); if (status == STATUS_INITIALIZED && !audio_frame_input_) { audio_frame_input_ = new LocalAudioFrameInput(cast_environment_, audio_sender_->AsWeakPtr()); } - status_change_cb.Run(status); + std::move(status_change_cb).Run(status); } void CastSenderImpl::OnVideoStatusChange( diff --git a/chromium/media/cast/cast_sender_impl.h b/chromium/media/cast/cast_sender_impl.h index 9ec3651f5d6..0d36d971bb7 100644 --- a/chromium/media/cast/cast_sender_impl.h +++ b/chromium/media/cast/cast_sender_impl.h @@ -27,7 +27,7 @@ class CastSenderImpl : public CastSender { CastTransport* const transport_sender); void InitializeAudio(const FrameSenderConfig& audio_config, - const StatusChangeCallback& status_change_cb) final; + StatusChangeOnceCallback status_change_cb) final; void InitializeVideo( const FrameSenderConfig& video_config, const StatusChangeCallback& status_change_cb, @@ -43,7 +43,7 @@ class CastSenderImpl : public CastSender { private: void ReceivedPacket(std::unique_ptr<Packet> packet); - void OnAudioStatusChange(const StatusChangeCallback& status_change_cb, + void OnAudioStatusChange(StatusChangeOnceCallback status_change_cb, OperationalStatus status); void OnVideoStatusChange(const StatusChangeCallback& status_change_cb, OperationalStatus status); diff --git a/chromium/media/cast/common/clock_drift_smoother.cc b/chromium/media/cast/common/clock_drift_smoother.cc index 79a60212bcd..dd8c39e61ed 100644 --- a/chromium/media/cast/common/clock_drift_smoother.cc +++ b/chromium/media/cast/common/clock_drift_smoother.cc @@ -6,7 +6,8 @@ #include <stdint.h> -#include "base/logging.h" +#include "base/check.h" +#include "base/notreached.h" namespace media { namespace cast { diff --git a/chromium/media/cast/logging/log_deserializer.cc b/chromium/media/cast/logging/log_deserializer.cc index b006df4951f..64c90ee88f8 100644 --- a/chromium/media/cast/logging/log_deserializer.cc +++ b/chromium/media/cast/logging/log_deserializer.cc @@ -10,6 +10,7 @@ #include <utility> #include "base/big_endian.h" +#include "base/logging.h" #include "third_party/zlib/zlib.h" using media::cast::FrameEventMap; diff --git a/chromium/media/cast/logging/logging_defines.cc b/chromium/media/cast/logging/logging_defines.cc index a7e4c714d2d..b492c0a18ac 100644 --- a/chromium/media/cast/logging/logging_defines.cc +++ b/chromium/media/cast/logging/logging_defines.cc @@ -4,7 +4,7 @@ #include "media/cast/logging/logging_defines.h" -#include "base/logging.h" +#include "base/notreached.h" #define ENUM_TO_STRING(enum) \ case enum: \ diff --git a/chromium/media/cast/logging/proto/proto_utils.cc b/chromium/media/cast/logging/proto/proto_utils.cc index 03251e64c03..63eb55717d0 100644 --- a/chromium/media/cast/logging/proto/proto_utils.cc +++ b/chromium/media/cast/logging/proto/proto_utils.cc @@ -4,7 +4,7 @@ #include "media/cast/logging/proto/proto_utils.h" -#include "base/logging.h" +#include "base/notreached.h" #define TO_PROTO_ENUM(enum) \ case enum: \ diff --git a/chromium/media/cast/logging/receiver_time_offset_estimator_impl.cc b/chromium/media/cast/logging/receiver_time_offset_estimator_impl.cc index 2979cc621b2..b401ead8d9d 100644 --- a/chromium/media/cast/logging/receiver_time_offset_estimator_impl.cc +++ b/chromium/media/cast/logging/receiver_time_offset_estimator_impl.cc @@ -5,7 +5,7 @@ #include <algorithm> #include <utility> -#include "base/logging.h" +#include "base/check.h" #include "base/time/tick_clock.h" #include "media/cast/logging/receiver_time_offset_estimator_impl.h" diff --git a/chromium/media/cast/logging/simple_event_subscriber.cc b/chromium/media/cast/logging/simple_event_subscriber.cc index d0f0df567c1..86901a82e4a 100644 --- a/chromium/media/cast/logging/simple_event_subscriber.cc +++ b/chromium/media/cast/logging/simple_event_subscriber.cc @@ -4,7 +4,7 @@ #include "media/cast/logging/simple_event_subscriber.h" -#include "base/logging.h" +#include "base/check.h" namespace media { namespace cast { diff --git a/chromium/media/cast/logging/stats_event_subscriber.cc b/chromium/media/cast/logging/stats_event_subscriber.cc index 985c69137c7..50fc1620b56 100644 --- a/chromium/media/cast/logging/stats_event_subscriber.cc +++ b/chromium/media/cast/logging/stats_event_subscriber.cc @@ -9,8 +9,9 @@ #include <memory> #include <utility> +#include "base/check_op.h" #include "base/format_macros.h" -#include "base/logging.h" +#include "base/notreached.h" #include "base/strings/stringprintf.h" #include "base/values.h" diff --git a/chromium/media/cast/net/cast_transport_config.h b/chromium/media/cast/net/cast_transport_config.h index 126d942a4e7..5042241637a 100644 --- a/chromium/media/cast/net/cast_transport_config.h +++ b/chromium/media/cast/net/cast_transport_config.h @@ -127,7 +127,7 @@ class PacketTransport { // SendPacket again until |cb| has been called. Any other errors that // occur will be reported through side channels, in such cases, this function // will return true indicating that the channel is not blocked. - virtual bool SendPacket(PacketRef packet, const base::Closure& cb) = 0; + virtual bool SendPacket(PacketRef packet, base::OnceClosure cb) = 0; // Returns the number of bytes ever sent. virtual int64_t GetBytesSent() = 0; diff --git a/chromium/media/cast/net/cast_transport_impl_unittest.cc b/chromium/media/cast/net/cast_transport_impl_unittest.cc index f78e63540c6..137378e5e52 100644 --- a/chromium/media/cast/net/cast_transport_impl_unittest.cc +++ b/chromium/media/cast/net/cast_transport_impl_unittest.cc @@ -47,10 +47,10 @@ class FakePacketSender : public PacketTransport { public: FakePacketSender() : paused_(false), packets_sent_(0), bytes_sent_(0) {} - bool SendPacket(PacketRef packet, const base::Closure& cb) final { + bool SendPacket(PacketRef packet, base::OnceClosure cb) final { if (paused_) { stored_packet_ = packet; - callback_ = cb; + callback_ = std::move(cb); return false; } ++packets_sent_; @@ -67,8 +67,8 @@ class FakePacketSender : public PacketTransport { void SetPaused(bool paused) { paused_ = paused; if (!paused && stored_packet_.get()) { - SendPacket(stored_packet_, callback_); - callback_.Run(); + SendPacket(stored_packet_, base::OnceClosure()); + std::move(callback_).Run(); } } @@ -76,7 +76,7 @@ class FakePacketSender : public PacketTransport { private: bool paused_; - base::Closure callback_; + base::OnceClosure callback_; PacketRef stored_packet_; int packets_sent_; int64_t bytes_sent_; diff --git a/chromium/media/cast/net/pacing/paced_sender.cc b/chromium/media/cast/net/pacing/paced_sender.cc index c1554090254..3e1e8232b42 100644 --- a/chromium/media/cast/net/pacing/paced_sender.cc +++ b/chromium/media/cast/net/pacing/paced_sender.cc @@ -224,10 +224,9 @@ bool PacedSender::SendRtcpPacket(uint32_t ssrc, PacketRef packet) { priority_packet_list_[key] = make_pair(PacketType_RTCP, packet); } else { // We pass the RTCP packets straight through. - if (!transport_->SendPacket( - packet, - base::Bind(&PacedSender::SendStoredPackets, - weak_factory_.GetWeakPtr()))) { + if (!transport_->SendPacket(packet, + base::BindOnce(&PacedSender::SendStoredPackets, + weak_factory_.GetWeakPtr()))) { state_ = State_TransportBlocked; } } @@ -248,8 +247,8 @@ void PacedSender::CancelSendingPacket(const PacketKey& packet_key) { PacketRef PacedSender::PopNextPacket(PacketType* packet_type, PacketKey* packet_key) { // Always pop from the priority list first. - PacketList* list = !priority_packet_list_.empty() ? - &priority_packet_list_ : &packet_list_; + PacketList* list = + !priority_packet_list_.empty() ? &priority_packet_list_ : &packet_list_; DCHECK(!list->empty()); // Determine which packet in the frame should be popped by examining the @@ -354,8 +353,8 @@ void PacedSender::SendStoredPackets() { next_next_max_burst_size_ = max_burst_size; } - base::Closure cb = base::Bind(&PacedSender::SendStoredPackets, - weak_factory_.GetWeakPtr()); + base::RepeatingClosure cb = base::BindRepeating( + &PacedSender::SendStoredPackets, weak_factory_.GetWeakPtr()); while (!empty()) { if (current_burst_size_ >= current_max_burst_size_) { transport_task_runner_->PostDelayedTask(FROM_HERE, diff --git a/chromium/media/cast/net/pacing/paced_sender_unittest.cc b/chromium/media/cast/net/pacing/paced_sender_unittest.cc index 37c7ba8761d..67c829abbf6 100644 --- a/chromium/media/cast/net/pacing/paced_sender_unittest.cc +++ b/chromium/media/cast/net/pacing/paced_sender_unittest.cc @@ -42,7 +42,7 @@ class TestPacketSender : public PacketTransport { public: TestPacketSender() : bytes_sent_(0) {} - bool SendPacket(PacketRef packet, const base::Closure& cb) final { + bool SendPacket(PacketRef packet, base::OnceClosure cb) final { EXPECT_FALSE(expected_packet_sizes_.empty()); size_t expected_packet_size = expected_packet_sizes_.front(); expected_packet_sizes_.pop_front(); @@ -184,16 +184,14 @@ TEST_F(PacedSenderTest, PassThroughRtcp) { mock_transport_.AddExpectedSizesAndPacketIds(kSize2, kRtcpPacketIdMagic, 1); Packet tmp(kSize2, kValue); - EXPECT_TRUE(paced_sender_->SendRtcpPacket( - 1, - new base::RefCountedData<Packet>(tmp))); + EXPECT_TRUE( + paced_sender_->SendRtcpPacket(1, new base::RefCountedData<Packet>(tmp))); } TEST_F(PacedSenderTest, BasicPace) { int num_of_packets = 27; - SendPacketVector packets = CreateSendPacketVector(kSize1, - num_of_packets, - false); + SendPacketVector packets = + CreateSendPacketVector(kSize1, num_of_packets, false); const base::TimeTicks earliest_event_timestamp = testing_clock_.NowTicks(); mock_transport_.AddExpectedSizesAndPacketIds(kSize1, UINT16_C(0), 10); @@ -294,7 +292,7 @@ TEST_F(PacedSenderTest, PaceWithNack) { int expected_video_network_event_count = num_of_packets_in_frame; int expected_video_retransmitted_event_count = 2 * num_of_packets_in_nack; - expected_video_retransmitted_event_count -= 2; // 2 packets deduped + expected_video_retransmitted_event_count -= 2; // 2 packets deduped int expected_audio_network_event_count = num_of_packets_in_frame; EXPECT_EQ(expected_video_network_event_count + expected_video_retransmitted_event_count + @@ -412,26 +410,24 @@ TEST_F(PacedSenderTest, SendPriority) { paced_sender_->RegisterPrioritySsrc(kAudioSsrc); // Retransmission packets with the earlier timestamp. - SendPacketVector resend_packets = - CreateSendPacketVector(kSize4, 10, false); + SendPacketVector resend_packets = CreateSendPacketVector(kSize4, 10, false); testing_clock_.Advance(base::TimeDelta::FromMilliseconds(10)); // Send 20 normal video packets. Only 10 will be sent in this // call, the rest will be sitting in the queue waiting for pacing. - EXPECT_TRUE(paced_sender_->SendPackets( - CreateSendPacketVector(kSize2, 20, false))); + EXPECT_TRUE( + paced_sender_->SendPackets(CreateSendPacketVector(kSize2, 20, false))); testing_clock_.Advance(base::TimeDelta::FromMilliseconds(10)); // Send normal audio packet. This is queued and will be sent // earlier than video packets. - EXPECT_TRUE(paced_sender_->SendPackets( - CreateSendPacketVector(kSize1, 1, true))); + EXPECT_TRUE( + paced_sender_->SendPackets(CreateSendPacketVector(kSize1, 1, true))); // Send RTCP packet. This is queued and will be sent first. EXPECT_TRUE(paced_sender_->SendRtcpPacket( - kVideoSsrc, - new base::RefCountedData<Packet>(Packet(kSize3, kValue)))); + kVideoSsrc, new base::RefCountedData<Packet>(Packet(kSize3, kValue)))); // Resend video packets. This is queued and will be sent // earlier than normal video packets. diff --git a/chromium/media/cast/net/rtcp/receiver_rtcp_event_subscriber.cc b/chromium/media/cast/net/rtcp/receiver_rtcp_event_subscriber.cc index ac4c05d589e..a888edf7166 100644 --- a/chromium/media/cast/net/rtcp/receiver_rtcp_event_subscriber.cc +++ b/chromium/media/cast/net/rtcp/receiver_rtcp_event_subscriber.cc @@ -7,6 +7,8 @@ #include <algorithm> #include <utility> +#include "base/logging.h" + namespace media { namespace cast { diff --git a/chromium/media/cast/net/rtcp/test_rtcp_packet_builder.cc b/chromium/media/cast/net/rtcp/test_rtcp_packet_builder.cc index dd03612346f..8c00d8e9b15 100644 --- a/chromium/media/cast/net/rtcp/test_rtcp_packet_builder.cc +++ b/chromium/media/cast/net/rtcp/test_rtcp_packet_builder.cc @@ -4,7 +4,7 @@ #include "media/cast/net/rtcp/test_rtcp_packet_builder.h" -#include "base/logging.h" +#include "base/check_op.h" #include "media/cast/net/rtcp/rtcp_utility.h" namespace media { diff --git a/chromium/media/cast/net/rtp/frame_buffer.cc b/chromium/media/cast/net/rtp/frame_buffer.cc index 1c345799e2e..255b5067157 100644 --- a/chromium/media/cast/net/rtp/frame_buffer.cc +++ b/chromium/media/cast/net/rtp/frame_buffer.cc @@ -4,7 +4,7 @@ #include "media/cast/net/rtp/frame_buffer.h" -#include "base/logging.h" +#include "base/check_op.h" namespace media { namespace cast { diff --git a/chromium/media/cast/net/rtp/packet_storage.cc b/chromium/media/cast/net/rtp/packet_storage.cc index 53c0d358191..15bb977779f 100644 --- a/chromium/media/cast/net/rtp/packet_storage.cc +++ b/chromium/media/cast/net/rtp/packet_storage.cc @@ -4,7 +4,8 @@ #include "media/cast/net/rtp/packet_storage.h" -#include "base/logging.h" +#include "base/check_op.h" +#include "base/notreached.h" #include "media/cast/constants.h" namespace media { diff --git a/chromium/media/cast/net/rtp/receiver_stats.cc b/chromium/media/cast/net/rtp/receiver_stats.cc index 5c7b4295d9c..c9d3f68c203 100644 --- a/chromium/media/cast/net/rtp/receiver_stats.cc +++ b/chromium/media/cast/net/rtp/receiver_stats.cc @@ -4,7 +4,6 @@ #include "media/cast/net/rtp/receiver_stats.h" -#include "base/logging.h" #include "media/cast/net/rtp/rtp_defines.h" namespace media { diff --git a/chromium/media/cast/net/rtp/rtp_packet_builder.cc b/chromium/media/cast/net/rtp/rtp_packet_builder.cc index dbeeedeb7a6..c49c7189db7 100644 --- a/chromium/media/cast/net/rtp/rtp_packet_builder.cc +++ b/chromium/media/cast/net/rtp/rtp_packet_builder.cc @@ -5,7 +5,7 @@ #include "media/cast/net/rtp/rtp_packet_builder.h" #include "base/big_endian.h" -#include "base/logging.h" +#include "base/check_op.h" namespace media { namespace cast { diff --git a/chromium/media/cast/net/rtp/rtp_packetizer.cc b/chromium/media/cast/net/rtp/rtp_packetizer.cc index 403de59a809..f4778415c01 100644 --- a/chromium/media/cast/net/rtp/rtp_packetizer.cc +++ b/chromium/media/cast/net/rtp/rtp_packetizer.cc @@ -7,7 +7,7 @@ #include <string> #include "base/big_endian.h" -#include "base/logging.h" +#include "base/check_op.h" #include "media/cast/net/pacing/paced_sender.h" #include "media/cast/net/rtp/rtp_defines.h" diff --git a/chromium/media/cast/net/rtp/rtp_packetizer_unittest.cc b/chromium/media/cast/net/rtp/rtp_packetizer_unittest.cc index 9b3e50685f0..84dab88d15a 100644 --- a/chromium/media/cast/net/rtp/rtp_packetizer_unittest.cc +++ b/chromium/media/cast/net/rtp/rtp_packetizer_unittest.cc @@ -27,7 +27,7 @@ static const uint16_t kSeqNum = 33; static const int kMaxPacketLength = 1500; static const int kSsrc = 0x12345; static const unsigned int kFrameSize = 5000; -} +} // namespace class TestRtpPacketTransport : public PacketTransport { public: @@ -65,7 +65,7 @@ class TestRtpPacketTransport : public PacketTransport { } } - bool SendPacket(PacketRef packet, const base::Closure& cb) final { + bool SendPacket(PacketRef packet, base::OnceClosure cb) final { ++packets_sent_; RtpParser parser(kSsrc, kPayload); RtpCastHeader rtp_header; @@ -122,8 +122,8 @@ class RtpPacketizerTest : public ::testing::Test { &testing_clock_, nullptr, transport_.get(), task_runner_)); pacer_->RegisterSsrc(config_.ssrc, false); - rtp_packetizer_.reset(new RtpPacketizer( - pacer_.get(), &packet_storage_, config_)); + rtp_packetizer_.reset( + new RtpPacketizer(pacer_.get(), &packet_storage_, config_)); video_frame_.dependency = EncodedFrame::DEPENDENT; video_frame_.frame_id = FrameId::first() + 1; video_frame_.referenced_frame_id = video_frame_.frame_id - 1; diff --git a/chromium/media/cast/net/rtp/rtp_parser.cc b/chromium/media/cast/net/rtp/rtp_parser.cc index 4d85bae8e18..f9ec4374d11 100644 --- a/chromium/media/cast/net/rtp/rtp_parser.cc +++ b/chromium/media/cast/net/rtp/rtp_parser.cc @@ -5,7 +5,7 @@ #include "media/cast/net/rtp/rtp_parser.h" #include "base/big_endian.h" -#include "base/logging.h" +#include "base/check.h" #include "media/cast/constants.h" #include "media/cast/net/rtp/rtp_defines.h" diff --git a/chromium/media/cast/net/udp_transport_impl.cc b/chromium/media/cast/net/udp_transport_impl.cc index 6b9993d3fa6..9c6fd7d248c 100644 --- a/chromium/media/cast/net/udp_transport_impl.cc +++ b/chromium/media/cast/net/udp_transport_impl.cc @@ -220,8 +220,7 @@ void UdpTransportImpl::ReceiveNextPacket(int length_or_status) { } } -bool UdpTransportImpl::SendPacket(PacketRef packet, - const base::RepeatingClosure& cb) { +bool UdpTransportImpl::SendPacket(PacketRef packet, base::OnceClosure cb) { DCHECK(io_thread_proxy_->RunsTasksInCurrentSequence()); if (!udp_socket_) return true; @@ -252,8 +251,9 @@ bool UdpTransportImpl::SendPacket(PacketRef packet, reinterpret_cast<char*>(&packet->data.front())); int result; - base::RepeatingCallback<void(int)> callback = base::BindRepeating( - &UdpTransportImpl::OnSent, weak_factory_.GetWeakPtr(), buf, packet, cb); + net::CompletionOnceCallback callback = + base::BindOnce(&UdpTransportImpl::OnSent, weak_factory_.GetWeakPtr(), buf, + packet, std::move(cb)); if (client_connected_) { // If we called Connect() before we must call Write() instead of // SendTo(). Otherwise on some platforms we might get @@ -288,11 +288,11 @@ bool UdpTransportImpl::SendPacket(PacketRef packet, result = udp_socket_->Write(buf.get(), static_cast<int>(packet->data.size()), - callback, traffic_annotation); + std::move(callback), traffic_annotation); } else if (!IsEmpty(remote_addr_)) { result = udp_socket_->SendTo(buf.get(), static_cast<int>(packet->data.size()), - remote_addr_, callback); + remote_addr_, std::move(callback)); } else { VLOG(1) << "Failed to send packet; socket is neither bound nor " << "connected."; @@ -313,7 +313,7 @@ int64_t UdpTransportImpl::GetBytesSent() { void UdpTransportImpl::OnSent(const scoped_refptr<net::IOBuffer>& buf, PacketRef packet, - const base::RepeatingClosure& cb, + base::OnceClosure cb, int result) { DCHECK(io_thread_proxy_->RunsTasksInCurrentSequence()); @@ -324,7 +324,7 @@ void UdpTransportImpl::OnSent(const scoped_refptr<net::IOBuffer>& buf, ScheduleReceiveNextPacket(); if (!cb.is_null()) { - cb.Run(); + std::move(cb).Run(); } } diff --git a/chromium/media/cast/net/udp_transport_impl.h b/chromium/media/cast/net/udp_transport_impl.h index d6a6864a0a3..0b60a35bded 100644 --- a/chromium/media/cast/net/udp_transport_impl.h +++ b/chromium/media/cast/net/udp_transport_impl.h @@ -49,7 +49,7 @@ class UdpTransportImpl final : public PacketTransport, public UdpTransport { ~UdpTransportImpl() final; // PacketTransport implementations. - bool SendPacket(PacketRef packet, const base::RepeatingClosure& cb) final; + bool SendPacket(PacketRef packet, base::OnceClosure cb) final; int64_t GetBytesSent() final; // Start receiving packets. Packets are submitted to |packet_receiver|. void StartReceiving(PacketReceiverCallbackWithStatus packet_receiver) final; @@ -97,7 +97,7 @@ class UdpTransportImpl final : public PacketTransport, public UdpTransport { void OnSent(const scoped_refptr<net::IOBuffer>& buf, PacketRef packet, - const base::RepeatingClosure& cb, + base::OnceClosure cb, int result); // Called by |reader_| when it completes reading a packet from the data pipe. diff --git a/chromium/media/cast/sender/audio_sender.cc b/chromium/media/cast/sender/audio_sender.cc index 7e60bcbef9c..b4b379059a3 100644 --- a/chromium/media/cast/sender/audio_sender.cc +++ b/chromium/media/cast/sender/audio_sender.cc @@ -7,7 +7,8 @@ #include <utility> #include "base/bind.h" -#include "base/logging.h" +#include "base/check_op.h" +#include "base/notreached.h" #include "media/cast/common/rtp_time.h" #include "media/cast/net/cast_transport_config.h" #include "media/cast/sender/audio_encoder.h" @@ -17,7 +18,7 @@ namespace cast { AudioSender::AudioSender(scoped_refptr<CastEnvironment> cast_environment, const FrameSenderConfig& audio_config, - StatusChangeCallback status_change_cb, + StatusChangeOnceCallback status_change_cb, CastTransport* const transport_sender) : FrameSender(cast_environment, transport_sender, diff --git a/chromium/media/cast/sender/audio_sender.h b/chromium/media/cast/sender/audio_sender.h index 124cb5afb65..ee3b6b5207c 100644 --- a/chromium/media/cast/sender/audio_sender.h +++ b/chromium/media/cast/sender/audio_sender.h @@ -33,7 +33,7 @@ class AudioSender : public FrameSender { public: AudioSender(scoped_refptr<CastEnvironment> cast_environment, const FrameSenderConfig& audio_config, - StatusChangeCallback status_change_cb, + StatusChangeOnceCallback status_change_cb, CastTransport* const transport_sender); ~AudioSender() final; diff --git a/chromium/media/cast/sender/audio_sender_unittest.cc b/chromium/media/cast/sender/audio_sender_unittest.cc index e5440ccdf83..10551754218 100644 --- a/chromium/media/cast/sender/audio_sender_unittest.cc +++ b/chromium/media/cast/sender/audio_sender_unittest.cc @@ -58,7 +58,7 @@ class TestPacketSender : public PacketTransport { public: TestPacketSender() : number_of_rtp_packets_(0), number_of_rtcp_packets_(0) {} - bool SendPacket(PacketRef packet, const base::Closure& cb) final { + bool SendPacket(PacketRef packet, base::OnceClosure cb) final { if (IsRtcpPacket(&packet->data[0], packet->data.size())) { ++number_of_rtcp_packets_; } else { @@ -111,9 +111,8 @@ class AudioSenderTest : public ::testing::Test { base::WrapUnique(transport_), task_runner_)); OperationalStatus operational_status = STATUS_UNINITIALIZED; audio_sender_.reset(new AudioSender( - cast_environment_, - audio_config_, - base::Bind(&SaveOperationalStatus, &operational_status), + cast_environment_, audio_config_, + base::BindOnce(&SaveOperationalStatus, &operational_status), transport_sender_.get())); task_runner_->RunTasks(); CHECK_EQ(STATUS_INITIALIZED, operational_status); @@ -122,7 +121,7 @@ class AudioSenderTest : public ::testing::Test { ~AudioSenderTest() override = default; base::SimpleTestTickClock testing_clock_; - TestPacketSender* transport_; // Owned by CastTransport. + TestPacketSender* transport_; // Owned by CastTransport. std::unique_ptr<CastTransportImpl> transport_sender_; scoped_refptr<FakeSingleThreadTaskRunner> task_runner_; std::unique_ptr<AudioSender> audio_sender_; diff --git a/chromium/media/cast/sender/video_encoder_impl.cc b/chromium/media/cast/sender/video_encoder_impl.cc index 50e74ffa685..4fc74e74b5d 100644 --- a/chromium/media/cast/sender/video_encoder_impl.cc +++ b/chromium/media/cast/sender/video_encoder_impl.cc @@ -9,7 +9,7 @@ #include "base/bind.h" #include "base/bind_helpers.h" #include "base/callback.h" -#include "base/logging.h" +#include "base/check.h" #include "media/base/video_frame.h" #include "media/cast/sender/fake_software_video_encoder.h" #include "media/cast/sender/vp8_encoder.h" diff --git a/chromium/media/cast/sender/video_sender_unittest.cc b/chromium/media/cast/sender/video_sender_unittest.cc index 5688b319cf8..e97396f68d0 100644 --- a/chromium/media/cast/sender/video_sender_unittest.cc +++ b/chromium/media/cast/sender/video_sender_unittest.cc @@ -41,7 +41,6 @@ static const int kHeight = 240; using testing::_; using testing::AtLeast; - void SaveOperationalStatus(OperationalStatus* out_status, OperationalStatus in_status) { DVLOG(1) << "OperationalStatus transitioning from " << *out_status << " to " @@ -52,15 +51,13 @@ void SaveOperationalStatus(OperationalStatus* out_status, class TestPacketSender : public PacketTransport { public: TestPacketSender() - : number_of_rtp_packets_(0), - number_of_rtcp_packets_(0), - paused_(false) {} + : number_of_rtp_packets_(0), number_of_rtcp_packets_(0), paused_(false) {} // A singular packet implies a RTCP packet. - bool SendPacket(PacketRef packet, const base::Closure& cb) final { + bool SendPacket(PacketRef packet, base::OnceClosure cb) final { if (paused_) { stored_packet_ = packet; - callback_ = cb; + callback_ = std::move(cb); return false; } if (IsRtcpPacket(&packet->data[0], packet->data.size())) { @@ -90,8 +87,8 @@ class TestPacketSender : public PacketTransport { void SetPause(bool paused) { paused_ = paused; if (!paused && stored_packet_.get()) { - SendPacket(stored_packet_, callback_); - callback_.Run(); + SendPacket(stored_packet_, base::OnceClosure()); + std::move(callback_).Run(); } } @@ -99,14 +96,13 @@ class TestPacketSender : public PacketTransport { int number_of_rtp_packets_; int number_of_rtcp_packets_; bool paused_; - base::Closure callback_; + base::OnceClosure callback_; PacketRef stored_packet_; DISALLOW_COPY_AND_ASSIGN(TestPacketSender); }; -void IgnorePlayoutDelayChanges(base::TimeDelta unused_playout_delay) { -} +void IgnorePlayoutDelayChanges(base::TimeDelta unused_playout_delay) {} class PeerVideoSender : public VideoSender { public: diff --git a/chromium/media/cdm/BUILD.gn b/chromium/media/cdm/BUILD.gn index 5ecddbe144e..806ce494dd1 100644 --- a/chromium/media/cdm/BUILD.gn +++ b/chromium/media/cdm/BUILD.gn @@ -78,14 +78,6 @@ source_set("cdm") { "cdm_helpers.h", "cdm_module.cc", "cdm_module.h", - - # Not putting these under |enable_cdm_proxy| so we don't have to if/def - # every usage of CdmProxy base types, e.g. in CdmCapability. This also - # helps enable more tests. - "cdm_proxy.cc", - "cdm_proxy.h", - "cdm_proxy_context.cc", - "cdm_proxy_context.h", "cdm_type_conversion.cc", "cdm_type_conversion.h", "cdm_wrapper.h", diff --git a/chromium/media/cdm/aes_decryptor.h b/chromium/media/cdm/aes_decryptor.h index f96e07a88a2..640e0ad1bf0 100644 --- a/chromium/media/cdm/aes_decryptor.h +++ b/chromium/media/cdm/aes_decryptor.h @@ -88,7 +88,6 @@ class MEDIA_EXPORT AesDecryptor : public ContentDecryptionModule, private: // Testing classes that needs to manipulate internal states for testing. friend class ClearKeyPersistentSessionCdm; - friend class ClearKeyCdmProxy; // Internally this class supports persistent license type sessions so that // it can be used by ClearKeyPersistentSessionCdm. The following methods diff --git a/chromium/media/cdm/aes_decryptor_unittest.cc b/chromium/media/cdm/aes_decryptor_unittest.cc index 1ff11c89d1a..0a812f3c310 100644 --- a/chromium/media/cdm/aes_decryptor_unittest.cc +++ b/chromium/media/cdm/aes_decryptor_unittest.cc @@ -59,9 +59,8 @@ MATCHER(NotEmpty, "") { } MATCHER(IsJSONDictionary, "") { std::string result(arg.begin(), arg.end()); - std::unique_ptr<base::Value> root( - base::JSONReader().ReadToValueDeprecated(result)); - return (root.get() && root->type() == base::Value::Type::DICTIONARY); + base::Optional<base::Value> root = base::JSONReader::Read(result); + return (root && root->type() == base::Value::Type::DICTIONARY); } MATCHER(IsNullTime, "") { return arg.is_null(); diff --git a/chromium/media/cdm/api/content_decryption_module.h b/chromium/media/cdm/api/content_decryption_module.h index 7cf9d3d8eae..0e7301230b9 100644 --- a/chromium/media/cdm/api/content_decryption_module.h +++ b/chromium/media/cdm/api/content_decryption_module.h @@ -8,7 +8,6 @@ #include <type_traits> #include "content_decryption_module_export.h" -#include "content_decryption_module_proxy.h" #if defined(_MSC_VER) typedef unsigned char uint8_t; @@ -1348,19 +1347,6 @@ class CDM_CLASS_API Host_11 { // CDM can call this method multiple times to operate on different files. virtual FileIO* CreateFileIO(FileIOClient* client) = 0; - // Requests a CdmProxy that proxies part of CDM functionalities to a different - // entity, e.g. a hardware CDM module. A CDM instance can have at most one - // CdmProxy throughout its lifetime, which must be requested and initialized - // during CDM instance initialization time, i.e. in or after CDM::Initialize() - // and before OnInitialized() is called, to ensure proper connection of the - // CdmProxy and the media player (e.g. hardware decoder). The CdmProxy is - // owned by the host and is guaranteed to be valid throughout the CDM - // instance's lifetime. The CDM must ensure that the |client| remain valid - // before the CDM instance is destroyed. Returns null if CdmProxy is not - // supported, called before CDM::Initialize(), RequestCdmProxy() is called - // more than once, or called after the CDM instance has been initialized. - virtual CdmProxy* RequestCdmProxy(CdmProxyClient* client) = 0; - // Requests a specific version of the storage ID. A storage ID is a stable, // device specific ID used by the CDM to securely store persistent data. The // ID will be returned by the host via ContentDecryptionModule::OnStorageId(). diff --git a/chromium/media/cdm/api/content_decryption_module_proxy.h b/chromium/media/cdm/api/content_decryption_module_proxy.h deleted file mode 100644 index 374f54a2a02..00000000000 --- a/chromium/media/cdm/api/content_decryption_module_proxy.h +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef CDM_CONTENT_DECRYPTION_MODULE_PROXY_H_ -#define CDM_CONTENT_DECRYPTION_MODULE_PROXY_H_ - -#include "content_decryption_module_export.h" - -#if defined(_MSC_VER) -typedef unsigned char uint8_t; -typedef unsigned int uint32_t; -typedef unsigned __int64 uint64_t; -#else -#include <stdint.h> -#endif - -namespace cdm { - -class CDM_CLASS_API CdmProxyClient; - -// A proxy class for the CDM. -// In general, the interpretation of the CdmProxy and CdmProxyClient method -// parameters are protocol dependent. For enum parameters, values outside the -// enum range may not work. -class CDM_CLASS_API CdmProxy { - public: - enum Function : uint32_t { - // For Intel Negotiate Crypto SessionKey Exchange (CSME) path to call - // ID3D11VideoContext::NegotiateCryptoSessionKeyExchange. - kIntelNegotiateCryptoSessionKeyExchange = 1, - // There will be more values in the future e.g. for D3D11 RSA method. - }; - - enum KeyType : uint32_t { - kDecryptOnly = 0, - kDecryptAndDecode = 1, - }; - - // Initializes the proxy. The results will be returned in - // CdmProxyClient::OnInitialized(). - virtual void Initialize() = 0; - - // Processes and updates the state of the proxy. - // |output_data_size| is required by some protocol to set up the output data. - // The operation may fail if the |output_data_size| is wrong. The results will - // be returned in CdmProxyClient::OnProcessed(). - virtual void Process(Function function, - uint32_t crypto_session_id, - const uint8_t* input_data, - uint32_t input_data_size, - uint32_t output_data_size) = 0; - - // Creates a crypto session for handling media. - // If extra data has to be passed to further setup the media crypto session, - // pass the data as |input_data|. The results will be returned in - // CdmProxyClient::OnMediaCryptoSessionCreated(). - virtual void CreateMediaCryptoSession(const uint8_t* input_data, - uint32_t input_data_size) = 0; - - // Sets a key for the session identified by |crypto_session_id|. - virtual void SetKey(uint32_t crypto_session_id, - const uint8_t* key_id, - uint32_t key_id_size, - KeyType key_type, - const uint8_t* key_blob, - uint32_t key_blob_size) = 0; - - // Removes a key for the session identified by |crypto_session_id|. - virtual void RemoveKey(uint32_t crypto_session_id, - const uint8_t* key_id, - uint32_t key_id_size) = 0; - - protected: - CdmProxy() {} - virtual ~CdmProxy() {} -}; - -// Responses to CdmProxy calls. All responses will be called asynchronously. -class CDM_CLASS_API CdmProxyClient { - public: - enum Status : uint32_t { - kOk, - kFail, - }; - - enum Protocol : uint32_t { - kNone = 0, // No protocol supported. Can be used in failure cases. - kIntel, // Method using Intel CSME. - // There will be more values in the future e.g. kD3D11RsaHardware, - // kD3D11RsaSoftware to use the D3D11 RSA method. - }; - - // Callback for Initialize(). If the proxy created a crypto session, then the - // ID for the crypto session is |crypto_session_id|. - virtual void OnInitialized(Status status, - Protocol protocol, - uint32_t crypto_session_id) = 0; - - // Callback for Process(). |output_data| is the output of processing. - virtual void OnProcessed(Status status, - const uint8_t* output_data, - uint32_t output_data_size) = 0; - - // Callback for CreateMediaCryptoSession(). On success: - // - |crypto_session_id| is the ID for the created crypto session. - // - |output_data| is extra value, if any. - // Otherwise, |crypto_session_id| and |output_data| should be ignored. - virtual void OnMediaCryptoSessionCreated(Status status, - uint32_t crypto_session_id, - uint64_t output_data) = 0; - - // Callback for SetKey(). - virtual void OnKeySet(Status status) = 0; - - // Callback for RemoveKey(). - virtual void OnKeyRemoved(Status status) = 0; - - // Called when there is a hardware reset and all the hardware context is lost. - virtual void NotifyHardwareReset() = 0; - - protected: - CdmProxyClient() {} - virtual ~CdmProxyClient() {} -}; - -} // namespace cdm - -#endif // CDM_CONTENT_DECRYPTION_MODULE_PROXY_H_ diff --git a/chromium/media/cdm/cdm_adapter.cc b/chromium/media/cdm/cdm_adapter.cc index 5ae3f92709f..caf9974ec57 100644 --- a/chromium/media/cdm/cdm_adapter.cc +++ b/chromium/media/cdm/cdm_adapter.cc @@ -413,29 +413,12 @@ std::unique_ptr<CallbackRegistration> CdmAdapter::RegisterEventCB( Decryptor* CdmAdapter::GetDecryptor() { DCHECK(task_runner_->BelongsToCurrentThread()); - - // When using HW secure codecs, we cannot and should not use the CDM instance - // to do decrypt and/or decode. Instead, we should use the CdmProxy. - // TODO(xhwang): Fix External Clear Key key system to be able to set - // |use_hw_secure_codecs| so that we don't have to check both. - // TODO(xhwang): Update this logic to support transcryption. - if (cdm_config_.use_hw_secure_codecs || cdm_proxy_created_) { - DVLOG(2) << __func__ << ": GetDecryptor() returns null"; - return nullptr; - } - return this; } int CdmAdapter::GetCdmId() const { DCHECK(task_runner_->BelongsToCurrentThread()); -#if BUILDFLAG(ENABLE_CDM_PROXY) - int cdm_id = helper_->GetCdmProxyCdmId(); - DVLOG(2) << __func__ << ": cdm_id = " << cdm_id; - return cdm_id; -#else return CdmContext::kInvalidCdmId; -#endif // BUILDFLAG(ENABLE_CDM_PROXY) } void CdmAdapter::RegisterNewKeyCB(StreamType stream_type, @@ -1067,27 +1050,6 @@ void CdmAdapter::RequestStorageId(uint32_t version) { weak_factory_.GetWeakPtr())); } -cdm::CdmProxy* CdmAdapter::RequestCdmProxy(cdm::CdmProxyClient* client) { - DVLOG(3) << __func__; - DCHECK(task_runner_->BelongsToCurrentThread()); - -#if BUILDFLAG(ENABLE_CDM_PROXY) - // CdmProxy should only be created once, at CDM initialization time. - if (cdm_proxy_created_ || - init_promise_id_ == CdmPromiseAdapter::kInvalidPromiseId) { - DVLOG(1) << __func__ - << ": CdmProxy can only be created once, and must be created " - "during CDM initialization."; - return nullptr; - } - - cdm_proxy_created_ = true; - return helper_->CreateCdmProxy(client); -#else - return nullptr; -#endif // BUILDFLAG(ENABLE_CDM_PROXY) -} - void CdmAdapter::OnStorageIdObtained(uint32_t version, const std::vector<uint8_t>& storage_id) { DVLOG(2) << __func__ << ": version = " << version; diff --git a/chromium/media/cdm/cdm_adapter.h b/chromium/media/cdm/cdm_adapter.h index b2ecdaae327..2b87592bb62 100644 --- a/chromium/media/cdm/cdm_adapter.h +++ b/chromium/media/cdm/cdm_adapter.h @@ -155,9 +155,6 @@ class MEDIA_EXPORT CdmAdapter : public ContentDecryptionModule, cdm::FileIO* CreateFileIO(cdm::FileIOClient* client) override; void RequestStorageId(uint32_t version) override; - // cdm::Host_11 specific implementation. - cdm::CdmProxy* RequestCdmProxy(cdm::CdmProxyClient* client) override; - private: CdmAdapter(const std::string& key_system, const url::Origin& security_origin, @@ -259,8 +256,6 @@ class MEDIA_EXPORT CdmAdapter : public ContentDecryptionModule, int last_read_file_size_kb_ = 0; bool file_size_uma_reported_ = false; - bool cdm_proxy_created_ = false; - // Used to keep track of promises while the CDM is processing the request. CdmPromiseAdapter cdm_promise_adapter_; diff --git a/chromium/media/cdm/cdm_adapter_unittest.cc b/chromium/media/cdm/cdm_adapter_unittest.cc index 193ac302e3e..c8d9ad159b1 100644 --- a/chromium/media/cdm/cdm_adapter_unittest.cc +++ b/chromium/media/cdm/cdm_adapter_unittest.cc @@ -8,8 +8,8 @@ #include <memory> #include "base/bind.h" +#include "base/check.h" #include "base/command_line.h" -#include "base/logging.h" #include "base/run_loop.h" #include "base/stl_util.h" #include "base/strings/string_number_conversions.h" @@ -558,13 +558,4 @@ TEST_P(CdmAdapterTestWithMockCdm, GetDecryptor) { EXPECT_TRUE(cdm_context->GetDecryptor()); } -TEST_P(CdmAdapterTestWithMockCdm, GetDecryptor_UseHwSecureCodecs) { - CdmConfig cdm_config; - cdm_config.use_hw_secure_codecs = true; - InitializeWithCdmConfig(cdm_config); - auto* cdm_context = cdm_->GetCdmContext(); - ASSERT_TRUE(cdm_context); - EXPECT_FALSE(cdm_context->GetDecryptor()); -} - } // namespace media diff --git a/chromium/media/cdm/cdm_auxiliary_helper.cc b/chromium/media/cdm/cdm_auxiliary_helper.cc index c87d9a93d08..4d163e112aa 100644 --- a/chromium/media/cdm/cdm_auxiliary_helper.cc +++ b/chromium/media/cdm/cdm_auxiliary_helper.cc @@ -18,16 +18,6 @@ cdm::FileIO* CdmAuxiliaryHelper::CreateCdmFileIO(cdm::FileIOClient* client) { return nullptr; } -#if BUILDFLAG(ENABLE_CDM_PROXY) -cdm::CdmProxy* CdmAuxiliaryHelper::CreateCdmProxy(cdm::CdmProxyClient* client) { - return nullptr; -} - -int CdmAuxiliaryHelper::GetCdmProxyCdmId() { - return CdmContext::kInvalidCdmId; -} -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - cdm::Buffer* CdmAuxiliaryHelper::CreateCdmBuffer(size_t capacity) { return nullptr; } diff --git a/chromium/media/cdm/cdm_auxiliary_helper.h b/chromium/media/cdm/cdm_auxiliary_helper.h index 2ecf88765f8..b6c829351a1 100644 --- a/chromium/media/cdm/cdm_auxiliary_helper.h +++ b/chromium/media/cdm/cdm_auxiliary_helper.h @@ -21,8 +21,6 @@ namespace cdm { class FileIO; class FileIOClient; -class CdmProxy; -class CdmProxyClient; } // namespace cdm namespace media { @@ -48,18 +46,6 @@ class MEDIA_EXPORT CdmAuxiliaryHelper : public CdmAllocator, // needed anymore. virtual cdm::FileIO* CreateCdmFileIO(cdm::FileIOClient* client); -#if BUILDFLAG(ENABLE_CDM_PROXY) - // Creates a cdm::CdmProxy object and returns it. - // The caller does not own the returned object and should not delete it - // directly. Instead, it should call cdm::CdmProxy::Destroy() after it's not - // needed anymore. - virtual cdm::CdmProxy* CreateCdmProxy(cdm::CdmProxyClient* client); - - // Returns a CDM ID associated with the last returned CdmProxy. Should only - // be called after the CdmProxy has been initialized. - virtual int GetCdmProxyCdmId(); -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - // CdmAllocator implementation. cdm::Buffer* CreateCdmBuffer(size_t capacity) override; std::unique_ptr<VideoFrameImpl> CreateCdmVideoFrame() override; diff --git a/chromium/media/cdm/cdm_context_ref_impl.cc b/chromium/media/cdm/cdm_context_ref_impl.cc index 32df59ade60..7dee07ef0c2 100644 --- a/chromium/media/cdm/cdm_context_ref_impl.cc +++ b/chromium/media/cdm/cdm_context_ref_impl.cc @@ -4,7 +4,7 @@ #include "media/cdm/cdm_context_ref_impl.h" -#include "base/logging.h" +#include "base/check_op.h" #include "media/base/content_decryption_module.h" namespace media { diff --git a/chromium/media/cdm/cdm_helpers.cc b/chromium/media/cdm/cdm_helpers.cc index eb98759559a..5a15d074d20 100644 --- a/chromium/media/cdm/cdm_helpers.cc +++ b/chromium/media/cdm/cdm_helpers.cc @@ -4,7 +4,7 @@ #include "media/cdm/cdm_helpers.h" -#include "base/logging.h" +#include "base/check.h" #include "ui/gfx/color_space.h" namespace media { diff --git a/chromium/media/cdm/cdm_module.cc b/chromium/media/cdm/cdm_module.cc index 1750e423820..6f451fcfad7 100644 --- a/chromium/media/cdm/cdm_module.cc +++ b/chromium/media/cdm/cdm_module.cc @@ -5,6 +5,7 @@ #include "media/cdm/cdm_module.h" #include "base/files/file_util.h" +#include "base/logging.h" #include "base/memory/ptr_util.h" #include "base/metrics/histogram_functions.h" #include "base/metrics/histogram_macros.h" diff --git a/chromium/media/cdm/cdm_module.h b/chromium/media/cdm/cdm_module.h index 894c18c7ae3..3e8ed8068b7 100644 --- a/chromium/media/cdm/cdm_module.h +++ b/chromium/media/cdm/cdm_module.h @@ -31,11 +31,7 @@ class MEDIA_EXPORT CdmModule { ~CdmModule(); - using CreateCdmFunc = void* (*)(int cdm_interface_version, - const char* key_system, - uint32_t key_system_size, - GetCdmHostFunc get_cdm_host_func, - void* user_data); + using CreateCdmFunc = decltype(&::CreateCdmInstance); CreateCdmFunc GetCreateCdmFunc(); @@ -57,9 +53,9 @@ class MEDIA_EXPORT CdmModule { bool was_initialize_called() const { return was_initialize_called_; } private: - using InitializeCdmModuleFunc = void (*)(); - using DeinitializeCdmModuleFunc = void (*)(); - using GetCdmVersionFunc = char* (*)(); + using InitializeCdmModuleFunc = decltype(&::INITIALIZE_CDM_MODULE); + using DeinitializeCdmModuleFunc = decltype(&::DeinitializeCdmModule); + using GetCdmVersionFunc = decltype(&::GetCdmVersion); CdmModule(); diff --git a/chromium/media/cdm/cdm_proxy.cc b/chromium/media/cdm/cdm_proxy.cc deleted file mode 100644 index 31063510da3..00000000000 --- a/chromium/media/cdm/cdm_proxy.cc +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/cdm/cdm_proxy.h" - -namespace media { - -CdmProxy::Client::Client() = default; -CdmProxy::Client::~Client() = default; -CdmProxy::CdmProxy() = default; -CdmProxy::~CdmProxy() = default; - -} // namespace media diff --git a/chromium/media/cdm/cdm_proxy.h b/chromium/media/cdm/cdm_proxy.h deleted file mode 100644 index 6d908e09f71..00000000000 --- a/chromium/media/cdm/cdm_proxy.h +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MEDIA_CDM_CDM_PROXY_H_ -#define MEDIA_CDM_CDM_PROXY_H_ - -#include <stdint.h> - -#include <memory> -#include <string> -#include <vector> - -#include "base/callback.h" -#include "base/memory/weak_ptr.h" -#include "media/base/cdm_context.h" -#include "media/base/media_export.h" - -namespace base { -class Token; -} - -namespace media { - -// A class that proxies part of ContentDecryptionModule (CDM) functionalities to -// a different entity, e.g. hardware CDM modules. -// In general, the interpretation of the method and callback parameters are -// protocol dependent. For enum parameters, values outside the enum range may -// not work. -class MEDIA_EXPORT CdmProxy { - public: - // Client of the proxy. - class MEDIA_EXPORT Client { - public: - Client(); - virtual ~Client(); - - // Called when there is a hardware reset. When hardware reset happens, all - // the hardware context is lost and all crypto sessions are destroyed. The - // CdmProxy returns to an uninitialized state and the caller must call - // Initialize() on the CdmProxy again to be able to continue using it. - virtual void NotifyHardwareReset() = 0; - }; - - enum class Status { - kOk, - kFail, - kMaxValue = kFail, - }; - - enum class Protocol { - // No supported protocol. Used in failure cases. - kNone, - // Method using Intel CSME. - kIntel, - // There will be more values in the future e.g. kD3D11RsaHardware, - // kD3D11RsaSoftware to use the D3D11 RSA method. - kMaxValue = kIntel, - }; - - enum class Function { - // For Intel CSME path to call - // ID3D11VideoContext::NegotiateCryptoSessionKeyExchange. - kIntelNegotiateCryptoSessionKeyExchange, - // There will be more values in the future e.g. for D3D11 RSA method. - kMaxValue = kIntelNegotiateCryptoSessionKeyExchange, - }; - - enum class KeyType { - kDecryptOnly, - kDecryptAndDecode, - kMaxValue = kDecryptAndDecode, - }; - - CdmProxy(); - virtual ~CdmProxy(); - - // Returns a weak pointer of the CdmContext associated with |this|. - // The weak pointer will be null if |this| is destroyed. - virtual base::WeakPtr<CdmContext> GetCdmContext() = 0; - - // Callback for Initialize(). If the proxy created a crypto session, then the - // ID for the crypto session is |crypto_session_id|. - using InitializeCB = base::OnceCallback< - void(Status status, Protocol protocol, uint32_t crypto_session_id)>; - - // Initializes the proxy. The status and the return values of the call is - // reported to |init_cb|. All other methods should only be called after the - // proxy is fully initialized. Otherwise they may fail. - // Note: The proxy also needs to be reinitialized after hardware reset. See - // Client::NotifyHardwareReset() for details. - virtual void Initialize(Client* client, InitializeCB init_cb) = 0; - - // Callback for Process(). |output_data| is the output of processing. - using ProcessCB = - base::OnceCallback<void(Status status, - const std::vector<uint8_t>& output_data)>; - - // Processes and updates the state of the proxy. - // |expected_output_size| is the size of the output data passed to the - // callback. Whether this value is required or not is protocol dependent. - // The status and the return values of the call is reported to |process_cb|. - virtual void Process(Function function, - uint32_t crypto_session_id, - const std::vector<uint8_t>& input_data, - uint32_t expected_output_data_size, - ProcessCB process_cb) = 0; - - // Callback for CreateMediaCryptoSession(). - // On success: - // |crypto_session_id| is the ID for the created crypto session. - // |output_data| is extra value, if any. - using CreateMediaCryptoSessionCB = base::OnceCallback< - void(Status status, uint32_t crypto_session_id, uint64_t output_data)>; - - // Creates a crypto session for handling media. - // If extra data has to be passed to further setup the media crypto session, - // pass the data as |input_data|. - // The status and the return values of the call is reported to - // |create_media_crypto_session_cb|. - virtual void CreateMediaCryptoSession( - const std::vector<uint8_t>& input_data, - CreateMediaCryptoSessionCB create_media_crypto_session_cb) = 0; - - // Callback for SetKey(). - using SetKeyCB = base::OnceCallback<void(Status status)>; - - // Sets a key in the proxy. - // |crypto_session_id| is the crypto session for decryption. - // |key_id| is the ID of the key. - // |key_type| is the type of the key. - // |key_blob| is the opaque key blob for decrypting or decoding. - // The status of the call is reported to |set_key_cb|. - virtual void SetKey(uint32_t crypto_session_id, - const std::vector<uint8_t>& key_id, - KeyType key_type, - const std::vector<uint8_t>& key_blob, - SetKeyCB set_key_cb) = 0; - - // Callback for RemoveKey(). - using RemoveKeyCB = base::OnceCallback<void(Status status)>; - - // Removes a key from the proxy. - // |crypto_session_id| is the crypto session for decryption. - // |key_id| is the ID of the key. - // The status of the call is reported to |remove_key_cb|. - virtual void RemoveKey(uint32_t crypto_session_id, - const std::vector<uint8_t>& key_id, - RemoveKeyCB remove_key_cb) = 0; - - private: - DISALLOW_COPY_AND_ASSIGN(CdmProxy); -}; - -using CdmProxyFactoryCB = base::RepeatingCallback<std::unique_ptr<CdmProxy>( - const base::Token& cdm_guid)>; - -} // namespace media - -#endif // MEDIA_CDM_CDM_PROXY_H_ diff --git a/chromium/media/cdm/cdm_proxy_context.cc b/chromium/media/cdm/cdm_proxy_context.cc deleted file mode 100644 index 8e5f67adbec..00000000000 --- a/chromium/media/cdm/cdm_proxy_context.cc +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/cdm/cdm_proxy_context.h" - -#include "build/build_config.h" - -namespace media { - -CdmProxyContext::CdmProxyContext() {} -CdmProxyContext::~CdmProxyContext() {} - -#if defined(OS_WIN) -base::Optional<CdmProxyContext::D3D11DecryptContext> -CdmProxyContext::GetD3D11DecryptContext(CdmProxy::KeyType key_type, - const std::string& key_id) { - return base::nullopt; -} -#endif - -} // namespace media
\ No newline at end of file diff --git a/chromium/media/cdm/cdm_proxy_context.h b/chromium/media/cdm/cdm_proxy_context.h deleted file mode 100644 index a04173d4610..00000000000 --- a/chromium/media/cdm/cdm_proxy_context.h +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MEDIA_CDM_CDM_PROXY_CONTEXT_H_ -#define MEDIA_CDM_CDM_PROXY_CONTEXT_H_ - -#include <stdint.h> - -#include <string> - -#include "base/compiler_specific.h" -#include "base/macros.h" -#include "base/optional.h" -#include "build/build_config.h" -#include "media/base/media_export.h" -#include "media/cdm/cdm_proxy.h" - -#if defined(OS_WIN) -#include <d3d11.h> -#endif - -namespace media { - -// An interface for accessing various CdmProxy data required for decrypting -// and/or decoding data. -class MEDIA_EXPORT CdmProxyContext { - public: -#if defined(OS_WIN) - struct D3D11DecryptContext { - // Crypto session pointer for decryption. - // The pointer is owned by the CdmContext implementation. - ID3D11CryptoSession* crypto_session; - - // Opaque key blob for decrypting or decoding. - // The pointer is owned by the CdmContext implementation. - const void* key_blob; - - // The size of the blob. - uint32_t key_blob_size; - - // GUID identifying the hardware key info. - GUID key_info_guid; - }; - - // Returns D3D11DecryptContext on success. Returns nullopt otherwise. The - // D3D11DecryptContext instance is only guaranteed to be valid before the - // caller returns. - // |key_type| is the requesting key type. - // |key_id| is the key ID of the media to decrypt. - virtual base::Optional<D3D11DecryptContext> GetD3D11DecryptContext( - CdmProxy::KeyType key_type, - const std::string& key_id) WARN_UNUSED_RESULT; -#endif // defined(OS_WIN) - - // TODO(crbug.com/787657): There will be more methods for this class. - - protected: - CdmProxyContext(); - virtual ~CdmProxyContext(); - - DISALLOW_COPY_AND_ASSIGN(CdmProxyContext); -}; - -} // namespace media - -#endif // MEDIA_CDM_CDM_PROXY_CONTEXT_H_ diff --git a/chromium/media/cdm/cenc_utils_unittest.cc b/chromium/media/cdm/cenc_utils_unittest.cc index d068c11b473..c5cfe9d0273 100644 --- a/chromium/media/cdm/cenc_utils_unittest.cc +++ b/chromium/media/cdm/cenc_utils_unittest.cc @@ -9,7 +9,7 @@ #include <limits> -#include "base/logging.h" +#include "base/check.h" #include "base/stl_util.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/chromium/media/cdm/json_web_key.cc b/chromium/media/cdm/json_web_key.cc index 3b3ba1c9785..d09a4921eb7 100644 --- a/chromium/media/cdm/json_web_key.cc +++ b/chromium/media/cdm/json_web_key.cc @@ -172,16 +172,15 @@ bool ExtractKeysFromJWKSet(const std::string& jwk_set, return false; } - std::unique_ptr<base::Value> root( - base::JSONReader().ReadToValueDeprecated(jwk_set)); - if (!root.get() || root->type() != base::Value::Type::DICTIONARY) { - DVLOG(1) << "Not valid JSON: " << jwk_set << ", root: " << root.get(); + base::Optional<base::Value> root = base::JSONReader::Read(jwk_set); + if (!root || root->type() != base::Value::Type::DICTIONARY) { + DVLOG(1) << "Not valid JSON: " << jwk_set; return false; } // Locate the set from the dictionary. base::DictionaryValue* dictionary = - static_cast<base::DictionaryValue*>(root.get()); + static_cast<base::DictionaryValue*>(&root.value()); base::ListValue* list_val = NULL; if (!dictionary->GetList(kKeysTag, &list_val)) { DVLOG(1) << "Missing '" << kKeysTag @@ -242,9 +241,8 @@ bool ExtractKeyIdsFromKeyIdsInitData(const std::string& input, return false; } - std::unique_ptr<base::Value> root( - base::JSONReader().ReadToValueDeprecated(input)); - if (!root.get() || root->type() != base::Value::Type::DICTIONARY) { + base::Optional<base::Value> root = base::JSONReader::Read(input); + if (!root || root->type() != base::Value::Type::DICTIONARY) { error_message->assign("Not valid JSON: "); error_message->append(ShortenTo64Characters(input)); return false; @@ -252,7 +250,7 @@ bool ExtractKeyIdsFromKeyIdsInitData(const std::string& input, // Locate the set from the dictionary. base::DictionaryValue* dictionary = - static_cast<base::DictionaryValue*>(root.get()); + static_cast<base::DictionaryValue*>(&root.value()); base::ListValue* list_val = NULL; if (!dictionary->GetList(kKeyIdsTag, &list_val)) { error_message->assign("Missing '"); @@ -376,16 +374,15 @@ bool ExtractFirstKeyIdFromLicenseRequest(const std::vector<uint8_t>& license, return false; } - std::unique_ptr<base::Value> root( - base::JSONReader().ReadToValueDeprecated(license_as_str)); - if (!root.get() || root->type() != base::Value::Type::DICTIONARY) { + base::Optional<base::Value> root = base::JSONReader::Read(license_as_str); + if (!root || root->type() != base::Value::Type::DICTIONARY) { DVLOG(1) << "Not valid JSON: " << license_as_str; return false; } // Locate the set from the dictionary. base::DictionaryValue* dictionary = - static_cast<base::DictionaryValue*>(root.get()); + static_cast<base::DictionaryValue*>(&root.value()); base::ListValue* list_val = NULL; if (!dictionary->GetList(kKeyIdsTag, &list_val)) { DVLOG(1) << "Missing '" << kKeyIdsTag << "' parameter or not a list"; diff --git a/chromium/media/cdm/json_web_key_unittest.cc b/chromium/media/cdm/json_web_key_unittest.cc index 3c4ed4f28f3..e0c4d348a97 100644 --- a/chromium/media/cdm/json_web_key_unittest.cc +++ b/chromium/media/cdm/json_web_key_unittest.cc @@ -8,7 +8,7 @@ #include <stdint.h> #include "base/base64.h" -#include "base/logging.h" +#include "base/check.h" #include "base/stl_util.h" #include "media/base/content_decryption_module.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/chromium/media/cdm/library_cdm/cdm_host_proxy.h b/chromium/media/cdm/library_cdm/cdm_host_proxy.h index ccbda3b6913..ac1ff8d5403 100644 --- a/chromium/media/cdm/library_cdm/cdm_host_proxy.h +++ b/chromium/media/cdm/library_cdm/cdm_host_proxy.h @@ -54,7 +54,6 @@ class CdmHostProxy { virtual void OnDeferredInitializationDone(cdm::StreamType stream_type, cdm::Status decoder_status) = 0; virtual cdm::FileIO* CreateFileIO(cdm::FileIOClient* client) = 0; - virtual cdm::CdmProxy* RequestCdmProxy(cdm::CdmProxyClient* client) = 0; virtual void RequestStorageId(uint32_t version) = 0; }; diff --git a/chromium/media/cdm/library_cdm/cdm_host_proxy_impl.h b/chromium/media/cdm/library_cdm/cdm_host_proxy_impl.h index 3a3bbb0eac5..22ebd602a0a 100644 --- a/chromium/media/cdm/library_cdm/cdm_host_proxy_impl.h +++ b/chromium/media/cdm/library_cdm/cdm_host_proxy_impl.h @@ -110,10 +110,6 @@ class CdmHostProxyImpl : public CdmHostProxy { return host_->CreateFileIO(client); } - cdm::CdmProxy* RequestCdmProxy(cdm::CdmProxyClient* client) final { - return host_->RequestCdmProxy(client); - } - void RequestStorageId(uint32_t version) final { host_->RequestStorageId(version); } @@ -124,15 +120,6 @@ class CdmHostProxyImpl : public CdmHostProxy { DISALLOW_COPY_AND_ASSIGN(CdmHostProxyImpl); }; -// Specialization for cdm::Host_10 methods. - -template <> -cdm::CdmProxy* CdmHostProxyImpl<cdm::Host_10>::RequestCdmProxy( - cdm::CdmProxyClient* /* client */) { - NOTREACHED() << "cdm::ContentDecryptionModule_10 CDM should never call this."; - return nullptr; -} - } // namespace media #endif // MEDIA_CDM_LIBRARY_CDM_CDM_HOST_PROXY_IMPL_H_ diff --git a/chromium/media/cdm/library_cdm/clear_key_cdm/BUILD.gn b/chromium/media/cdm/library_cdm/clear_key_cdm/BUILD.gn index 8e01c758d09..25b1dfee1ef 100644 --- a/chromium/media/cdm/library_cdm/clear_key_cdm/BUILD.gn +++ b/chromium/media/cdm/library_cdm/clear_key_cdm/BUILD.gn @@ -15,8 +15,6 @@ loadable_module("clear_key_cdm") { "cdm_file_adapter.h", "cdm_file_io_test.cc", "cdm_file_io_test.h", - "cdm_proxy_handler.cc", - "cdm_proxy_handler.h", "cdm_video_decoder.cc", "cdm_video_decoder.h", "clear_key_cdm.cc", @@ -31,7 +29,6 @@ loadable_module("clear_key_cdm") { defines = [ "CDM_IMPLEMENTATION" ] deps = [ - ":cdm_proxy_common", "//base", "//media", "//media:media_buildflags", @@ -53,20 +50,3 @@ loadable_module("clear_key_cdm") { deps += [ "//third_party/ffmpeg" ] } } - -source_set("clear_key_cdm_proxy") { - sources = [ - "clear_key_cdm_proxy.cc", - "clear_key_cdm_proxy.h", - ] - - deps = [ - ":cdm_proxy_common", - "//base", - "//media", - ] -} - -source_set("cdm_proxy_common") { - sources = [ "cdm_proxy_common.h" ] -} diff --git a/chromium/media/cdm/library_cdm/clear_key_cdm/cdm_proxy_common.h b/chromium/media/cdm/library_cdm/clear_key_cdm/cdm_proxy_common.h deleted file mode 100644 index f0a6a8a2f79..00000000000 --- a/chromium/media/cdm/library_cdm/clear_key_cdm/cdm_proxy_common.h +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MEDIA_CDM_LIBRARY_CDM_CLEAR_KEY_CDM_CDM_PROXY_COMMON_H_ -#define MEDIA_CDM_LIBRARY_CDM_CLEAR_KEY_CDM_CDM_PROXY_COMMON_H_ - -#include <stdint.h> -#include <array> - -namespace media { - -// Constants used to test CdmProxy stack using ClearKeyCdm and ClearKeyCdmProxy. -constexpr uint32_t kClearKeyCdmProxyCryptoSessionId = 1; -constexpr uint32_t kClearKeyCdmProxyMediaCryptoSessionId = 23; -constexpr std::array<uint8_t, 3> kClearKeyCdmProxyInputData = {4, 5, 6}; -constexpr std::array<uint8_t, 4> kClearKeyCdmProxyOutputData = {7, 8, 9, 10}; - -} // namespace media - -#endif // MEDIA_CDM_LIBRARY_CDM_CLEAR_KEY_CDM_CDM_PROXY_COMMON_H_ diff --git a/chromium/media/cdm/library_cdm/clear_key_cdm/cdm_proxy_handler.cc b/chromium/media/cdm/library_cdm/clear_key_cdm/cdm_proxy_handler.cc deleted file mode 100644 index 115dcdf1de9..00000000000 --- a/chromium/media/cdm/library_cdm/clear_key_cdm/cdm_proxy_handler.cc +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/cdm/library_cdm/clear_key_cdm/cdm_proxy_handler.h" - -#include <stdint.h> -#include <algorithm> - -#include "base/logging.h" -#include "base/macros.h" -#include "media/cdm/library_cdm/cdm_host_proxy.h" -#include "media/cdm/library_cdm/clear_key_cdm/cdm_proxy_common.h" - -namespace media { - -CdmProxyHandler::CdmProxyHandler(CdmHostProxy* cdm_host_proxy) - : cdm_host_proxy_(cdm_host_proxy) {} - -CdmProxyHandler::~CdmProxyHandler() {} - -void CdmProxyHandler::Initialize(InitCB init_cb) { - DVLOG(1) << __func__; - init_cb_ = std::move(init_cb); - - cdm_proxy_ = cdm_host_proxy_->RequestCdmProxy(this); - if (!cdm_proxy_) { - FinishInitialization(false); - return; - } - - cdm_proxy_->Initialize(); -} - -void CdmProxyHandler::SetKey(const std::vector<uint8_t>& response, - SetKeyCB set_key_cb) { - DVLOG(2) << __func__; - DCHECK(!set_key_cb_); - set_key_cb_ = std::move(set_key_cb); - cdm_proxy_->SetKey(crypto_session_id_, nullptr, 0, - cdm::CdmProxy::kDecryptAndDecode, response.data(), - response.size()); -} - -void CdmProxyHandler::FinishInitialization(bool success) { - DVLOG(1) << __func__ << ": success = " << success; - std::move(init_cb_).Run(success); -} - -void CdmProxyHandler::OnInitialized(Status status, - Protocol protocol, - uint32_t crypto_session_id) { - DVLOG(1) << __func__ << ": status = " << status; - - if (status != Status::kOk || - crypto_session_id != kClearKeyCdmProxyCryptoSessionId) { - FinishInitialization(false); - return; - } - - // Only one CdmProxy can be created during the lifetime of the CDM instance. - if (cdm_host_proxy_->RequestCdmProxy(this)) { - FinishInitialization(false); - return; - } - - cdm_proxy_->Process(cdm::CdmProxy::kIntelNegotiateCryptoSessionKeyExchange, - crypto_session_id, kClearKeyCdmProxyInputData.data(), - kClearKeyCdmProxyInputData.size(), 0); -} - -void CdmProxyHandler::OnProcessed(Status status, - const uint8_t* output_data, - uint32_t output_data_size) { - DVLOG(2) << __func__ << ": status = " << status; - - if (status != Status::kOk || - !std::equal(output_data, output_data + output_data_size, - kClearKeyCdmProxyOutputData.begin())) { - FinishInitialization(false); - return; - } - - cdm_proxy_->CreateMediaCryptoSession(kClearKeyCdmProxyInputData.data(), - kClearKeyCdmProxyInputData.size()); -} - -void CdmProxyHandler::OnMediaCryptoSessionCreated(Status status, - uint32_t crypto_session_id, - uint64_t output_data) { - DVLOG(2) << __func__ << ": status = " << status; - - if (status != Status::kOk || - crypto_session_id != kClearKeyCdmProxyMediaCryptoSessionId) { - FinishInitialization(false); - return; - } - - FinishInitialization(true); -} - -void CdmProxyHandler::OnKeySet(Status status) { - DVLOG(2) << __func__ << ": status = " << status; - DCHECK(set_key_cb_); - - std::move(set_key_cb_).Run(status == Status::kOk); -} - -void CdmProxyHandler::OnKeyRemoved(Status status) { - DVLOG(2) << __func__; - NOTREACHED(); -} - -void CdmProxyHandler::NotifyHardwareReset() { - DVLOG(1) << __func__; - NOTREACHED(); -} - -} // namespace media diff --git a/chromium/media/cdm/library_cdm/clear_key_cdm/cdm_proxy_handler.h b/chromium/media/cdm/library_cdm/clear_key_cdm/cdm_proxy_handler.h deleted file mode 100644 index 850d905716c..00000000000 --- a/chromium/media/cdm/library_cdm/clear_key_cdm/cdm_proxy_handler.h +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MEDIA_CDM_LIBRARY_CDM_CLEAR_KEY_CDM_CDM_PROXY_HANDLER_H_ -#define MEDIA_CDM_LIBRARY_CDM_CLEAR_KEY_CDM_CDM_PROXY_HANDLER_H_ - -#include "base/callback.h" -#include "base/macros.h" -#include "media/cdm/api/content_decryption_module.h" - -namespace media { - -class CdmHostProxy; - -class CdmProxyHandler : public cdm::CdmProxyClient { - public: - using InitCB = base::OnceCallback<void(bool success)>; - using SetKeyCB = base::OnceCallback<void(bool success)>; - - explicit CdmProxyHandler(CdmHostProxy* cdm_host_proxy); - ~CdmProxyHandler() override; - - // Initializes the CdmProxyHandler and returns the result through |init_cb|. - // This will request and initialize the CdmProxy, create media crypto session - // and do some trivial procesing for better test coverage. - void Initialize(InitCB init_cb); - - // Push a response that contains a license to the CdmProxy. - void SetKey(const std::vector<uint8_t>& response, SetKeyCB set_key_cb); - - private: - void FinishInitialization(bool success); - - // cdm::CdmProxyClient implementation. - void OnInitialized(Status status, - Protocol protocol, - uint32_t crypto_session_id) final; - void OnProcessed(Status status, - const uint8_t* output_data, - uint32_t output_data_size) final; - void OnMediaCryptoSessionCreated(Status status, - uint32_t crypto_session_id, - uint64_t output_data) final; - void OnKeySet(Status status) final; - void OnKeyRemoved(Status status) final; - void NotifyHardwareReset() final; - - CdmHostProxy* const cdm_host_proxy_ = nullptr; - InitCB init_cb_; - SetKeyCB set_key_cb_; - cdm::CdmProxy* cdm_proxy_ = nullptr; - uint32_t crypto_session_id_ = 0u; - - DISALLOW_COPY_AND_ASSIGN(CdmProxyHandler); -}; - -} // namespace media - -#endif // MEDIA_CDM_LIBRARY_CDM_CLEAR_KEY_CDM_CDM_PROXY_HANDLER_H_ diff --git a/chromium/media/cdm/library_cdm/clear_key_cdm/clear_key_cdm.cc b/chromium/media/cdm/library_cdm/clear_key_cdm/clear_key_cdm.cc index 4d3c278b531..5ad5056e830 100644 --- a/chromium/media/cdm/library_cdm/clear_key_cdm/clear_key_cdm.cc +++ b/chromium/media/cdm/library_cdm/clear_key_cdm/clear_key_cdm.cc @@ -29,7 +29,6 @@ #include "media/cdm/library_cdm/cdm_host_proxy.h" #include "media/cdm/library_cdm/cdm_host_proxy_impl.h" #include "media/cdm/library_cdm/clear_key_cdm/cdm_file_io_test.h" -#include "media/cdm/library_cdm/clear_key_cdm/cdm_proxy_handler.h" #include "media/cdm/library_cdm/clear_key_cdm/cdm_video_decoder.h" #include "media/media_buildflags.h" @@ -67,8 +66,6 @@ const char kExternalClearKeyStorageIdTestKeySystem[] = "org.chromium.externalclearkey.storageidtest"; const char kExternalClearKeyDifferentGuidTestKeySystem[] = "org.chromium.externalclearkey.differentguid"; -const char kExternalClearKeyCdmProxyKeySystem[] = - "org.chromium.externalclearkey.cdmproxy"; const int64_t kMsPerSecond = 1000; const int64_t kMaxTimerDelayMs = 5 * kMsPerSecond; @@ -181,8 +178,7 @@ void* CreateCdmInstance(int cdm_interface_version, key_system_string != kExternalClearKeyCrashKeySystem && key_system_string != kExternalClearKeyVerifyCdmHostTestKeySystem && key_system_string != kExternalClearKeyStorageIdTestKeySystem && - key_system_string != kExternalClearKeyDifferentGuidTestKeySystem && - key_system_string != kExternalClearKeyCdmProxyKeySystem) { + key_system_string != kExternalClearKeyDifferentGuidTestKeySystem) { DVLOG(1) << "Unsupported key system:" << key_system_string; return nullptr; } @@ -349,13 +345,6 @@ void ClearKeyCdm::Initialize(bool allow_distinctive_identifier, // to check persistent state permission. allow_persistent_state_ = allow_persistent_state; - // CdmProxy must be created during initialization time. OnInitialized() will - // be called in OnCdmProxyHandlerInitialized(). - if (key_system_ == kExternalClearKeyCdmProxyKeySystem) { - InitializeCdmProxyHandler(); - return; - } - cdm_host_proxy_->OnInitialized(true); } @@ -441,54 +430,15 @@ void ClearKeyCdm::UpdateSession(uint32_t promise_id, DVLOG(1) << __func__; std::string web_session_str(session_id, session_id_length); std::vector<uint8_t> response_vector(response, response + response_size); - auto pending_update_params = std::make_unique<UpdateParams>( - promise_id, std::move(web_session_str), response_vector); - - // Push the license to the CdmProxy. The license will still be pushed to the - // |cdm_| after OnKeySet() is called, which then triggers - // OnSessionKeysChange(). This order is critical to avoid race conditions like - // OnSessionKeysChange() being called before the keys are actually available - // in the CdmProxy. - if (cdm_proxy_handler_) { - if (pending_update_params_) { - OnPromiseFailed(promise_id, CdmPromise::Exception::INVALID_STATE_ERROR, 0, - "Parallel updates not supported."); - return; - } - - pending_update_params_ = std::move(pending_update_params); - cdm_proxy_handler_->SetKey( - response_vector, - base::BindOnce(&ClearKeyCdm::OnCdmProxyKeySet, base::Unretained(this))); - return; - } - - UpdateSessionInternal(std::move(pending_update_params)); -} - -void ClearKeyCdm::OnCdmProxyKeySet(bool success) { - DCHECK(pending_update_params_); - - if (!success) { - auto promise_id = pending_update_params_->promise_id; - pending_update_params_.reset(); - OnPromiseFailed(promise_id, CdmPromise::Exception::INVALID_STATE_ERROR, 0, - "Parallel updates not supported."); - return; - } - - UpdateSessionInternal(std::move(pending_update_params_)); -} -void ClearKeyCdm::UpdateSessionInternal(std::unique_ptr<UpdateParams> params) { std::unique_ptr<media::SimpleCdmPromise> promise( new media::CdmCallbackPromise<>( base::BindOnce(&ClearKeyCdm::OnUpdateSuccess, base::Unretained(this), - params->promise_id, params->session_id), + promise_id, web_session_str), base::BindOnce(&ClearKeyCdm::OnPromiseFailed, base::Unretained(this), - params->promise_id))); + promise_id))); - cdm_->UpdateSession(params->session_id, params->response, std::move(promise)); + cdm_->UpdateSession(session_id, response_vector, std::move(promise)); } void ClearKeyCdm::OnUpdateSuccess(uint32_t promise_id, @@ -616,10 +566,6 @@ cdm::Status ClearKeyCdm::Decrypt(const cdm::InputBuffer_2& encrypted_buffer, DVLOG(1) << __func__; DCHECK(encrypted_buffer.data); - // When CdmProxy is used, the CDM cannot do any decryption or decoding. - if (key_system_ == kExternalClearKeyCdmProxyKeySystem) - return cdm::kDecryptError; - scoped_refptr<DecoderBuffer> buffer; cdm::Status status = DecryptToMediaDecoderBuffer(encrypted_buffer, &buffer); @@ -639,10 +585,8 @@ cdm::Status ClearKeyCdm::Decrypt(const cdm::InputBuffer_2& encrypted_buffer, cdm::Status ClearKeyCdm::InitializeAudioDecoder( const cdm::AudioDecoderConfig_2& audio_decoder_config) { - if (key_system_ == kExternalClearKeyDecryptOnlyKeySystem || - key_system_ == kExternalClearKeyCdmProxyKeySystem) { + if (key_system_ == kExternalClearKeyDecryptOnlyKeySystem) return cdm::kInitializationError; - } #if defined(CLEAR_KEY_CDM_USE_FFMPEG_DECODER) if (!audio_decoder_) @@ -665,10 +609,8 @@ cdm::Status ClearKeyCdm::InitializeVideoDecoder( cdm::Status ClearKeyCdm::InitializeVideoDecoder( const cdm::VideoDecoderConfig_3& video_decoder_config) { - if (key_system_ == kExternalClearKeyDecryptOnlyKeySystem || - key_system_ == kExternalClearKeyCdmProxyKeySystem) { + if (key_system_ == kExternalClearKeyDecryptOnlyKeySystem) return cdm::kInitializationError; - } if (!video_decoder_) { video_decoder_ = @@ -1029,29 +971,4 @@ void ClearKeyCdm::StartStorageIdTest() { cdm_host_proxy_->RequestStorageId(0); } -void ClearKeyCdm::InitializeCdmProxyHandler() { - DVLOG(1) << __func__; - DCHECK(!cdm_proxy_handler_); - - cdm_proxy_handler_ = std::make_unique<CdmProxyHandler>(cdm_host_proxy_.get()); - cdm_proxy_handler_->Initialize(base::BindOnce( - &ClearKeyCdm::OnCdmProxyHandlerInitialized, base::Unretained(this))); -} - -void ClearKeyCdm::OnCdmProxyHandlerInitialized(bool success) { - DVLOG(1) << __func__; - DCHECK(cdm_proxy_handler_); - - cdm_host_proxy_->OnInitialized(success); -} - -ClearKeyCdm::UpdateParams::UpdateParams(uint32_t promise_id, - std::string session_id, - std::vector<uint8_t> response) - : promise_id(promise_id), - session_id(std::move(session_id)), - response(std::move(response)) {} - -ClearKeyCdm::UpdateParams::~UpdateParams() {} - } // namespace media diff --git a/chromium/media/cdm/library_cdm/clear_key_cdm/clear_key_cdm.h b/chromium/media/cdm/library_cdm/clear_key_cdm/clear_key_cdm.h index b33cd54d188..79e86d52656 100644 --- a/chromium/media/cdm/library_cdm/clear_key_cdm/clear_key_cdm.h +++ b/chromium/media/cdm/library_cdm/clear_key_cdm/clear_key_cdm.h @@ -23,7 +23,6 @@ namespace media { class CdmHostProxy; -class CdmProxyHandler; class CdmVideoDecoder; class DecoderBuffer; class FFmpegCdmAudioDecoder; @@ -103,17 +102,6 @@ class ClearKeyCdm : public cdm::ContentDecryptionModule_10, uint32_t storage_id_size) override; private: - struct UpdateParams { - UpdateParams(uint32_t promise_id, - std::string session_id, - std::vector<uint8_t> response); - ~UpdateParams(); - - const uint32_t promise_id; - const std::string session_id; - const std::vector<uint8_t> response; - }; - // ContentDecryptionModule callbacks. void OnSessionMessage(const std::string& session_id, CdmMessageType message_type, @@ -163,12 +151,6 @@ class ClearKeyCdm : public cdm::ContentDecryptionModule_10, void ReportVerifyCdmHostTestResult(); void StartStorageIdTest(); - void InitializeCdmProxyHandler(); - void OnCdmProxyHandlerInitialized(bool success); - void OnCdmProxyKeySet(bool success); - - void UpdateSessionInternal(std::unique_ptr<UpdateParams> params); - int host_interface_version_ = 0; std::unique_ptr<CdmHostProxy> cdm_host_proxy_; @@ -193,12 +175,9 @@ class ClearKeyCdm : public cdm::ContentDecryptionModule_10, std::unique_ptr<FFmpegCdmAudioDecoder> audio_decoder_; #endif // CLEAR_KEY_CDM_USE_FFMPEG_DECODER - std::unique_ptr<UpdateParams> pending_update_params_; - std::unique_ptr<CdmVideoDecoder> video_decoder_; std::unique_ptr<FileIOTestRunner> file_io_test_runner_; - std::unique_ptr<CdmProxyHandler> cdm_proxy_handler_; bool is_running_output_protection_test_ = false; bool is_running_platform_verification_test_ = false; diff --git a/chromium/media/cdm/library_cdm/clear_key_cdm/clear_key_cdm_proxy.cc b/chromium/media/cdm/library_cdm/clear_key_cdm/clear_key_cdm_proxy.cc deleted file mode 100644 index 0b4ac9c4a1c..00000000000 --- a/chromium/media/cdm/library_cdm/clear_key_cdm/clear_key_cdm_proxy.cc +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/cdm/library_cdm/clear_key_cdm/clear_key_cdm_proxy.h" - -#include "base/bind_helpers.h" -#include "base/logging.h" -#include "media/base/content_decryption_module.h" -#include "media/cdm/library_cdm/clear_key_cdm/cdm_proxy_common.h" - -namespace media { - -namespace { - -constexpr char kDummySessionId[] = "dummy session id"; - -class IgnoreResponsePromise : public SimpleCdmPromise { - public: - IgnoreResponsePromise() = default; - ~IgnoreResponsePromise() override = default; - - // SimpleCdmPromise implementation. - void resolve() final { MarkPromiseSettled(); } - void reject(CdmPromise::Exception exception_code, - uint32_t system_code, - const std::string& error_message) final { - MarkPromiseSettled(); - } -}; - -} // namespace - -ClearKeyCdmProxy::ClearKeyCdmProxy() {} - -ClearKeyCdmProxy::~ClearKeyCdmProxy() {} - -base::WeakPtr<CdmContext> ClearKeyCdmProxy::GetCdmContext() { - DVLOG(1) << __func__; - return weak_factory_.GetWeakPtr(); -} - -void ClearKeyCdmProxy::Initialize(Client* client, InitializeCB init_cb) { - DVLOG(1) << __func__; - - std::move(init_cb).Run(Status::kOk, Protocol::kIntel, - kClearKeyCdmProxyCryptoSessionId); -} - -void ClearKeyCdmProxy::Process(Function function, - uint32_t crypto_session_id, - const std::vector<uint8_t>& input_data, - uint32_t expected_output_data_size, - ProcessCB process_cb) { - DVLOG(2) << __func__; - - if (crypto_session_id != kClearKeyCdmProxyCryptoSessionId || - !std::equal(input_data.begin(), input_data.end(), - kClearKeyCdmProxyInputData.begin(), - kClearKeyCdmProxyInputData.end())) { - std::move(process_cb).Run(Status::kFail, {}); - return; - } - - std::move(process_cb) - .Run(Status::kOk, - std::vector<uint8_t>(kClearKeyCdmProxyOutputData.begin(), - kClearKeyCdmProxyOutputData.end())); -} - -void ClearKeyCdmProxy::CreateMediaCryptoSession( - const std::vector<uint8_t>& input_data, - CreateMediaCryptoSessionCB create_media_crypto_session_cb) { - DVLOG(2) << __func__; - - if (!std::equal(input_data.begin(), input_data.end(), - kClearKeyCdmProxyInputData.begin(), - kClearKeyCdmProxyInputData.end())) { - std::move(create_media_crypto_session_cb).Run(Status::kFail, 0, 0); - return; - } - - std::move(create_media_crypto_session_cb) - .Run(Status::kOk, kClearKeyCdmProxyMediaCryptoSessionId, 0); -} - -void ClearKeyCdmProxy::SetKey(uint32_t crypto_session_id, - const std::vector<uint8_t>& key_id, - KeyType /* key_type */, - const std::vector<uint8_t>& key_blob, - SetKeyCB set_key_cb) { - DVLOG(1) << __func__; - - if (!aes_decryptor_) - CreateDecryptor(); - - aes_decryptor_->UpdateSession(kDummySessionId, key_blob, - std::make_unique<IgnoreResponsePromise>()); - std::move(set_key_cb).Run(Status::kOk); -} - -void ClearKeyCdmProxy::RemoveKey(uint32_t crypto_session_id, - const std::vector<uint8_t>& key_id, - RemoveKeyCB remove_key_cb) { - std::move(remove_key_cb).Run(Status::kOk); -} - -Decryptor* ClearKeyCdmProxy::GetDecryptor() { - DVLOG(1) << __func__; - - if (!aes_decryptor_) - CreateDecryptor(); - - return aes_decryptor_.get(); -} - -void ClearKeyCdmProxy::CreateDecryptor() { - DVLOG(1) << __func__; - DCHECK(!aes_decryptor_); - - aes_decryptor_ = - base::MakeRefCounted<AesDecryptor>(base::DoNothing(), base::DoNothing(), - base::DoNothing(), base::DoNothing()); - - // Also create a dummy session to be used for SetKey(). - aes_decryptor_->CreateSession(kDummySessionId, CdmSessionType::kTemporary); -} - -} // namespace media diff --git a/chromium/media/cdm/library_cdm/clear_key_cdm/clear_key_cdm_proxy.h b/chromium/media/cdm/library_cdm/clear_key_cdm/clear_key_cdm_proxy.h deleted file mode 100644 index bc78f23bd6b..00000000000 --- a/chromium/media/cdm/library_cdm/clear_key_cdm/clear_key_cdm_proxy.h +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MEDIA_CDM_LIBRARY_CDM_CLEAR_KEY_CDM_CLEAR_KEY_CDM_PROXY_H_ -#define MEDIA_CDM_LIBRARY_CDM_CLEAR_KEY_CDM_CLEAR_KEY_CDM_PROXY_H_ - -#include "base/callback.h" -#include "base/macros.h" -#include "base/memory/ref_counted.h" -#include "base/memory/weak_ptr.h" -#include "media/base/cdm_context.h" -#include "media/cdm/aes_decryptor.h" -#include "media/cdm/cdm_proxy.h" - -namespace media { - -// CdmProxy implementation for Clear Key CDM to test CDM Proxy support. -class ClearKeyCdmProxy : public CdmProxy, public CdmContext { - public: - ClearKeyCdmProxy(); - ~ClearKeyCdmProxy() final; - - // CdmProxy implementation. - base::WeakPtr<CdmContext> GetCdmContext() final; - void Initialize(Client* client, InitializeCB init_cb) final; - void Process(Function function, - uint32_t crypto_session_id, - const std::vector<uint8_t>& input_data, - uint32_t expected_output_data_size, - ProcessCB process_cb) final; - void CreateMediaCryptoSession( - const std::vector<uint8_t>& input_data, - CreateMediaCryptoSessionCB create_media_crypto_session_cb) final; - void SetKey(uint32_t crypto_session_id, - const std::vector<uint8_t>& key_id, - KeyType key_type, - const std::vector<uint8_t>& key_blob, - SetKeyCB set_key_cb) final; - void RemoveKey(uint32_t crypto_session_id, - const std::vector<uint8_t>& key_id, - RemoveKeyCB remove_key_cb) final; - - // CdmContext implementation. - Decryptor* GetDecryptor() final; - - private: - void CreateDecryptor(); - - scoped_refptr<AesDecryptor> aes_decryptor_; - - base::WeakPtrFactory<ClearKeyCdmProxy> weak_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(ClearKeyCdmProxy); -}; - -} // namespace media - -#endif // MEDIA_CDM_LIBRARY_CDM_CLEAR_KEY_CDM_CLEAR_KEY_CDM_PROXY_H_ diff --git a/chromium/media/cdm/simple_cdm_buffer.cc b/chromium/media/cdm/simple_cdm_buffer.cc index 76d0e1e91b0..ea2a5d357d4 100644 --- a/chromium/media/cdm/simple_cdm_buffer.cc +++ b/chromium/media/cdm/simple_cdm_buffer.cc @@ -6,7 +6,7 @@ #include <limits> -#include "base/logging.h" +#include "base/check_op.h" #include "base/numerics/safe_conversions.h" namespace media { diff --git a/chromium/media/filters/BUILD.gn b/chromium/media/filters/BUILD.gn index 9b97365808f..c49ebe92141 100644 --- a/chromium/media/filters/BUILD.gn +++ b/chromium/media/filters/BUILD.gn @@ -14,6 +14,7 @@ jumbo_source_set("filters") { visibility = [ "//media", "//media/renderers", + "//media/webcodecs", ] sources = [ @@ -187,6 +188,13 @@ jumbo_source_set("filters") { "ffmpeg_video_decoder.h", ] } + + if (is_android) { + sources += [ + "android/video_frame_extractor.cc", + "android/video_frame_extractor.h", + ] + } } if (is_android) { @@ -199,8 +207,6 @@ jumbo_source_set("filters") { sources += [ "android/media_codec_audio_decoder.cc", "android/media_codec_audio_decoder.h", - "android/video_frame_extractor.cc", - "android/video_frame_extractor.h", ] deps += [ "//media/base/android" ] } diff --git a/chromium/media/filters/aom_video_decoder.cc b/chromium/media/filters/aom_video_decoder.cc index 6d754abcc69..1ce06159a32 100644 --- a/chromium/media/filters/aom_video_decoder.cc +++ b/chromium/media/filters/aom_video_decoder.cc @@ -11,6 +11,7 @@ #include "media/base/bind_to_current_loop.h" #include "media/base/decoder_buffer.h" #include "media/base/media_log.h" +#include "media/base/status.h" #include "media/base/video_util.h" #include "media/filters/frame_buffer_pool.h" #include "third_party/libyuv/include/libyuv/convert.h" @@ -147,7 +148,7 @@ void AomVideoDecoder::Initialize(const VideoDecoderConfig& config, InitCB bound_init_cb = BindToCurrentLoop(std::move(init_cb)); if (config.is_encrypted() || config.codec() != kCodecAV1) { - std::move(bound_init_cb).Run(false); + std::move(bound_init_cb).Run(StatusCode::kDecoderFailedInitialization); return; } @@ -172,7 +173,7 @@ void AomVideoDecoder::Initialize(const VideoDecoderConfig& config, 0 /* flags */) != AOM_CODEC_OK) { MEDIA_LOG(ERROR, media_log_) << "aom_codec_dec_init() failed: " << aom_codec_error(aom_decoder_.get()); - std::move(bound_init_cb).Run(false); + std::move(bound_init_cb).Run(StatusCode::kDecoderFailedInitialization); return; } @@ -184,7 +185,7 @@ void AomVideoDecoder::Initialize(const VideoDecoderConfig& config, memory_pool_.get()) != AOM_CODEC_OK) { DLOG(ERROR) << "Failed to configure external buffers. " << aom_codec_error(context.get()); - std::move(bound_init_cb).Run(false); + std::move(bound_init_cb).Run(StatusCode::kDecoderFailedInitialization); return; } @@ -192,7 +193,7 @@ void AomVideoDecoder::Initialize(const VideoDecoderConfig& config, state_ = DecoderState::kNormal; output_cb_ = BindToCurrentLoop(output_cb); aom_decoder_ = std::move(context); - std::move(bound_init_cb).Run(true); + std::move(bound_init_cb).Run(OkStatus()); } void AomVideoDecoder::Decode(scoped_refptr<DecoderBuffer> buffer, diff --git a/chromium/media/filters/aom_video_decoder_unittest.cc b/chromium/media/filters/aom_video_decoder_unittest.cc index f8e2612e0fd..3de1a5440c0 100644 --- a/chromium/media/filters/aom_video_decoder_unittest.cc +++ b/chromium/media/filters/aom_video_decoder_unittest.cc @@ -49,7 +49,12 @@ class AomVideoDecoderTest : public testing::Test { void InitializeWithConfigWithResult(const VideoDecoderConfig& config, bool success) { - decoder_->Initialize(config, false, nullptr, NewExpectedBoolCB(success), + decoder_->Initialize(config, false, nullptr, + base::BindOnce( + [](bool success, Status status) { + EXPECT_EQ(status.is_ok(), success); + }, + success), base::BindRepeating(&AomVideoDecoderTest::FrameReady, base::Unretained(this)), base::NullCallback()); diff --git a/chromium/media/filters/audio_clock.cc b/chromium/media/filters/audio_clock.cc index 67fab3e7b87..017e1c9e2ba 100644 --- a/chromium/media/filters/audio_clock.cc +++ b/chromium/media/filters/audio_clock.cc @@ -10,7 +10,7 @@ #include <algorithm> #include <cmath> -#include "base/logging.h" +#include "base/check_op.h" namespace media { diff --git a/chromium/media/filters/audio_file_reader_unittest.cc b/chromium/media/filters/audio_file_reader_unittest.cc index 11b1931a607..edae7b8a577 100644 --- a/chromium/media/filters/audio_file_reader_unittest.cc +++ b/chromium/media/filters/audio_file_reader_unittest.cc @@ -7,7 +7,6 @@ #include <memory> #include "base/hash/md5.h" -#include "base/logging.h" #include "base/macros.h" #include "build/build_config.h" #include "media/base/audio_bus.h" diff --git a/chromium/media/filters/chunk_demuxer.cc b/chromium/media/filters/chunk_demuxer.cc index 4651bb5fdb3..782cfa91af5 100644 --- a/chromium/media/filters/chunk_demuxer.cc +++ b/chromium/media/filters/chunk_demuxer.cc @@ -576,6 +576,11 @@ int64_t ChunkDemuxer::GetMemoryUsage() const { return mem; } +base::Optional<container_names::MediaContainerName> +ChunkDemuxer::GetContainerForMetrics() const { + return base::nullopt; +} + void ChunkDemuxer::AbortPendingReads() { base::AutoLock auto_lock(lock_); DCHECK(state_ == INITIALIZED || state_ == ENDED || state_ == SHUTDOWN || diff --git a/chromium/media/filters/chunk_demuxer.h b/chromium/media/filters/chunk_demuxer.h index d79bf696f35..dc6dcfa62ba 100644 --- a/chromium/media/filters/chunk_demuxer.h +++ b/chromium/media/filters/chunk_demuxer.h @@ -223,6 +223,8 @@ class MEDIA_EXPORT ChunkDemuxer : public Demuxer { std::vector<DemuxerStream*> GetAllStreams() override; base::TimeDelta GetStartTime() const override; int64_t GetMemoryUsage() const override; + base::Optional<container_names::MediaContainerName> GetContainerForMetrics() + const override; void AbortPendingReads() override; // ChunkDemuxer reads are abortable. StartWaitingForSeek() and diff --git a/chromium/media/filters/decoder_selector.cc b/chromium/media/filters/decoder_selector.cc index 78d20de6231..bb06378b92d 100644 --- a/chromium/media/filters/decoder_selector.cc +++ b/chromium/media/filters/decoder_selector.cc @@ -165,6 +165,14 @@ void DecoderSelector<StreamType>::OnDecoderInitializeDone(Status status) { DCHECK(task_runner_->BelongsToCurrentThread()); if (!status.is_ok()) { + // TODO(tmathmeyer) this might be noisy in media log. Consider batching + // all failures as causes to a single Status object and only surfacing it if + // decoder selection fails entirely. + media_log_->NotifyError( + Status(StatusCode::kDecoderFailedInitialization) + .WithData("Decoder name", decoder_->GetDisplayName()) + .AddCause(std::move(status))); + // Try the next decoder on the list. decoder_.reset(); InitializeDecoder(); diff --git a/chromium/media/filters/decoder_selector_unittest.cc b/chromium/media/filters/decoder_selector_unittest.cc index cd94e12d429..d625e4840c8 100644 --- a/chromium/media/filters/decoder_selector_unittest.cc +++ b/chromium/media/filters/decoder_selector_unittest.cc @@ -6,9 +6,10 @@ #include <tuple> #include "base/bind.h" -#include "base/logging.h" +#include "base/check.h" #include "base/macros.h" #include "base/memory/scoped_refptr.h" +#include "base/notreached.h" #include "base/test/gmock_callback_support.h" #include "base/test/task_environment.h" #include "build/build_config.h" diff --git a/chromium/media/filters/decoder_stream.cc b/chromium/media/filters/decoder_stream.cc index ae43bb477a4..f5be9e03c14 100644 --- a/chromium/media/filters/decoder_stream.cc +++ b/chromium/media/filters/decoder_stream.cc @@ -91,6 +91,9 @@ const char* GetStatusString( case DecoderStream<StreamType>::DECODE_ERROR: return "decode_error"; } + + NOTREACHED(); + return ""; } template <DemuxerStream::Type StreamType> diff --git a/chromium/media/filters/ffmpeg_demuxer.cc b/chromium/media/filters/ffmpeg_demuxer.cc index 550ddf62655..97812376c52 100644 --- a/chromium/media/filters/ffmpeg_demuxer.cc +++ b/chromium/media/filters/ffmpeg_demuxer.cc @@ -641,6 +641,15 @@ void FFmpegDemuxerStream::EnqueuePacket(ScopedAVPacket packet) { if (packet->flags & AV_PKT_FLAG_KEY) buffer->set_is_key_frame(true); + // One last sanity check on the packet timestamps in case any of the above + // calculations have pushed the values to the limits. + if (buffer->timestamp() == kNoTimestamp || + buffer->timestamp() == kInfiniteDuration) { + MEDIA_LOG(ERROR, media_log_) << "FFmpegDemuxer: PTS is not defined"; + demuxer_->NotifyDemuxerError(DEMUXER_ERROR_COULD_NOT_PARSE); + return; + } + last_packet_timestamp_ = buffer->timestamp(); last_packet_duration_ = buffer->duration(); @@ -1150,6 +1159,11 @@ int64_t FFmpegDemuxer::GetMemoryUsage() const { return allocation_size; } +base::Optional<container_names::MediaContainerName> +FFmpegDemuxer::GetContainerForMetrics() const { + return container(); +} + void FFmpegDemuxer::OnEncryptedMediaInitData( EmeInitDataType init_data_type, const std::string& encryption_key_id) { diff --git a/chromium/media/filters/ffmpeg_demuxer.h b/chromium/media/filters/ffmpeg_demuxer.h index 2578a11f7ef..ab12030c1d0 100644 --- a/chromium/media/filters/ffmpeg_demuxer.h +++ b/chromium/media/filters/ffmpeg_demuxer.h @@ -231,6 +231,8 @@ class MEDIA_EXPORT FFmpegDemuxer : public Demuxer { std::vector<DemuxerStream*> GetAllStreams() override; base::TimeDelta GetStartTime() const override; int64_t GetMemoryUsage() const override; + base::Optional<container_names::MediaContainerName> GetContainerForMetrics() + const override; // Calls |encrypted_media_init_data_cb_| with the initialization data // encountered in the file. diff --git a/chromium/media/filters/ffmpeg_glue.cc b/chromium/media/filters/ffmpeg_glue.cc index e0ac8452536..53ec21394d7 100644 --- a/chromium/media/filters/ffmpeg_glue.cc +++ b/chromium/media/filters/ffmpeg_glue.cc @@ -4,9 +4,10 @@ #include "media/filters/ffmpeg_glue.h" -#include "base/logging.h" +#include "base/check_op.h" #include "base/macros.h" #include "base/metrics/histogram_functions.h" +#include "base/notreached.h" #include "media/base/container_names.h" #include "media/ffmpeg/ffmpeg_common.h" diff --git a/chromium/media/filters/ffmpeg_glue_unittest.cc b/chromium/media/filters/ffmpeg_glue_unittest.cc index 87cff01fea7..0e56adf9bee 100644 --- a/chromium/media/filters/ffmpeg_glue_unittest.cc +++ b/chromium/media/filters/ffmpeg_glue_unittest.cc @@ -8,7 +8,7 @@ #include <memory> -#include "base/logging.h" +#include "base/check.h" #include "base/macros.h" #include "base/test/metrics/histogram_tester.h" #include "media/base/container_names.h" diff --git a/chromium/media/filters/file_data_source.cc b/chromium/media/filters/file_data_source.cc index 443d378781e..4fce5b44158 100644 --- a/chromium/media/filters/file_data_source.cc +++ b/chromium/media/filters/file_data_source.cc @@ -7,7 +7,7 @@ #include <algorithm> #include <utility> -#include "base/logging.h" +#include "base/check_op.h" namespace media { diff --git a/chromium/media/filters/frame_buffer_pool.cc b/chromium/media/filters/frame_buffer_pool.cc index a008db80a19..03740545fc0 100644 --- a/chromium/media/filters/frame_buffer_pool.cc +++ b/chromium/media/filters/frame_buffer_pool.cc @@ -6,8 +6,8 @@ #include "base/bind.h" #include "base/callback_helpers.h" +#include "base/check_op.h" #include "base/location.h" -#include "base/logging.h" #include "base/macros.h" #include "base/sequenced_task_runner.h" #include "base/stl_util.h" diff --git a/chromium/media/filters/fuchsia/fuchsia_video_decoder.cc b/chromium/media/filters/fuchsia/fuchsia_video_decoder.cc index 8c3647f8f2a..8706bc45b3a 100644 --- a/chromium/media/filters/fuchsia/fuchsia_video_decoder.cc +++ b/chromium/media/filters/fuchsia/fuchsia_video_decoder.cc @@ -897,11 +897,18 @@ void FuchsiaVideoDecoder::OnOutputPacket(fuchsia::media::Packet output_packet, pixel_aspect_ratio = container_pixel_aspect_ratio_; } - base::TimeDelta timestamp; - if (output_packet.has_timestamp_ish()) { - timestamp = base::TimeDelta::FromNanoseconds(output_packet.timestamp_ish()); + // SendInputPacket() sets timestamp for all packets sent to the decoder, so we + // expect to receive timestamp for all decoded frames. Missing timestamp + // indicates a bug in the decoder implementation. + if (!output_packet.has_timestamp_ish()) { + LOG(ERROR) << "Received frame without timestamp."; + OnError(); + return; } + base::TimeDelta timestamp = + base::TimeDelta::FromNanoseconds(output_packet.timestamp_ish()); + num_used_output_buffers_++; auto frame = output_mailboxes_[buffer_index]->CreateFrame( diff --git a/chromium/media/filters/ivf_parser.cc b/chromium/media/filters/ivf_parser.cc index 8361088b040..4e3a149e8d8 100644 --- a/chromium/media/filters/ivf_parser.cc +++ b/chromium/media/filters/ivf_parser.cc @@ -4,6 +4,8 @@ #include "media/filters/ivf_parser.h" +#include <cstring> + #include "base/logging.h" #include "base/numerics/safe_conversions.h" #include "base/sys_byteorder.h" diff --git a/chromium/media/filters/media_file_checker_unittest.cc b/chromium/media/filters/media_file_checker_unittest.cc index 9ee40bf98cf..38047fd8ed1 100644 --- a/chromium/media/filters/media_file_checker_unittest.cc +++ b/chromium/media/filters/media_file_checker_unittest.cc @@ -7,7 +7,6 @@ #include <utility> #include "base/files/file.h" -#include "base/logging.h" #include "build/build_config.h" #include "media/base/test_data_util.h" #include "media/media_buildflags.h" diff --git a/chromium/media/filters/memory_data_source.cc b/chromium/media/filters/memory_data_source.cc index 9f1c0359b90..99401d6ad1f 100644 --- a/chromium/media/filters/memory_data_source.cc +++ b/chromium/media/filters/memory_data_source.cc @@ -6,7 +6,7 @@ #include <algorithm> -#include "base/logging.h" +#include "base/check.h" namespace media { diff --git a/chromium/media/filters/pipeline_controller.cc b/chromium/media/filters/pipeline_controller.cc index c8f2036bd01..c2f32d9ec8d 100644 --- a/chromium/media/filters/pipeline_controller.cc +++ b/chromium/media/filters/pipeline_controller.cc @@ -96,9 +96,13 @@ void PipelineController::Suspend() { void PipelineController::Resume() { DCHECK(thread_checker_.CalledOnValidThread()); pending_suspend_ = false; - if (state_ == State::SUSPENDING || state_ == State::SUSPENDED) { + // TODO(sandersd) fix resume during suspended start. + if (state_ == State::SUSPENDING || state_ == State::SUSPENDED || + (state_ == State::SWITCHING_TRACKS && + previous_track_change_state_ == State::SUSPENDED)) { pending_resume_ = true; Dispatch(); + return; } } @@ -279,7 +283,7 @@ void PipelineController::Dispatch() { // We can only switch tracks if we are not in a transitioning state already. if ((pending_audio_track_change_ || pending_video_track_change_) && (state_ == State::PLAYING || state_ == State::SUSPENDED)) { - State old_state = state_; + previous_track_change_state_ = state_; state_ = State::SWITCHING_TRACKS; // Attempt to do a track change _before_ attempting a seek operation, @@ -290,7 +294,7 @@ void PipelineController::Dispatch() { pipeline_->OnEnabledAudioTracksChanged( pending_audio_track_change_ids_, base::BindOnce(&PipelineController::OnTrackChangeComplete, - weak_factory_.GetWeakPtr(), old_state)); + weak_factory_.GetWeakPtr())); return; } @@ -299,7 +303,7 @@ void PipelineController::Dispatch() { pipeline_->OnSelectedVideoTrackChanged( pending_video_track_change_id_, base::BindOnce(&PipelineController::OnTrackChangeComplete, - weak_factory_.GetWeakPtr(), old_state)); + weak_factory_.GetWeakPtr())); return; } } @@ -431,14 +435,15 @@ void PipelineController::OnSelectedVideoTrackChanged( } void PipelineController::FireOnTrackChangeCompleteForTesting(State set_to) { - OnTrackChangeComplete(set_to); + previous_track_change_state_ = set_to; + OnTrackChangeComplete(); } -void PipelineController::OnTrackChangeComplete(State previous_state) { +void PipelineController::OnTrackChangeComplete() { DCHECK(thread_checker_.CalledOnValidThread()); if (state_ == State::SWITCHING_TRACKS) - state_ = previous_state; + state_ = previous_track_change_state_; // Other track changed or seek/suspend/resume, etc may be waiting. Dispatch(); diff --git a/chromium/media/filters/pipeline_controller.h b/chromium/media/filters/pipeline_controller.h index 81bd337d3da..52bb4757b4d 100644 --- a/chromium/media/filters/pipeline_controller.h +++ b/chromium/media/filters/pipeline_controller.h @@ -152,7 +152,7 @@ class MEDIA_EXPORT PipelineController { // PipelineStaus callback that also carries the target state. void OnPipelineStatus(State state, PipelineStatus pipeline_status); - void OnTrackChangeComplete(State previous_state); + void OnTrackChangeComplete(); // The Pipeline we are managing state for. std::unique_ptr<Pipeline> pipeline_; @@ -188,6 +188,10 @@ class MEDIA_EXPORT PipelineController { // Tracks the current state of |pipeline_|. State state_ = State::STOPPED; + // The previous state of |pipeline_| if it's currently undergoing a track + // change. + State previous_track_change_state_ = State::STOPPED; + // Indicates that a seek has occurred. When set, a seeked callback will be // issued at the next stable state. bool pending_seeked_cb_ = false; diff --git a/chromium/media/filters/pipeline_controller_unittest.cc b/chromium/media/filters/pipeline_controller_unittest.cc index bf9a7f99232..bf795f37fe8 100644 --- a/chromium/media/filters/pipeline_controller_unittest.cc +++ b/chromium/media/filters/pipeline_controller_unittest.cc @@ -8,9 +8,9 @@ #include "base/bind.h" #include "base/bind_helpers.h" -#include "base/logging.h" #include "base/macros.h" #include "base/memory/ref_counted.h" +#include "base/notreached.h" #include "base/run_loop.h" #include "base/test/gmock_callback_support.h" #include "base/test/gmock_move_support.h" @@ -535,4 +535,17 @@ TEST_F(PipelineControllerTest, SuspendDuringAudioTrackChange) { EXPECT_FALSE(was_resumed_); } +TEST_F(PipelineControllerTest, ResumePlaybackDuringSwitchingTracksState) { + Complete(StartPipeline()); + Complete(SuspendPipeline()); + EXPECT_CALL(*pipeline_, OnSelectedVideoTrackChanged(_, _)).Times(1); + EXPECT_CALL(*pipeline_, GetMediaTime()).Times(1); + EXPECT_CALL(*pipeline_, OnResume(_, _)).Times(1); + + pipeline_controller_.OnSelectedVideoTrackChanged({}); + pipeline_controller_.Resume(); + pipeline_controller_.FireOnTrackChangeCompleteForTesting( + PipelineController::State::SUSPENDED); +} + } // namespace media diff --git a/chromium/media/filters/video_cadence_estimator.cc b/chromium/media/filters/video_cadence_estimator.cc index 72f9963a770..8b8c2e8b5b2 100644 --- a/chromium/media/filters/video_cadence_estimator.cc +++ b/chromium/media/filters/video_cadence_estimator.cc @@ -11,6 +11,7 @@ #include <numeric> #include <string> +#include "base/logging.h" #include "base/metrics/histogram_macros.h" #include "media/base/media_switches.h" diff --git a/chromium/media/filters/vp9_raw_bits_reader.cc b/chromium/media/filters/vp9_raw_bits_reader.cc index 1fc8f775f63..f29a504cc2d 100644 --- a/chromium/media/filters/vp9_raw_bits_reader.cc +++ b/chromium/media/filters/vp9_raw_bits_reader.cc @@ -6,7 +6,7 @@ #include <limits.h> -#include "base/logging.h" +#include "base/check_op.h" #include "media/base/bit_reader.h" namespace media { diff --git a/chromium/media/filters/wsola_internals.cc b/chromium/media/filters/wsola_internals.cc index f750d87c3d2..dd269a67ef3 100644 --- a/chromium/media/filters/wsola_internals.cc +++ b/chromium/media/filters/wsola_internals.cc @@ -6,10 +6,11 @@ #include <algorithm> #include <cmath> +#include <cstring> #include <limits> #include <memory> -#include "base/logging.h" +#include "base/check_op.h" #include "base/numerics/math_constants.h" #include "build/build_config.h" #include "media/base/audio_bus.h" diff --git a/chromium/media/formats/mp2t/descriptors.cc b/chromium/media/formats/mp2t/descriptors.cc index 7d06dc972d2..075ea59ecce 100644 --- a/chromium/media/formats/mp2t/descriptors.cc +++ b/chromium/media/formats/mp2t/descriptors.cc @@ -6,7 +6,7 @@ #include <vector> -#include "base/logging.h" +#include "base/check.h" #include "media/base/bit_reader.h" #include "media/base/encryption_pattern.h" #include "media/formats/mp2t/mp2t_common.h" diff --git a/chromium/media/formats/mp2t/es_adapter_video_unittest.cc b/chromium/media/formats/mp2t/es_adapter_video_unittest.cc index 8d07d2967cb..f3afd2ac710 100644 --- a/chromium/media/formats/mp2t/es_adapter_video_unittest.cc +++ b/chromium/media/formats/mp2t/es_adapter_video_unittest.cc @@ -10,7 +10,6 @@ #include <vector> #include "base/bind.h" -#include "base/logging.h" #include "base/stl_util.h" #include "base/strings/string_util.h" #include "base/time/time.h" diff --git a/chromium/media/formats/mp2t/es_parser_adts_unittest.cc b/chromium/media/formats/mp2t/es_parser_adts_unittest.cc index 70b80b99a3f..5dbc26428c8 100644 --- a/chromium/media/formats/mp2t/es_parser_adts_unittest.cc +++ b/chromium/media/formats/mp2t/es_parser_adts_unittest.cc @@ -5,7 +5,6 @@ #include <vector> #include "base/bind.h" -#include "base/logging.h" #include "base/macros.h" #include "base/time/time.h" #include "media/base/stream_parser_buffer.h" diff --git a/chromium/media/formats/mp2t/es_parser_h264_unittest.cc b/chromium/media/formats/mp2t/es_parser_h264_unittest.cc index fad0a43f44a..05f57751b88 100644 --- a/chromium/media/formats/mp2t/es_parser_h264_unittest.cc +++ b/chromium/media/formats/mp2t/es_parser_h264_unittest.cc @@ -10,7 +10,7 @@ #include <vector> #include "base/bind.h" -#include "base/logging.h" +#include "base/check.h" #include "base/macros.h" #include "base/strings/string_util.h" #include "base/time/time.h" diff --git a/chromium/media/formats/mp2t/es_parser_mpeg1audio_unittest.cc b/chromium/media/formats/mp2t/es_parser_mpeg1audio_unittest.cc index f54cc6c6409..46477bf9487 100644 --- a/chromium/media/formats/mp2t/es_parser_mpeg1audio_unittest.cc +++ b/chromium/media/formats/mp2t/es_parser_mpeg1audio_unittest.cc @@ -5,7 +5,6 @@ #include <vector> #include "base/bind.h" -#include "base/logging.h" #include "base/macros.h" #include "base/time/time.h" #include "media/base/media_util.h" diff --git a/chromium/media/formats/mp2t/es_parser_test_base.cc b/chromium/media/formats/mp2t/es_parser_test_base.cc index 3d59a225313..157812f6f33 100644 --- a/chromium/media/formats/mp2t/es_parser_test_base.cc +++ b/chromium/media/formats/mp2t/es_parser_test_base.cc @@ -4,8 +4,8 @@ #include "media/formats/mp2t/es_parser_test_base.h" +#include "base/check_op.h" #include "base/files/memory_mapped_file.h" -#include "base/logging.h" #include "base/strings/string_util.h" #include "base/strings/stringprintf.h" #include "media/base/stream_parser_buffer.h" diff --git a/chromium/media/formats/mp2t/timestamp_unroller.cc b/chromium/media/formats/mp2t/timestamp_unroller.cc index 4c39c007ecd..31a994dabbe 100644 --- a/chromium/media/formats/mp2t/timestamp_unroller.cc +++ b/chromium/media/formats/mp2t/timestamp_unroller.cc @@ -4,7 +4,7 @@ #include "media/formats/mp2t/timestamp_unroller.h" -#include "base/logging.h" +#include "base/check_op.h" namespace media { namespace mp2t { diff --git a/chromium/media/formats/mp2t/timestamp_unroller_unittest.cc b/chromium/media/formats/mp2t/timestamp_unroller_unittest.cc index 8fd0e3fd12e..7c355831a94 100644 --- a/chromium/media/formats/mp2t/timestamp_unroller_unittest.cc +++ b/chromium/media/formats/mp2t/timestamp_unroller_unittest.cc @@ -6,7 +6,6 @@ #include <stdint.h> #include <vector> -#include "base/logging.h" #include "base/stl_util.h" #include "base/test/perf_test_suite.h" #include "media/formats/mp2t/timestamp_unroller.h" diff --git a/chromium/media/formats/mp2t/ts_section_cets_ecm.cc b/chromium/media/formats/mp2t/ts_section_cets_ecm.cc index a50dec10152..d458c06fb5c 100644 --- a/chromium/media/formats/mp2t/ts_section_cets_ecm.cc +++ b/chromium/media/formats/mp2t/ts_section_cets_ecm.cc @@ -4,7 +4,7 @@ #include "media/formats/mp2t/ts_section_cets_ecm.h" -#include "base/logging.h" +#include "base/check.h" #include "media/base/bit_reader.h" #include "media/formats/mp2t/mp2t_common.h" diff --git a/chromium/media/formats/mp2t/ts_section_cets_pssh.cc b/chromium/media/formats/mp2t/ts_section_cets_pssh.cc index 89f1d3aa537..976d4b53898 100644 --- a/chromium/media/formats/mp2t/ts_section_cets_pssh.cc +++ b/chromium/media/formats/mp2t/ts_section_cets_pssh.cc @@ -4,7 +4,7 @@ #include "media/formats/mp2t/ts_section_cets_pssh.h" -#include "base/logging.h" +#include "base/check.h" #include "media/base/bit_reader.h" #include "media/formats/mp2t/mp2t_common.h" diff --git a/chromium/media/formats/mp2t/ts_section_pmt.cc b/chromium/media/formats/mp2t/ts_section_pmt.cc index ed6be68b92a..4e8cbf0171e 100644 --- a/chromium/media/formats/mp2t/ts_section_pmt.cc +++ b/chromium/media/formats/mp2t/ts_section_pmt.cc @@ -6,7 +6,7 @@ #include <map> -#include "base/logging.h" +#include "base/check.h" #include "media/base/bit_reader.h" #include "media/formats/mp2t/mp2t_common.h" diff --git a/chromium/media/formats/mp4/mp4_box_reader_fuzzer.cc b/chromium/media/formats/mp4/mp4_box_reader_fuzzer.cc index 30ae9eba7cd..25a5e4eeb84 100644 --- a/chromium/media/formats/mp4/mp4_box_reader_fuzzer.cc +++ b/chromium/media/formats/mp4/mp4_box_reader_fuzzer.cc @@ -7,7 +7,7 @@ #include <memory> -#include "base/logging.h" +#include "base/check.h" #include "media/base/media_util.h" #include "media/formats/mp4/box_reader.h" diff --git a/chromium/media/formats/mp4/nalu_test_helper.cc b/chromium/media/formats/mp4/nalu_test_helper.cc index abdbc6142b1..acb81028e42 100644 --- a/chromium/media/formats/mp4/nalu_test_helper.cc +++ b/chromium/media/formats/mp4/nalu_test_helper.cc @@ -4,7 +4,7 @@ #include "media/formats/mp4/nalu_test_helper.h" -#include "base/logging.h" +#include "base/check_op.h" #include "base/strings/string_split.h" #include "base/strings/string_util.h" #include "media/video/h264_parser.h" diff --git a/chromium/media/formats/mp4/sample_to_group_iterator.cc b/chromium/media/formats/mp4/sample_to_group_iterator.cc index f5ce394aae2..b59916f42da 100644 --- a/chromium/media/formats/mp4/sample_to_group_iterator.cc +++ b/chromium/media/formats/mp4/sample_to_group_iterator.cc @@ -4,7 +4,7 @@ #include "media/formats/mp4/sample_to_group_iterator.h" -#include "base/logging.h" +#include "base/check.h" namespace media { namespace mp4 { diff --git a/chromium/media/formats/webm/cluster_builder.cc b/chromium/media/formats/webm/cluster_builder.cc index b685c2350a8..94c85d35107 100644 --- a/chromium/media/formats/webm/cluster_builder.cc +++ b/chromium/media/formats/webm/cluster_builder.cc @@ -7,7 +7,7 @@ #include <memory> #include <utility> -#include "base/logging.h" +#include "base/check_op.h" #include "media/base/data_buffer.h" #include "media/formats/webm/webm_constants.h" diff --git a/chromium/media/formats/webm/opus_packet_builder.cc b/chromium/media/formats/webm/opus_packet_builder.cc index c629bfbb183..9f71100df5d 100644 --- a/chromium/media/formats/webm/opus_packet_builder.cc +++ b/chromium/media/formats/webm/opus_packet_builder.cc @@ -4,7 +4,7 @@ #include "media/formats/webm/opus_packet_builder.h" -#include "base/logging.h" +#include "base/check_op.h" #include "media/formats/webm/webm_cluster_parser.h" namespace media { diff --git a/chromium/media/formats/webm/tracks_builder.cc b/chromium/media/formats/webm/tracks_builder.cc index 60c0abb7721..7ba64607f1f 100644 --- a/chromium/media/formats/webm/tracks_builder.cc +++ b/chromium/media/formats/webm/tracks_builder.cc @@ -4,7 +4,9 @@ #include "media/formats/webm/tracks_builder.h" -#include "base/logging.h" +#include <cstring> + +#include "base/check_op.h" #include "media/formats/webm/webm_constants.h" namespace media { diff --git a/chromium/media/formats/webm/webm_content_encodings.cc b/chromium/media/formats/webm/webm_content_encodings.cc index 9a03ea153ea..7c9aacaeeac 100644 --- a/chromium/media/formats/webm/webm_content_encodings.cc +++ b/chromium/media/formats/webm/webm_content_encodings.cc @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "base/logging.h" #include "media/formats/webm/webm_content_encodings.h" +#include "base/check_op.h" namespace media { diff --git a/chromium/media/formats/webm/webm_parser.cc b/chromium/media/formats/webm/webm_parser.cc index 39fc6c5c49f..f59bec22748 100644 --- a/chromium/media/formats/webm/webm_parser.cc +++ b/chromium/media/formats/webm/webm_parser.cc @@ -13,6 +13,7 @@ #include <stddef.h> +#include <cstring> #include <iomanip> #include <limits> diff --git a/chromium/media/formats/webm/webm_stream_parser.cc b/chromium/media/formats/webm/webm_stream_parser.cc index 9e3a0c90489..4a6799bd373 100644 --- a/chromium/media/formats/webm/webm_stream_parser.cc +++ b/chromium/media/formats/webm/webm_stream_parser.cc @@ -157,7 +157,6 @@ int WebMStreamParser::ParseInfoAndTracks(const uint8_t* data, int size) { } // Skip the element. return result + element_size; - break; case kWebMIdCluster: if (!cluster_parser_) { MEDIA_LOG(ERROR, media_log_) << "Found Cluster element before Info."; @@ -166,14 +165,12 @@ int WebMStreamParser::ParseInfoAndTracks(const uint8_t* data, int size) { ChangeState(kParsingClusters); new_segment_cb_.Run(); return 0; - break; case kWebMIdSegment: // Segment of unknown size indicates live stream. if (element_size == kWebMUnknownSize) unknown_segment_size_ = true; // Just consume the segment header. return result; - break; case kWebMIdInfo: // We've found the element we are looking for. break; diff --git a/chromium/media/fuchsia/audio/BUILD.gn b/chromium/media/fuchsia/audio/BUILD.gn index 039276f85ff..23c6d3db808 100644 --- a/chromium/media/fuchsia/audio/BUILD.gn +++ b/chromium/media/fuchsia/audio/BUILD.gn @@ -21,6 +21,20 @@ source_set("audio") { ] } +source_set("test_support") { + testonly = true + public_deps = [ + "//base", + "//third_party/fuchsia-sdk/sdk/fidl/fuchsia.media", + "//third_party/fuchsia-sdk/sdk/fidl/fuchsia.media.audio", + "//third_party/fuchsia-sdk/sdk/pkg/fidl_cpp", + ] + sources = [ + "fake_audio_consumer.cc", + "fake_audio_consumer.h", + ] +} + source_set("unittests") { testonly = true diff --git a/chromium/media/fuchsia/audio/fake_audio_consumer.cc b/chromium/media/fuchsia/audio/fake_audio_consumer.cc new file mode 100644 index 00000000000..f8368980332 --- /dev/null +++ b/chromium/media/fuchsia/audio/fake_audio_consumer.cc @@ -0,0 +1,268 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/fuchsia/audio/fake_audio_consumer.h" + +#include <lib/vfs/cpp/pseudo_dir.h> +#include <lib/vfs/cpp/service.h> + +#include "base/fuchsia/fuchsia_logging.h" + +namespace media { + +namespace { + +// Lead time range returned from WatchStatus(); +constexpr base::TimeDelta kMinLeadTime = base::TimeDelta::FromMilliseconds(100); +constexpr base::TimeDelta kMaxLeadTime = base::TimeDelta::FromMilliseconds(500); + +} // namespace + +// Buffering delay. +constexpr base::TimeDelta kBufferDelay = base::TimeDelta::FromMilliseconds(30); + +FakeAudioConsumer::FakeAudioConsumer( + uint64_t session_id, + fidl::InterfaceRequest<fuchsia::media::AudioConsumer> request) + : session_id_(session_id), + audio_consumer_binding_(this), + stream_sink_binding_(this), + volume_control_binding_(this) { + audio_consumer_binding_.Bind(std::move(request)); +} + +FakeAudioConsumer::~FakeAudioConsumer() = default; + +base::TimeDelta FakeAudioConsumer::GetMediaPosition() { + base::TimeDelta result = media_pos_; + if (state_ == State::kPlaying) { + result += (base::TimeTicks::Now() - reference_time_) * media_delta_ / + reference_delta_; + } + return result; +} + +void FakeAudioConsumer::CreateStreamSink( + std::vector<zx::vmo> buffers, + fuchsia::media::AudioStreamType stream_type, + std::unique_ptr<fuchsia::media::Compression> compression, + fidl::InterfaceRequest<fuchsia::media::StreamSink> stream_sink_request) { + num_buffers_ = buffers.size(); + CHECK_GT(num_buffers_, 0U); + stream_sink_binding_.Bind(std::move(stream_sink_request)); +} + +void FakeAudioConsumer::Start(fuchsia::media::AudioConsumerStartFlags flags, + int64_t reference_time, + int64_t media_time) { + CHECK(state_ == State::kStopped); + + if (reference_time != fuchsia::media::NO_TIMESTAMP) { + reference_time_ = base::TimeTicks::FromZxTime(reference_time); + } else { + reference_time_ = base::TimeTicks::Now() + kBufferDelay; + } + + if (media_time != fuchsia::media::NO_TIMESTAMP) { + media_pos_ = base::TimeDelta::FromZxDuration(media_time); + } else { + if (media_pos_.is_min()) { + media_pos_ = base::TimeDelta(); + } + } + + state_ = State::kPlaying; + + OnStatusUpdate(); + ScheduleNextStreamPosUpdate(); +} + +void FakeAudioConsumer::Stop() { + CHECK(state_ != State::kPlaying); + + state_ = State::kStopped; + OnStatusUpdate(); +} + +void FakeAudioConsumer::WatchStatus(WatchStatusCallback callback) { + status_callback_ = std::move(callback); + if (have_status_update_) { + CallStatusCallback(); + } +} + +void FakeAudioConsumer::SetRate(float rate) { + // Playback rate must not be negative. + CHECK_GE(rate, 0.0); + + // Update reference position. + auto now = base::TimeTicks::Now(); + media_pos_ = + media_pos_ + (now - reference_time_) * media_delta_ / reference_delta_; + reference_time_ = now; + + // Approximate the rate as n/1000; + reference_delta_ = 1000; + media_delta_ = static_cast<int>(rate * 1000.0); + + OnStatusUpdate(); + + if (update_timer_.IsRunning()) + update_timer_.Reset(); + ScheduleNextStreamPosUpdate(); +} + +void FakeAudioConsumer::BindVolumeControl( + fidl::InterfaceRequest<fuchsia::media::audio::VolumeControl> + volume_control_request) { + volume_control_binding_.Bind(std::move(volume_control_request)); +} + +void FakeAudioConsumer::SendPacket(fuchsia::media::StreamPacket stream_packet, + SendPacketCallback callback) { + CHECK_LT(stream_packet.payload_buffer_id, num_buffers_); + + Packet packet; + if (stream_packet.pts == fuchsia::media::NO_TIMESTAMP) { + if (media_pos_.is_min()) { + packet.pts = base::TimeDelta(); + } else { + packet.pts = media_pos_; + } + } else { + packet.pts = base::TimeDelta::FromZxDuration(stream_packet.pts); + } + pending_packets_.push_back(std::move(packet)); + + callback(); + + ScheduleNextStreamPosUpdate(); +} + +void FakeAudioConsumer::SendPacketNoReply(fuchsia::media::StreamPacket packet) { + NOTREACHED(); +} + +void FakeAudioConsumer::EndOfStream() { + Packet packet; + packet.is_eos = true; + pending_packets_.push_back(std::move(packet)); +} + +void FakeAudioConsumer::DiscardAllPackets(DiscardAllPacketsCallback callback) { + DiscardAllPacketsNoReply(); + std::move(callback)(); +} + +void FakeAudioConsumer::DiscardAllPacketsNoReply() { + pending_packets_.clear(); +} + +void FakeAudioConsumer::SetVolume(float volume) { + volume_ = volume; +} + +void FakeAudioConsumer::SetMute(bool mute) { + is_muted_ = mute; +} + +void FakeAudioConsumer::NotImplemented_(const std::string& name) { + LOG(FATAL) << "Reached non-implemented " << name; +} + +void FakeAudioConsumer::ScheduleNextStreamPosUpdate() { + if (pending_packets_.empty() || update_timer_.IsRunning() || + media_delta_ == 0 || state_ != State::kPlaying) { + return; + } + base::TimeDelta delay; + if (!pending_packets_.front().is_eos) { + auto next_packet_time = + reference_time_ + (pending_packets_.front().pts - media_pos_) * + reference_delta_ / media_delta_; + delay = (next_packet_time - base::TimeTicks::Now()); + } + update_timer_.Start(FROM_HERE, delay, + base::BindOnce(&FakeAudioConsumer::UpdateStreamPos, + base::Unretained(this))); +} + +void FakeAudioConsumer::UpdateStreamPos() { + if (state_ != State::kPlaying) + return; + + auto now = base::TimeTicks::Now(); + auto new_media_pos = + media_pos_ + (now - reference_time_) * media_delta_ / reference_delta_; + + // Drop all packets with PTS before the current position. + while (!pending_packets_.empty()) { + if (!pending_packets_.front().is_eos && + pending_packets_.front().pts > new_media_pos) { + break; + } + + Packet packet = pending_packets_.front(); + pending_packets_.pop_front(); + + if (packet.is_eos) { + // No data should be submitted after EOS. + CHECK(pending_packets_.empty()); + audio_consumer_binding_.events().OnEndOfStream(); + state_ = State::kEndOfStream; + media_pos_ = new_media_pos; + reference_time_ = now; + } + } + + ScheduleNextStreamPosUpdate(); +} + +void FakeAudioConsumer::OnStatusUpdate() { + have_status_update_ = true; + if (status_callback_) { + CallStatusCallback(); + } +} + +void FakeAudioConsumer::CallStatusCallback() { + DCHECK(status_callback_); + DCHECK(have_status_update_); + + fuchsia::media::AudioConsumerStatus status; + if (state_ == State::kPlaying) { + fuchsia::media::TimelineFunction timeline; + timeline.reference_time = reference_time_.ToZxTime(); + timeline.subject_time = media_pos_.ToZxDuration(); + timeline.reference_delta = reference_delta_; + timeline.subject_delta = media_delta_; + + status.set_presentation_timeline(std::move(timeline)); + } + + status.set_min_lead_time(kMinLeadTime.ToZxDuration()); + status.set_max_lead_time(kMaxLeadTime.ToZxDuration()); + + have_status_update_ = false; + std::move(status_callback_)(std::move(status)); + status_callback_ = {}; +} + +FakeAudioConsumerService::FakeAudioConsumerService(vfs::PseudoDir* pseudo_dir) + : binding_(pseudo_dir, this) {} + +FakeAudioConsumerService::~FakeAudioConsumerService() {} + +void FakeAudioConsumerService::CreateAudioConsumer( + uint64_t session_id, + fidl::InterfaceRequest<fuchsia::media::AudioConsumer> request) { + audio_consumers_.push_back( + std::make_unique<FakeAudioConsumer>(session_id, std::move(request))); +} + +void FakeAudioConsumerService::NotImplemented_(const std::string& name) { + LOG(FATAL) << "Reached non-implemented " << name; +} + +} // namespace media
\ No newline at end of file diff --git a/chromium/media/fuchsia/audio/fake_audio_consumer.h b/chromium/media/fuchsia/audio/fake_audio_consumer.h new file mode 100644 index 00000000000..56615269cbd --- /dev/null +++ b/chromium/media/fuchsia/audio/fake_audio_consumer.h @@ -0,0 +1,164 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_FUCHSIA_AUDIO_FAKE_AUDIO_CONSUMER_H_ +#define MEDIA_FUCHSIA_AUDIO_FAKE_AUDIO_CONSUMER_H_ + +#include <fuchsia/media/audio/cpp/fidl.h> +#include <fuchsia/media/audio/cpp/fidl_test_base.h> +#include <fuchsia/media/cpp/fidl.h> +#include <fuchsia/media/cpp/fidl_test_base.h> +#include <lib/fidl/cpp/binding.h> + +#include <vector> + +#include "base/fuchsia/scoped_service_binding.h" +#include "base/time/time.h" +#include "base/timer/timer.h" + +namespace vfs { +class PseudoDir; +} // namespace vfs + +namespace media { + +// Fake implementation of fuchsia::media::AudioConsumer interface. Used for +// tests. +class FakeAudioConsumer + : public fuchsia::media::testing::AudioConsumer_TestBase, + public fuchsia::media::testing::StreamSink_TestBase, + public fuchsia::media::audio::testing::VolumeControl_TestBase { + public: + FakeAudioConsumer( + uint64_t session_id, + fidl::InterfaceRequest<fuchsia::media::AudioConsumer> request); + ~FakeAudioConsumer() final; + + FakeAudioConsumer(const FakeAudioConsumer&) = delete; + FakeAudioConsumer& operator=(const FakeAudioConsumer&) = delete; + + uint64_t session_id() { return session_id_; } + float volume() const { return volume_; } + bool is_muted() const { return is_muted_; } + + base::TimeDelta GetMediaPosition(); + + private: + enum class State { + kStopped, + kPlaying, + kEndOfStream, + }; + + struct Packet { + base::TimeDelta pts; + bool is_eos = false; + }; + + // fuchsia::media::AudioConsumer interface; + void CreateStreamSink( + std::vector<zx::vmo> buffers, + fuchsia::media::AudioStreamType stream_type, + std::unique_ptr<fuchsia::media::Compression> compression, + fidl::InterfaceRequest<fuchsia::media::StreamSink> stream_sink_request) + final; + void Start(fuchsia::media::AudioConsumerStartFlags flags, + int64_t reference_time, + int64_t media_time) final; + void Stop() final; + void WatchStatus(WatchStatusCallback callback) final; + void SetRate(float rate) final; + void BindVolumeControl( + fidl::InterfaceRequest<fuchsia::media::audio::VolumeControl> + volume_control_request) final; + + // fuchsia::media::StreamSink interface. + void SendPacket(fuchsia::media::StreamPacket packet, + SendPacketCallback callback) final; + void SendPacketNoReply(fuchsia::media::StreamPacket packet) final; + void EndOfStream() final; + void DiscardAllPackets(DiscardAllPacketsCallback callback) final; + void DiscardAllPacketsNoReply() final; + + // fuchsia::media::audio::VolumeControl interface. + void SetVolume(float volume) final; + void SetMute(bool mute) final; + + // Not-implemented handler for _TestBase parents. + void NotImplemented_(const std::string& name) final; + + void ScheduleNextStreamPosUpdate(); + + // Updates stream position and drops old packets from the stream. + void UpdateStreamPos(); + + void OnStatusUpdate(); + void CallStatusCallback(); + + const uint64_t session_id_; + + fidl::Binding<fuchsia::media::AudioConsumer> audio_consumer_binding_; + fidl::Binding<fuchsia::media::StreamSink> stream_sink_binding_; + fidl::Binding<fuchsia::media::audio::VolumeControl> volume_control_binding_; + + size_t num_buffers_ = 0; + + State state_ = State::kStopped; + + bool have_status_update_ = true; + WatchStatusCallback status_callback_; + + base::TimeTicks reference_time_; + + // Numerator and denumerator for current playback rate. + uint32_t media_delta_ = 1; + uint32_t reference_delta_ = 1; + + // Last known media position. Min value indicates that the stream position + // hasn't been set. If stream is playing then value corresponds to + // |reference_time_|. + base::TimeDelta media_pos_ = base::TimeDelta::Min(); + + std::list<Packet> pending_packets_; + + // Timer to call UpdateStreamPos() for the next packet. + base::OneShotTimer update_timer_; + + float volume_ = 1.0; + bool is_muted_ = false; +}; + +class FakeAudioConsumerService + : public fuchsia::media::testing::SessionAudioConsumerFactory_TestBase { + public: + explicit FakeAudioConsumerService(vfs::PseudoDir* pseudo_dir); + ~FakeAudioConsumerService() final; + + FakeAudioConsumerService(const FakeAudioConsumerService&) = delete; + FakeAudioConsumerService& operator=(const FakeAudioConsumerService&) = delete; + + size_t num_instances() { return audio_consumers_.size(); } + FakeAudioConsumer* instance(size_t index) { + return audio_consumers_[index].get(); + } + + private: + // fuchsia::media::SessionAudioConsumerFactory implementation. + void CreateAudioConsumer(uint64_t session_id, + fidl::InterfaceRequest<fuchsia::media::AudioConsumer> + audio_consumer_request) final; + + // Not-implemented handler for SessionAudioConsumerFactory_TestBase. + void NotImplemented_(const std::string& name) final; + + base::fuchsia::ScopedServiceBinding< + fuchsia::media::SessionAudioConsumerFactory> + binding_; + + std::vector<std::unique_ptr<FakeAudioConsumer>> audio_consumers_; +}; + +} // namespace media + +#endif // MEDIA_FUCHSIA_AUDIO_FAKE_AUDIO_CONSUMER_H_ diff --git a/chromium/media/fuchsia/audio/fuchsia_audio_renderer.cc b/chromium/media/fuchsia/audio/fuchsia_audio_renderer.cc index 9c1c262e547..b4f6476516c 100644 --- a/chromium/media/fuchsia/audio/fuchsia_audio_renderer.cc +++ b/chromium/media/fuchsia/audio/fuchsia_audio_renderer.cc @@ -254,7 +254,6 @@ void FuchsiaAudioRenderer::StartTicking() { SetPlaybackState(PlaybackState::kStarting); } - audio_consumer_->Start(flags, fuchsia::media::NO_TIMESTAMP, media_pos.ToZxDuration()); } @@ -528,6 +527,13 @@ void FuchsiaAudioRenderer::OnDemuxerStreamReadDone( OnStreamSendDone(buffer_index); }); + // AudioConsumer doesn't report exact time when the data is decoded, but it's + // safe to report it as decoded right away since the packet is expected to be + // decoded soon after AudioConsumer receives it. + PipelineStatistics stats; + stats.audio_bytes_decoded = buffer->data_size(); + client_->OnStatisticsUpdate(stats); + last_packet_timestamp_ = buffer->timestamp(); ScheduleReadDemuxerStream(); diff --git a/chromium/media/fuchsia/camera/BUILD.gn b/chromium/media/fuchsia/camera/BUILD.gn new file mode 100644 index 00000000000..c8ec86f4a93 --- /dev/null +++ b/chromium/media/fuchsia/camera/BUILD.gn @@ -0,0 +1,19 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +source_set("test_support") { + testonly = true + public_deps = [ + "//base", + "//testing/gtest", + "//third_party/fuchsia-sdk/sdk/fidl/fuchsia.camera3", + "//third_party/fuchsia-sdk/sdk/fidl/fuchsia.sysmem", + "//third_party/fuchsia-sdk/sdk/pkg/fidl_cpp", + "//ui/gfx/geometry", + ] + sources = [ + "fake_fuchsia_camera.cc", + "fake_fuchsia_camera.h", + ] +} diff --git a/chromium/media/fuchsia/camera/fake_fuchsia_camera.cc b/chromium/media/fuchsia/camera/fake_fuchsia_camera.cc new file mode 100644 index 00000000000..921a1b6cfb1 --- /dev/null +++ b/chromium/media/fuchsia/camera/fake_fuchsia_camera.cc @@ -0,0 +1,517 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/fuchsia/camera/fake_fuchsia_camera.h" + +#include <fuchsia/sysmem/cpp/fidl.h> +#include <lib/sys/cpp/component_context.h> + +#include "base/fuchsia/default_context.h" +#include "base/memory/platform_shared_memory_region.h" +#include "base/memory/writable_shared_memory_region.h" +#include "base/message_loop/message_loop_current.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace media { + +namespace { + +constexpr uint64_t kDefaultFakeDeviceId = 42; + +constexpr uint8_t kYPlaneSalt = 1; +constexpr uint8_t kUPlaneSalt = 2; +constexpr uint8_t kVPlaneSalt = 3; + +uint8_t GetTestFrameValue(gfx::Size size, int x, int y, uint8_t salt) { + return static_cast<uint8_t>(y + x * size.height() + salt); +} + +// Fills one plane of a test frame. |data| points at the location of the pixel +// (0, 0). |orientation| specifies frame orientation transformation that will be +// applied on the receiving end, so this function applies _reverse_ of the +// |orientation| transformation. +void FillPlane(uint8_t* data, + gfx::Size size, + int x_step, + int y_step, + fuchsia::camera3::Orientation orientation, + uint8_t salt) { + // First flip X axis for flipped orientation. + if (orientation == fuchsia::camera3::Orientation::UP_FLIPPED || + orientation == fuchsia::camera3::Orientation::DOWN_FLIPPED || + orientation == fuchsia::camera3::Orientation::RIGHT_FLIPPED || + orientation == fuchsia::camera3::Orientation::LEFT_FLIPPED) { + // Move the origin to the top right corner and flip the X axis. + data += (size.width() - 1) * x_step; + x_step = -x_step; + } + + switch (orientation) { + case fuchsia::camera3::Orientation::UP: + case fuchsia::camera3::Orientation::UP_FLIPPED: + break; + + case fuchsia::camera3::Orientation::DOWN: + case fuchsia::camera3::Orientation::DOWN_FLIPPED: + // Move |data| to point to the bottom right corner and reverse direction + // of both axes. + data += (size.width() - 1) * x_step + (size.height() - 1) * y_step; + x_step = -x_step; + y_step = -y_step; + break; + + case fuchsia::camera3::Orientation::LEFT: + case fuchsia::camera3::Orientation::LEFT_FLIPPED: + // Rotate 90 degrees clockwise by moving |data| to point to the right top + // corner, swapping the axes and reversing direction of the Y axis. + data += (size.width() - 1) * x_step; + size = gfx::Size(size.height(), size.width()); + std::swap(x_step, y_step); + y_step = -y_step; + break; + + case fuchsia::camera3::Orientation::RIGHT: + case fuchsia::camera3::Orientation::RIGHT_FLIPPED: + // Rotate 90 degrees counter-clockwise by moving |data| to point to the + // bottom left corner, swapping the axes and reversing direction of the X + // axis. + data += (size.height() - 1) * y_step; + size = gfx::Size(size.height(), size.width()); + std::swap(x_step, y_step); + x_step = -x_step; + break; + } + + for (int y = 0; y < size.height(); ++y) { + for (int x = 0; x < size.width(); ++x) { + data[x * x_step + y * y_step] = GetTestFrameValue(size, x, y, salt); + } + } +} + +void ValidatePlane(const uint8_t* data, + gfx::Size size, + size_t x_step, + size_t y_step, + uint8_t salt) { + for (int y = 0; y < size.height(); ++y) { + for (int x = 0; x < size.width(); ++x) { + SCOPED_TRACE(testing::Message() << "x=" << x << " y=" << y); + EXPECT_EQ(data[x * x_step + y * y_step], + GetTestFrameValue(size, x, y, salt)); + } + } +} + +} // namespace + +// static +const gfx::Size FakeCameraStream::kMaxFrameSize = gfx::Size(100, 60); +// static +const gfx::Size FakeCameraStream::kDefaultFrameSize = gfx::Size(60, 40); + +// static +void FakeCameraStream::ValidateFrameData(const uint8_t* data, + gfx::Size size, + uint8_t salt) { + const uint8_t* y_plane = data; + { + SCOPED_TRACE("Y plane"); + ValidatePlane(y_plane, size, 1, size.width(), salt + kYPlaneSalt); + } + + gfx::Size uv_size(size.width() / 2, size.height() / 2); + const uint8_t* u_plane = y_plane + size.width() * size.height(); + { + SCOPED_TRACE("U plane"); + ValidatePlane(u_plane, uv_size, 1, uv_size.width(), salt + kUPlaneSalt); + } + + const uint8_t* v_plane = u_plane + uv_size.width() * uv_size.height(); + { + SCOPED_TRACE("V plane"); + ValidatePlane(v_plane, uv_size, 1, uv_size.width(), salt + kVPlaneSalt); + } +} + +struct FakeCameraStream::Buffer { + explicit Buffer(base::WritableSharedMemoryMapping mapping) + : mapping(std::move(mapping)), + release_fence_watch_controller(FROM_HERE) {} + + base::WritableSharedMemoryMapping mapping; + + // Frame is used by the client when the |release_fence| is not null. + zx::eventpair release_fence; + + base::MessagePumpForIO::ZxHandleWatchController + release_fence_watch_controller; +}; + +FakeCameraStream::FakeCameraStream() : binding_(this) {} +FakeCameraStream::~FakeCameraStream() = default; + +void FakeCameraStream::Bind( + fidl::InterfaceRequest<fuchsia::camera3::Stream> request) { + binding_.Bind(std::move(request)); +} + +void FakeCameraStream::SetFakeResolution(gfx::Size resolution) { + resolution_ = resolution; + resolution_update_ = + fuchsia::math::Size{resolution_.width(), resolution_.height()}; + SendResolution(); +} + +void FakeCameraStream::SetFakeOrientation( + fuchsia::camera3::Orientation orientation) { + orientation_ = orientation; + orientation_update_ = orientation; + SendOrientation(); +} + +bool FakeCameraStream::WaitBuffersAllocated() { + EXPECT_FALSE(wait_buffers_allocated_run_loop_); + + if (!buffers_.empty()) + return true; + + wait_buffers_allocated_run_loop_.emplace(); + wait_buffers_allocated_run_loop_->Run(); + wait_buffers_allocated_run_loop_.reset(); + + return !buffers_.empty(); +} + +bool FakeCameraStream::WaitFreeBuffer() { + EXPECT_FALSE(wait_free_buffer_run_loop_); + + if (num_used_buffers_ < buffers_.size()) + return true; + + wait_free_buffer_run_loop_.emplace(); + wait_free_buffer_run_loop_->Run(); + wait_free_buffer_run_loop_.reset(); + + return num_used_buffers_ < buffers_.size(); +} + +void FakeCameraStream::ProduceFrame(base::TimeTicks timestamp, uint8_t salt) { + ASSERT_LT(num_used_buffers_, buffers_.size()); + ASSERT_FALSE(next_frame_); + + size_t index = buffers_.size(); + for (size_t i = 0; i < buffers_.size(); ++i) { + if (!buffers_[i]->release_fence) { + index = i; + break; + } + } + EXPECT_LT(index, buffers_.size()); + + auto* buffer = buffers_[index].get(); + + gfx::Size coded_size((resolution_.width() + 1) & ~1, + (resolution_.height() + 1) & ~1); + + // Fill Y plane. + uint8_t* y_plane = reinterpret_cast<uint8_t*>(buffer->mapping.memory()); + size_t stride = kMaxFrameSize.width(); + FillPlane(y_plane, coded_size, /*x_step=*/1, /*y_step=*/stride, orientation_, + salt + kYPlaneSalt); + + // Fill UV plane. + gfx::Size uv_size(coded_size.width() / 2, coded_size.height() / 2); + uint8_t* uv_plane = y_plane + kMaxFrameSize.width() * kMaxFrameSize.height(); + FillPlane(uv_plane, uv_size, /*x_step=*/2, /*y_step=*/stride, orientation_, + salt + kUPlaneSalt); + FillPlane(uv_plane + 1, uv_size, /*x_step=*/2, /*y_step=*/stride, + orientation_, salt + kVPlaneSalt); + + // Create FrameInfo. + fuchsia::camera3::FrameInfo frame; + frame.frame_counter = frame_counter_++; + frame.buffer_index = 0; + frame.timestamp = timestamp.ToZxTime(); + EXPECT_EQ( + zx::eventpair::create(0u, &frame.release_fence, &buffer->release_fence), + ZX_OK); + + // Watch release fence to get notified when the frame is released. + base::MessageLoopCurrentForIO::Get()->WatchZxHandle( + buffer->release_fence.get(), /*persistent=*/false, + ZX_EVENTPAIR_PEER_CLOSED, &buffer->release_fence_watch_controller, this); + + num_used_buffers_++; + next_frame_ = std::move(frame); + SendNextFrame(); +} + +void FakeCameraStream::WatchResolution(WatchResolutionCallback callback) { + EXPECT_FALSE(watch_resolution_callback_); + watch_resolution_callback_ = std::move(callback); + SendResolution(); +} + +void FakeCameraStream::WatchOrientation(WatchOrientationCallback callback) { + EXPECT_FALSE(watch_orientation_callback_); + watch_orientation_callback_ = std::move(callback); + SendOrientation(); +} + +void FakeCameraStream::SetBufferCollection( + fidl::InterfaceHandle<fuchsia::sysmem::BufferCollectionToken> + token_handle) { + EXPECT_TRUE(token_handle); + + // Drop old buffers. + buffers_.clear(); + if (buffer_collection_) { + buffer_collection_->Close(); + buffer_collection_.Unbind(); + } + + // Use a SyncPtr to be able to wait for Sync() synchronously. + fuchsia::sysmem::BufferCollectionTokenSyncPtr token; + token.Bind(std::move(token_handle)); + + // Duplicate the token to access from the stream. + fidl::InterfaceHandle<fuchsia::sysmem::BufferCollectionToken> local_token; + zx_status_t status = + token->Duplicate(/*rights_attenuation_mask=*/0, local_token.NewRequest()); + EXPECT_EQ(status, ZX_OK); + + status = token->Sync(); + EXPECT_EQ(status, ZX_OK); + + // Return the token back to the client. + new_buffer_collection_token_ = token.Unbind(); + SendBufferCollection(); + + // Initialize the new collection using |local_token|. + auto allocator = base::fuchsia::ComponentContextForCurrentProcess() + ->svc() + ->Connect<fuchsia::sysmem::Allocator>(); + + allocator->BindSharedCollection(std::move(local_token), + buffer_collection_.NewRequest()); + EXPECT_EQ(status, ZX_OK); + + buffer_collection_.set_error_handler( + fit::bind_member(this, &FakeCameraStream::OnBufferCollectionError)); + + fuchsia::sysmem::BufferCollectionConstraints constraints; + constraints.usage.cpu = + fuchsia::sysmem::cpuUsageRead | fuchsia::sysmem::cpuUsageWrite; + + // The client is expected to request buffers it may need. We don't need to + // reserve any for the server side. + constraints.min_buffer_count_for_camping = 0; + + // Initialize image format. + constraints.image_format_constraints_count = 1; + constraints.image_format_constraints[0].pixel_format.type = + fuchsia::sysmem::PixelFormatType::NV12; + constraints.image_format_constraints[0].color_spaces_count = 1; + constraints.image_format_constraints[0].color_space[0].type = + fuchsia::sysmem::ColorSpaceType::REC601_NTSC; + constraints.image_format_constraints[0].required_max_coded_width = + kMaxFrameSize.width(); + constraints.image_format_constraints[0].required_max_coded_height = + kMaxFrameSize.height(); + + buffer_collection_->SetConstraints(/*has_constraints=*/true, + std::move(constraints)); + buffer_collection_->WaitForBuffersAllocated( + fit::bind_member(this, &FakeCameraStream::OnBufferCollectionAllocated)); +} + +void FakeCameraStream::WatchBufferCollection( + WatchBufferCollectionCallback callback) { + EXPECT_FALSE(watch_buffer_collection_callback_); + watch_buffer_collection_callback_ = std::move(callback); + SendBufferCollection(); +} + +void FakeCameraStream::GetNextFrame(GetNextFrameCallback callback) { + EXPECT_FALSE(get_next_frame_callback_); + get_next_frame_callback_ = std::move(callback); + SendNextFrame(); +} + +void FakeCameraStream::NotImplemented_(const std::string& name) { + ADD_FAILURE() << "NotImplemented_: " << name; +} + +void FakeCameraStream::OnBufferCollectionError(zx_status_t status) { + ADD_FAILURE() << "BufferCollection failed."; + if (wait_buffers_allocated_run_loop_) + wait_buffers_allocated_run_loop_->Quit(); + if (wait_free_buffer_run_loop_) + wait_free_buffer_run_loop_->Quit(); +} + +void FakeCameraStream::OnBufferCollectionAllocated( + zx_status_t status, + fuchsia::sysmem::BufferCollectionInfo_2 buffer_collection_info) { + if (status != ZX_OK) { + OnBufferCollectionError(status); + return; + } + + EXPECT_TRUE(buffers_.empty()); + EXPECT_TRUE(buffer_collection_info.settings.has_image_format_constraints); + EXPECT_EQ(buffer_collection_info.settings.image_format_constraints + .pixel_format.type, + fuchsia::sysmem::PixelFormatType::NV12); + + size_t buffer_size = + buffer_collection_info.settings.buffer_settings.size_bytes; + for (size_t i = 0; i < buffer_collection_info.buffer_count; ++i) { + auto& buffer = buffer_collection_info.buffers[i]; + EXPECT_EQ(buffer.vmo_usable_start, 0U); + auto region = base::WritableSharedMemoryRegion::Deserialize( + base::subtle::PlatformSharedMemoryRegion::Take( + std::move(buffer.vmo), + base::subtle::PlatformSharedMemoryRegion::Mode::kWritable, + buffer_size, base::UnguessableToken::Create())); + auto mapping = region.Map(); + EXPECT_TRUE(mapping.IsValid()); + buffers_.push_back(std::make_unique<Buffer>(std::move(mapping))); + } + + if (wait_buffers_allocated_run_loop_) + wait_buffers_allocated_run_loop_->Quit(); +} + +void FakeCameraStream::SendResolution() { + if (!watch_resolution_callback_ || !resolution_update_) + return; + watch_resolution_callback_(resolution_update_.value()); + watch_resolution_callback_ = {}; + resolution_update_.reset(); +} + +void FakeCameraStream::SendOrientation() { + if (!watch_orientation_callback_ || !orientation_update_) + return; + watch_orientation_callback_(orientation_update_.value()); + watch_orientation_callback_ = {}; + orientation_update_.reset(); +} + +void FakeCameraStream::SendBufferCollection() { + if (!watch_buffer_collection_callback_ || !new_buffer_collection_token_) + return; + watch_buffer_collection_callback_( + std::move(new_buffer_collection_token_.value())); + watch_buffer_collection_callback_ = {}; + new_buffer_collection_token_.reset(); +} + +void FakeCameraStream::SendNextFrame() { + if (!get_next_frame_callback_ || !next_frame_) + return; + get_next_frame_callback_(std::move(next_frame_.value())); + get_next_frame_callback_ = {}; + next_frame_.reset(); +} + +void FakeCameraStream::OnZxHandleSignalled(zx_handle_t handle, + zx_signals_t signals) { + EXPECT_EQ(signals, ZX_EVENTPAIR_PEER_CLOSED); + + // Find the buffer that corresponds to the |handle|. + size_t index = buffers_.size(); + for (size_t i = 0; i < buffers_.size(); ++i) { + if (buffers_[i]->release_fence.get() == handle) { + index = i; + break; + } + } + ASSERT_LT(index, buffers_.size()); + buffers_[index]->release_fence = {}; + buffers_[index]->release_fence_watch_controller.StopWatchingZxHandle(); + num_used_buffers_--; + + if (wait_free_buffer_run_loop_) + wait_free_buffer_run_loop_->Quit(); +} +FakeCameraDevice::FakeCameraDevice(FakeCameraStream* stream) + : binding_(this), stream_(stream) {} + +FakeCameraDevice::~FakeCameraDevice() = default; + +void FakeCameraDevice::Bind( + fidl::InterfaceRequest<fuchsia::camera3::Device> request) { + binding_.Bind(std::move(request)); +} + +void FakeCameraDevice::GetIdentifier(GetIdentifierCallback callback) { + callback("Fake Camera"); +} + +void FakeCameraDevice::GetConfigurations(GetConfigurationsCallback callback) { + std::vector<fuchsia::camera3::Configuration> configurations(1); + configurations[0].streams.resize(1); + configurations[0].streams[0].frame_rate.numerator = 30; + configurations[0].streams[0].frame_rate.denominator = 1; + configurations[0].streams[0].image_format.pixel_format.type = + fuchsia::sysmem::PixelFormatType::NV12; + configurations[0].streams[0].image_format.coded_width = 640; + configurations[0].streams[0].image_format.coded_height = 480; + configurations[0].streams[0].image_format.bytes_per_row = 640; + callback(std::move(configurations)); +} + +void FakeCameraDevice::ConnectToStream( + uint32_t index, + fidl::InterfaceRequest<fuchsia::camera3::Stream> request) { + EXPECT_EQ(index, 0U); + stream_->Bind(std::move(request)); +} + +void FakeCameraDevice::NotImplemented_(const std::string& name) { + ADD_FAILURE() << "NotImplemented_: " << name; +} + +FakeCameraDeviceWatcher::FakeCameraDeviceWatcher( + sys::OutgoingDirectory* outgoing_directory) { + outgoing_directory->AddPublicService<fuchsia::camera3::DeviceWatcher>( + [this](fidl::InterfaceRequest<fuchsia::camera3::DeviceWatcher> request) { + bindings_.AddBinding(std::make_unique<Client>(&device_), + std::move(request)); + }); +} + +FakeCameraDeviceWatcher::~FakeCameraDeviceWatcher() = default; + +FakeCameraDeviceWatcher::Client::Client(FakeCameraDevice* device) + : device_(device) {} +FakeCameraDeviceWatcher::Client::~Client() {} + +void FakeCameraDeviceWatcher::Client::WatchDevices( + WatchDevicesCallback callback) { + if (devices_sent_) + return; + + std::vector<fuchsia::camera3::WatchDevicesEvent> events(1); + events[0].set_added(kDefaultFakeDeviceId); + callback(std::move(events)); + + devices_sent_ = true; +} + +void FakeCameraDeviceWatcher::Client::ConnectToDevice( + uint64_t id, + fidl::InterfaceRequest<fuchsia::camera3::Device> request) { + if (id == kDefaultFakeDeviceId) + device_->Bind(std::move(request)); +} + +void FakeCameraDeviceWatcher::Client::NotImplemented_(const std::string& name) { + ADD_FAILURE() << "NotImplemented_: " << name; +} + +} // namespace media
\ No newline at end of file diff --git a/chromium/media/fuchsia/camera/fake_fuchsia_camera.h b/chromium/media/fuchsia/camera/fake_fuchsia_camera.h new file mode 100644 index 00000000000..016c68d0fc1 --- /dev/null +++ b/chromium/media/fuchsia/camera/fake_fuchsia_camera.h @@ -0,0 +1,194 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_FUCHSIA_CAMERA_FAKE_FUCHSIA_CAMERA_H_ +#define MEDIA_FUCHSIA_CAMERA_FAKE_FUCHSIA_CAMERA_H_ + +#include <fuchsia/camera3/cpp/fidl.h> +#include <fuchsia/camera3/cpp/fidl_test_base.h> +#include <lib/fidl/cpp/binding.h> +#include <lib/fidl/cpp/binding_set.h> +#include <lib/sys/cpp/outgoing_directory.h> + +#include <vector> + +#include "base/message_loop/message_pump_for_io.h" +#include "base/optional.h" +#include "base/run_loop.h" +#include "ui/gfx/geometry/size.h" + +namespace media { + +class FakeCameraStream : public fuchsia::camera3::testing::Stream_TestBase, + public base::MessagePumpForIO::ZxHandleWatcher { + public: + static const gfx::Size kMaxFrameSize; + static const gfx::Size kDefaultFrameSize; + + // Verifies that the I420 image stored at |data| matches the frame produced + // by ProduceFrame(). + static void ValidateFrameData(const uint8_t* data, + gfx::Size size, + uint8_t salt); + + FakeCameraStream(); + ~FakeCameraStream() final; + + FakeCameraStream(const FakeCameraStream&) = delete; + FakeCameraStream& operator=(const FakeCameraStream&) = delete; + + void Bind(fidl::InterfaceRequest<fuchsia::camera3::Stream> request); + + void SetFakeResolution(gfx::Size resolution); + void SetFakeOrientation(fuchsia::camera3::Orientation orientation); + + // Waits for the buffer collection to be allocated. Returns true if the buffer + // collection was allocated successfully. + bool WaitBuffersAllocated(); + + // Waits until there is at least one free buffer that can be used for the next + // frame. + bool WaitFreeBuffer(); + + void ProduceFrame(base::TimeTicks timestamp, uint8_t salt); + + private: + struct Buffer; + + // fuchsia::camera3::Stream implementation. + void WatchResolution(WatchResolutionCallback callback) final; + void WatchOrientation(WatchOrientationCallback callback) final; + void SetBufferCollection( + fidl::InterfaceHandle<fuchsia::sysmem::BufferCollectionToken> + token_handle) final; + void WatchBufferCollection(WatchBufferCollectionCallback callback) final; + void GetNextFrame(GetNextFrameCallback callback) final; + + // fuchsia::camera3::testing::Stream_TestBase override. + void NotImplemented_(const std::string& name) override; + + void OnBufferCollectionError(zx_status_t status); + + void OnBufferCollectionAllocated( + zx_status_t status, + fuchsia::sysmem::BufferCollectionInfo_2 buffer_collection_info); + + // Calls callback for the pending WatchResolution() if the call is pending and + // resolution has been updated. + void SendResolution(); + + // Calls callback for the pending WatchOrientation() if the call is pending + // and orientation has been updated. + void SendOrientation(); + + // Calls callback for the pending WatchBufferCollection() if we have a new + // token and the call is pending. + void SendBufferCollection(); + + // Calls callback for the pending GetNextFrame() if we have a new frame and + // the call is pending. + void SendNextFrame(); + + // ZxHandleWatcher interface. Used to wait for frame release_fences to get + // notified when the client releases a buffer. + void OnZxHandleSignalled(zx_handle_t handle, zx_signals_t signals) final; + + fidl::Binding<fuchsia::camera3::Stream> binding_; + + gfx::Size resolution_ = kDefaultFrameSize; + fuchsia::camera3::Orientation orientation_ = + fuchsia::camera3::Orientation::UP; + + base::Optional<fuchsia::math::Size> resolution_update_ = fuchsia::math::Size{ + kDefaultFrameSize.width(), kDefaultFrameSize.height()}; + WatchResolutionCallback watch_resolution_callback_; + + base::Optional<fuchsia::camera3::Orientation> orientation_update_ = + fuchsia::camera3::Orientation::UP; + WatchOrientationCallback watch_orientation_callback_; + + base::Optional<fidl::InterfaceHandle<fuchsia::sysmem::BufferCollectionToken>> + new_buffer_collection_token_; + WatchBufferCollectionCallback watch_buffer_collection_callback_; + + base::Optional<fuchsia::camera3::FrameInfo> next_frame_; + GetNextFrameCallback get_next_frame_callback_; + + fuchsia::sysmem::BufferCollectionPtr buffer_collection_; + + base::Optional<base::RunLoop> wait_buffers_allocated_run_loop_; + base::Optional<base::RunLoop> wait_free_buffer_run_loop_; + + std::vector<std::unique_ptr<Buffer>> buffers_; + size_t num_used_buffers_ = 0; + + size_t frame_counter_ = 0; +}; + +class FakeCameraDevice : public fuchsia::camera3::testing::Device_TestBase { + public: + explicit FakeCameraDevice(FakeCameraStream* stream); + ~FakeCameraDevice() final; + + FakeCameraDevice(const FakeCameraDevice&) = delete; + FakeCameraDevice& operator=(const FakeCameraDevice&) = delete; + + void Bind(fidl::InterfaceRequest<fuchsia::camera3::Device> request); + + private: + // fuchsia::camera3::Device implementation. + void GetIdentifier(GetIdentifierCallback callback) final; + void GetConfigurations(GetConfigurationsCallback callback) final; + void ConnectToStream( + uint32_t index, + fidl::InterfaceRequest<fuchsia::camera3::Stream> request) final; + + // fuchsia::camera3::testing::Device_TestBase override. + void NotImplemented_(const std::string& name) override; + + fidl::Binding<fuchsia::camera3::Device> binding_; + FakeCameraStream* const stream_; +}; + +class FakeCameraDeviceWatcher { + public: + explicit FakeCameraDeviceWatcher(sys::OutgoingDirectory* outgoing_directory); + ~FakeCameraDeviceWatcher(); + + FakeCameraDeviceWatcher(const FakeCameraDeviceWatcher&) = delete; + FakeCameraDeviceWatcher& operator=(const FakeCameraDeviceWatcher&) = delete; + + private: + class Client : public fuchsia::camera3::testing::DeviceWatcher_TestBase { + public: + explicit Client(FakeCameraDevice* device); + ~Client() final; + + Client(const Client&) = delete; + Client& operator=(const Client&) = delete; + + // fuchsia::camera3::testing::DeviceWatcher_TestBase override. + void NotImplemented_(const std::string& name) final; + + // fuchsia::camera3::DeviceWatcher implementation. + void WatchDevices(WatchDevicesCallback callback) final; + void ConnectToDevice( + uint64_t id, + fidl::InterfaceRequest<fuchsia::camera3::Device> request) final; + + private: + bool devices_sent_ = false; + FakeCameraDevice* const device_; + }; + + fidl::BindingSet<fuchsia::camera3::DeviceWatcher, std::unique_ptr<Client>> + bindings_; + + FakeCameraStream stream_; + FakeCameraDevice device_{&stream_}; +}; + +} // namespace media + +#endif // MEDIA_FUCHSIA_CAMERA_FAKE_FUCHSIA_CAMERA_H_ diff --git a/chromium/media/fuchsia/cdm/fuchsia_decryptor.cc b/chromium/media/fuchsia/cdm/fuchsia_decryptor.cc index b36fcfb878c..e10b2a1a863 100644 --- a/chromium/media/fuchsia/cdm/fuchsia_decryptor.cc +++ b/chromium/media/fuchsia/cdm/fuchsia_decryptor.cc @@ -4,9 +4,10 @@ #include "media/fuchsia/cdm/fuchsia_decryptor.h" +#include "base/check.h" #include "base/fuchsia/fuchsia_logging.h" #include "base/location.h" -#include "base/logging.h" +#include "base/notreached.h" #include "base/threading/thread_task_runner_handle.h" #include "media/base/decoder_buffer.h" #include "media/base/video_frame.h" diff --git a/chromium/media/fuchsia/common/sysmem_buffer_pool.cc b/chromium/media/fuchsia/common/sysmem_buffer_pool.cc index 6c30ad8a0df..94c1a1f43d9 100644 --- a/chromium/media/fuchsia/common/sysmem_buffer_pool.cc +++ b/chromium/media/fuchsia/common/sysmem_buffer_pool.cc @@ -140,13 +140,18 @@ BufferAllocator::BufferAllocator() { BufferAllocator::~BufferAllocator() = default; +fuchsia::sysmem::BufferCollectionTokenPtr BufferAllocator::CreateNewToken() { + fuchsia::sysmem::BufferCollectionTokenPtr collection_token; + allocator_->AllocateSharedCollection(collection_token.NewRequest()); + return collection_token; +} + std::unique_ptr<SysmemBufferPool::Creator> BufferAllocator::MakeBufferPoolCreator(size_t num_of_tokens) { DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); // Create a new sysmem buffer collection token for the allocated buffers. - fuchsia::sysmem::BufferCollectionTokenPtr collection_token; - allocator_->AllocateSharedCollection(collection_token.NewRequest()); + fuchsia::sysmem::BufferCollectionTokenPtr collection_token = CreateNewToken(); // Create collection token for sharing with other components. std::vector<fuchsia::sysmem::BufferCollectionTokenPtr> shared_tokens; @@ -166,4 +171,15 @@ BufferAllocator::MakeBufferPoolCreator(size_t num_of_tokens) { std::move(buffer_collection), std::move(shared_tokens)); } +std::unique_ptr<SysmemBufferPool::Creator> +BufferAllocator::MakeBufferPoolCreatorFromToken( + fuchsia::sysmem::BufferCollectionTokenPtr token) { + fuchsia::sysmem::BufferCollectionPtr buffer_collection; + allocator_->BindSharedCollection(std::move(token), + buffer_collection.NewRequest()); + return std::make_unique<SysmemBufferPool::Creator>( + std::move(buffer_collection), + std::vector<fuchsia::sysmem::BufferCollectionTokenPtr>{}); +} + } // namespace media diff --git a/chromium/media/fuchsia/common/sysmem_buffer_pool.h b/chromium/media/fuchsia/common/sysmem_buffer_pool.h index c4c6ceabb54..ec657ed370d 100644 --- a/chromium/media/fuchsia/common/sysmem_buffer_pool.h +++ b/chromium/media/fuchsia/common/sysmem_buffer_pool.h @@ -98,9 +98,14 @@ class BufferAllocator { BufferAllocator(); ~BufferAllocator(); + fuchsia::sysmem::BufferCollectionTokenPtr CreateNewToken(); + std::unique_ptr<SysmemBufferPool::Creator> MakeBufferPoolCreator( size_t num_shared_token); + std::unique_ptr<SysmemBufferPool::Creator> MakeBufferPoolCreatorFromToken( + fuchsia::sysmem::BufferCollectionTokenPtr token); + // TODO(sergeyu): Update FuchsiaVideoDecoder to use SysmemBufferPool and // remove this function. fuchsia::sysmem::Allocator* raw() { return allocator_.get(); } diff --git a/chromium/media/fuchsia/common/sysmem_buffer_reader.cc b/chromium/media/fuchsia/common/sysmem_buffer_reader.cc index 8667a877398..406096a2617 100644 --- a/chromium/media/fuchsia/common/sysmem_buffer_reader.cc +++ b/chromium/media/fuchsia/common/sysmem_buffer_reader.cc @@ -17,27 +17,74 @@ SysmemBufferReader::~SysmemBufferReader() = default; bool SysmemBufferReader::Read(size_t index, size_t offset, base::span<uint8_t> data) { - DCHECK_LT(index, collection_info_.buffer_count); - const fuchsia::sysmem::BufferMemorySettings& settings = - collection_info_.settings.buffer_settings; - fuchsia::sysmem::VmoBuffer& buffer = collection_info_.buffers[index]; - DCHECK_LE(buffer.vmo_usable_start + offset + data.size(), - settings.size_bytes); + DCHECK_LT(index, num_buffers()); + DCHECK_LE(offset + data.size(), + collection_info_.settings.buffer_settings.size_bytes); + const fuchsia::sysmem::VmoBuffer& buffer = collection_info_.buffers[index]; size_t vmo_offset = buffer.vmo_usable_start + offset; - // Invalidate cache. - if (settings.coherency_domain == fuchsia::sysmem::CoherencyDomain::RAM) { - zx_status_t status = buffer.vmo.op_range( - ZX_VMO_OP_CACHE_CLEAN_INVALIDATE, vmo_offset, data.size(), nullptr, 0); - ZX_LOG_IF(ERROR, status != ZX_OK, status) << "Fail to invalidate cache"; - } + InvalidateCacheIfNecessary(buffer.vmo, vmo_offset, data.size()); zx_status_t status = buffer.vmo.read(data.data(), vmo_offset, data.size()); + ZX_LOG_IF(ERROR, status != ZX_OK, status) << "Fail to read"; return status == ZX_OK; } +base::span<const uint8_t> SysmemBufferReader::GetMappingForBuffer( + size_t index) { + if (mappings_.empty()) + mappings_.resize(num_buffers()); + + DCHECK_LT(index, mappings_.size()); + + const fuchsia::sysmem::BufferMemorySettings& settings = + collection_info_.settings.buffer_settings; + fuchsia::sysmem::VmoBuffer& buffer = collection_info_.buffers[index]; + + auto& mapping = mappings_[index]; + size_t buffer_start = buffer.vmo_usable_start; + + if (!mapping.IsValid()) { + size_t mapping_size = buffer_start + settings.size_bytes; + auto region = base::ReadOnlySharedMemoryRegion::Deserialize( + base::subtle::PlatformSharedMemoryRegion::Take( + std::move(buffer.vmo), + base::subtle::PlatformSharedMemoryRegion::Mode::kReadOnly, + mapping_size, base::UnguessableToken::Create())); + + mapping = region.Map(); + + // Return the VMO handle back to buffer_. + buffer.vmo = base::ReadOnlySharedMemoryRegion::TakeHandleForSerialization( + std::move(region)) + .PassPlatformHandle(); + } + + if (!mapping.IsValid()) { + DLOG(WARNING) << "Failed to map VMO returned by sysmem"; + return {}; + } + + InvalidateCacheIfNecessary(buffer.vmo, buffer_start, settings.size_bytes); + + return base::make_span( + reinterpret_cast<const uint8_t*>(mapping.memory()) + buffer_start, + settings.size_bytes); +} + +void SysmemBufferReader::InvalidateCacheIfNecessary(const zx::vmo& vmo, + size_t offset, + size_t size) { + if (collection_info_.settings.buffer_settings.coherency_domain == + fuchsia::sysmem::CoherencyDomain::RAM) { + zx_status_t status = vmo.op_range(ZX_VMO_OP_CACHE_CLEAN_INVALIDATE, offset, + size, nullptr, 0); + ZX_LOG_IF(ERROR, status != ZX_OK, status) << "Fail to invalidate cache"; + } +} + // static std::unique_ptr<SysmemBufferReader> SysmemBufferReader::Create( fuchsia::sysmem::BufferCollectionInfo_2 info) { diff --git a/chromium/media/fuchsia/common/sysmem_buffer_reader.h b/chromium/media/fuchsia/common/sysmem_buffer_reader.h index 0c314162e68..bc9727facd0 100644 --- a/chromium/media/fuchsia/common/sysmem_buffer_reader.h +++ b/chromium/media/fuchsia/common/sysmem_buffer_reader.h @@ -11,6 +11,7 @@ #include <memory> #include "base/containers/span.h" +#include "base/memory/read_only_shared_memory_region.h" namespace media { @@ -26,11 +27,35 @@ class SysmemBufferReader { explicit SysmemBufferReader(fuchsia::sysmem::BufferCollectionInfo_2 info); ~SysmemBufferReader(); + size_t num_buffers() const { return collection_info_.buffer_count; } + + const fuchsia::sysmem::SingleBufferSettings& buffer_settings() { + return collection_info_.settings; + } + // Read the buffer content at |index| into |data|, starting from |offset|. bool Read(size_t index, size_t offset, base::span<uint8_t> data); + // Returns a span for the memory-mapping of the buffer with the specified + // |index|, invalidating the CPU cache for the specified buffer in the sysmem + // collection if necessary. Buffers are mapped lazily and remain mapped for + // the lifetime of SysmemBufferReader. Should be called every time before + // accessing the mapping to ensure that the CPU cache is invalidated for + // buffers with RAM coherency. + base::span<const uint8_t> GetMappingForBuffer(size_t index); + private: + // Invalidates CPU cache for the specified range of the specified vmo in + // case the collection was allocated with RAM coherency. No-op for collections + // with CPU coherency. Called from Read() and GetMapping() to ensure clients + // get up-to-date buffer content in case the buffer was updated by other + // participants directly in RAM (bypassing CPU cache). + void InvalidateCacheIfNecessary(const zx::vmo& buffer, + size_t offset, + size_t size); + fuchsia::sysmem::BufferCollectionInfo_2 collection_info_; + std::vector<base::ReadOnlySharedMemoryMapping> mappings_; DISALLOW_COPY_AND_ASSIGN(SysmemBufferReader); }; diff --git a/chromium/media/fuchsia/metrics/BUILD.gn b/chromium/media/fuchsia/metrics/BUILD.gn new file mode 100644 index 00000000000..ee6952ba233 --- /dev/null +++ b/chromium/media/fuchsia/metrics/BUILD.gn @@ -0,0 +1,28 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +source_set("metrics") { + sources = [ + "fuchsia_playback_events_recorder.cc", + "fuchsia_playback_events_recorder.h", + ] + deps = [ + "//media/mojo/mojom", + "//mojo/public/cpp/bindings", + ] +} + +source_set("unittests") { + testonly = true + + deps = [ + ":metrics", + "//base", + "//base/test:test_support", + "//media", + "//testing/gtest", + ] + + sources = [ "fuchsia_playback_events_recorder_test.cc" ] +} diff --git a/chromium/media/fuchsia/metrics/DEPS b/chromium/media/fuchsia/metrics/DEPS new file mode 100644 index 00000000000..ef8ad28d9d4 --- /dev/null +++ b/chromium/media/fuchsia/metrics/DEPS @@ -0,0 +1,3 @@ +include_rules = [ + "+mojo/public", +] diff --git a/chromium/media/fuchsia/metrics/fuchsia_playback_events_recorder.cc b/chromium/media/fuchsia/metrics/fuchsia_playback_events_recorder.cc new file mode 100644 index 00000000000..d883b08b38a --- /dev/null +++ b/chromium/media/fuchsia/metrics/fuchsia_playback_events_recorder.cc @@ -0,0 +1,139 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/fuchsia/metrics/fuchsia_playback_events_recorder.h" + +#include "base/metrics/user_metrics.h" +#include "base/strings/stringprintf.h" +#include "mojo/public/cpp/bindings/self_owned_receiver.h" + +namespace media { + +namespace { + +void RecordEventWithValueAt(const char* name, + int64_t value, + base::TimeTicks time) { + base::RecordComputedActionAt( + base::StringPrintf("WebEngine.Media.%s:%ld", name, value), time); +} + +void RecordEventWithValue(const char* name, int64_t value) { + RecordEventWithValueAt(name, value, base::TimeTicks::Now()); +} + +constexpr base::TimeDelta kBitrateReportPeriod = + base::TimeDelta::FromSeconds(5); + +} // namespace + +FuchsiaPlaybackEventsRecorder::BitrateEstimator::BitrateEstimator() {} +FuchsiaPlaybackEventsRecorder::BitrateEstimator::~BitrateEstimator() {} + +void FuchsiaPlaybackEventsRecorder::BitrateEstimator::Update( + const PipelineStatistics& stats) { + base::TimeTicks now = base::TimeTicks::Now(); + + // The code below trusts that |stats| are valid even though they came from an + // untrusted process. That's accepable because the stats are used only to + // record metrics. + if (last_stats_) { + time_elapsed_ += now - last_stats_time_; + audio_bytes_ += + stats.audio_bytes_decoded - last_stats_->audio_bytes_decoded; + video_bytes_ += + stats.video_bytes_decoded - last_stats_->video_bytes_decoded; + if (time_elapsed_ >= kBitrateReportPeriod) { + size_t audio_bitrate_kbps = + 8 * audio_bytes_ / time_elapsed_.InMilliseconds(); + RecordEventWithValueAt("AudioBitrate", audio_bitrate_kbps, now); + + size_t video_bitrate_kbps = + 8 * video_bytes_ / time_elapsed_.InMilliseconds(); + RecordEventWithValueAt("VideoBitrate", video_bitrate_kbps, now); + + time_elapsed_ = base::TimeDelta(); + audio_bytes_ = 0; + video_bytes_ = 0; + } + } + + last_stats_ = stats; + last_stats_time_ = now; +} + +void FuchsiaPlaybackEventsRecorder::BitrateEstimator::OnPause() { + last_stats_ = {}; +} + +// static +void FuchsiaPlaybackEventsRecorder::Create( + mojo::PendingReceiver<mojom::PlaybackEventsRecorder> receiver) { + mojo::MakeSelfOwnedReceiver(std::make_unique<FuchsiaPlaybackEventsRecorder>(), + std::move(receiver)); +} + +FuchsiaPlaybackEventsRecorder::FuchsiaPlaybackEventsRecorder() = default; +FuchsiaPlaybackEventsRecorder::~FuchsiaPlaybackEventsRecorder() = default; + +void FuchsiaPlaybackEventsRecorder::OnPlaying() { + base::RecordComputedAction("WebEngine.Media.Playing"); +} + +void FuchsiaPlaybackEventsRecorder::OnPaused() { + base::RecordComputedAction("WebEngine.Media.Pause"); + bitrate_estimator_.OnPause(); +} + +void FuchsiaPlaybackEventsRecorder::OnSeeking() { + buffering_state_ = BufferingState::kInitialBuffering; +} + +void FuchsiaPlaybackEventsRecorder::OnEnded() { + base::RecordComputedAction("WebEngine.Media.Ended"); +} + +void FuchsiaPlaybackEventsRecorder::OnBuffering() { + DCHECK(buffering_state_ == BufferingState::kBuffered); + + buffering_start_time_ = base::TimeTicks::Now(); + buffering_state_ = BufferingState::kBuffering; + + bitrate_estimator_.OnPause(); +} + +void FuchsiaPlaybackEventsRecorder::OnBufferingComplete() { + auto now = base::TimeTicks::Now(); + + if (buffering_state_ == BufferingState::kBuffering) { + base::TimeDelta time_between_buffering = + buffering_start_time_ - last_buffering_end_time_; + RecordEventWithValueAt("PlayTimeBeforeAutoPause", + time_between_buffering.InMilliseconds(), now); + + base::TimeDelta buffering_user_time = now - buffering_start_time_; + RecordEventWithValueAt("AutoPauseTime", + buffering_user_time.InMilliseconds(), now); + } + + buffering_state_ = BufferingState::kBuffered; + last_buffering_end_time_ = now; +} + +void FuchsiaPlaybackEventsRecorder::OnError(PipelineStatus status) { + RecordEventWithValue("Error", status); +} + +void FuchsiaPlaybackEventsRecorder::OnNaturalSizeChanged( + const gfx::Size& size) { + base::RecordComputedAction(base::StringPrintf( + "WebEngine.Media.VideoResolution:%dx%d", size.width(), size.height())); +} + +void FuchsiaPlaybackEventsRecorder::OnPipelineStatistics( + const PipelineStatistics& stats) { + bitrate_estimator_.Update(stats); +} + +} // namespace media diff --git a/chromium/media/fuchsia/metrics/fuchsia_playback_events_recorder.h b/chromium/media/fuchsia/metrics/fuchsia_playback_events_recorder.h new file mode 100644 index 00000000000..2686bafec9e --- /dev/null +++ b/chromium/media/fuchsia/metrics/fuchsia_playback_events_recorder.h @@ -0,0 +1,70 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_FUCHSIA_METRICS_FUCHSIA_PLAYBACK_EVENTS_RECORDER_H_ +#define MEDIA_FUCHSIA_METRICS_FUCHSIA_PLAYBACK_EVENTS_RECORDER_H_ + +#include "media/mojo/mojom/playback_events_recorder.mojom.h" +#include "mojo/public/cpp/bindings/pending_remote.h" +#include "mojo/public/cpp/bindings/remote.h" + +namespace media { + +class FuchsiaPlaybackEventsRecorder : public mojom::PlaybackEventsRecorder { + public: + static void Create( + mojo::PendingReceiver<mojom::PlaybackEventsRecorder> receiver); + + FuchsiaPlaybackEventsRecorder(); + ~FuchsiaPlaybackEventsRecorder() final; + + FuchsiaPlaybackEventsRecorder(const FuchsiaPlaybackEventsRecorder&) = delete; + FuchsiaPlaybackEventsRecorder& operator=( + const FuchsiaPlaybackEventsRecorder&) = delete; + + // mojom::PlaybackEventsRecorder implementation. + void OnPlaying() final; + void OnPaused() final; + void OnSeeking() final; + void OnEnded() final; + void OnBuffering() final; + void OnBufferingComplete() final; + void OnError(PipelineStatus status) final; + void OnNaturalSizeChanged(const gfx::Size& size) final; + void OnPipelineStatistics(const PipelineStatistics& stats) final; + + private: + class BitrateEstimator { + public: + BitrateEstimator(); + ~BitrateEstimator(); + + void Update(const PipelineStatistics& stats); + void OnPause(); + + private: + base::TimeDelta time_elapsed_; + size_t audio_bytes_ = 0; + size_t video_bytes_ = 0; + + base::Optional<PipelineStatistics> last_stats_; + base::TimeTicks last_stats_time_; + }; + + enum class BufferingState { + kInitialBuffering, + kBuffering, + kBuffered, + }; + + BufferingState buffering_state_ = BufferingState::kInitialBuffering; + base::TimeTicks buffering_start_time_; + base::TimeTicks last_buffering_end_time_; + + BitrateEstimator bitrate_estimator_; +}; + +} // namespace media + +#endif // MEDIA_FUCHSIA_METRICS_FUCHSIA_PLAYBACK_EVENTS_RECORDER_H_
\ No newline at end of file diff --git a/chromium/media/fuchsia/metrics/fuchsia_playback_events_recorder_test.cc b/chromium/media/fuchsia/metrics/fuchsia_playback_events_recorder_test.cc new file mode 100644 index 00000000000..a18704575a2 --- /dev/null +++ b/chromium/media/fuchsia/metrics/fuchsia_playback_events_recorder_test.cc @@ -0,0 +1,202 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/fuchsia/metrics/fuchsia_playback_events_recorder.h" + +#include "base/metrics/user_metrics.h" +#include "base/test/simple_test_tick_clock.h" +#include "base/test/task_environment.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace media { + +constexpr base::TimeDelta kSecond = base::TimeDelta::FromSeconds(1); + +class FuchsiaPlaybackEventsRecorderTest : public testing::Test { + public: + FuchsiaPlaybackEventsRecorderTest() + : task_environment_(base::test::TaskEnvironment::MainThreadType::IO, + base::test::TaskEnvironment::TimeSource::MOCK_TIME) { + time_base_ = base::TimeTicks::Now(); + + base::SetRecordActionTaskRunner( + task_environment_.GetMainThreadTaskRunner()); + action_callback_ = base::BindRepeating( + &FuchsiaPlaybackEventsRecorderTest::OnAction, base::Unretained(this)); + base::AddActionCallback(action_callback_); + } + + ~FuchsiaPlaybackEventsRecorderTest() override { + base::RemoveActionCallback(action_callback_); + } + + protected: + struct Event { + base::TimeTicks time; + std::string name; + + bool operator==(const Event& other) const { + return time == other.time && name == other.name; + } + }; + + void OnAction(const std::string& name, base::TimeTicks time) { + recorded_events_.push_back({time, name}); + } + + void ExpectEvents(const std::vector<Event>& expected) { + EXPECT_EQ(recorded_events_.size(), expected.size()); + size_t end = std::min(recorded_events_.size(), expected.size()); + for (size_t i = 0; i < end; ++i) { + SCOPED_TRACE(i); + EXPECT_EQ(recorded_events_[i].time, expected[i].time); + EXPECT_EQ(recorded_events_[i].name, expected[i].name); + } + } + + base::test::TaskEnvironment task_environment_; + + base::SimpleTestTickClock test_clock_; + base::TimeTicks time_base_; + + base::ActionCallback action_callback_; + FuchsiaPlaybackEventsRecorder recorder_; + std::vector<Event> recorded_events_; +}; + +TEST_F(FuchsiaPlaybackEventsRecorderTest, PlayPause) { + recorder_.OnNaturalSizeChanged(gfx::Size(640, 480)); + recorder_.OnPlaying(); + task_environment_.AdvanceClock(2 * kSecond); + recorder_.OnPaused(); + + ExpectEvents({ + {time_base_, "WebEngine.Media.VideoResolution:640x480"}, + {time_base_, "WebEngine.Media.Playing"}, + {time_base_ + 2 * kSecond, "WebEngine.Media.Pause"}, + }); +} + +TEST_F(FuchsiaPlaybackEventsRecorderTest, Error) { + recorder_.OnPlaying(); + task_environment_.AdvanceClock(2 * kSecond); + recorder_.OnError(PIPELINE_ERROR_DECODE); + + ExpectEvents({ + {time_base_, "WebEngine.Media.Playing"}, + {time_base_ + 2 * kSecond, "WebEngine.Media.Error:3"}, + }); +} + +TEST_F(FuchsiaPlaybackEventsRecorderTest, Buffering) { + recorder_.OnPlaying(); + recorder_.OnBufferingComplete(); + task_environment_.AdvanceClock(2 * kSecond); + recorder_.OnBuffering(); + task_environment_.AdvanceClock(3 * kSecond); + recorder_.OnBufferingComplete(); + + ExpectEvents({ + {time_base_, "WebEngine.Media.Playing"}, + {time_base_ + 5 * kSecond, + "WebEngine.Media.PlayTimeBeforeAutoPause:2000"}, + {time_base_ + 5 * kSecond, "WebEngine.Media.AutoPauseTime:3000"}, + }); +} + +TEST_F(FuchsiaPlaybackEventsRecorderTest, Bitrate) { + recorder_.OnPlaying(); + recorder_.OnBufferingComplete(); + + PipelineStatistics stats; + recorder_.OnPipelineStatistics(stats); + + for (int i = 0; i < 5; ++i) { + stats.audio_bytes_decoded += 5000; + stats.video_bytes_decoded += 10000; + + task_environment_.AdvanceClock(kSecond); + recorder_.OnPipelineStatistics(stats); + } + + ExpectEvents({ + {time_base_, "WebEngine.Media.Playing"}, + {time_base_ + 5 * kSecond, "WebEngine.Media.AudioBitrate:40"}, + {time_base_ + 5 * kSecond, "WebEngine.Media.VideoBitrate:80"}, + }); +} + +TEST_F(FuchsiaPlaybackEventsRecorderTest, BitrateAfterPause) { + recorder_.OnPlaying(); + recorder_.OnBufferingComplete(); + + PipelineStatistics stats; + recorder_.OnPipelineStatistics(stats); + + for (int i = 0; i < 3; ++i) { + stats.audio_bytes_decoded += 5000; + stats.video_bytes_decoded += 10000; + + task_environment_.AdvanceClock(kSecond); + recorder_.OnPipelineStatistics(stats); + } + + recorder_.OnPaused(); + task_environment_.AdvanceClock(10 * kSecond); + recorder_.OnPlaying(); + + for (int i = 0; i < 3; ++i) { + stats.audio_bytes_decoded += 5000; + stats.video_bytes_decoded += 10000; + + task_environment_.AdvanceClock(kSecond); + recorder_.OnPipelineStatistics(stats); + } + + ExpectEvents({ + {time_base_, "WebEngine.Media.Playing"}, + {time_base_ + 3 * kSecond, "WebEngine.Media.Pause"}, + {time_base_ + 13 * kSecond, "WebEngine.Media.Playing"}, + {time_base_ + 16 * kSecond, "WebEngine.Media.AudioBitrate:40"}, + {time_base_ + 16 * kSecond, "WebEngine.Media.VideoBitrate:80"}, + }); +} + +TEST_F(FuchsiaPlaybackEventsRecorderTest, BitrateAfterBuffering) { + recorder_.OnPlaying(); + recorder_.OnBufferingComplete(); + + PipelineStatistics stats; + recorder_.OnPipelineStatistics(stats); + + for (int i = 0; i < 3; ++i) { + stats.audio_bytes_decoded += 5000; + stats.video_bytes_decoded += 10000; + + task_environment_.AdvanceClock(kSecond); + recorder_.OnPipelineStatistics(stats); + } + + recorder_.OnBuffering(); + task_environment_.AdvanceClock(10 * kSecond); + recorder_.OnBufferingComplete(); + + for (int i = 0; i < 3; ++i) { + stats.audio_bytes_decoded += 5000; + stats.video_bytes_decoded += 10000; + + task_environment_.AdvanceClock(kSecond); + recorder_.OnPipelineStatistics(stats); + } + + ExpectEvents({ + {time_base_, "WebEngine.Media.Playing"}, + {time_base_ + 13 * kSecond, + "WebEngine.Media.PlayTimeBeforeAutoPause:3000"}, + {time_base_ + 13 * kSecond, "WebEngine.Media.AutoPauseTime:10000"}, + {time_base_ + 16 * kSecond, "WebEngine.Media.AudioBitrate:40"}, + {time_base_ + 16 * kSecond, "WebEngine.Media.VideoBitrate:80"}, + }); +} +} // namespace media
\ No newline at end of file diff --git a/chromium/media/gpu/BUILD.gn b/chromium/media/gpu/BUILD.gn index dbe714444da..767a3316092 100644 --- a/chromium/media/gpu/BUILD.gn +++ b/chromium/media/gpu/BUILD.gn @@ -109,6 +109,8 @@ component("gpu") { "android/codec_image.h", "android/codec_image_group.cc", "android/codec_image_group.h", + "android/codec_output_buffer_renderer.cc", + "android/codec_output_buffer_renderer.h", "android/codec_surface_bundle.cc", "android/codec_surface_bundle.h", "android/codec_wrapper.cc", @@ -117,6 +119,8 @@ component("gpu") { "android/device_info.h", "android/direct_shared_image_video_provider.cc", "android/direct_shared_image_video_provider.h", + "android/frame_info_helper.cc", + "android/frame_info_helper.h", "android/maybe_render_early_manager.cc", "android/maybe_render_early_manager.h", "android/media_codec_video_decoder.cc", @@ -133,8 +137,6 @@ component("gpu") { "android/video_frame_factory.h", "android/video_frame_factory_impl.cc", "android/video_frame_factory_impl.h", - "android/ycbcr_helper.cc", - "android/ycbcr_helper.h", ] libs += [ "android" ] deps += [ @@ -198,6 +200,8 @@ component("gpu") { "windows/dxva_picture_buffer_win.h", "windows/dxva_video_decode_accelerator_win.cc", "windows/dxva_video_decode_accelerator_win.h", + "windows/hresult_status_debug_device.cc", + "windows/hresult_status_debug_device.h", "windows/init_guid.cc", "windows/media_foundation_video_encode_accelerator_win.cc", "windows/media_foundation_video_encode_accelerator_win.h", @@ -211,6 +215,7 @@ component("gpu") { ] public_deps += [ "//media/base/win:media_foundation_util" ] deps += [ + "//media/base/win:hresult_status_helper", "//media/parsers", "//third_party/angle:includes", "//ui/display", @@ -231,14 +236,6 @@ component("gpu") { "/DELAYLOAD:mf.dll", "/DELAYLOAD:mfplat.dll", ] - if (enable_library_cdms) { - sources += [ - "windows/d3d11_cdm_proxy.cc", - "windows/d3d11_cdm_proxy.h", - "windows/d3d11_decryptor.cc", - "windows/d3d11_decryptor.h", - ] - } } if (use_ozone) { @@ -485,9 +482,11 @@ source_set("unit_tests") { "//base/test:test_support", "//media:test_support", "//media/gpu", + "//media/gpu:test_support", "//media/gpu/ipc/service:unit_tests", "//testing/gmock", "//testing/gtest", + "//ui/gl:test_support", ] sources = [ "h264_decoder_unittest.cc" ] @@ -507,11 +506,10 @@ source_set("unit_tests") { } if (is_win && enable_library_cdms) { sources += [ - "windows/d3d11_cdm_proxy_unittest.cc", "windows/d3d11_copying_texture_wrapper_unittest.cc", "windows/d3d11_decoder_configurator_unittest.cc", - "windows/d3d11_decryptor_unittest.cc", "windows/d3d11_texture_selector_unittest.cc", + "windows/d3d11_texture_wrapper_unittest.cc", "windows/d3d11_video_decoder_unittest.cc", "windows/d3d11_video_device_format_support_unittest.cc", "windows/d3d11_video_processor_proxy_unittest.cc", diff --git a/chromium/media/gpu/OWNERS b/chromium/media/gpu/OWNERS index 18fb4fa7ad2..2b325453c62 100644 --- a/chromium/media/gpu/OWNERS +++ b/chromium/media/gpu/OWNERS @@ -7,6 +7,7 @@ sandersd@chromium.org acourbot@chromium.org hiroh@chromium.org jcliang@chromium.org +jkardatzke@chromium.org kcwu@chromium.org mcasas@chromium.org posciak@chromium.org diff --git a/chromium/media/gpu/android/android_video_surface_chooser_impl_unittest.cc b/chromium/media/gpu/android/android_video_surface_chooser_impl_unittest.cc index 0cc3adf01e6..9f5eff7e972 100644 --- a/chromium/media/gpu/android/android_video_surface_chooser_impl_unittest.cc +++ b/chromium/media/gpu/android/android_video_surface_chooser_impl_unittest.cc @@ -9,7 +9,6 @@ #include <memory> #include "base/bind.h" -#include "base/logging.h" #include "base/test/simple_test_tick_clock.h" #include "media/base/android/mock_android_overlay.h" #include "testing/gmock/include/gmock/gmock.h" diff --git a/chromium/media/gpu/android/codec_allocator_unittest.cc b/chromium/media/gpu/android/codec_allocator_unittest.cc index 90d25b8ccaf..439a310fcd4 100644 --- a/chromium/media/gpu/android/codec_allocator_unittest.cc +++ b/chromium/media/gpu/android/codec_allocator_unittest.cc @@ -10,7 +10,7 @@ #include "base/bind.h" #include "base/bind_helpers.h" -#include "base/logging.h" +#include "base/check.h" #include "base/single_thread_task_runner.h" #include "base/test/simple_test_tick_clock.h" #include "base/test/task_environment.h" diff --git a/chromium/media/gpu/android/codec_image.cc b/chromium/media/gpu/android/codec_image.cc index 187fb240c74..ae3a90da5c9 100644 --- a/chromium/media/gpu/android/codec_image.cc +++ b/chromium/media/gpu/android/codec_image.cc @@ -16,45 +16,19 @@ #include "ui/gl/scoped_make_current.h" namespace media { -namespace { - -// Makes |texture_owner|'s context current if it isn't already. -std::unique_ptr<ui::ScopedMakeCurrent> MakeCurrentIfNeeded( - gpu::TextureOwner* texture_owner) { - gl::GLContext* context = texture_owner->GetContext(); - // Note: this works for virtual contexts too, because IsCurrent() returns true - // if their shared platform context is current, regardless of which virtual - // context is current. - if (context->IsCurrent(nullptr)) - return nullptr; - - auto scoped_current = std::make_unique<ui::ScopedMakeCurrent>( - context, texture_owner->GetSurface()); - // Log an error if ScopedMakeCurrent failed for debugging - // https://crbug.com/878042. - // TODO(ericrk): Remove this once debugging is completed. - if (!context->IsCurrent(nullptr)) { - LOG(ERROR) << "Failed to make context current in CodecImage. Subsequent " - "UpdateTexImage may fail."; - } - return scoped_current; -} - -} // namespace -CodecImage::CodecImage() = default; +CodecImage::CodecImage(const gfx::Size& coded_size) : coded_size_(coded_size) {} CodecImage::~CodecImage() { NotifyUnused(); } void CodecImage::Initialize( - std::unique_ptr<CodecOutputBuffer> output_buffer, + std::unique_ptr<CodecOutputBufferRenderer> output_buffer_renderer, scoped_refptr<CodecBufferWaitCoordinator> codec_buffer_wait_coordinator, PromotionHintAggregator::NotifyPromotionHintCB promotion_hint_cb) { - DCHECK(output_buffer); - phase_ = Phase::kInCodec; - output_buffer_ = std::move(output_buffer); + DCHECK(output_buffer_renderer); + output_buffer_renderer_ = std::move(output_buffer_renderer); codec_buffer_wait_coordinator_ = std::move(codec_buffer_wait_coordinator); promotion_hint_cb_ = std::move(promotion_hint_cb); } @@ -77,10 +51,7 @@ void CodecImage::NotifyUnused() { } gfx::Size CodecImage::GetSize() { - // Return a nonzero size, to avoid GL errors, even if we dropped the codec - // buffer already. Note that if we dropped it, there's no data in the - // texture anyway, so the old size doesn't matter. - return output_buffer_ ? output_buffer_->size() : gfx::Size(1, 1); + return coded_size_; } unsigned CodecImage::GetInternalFormat() { @@ -119,7 +90,11 @@ bool CodecImage::CopyTexImage(unsigned target) { codec_buffer_wait_coordinator_->texture_owner()->GetTextureId())) return false; - RenderToTextureOwnerFrontBuffer(BindingsMode::kEnsureTexImageBound); + if (!output_buffer_renderer_) + return true; + + output_buffer_renderer_->RenderToTextureOwnerFrontBuffer( + BindingsMode::kEnsureTexImageBound); return true; } @@ -183,23 +158,13 @@ void CodecImage::OnMemoryDump(base::trace_event::ProcessMemoryDump* pmd, const std::string& dump_name) {} void CodecImage::GetTextureMatrix(float matrix[16]) { - // Default to identity. - static constexpr float kYInvertedIdentity[16]{ - 1, 0, 0, 0, // - 0, -1, 0, 0, // - 0, 0, 1, 0, // - 0, 1, 0, 1 // + static constexpr float kIdentity[16]{ + 1, 0, 0, 0, // + 0, 1, 0, 0, // + 0, 0, 1, 0, // + 0, 0, 0, 1 // }; - memcpy(matrix, kYInvertedIdentity, sizeof(kYInvertedIdentity)); - if (!codec_buffer_wait_coordinator_) - return; - - // The matrix is available after we render to the front buffer. If that fails - // we'll return the matrix from the previous frame, which is more likely to be - // correct than the identity matrix anyway. - RenderToTextureOwnerFrontBuffer(BindingsMode::kDontRestoreIfBound); - codec_buffer_wait_coordinator_->texture_owner()->GetTransformMatrix(matrix); - YInvertMatrix(matrix); + memcpy(matrix, kIdentity, sizeof(kIdentity)); } void CodecImage::NotifyPromotionHint(bool promotion_hint, @@ -222,8 +187,10 @@ void CodecImage::ReleaseResources() { } bool CodecImage::IsUsingGpuMemory() const { + if (!output_buffer_renderer_) + return false; // Only the images which are bound to texture accounts for gpu memory. - return was_tex_image_bound_; + return output_buffer_renderer_->was_tex_image_bound(); } void CodecImage::UpdateAndBindTexImage() { @@ -239,120 +206,33 @@ gpu::TextureBase* CodecImage::GetTextureBase() const { } bool CodecImage::RenderToFrontBuffer() { - // This code is used to trigger early rendering of the image before it is used - // for compositing, there is no need to bind the image. - return codec_buffer_wait_coordinator_ - ? RenderToTextureOwnerFrontBuffer(BindingsMode::kRestoreIfBound) - : RenderToOverlay(); + if (!output_buffer_renderer_) + return false; + return output_buffer_renderer_->RenderToFrontBuffer(); } bool CodecImage::RenderToTextureOwnerBackBuffer(BlockingMode blocking_mode) { - DCHECK_NE(phase_, Phase::kInFrontBuffer); - if (phase_ == Phase::kInBackBuffer) - return true; - if (phase_ == Phase::kInvalidated) - return false; - - // Normally, we should have a wait coordinator if we're called. However, if - // the renderer is torn down (either VideoFrameSubmitter or the whole process) - // before we get returns back from viz, then we can be notified that we're - // no longer in use (erroneously) when the VideoFrame is destroyed. So, if - // we don't have a wait coordinator, then just fail. - if (!codec_buffer_wait_coordinator_) + if (!output_buffer_renderer_) return false; - // Wait for a previous frame available so we don't confuse it with the one - // we're about to release. - if (codec_buffer_wait_coordinator_->IsExpectingFrameAvailable()) { - if (blocking_mode == BlockingMode::kForbidBlocking) - return false; - codec_buffer_wait_coordinator_->WaitForFrameAvailable(); - } - if (!output_buffer_->ReleaseToSurface()) { - phase_ = Phase::kInvalidated; - return false; - } - phase_ = Phase::kInBackBuffer; - codec_buffer_wait_coordinator_->SetReleaseTimeToNow(); - return true; + return output_buffer_renderer_->RenderToTextureOwnerBackBuffer(blocking_mode); } bool CodecImage::RenderToTextureOwnerFrontBuffer(BindingsMode bindings_mode) { - // Normally, we should have a wait coordinator if we're called. However, if - // the renderer is torn down (either VideoFrameSubmitter or the whole process) - // before we get returns back from viz, then we can be notified that we're - // no longer in use (erroneously) when the VideoFrame is destroyed. So, if - // we don't have a wait coordinator, then just fail. - if (!codec_buffer_wait_coordinator_) + if (!output_buffer_renderer_) return false; - - if (phase_ == Phase::kInFrontBuffer) { - EnsureBoundIfNeeded(bindings_mode); - return true; - } - if (phase_ == Phase::kInvalidated) - return false; - - // Render it to the back buffer if it's not already there. - if (!RenderToTextureOwnerBackBuffer()) - return false; - - // The image is now in the back buffer, so promote it to the front buffer. - phase_ = Phase::kInFrontBuffer; - if (codec_buffer_wait_coordinator_->IsExpectingFrameAvailable()) - codec_buffer_wait_coordinator_->WaitForFrameAvailable(); - - std::unique_ptr<ui::ScopedMakeCurrent> scoped_make_current = - MakeCurrentIfNeeded( - codec_buffer_wait_coordinator_->texture_owner().get()); - // If updating the image will implicitly update the texture bindings then - // restore if requested or the update needed a context switch. - bool should_restore_bindings = - codec_buffer_wait_coordinator_->texture_owner() - ->binds_texture_on_update() && - (bindings_mode == BindingsMode::kRestoreIfBound || !!scoped_make_current); - - GLint bound_service_id = 0; - if (should_restore_bindings) - glGetIntegerv(GL_TEXTURE_BINDING_EXTERNAL_OES, &bound_service_id); - codec_buffer_wait_coordinator_->texture_owner()->UpdateTexImage(); - EnsureBoundIfNeeded(bindings_mode); - if (should_restore_bindings) - glBindTexture(GL_TEXTURE_EXTERNAL_OES, bound_service_id); - return true; -} - -void CodecImage::EnsureBoundIfNeeded(BindingsMode mode) { - DCHECK(codec_buffer_wait_coordinator_); - - if (codec_buffer_wait_coordinator_->texture_owner() - ->binds_texture_on_update()) { - was_tex_image_bound_ = true; - return; - } - if (mode != BindingsMode::kEnsureTexImageBound) - return; - codec_buffer_wait_coordinator_->texture_owner()->EnsureTexImageBound(); - was_tex_image_bound_ = true; + return output_buffer_renderer_->RenderToTextureOwnerFrontBuffer( + bindings_mode); } bool CodecImage::RenderToOverlay() { - if (phase_ == Phase::kInFrontBuffer) - return true; - if (phase_ == Phase::kInvalidated) + if (!output_buffer_renderer_) return false; - - if (!output_buffer_->ReleaseToSurface()) { - phase_ = Phase::kInvalidated; - return false; - } - phase_ = Phase::kInFrontBuffer; - return true; + return output_buffer_renderer_->RenderToOverlay(); } void CodecImage::ReleaseCodecBuffer() { - output_buffer_ = nullptr; - phase_ = Phase::kInvalidated; + output_buffer_renderer_.reset(); } std::unique_ptr<base::android::ScopedHardwareBufferFenceSync> @@ -369,9 +249,7 @@ CodecImage::GetAHardwareBuffer() { } gfx::Rect CodecImage::GetCropRect() { - if (!codec_buffer_wait_coordinator_) - return gfx::Rect(); - return codec_buffer_wait_coordinator_->texture_owner()->GetCropRect(); + return gfx::Rect(); } bool CodecImage::HasMutableState() const { diff --git a/chromium/media/gpu/android/codec_image.h b/chromium/media/gpu/android/codec_image.h index 61861825e4b..8693118f918 100644 --- a/chromium/media/gpu/android/codec_image.h +++ b/chromium/media/gpu/android/codec_image.h @@ -16,7 +16,7 @@ #include "gpu/command_buffer/service/gl_stream_texture_image.h" #include "gpu/command_buffer/service/stream_texture_shared_image_interface.h" #include "media/gpu/android/codec_buffer_wait_coordinator.h" -#include "media/gpu/android/codec_wrapper.h" +#include "media/gpu/android/codec_output_buffer_renderer.h" #include "media/gpu/android/promotion_hint_aggregator.h" #include "media/gpu/media_gpu_export.h" @@ -33,8 +33,7 @@ namespace media { class MEDIA_GPU_EXPORT CodecImage : public gpu::StreamTextureSharedImageInterface { public: - // Whether RenderToTextureOwnerBackBuffer may block or not. - enum class BlockingMode { kForbidBlocking, kAllowBlocking }; + using BlockingMode = CodecOutputBufferRenderer::BlockingMode; // Callback to notify that a codec image is now unused in the sense of not // being out for display. This lets us signal interested folks once a video @@ -47,14 +46,14 @@ class MEDIA_GPU_EXPORT CodecImage // destroying it. using UnusedCB = base::OnceCallback<void(CodecImage*)>; - CodecImage(); + CodecImage(const gfx::Size& coded_size); // (Re-)Initialize this CodecImage to use |output_buffer| et. al. // // May be called on a random thread, but only if the CodecImage is otherwise // not in use. void Initialize( - std::unique_ptr<CodecOutputBuffer> output_buffer, + std::unique_ptr<CodecOutputBufferRenderer> output_buffer_renderer, scoped_refptr<CodecBufferWaitCoordinator> codec_buffer_wait_coordinator, PromotionHintAggregator::NotifyPromotionHintCB promotion_hint_cb); @@ -123,12 +122,11 @@ class MEDIA_GPU_EXPORT CodecImage // Whether the codec buffer has been rendered to the front buffer. bool was_rendered_to_front_buffer() const { - return phase_ == Phase::kInFrontBuffer; + return output_buffer_renderer_ + ? output_buffer_renderer_->was_rendered_to_front_buffer() + : false; } - // Whether the TextureOwner's texture is in the front buffer and bound to the - // latest image. - bool was_tex_image_bound() const { return was_tex_image_bound_; } // Whether this image is backed by a texture owner. // We want to check for texture_owner owned by @@ -164,7 +162,9 @@ class MEDIA_GPU_EXPORT CodecImage virtual void ReleaseCodecBuffer(); CodecOutputBuffer* get_codec_output_buffer_for_testing() const { - return output_buffer_.get(); + return output_buffer_renderer_ + ? output_buffer_renderer_->get_codec_output_buffer_for_testing() + : nullptr; } protected: @@ -173,22 +173,12 @@ class MEDIA_GPU_EXPORT CodecImage private: FRIEND_TEST_ALL_PREFIXES(CodecImageTest, RenderAfterUnusedDoesntCrash); - // The lifecycle phases of an image. - // The only possible transitions are from left to right. Both - // kInFrontBuffer and kInvalidated are terminal. - enum class Phase { kInCodec, kInBackBuffer, kInFrontBuffer, kInvalidated }; + std::unique_ptr<CodecOutputBufferRenderer> output_buffer_renderer_; // Renders this image to the texture owner front buffer by first rendering // it to the back buffer if it's not already there, and then waiting for the // frame available event before calling UpdateTexImage(). bool RenderToTextureOwnerFrontBuffer(BindingsMode bindings_mode); - void EnsureBoundIfNeeded(BindingsMode mode); - - // The phase of the image buffer's lifecycle. - Phase phase_ = Phase::kInvalidated; - - // The buffer backing this image. - std::unique_ptr<CodecOutputBuffer> output_buffer_; // The CodecBufferWaitCoordinator that |output_buffer_| will be rendered to. // Or null, if this image is backed by an overlay. @@ -197,13 +187,14 @@ class MEDIA_GPU_EXPORT CodecImage // The bounds last sent to the overlay. gfx::Rect most_recent_bounds_; + // Coded size of the image. + gfx::Size coded_size_; + // Callback to notify about promotion hints and overlay position. PromotionHintAggregator::NotifyPromotionHintCB promotion_hint_cb_; std::vector<UnusedCB> unused_cbs_; - bool was_tex_image_bound_ = false; - DISALLOW_COPY_AND_ASSIGN(CodecImage); }; diff --git a/chromium/media/gpu/android/codec_image_group_unittest.cc b/chromium/media/gpu/android/codec_image_group_unittest.cc index 00cef1d3002..22c72c7ab9c 100644 --- a/chromium/media/gpu/android/codec_image_group_unittest.cc +++ b/chromium/media/gpu/android/codec_image_group_unittest.cc @@ -20,6 +20,9 @@ namespace media { namespace { +// Size used to create MockCodecImage. +constexpr gfx::Size kMockImageSize(100, 100); + // Subclass of CodecImageGroup which will notify us when it's destroyed. class CodecImageGroupWithDestructionHook : public CodecImageGroup { public: @@ -119,7 +122,7 @@ TEST_F(CodecImageGroupTest, ImagesRetainRefToGroup) { bool was_destroyed = false; rec.image_group->SetDestructionCallback( base::BindOnce([](bool* flag) -> void { *flag = true; }, &was_destroyed)); - scoped_refptr<CodecImage> image = new MockCodecImage(); + scoped_refptr<CodecImage> image = new MockCodecImage(kMockImageSize); // We're supposed to call this from |gpu_task_runner_|, but all // CodecImageGroup really cares about is being single sequence. rec.image_group->AddCodecImage(image.get()); @@ -138,8 +141,8 @@ TEST_F(CodecImageGroupTest, ImageGroupDropsForwardsSurfaceDestruction) { // also verify that the image group drops its ref to the surface bundle, so // that it doesn't prevent destruction of the overlay that provided it. Record rec = CreateImageGroup(); - scoped_refptr<MockCodecImage> image_1 = new MockCodecImage(); - scoped_refptr<MockCodecImage> image_2 = new MockCodecImage(); + scoped_refptr<MockCodecImage> image_1 = new MockCodecImage(kMockImageSize); + scoped_refptr<MockCodecImage> image_2 = new MockCodecImage(kMockImageSize); rec.image_group->AddCodecImage(image_1.get()); rec.image_group->AddCodecImage(image_2.get()); diff --git a/chromium/media/gpu/android/codec_image_unittest.cc b/chromium/media/gpu/android/codec_image_unittest.cc index 840cf24f092..7fcd7eeba13 100644 --- a/chromium/media/gpu/android/codec_image_unittest.cc +++ b/chromium/media/gpu/android/codec_image_unittest.cc @@ -6,7 +6,6 @@ #include "base/bind.h" #include "base/bind_helpers.h" -#include "base/logging.h" #include "base/test/mock_callback.h" #include "base/test/task_environment.h" #include "base/threading/sequenced_task_runner_handle.h" @@ -87,10 +86,15 @@ class CodecImageTest : public testing::Test { CodecImage::UnusedCB unused_cb = base::DoNothing()) { std::unique_ptr<CodecOutputBuffer> buffer; wrapper_->DequeueOutputBuffer(nullptr, nullptr, &buffer); - scoped_refptr<CodecImage> image = new CodecImage(); + + auto codec_buffer_wait_coordinator = + kind == kTextureOwner ? codec_buffer_wait_coordinator_ : nullptr; + auto buffer_renderer = std::make_unique<CodecOutputBufferRenderer>( + std::move(buffer), codec_buffer_wait_coordinator); + + scoped_refptr<CodecImage> image = new CodecImage(buffer_renderer->size()); image->Initialize( - std::move(buffer), - kind == kTextureOwner ? codec_buffer_wait_coordinator_ : nullptr, + std::move(buffer_renderer), codec_buffer_wait_coordinator, base::BindRepeating(&PromotionHintReceiver::OnPromotionHint, base::Unretained(&promotion_hint_receiver_))); @@ -217,48 +221,6 @@ TEST_F(CodecImageTestExplicitBind, CopyTexImageTriggersFrontBufferRendering) { ASSERT_TRUE(i->was_rendered_to_front_buffer()); } -TEST_F(CodecImageTest, GetTextureMatrixTriggersFrontBufferRendering) { - auto i = NewImage(kTextureOwner); - InSequence s; - EXPECT_CALL(*codec_, ReleaseOutputBuffer(_, true)); - EXPECT_CALL(*codec_buffer_wait_coordinator_, WaitForFrameAvailable()); - EXPECT_CALL(*codec_buffer_wait_coordinator_->texture_owner(), - UpdateTexImage()); - EXPECT_CALL(*codec_buffer_wait_coordinator_->texture_owner(), - GetTransformMatrix(_)); - float matrix[16]; - i->GetTextureMatrix(matrix); - ASSERT_TRUE(i->was_rendered_to_front_buffer()); -} - -TEST_F(CodecImageTestExplicitBind, - GetTextureMatrixTriggersFrontBufferRendering) { - // GetTextureMatrix should not bind the image. - codec_buffer_wait_coordinator_->texture_owner()->expect_update_tex_image = - false; - - auto i = NewImage(kTextureOwner); - InSequence s; - EXPECT_CALL(*codec_, ReleaseOutputBuffer(_, true)); - EXPECT_CALL(*codec_buffer_wait_coordinator_, WaitForFrameAvailable()); - EXPECT_CALL(*codec_buffer_wait_coordinator_->texture_owner(), - UpdateTexImage()); - EXPECT_CALL(*codec_buffer_wait_coordinator_->texture_owner(), - GetTransformMatrix(_)); - float matrix[16]; - i->GetTextureMatrix(matrix); - ASSERT_TRUE(i->was_rendered_to_front_buffer()); -} - -TEST_F(CodecImageTest, GetTextureMatrixReturnsIdentityForOverlayImages) { - auto i = NewImage(kOverlay); - float matrix[16]{0}; - i->GetTextureMatrix(matrix); - // See GetTextureMatrix() for the expected result. - ASSERT_EQ(matrix[0], 1); - ASSERT_EQ(matrix[5], -1); -} - TEST_F(CodecImageTest, ScheduleOverlayPlaneTriggersFrontBufferRendering) { auto i = NewImage(kOverlay); EXPECT_CALL(*codec_, ReleaseOutputBuffer(_, true)); @@ -413,15 +375,6 @@ TEST_F(CodecImageTest, GetAHardwareBufferAfterRelease) { EXPECT_FALSE(i->GetAHardwareBuffer()); } -TEST_F(CodecImageTest, GetCropRect) { - auto i = NewImage(kTextureOwner); - EXPECT_EQ( - codec_buffer_wait_coordinator_->texture_owner()->get_crop_rect_count, 0); - i->GetCropRect(); - EXPECT_EQ( - codec_buffer_wait_coordinator_->texture_owner()->get_crop_rect_count, 1); -} - TEST_F(CodecImageTest, RenderAfterUnusedDoesntCrash) { auto i = NewImage(kTextureOwner); i->NotifyUnused(); @@ -430,4 +383,20 @@ TEST_F(CodecImageTest, RenderAfterUnusedDoesntCrash) { CodecImage::BindingsMode::kEnsureTexImageBound)); } +TEST_F(CodecImageTest, CodedSizeVsVisibleSize) { + const gfx::Size coded_size(128, 128); + const gfx::Size visible_size(100, 100); + auto buffer = CodecOutputBuffer::CreateForTesting(0, visible_size); + auto buffer_renderer = + std::make_unique<CodecOutputBufferRenderer>(std::move(buffer), nullptr); + + scoped_refptr<CodecImage> image = new CodecImage(coded_size); + image->Initialize(std::move(buffer_renderer), nullptr, + PromotionHintAggregator::NotifyPromotionHintCB()); + + // Verify that CodecImage::GetSize returns coded_size and not visible_size + // that comes in CodecOutputBuffer size. + EXPECT_EQ(image->GetSize(), coded_size); +} + } // namespace media diff --git a/chromium/media/gpu/android/codec_output_buffer_renderer.cc b/chromium/media/gpu/android/codec_output_buffer_renderer.cc new file mode 100644 index 00000000000..57f32eabb24 --- /dev/null +++ b/chromium/media/gpu/android/codec_output_buffer_renderer.cc @@ -0,0 +1,166 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/gpu/android/codec_output_buffer_renderer.h" +#include <string.h> + +#include "base/android/scoped_hardware_buffer_fence_sync.h" +#include "base/bind_helpers.h" +#include "gpu/command_buffer/service/gles2_cmd_decoder.h" +#include "gpu/command_buffer/service/texture_manager.h" +#include "ui/gl/gl_context.h" +#include "ui/gl/scoped_make_current.h" + +namespace media { +namespace { + +// Makes |texture_owner|'s context current if it isn't already. +std::unique_ptr<ui::ScopedMakeCurrent> MakeCurrentIfNeeded( + gpu::TextureOwner* texture_owner) { + gl::GLContext* context = texture_owner->GetContext(); + // Note: this works for virtual contexts too, because IsCurrent() returns true + // if their shared platform context is current, regardless of which virtual + // context is current. + if (context->IsCurrent(nullptr)) + return nullptr; + + auto scoped_current = std::make_unique<ui::ScopedMakeCurrent>( + context, texture_owner->GetSurface()); + // Log an error if ScopedMakeCurrent failed for debugging + // https://crbug.com/878042. + // TODO(ericrk): Remove this once debugging is completed. + if (!context->IsCurrent(nullptr)) { + LOG(ERROR) << "Failed to make context current in CodecImage. Subsequent " + "UpdateTexImage may fail."; + } + return scoped_current; +} + +} // namespace + +CodecOutputBufferRenderer::CodecOutputBufferRenderer( + std::unique_ptr<CodecOutputBuffer> output_buffer, + scoped_refptr<CodecBufferWaitCoordinator> codec_buffer_wait_coordinator) + : output_buffer_(std::move(output_buffer)), + codec_buffer_wait_coordinator_(std::move(codec_buffer_wait_coordinator)) { + +} + +CodecOutputBufferRenderer::~CodecOutputBufferRenderer() = default; + +bool CodecOutputBufferRenderer::RenderToTextureOwnerBackBuffer( + BlockingMode blocking_mode) { + DCHECK_NE(phase_, Phase::kInFrontBuffer); + if (phase_ == Phase::kInBackBuffer) + return true; + if (phase_ == Phase::kInvalidated) + return false; + + // Normally, we should have a wait coordinator if we're called. However, if + // the renderer is torn down (either VideoFrameSubmitter or the whole process) + // before we get returns back from viz, then we can be notified that we're + // no longer in use (erroneously) when the VideoFrame is destroyed. So, if + // we don't have a wait coordinator, then just fail. + if (!codec_buffer_wait_coordinator_) + return false; + + // Wait for a previous frame available so we don't confuse it with the one + // we're about to release. + if (codec_buffer_wait_coordinator_->IsExpectingFrameAvailable()) { + if (blocking_mode == BlockingMode::kForbidBlocking) + return false; + codec_buffer_wait_coordinator_->WaitForFrameAvailable(); + } + if (!output_buffer_->ReleaseToSurface()) { + phase_ = Phase::kInvalidated; + return false; + } + phase_ = Phase::kInBackBuffer; + codec_buffer_wait_coordinator_->SetReleaseTimeToNow(); + return true; +} + +bool CodecOutputBufferRenderer::RenderToTextureOwnerFrontBuffer( + BindingsMode bindings_mode) { + // Normally, we should have a wait coordinator if we're called. However, if + // the renderer is torn down (either VideoFrameSubmitter or the whole process) + // before we get returns back from viz, then we can be notified that we're + // no longer in use (erroneously) when the VideoFrame is destroyed. So, if + // we don't have a wait coordinator, then just fail. + if (!codec_buffer_wait_coordinator_) + return false; + + if (phase_ == Phase::kInFrontBuffer) { + EnsureBoundIfNeeded(bindings_mode); + return true; + } + if (phase_ == Phase::kInvalidated) + return false; + + // Render it to the back buffer if it's not already there. + if (!RenderToTextureOwnerBackBuffer()) + return false; + + // The image is now in the back buffer, so promote it to the front buffer. + phase_ = Phase::kInFrontBuffer; + if (codec_buffer_wait_coordinator_->IsExpectingFrameAvailable()) + codec_buffer_wait_coordinator_->WaitForFrameAvailable(); + + std::unique_ptr<ui::ScopedMakeCurrent> scoped_make_current = + MakeCurrentIfNeeded( + codec_buffer_wait_coordinator_->texture_owner().get()); + // If updating the image will implicitly update the texture bindings then + // restore if requested or the update needed a context switch. + bool should_restore_bindings = + codec_buffer_wait_coordinator_->texture_owner() + ->binds_texture_on_update() && + (bindings_mode == BindingsMode::kRestoreIfBound || !!scoped_make_current); + + GLint bound_service_id = 0; + if (should_restore_bindings) + glGetIntegerv(GL_TEXTURE_BINDING_EXTERNAL_OES, &bound_service_id); + codec_buffer_wait_coordinator_->texture_owner()->UpdateTexImage(); + EnsureBoundIfNeeded(bindings_mode); + if (should_restore_bindings) + glBindTexture(GL_TEXTURE_EXTERNAL_OES, bound_service_id); + return true; +} + +void CodecOutputBufferRenderer::EnsureBoundIfNeeded(BindingsMode mode) { + DCHECK(codec_buffer_wait_coordinator_); + + if (codec_buffer_wait_coordinator_->texture_owner() + ->binds_texture_on_update()) { + was_tex_image_bound_ = true; + return; + } + if (mode != BindingsMode::kEnsureTexImageBound) + return; + codec_buffer_wait_coordinator_->texture_owner()->EnsureTexImageBound(); + was_tex_image_bound_ = true; +} + +bool CodecOutputBufferRenderer::RenderToOverlay() { + if (phase_ == Phase::kInFrontBuffer) + return true; + if (phase_ == Phase::kInvalidated) + return false; + + if (!output_buffer_->ReleaseToSurface()) { + phase_ = Phase::kInvalidated; + return false; + } + phase_ = Phase::kInFrontBuffer; + return true; +} + +bool CodecOutputBufferRenderer::RenderToFrontBuffer() { + // This code is used to trigger early rendering of the image before it is used + // for compositing, there is no need to bind the image. + return codec_buffer_wait_coordinator_ + ? RenderToTextureOwnerFrontBuffer(BindingsMode::kRestoreIfBound) + : RenderToOverlay(); +} + +} // namespace media diff --git a/chromium/media/gpu/android/codec_output_buffer_renderer.h b/chromium/media/gpu/android/codec_output_buffer_renderer.h new file mode 100644 index 00000000000..69e6ea73121 --- /dev/null +++ b/chromium/media/gpu/android/codec_output_buffer_renderer.h @@ -0,0 +1,102 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_GPU_ANDROID_CODEC_OUTPUT_BUFFER_RENDERER_H_ +#define MEDIA_GPU_ANDROID_CODEC_OUTPUT_BUFFER_RENDERER_H_ + +#include <stdint.h> + +#include <memory> + +#include "gpu/command_buffer/service/stream_texture_shared_image_interface.h" +#include "media/gpu/android/codec_buffer_wait_coordinator.h" +#include "media/gpu/android/codec_wrapper.h" +#include "media/gpu/media_gpu_export.h" + +namespace media { + +// A class that holds CodecOutputBuffer and renders it to TextureOwner or +// overlay as necessary. Unit tests for this class are part of CodecImage unit +// tests. +class MEDIA_GPU_EXPORT CodecOutputBufferRenderer { + public: + using BindingsMode = gpu::StreamTextureSharedImageInterface::BindingsMode; + // Whether RenderToTextureOwnerBackBuffer may block or not. + enum class BlockingMode { kForbidBlocking, kAllowBlocking }; + + CodecOutputBufferRenderer( + std::unique_ptr<CodecOutputBuffer> output_buffer, + scoped_refptr<CodecBufferWaitCoordinator> codec_buffer_wait_coordinator); + ~CodecOutputBufferRenderer(); + + CodecOutputBufferRenderer(const CodecOutputBufferRenderer&) = delete; + CodecOutputBufferRenderer& operator=(const CodecOutputBufferRenderer&) = + delete; + + // Renders this image to the overlay. Returns true if the buffer is in the + // overlay front buffer. Returns false if the buffer was invalidated. + bool RenderToOverlay(); + + // Renders this image to the texture owner front buffer by first rendering + // it to the back buffer if it's not already there, and then waiting for the + // frame available event before calling UpdateTexImage(). + bool RenderToTextureOwnerFrontBuffer(BindingsMode bindings_mode); + + // Renders this image to the front buffer of its backing surface. + // Returns true if the buffer is in the front buffer. Returns false if the + // buffer was invalidated. After an image is invalidated it's no longer + // possible to render it. + bool RenderToFrontBuffer(); + + // Renders this image to the back buffer of its texture owner. Only valid if + // is_texture_owner_backed(). Returns true if the buffer is in the back + // buffer. Returns false if the buffer was invalidated. + // |blocking_mode| indicates whether this should (a) wait for any previously + // pending rendered frame before rendering this one, or (b) fail if a wait + // is required. + bool RenderToTextureOwnerBackBuffer( + BlockingMode blocking_mode = BlockingMode::kAllowBlocking); + + // Whether the codec buffer has been rendered to the front buffer. + bool was_rendered_to_front_buffer() const { + return phase_ == Phase::kInFrontBuffer; + } + + gfx::Size size() const { return output_buffer_->size(); } + + bool was_tex_image_bound() const { return was_tex_image_bound_; } + + scoped_refptr<gpu::TextureOwner> texture_owner() const { + return codec_buffer_wait_coordinator_ + ? codec_buffer_wait_coordinator_->texture_owner() + : nullptr; + } + + CodecOutputBuffer* get_codec_output_buffer_for_testing() const { + return output_buffer_.get(); + } + + private: + // The lifecycle phases of an buffer. + // The only possible transitions are from left to right. Both + // kInFrontBuffer and kInvalidated are terminal. + enum class Phase { kInCodec, kInBackBuffer, kInFrontBuffer, kInvalidated }; + + void EnsureBoundIfNeeded(BindingsMode mode); + + // The phase of the image buffer's lifecycle. + Phase phase_ = Phase::kInCodec; + + // The buffer backing this image. + std::unique_ptr<CodecOutputBuffer> output_buffer_; + + // The CodecBufferWaitCoordinator that |output_buffer_| will be rendered to. + // Or null, if this image is backed by an overlay. + scoped_refptr<CodecBufferWaitCoordinator> codec_buffer_wait_coordinator_; + + bool was_tex_image_bound_ = false; +}; + +} // namespace media +#endif // MEDIA_GPU_ANDROID_CODEC_OUTPUT_BUFFER_RENDERER_H diff --git a/chromium/media/gpu/android/codec_wrapper_unittest.cc b/chromium/media/gpu/android/codec_wrapper_unittest.cc index f6973a2ad73..66c48ad7f3e 100644 --- a/chromium/media/gpu/android/codec_wrapper_unittest.cc +++ b/chromium/media/gpu/android/codec_wrapper_unittest.cc @@ -5,7 +5,6 @@ #include <memory> #include "base/bind.h" -#include "base/logging.h" #include "base/memory/ref_counted.h" #include "base/synchronization/waitable_event.h" #include "base/test/mock_callback.h" diff --git a/chromium/media/gpu/android/direct_shared_image_video_provider.cc b/chromium/media/gpu/android/direct_shared_image_video_provider.cc index e4b6b40d02b..e9383411ff6 100644 --- a/chromium/media/gpu/android/direct_shared_image_video_provider.cc +++ b/chromium/media/gpu/android/direct_shared_image_video_provider.cc @@ -148,7 +148,7 @@ void GpuSharedImageVideoFactory::CreateImage( // Generate a shared image mailbox. auto mailbox = gpu::Mailbox::GenerateForSharedImage(); - auto codec_image = base::MakeRefCounted<CodecImage>(); + auto codec_image = base::MakeRefCounted<CodecImage>(spec.coded_size); TRACE_EVENT0("media", "GpuSharedImageVideoFactory::CreateVideoFrame"); @@ -200,14 +200,14 @@ bool GpuSharedImageVideoFactory::CreateImageInternal( if (!group) return false; - const auto& size = spec.size; + const auto& coded_size = spec.coded_size; // Create a Texture and a CodecImage to back it. // TODO(liberato): Once legacy mailbox support is removed, we don't need to // create this texture. So, we won't need |texture_owner| either. std::unique_ptr<AbstractTexture> texture = decoder_helper_->CreateTexture( - GL_TEXTURE_EXTERNAL_OES, GL_RGBA, size.width(), size.height(), GL_RGBA, - GL_UNSIGNED_BYTE); + GL_TEXTURE_EXTERNAL_OES, GL_RGBA, coded_size.width(), coded_size.height(), + GL_RGBA, GL_UNSIGNED_BYTE); // Attach the image to the texture. // Either way, we expect this to be UNBOUND (i.e., decoder-managed). For @@ -239,7 +239,7 @@ bool GpuSharedImageVideoFactory::CreateImageInternal( // TODO(vikassoni): This shared image need to be thread safe eventually for // webview to work with shared images. auto shared_image = std::make_unique<gpu::SharedImageVideo>( - mailbox, size, gfx::ColorSpace::CreateSRGB(), std::move(image), + mailbox, coded_size, gfx::ColorSpace::CreateSRGB(), std::move(image), std::move(texture), std::move(shared_context), false /* is_thread_safe */); diff --git a/chromium/media/gpu/android/frame_info_helper.cc b/chromium/media/gpu/android/frame_info_helper.cc new file mode 100644 index 00000000000..efe1873cb1c --- /dev/null +++ b/chromium/media/gpu/android/frame_info_helper.cc @@ -0,0 +1,120 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/gpu/android/frame_info_helper.h" + +#include "gpu/command_buffer/service/shared_image_video.h" +#include "gpu/ipc/service/command_buffer_stub.h" +#include "gpu/ipc/service/gpu_channel.h" +#include "gpu/ipc/service/gpu_channel_manager.h" + +namespace media { + +FrameInfoHelper::FrameInfo::FrameInfo() = default; +FrameInfoHelper::FrameInfo::~FrameInfo() = default; +FrameInfoHelper::FrameInfo::FrameInfo(FrameInfo&&) = default; +FrameInfoHelper::FrameInfo::FrameInfo(const FrameInfoHelper::FrameInfo&) = + default; +FrameInfoHelper::FrameInfo& FrameInfoHelper::FrameInfo::operator=( + const FrameInfoHelper::FrameInfo&) = default; + +// Concrete implementation of FrameInfoHelper that renders output buffers and +// gets the FrameInfo they need. +class FrameInfoHelperImpl : public FrameInfoHelper, + public gpu::CommandBufferStub::DestructionObserver { + public: + FrameInfoHelperImpl(SharedImageVideoProvider::GetStubCB get_stub_cb) { + stub_ = get_stub_cb.Run(); + if (stub_) + stub_->AddDestructionObserver(this); + } + + ~FrameInfoHelperImpl() override { + if (stub_) + stub_->RemoveDestructionObserver(this); + } + + void GetFrameInfo( + std::unique_ptr<CodecOutputBufferRenderer> buffer_renderer, + base::OnceCallback< + void(std::unique_ptr<CodecOutputBufferRenderer>, FrameInfo, bool)> cb) + override { + if (!buffer_renderer) { + std::move(cb).Run(nullptr, FrameInfo(), false); + return; + } + + auto texture_owner = buffer_renderer->texture_owner(); + + FrameInfo info; + + // Indicates that the FrameInfo is reliable and can be cached by caller. + // It's true if we either return cached values or we attempted to render + // frame and succeeded. + bool success = true; + + // We default to visible size if if we can't get real size + info.coded_size = buffer_renderer->size(); + info.visible_rect = gfx::Rect(info.coded_size); + + if (texture_owner) { + if (visible_size_ == buffer_renderer->size()) { + info = frame_info_; + } else if (buffer_renderer->RenderToTextureOwnerFrontBuffer( + CodecOutputBufferRenderer::BindingsMode:: + kDontRestoreIfBound)) { + visible_size_ = buffer_renderer->size(); + texture_owner->GetCodedSizeAndVisibleRect( + visible_size_, &frame_info_.coded_size, &frame_info_.visible_rect); + + frame_info_.ycbcr_info = GetYCbCrInfo(texture_owner.get()); + info = frame_info_; + } else { + // We attempted to render frame and failed, mark request as failed so + // caller won't cache best-guess values. + success = false; + } + } + + std::move(cb).Run(std::move(buffer_renderer), frame_info_, success); + } + + void OnWillDestroyStub(bool have_context) override { + DCHECK(stub_); + stub_ = nullptr; + } + + private: + // Gets YCbCrInfo from last rendered frame. + base::Optional<gpu::VulkanYCbCrInfo> GetYCbCrInfo( + gpu::TextureOwner* texture_owner) { + gpu::ContextResult result; + if (!stub_) + return base::nullopt; + + auto shared_context = + stub_->channel()->gpu_channel_manager()->GetSharedContextState(&result); + auto context_provider = + (result == gpu::ContextResult::kSuccess) ? shared_context : nullptr; + if (!context_provider) + return base::nullopt; + + return gpu::SharedImageVideo::GetYcbcrInfo(texture_owner, context_provider); + } + + gpu::CommandBufferStub* stub_ = nullptr; + + FrameInfo frame_info_; + gfx::Size visible_size_; +}; + +// static +base::SequenceBound<FrameInfoHelper> FrameInfoHelper::Create( + scoped_refptr<base::SequencedTaskRunner> gpu_task_runner, + SharedImageVideoProvider::GetStubCB get_stub_cb) { + return base::SequenceBound<FrameInfoHelperImpl>(std::move(gpu_task_runner), + std::move(get_stub_cb)); +} + +} // namespace media diff --git a/chromium/media/gpu/android/frame_info_helper.h b/chromium/media/gpu/android/frame_info_helper.h new file mode 100644 index 00000000000..5fc4ffca328 --- /dev/null +++ b/chromium/media/gpu/android/frame_info_helper.h @@ -0,0 +1,63 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_GPU_ANDROID_FRAME_INFO_HELPER_H_ +#define MEDIA_GPU_ANDROID_FRAME_INFO_HELPER_H_ + +#include "base/optional.h" +#include "base/threading/sequence_bound.h" +#include "media/gpu/android/codec_image.h" +#include "media/gpu/android/shared_image_video_provider.h" +#include "media/gpu/media_gpu_export.h" + +namespace media { + +// Helper class to fetch YCbCrInfo for Vulkan from a CodecImage. +class MEDIA_GPU_EXPORT FrameInfoHelper { + public: + struct FrameInfo { + FrameInfo(); + ~FrameInfo(); + + FrameInfo(FrameInfo&&); + FrameInfo(const FrameInfo&); + FrameInfo& operator=(const FrameInfo&); + + gfx::Size coded_size; + gfx::Rect visible_rect; + base::Optional<gpu::VulkanYCbCrInfo> ycbcr_info; + }; + + static base::SequenceBound<FrameInfoHelper> Create( + scoped_refptr<base::SequencedTaskRunner> gpu_task_runner, + SharedImageVideoProvider::GetStubCB get_stub_cb); + + virtual ~FrameInfoHelper() = default; + + // Call |cb| with the FrameInfo. Will render |buffer_renderer| to the front + // buffer if we don't have frame info cached. For Vulkan this also will + // attempt to get YCbCrInfo and cache it. If all necessary info is cached the + // call will leave buffer_renderer intact and it can be rendered later. + // Rendering can fail for reasons. This function will make best efforts to + // fill FrameInfo which can be used to create VideoFrame, but shouldn't be + // cached by caller. Last parameter in |cb| is bool that indicates that info + // is reliable. + // + // While this API might seem to be out of its Vulkan mind, it's this + // complicated to (a) prevent rendering frames out of order to the front + // buffer, and (b) make it easy to handle the fact that sometimes, we just + // can't get a YCbCrInfo from a CodecImage due to timeouts. + virtual void GetFrameInfo( + std::unique_ptr<CodecOutputBufferRenderer> buffer_renderer, + base::OnceCallback<void(std::unique_ptr<CodecOutputBufferRenderer>, + FrameInfo, + bool)> cb) = 0; + + protected: + FrameInfoHelper() = default; +}; + +} // namespace media + +#endif // MEDIA_GPU_ANDROID_FRAME_INFO_HELPER_H_ diff --git a/chromium/media/gpu/android/mock_codec_image.cc b/chromium/media/gpu/android/mock_codec_image.cc index 09901993ac7..aca947f798b 100644 --- a/chromium/media/gpu/android/mock_codec_image.cc +++ b/chromium/media/gpu/android/mock_codec_image.cc @@ -6,7 +6,8 @@ namespace media { -MockCodecImage::MockCodecImage() = default; +MockCodecImage::MockCodecImage(const gfx::Size& coded_size) + : CodecImage(coded_size) {} MockCodecImage::~MockCodecImage() = default; diff --git a/chromium/media/gpu/android/mock_codec_image.h b/chromium/media/gpu/android/mock_codec_image.h index aa83b341a50..6020705e0b3 100644 --- a/chromium/media/gpu/android/mock_codec_image.h +++ b/chromium/media/gpu/android/mock_codec_image.h @@ -15,7 +15,7 @@ namespace media { // CodecImage with a mocked ReleaseCodecBuffer. class MockCodecImage : public CodecImage { public: - MockCodecImage(); + MockCodecImage(const gfx::Size& coded_size); MOCK_METHOD0(ReleaseCodecBuffer, void()); diff --git a/chromium/media/gpu/android/promotion_hint_aggregator_impl_unittest.cc b/chromium/media/gpu/android/promotion_hint_aggregator_impl_unittest.cc index 113b1c953aa..4e7e04acbfa 100644 --- a/chromium/media/gpu/android/promotion_hint_aggregator_impl_unittest.cc +++ b/chromium/media/gpu/android/promotion_hint_aggregator_impl_unittest.cc @@ -9,7 +9,6 @@ #include <memory> #include "base/bind.h" -#include "base/logging.h" #include "base/test/simple_test_tick_clock.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/chromium/media/gpu/android/shared_image_video_provider.cc b/chromium/media/gpu/android/shared_image_video_provider.cc index c543495f253..427987f6624 100644 --- a/chromium/media/gpu/android/shared_image_video_provider.cc +++ b/chromium/media/gpu/android/shared_image_video_provider.cc @@ -9,13 +9,13 @@ namespace media { SharedImageVideoProvider::ImageSpec::ImageSpec() = default; SharedImageVideoProvider::ImageSpec::ImageSpec(const gfx::Size& our_size, uint64_t our_generation_id) - : size(our_size), generation_id(our_generation_id) {} + : coded_size(our_size), generation_id(our_generation_id) {} SharedImageVideoProvider::ImageSpec::ImageSpec(const ImageSpec&) = default; SharedImageVideoProvider::ImageSpec::~ImageSpec() = default; bool SharedImageVideoProvider::ImageSpec::operator==( const ImageSpec& rhs) const { - return size == rhs.size && generation_id == rhs.generation_id; + return coded_size == rhs.coded_size && generation_id == rhs.generation_id; } bool SharedImageVideoProvider::ImageSpec::operator!=( diff --git a/chromium/media/gpu/android/shared_image_video_provider.h b/chromium/media/gpu/android/shared_image_video_provider.h index 83332f58811..ffbac1331a5 100644 --- a/chromium/media/gpu/android/shared_image_video_provider.h +++ b/chromium/media/gpu/android/shared_image_video_provider.h @@ -31,12 +31,12 @@ class MEDIA_GPU_EXPORT SharedImageVideoProvider { // Description of the underlying properties of the shared image. struct ImageSpec { ImageSpec(); - ImageSpec(const gfx::Size& size, uint64_t generation_id); + ImageSpec(const gfx::Size& coded_size, uint64_t generation_id); ImageSpec(const ImageSpec&); ~ImageSpec(); // Size of the underlying texture. - gfx::Size size; + gfx::Size coded_size; // This is a hack to allow us to discard pooled images if the TextureOwner // changes. We don't want to keep a ref to the TextureOwner here, so we diff --git a/chromium/media/gpu/android/surface_chooser_helper_unittest.cc b/chromium/media/gpu/android/surface_chooser_helper_unittest.cc index f9f4506fc85..353e9b0bfde 100644 --- a/chromium/media/gpu/android/surface_chooser_helper_unittest.cc +++ b/chromium/media/gpu/android/surface_chooser_helper_unittest.cc @@ -9,7 +9,6 @@ #include <memory> #include "base/bind.h" -#include "base/logging.h" #include "base/test/simple_test_tick_clock.h" #include "media/gpu/android/mock_android_video_surface_chooser.h" #include "media/gpu/android/mock_promotion_hint_aggregator.h" diff --git a/chromium/media/gpu/android/video_frame_factory_impl.cc b/chromium/media/gpu/android/video_frame_factory_impl.cc index 78b08ea3fc3..b7c768bae0c 100644 --- a/chromium/media/gpu/android/video_frame_factory_impl.cc +++ b/chromium/media/gpu/android/video_frame_factory_impl.cc @@ -81,13 +81,13 @@ VideoFrameFactoryImpl::VideoFrameFactoryImpl( const gpu::GpuPreferences& gpu_preferences, std::unique_ptr<SharedImageVideoProvider> image_provider, std::unique_ptr<MaybeRenderEarlyManager> mre_manager, - base::SequenceBound<YCbCrHelper> ycbcr_helper) + base::SequenceBound<FrameInfoHelper> frame_info_helper) : image_provider_(std::move(image_provider)), gpu_task_runner_(std::move(gpu_task_runner)), enable_threaded_texture_mailboxes_( gpu_preferences.enable_threaded_texture_mailboxes), mre_manager_(std::move(mre_manager)), - ycbcr_helper_(std::move(ycbcr_helper)) {} + frame_info_helper_(std::move(frame_info_helper)) {} VideoFrameFactoryImpl::~VideoFrameFactoryImpl() { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); @@ -147,6 +147,9 @@ void VideoFrameFactoryImpl::CreateVideoFrame( gfx::Size coded_size = output_buffer->size(); gfx::Rect visible_rect(coded_size); + auto output_buffer_renderer = std::make_unique<CodecOutputBufferRenderer>( + std::move(output_buffer), codec_buffer_wait_coordinator_); + // The pixel format doesn't matter here as long as it's valid for texture // frames. But SkiaRenderer wants to ensure that the format of the resource // used here which will eventually create a promise image must match the @@ -165,21 +168,76 @@ void VideoFrameFactoryImpl::CreateVideoFrame( return; } - // Update the current spec to match the size. - image_spec_.size = coded_size; + auto image_ready_cb = + base::BindOnce(&VideoFrameFactoryImpl::CreateVideoFrame_OnImageReady, + weak_factory_.GetWeakPtr(), std::move(output_cb), + timestamp, natural_size, codec_buffer_wait_coordinator_, + std::move(promotion_hint_cb), pixel_format, overlay_mode_, + enable_threaded_texture_mailboxes_, gpu_task_runner_); - auto image_ready_cb = base::BindOnce( - &VideoFrameFactoryImpl::CreateVideoFrame_OnImageReady, - weak_factory_.GetWeakPtr(), std::move(output_cb), timestamp, coded_size, - natural_size, std::move(output_buffer), codec_buffer_wait_coordinator_, - std::move(promotion_hint_cb), pixel_format, overlay_mode_, - enable_threaded_texture_mailboxes_, gpu_task_runner_); - - image_provider_->RequestImage( - std::move(image_ready_cb), image_spec_, - codec_buffer_wait_coordinator_ - ? codec_buffer_wait_coordinator_->texture_owner() - : nullptr); + RequestImage(std::move(output_buffer_renderer), std::move(image_ready_cb)); +} + +void VideoFrameFactoryImpl::RequestImage( + std::unique_ptr<CodecOutputBufferRenderer> buffer_renderer, + ImageWithInfoReadyCB image_ready_cb) { + if (buffer_renderer && visible_size_ == buffer_renderer->size()) { + auto cb = base::BindOnce(std::move(image_ready_cb), + std::move(buffer_renderer), frame_info_); + + image_provider_->RequestImage( + std::move(cb), image_spec_, + codec_buffer_wait_coordinator_ + ? codec_buffer_wait_coordinator_->texture_owner() + : nullptr); + return; + } + + // We need to reset size to make sure VFFI pipeline is still ordered. + // e.g: CreateVideoFrame is called with new size. We post task to GPU thread + // to get new frame info. While we wait CreateVideoFrame might be called with + // old size again and if we don't reset size here we will skip GPU hop and new + // frame will be created earlier than first one. + visible_size_ = gfx::Size(); + + auto info_cb = BindToCurrentLoop( + base::BindOnce(&VideoFrameFactoryImpl::CreateVideoFrame_OnFrameInfoReady, + weak_factory_.GetWeakPtr(), std::move(image_ready_cb), + codec_buffer_wait_coordinator_)); + + frame_info_helper_.Post(FROM_HERE, &FrameInfoHelper::GetFrameInfo, + std::move(buffer_renderer), std::move(info_cb)); +} + +void VideoFrameFactoryImpl::CreateVideoFrame_OnFrameInfoReady( + ImageWithInfoReadyCB image_ready_cb, + scoped_refptr<CodecBufferWaitCoordinator> codec_buffer_wait_coordinator, + std::unique_ptr<CodecOutputBufferRenderer> output_buffer_renderer, + FrameInfoHelper::FrameInfo frame_info, + bool success) { + // To get frame info we need to render frame which might fail for variety of + // reason. FrameInfoHelper will provide best values we can proceed with, but + // we should not cache it and attempt to get info for next frame. + if (success) { + frame_info_ = frame_info; + visible_size_ = output_buffer_renderer->size(); + } + + // If we don't have output buffer here we can't rely on reply from + // FrameInfoHelper as there might be not cached value and we can't render + // nothing. But in this case call comes from RunAfterPendingVideoFrames and we + // just want to ask for the same image spec as before to order callback after + // all RequestImage, so skip updating image_spec_ in this case. + if (output_buffer_renderer) + image_spec_.coded_size = frame_info.coded_size; + + auto cb = base::BindOnce(std::move(image_ready_cb), + std::move(output_buffer_renderer), frame_info); + + auto texture_owner = codec_buffer_wait_coordinator + ? codec_buffer_wait_coordinator->texture_owner() + : nullptr; + image_provider_->RequestImage(std::move(cb), image_spec_, texture_owner); } // static @@ -187,15 +245,15 @@ void VideoFrameFactoryImpl::CreateVideoFrame_OnImageReady( base::WeakPtr<VideoFrameFactoryImpl> thiz, OnceOutputCB output_cb, base::TimeDelta timestamp, - gfx::Size coded_size, gfx::Size natural_size, - std::unique_ptr<CodecOutputBuffer> output_buffer, scoped_refptr<CodecBufferWaitCoordinator> codec_buffer_wait_coordinator, PromotionHintAggregator::NotifyPromotionHintCB promotion_hint_cb, VideoPixelFormat pixel_format, OverlayMode overlay_mode, bool enable_threaded_texture_mailboxes, scoped_refptr<base::SequencedTaskRunner> gpu_task_runner, + std::unique_ptr<CodecOutputBufferRenderer> output_buffer_renderer, + FrameInfoHelper::FrameInfo frame_info, SharedImageVideoProvider::ImageRecord record) { TRACE_EVENT0("media", "VideoVideoFrameFactoryImpl::OnVideoFrameImageReady"); @@ -210,7 +268,7 @@ void VideoFrameFactoryImpl::CreateVideoFrame_OnImageReady( // When we remove the output buffer management from CodecImage, then that's // what we'd have a reference to here rather than CodecImage. record.codec_image_holder->codec_image_raw()->Initialize( - std::move(output_buffer), codec_buffer_wait_coordinator, + std::move(output_buffer_renderer), codec_buffer_wait_coordinator, std::move(promotion_hint_cb)); // Send the CodecImage (via holder, since we can't touch the refcount here) to @@ -221,75 +279,16 @@ void VideoFrameFactoryImpl::CreateVideoFrame_OnImageReady( // record before we move it into |completion_cb|. auto codec_image_holder = std::move(record.codec_image_holder); - // Doesn't need to be weak-ptr'd, since we're either calling it inline, or - // calling it from the YCbCr callback which is, itself weak-ptr'd. - auto completion_cb = base::BindOnce( - &VideoFrameFactoryImpl::CreateVideoFrame_Finish, thiz, - std::move(output_cb), timestamp, coded_size, natural_size, - std::move(codec_buffer_wait_coordinator), pixel_format, overlay_mode, - enable_threaded_texture_mailboxes, std::move(record)); - - // TODO(liberato): Use |ycbcr_helper_| as a signal about whether we're - // supposed to get YCbCr info or not, rather than requiring the provider to - // tell us. Note that right now, we do have the helper even if we don't - // need it. See GpuMojoMediaClient. - if (!thiz->ycbcr_info_ && record.is_vulkan) { - // We need YCbCr info to create the frame. Post back to the gpu thread to - // do it. Note that we might post multiple times before succeeding once, - // both because of failures and because we might get multiple requests to - // create frames on the mcvd thread, before the gpu thread returns one ycbcr - // info to us. Either way, it's fine, since the helper also caches the - // info locally. It won't render more frames than needed. - auto ycbcr_cb = BindToCurrentLoop(base::BindOnce( - &VideoFrameFactoryImpl::CreateVideoFrame_OnYCbCrInfo, - thiz->weak_factory_.GetWeakPtr(), std::move(completion_cb))); - thiz->ycbcr_helper_.Post(FROM_HERE, &YCbCrHelper::GetYCbCrInfo, - std::move(codec_image_holder), - std::move(ycbcr_cb)); - return; - } - - std::move(completion_cb).Run(); -} - -void VideoFrameFactoryImpl::CreateVideoFrame_OnYCbCrInfo( - base::OnceClosure completion_cb, - YCbCrHelper::OptionalInfo ycbcr_info) { - ycbcr_info_ = std::move(ycbcr_info); - if (ycbcr_info_) { - // Clear the helper just to free it up, though we might continue to get - // callbacks from it if we've posted multiple requests. - // - // We only do this if we actually get the info; we should continue to ask - // if we don't. This can happen if, for example, the frame failed to render - // due to a timeout. - ycbcr_helper_.Reset(); - } - std::move(completion_cb).Run(); -} - -void VideoFrameFactoryImpl::CreateVideoFrame_Finish( - OnceOutputCB output_cb, - base::TimeDelta timestamp, - gfx::Size coded_size, - gfx::Size natural_size, - scoped_refptr<CodecBufferWaitCoordinator> codec_buffer_wait_coordinator, - VideoPixelFormat pixel_format, - OverlayMode overlay_mode, - bool enable_threaded_texture_mailboxes, - SharedImageVideoProvider::ImageRecord record) { gpu::MailboxHolder mailbox_holders[VideoFrame::kMaxPlanes]; mailbox_holders[0] = gpu::MailboxHolder(record.mailbox, gpu::SyncToken(), GL_TEXTURE_EXTERNAL_OES); - gfx::Rect visible_rect(coded_size); - auto frame = VideoFrame::WrapNativeTextures( - pixel_format, mailbox_holders, VideoFrame::ReleaseMailboxCB(), coded_size, - visible_rect, natural_size, timestamp); + pixel_format, mailbox_holders, VideoFrame::ReleaseMailboxCB(), + frame_info.coded_size, frame_info.visible_rect, natural_size, timestamp); // For Vulkan. - frame->set_ycbcr_info(ycbcr_info_); + frame->set_ycbcr_info(frame_info.ycbcr_info); // If, for some reason, we failed to create a frame, then fail. Note that we // don't need to call |release_cb|; dropping it is okay since the api says so. @@ -370,17 +369,15 @@ void VideoFrameFactoryImpl::RunAfterPendingVideoFrames( auto image_ready_cb = base::BindOnce( [](base::OnceClosure closure, + std::unique_ptr<CodecOutputBufferRenderer> output_buffer_renderer, + FrameInfoHelper::FrameInfo frame_info, SharedImageVideoProvider::ImageRecord record) { // Ignore |record| since we don't actually need an image. std::move(closure).Run(); }, std::move(closure)); - image_provider_->RequestImage( - std::move(image_ready_cb), image_spec_, - codec_buffer_wait_coordinator_ - ? codec_buffer_wait_coordinator_->texture_owner() - : nullptr); + RequestImage(nullptr, std::move(image_ready_cb)); } } // namespace media diff --git a/chromium/media/gpu/android/video_frame_factory_impl.h b/chromium/media/gpu/android/video_frame_factory_impl.h index 9b4c5f50329..624d7d2b650 100644 --- a/chromium/media/gpu/android/video_frame_factory_impl.h +++ b/chromium/media/gpu/android/video_frame_factory_impl.h @@ -16,10 +16,10 @@ #include "media/gpu/android/codec_buffer_wait_coordinator.h" #include "media/gpu/android/codec_image.h" #include "media/gpu/android/codec_wrapper.h" +#include "media/gpu/android/frame_info_helper.h" #include "media/gpu/android/maybe_render_early_manager.h" #include "media/gpu/android/shared_image_video_provider.h" #include "media/gpu/android/video_frame_factory.h" -#include "media/gpu/android/ycbcr_helper.h" #include "media/gpu/media_gpu_export.h" #include "ui/gl/gl_bindings.h" @@ -41,13 +41,18 @@ class MEDIA_GPU_EXPORT VideoFrameFactoryImpl : public VideoFrameFactory { base::OnceCallback<void(gpu::Mailbox mailbox, VideoFrame::ReleaseMailboxCB release_cb)>; + using ImageWithInfoReadyCB = + base::OnceCallback<void(std::unique_ptr<CodecOutputBufferRenderer>, + FrameInfoHelper::FrameInfo, + SharedImageVideoProvider::ImageRecord)>; + // |get_stub_cb| will be run on |gpu_task_runner|. VideoFrameFactoryImpl( scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner, const gpu::GpuPreferences& gpu_preferences, std::unique_ptr<SharedImageVideoProvider> image_provider, std::unique_ptr<MaybeRenderEarlyManager> mre_manager, - base::SequenceBound<YCbCrHelper> ycbcr_helper); + base::SequenceBound<FrameInfoHelper> frame_info_helper); ~VideoFrameFactoryImpl() override; void Initialize(OverlayMode overlay_mode, InitCB init_cb) override; @@ -68,6 +73,8 @@ class MEDIA_GPU_EXPORT VideoFrameFactoryImpl : public VideoFrameFactory { } private: + void RequestImage(std::unique_ptr<CodecOutputBufferRenderer> buffer_renderer, + ImageWithInfoReadyCB image_ready_cb); // ImageReadyCB that will construct a VideoFrame, and forward it to // |output_cb| if construction succeeds. This is static for two reasons. // First, we want to snapshot the state of the world when the request is made, @@ -83,33 +90,23 @@ class MEDIA_GPU_EXPORT VideoFrameFactoryImpl : public VideoFrameFactory { base::WeakPtr<VideoFrameFactoryImpl> thiz, OnceOutputCB output_cb, base::TimeDelta timestamp, - gfx::Size coded_size, gfx::Size natural_size, - std::unique_ptr<CodecOutputBuffer> output_buffer, scoped_refptr<CodecBufferWaitCoordinator> codec_buffer_wait_coordinator, PromotionHintAggregator::NotifyPromotionHintCB promotion_hint_cb, VideoPixelFormat pixel_format, OverlayMode overlay_mode, bool enable_threaded_texture_mailboxes, scoped_refptr<base::SequencedTaskRunner> gpu_task_runner, + std::unique_ptr<CodecOutputBufferRenderer> output_buffer_renderer, + FrameInfoHelper::FrameInfo frame_info, SharedImageVideoProvider::ImageRecord record); - // Callback to receive YCbCrInfo from |provider_| while creating a VideoFrame. - void CreateVideoFrame_OnYCbCrInfo(base::OnceClosure completion_cb, - YCbCrHelper::OptionalInfo ycbcr_info); - - // Really create the VideoFrame, once we've tried to get the YCbCrInfo if it's - // needed for it. - void CreateVideoFrame_Finish( - OnceOutputCB output_cb, - base::TimeDelta timestamp, - gfx::Size coded_size, - gfx::Size natural_size, + void CreateVideoFrame_OnFrameInfoReady( + ImageWithInfoReadyCB image_ready_cb, scoped_refptr<CodecBufferWaitCoordinator> codec_buffer_wait_coordinator, - VideoPixelFormat pixel_format, - OverlayMode overlay_mode, - bool enable_threaded_texture_mailboxes, - SharedImageVideoProvider::ImageRecord record); + std::unique_ptr<CodecOutputBufferRenderer> output_buffer_renderer, + FrameInfoHelper::FrameInfo frame_info, + bool success); MaybeRenderEarlyManager* mre_manager() const { return mre_manager_.get(); } @@ -131,11 +128,12 @@ class MEDIA_GPU_EXPORT VideoFrameFactoryImpl : public VideoFrameFactory { std::unique_ptr<MaybeRenderEarlyManager> mre_manager_; - // Sampler conversion information which is used in vulkan context. - YCbCrHelper::OptionalInfo ycbcr_info_; + // Caches FrameInfo and visible size it was cached for. + gfx::Size visible_size_; + FrameInfoHelper::FrameInfo frame_info_; // Optional helper to get the Vulkan YCbCrInfo. - base::SequenceBound<YCbCrHelper> ycbcr_helper_; + base::SequenceBound<FrameInfoHelper> frame_info_helper_; // The current image spec that we'll use to request images. SharedImageVideoProvider::ImageSpec image_spec_; diff --git a/chromium/media/gpu/android/video_frame_factory_impl_unittest.cc b/chromium/media/gpu/android/video_frame_factory_impl_unittest.cc index eda146a5425..ade0a27c05d 100644 --- a/chromium/media/gpu/android/video_frame_factory_impl_unittest.cc +++ b/chromium/media/gpu/android/video_frame_factory_impl_unittest.cc @@ -41,21 +41,49 @@ class MockMaybeRenderEarlyManager : public MaybeRenderEarlyManager { MOCK_METHOD0(MaybeRenderEarly, void()); }; -class MockYCbCrHelper : public YCbCrHelper, public DestructionObservable { +class MockFrameInfoHelper : public FrameInfoHelper, + public DestructionObservable { public: - MockYCbCrHelper(MockYCbCrHelper** thiz) { *thiz = this; } - - void GetYCbCrInfo( - scoped_refptr<CodecImageHolder> codec_image_holder, - base::OnceCallback<void(OptionalInfo ycbcr_info)> cb) override { - MockGetYCbCrInfo(codec_image_holder); + MockFrameInfoHelper(MockFrameInfoHelper** thiz) { *thiz = this; } + + void GetFrameInfo( + std::unique_ptr<CodecOutputBufferRenderer> buffer_renderer, + base::OnceCallback< + void(std::unique_ptr<CodecOutputBufferRenderer>, FrameInfo, bool)> cb) + override { + MockGetFrameInfo(buffer_renderer.get()); cb_ = std::move(cb); + buffer_renderer_ = std::move(buffer_renderer); + + if (run_callback_automatically_) { + RunWithYcbCrInfo(true); + base::RunLoop().RunUntilIdle(); + } } - MOCK_METHOD1(MockGetYCbCrInfo, - void(scoped_refptr<CodecImageHolder> codec_image_holder)); + void RunWithYcbCrInfo(bool success) { + DCHECK(buffer_renderer_); + + FrameInfo info; + info.coded_size = buffer_renderer_->size(); + info.visible_rect = gfx::Rect(info.coded_size); + + std::move(cb_).Run(std::move(buffer_renderer_), info, success); + } - base::OnceCallback<void(OptionalInfo ycbcr_info)> cb_; + void set_run_callback_automatically(bool run_callback_automatically) { + run_callback_automatically_ = run_callback_automatically; + } + + MOCK_METHOD1(MockGetFrameInfo, + void(CodecOutputBufferRenderer* buffer_renderer)); + + private: + bool run_callback_automatically_ = true; + base::OnceCallback< + void(std::unique_ptr<CodecOutputBufferRenderer>, FrameInfo, bool)> + cb_; + std::unique_ptr<CodecOutputBufferRenderer> buffer_renderer_; }; class VideoFrameFactoryImplTest : public testing::Test { @@ -68,8 +96,8 @@ class VideoFrameFactoryImplTest : public testing::Test { auto mre_manager = std::make_unique<MockMaybeRenderEarlyManager>(); mre_manager_raw_ = mre_manager.get(); - auto ycbcr_helper = - base::SequenceBound<MockYCbCrHelper>(task_runner_, &ycbcr_helper_raw_); + auto ycbcr_helper = base::SequenceBound<MockFrameInfoHelper>( + task_runner_, &ycbcr_helper_raw_); base::RunLoop().RunUntilIdle(); // Init |ycbcr_helper_raw_|. ycbcr_destruction_observer_ = ycbcr_helper_raw_->CreateDestructionObserver(); @@ -128,7 +156,8 @@ class VideoFrameFactoryImplTest : public testing::Test { *flag = true; }, base::Unretained(release_cb_called_flag)); - auto codec_image = base::MakeRefCounted<MockCodecImage>(); + auto codec_image = + base::MakeRefCounted<MockCodecImage>(gfx::Size(100, 100)); record.codec_image_holder = base::MakeRefCounted<CodecImageHolder>(task_runner_, codec_image); return record; @@ -148,7 +177,7 @@ class VideoFrameFactoryImplTest : public testing::Test { // Sent to |impl_| by RequestVideoFrame.. base::MockCallback<VideoFrameFactory::OnceOutputCB> output_cb_; - MockYCbCrHelper* ycbcr_helper_raw_ = nullptr; + MockFrameInfoHelper* ycbcr_helper_raw_ = nullptr; std::unique_ptr<DestructionObserver> ycbcr_destruction_observer_; gpu::GpuPreferences gpu_preferences_; @@ -244,77 +273,72 @@ TEST_F(VideoFrameFactoryImplTest, base::RunLoop().RunUntilIdle(); } -TEST_F(VideoFrameFactoryImplTest, DoesNotCallYCbCrHelperIfNotVulkan) { - EXPECT_CALL(*ycbcr_helper_raw_, MockGetYCbCrInfo(_)).Times(0); - RequestVideoFrame(); - auto image_record = MakeImageRecord(); - image_record.is_vulkan = false; - image_provider_raw_->ProvideOneRequestedImage(&image_record); - base::RunLoop().RunUntilIdle(); -} +TEST_F(VideoFrameFactoryImplTest, DoesCallFrameInfoHelperIfVulkan) { + // We will be driving callback by ourselves in this test. + ycbcr_helper_raw_->set_run_callback_automatically(false); + // Expect call to get info for the first frame. + EXPECT_CALL(*ycbcr_helper_raw_, MockGetFrameInfo(_)).Times(1); -TEST_F(VideoFrameFactoryImplTest, DoesCallYCbCrHelperIfVulkan) { RequestVideoFrame(); - auto image_record = MakeImageRecord(); - base::OnceCallback<void(YCbCrHelper::OptionalInfo)> cb; - EXPECT_CALL(*ycbcr_helper_raw_, - MockGetYCbCrInfo(image_record.codec_image_holder)) - .Times(1); - image_record.is_vulkan = true; - image_provider_raw_->ProvideOneRequestedImage(&image_record); + + // Provide info. It should send image request. + ycbcr_helper_raw_->RunWithYcbCrInfo(true); base::RunLoop().RunUntilIdle(); - // Provide YCbCrInfo. It should provide the VideoFrame too. + testing::Mock::VerifyAndClearExpectations(ycbcr_helper_raw_); + + // Fulfilling image request should provide video frame. EXPECT_CALL(output_cb_, Run(_)).Times(1); - gpu::VulkanYCbCrInfo ycbcr; - std::move(ycbcr_helper_raw_->cb_).Run(ycbcr); + + auto image_record = MakeImageRecord(); + image_provider_raw_->ProvideOneRequestedImage(&image_record); base::RunLoop().RunUntilIdle(); - // It's okay if the ycbcr helper is destroyed. If not, then verify - // expectations explicitly now. - if (ycbcr_destruction_observer_->destructed()) - ycbcr_helper_raw_ = nullptr; - else - testing::Mock::VerifyAndClearExpectations(ycbcr_helper_raw_); // Verify that no more calls happen, since we don't want thread hops on every // frame. Note that multiple could be dispatched before now. It should still // send along a VideoFrame, though. + EXPECT_CALL(*ycbcr_helper_raw_, MockGetFrameInfo(_)).Times(0); + EXPECT_CALL(output_cb_, Run(_)).Times(1); + RequestVideoFrame(); auto other_image_record = MakeImageRecord(); // If the helper hasn't been destroyed, then we don't expect it to be called. - if (ycbcr_helper_raw_) - EXPECT_CALL(*ycbcr_helper_raw_, MockGetYCbCrInfo(_)).Times(0); - EXPECT_CALL(output_cb_, Run(_)).Times(1); image_provider_raw_->ProvideOneRequestedImage(&other_image_record); base::RunLoop().RunUntilIdle(); } TEST_F(VideoFrameFactoryImplTest, NullYCbCrInfoDoesntCrash) { - // Sending a null YCbCrInfo then requesting a frame shouldn't cause a crash. - // See https://crbug.com/1007196 . + // We will be driving callback by ourselves in this test. + ycbcr_helper_raw_->set_run_callback_automatically(false); + + // Expect call to get info for the first frame. + EXPECT_CALL(*ycbcr_helper_raw_, MockGetFrameInfo(_)).Times(1); + RequestVideoFrame(); + + // Provide info. It should send image request. + ycbcr_helper_raw_->RunWithYcbCrInfo(false); + base::RunLoop().RunUntilIdle(); + + testing::Mock::VerifyAndClearExpectations(ycbcr_helper_raw_); + + // Fulfilling image request should provide video frame. + EXPECT_CALL(output_cb_, Run(_)).Times(1); + auto image_record = MakeImageRecord(); - EXPECT_CALL(*ycbcr_helper_raw_, - MockGetYCbCrInfo(image_record.codec_image_holder)) - .Times(1); - image_record.is_vulkan = true; image_provider_raw_->ProvideOneRequestedImage(&image_record); base::RunLoop().RunUntilIdle(); - // Provide an empty YCbCrInfo. + // Verify that we will get call to GetFrameInfo as previous one failed. + EXPECT_CALL(*ycbcr_helper_raw_, MockGetFrameInfo(_)).Times(1); EXPECT_CALL(output_cb_, Run(_)).Times(1); - std::move(ycbcr_helper_raw_->cb_).Run(base::nullopt); - base::RunLoop().RunUntilIdle(); - // It shouldn't crash on the next frame. crbug.com/1007196 RequestVideoFrame(); + ycbcr_helper_raw_->RunWithYcbCrInfo(true); + base::RunLoop().RunUntilIdle(); + auto other_image_record = MakeImageRecord(); - other_image_record.is_vulkan = true; - // Should still call the helper, since it didn't get YCbCrInfo last time. - EXPECT_CALL(*ycbcr_helper_raw_, - MockGetYCbCrInfo(other_image_record.codec_image_holder)) - .Times(1); - // Since we aren't sending YCbCr info, it won't call us back with a frame. + // If the helper hasn't been destroyed, then we don't expect it to be called. image_provider_raw_->ProvideOneRequestedImage(&other_image_record); base::RunLoop().RunUntilIdle(); } diff --git a/chromium/media/gpu/android/ycbcr_helper.cc b/chromium/media/gpu/android/ycbcr_helper.cc deleted file mode 100644 index bee89a685a8..00000000000 --- a/chromium/media/gpu/android/ycbcr_helper.cc +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/gpu/android/ycbcr_helper.h" - -#include "gpu/command_buffer/service/shared_image_video.h" -#include "gpu/ipc/service/command_buffer_stub.h" -#include "gpu/ipc/service/gpu_channel.h" -#include "gpu/ipc/service/gpu_channel_manager.h" - -namespace media { - -// Concrete implementation of YCbCrHelper that renders output buffers and gets -// the YCbCrInfo they need. -class YCbCrHelperImpl : public YCbCrHelper, - public gpu::CommandBufferStub::DestructionObserver { - public: - YCbCrHelperImpl(SharedImageVideoProvider::GetStubCB get_stub_cb) { - stub_ = get_stub_cb.Run(); - if (stub_) - stub_->AddDestructionObserver(this); - } - - ~YCbCrHelperImpl() override { - if (stub_) - stub_->RemoveDestructionObserver(this); - } - - // YCbCrHelper - void GetYCbCrInfo( - scoped_refptr<CodecImageHolder> codec_image_holder, - base::OnceCallback<void(OptionalInfo ycbcr_info)> cb) override { - // If we don't have the info cached, then try to get it. If we have gotten - // it, then don't try again. Assume that our caller asked for it before it - // got the results back. We don't want to render more frames to the front - // buffer if we don't need to. - if (!ycbcr_info_) - ycbcr_info_ = RenderImageAndGetYCbCrInfo(std::move(codec_image_holder)); - - // Whether we got it or not, send it along. - std::move(cb).Run(ycbcr_info_); - } - - void OnWillDestroyStub(bool have_context) override { - DCHECK(stub_); - stub_ = nullptr; - } - - private: - // Render the codec output buffer, and use it to get the YCbCrInfo. - OptionalInfo RenderImageAndGetYCbCrInfo( - scoped_refptr<CodecImageHolder> codec_image_holder) { - gpu::ContextResult result; - if (!stub_) - return base::nullopt; - - auto shared_context = - stub_->channel()->gpu_channel_manager()->GetSharedContextState(&result); - auto context_provider = - (result == gpu::ContextResult::kSuccess) ? shared_context : nullptr; - if (!context_provider) - return base::nullopt; - - return gpu::SharedImageVideo::GetYcbcrInfo( - codec_image_holder->codec_image_raw(), context_provider); - } - - gpu::CommandBufferStub* stub_ = nullptr; - - OptionalInfo ycbcr_info_; -}; - -// static -base::SequenceBound<YCbCrHelper> YCbCrHelper::Create( - scoped_refptr<base::SequencedTaskRunner> gpu_task_runner, - SharedImageVideoProvider::GetStubCB get_stub_cb) { - return base::SequenceBound<YCbCrHelperImpl>(std::move(gpu_task_runner), - std::move(get_stub_cb)); -} - -} // namespace media diff --git a/chromium/media/gpu/android/ycbcr_helper.h b/chromium/media/gpu/android/ycbcr_helper.h deleted file mode 100644 index 4576ece0086..00000000000 --- a/chromium/media/gpu/android/ycbcr_helper.h +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MEDIA_GPU_ANDROID_YCBCR_HELPER_H_ -#define MEDIA_GPU_ANDROID_YCBCR_HELPER_H_ - -#include "base/optional.h" -#include "base/threading/sequence_bound.h" -#include "media/gpu/android/codec_image.h" -#include "media/gpu/android/shared_image_video_provider.h" -#include "media/gpu/media_gpu_export.h" - -namespace media { - -// Helper class to fetch YCbCrInfo for Vulkan from a CodecImage. -class MEDIA_GPU_EXPORT YCbCrHelper { - public: - using OptionalInfo = base::Optional<gpu::VulkanYCbCrInfo>; - - static base::SequenceBound<YCbCrHelper> Create( - scoped_refptr<base::SequencedTaskRunner> gpu_task_runner, - SharedImageVideoProvider::GetStubCB get_stub_cb); - - virtual ~YCbCrHelper() = default; - - // Call |cb| with the YCbCrInfo (or nullopt, if we can't get it). Will render - // |codec_image_holder| to the front buffer if it hasn't successfully gotten - // the YCbCrInfo on a previous call. Otherwise, will return the cached - // YCbCrInfo and leave |codec_image_holder| unmodified. Once we call |cb| - // with a non-nullopt YCbCrInfo, we will always return that same value; there - // is no need to call us afterwards. - // - // While this API might seem to be out of its Vulkan mind, it's this - // complicated to (a) prevent rendering frames out of order to the front - // buffer, and (b) make it easy to handle the fact that sometimes, we just - // can't get a YCbCrInfo from a CodecImage due to timeouts. - virtual void GetYCbCrInfo( - scoped_refptr<CodecImageHolder> codec_image_holder, - base::OnceCallback<void(OptionalInfo ycbcr_info)> cb) = 0; - - protected: - YCbCrHelper() = default; -}; - -} // namespace media - -#endif // MEDIA_GPU_ANDROID_YCBCR_HELPER_H_ diff --git a/chromium/media/gpu/chromeos/fourcc_unittests.cc b/chromium/media/gpu/chromeos/fourcc_unittests.cc index 9ad8c921bea..ade4a4b663c 100644 --- a/chromium/media/gpu/chromeos/fourcc_unittests.cc +++ b/chromium/media/gpu/chromeos/fourcc_unittests.cc @@ -5,7 +5,6 @@ #include "base/optional.h" #include "media/gpu/chromeos/fourcc.h" -#include "base/logging.h" #include "media/gpu/buildflags.h" #include "testing/gtest/include/gtest/gtest.h" diff --git a/chromium/media/gpu/chromeos/platform_video_frame_utils.cc b/chromium/media/gpu/chromeos/platform_video_frame_utils.cc index 0c555fd8f05..9e7994040b8 100644 --- a/chromium/media/gpu/chromeos/platform_video_frame_utils.cc +++ b/chromium/media/gpu/chromeos/platform_video_frame_utils.cc @@ -4,12 +4,15 @@ #include "media/gpu/chromeos/platform_video_frame_utils.h" -#include "base/atomic_sequence_num.h" +#include <limits> + #include "base/bind.h" #include "base/bind_helpers.h" #include "base/callback_helpers.h" #include "base/files/scoped_file.h" +#include "base/no_destructor.h" #include "base/posix/eintr_wrapper.h" +#include "base/synchronization/lock.h" #include "gpu/ipc/common/gpu_client_ids.h" #include "gpu/ipc/common/gpu_memory_buffer_support.h" #include "gpu/ipc/service/gpu_memory_buffer_factory.h" @@ -37,11 +40,19 @@ gfx::GpuMemoryBufferHandle AllocateGpuMemoryBufferHandle( if (!buffer_format) return gmb_handle; - static base::AtomicSequenceNumber buffer_id_generator; + int gpu_memory_buffer_id; + { + static base::NoDestructor<base::Lock> id_lock; + static int next_gpu_memory_buffer_id = 0; + base::AutoLock lock(*id_lock); + CHECK_LT(next_gpu_memory_buffer_id, std::numeric_limits<int>::max()); + gpu_memory_buffer_id = next_gpu_memory_buffer_id++; + } + // TODO(hiroh): Rename the client id to more generic one. gmb_handle = factory->CreateGpuMemoryBuffer( - gfx::GpuMemoryBufferId(buffer_id_generator.GetNext()), coded_size, - *buffer_format, buffer_usage, gpu::kPlatformVideoFramePoolClientId, + gfx::GpuMemoryBufferId(gpu_memory_buffer_id), coded_size, *buffer_format, + buffer_usage, gpu::kPlatformVideoFramePoolClientId, gfx::kNullAcceleratedWidget); DCHECK(gmb_handle.is_null() || gmb_handle.type != gfx::NATIVE_PIXMAP || VideoFrame::NumPlanes(pixel_format) == diff --git a/chromium/media/gpu/chromeos/platform_video_frame_utils_unittest.cc b/chromium/media/gpu/chromeos/platform_video_frame_utils_unittest.cc index 005b9dd92fc..abcc36328be 100644 --- a/chromium/media/gpu/chromeos/platform_video_frame_utils_unittest.cc +++ b/chromium/media/gpu/chromeos/platform_video_frame_utils_unittest.cc @@ -70,7 +70,7 @@ scoped_refptr<VideoFrame> CreateMockDmaBufVideoFrame( class FakeGpuMemoryBufferFactory : public gpu::GpuMemoryBufferFactory { public: FakeGpuMemoryBufferFactory() = default; - ~FakeGpuMemoryBufferFactory() { + ~FakeGpuMemoryBufferFactory() override { for (const auto& buffers : gpu_memory_buffers_) { if (!buffers.second.empty()) { LOG(ERROR) << "client_id=" << buffers.first diff --git a/chromium/media/gpu/chromeos/video_decoder_pipeline.cc b/chromium/media/gpu/chromeos/video_decoder_pipeline.cc index 5ebd3caedbb..906861ba788 100644 --- a/chromium/media/gpu/chromeos/video_decoder_pipeline.cc +++ b/chromium/media/gpu/chromeos/video_decoder_pipeline.cc @@ -319,7 +319,7 @@ void VideoDecoderPipeline::ResetTask(base::OnceClosure closure) { DCHECK(!client_reset_cb_); DVLOGF(3); - need_notify_decoder_flushed_ = false; + need_apply_new_resolution = false; client_reset_cb_ = std::move(closure); decoder_->Reset( base::BindOnce(&VideoDecoderPipeline::OnResetDone, decoder_weak_this_)); @@ -431,7 +431,7 @@ void VideoDecoderPipeline::OnFrameConverted(scoped_refptr<VideoFrame> frame) { // After outputting a frame, flush might be completed. CallFlushCbIfNeeded(DecodeStatus::OK); - CallOnPipelineFlushedIfNeeded(); + CallApplyResolutionChangeIfNeeded(); } bool VideoDecoderPipeline::HasPendingFrames() const { @@ -468,19 +468,19 @@ void VideoDecoderPipeline::CallFlushCbIfNeeded(DecodeStatus status) { void VideoDecoderPipeline::PrepareChangeResolution() { DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_sequence_checker_); DVLOGF(3); - DCHECK(!need_notify_decoder_flushed_); + DCHECK(!need_apply_new_resolution); - need_notify_decoder_flushed_ = true; - CallOnPipelineFlushedIfNeeded(); + need_apply_new_resolution = true; + CallApplyResolutionChangeIfNeeded(); } -void VideoDecoderPipeline::CallOnPipelineFlushedIfNeeded() { +void VideoDecoderPipeline::CallApplyResolutionChangeIfNeeded() { DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_sequence_checker_); DVLOGF(3); - if (need_notify_decoder_flushed_ && !HasPendingFrames()) { - need_notify_decoder_flushed_ = false; - decoder_->OnPipelineFlushed(); + if (need_apply_new_resolution && !HasPendingFrames()) { + need_apply_new_resolution = false; + decoder_->ApplyResolutionChange(); } } diff --git a/chromium/media/gpu/chromeos/video_decoder_pipeline.h b/chromium/media/gpu/chromeos/video_decoder_pipeline.h index 396055e7227..030ed9058e1 100644 --- a/chromium/media/gpu/chromeos/video_decoder_pipeline.h +++ b/chromium/media/gpu/chromeos/video_decoder_pipeline.h @@ -59,7 +59,7 @@ class MEDIA_GPU_EXPORT DecoderInterface { virtual DmabufVideoFramePool* GetVideoFramePool() const = 0; // After this method is called from |decoder_|, the client needs to call - // DecoderInterface::OnPipelineFlushed() when all pending frames are + // DecoderInterface::ApplyResolutionChange() when all pending frames are // flushed. virtual void PrepareChangeResolution() = 0; @@ -114,7 +114,7 @@ class MEDIA_GPU_EXPORT DecoderInterface { // After DecoderInterface calls |prepare_change_resolution_cb| passed // from the constructor, this method is called when the pipeline flushes // pending frames. - virtual void OnPipelineFlushed() = 0; + virtual void ApplyResolutionChange() = 0; protected: // Decoder task runner. All public methods of @@ -217,8 +217,8 @@ class MEDIA_GPU_EXPORT VideoDecoderPipeline : public VideoDecoder, // i.e. |image_processor_| or |frame_converter_| has pending frames. bool HasPendingFrames() const; - // Call DecoderInterface::OnPipelineFlushed() when we need to. - void CallOnPipelineFlushedIfNeeded(); + // Call DecoderInterface::ApplyResolutionChange() when we need to. + void CallApplyResolutionChangeIfNeeded(); // Call |client_flush_cb_| with |status|. void CallFlushCbIfNeeded(DecodeStatus status); @@ -271,8 +271,8 @@ class MEDIA_GPU_EXPORT VideoDecoderPipeline : public VideoDecoder, base::OnceClosure client_reset_cb_; // True if we need to notify |decoder_| that the pipeline is flushed via - // DecoderInterface::OnPipelineFlushed(). - bool need_notify_decoder_flushed_ = false; + // DecoderInterface::ApplyResolutionChange(). + bool need_apply_new_resolution = false; // True if the decoder needs bitstream conversion before decoding. bool needs_bitstream_conversion_ = false; diff --git a/chromium/media/gpu/gles2_decoder_helper.cc b/chromium/media/gpu/gles2_decoder_helper.cc index 4d32c2ac2c6..bebb6a66629 100644 --- a/chromium/media/gpu/gles2_decoder_helper.cc +++ b/chromium/media/gpu/gles2_decoder_helper.cc @@ -6,7 +6,7 @@ #include <memory> -#include "base/logging.h" +#include "base/check_op.h" #include "base/macros.h" #include "base/threading/thread_checker.h" #include "gpu/command_buffer/common/gles2_cmd_utils.h" diff --git a/chromium/media/gpu/gpu_video_encode_accelerator_factory.cc b/chromium/media/gpu/gpu_video_encode_accelerator_factory.cc index cf43708610e..21ecf1b6ed8 100644 --- a/chromium/media/gpu/gpu_video_encode_accelerator_factory.cc +++ b/chromium/media/gpu/gpu_video_encode_accelerator_factory.cc @@ -22,6 +22,7 @@ #endif #if defined(OS_WIN) #include "base/feature_list.h" +#include "gpu/config/gpu_driver_bug_workarounds.h" #include "media/base/media_switches.h" #include "media/gpu/windows/media_foundation_video_encode_accelerator_win.h" #endif @@ -67,9 +68,11 @@ std::unique_ptr<VideoEncodeAccelerator> CreateVTVEA() { // Creates a MediaFoundationVEA for Win 7 or later. If |compatible_with_win7| is // true, VEA is limited to a subset of features that is compatible with Win 7. std::unique_ptr<VideoEncodeAccelerator> CreateMediaFoundationVEA( - bool compatible_with_win7) { + bool compatible_with_win7, + bool enable_async_mft) { return base::WrapUnique<VideoEncodeAccelerator>( - new MediaFoundationVideoEncodeAccelerator(compatible_with_win7)); + new MediaFoundationVideoEncodeAccelerator(compatible_with_win7, + enable_async_mft)); } #endif @@ -100,11 +103,12 @@ std::vector<VEAFactoryFunction> GetVEAFactoryFunctions( vea_factory_functions.push_back(base::BindRepeating(&CreateVTVEA)); #endif #if defined(OS_WIN) - if (base::FeatureList::IsEnabled(kMediaFoundationH264Encoding)) { - vea_factory_functions.push_back(base::BindRepeating( - &CreateMediaFoundationVEA, - gpu_preferences.enable_media_foundation_vea_on_windows7)); - } + vea_factory_functions.push_back(base::BindRepeating( + &CreateMediaFoundationVEA, + gpu_preferences.enable_media_foundation_vea_on_windows7, + base::FeatureList::IsEnabled(kMediaFoundationAsyncH264Encoding) && + !gpu::GpuDriverBugWorkarounds() + .disable_mediafoundation_async_h264_encoding)); #endif return vea_factory_functions; } diff --git a/chromium/media/gpu/gpu_video_encode_accelerator_helpers.cc b/chromium/media/gpu/gpu_video_encode_accelerator_helpers.cc index 2979dc754f7..151101e42aa 100644 --- a/chromium/media/gpu/gpu_video_encode_accelerator_helpers.cc +++ b/chromium/media/gpu/gpu_video_encode_accelerator_helpers.cc @@ -6,7 +6,7 @@ #include <algorithm> -#include "base/logging.h" +#include "base/check_op.h" namespace media { namespace { diff --git a/chromium/media/gpu/h264_decoder.cc b/chromium/media/gpu/h264_decoder.cc index 62c42d9393f..59ab81d16ba 100644 --- a/chromium/media/gpu/h264_decoder.cc +++ b/chromium/media/gpu/h264_decoder.cc @@ -437,7 +437,7 @@ void H264Decoder::ConstructReferencePicListsB( std::sort(ref_pic_list_b1_.begin(), iter, POCAscCompare()); // Now add [3] and sort by ascending long_term_pic_num - dpb_.GetShortTermRefPicsAppending(&ref_pic_list_b1_); + dpb_.GetLongTermRefPicsAppending(&ref_pic_list_b1_); std::sort(ref_pic_list_b1_.begin() + num_short_refs, ref_pic_list_b1_.end(), LongTermPicNumAscCompare()); diff --git a/chromium/media/gpu/h264_decoder_unittest.cc b/chromium/media/gpu/h264_decoder_unittest.cc index b12d769a633..96703288876 100644 --- a/chromium/media/gpu/h264_decoder_unittest.cc +++ b/chromium/media/gpu/h264_decoder_unittest.cc @@ -9,11 +9,11 @@ #include <memory> #include <string> +#include "base/check.h" #include "base/command_line.h" #include "base/containers/queue.h" #include "base/containers/span.h" #include "base/files/file_util.h" -#include "base/logging.h" #include "media/base/test_data_util.h" #include "media/gpu/h264_decoder.h" #include "testing/gmock/include/gmock/gmock.h" diff --git a/chromium/media/gpu/v4l2/BUILD.gn b/chromium/media/gpu/v4l2/BUILD.gn index 51814e19868..88b72e36308 100644 --- a/chromium/media/gpu/v4l2/BUILD.gn +++ b/chromium/media/gpu/v4l2/BUILD.gn @@ -95,6 +95,7 @@ source_set("v4l2") { "//media/gpu:buildflags", "//media/gpu:common", "//media/gpu/chromeos:common", + "//media/parsers", "//third_party/libyuv", "//ui/gfx/geometry", "//ui/ozone", @@ -116,7 +117,6 @@ source_set("v4l2") { "//components/chromeos_camera:jpeg_encode_accelerator", "//components/chromeos_camera:mjpeg_decode_accelerator", "//media/gpu:video_frame_mapper_common", - "//media/parsers", ] } } diff --git a/chromium/media/gpu/v4l2/v4l2_device.cc b/chromium/media/gpu/v4l2/v4l2_device.cc index ee5c4430e70..9b81f8046f2 100644 --- a/chromium/media/gpu/v4l2/v4l2_device.cc +++ b/chromium/media/gpu/v4l2/v4l2_device.cc @@ -47,6 +47,10 @@ namespace { // Maximum number of requests that can be created. constexpr size_t kMaxNumRequests = 32; +gfx::Rect V4L2RectToGfxRect(const v4l2_rect& rect) { + return gfx::Rect(rect.left, rect.top, rect.width, rect.height); +} + } // namespace V4L2ExtCtrl::V4L2ExtCtrl(uint32_t id) { @@ -236,9 +240,9 @@ scoped_refptr<VideoFrame> V4L2Buffer::GetVideoFrame() { // We can create the VideoFrame only when using MMAP buffers. if (v4l2_buffer_.memory != V4L2_MEMORY_MMAP) { VLOGF(1) << "Cannot create video frame from non-MMAP buffer"; - // video_frame_ should be null since that's its default value. - DCHECK_EQ(video_frame_, nullptr); - return video_frame_; + // Allow NOTREACHED() on invalid argument because this is an internal + // method. + NOTREACHED(); } // Create the video frame instance if requiring it for the first time. @@ -881,6 +885,58 @@ base::Optional<struct v4l2_format> V4L2Queue::SetFormat(uint32_t fourcc, return current_format_; } +std::pair<base::Optional<struct v4l2_format>, int> V4L2Queue::GetFormat() { + struct v4l2_format format; + memset(&format, 0, sizeof(format)); + format.type = type_; + if (device_->Ioctl(VIDIOC_G_FMT, &format) != 0) { + VPQLOGF(2) << "Failed to get format"; + return std::make_pair(base::nullopt, errno); + } + + return std::make_pair(format, 0); +} + +base::Optional<gfx::Rect> V4L2Queue::GetVisibleRect() { + // Some drivers prior to 4.13 only accept the non-MPLANE variant when using + // VIDIOC_G_SELECTION. This block can be removed once we stop supporting + // kernels < 4.13. + // For details, see the note at + // https://www.kernel.org/doc/html/latest/media/uapi/v4l/vidioc-g-selection.html + enum v4l2_buf_type compose_type; + switch (type_) { + case V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE: + compose_type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + break; + case V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE: + compose_type = V4L2_BUF_TYPE_VIDEO_OUTPUT; + break; + default: + compose_type = type_; + break; + } + + struct v4l2_selection selection = {}; + selection.type = compose_type; + selection.target = V4L2_SEL_TGT_COMPOSE; + if (device_->Ioctl(VIDIOC_G_SELECTION, &selection) == 0) { + DVQLOGF(3) << "VIDIOC_G_SELECTION is supported"; + return V4L2RectToGfxRect(selection.r); + } + + // TODO(acourbot) using VIDIOC_G_CROP is considered legacy and can be + // removed once no active devices use it anymore. + DVQLOGF(3) << "Fallback to VIDIOC_G_CROP"; + struct v4l2_crop crop = {}; + crop.type = type_; + if (device_->Ioctl(VIDIOC_G_CROP, &crop) == 0) { + return V4L2RectToGfxRect(crop.c); + } + + VQLOGF(1) << "Failed to get visible rect"; + return base::nullopt; +} + size_t V4L2Queue::AllocateBuffers(size_t count, enum v4l2_memory memory) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DCHECK(!free_buffers_); @@ -906,13 +962,12 @@ size_t V4L2Queue::AllocateBuffers(size_t count, enum v4l2_memory memory) { // This should not be required, but Tegra's VIDIOC_QUERYBUF will fail on // output buffers if the number of specified planes does not exactly match the // format. - struct v4l2_format format = {.type = type_}; - int ret = device_->Ioctl(VIDIOC_G_FMT, &format); - if (ret) { - VPQLOGF(1) << "VIDIOC_G_FMT failed"; + base::Optional<v4l2_format> format = GetFormat().first; + if (!format) { + VQLOGF(1) << "Cannot get format."; return 0; } - planes_count_ = format.fmt.pix_mp.num_planes; + planes_count_ = format->fmt.pix_mp.num_planes; DCHECK_LE(planes_count_, static_cast<size_t>(VIDEO_MAX_PLANES)); struct v4l2_requestbuffers reqbufs = {}; @@ -921,7 +976,7 @@ size_t V4L2Queue::AllocateBuffers(size_t count, enum v4l2_memory memory) { reqbufs.memory = memory; DVQLOGF(3) << "Requesting " << count << " buffers."; - ret = device_->Ioctl(VIDIOC_REQBUFS, &reqbufs); + int ret = device_->Ioctl(VIDIOC_REQBUFS, &reqbufs); if (ret) { VPQLOGF(1) << "VIDIOC_REQBUFS failed"; return 0; @@ -934,7 +989,7 @@ size_t V4L2Queue::AllocateBuffers(size_t count, enum v4l2_memory memory) { // Now query all buffer information. for (size_t i = 0; i < reqbufs.count; i++) { - auto buffer = V4L2Buffer::Create(device_, type_, memory_, format, i); + auto buffer = V4L2Buffer::Create(device_, type_, memory_, *format, i); if (!buffer) { DeallocateBuffers(); @@ -1936,6 +1991,18 @@ void V4L2Device::SchedulePoll() { device_poller_->SchedulePoll(); } +base::Optional<struct v4l2_event> V4L2Device::DequeueEvent() { + DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); + struct v4l2_event event = {}; + + if (Ioctl(VIDIOC_DQEVENT, &event) != 0) { + VPLOGF(3) << "Failed to dequeue event"; + return base::nullopt; + } + + return event; +} + V4L2RequestsQueue* V4L2Device::GetRequestsQueue() { DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); @@ -1980,6 +2047,23 @@ bool V4L2Device::SetExtCtrls(uint32_t ctrl_class, return Ioctl(VIDIOC_S_EXT_CTRLS, &ext_ctrls) == 0; } +base::Optional<struct v4l2_ext_control> V4L2Device::GetCtrl(uint32_t ctrl_id) { + DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); + struct v4l2_ext_control ctrl = {}; + struct v4l2_ext_controls ext_ctrls = {}; + + ctrl.id = ctrl_id; + ext_ctrls.controls = &ctrl; + ext_ctrls.count = 1; + + if (Ioctl(VIDIOC_G_EXT_CTRLS, &ext_ctrls) != 0) { + VPLOGF(3) << "Failed to get control"; + return base::nullopt; + } + + return ctrl; +} + class V4L2Request { public: // Apply the passed controls to the request. diff --git a/chromium/media/gpu/v4l2/v4l2_device.h b/chromium/media/gpu/v4l2/v4l2_device.h index 346ec0c9495..310d4a4a1a5 100644 --- a/chromium/media/gpu/v4l2/v4l2_device.h +++ b/chromium/media/gpu/v4l2/v4l2_device.h @@ -293,6 +293,26 @@ class MEDIA_GPU_EXPORT V4L2Queue size_t buffer_size) WARN_UNUSED_RESULT; + // Returns the currently set format on the queue. The result is returned as + // a std::pair where the first member is the format, or base::nullopt if the + // format could not be obtained due to an ioctl error. The second member is + // only used in case of an error and contains the |errno| set by the failing + // ioctl. If the first member is not base::nullopt, the second member will + // always be zero. + // + // If the second member is 0, then the first member is guaranteed to have + // a valid value. So clients that are not interested in the precise error + // message can just check that the first member is valid and go on. + // + // This pair is used because not all failures to get the format are + // necessarily errors, so we need to way to let the use decide whether it + // is one or not. + std::pair<base::Optional<struct v4l2_format>, int> GetFormat(); + + // Codec-specific method to get the visible rectangle of the queue, using the + // VIDIOC_G_SELECTION ioctl if available, or VIDIOC_G_CROP as a fallback. + base::Optional<gfx::Rect> GetVisibleRect(); + // Allocate |count| buffers for the current format of this queue, with a // specific |memory| allocation, and returns the number of buffers allocated // or zero if an error occurred, or if references to any previously allocated @@ -693,6 +713,9 @@ class MEDIA_GPU_EXPORT V4L2Device // to be called from V4L2Queue, clients should not need to call it directly. void SchedulePoll(); + // Attempt to dequeue a V4L2 event and return it. + base::Optional<struct v4l2_event> DequeueEvent(); + // Returns requests queue to get free requests. A null pointer is returned if // the queue creation failed or if requests are not supported. V4L2RequestsQueue* GetRequestsQueue(); @@ -703,6 +726,10 @@ class MEDIA_GPU_EXPORT V4L2Device // whether the operation succeeded. bool SetExtCtrls(uint32_t ctrl_class, std::vector<V4L2ExtCtrl> ctrls); + // Get the value of a single control, or base::nullopt of the control is not + // exposed by the device. + base::Optional<struct v4l2_ext_control> GetCtrl(uint32_t ctrl_id); + protected: friend class base::RefCountedThreadSafe<V4L2Device>; V4L2Device(); diff --git a/chromium/media/gpu/v4l2/v4l2_image_processor_backend.cc b/chromium/media/gpu/v4l2/v4l2_image_processor_backend.cc index 12eeef482fb..6498537e426 100644 --- a/chromium/media/gpu/v4l2/v4l2_image_processor_backend.cc +++ b/chromium/media/gpu/v4l2/v4l2_image_processor_backend.cc @@ -574,6 +574,33 @@ void V4L2ImageProcessorBackend::ProcessJobsTask() { DCHECK_CALLED_ON_VALID_SEQUENCE(backend_sequence_checker_); while (!input_job_queue_.empty()) { + if (!input_queue_->IsStreaming()) { + const VideoFrame& input_frame = + *(input_job_queue_.front()->input_frame.get()); + const gfx::Size input_buffer_size(input_frame.stride(0), + input_frame.coded_size().height()); + if (!ReconfigureV4L2Format(input_buffer_size, input_frame.visible_rect(), + V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE)) { + NotifyError(); + return; + } + } + + if (input_job_queue_.front() + ->output_frame && // output_frame is nullptr in ALLOCATE mode. + !output_queue_->IsStreaming()) { + const VideoFrame& output_frame = + *(input_job_queue_.front()->output_frame.get()); + const gfx::Size output_buffer_size(output_frame.stride(0), + output_frame.coded_size().height()); + if (!ReconfigureV4L2Format(output_buffer_size, + output_frame.visible_rect(), + V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE)) { + NotifyError(); + return; + } + } + // We need one input and one output buffer to schedule the job auto input_buffer = input_queue_->GetFreeBuffer(); auto output_buffer = output_queue_->GetFreeBuffer(); @@ -641,6 +668,39 @@ bool V4L2ImageProcessorBackend::ApplyCrop(const gfx::Rect& visible_rect, return true; } +bool V4L2ImageProcessorBackend::ReconfigureV4L2Format( + const gfx::Size& size, + const gfx::Rect& visible_rect, + enum v4l2_buf_type type) { + v4l2_format format{}; + format.type = type; + if (device_->Ioctl(VIDIOC_G_FMT, &format) != 0) { + VPLOGF(1) << "ioctl() failed: VIDIOC_G_FMT"; + return false; + } + + if (static_cast<int>(format.fmt.pix_mp.width) == size.width() && + static_cast<int>(format.fmt.pix_mp.height) == size.height()) { + return true; + } + format.fmt.pix_mp.width = size.width(); + format.fmt.pix_mp.height = size.height(); + if (device_->Ioctl(VIDIOC_S_FMT, &format) != 0) { + VPLOGF(1) << "ioctl() failed: VIDIOC_S_FMT"; + return false; + } + if (!ApplyCrop(visible_rect, type)) { + return false; + } + + auto queue = device_->GetQueue(type); + const size_t num_buffers = queue->AllocatedBuffersCount(); + const v4l2_memory memory_type = queue->GetMemoryType(); + DCHECK_GT(num_buffers, 0u); + return queue->DeallocateBuffers() && + AllocateV4L2Buffers(queue.get(), num_buffers, memory_type); +} + bool V4L2ImageProcessorBackend::CreateInputBuffers() { VLOGF(2); DCHECK_CALLED_ON_VALID_SEQUENCE(backend_sequence_checker_); diff --git a/chromium/media/gpu/v4l2/v4l2_image_processor_backend.h b/chromium/media/gpu/v4l2/v4l2_image_processor_backend.h index 5214eeb6414..bd1c78ac4e9 100644 --- a/chromium/media/gpu/v4l2/v4l2_image_processor_backend.h +++ b/chromium/media/gpu/v4l2/v4l2_image_processor_backend.h @@ -135,6 +135,10 @@ class MEDIA_GPU_EXPORT V4L2ImageProcessorBackend bool CreateOutputBuffers(); // Specify |visible_rect| to v4l2 |type| queue. bool ApplyCrop(const gfx::Rect& visible_rect, enum v4l2_buf_type type); + // Reconfigure |size| and |visible_rect| to v4l2 |type| queue. + bool ReconfigureV4L2Format(const gfx::Size& size, + const gfx::Rect& visible_rect, + enum v4l2_buf_type type); // Callback of VideoFrame destruction. Since VideoFrame destruction // callback might be executed on any sequence, we use a thunk to post the diff --git a/chromium/media/gpu/v4l2/v4l2_mjpeg_decode_accelerator.cc b/chromium/media/gpu/v4l2/v4l2_mjpeg_decode_accelerator.cc index ff21cd97f99..9be3d4857bf 100644 --- a/chromium/media/gpu/v4l2/v4l2_mjpeg_decode_accelerator.cc +++ b/chromium/media/gpu/v4l2/v4l2_mjpeg_decode_accelerator.cc @@ -756,18 +756,16 @@ void V4L2MjpegDecodeAccelerator::DevicePollTask() { bool V4L2MjpegDecodeAccelerator::DequeueSourceChangeEvent() { DCHECK(decoder_task_runner_->BelongsToCurrentThread()); - struct v4l2_event ev; - memset(&ev, 0, sizeof(ev)); - - if (device_->Ioctl(VIDIOC_DQEVENT, &ev) == 0) { - if (ev.type == V4L2_EVENT_SOURCE_CHANGE) { - VLOGF(2) << ": got source change event: " << ev.u.src_change.changes; - if (ev.u.src_change.changes & V4L2_EVENT_SRC_CH_RESOLUTION) { + if (base::Optional<struct v4l2_event> event = device_->DequeueEvent()) { + if (event->type == V4L2_EVENT_SOURCE_CHANGE) { + VLOGF(2) << ": got source change event: " << event->u.src_change.changes; + if (event->u.src_change.changes & V4L2_EVENT_SRC_CH_RESOLUTION) { return true; } VLOGF(1) << "unexpected source change event."; } else { - VLOGF(1) << "got an event (" << ev.type << ") we haven't subscribed to."; + VLOGF(1) << "got an event (" << event->type + << ") we haven't subscribed to."; } } else { VLOGF(1) << "dequeue event failed."; diff --git a/chromium/media/gpu/v4l2/v4l2_slice_video_decode_accelerator.cc b/chromium/media/gpu/v4l2/v4l2_slice_video_decode_accelerator.cc index 4f9ff1d5a67..dd2c2e853eb 100644 --- a/chromium/media/gpu/v4l2/v4l2_slice_video_decode_accelerator.cc +++ b/chromium/media/gpu/v4l2/v4l2_slice_video_decode_accelerator.cc @@ -649,15 +649,12 @@ bool V4L2SliceVideoDecodeAccelerator::CreateOutputBuffers() { // Since VdaVideoDecoder doesn't allocate PictureBuffer with size adjusted by // itself, we have to adjust here. - struct v4l2_format format; - memset(&format, 0, sizeof(format)); - format.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; - - if (device_->Ioctl(VIDIOC_G_FMT, &format) != 0) { - PLOG(ERROR) << "Failed getting OUTPUT format"; + auto ret = input_queue_->GetFormat().first; + if (!ret) { NOTIFY_ERROR(PLATFORM_FAILURE); return false; } + struct v4l2_format format = std::move(*ret); format.fmt.pix_mp.width = pic_size.width(); format.fmt.pix_mp.height = pic_size.height(); @@ -669,13 +666,12 @@ bool V4L2SliceVideoDecodeAccelerator::CreateOutputBuffers() { } // Get the coded size from the CAPTURE queue - memset(&format, 0, sizeof(format)); - format.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - if (device_->Ioctl(VIDIOC_G_FMT, &format) != 0) { - PLOG(ERROR) << "Failed getting CAPTURE format"; + ret = output_queue_->GetFormat().first; + if (!ret) { NOTIFY_ERROR(PLATFORM_FAILURE); return false; } + format = std::move(*ret); coded_size_.SetSize(base::checked_cast<int>(format.fmt.pix_mp.width), base::checked_cast<int>(format.fmt.pix_mp.height)); diff --git a/chromium/media/gpu/v4l2/v4l2_slice_video_decoder.cc b/chromium/media/gpu/v4l2/v4l2_slice_video_decoder.cc index aa5526ef711..28e1b3b7e4a 100644 --- a/chromium/media/gpu/v4l2/v4l2_slice_video_decoder.cc +++ b/chromium/media/gpu/v4l2/v4l2_slice_video_decoder.cc @@ -425,13 +425,12 @@ void V4L2SliceVideoDecoder::ChangeResolution(gfx::Size pic_size, client_->PrepareChangeResolution(); } -void V4L2SliceVideoDecoder::OnPipelineFlushed() { +void V4L2SliceVideoDecoder::ApplyResolutionChange() { DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_sequence_checker_); DVLOGF(3); DCHECK(continue_change_resolution_cb_); - decoder_task_runner_->PostTask(FROM_HERE, - std::move(continue_change_resolution_cb_)); + std::move(continue_change_resolution_cb_).Run(); } void V4L2SliceVideoDecoder::ContinueChangeResolution( diff --git a/chromium/media/gpu/v4l2/v4l2_slice_video_decoder.h b/chromium/media/gpu/v4l2/v4l2_slice_video_decoder.h index 801ededec56..d5b82bbf824 100644 --- a/chromium/media/gpu/v4l2/v4l2_slice_video_decoder.h +++ b/chromium/media/gpu/v4l2/v4l2_slice_video_decoder.h @@ -55,7 +55,7 @@ class MEDIA_GPU_EXPORT V4L2SliceVideoDecoder const OutputCB& output_cb) override; void Reset(base::OnceClosure closure) override; void Decode(scoped_refptr<DecoderBuffer> buffer, DecodeCB decode_cb) override; - void OnPipelineFlushed() override; + void ApplyResolutionChange() override; // V4L2VideoDecoderBackend::Client implementation void OnBackendError() override; diff --git a/chromium/media/gpu/v4l2/v4l2_vda_helpers.cc b/chromium/media/gpu/v4l2/v4l2_vda_helpers.cc index 86d2ccd8234..f520d93be0f 100644 --- a/chromium/media/gpu/v4l2/v4l2_vda_helpers.cc +++ b/chromium/media/gpu/v4l2/v4l2_vda_helpers.cc @@ -184,6 +184,7 @@ bool H264InputBufferFragmentSplitter::AdvanceFrameFragment(const uint8_t* data, h264_parser_->SetStream(data, size); H264NALU nalu; H264Parser::Result result; + bool has_frame_data = false; *endpos = 0; // Keep on peeking the next NALs while they don't indicate a frame @@ -197,7 +198,8 @@ bool H264InputBufferFragmentSplitter::AdvanceFrameFragment(const uint8_t* data, } if (result == H264Parser::kEOStream) { // We've reached the end of the buffer before finding a frame boundary. - partial_frame_pending_ = true; + if (has_frame_data) + partial_frame_pending_ = true; *endpos = size; return true; } @@ -207,6 +209,8 @@ bool H264InputBufferFragmentSplitter::AdvanceFrameFragment(const uint8_t* data, if (nalu.size < 1) return false; + has_frame_data = true; + // For these two, if the "first_mb_in_slice" field is zero, start a // new frame and return. This field is Exp-Golomb coded starting on // the eighth data bit of the NAL; a zero value is encoded with a diff --git a/chromium/media/gpu/v4l2/v4l2_video_decode_accelerator.cc b/chromium/media/gpu/v4l2/v4l2_video_decode_accelerator.cc index 7147d9f81f7..4a581cab841 100644 --- a/chromium/media/gpu/v4l2/v4l2_video_decode_accelerator.cc +++ b/chromium/media/gpu/v4l2/v4l2_video_decode_accelerator.cc @@ -419,8 +419,6 @@ void V4L2VideoDecodeAccelerator::AssignPictureBuffersTask( for (auto&& buffer : v4l2_buffers) { const int i = buffer.BufferId(); - DCHECK(buffers[i].size() == egl_image_size_); - OutputRecord& output_record = output_buffer_map_[i]; DCHECK_EQ(output_record.egl_image, EGL_NO_IMAGE_KHR); DCHECK_EQ(output_record.picture_id, -1); @@ -1391,17 +1389,15 @@ bool V4L2VideoDecodeAccelerator::DequeueResolutionChangeEvent() { DCHECK_NE(decoder_state_, kUninitialized); DVLOGF(3); - struct v4l2_event ev; - memset(&ev, 0, sizeof(ev)); - - while (device_->Ioctl(VIDIOC_DQEVENT, &ev) == 0) { - if (ev.type == V4L2_EVENT_SOURCE_CHANGE) { - if (ev.u.src_change.changes & V4L2_EVENT_SRC_CH_RESOLUTION) { + while (base::Optional<struct v4l2_event> event = device_->DequeueEvent()) { + if (event->type == V4L2_EVENT_SOURCE_CHANGE) { + if (event->u.src_change.changes & V4L2_EVENT_SRC_CH_RESOLUTION) { VLOGF(2) << "got resolution change event."; return true; } } else { - VLOGF(1) << "got an event (" << ev.type << ") we haven't subscribed to."; + VLOGF(1) << "got an event (" << event->type + << ") we haven't subscribed to."; } } return false; @@ -2084,18 +2080,19 @@ bool V4L2VideoDecodeAccelerator::GetFormatInfo(struct v4l2_format* format, DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); *again = false; - memset(format, 0, sizeof(*format)); - format->type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - if (device_->Ioctl(VIDIOC_G_FMT, format) != 0) { - if (errno == EINVAL) { + + auto ret = output_queue_->GetFormat(); + switch (ret.second) { + case 0: + *format = *ret.first; + break; + case EINVAL: // EINVAL means we haven't seen sufficient stream to decode the format. *again = true; return true; - } else { - PLOG(ERROR) << "ioctl() failed: VIDIOC_G_FMT"; + default: NOTIFY_ERROR(PLATFORM_FAILURE); return false; - } } // Make sure we are still getting the format we set on initialization. @@ -2146,30 +2143,11 @@ gfx::Size V4L2VideoDecodeAccelerator::GetVisibleSize( const gfx::Size& coded_size) { DCHECK(decoder_thread_.task_runner()->BelongsToCurrentThread()); - struct v4l2_rect* visible_rect; - struct v4l2_selection selection_arg; - memset(&selection_arg, 0, sizeof(selection_arg)); - selection_arg.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - selection_arg.target = V4L2_SEL_TGT_COMPOSE; - - if (device_->Ioctl(VIDIOC_G_SELECTION, &selection_arg) == 0) { - DVLOGF(3) << "VIDIOC_G_SELECTION is supported"; - visible_rect = &selection_arg.r; - } else { - DVLOGF(3) << "Fallback to VIDIOC_G_CROP"; - struct v4l2_crop crop_arg; - memset(&crop_arg, 0, sizeof(crop_arg)); - crop_arg.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; - - if (device_->Ioctl(VIDIOC_G_CROP, &crop_arg) != 0) { - VPLOGF(1) << "ioctl() VIDIOC_G_CROP failed"; - return coded_size; - } - visible_rect = &crop_arg.c; + auto ret = output_queue_->GetVisibleRect(); + if (!ret) { + return coded_size; } - - gfx::Rect rect(visible_rect->left, visible_rect->top, visible_rect->width, - visible_rect->height); + gfx::Rect rect = std::move(*ret); DVLOGF(3) << "visible rectangle is " << rect.ToString(); if (!gfx::Rect(coded_size).Contains(rect)) { DVLOGF(3) << "visible rectangle " << rect.ToString() @@ -2398,11 +2376,10 @@ bool V4L2VideoDecodeAccelerator::CreateOutputBuffers() { DCHECK(output_buffer_map_.empty()); // Number of output buffers we need. - struct v4l2_control ctrl; - memset(&ctrl, 0, sizeof(ctrl)); - ctrl.id = V4L2_CID_MIN_BUFFERS_FOR_CAPTURE; - IOCTL_OR_ERROR_RETURN_FALSE(VIDIOC_G_CTRL, &ctrl); - output_dpb_size_ = ctrl.value; + auto ctrl = device_->GetCtrl(V4L2_CID_MIN_BUFFERS_FOR_CAPTURE); + if (!ctrl) + return false; + output_dpb_size_ = ctrl->value; // Output format setup in Initialize(). diff --git a/chromium/media/gpu/v4l2/v4l2_video_decoder_backend_stateless.cc b/chromium/media/gpu/v4l2/v4l2_video_decoder_backend_stateless.cc index 3ca859a4caf..b8c3400a990 100644 --- a/chromium/media/gpu/v4l2/v4l2_video_decoder_backend_stateless.cc +++ b/chromium/media/gpu/v4l2/v4l2_video_decoder_backend_stateless.cc @@ -529,13 +529,12 @@ bool V4L2StatelessVideoDecoderBackend::ApplyResolution( DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DCHECK_EQ(input_queue_->QueuedBuffersCount(), 0u); - struct v4l2_format format = {}; - - format.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; - if (device_->Ioctl(VIDIOC_G_FMT, &format) != 0) { + auto ret = input_queue_->GetFormat().first; + if (!ret) { VPLOGF(1) << "Failed getting OUTPUT format"; return false; } + struct v4l2_format format = std::move(*ret); format.fmt.pix_mp.width = pic_size.width(); format.fmt.pix_mp.height = pic_size.height(); diff --git a/chromium/media/gpu/v4l2/v4l2_video_encode_accelerator.cc b/chromium/media/gpu/v4l2/v4l2_video_encode_accelerator.cc index 3694d71f1b1..8c7ea443927 100644 --- a/chromium/media/gpu/v4l2/v4l2_video_encode_accelerator.cc +++ b/chromium/media/gpu/v4l2/v4l2_video_encode_accelerator.cc @@ -203,6 +203,13 @@ bool V4L2VideoEncodeAccelerator::Initialize(const Config& config, TRACE_EVENT0("media,gpu", "V4L2VEA::Initialize"); VLOGF(2) << ": " << config.AsHumanReadableString(); + // V4L2VEA doesn't support temporal layers but we let it pass here to support + // simulcast. + if (config.HasSpatialLayer()) { + VLOGF(1) << "Spatial layer encoding is supported"; + return false; + } + encoder_input_visible_rect_ = gfx::Rect(config.input_visible_size); client_ptr_factory_.reset(new base::WeakPtrFactory<Client>(client)); @@ -461,6 +468,11 @@ bool V4L2VideoEncodeAccelerator::AllocateImageProcessorOutputBuffers( DCHECK(image_processor_); DCHECK_EQ(image_processor_->output_mode(), ImageProcessor::OutputMode::IMPORT); + + // The existing buffers in |image_processor_output_buffers_| may be alive + // until they are actually consumed by the encoder driver, after they are + // destroyed here. + image_processor_output_buffers_.clear(); image_processor_output_buffers_.resize(count); const ImageProcessor::PortConfig& output_config = image_processor_->output_config(); @@ -757,19 +769,36 @@ bool V4L2VideoEncodeAccelerator::ReconfigureFormatIfNeeded( } if (!input_buffer_map_.empty()) { - // TODO(crbug.com/1060057): Handle coded_size change. - if (frame.coded_size() != input_frame_size_) { - VLOGF(1) << "Input frame size is changed during encoding" - << ", frame.coded_size()=" << frame.coded_size().ToString() - << ", input_frame_size=" << input_frame_size_.ToString(); + // ReconfigureFormatIfNeeded() has been called with the first VideoFrame. + // We checks here we need to (re)create ImageProcessor because the visible + // rectangle of |frame| differs from the first VideoFrame. + // |frame.natural_size()| is the size to be encoded. It must be the same as + // |encoder_input_visible_rect_.size()|, otherwise VEA client must recreate + // VEA with the new encoder resolution. + if (frame.natural_size() != encoder_input_visible_rect_.size()) { + VLOGF(1) << "Encoder resolution is changed during encoding" + << ", frame.natural_size()=" << frame.natural_size().ToString() + << ", encoder_input_visible_rect_=" + << input_frame_size_.ToString(); return false; } - return true; - } + if (frame.coded_size() == input_frame_size_) { + return true; + } + + // If a dimension of the underlying VideoFrame varies during video encoding + // (i.e. frame.coded_size() != input_frame_size_), we (re)create + // ImageProcessor to crop the VideoFrame, |frame.visible_rect()| -> + // |encoder_input_visible_rect_|. + // TODO(hiroh): if |frame.coded_size()| is the same as VideoFrame:: + // DetermineAlignedSize(input_format, encoder_input_visible_rect_.size()) + // and don't need a pixel format conversion, image processor is not + // necessary but we should rather NegotiateInputFormat(). + } else if (frame.coded_size() == input_frame_size_) { + // This path is for the first frame on Encode(). + // Height and width that V4L2VEA needs to configure. + const gfx::Size buffer_size(frame.stride(0), frame.coded_size().height()); - // Height and width that V4L2VEA needs to configure. - const gfx::Size buffer_size(frame.stride(0), frame.coded_size().height()); - if (frame.coded_size() == input_frame_size_) { // A buffer given by client is allocated with the same dimension using // minigbm. However, it is possible that stride and height are different // from ones adjusted by a driver. @@ -792,13 +821,30 @@ bool V4L2VideoEncodeAccelerator::ReconfigureFormatIfNeeded( // The |frame| dimension is different from the resolution configured to // V4L2VEA. This is the case that V4L2VEA needs to create ImageProcessor for - // scaling. Update |input_frame_size_| to check if succeeding frames' - // dimensions are not different from one of the first frame. + // cropping and scaling. Update |input_frame_size_| to check if succeeding + // frames' dimensions are not different from the current one. input_frame_size_ = frame.coded_size(); - return CreateImageProcessor(frame.layout(), device_input_layout_->format(), - device_input_layout_->coded_size(), - frame.visible_rect(), - encoder_input_visible_rect_); + if (!CreateImageProcessor(frame.layout(), device_input_layout_->format(), + device_input_layout_->coded_size(), + frame.visible_rect(), + encoder_input_visible_rect_)) { + return false; + } + + if (gfx::Size(image_processor_->output_config().planes[0].stride, + image_processor_->output_config().size.height()) != + device_input_layout_->coded_size()) { + VLOGF(1) << "Image Processor's output buffer's size is different from " + << "input buffer size configure to the encoder driver. " + << "ip's output buffer size: " + << gfx::Size(image_processor_->output_config().planes[0].stride, + image_processor_->output_config().size.height()) + .ToString() + << ", encoder's input buffer size: " + << device_input_layout_->coded_size().ToString(); + return false; + } + return true; } void V4L2VideoEncodeAccelerator::InputImageProcessorTask() { @@ -1553,8 +1599,28 @@ bool V4L2VideoEncodeAccelerator::SetFormats(VideoPixelFormat input_format, if (!SetOutputFormat(output_profile)) return false; - auto v4l2_format = - NegotiateInputFormat(input_format, encoder_input_visible_rect_.size()); + gfx::Size input_size = encoder_input_visible_rect_.size(); + if (native_input_mode_) { + DCHECK(!image_processor_gmb_factory_); + image_processor_gmb_factory_ = + gpu::GpuMemoryBufferFactory::CreateNativeType(nullptr); + if (!image_processor_gmb_factory_) { + VLOGF(1) << "Failed to create GpuMemoryBufferFactory"; + return false; + } + + auto input_layout = GetPlatformVideoFrameLayout( + image_processor_gmb_factory_.get(), input_format, + encoder_input_visible_rect_.size(), + gfx::BufferUsage::SCANOUT_VEA_READ_CAMERA_AND_CPU_READ_WRITE); + if (!input_layout) + return false; + input_size = gfx::Size(input_layout->planes()[0].stride, + input_layout->coded_size().height()); + } + + DCHECK(input_frame_size_.IsEmpty()); + auto v4l2_format = NegotiateInputFormat(input_format, input_size); if (!v4l2_format) return false; diff --git a/chromium/media/gpu/vaapi/BUILD.gn b/chromium/media/gpu/vaapi/BUILD.gn index 7ef509fafae..2524a1c31b5 100644 --- a/chromium/media/gpu/vaapi/BUILD.gn +++ b/chromium/media/gpu/vaapi/BUILD.gn @@ -109,7 +109,6 @@ source_set("vaapi") { } if (use_x11) { - configs += [ "//build/config/linux:x11" ] deps += [ "//ui/gfx/x" ] sources += [ "vaapi_picture_tfp.cc", @@ -167,7 +166,6 @@ source_set("common") { } if (use_x11) { - configs += [ "//build/config/linux:x11" ] deps += [ "//ui/gfx/x" ] } @@ -195,6 +193,7 @@ source_set("unit_test") { "h264_encoder_unittest.cc", "vaapi_image_decode_accelerator_worker_unittest.cc", "vaapi_video_decode_accelerator_unittest.cc", + "vaapi_video_encode_accelerator_unittest.cc", ] deps = [ ":common", @@ -285,6 +284,7 @@ test("vaapi_unittest") { ":common", ":vaapi_utils_unittest", "//base", + "//base/test:test_support", "//media/gpu/test:helpers", "//testing/gtest", ] diff --git a/chromium/media/gpu/vaapi/h264_encoder.h b/chromium/media/gpu/vaapi/h264_encoder.h index 517ff2a5ede..908e1f7bf78 100644 --- a/chromium/media/gpu/vaapi/h264_encoder.h +++ b/chromium/media/gpu/vaapi/h264_encoder.h @@ -53,7 +53,7 @@ class H264Encoder : public AcceleratedVideoEncoder { // Bitrate window size in bits. unsigned int cpb_size_bits; - // Quantization parameter. + // Quantization parameter. Their ranges are 0-51. int initial_qp; ScalingSettings scaling_settings; diff --git a/chromium/media/gpu/vaapi/vaapi_image_decode_accelerator_worker_unittest.cc b/chromium/media/gpu/vaapi/vaapi_image_decode_accelerator_worker_unittest.cc index 31f840b5695..0767729e06d 100644 --- a/chromium/media/gpu/vaapi/vaapi_image_decode_accelerator_worker_unittest.cc +++ b/chromium/media/gpu/vaapi/vaapi_image_decode_accelerator_worker_unittest.cc @@ -14,11 +14,12 @@ #include "testing/gtest/include/gtest/gtest.h" #include "base/bind.h" +#include "base/check_op.h" #include "base/containers/span.h" -#include "base/logging.h" #include "base/macros.h" #include "base/memory/ptr_util.h" #include "base/memory/scoped_refptr.h" +#include "base/notreached.h" #include "base/test/scoped_feature_list.h" #include "base/test/task_environment.h" #include "gpu/config/gpu_finch_features.h" diff --git a/chromium/media/gpu/vaapi/vaapi_image_processor_backend.cc b/chromium/media/gpu/vaapi/vaapi_image_processor_backend.cc index 6f63dafba87..4cb6bceda56 100644 --- a/chromium/media/gpu/vaapi/vaapi_image_processor_backend.cc +++ b/chromium/media/gpu/vaapi/vaapi_image_processor_backend.cc @@ -25,6 +25,7 @@ namespace media { +#if defined(OS_CHROMEOS) namespace { // UMA errors that the VaapiImageProcessorBackend class reports. enum class VaIPFailure { @@ -67,6 +68,7 @@ bool IsSupported(uint32_t input_va_fourcc, } } // namespace +#endif // static std::unique_ptr<ImageProcessorBackend> VaapiImageProcessorBackend::Create( @@ -78,8 +80,7 @@ std::unique_ptr<ImageProcessorBackend> VaapiImageProcessorBackend::Create( // VaapiImageProcessorBackend supports ChromeOS only. #if !defined(OS_CHROMEOS) return nullptr; -#endif - +#else auto input_vafourcc = input_config.fourcc.ToVAFourCC(); if (!input_vafourcc) { VLOGF(2) << "Input fourcc " << input_config.fourcc.ToString() @@ -146,6 +147,7 @@ std::unique_ptr<ImageProcessorBackend> VaapiImageProcessorBackend::Create( return base::WrapUnique<ImageProcessorBackend>(new VaapiImageProcessorBackend( std::move(vaapi_wrapper), input_config, output_config, OutputMode::IMPORT, std::move(error_cb), std::move(backend_task_runner))); +#endif } VaapiImageProcessorBackend::VaapiImageProcessorBackend( diff --git a/chromium/media/gpu/vaapi/vaapi_jpeg_encoder.cc b/chromium/media/gpu/vaapi/vaapi_jpeg_encoder.cc index 854bababd3f..643dc91ca29 100644 --- a/chromium/media/gpu/vaapi/vaapi_jpeg_encoder.cc +++ b/chromium/media/gpu/vaapi/vaapi_jpeg_encoder.cc @@ -10,7 +10,7 @@ #include <stddef.h> #include <string.h> -#include "base/logging.h" +#include "base/check_op.h" #include "base/numerics/ranges.h" #include "base/numerics/safe_conversions.h" #include "base/stl_util.h" diff --git a/chromium/media/gpu/vaapi/vaapi_unittest.cc b/chromium/media/gpu/vaapi/vaapi_unittest.cc index 7de8dec2a07..d3d459fadf8 100644 --- a/chromium/media/gpu/vaapi/vaapi_unittest.cc +++ b/chromium/media/gpu/vaapi/vaapi_unittest.cc @@ -12,15 +12,14 @@ #include <va/va.h> -#include "base/at_exit.h" -#include "base/command_line.h" #include "base/files/file.h" #include "base/files/scoped_file.h" -#include "base/logging.h" #include "base/optional.h" #include "base/process/launch.h" #include "base/stl_util.h" #include "base/strings/string_split.h" +#include "base/test/launcher/unit_test_launcher.h" +#include "base/test/test_suite.h" #include "media/gpu/vaapi/vaapi_wrapper.h" namespace media { @@ -216,17 +215,11 @@ TEST_F(VaapiTest, DefaultEntrypointIsSupported) { } // namespace media int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - base::CommandLine::Init(argc, argv); - base::ShadowingAtExitManager at_exit_manager; - - // Needed to enable DVLOG through --vmodule. - logging::LoggingSettings settings; - settings.logging_dest = - logging::LOG_TO_SYSTEM_DEBUG_LOG | logging::LOG_TO_STDERR; - LOG_ASSERT(logging::InitLogging(settings)); + base::TestSuite test_suite(argc, argv); media::VaapiWrapper::PreSandboxInitialization(); - return RUN_ALL_TESTS(); + return base::LaunchUnitTests( + argc, argv, + base::BindOnce(&base::TestSuite::Run, base::Unretained(&test_suite))); } diff --git a/chromium/media/gpu/vaapi/vaapi_video_decode_accelerator.cc b/chromium/media/gpu/vaapi/vaapi_video_decode_accelerator.cc index 5020b2b8bf9..63ea51830ee 100644 --- a/chromium/media/gpu/vaapi/vaapi_video_decode_accelerator.cc +++ b/chromium/media/gpu/vaapi/vaapi_video_decode_accelerator.cc @@ -458,11 +458,25 @@ void VaapiVideoDecodeAccelerator::DecodeTask() { "The visible rectangle is not contained by the picture size", UNREADABLE_INPUT, ); VLOGF(2) << "Decoder requesting a new set of surfaces"; + size_t required_num_of_pictures = decoder_->GetRequiredNumOfPictures(); + if (buffer_allocation_mode_ == BufferAllocationMode::kNone && + profile_ >= H264PROFILE_MIN && profile_ <= H264PROFILE_MAX) { + // For H.264, the decoder might request too few pictures. In + // BufferAllocationMode::kNone, this can cause us to do a lot of busy + // work waiting for picture buffers to come back from the client (see + // crbug.com/910986#c32). This is a workaround to increase the + // likelihood that we don't have to wait on buffers to come back from + // the client. |kNumOfPics| is picked to mirror the value returned by + // VP9Decoder::GetRequiredNumOfPictures(). + constexpr size_t kMinNumOfPics = 13u; + required_num_of_pictures = + std::max(kMinNumOfPics, required_num_of_pictures); + } task_runner_->PostTask( FROM_HERE, base::BindOnce( &VaapiVideoDecodeAccelerator::InitiateSurfaceSetChange, - weak_this_, decoder_->GetRequiredNumOfPictures(), pic_size, + weak_this_, required_num_of_pictures, pic_size, decoder_->GetNumReferenceFrames(), visible_rect)); // We'll get rescheduled once ProvidePictureBuffers() finishes. return; @@ -1158,12 +1172,11 @@ VaapiVideoDecodeAccelerator::DecideBufferAllocationMode() { // On Gemini Lake, Kaby Lake and later we can pass to libva the client's // PictureBuffers to decode onto, which skips the use of the Vpp unit and its // associated format reconciliation copy, avoiding all internal buffer - // allocations. This only works for VP8 and VP9: H264 GetNumReferenceFrames() - // depends on the bitstream and sometimes it's not enough to cover the amount - // of frames needed by the client pipeline (see b/133733739). + // allocations. // TODO(crbug.com/911754): Enable for VP9 Profile 2. if (IsGeminiLakeOrLater() && - (profile_ == VP9PROFILE_PROFILE0 || profile_ == VP8PROFILE_ANY)) { + (profile_ == VP9PROFILE_PROFILE0 || profile_ == VP8PROFILE_ANY || + (profile_ >= H264PROFILE_MIN && profile_ <= H264PROFILE_MAX))) { // Add one to the reference frames for the one being currently egressed, and // an extra allocation for both |client_| and |decoder_|, see // crrev.com/c/1576560. @@ -1172,18 +1185,18 @@ VaapiVideoDecodeAccelerator::DecideBufferAllocationMode() { return BufferAllocationMode::kNone; } - // If we're here, we have to use the Vpp unit and allocate buffers for - // |decoder_|; usually we'd have to allocate the |decoder_|s - // GetRequiredNumOfPictures() internally, we can allocate just |decoder_|s - // GetNumReferenceFrames() + 1. Moreover, we also request the |client_| to - // allocate less than the usual |decoder_|s GetRequiredNumOfPictures(). - - // Another +1 is experimentally needed for high-to-high resolution changes. + // For H.264 on older devices, another +1 is experimentally needed for + // high-to-high resolution changes. // TODO(mcasas): Figure out why and why only H264, see crbug.com/912295 and // http://crrev.com/c/1363807/9/media/gpu/h264_decoder.cc#1449. if (profile_ >= H264PROFILE_MIN && profile_ <= H264PROFILE_MAX) return BufferAllocationMode::kReduced; + // If we're here, we have to use the Vpp unit and allocate buffers for + // |decoder_|; usually we'd have to allocate the |decoder_|s + // GetRequiredNumOfPictures() internally, we can allocate just |decoder_|s + // GetNumReferenceFrames() + 1. Moreover, we also request the |client_| to + // allocate less than the usual |decoder_|s GetRequiredNumOfPictures(). return BufferAllocationMode::kSuperReduced; } diff --git a/chromium/media/gpu/vaapi/vaapi_video_decode_accelerator_unittest.cc b/chromium/media/gpu/vaapi/vaapi_video_decode_accelerator_unittest.cc index 280fad78379..12daaefd865 100644 --- a/chromium/media/gpu/vaapi/vaapi_video_decode_accelerator_unittest.cc +++ b/chromium/media/gpu/vaapi/vaapi_video_decode_accelerator_unittest.cc @@ -37,10 +37,10 @@ struct TestParams { constexpr int32_t kBitstreamId = 123; constexpr size_t kInputSize = 256; -constexpr size_t kNumPictures = 4; +constexpr size_t kNumPictures = 14; const gfx::Size kPictureSize(64, 48); -constexpr size_t kNewNumPictures = 3; +constexpr size_t kNewNumPictures = 13; const gfx::Size kNewPictureSize(64, 48); MATCHER_P2(IsExpectedDecoderBuffer, data_size, decrypt_config, "") { @@ -488,6 +488,7 @@ TEST_P(VaapiVideoDecodeAcceleratorTest, constexpr TestParams kTestCases[] = { {H264PROFILE_MIN, false /* decode_using_client_picture_buffers */}, + {H264PROFILE_MIN, true /* decode_using_client_picture_buffers */}, {VP8PROFILE_MIN, false /* decode_using_client_picture_buffers */}, {VP9PROFILE_MIN, false /* decode_using_client_picture_buffers */}, {VP9PROFILE_MIN, true /* decode_using_client_picture_buffers */}}; diff --git a/chromium/media/gpu/vaapi/vaapi_video_decoder.cc b/chromium/media/gpu/vaapi/vaapi_video_decoder.cc index b7f676aa93f..c97f1a06cd9 100644 --- a/chromium/media/gpu/vaapi/vaapi_video_decoder.cc +++ b/chromium/media/gpu/vaapi/vaapi_video_decoder.cc @@ -240,7 +240,7 @@ void VaapiVideoDecoder::HandleDecodeTask() { case AcceleratedVideoDecoder::kConfigChange: // A new set of output buffers is requested. We either didn't have any // output buffers yet or encountered a resolution change. - // After the pipeline flushes all frames, OnPipelineFlushed() will be + // After the pipeline flushes all frames, ApplyResolutionChange() will be // called and we can start changing resolution. DCHECK(client_); SetState(State::kChangingResolution); @@ -392,7 +392,7 @@ void VaapiVideoDecoder::OutputFrameTask(scoped_refptr<VideoFrame> video_frame, output_cb_.Run(std::move(video_frame)); } -void VaapiVideoDecoder::OnPipelineFlushed() { +void VaapiVideoDecoder::ApplyResolutionChange() { DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_sequence_checker_); DCHECK(state_ == State::kChangingResolution || state_ == State::kWaitingForInput); diff --git a/chromium/media/gpu/vaapi/vaapi_video_decoder.h b/chromium/media/gpu/vaapi/vaapi_video_decoder.h index 01e282d2613..db186f14734 100644 --- a/chromium/media/gpu/vaapi/vaapi_video_decoder.h +++ b/chromium/media/gpu/vaapi/vaapi_video_decoder.h @@ -54,7 +54,7 @@ class VaapiVideoDecoder : public DecoderInterface, const OutputCB& output_cb) override; void Decode(scoped_refptr<DecoderBuffer> buffer, DecodeCB decode_cb) override; void Reset(base::OnceClosure reset_cb) override; - void OnPipelineFlushed() override; + void ApplyResolutionChange() override; // DecodeSurfaceHandler<VASurface> implementation. scoped_refptr<VASurface> CreateSurface() override; diff --git a/chromium/media/gpu/vaapi/vaapi_video_encode_accelerator.cc b/chromium/media/gpu/vaapi/vaapi_video_encode_accelerator.cc index 234754e1463..c7ae04b8be9 100644 --- a/chromium/media/gpu/vaapi/vaapi_video_encode_accelerator.cc +++ b/chromium/media/gpu/vaapi/vaapi_video_encode_accelerator.cc @@ -280,6 +280,13 @@ bool VaapiVideoEncodeAccelerator::Initialize(const Config& config, VLOGF(2) << "Initializing VAVEA, " << config.AsHumanReadableString(); + // VaapiVEA doesn't support temporal layers but we let it pass here to support + // simulcast. + if (config.HasSpatialLayer()) { + VLOGF(1) << "Spatial layer encoding is supported"; + return false; + } + client_ptr_factory_.reset(new base::WeakPtrFactory<Client>(client)); client_ = client_ptr_factory_->GetWeakPtr(); @@ -444,7 +451,9 @@ void VaapiVideoEncodeAccelerator::InitializeTask(const Config& config) { num_frames_in_flight_, expected_input_coded_size_, output_buffer_byte_size_)); - encoder_info_.scaling_settings = encoder_->GetScalingSettings(); + // TODO(crbug.com/1034686): Set ScalingSettings causes getStats() hangs. + // Investigate and fix the issue. + // encoder_info_.scaling_settings = encoder_->GetScalingSettings(); // TODO(crbug.com/1030199): VaapiVideoEncodeAccelerator doesn't support either // temporal-SVC or spatial-SVC. Update |fps_allocation| properly once they are diff --git a/chromium/media/gpu/vaapi/vaapi_video_encode_accelerator_unittest.cc b/chromium/media/gpu/vaapi/vaapi_video_encode_accelerator_unittest.cc new file mode 100644 index 00000000000..01bfbb3a6e0 --- /dev/null +++ b/chromium/media/gpu/vaapi/vaapi_video_encode_accelerator_unittest.cc @@ -0,0 +1,82 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/gpu/vaapi/vaapi_video_encode_accelerator.h" + +#include "base/test/task_environment.h" +#include "media/video/video_encode_accelerator.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace media { +namespace { + +constexpr gfx::Size kDefaultEncodeSize(1280, 720); +constexpr uint32_t kDefaultBitrateBps = 4 * 1000 * 1000; +constexpr uint32_t kDefaultFramerate = 30; +const VideoEncodeAccelerator::Config kDefaultVEAConfig(PIXEL_FORMAT_I420, + kDefaultEncodeSize, + VP8PROFILE_ANY, + kDefaultBitrateBps, + kDefaultFramerate); + +class MockVideoEncodeAcceleratorClient : public VideoEncodeAccelerator::Client { + public: + MockVideoEncodeAcceleratorClient() = default; + virtual ~MockVideoEncodeAcceleratorClient() = default; + + MOCK_METHOD3(RequireBitstreamBuffers, + void(unsigned int, const gfx::Size&, size_t output_buffer_size)); + MOCK_METHOD2(BitstreamBufferReady, + void(int32_t, const BitstreamBufferMetadata&)); + MOCK_METHOD1(NotifyError, void(VideoEncodeAccelerator::Error)); + MOCK_METHOD1(NotifyEncoderInfoChange, void(const VideoEncoderInfo& info)); +}; + +struct VaapiVEAInitializeTestParam { + uint8_t num_of_temporal_layers = 0; + uint8_t num_of_spatial_layers = 0; + bool expected_result; +}; + +class VaapiVEAInitializeTest + : public ::testing::TestWithParam<VaapiVEAInitializeTestParam> { + protected: + VaapiVEAInitializeTest() = default; + ~VaapiVEAInitializeTest() override = default; + base::test::TaskEnvironment task_environment_; +}; + +TEST_P(VaapiVEAInitializeTest, SpatialLayerAndTemporalLayerEncoding) { + VideoEncodeAccelerator::Config config = kDefaultVEAConfig; + const uint8_t num_of_temporal_layers = GetParam().num_of_temporal_layers; + const uint8_t num_of_spatial_layers = GetParam().num_of_spatial_layers; + constexpr int kDenom[] = {4, 2, 1}; + for (uint8_t i = 0; i < num_of_spatial_layers; ++i) { + VideoEncodeAccelerator::Config::SpatialLayer spatial_layer; + int denom = kDenom[i]; + spatial_layer.width = kDefaultEncodeSize.width() / denom; + spatial_layer.height = kDefaultEncodeSize.height() / denom; + spatial_layer.bitrate_bps = kDefaultBitrateBps / denom; + spatial_layer.framerate = kDefaultFramerate; + spatial_layer.max_qp = 30; + spatial_layer.num_of_temporal_layers = num_of_temporal_layers; + config.spatial_layers.push_back(spatial_layer); + } + + VaapiVideoEncodeAccelerator vea; + MockVideoEncodeAcceleratorClient client; + EXPECT_EQ(vea.Initialize(config, &client), GetParam().expected_result); +} + +constexpr VaapiVEAInitializeTestParam kTestCases[] = { + {1u, 3u, false}, // Spatial Layer only. + {3u, 3u, false}, // Temporal + Spatial Layer. +}; + +INSTANTIATE_TEST_SUITE_P(SpatialLayerAndTemporalLayerEncoding, + VaapiVEAInitializeTest, + ::testing::ValuesIn(kTestCases)); +} // namespace +} // namespace media diff --git a/chromium/media/gpu/vaapi/vaapi_wrapper.cc b/chromium/media/gpu/vaapi/vaapi_wrapper.cc index aa951cf8ee7..f238e6f0851 100644 --- a/chromium/media/gpu/vaapi/vaapi_wrapper.cc +++ b/chromium/media/gpu/vaapi/vaapi_wrapper.cc @@ -21,6 +21,7 @@ #include "base/bind_helpers.h" #include "base/bits.h" #include "base/callback_helpers.h" +#include "base/cpu.h" #include "base/environment.h" #include "base/files/scoped_file.h" #include "base/logging.h" @@ -141,6 +142,31 @@ namespace media { namespace { +// Returns true if the SoC has a 9.5 GPU. CPU model IDs are referenced from the +// following file in the kernel source: arch/x86/include/asm/intel-family.h. +bool IsGen95Gpu() { + constexpr int kPentiumAndLaterFamily = 0x06; + constexpr int kKabyLakeModelId = 0x9E; + // Amber Lake, Whiskey Lake and some Comet Lake CPU IDs are the same as KBL L. + constexpr int kKabyLake_LModelId = 0x8E; + constexpr int kGeminiLakeModelId = 0x7A; + constexpr int kCometLakeModelId = 0xA5; + constexpr int kCometLake_LModelId = 0xA6; + static base::NoDestructor<base::CPU> cpuid; + static const bool is_gen95_gpu = cpuid->family() == kPentiumAndLaterFamily && + (cpuid->model() == kKabyLakeModelId || + cpuid->model() == kKabyLake_LModelId || + cpuid->model() == kGeminiLakeModelId || + cpuid->model() == kCometLakeModelId || + cpuid->model() == kCometLake_LModelId); + return is_gen95_gpu; +} + +bool IsModeEncoding(VaapiWrapper::CodecMode mode) { + return mode == VaapiWrapper::CodecMode::kEncode || + mode == VaapiWrapper::CodecMode::kEncodeConstantQuantizationParameter; +} + bool GetNV12VisibleWidthBytes(int visible_width, uint32_t plane, size_t* bytes) { @@ -220,7 +246,8 @@ static const struct { {H264PROFILE_HIGH, VAProfileH264High}, {VP8PROFILE_ANY, VAProfileVP8Version0_3}, {VP9PROFILE_PROFILE0, VAProfileVP9Profile0}, - {VP9PROFILE_PROFILE1, VAProfileVP9Profile1}, + // VP9 hw encode/decode on profile 1 is not enabled on chromium-vaapi. + // {VP9PROFILE_PROFILE1, VAProfileVP9Profile1}, // TODO(crbug.com/1011454, crbug.com/1011469): Reenable VP9PROFILE_PROFILE2 // and _PROFILE3 when P010 is completely supported. //{VP9PROFILE_PROFILE2, VAProfileVP9Profile2}, @@ -308,14 +335,14 @@ std::string VAProfileToString(VAProfile va_profile) { bool IsBlackListedDriver(const std::string& va_vendor_string, VaapiWrapper::CodecMode mode, VAProfile va_profile) { - if (mode != VaapiWrapper::CodecMode::kEncode) + if (!IsModeEncoding(mode)) return false; // TODO(crbug.com/828482): Remove once H264 encoder on AMD is enabled by // default. if (VendorStringToImplementationType(va_vendor_string) == VAImplementation::kMesaGallium && - va_vendor_string.find("AMD STONEY") != std::string::npos && + base::Contains(va_vendor_string, "AMD STONEY") && !base::FeatureList::IsEnabled(kVaapiH264AMDEncoder)) { constexpr VAProfile kH264Profiles[] = {VAProfileH264Baseline, VAProfileH264Main, VAProfileH264High, @@ -330,13 +357,17 @@ bool IsBlackListedDriver(const std::string& va_vendor_string, return true; } - // TODO(crbug.com/811912): Remove once VP9 encoding is to be enabled by - // default. + // TODO(crbug.com/811912): Remove once VP9 encoding is enabled by default. if (va_profile == VAProfileVP9Profile0 && !base::FeatureList::IsEnabled(kVaapiVP9Encoder)) { return true; } + // TODO(b/158655609): Several Gen 9.5 GPU devices suffer from a GPU hang when + // VP8 encoding in some power saving states. Blacklist them temporarily. + if (IsGen95Gpu() && va_profile == VAProfileVP8Version0_3) + return true; + return false; } @@ -556,6 +587,9 @@ std::vector<VAEntrypoint> GetEntryPointsForProfile(const base::Lock* va_lock, {VAEntrypointVLD}, // For kDecode. {VAEntrypointEncSlice, VAEntrypointEncPicture, VAEntrypointEncSliceLP}, // For kEncode. + {VAEntrypointEncSlice, + VAEntrypointEncSliceLP}, // For + // kEncodeConstantQuantizationParameter. {VAEntrypointVideoProc} // For kVideoProcess. }; std::vector<VAEntrypoint> entrypoints; @@ -586,12 +620,15 @@ static bool GetRequiredAttribs(const base::Lock* va_lock, required_attribs->push_back({VAConfigAttribRTFormat, VA_RT_FORMAT_YUV420}); } - if (mode != VaapiWrapper::kEncode) + if (!IsModeEncoding(mode)) return true; - // All encoding use constant bit rate except for JPEG. - if (profile != VAProfileJPEGBaseline) - required_attribs->push_back({VAConfigAttribRateControl, VA_RC_CBR}); + if (profile != VAProfileJPEGBaseline) { + if (mode == VaapiWrapper::kEncode) + required_attribs->push_back({VAConfigAttribRateControl, VA_RC_CBR}); + if (mode == VaapiWrapper::kEncodeConstantQuantizationParameter) + required_attribs->push_back({VAConfigAttribRateControl, VA_RC_CQP}); + } // VAConfigAttribEncPackedHeaders is H.264 specific. if ((profile >= VAProfileH264Baseline && profile <= VAProfileH264High) || @@ -1359,6 +1396,11 @@ bool VaapiWrapper::GetJpegDecodeSuitableImageFourCC(unsigned int rt_format, preferred_fourcc == VA_FOURCC_P010) { preferred_fourcc = VA_FOURCC_I420; } + } else if (GetImplementationType() == VAImplementation::kIntelIHD) { + // TODO(b/155939640): iHD v19.4 fails to allocate AYUV surfaces for the VPP + // on gen 9.5. + if (preferred_fourcc == VA_FOURCC_AYUV) + preferred_fourcc = VA_FOURCC_I420; } if (!VASupportedImageFormats::Get().IsImageFormatSupported( @@ -1449,6 +1491,7 @@ VAEntrypoint VaapiWrapper::GetDefaultVaEntryPoint(CodecMode mode, case VaapiWrapper::kDecode: return VAEntrypointVLD; case VaapiWrapper::kEncode: + case VaapiWrapper::kEncodeConstantQuantizationParameter: if (profile == VAProfileJPEGBaseline) return VAEntrypointEncPicture; else @@ -2191,12 +2234,18 @@ VaapiWrapper::~VaapiWrapper() { } bool VaapiWrapper::Initialize(CodecMode mode, VAProfile va_profile) { +#if DCHECK_IS_ON() + if (mode == kEncodeConstantQuantizationParameter) { + DCHECK_NE(va_profile, VAProfileJPEGBaseline) + << "JPEG Encoding doesn't support CQP bitrate control"; + } +#endif // DCHECK_IS_ON() + if (mode != kVideoProcess) TryToSetVADisplayAttributeToLocalGPU(); VAEntrypoint entrypoint = GetDefaultVaEntryPoint(mode, va_profile); - - if (mode == CodecMode::kEncode && IsLowPowerEncSupported(va_profile) && + if (IsModeEncoding(mode) && IsLowPowerEncSupported(va_profile, mode) && base::FeatureList::IsEnabled(kVaapiLowPowerEncoder)) { entrypoint = VAEntrypointEncSliceLP; DVLOG(2) << "Enable VA-API Low-Power Encode Entrypoint"; @@ -2205,8 +2254,9 @@ bool VaapiWrapper::Initialize(CodecMode mode, VAProfile va_profile) { base::AutoLock auto_lock(*va_lock_); std::vector<VAConfigAttrib> required_attribs; if (!GetRequiredAttribs(va_lock_, va_display_, mode, va_profile, entrypoint, - &required_attribs)) + &required_attribs)) { return false; + } VAStatus va_res = vaCreateConfig(va_display_, va_profile, entrypoint, @@ -2420,7 +2470,8 @@ void VaapiWrapper::TryToSetVADisplayAttributeToLocalGPU() { } // Check the support for low-power encode -bool VaapiWrapper::IsLowPowerEncSupported(VAProfile va_profile) const { +bool VaapiWrapper::IsLowPowerEncSupported(VAProfile va_profile, + CodecMode mode) const { // Enabled only for H264/AVC & VP9 Encoders if (va_profile != VAProfileH264ConstrainedBaseline && va_profile != VAProfileH264Main && va_profile != VAProfileH264High && @@ -2431,8 +2482,8 @@ bool VaapiWrapper::IsLowPowerEncSupported(VAProfile va_profile) const { std::vector<VAConfigAttrib> required_attribs; base::AutoLock auto_lock(*va_lock_); - GetRequiredAttribs(va_lock_, va_display_, VaapiWrapper::CodecMode::kEncode, - va_profile, kLowPowerEncEntryPoint, &required_attribs); + GetRequiredAttribs(va_lock_, va_display_, mode, va_profile, + kLowPowerEncEntryPoint, &required_attribs); // Query the driver for required attributes. std::vector<VAConfigAttrib> attribs = required_attribs; for (size_t i = 0; i < required_attribs.size(); ++i) diff --git a/chromium/media/gpu/vaapi/vaapi_wrapper.h b/chromium/media/gpu/vaapi/vaapi_wrapper.h index c3190da3b94..7f087039c58 100644 --- a/chromium/media/gpu/vaapi/vaapi_wrapper.h +++ b/chromium/media/gpu/vaapi/vaapi_wrapper.h @@ -97,7 +97,9 @@ class MEDIA_GPU_EXPORT VaapiWrapper public: enum CodecMode { kDecode, - kEncode, + kEncode, // Encode with Constant Bitrate algorithm. + kEncodeConstantQuantizationParameter, // Encode with Constant Quantization + // Parameter algorithm. kVideoProcess, kCodecModeMax, }; @@ -426,8 +428,8 @@ class MEDIA_GPU_EXPORT VaapiWrapper // Attempt to set render mode to "render to texture.". Failure is non-fatal. void TryToSetVADisplayAttributeToLocalGPU(); - // Check low-power encode support for the given profile - bool IsLowPowerEncSupported(VAProfile va_profile) const; + // Check low-power encode support for |profile| and |mode|. + bool IsLowPowerEncSupported(VAProfile va_profile, CodecMode mode) const; const CodecMode mode_; diff --git a/chromium/media/gpu/vaapi/vp8_encoder.cc b/chromium/media/gpu/vaapi/vp8_encoder.cc index 5d9591482e6..aaf6c7bf00c 100644 --- a/chromium/media/gpu/vaapi/vp8_encoder.cc +++ b/chromium/media/gpu/vaapi/vp8_encoder.cc @@ -16,12 +16,14 @@ constexpr size_t kKFPeriod = 3000; // Arbitrarily chosen bitrate window size for rate control, in ms. constexpr int kCPBWindowSizeMs = 1500; -// Based on WebRTC's defaults. +// Quantization parameter. They are vp8 ac/dc indices and their ranges are +// 0-127. Based on WebRTC's defaults. constexpr int kMinQP = 4; // b/110059922, crbug.com/1001900: Tuned 112->117 for bitrate issue in a lower // resolution (180p). constexpr int kMaxQP = 117; -constexpr int kDefaultQP = (3 * kMinQP + kMaxQP) / 4; +// This stands for 32 as a real ac value (see rfc 14.1. table ac_qlookup). +constexpr int kDefaultQP = 28; } // namespace VP8Encoder::EncodeParams::EncodeParams() @@ -161,12 +163,7 @@ void VP8Encoder::InitializeFrameHeader() { DCHECK(!visible_size_.IsEmpty()); current_frame_hdr_.width = visible_size_.width(); current_frame_hdr_.height = visible_size_.height(); - // Since initial_qp is always kDefaultQP (=32), y_ac_qi should be 28 - // (the table index for kDefaultQP, see rfc 14.1. table ac_qlookup) - static_assert(kDefaultQP == 32, "kDefault QP is not 32"); - DCHECK_EQ(current_params_.initial_qp, kDefaultQP); - constexpr uint8_t kDefaultQPACQIndex = 28; - current_frame_hdr_.quantization_hdr.y_ac_qi = kDefaultQPACQIndex; + current_frame_hdr_.quantization_hdr.y_ac_qi = kDefaultQP; current_frame_hdr_.show_frame = true; // TODO(sprang): Make this dynamic. Value based on reference implementation // in libyami (https://github.com/intel/libyami). diff --git a/chromium/media/gpu/vaapi/vp8_encoder.h b/chromium/media/gpu/vaapi/vp8_encoder.h index c38282253f6..b7dcbea3eb9 100644 --- a/chromium/media/gpu/vaapi/vp8_encoder.h +++ b/chromium/media/gpu/vaapi/vp8_encoder.h @@ -38,6 +38,8 @@ class VP8Encoder : public AcceleratedVideoEncoder { // Coded picture buffer size in bits. unsigned int cpb_size_bits; + // Quantization parameter. They are vp8 ac/dc indices and their ranges are + // 0-127. int initial_qp; ScalingSettings scaling_settings; diff --git a/chromium/media/gpu/vaapi/vp9_encoder.cc b/chromium/media/gpu/vaapi/vp9_encoder.cc index 6050678593a..140ac37af4c 100644 --- a/chromium/media/gpu/vaapi/vp9_encoder.cc +++ b/chromium/media/gpu/vaapi/vp9_encoder.cc @@ -16,10 +16,14 @@ constexpr size_t kKFPeriod = 3000; // Arbitrarily chosen bitrate window size for rate control, in ms. constexpr int kCPBWindowSizeMs = 500; -// Based on WebRTC's defaults. +// Quantization parameter. They are vp9 ac/dc indices and their ranges are +// 0-255. Based on WebRTC's defaults. constexpr int kMinQP = 4; constexpr int kMaxQP = 112; -constexpr int kDefaultQP = (3 * kMinQP + kMaxQP) / 4; +// This stands for 31 as a real ac value (see rfc 8.6.1 table +// ac_qlookup[3][256]). Note: This needs to be revisited once we have 10&12 bit +// encoder support. +constexpr int kDefaultQP = 24; // filter level may affect on quality at lower bitrates; for now, // we set a constant value (== 10) which is what other VA-API @@ -166,12 +170,7 @@ void VP9Encoder::InitializeFrameHeader() { current_frame_hdr_.frame_height = visible_size_.height(); current_frame_hdr_.render_width = visible_size_.width(); current_frame_hdr_.render_height = visible_size_.height(); - // Since initial_qp is always kDefaultQP (=31), base_q_idx should be 24 - // (the table index for kDefaultQP, see rfc 8.6.1 table ac_qlookup[3][256]) - // Note: This needs to be revisited once we have 10&12 bit encoder support - DCHECK_EQ(current_params_.initial_qp, kDefaultQP); - constexpr uint8_t kDefaultQPACQIndex = 24; - current_frame_hdr_.quant_params.base_q_idx = kDefaultQPACQIndex; + current_frame_hdr_.quant_params.base_q_idx = kDefaultQP; current_frame_hdr_.loop_filter.level = kDefaultLfLevel; current_frame_hdr_.show_frame = true; } diff --git a/chromium/media/gpu/vaapi/vp9_encoder.h b/chromium/media/gpu/vaapi/vp9_encoder.h index 0ff69c0d165..2f3eda4b440 100644 --- a/chromium/media/gpu/vaapi/vp9_encoder.h +++ b/chromium/media/gpu/vaapi/vp9_encoder.h @@ -40,6 +40,8 @@ class VP9Encoder : public AcceleratedVideoEncoder { // Coded picture buffer size in bits. unsigned int cpb_size_bits; + // Quantization parameter. They are vp9 ac/dc indices and their ranges are + // 0-255. int initial_qp; ScalingSettings scaling_settings; diff --git a/chromium/media/gpu/video_encode_accelerator_unittest.cc b/chromium/media/gpu/video_encode_accelerator_unittest.cc index 520e9153541..25b8342343c 100644 --- a/chromium/media/gpu/video_encode_accelerator_unittest.cc +++ b/chromium/media/gpu/video_encode_accelerator_unittest.cc @@ -301,8 +301,12 @@ bool ShouldSkipTest(VideoPixelFormat format) { // Disable mid_stream_bitrate_switch test cases for elm/hana. {"elm", "MidStreamParamSwitchBitrate", PIXEL_FORMAT_UNKNOWN}, {"elm", "MultipleEncoders", PIXEL_FORMAT_UNKNOWN}, + {"elm-kernelnext", "MidStreamParamSwitchBitrate", PIXEL_FORMAT_UNKNOWN}, + {"elm-kernelnext", "MultipleEncoders", PIXEL_FORMAT_UNKNOWN}, {"hana", "MidStreamParamSwitchBitrate", PIXEL_FORMAT_UNKNOWN}, {"hana", "MultipleEncoders", PIXEL_FORMAT_UNKNOWN}, + {"hana-kernelnext", "MidStreamParamSwitchBitrate", PIXEL_FORMAT_UNKNOWN}, + {"hana-kernelnext", "MultipleEncoders", PIXEL_FORMAT_UNKNOWN}, // crbug.com/965348#c6: Tegra driver calculates the wrong plane size of // NV12. Disable all tests on nyan family for NV12 test. @@ -691,7 +695,7 @@ class VideoEncodeAcceleratorTestEnvironment : public ::testing::Environment { #if defined(USE_OZONE) // Initialize Ozone so that DMABuf can be created through Ozone DRM. ui::OzonePlatform::InitParams params; - params.single_process = false; + params.single_process = true; ui::OzonePlatform::InitializeForUI(params); base::Thread::Options options; @@ -1136,8 +1140,8 @@ void VideoFrameQualityValidator::Initialize(const gfx::Size& coded_size, decoder_->Initialize( config, false, nullptr, - base::BindRepeating(&VideoFrameQualityValidator::InitializeCB, - base::Unretained(this)), + base::BindOnce(&VideoFrameQualityValidator::InitializeCB, + base::Unretained(this)), base::BindRepeating(&VideoFrameQualityValidator::VerifyOutputFrame, base::Unretained(this)), base::NullCallback()); @@ -2567,8 +2571,8 @@ void VEANoInputClient::RequireBitstreamBuffers( // Timer is used to make sure there is no output frame in 100ms. timer_.reset(new base::OneShotTimer()); timer_->Start(FROM_HERE, base::TimeDelta::FromMilliseconds(100), - base::Bind(&VEANoInputClient::SetState, base::Unretained(this), - CS_FINISHED)); + base::BindOnce(&VEANoInputClient::SetState, + base::Unretained(this), CS_FINISHED)); } void VEANoInputClient::BitstreamBufferReady( diff --git a/chromium/media/gpu/vp8_decoder_unittest.cc b/chromium/media/gpu/vp8_decoder_unittest.cc index 8598f483c50..c7e9b289522 100644 --- a/chromium/media/gpu/vp8_decoder_unittest.cc +++ b/chromium/media/gpu/vp8_decoder_unittest.cc @@ -8,7 +8,6 @@ #include "base/command_line.h" #include "base/files/file_util.h" -#include "base/logging.h" #include "media/base/test_data_util.h" #include "media/gpu/vp8_decoder.h" #include "testing/gmock/include/gmock/gmock.h" diff --git a/chromium/media/gpu/windows/d3d11_cdm_proxy.cc b/chromium/media/gpu/windows/d3d11_cdm_proxy.cc deleted file mode 100644 index 1e274aba7ae..00000000000 --- a/chromium/media/gpu/windows/d3d11_cdm_proxy.cc +++ /dev/null @@ -1,658 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// TODO(crbug.com/787657): Handle hardware key reset and notify the client. -#include "media/gpu/windows/d3d11_cdm_proxy.h" - -#include <initguid.h> - -#include "base/bind.h" -#include "base/logging.h" -#include "base/memory/ptr_util.h" -#include "base/power_monitor/power_monitor.h" -#include "base/power_monitor/power_observer.h" -#include "base/stl_util.h" -#include "base/synchronization/waitable_event.h" -#include "base/win/object_watcher.h" -#include "media/base/callback_registry.h" -#include "media/base/cdm_context.h" -#include "media/cdm/cdm_proxy_context.h" -#include "media/gpu/windows/d3d11_decryptor.h" - -namespace media { - -namespace { - -// Checks whether there is a hardware protected key exhange method. -// https://msdn.microsoft.com/en-us/library/windows/desktop/dn894125(v=vs.85).aspx -// The key exhange capabilities are checked using these. -// https://msdn.microsoft.com/en-us/library/windows/desktop/hh447640%28v=vs.85%29.aspx?f=255&MSPPError=-2147217396 -// https://msdn.microsoft.com/en-us/library/windows/desktop/hh447782(v=vs.85).aspx -bool CanDoHardwareProtectedKeyExchange(ComD3D11VideoDevice video_device, - const GUID& crypto_type) { - D3D11_VIDEO_CONTENT_PROTECTION_CAPS caps = {}; - HRESULT hresult = video_device->GetContentProtectionCaps( - &crypto_type, &D3D11_DECODER_PROFILE_H264_VLD_NOFGT, &caps); - if (FAILED(hresult)) { - DVLOG(1) << "Failed to get content protection caps."; - return false; - } - - for (uint32_t i = 0; i < caps.KeyExchangeTypeCount; ++i) { - GUID kex_guid = {}; - hresult = video_device->CheckCryptoKeyExchange( - &crypto_type, &D3D11_DECODER_PROFILE_H264_VLD_NOFGT, i, &kex_guid); - if (FAILED(hresult)) { - DVLOG(1) << "Failed to get key exchange GUID"; - return false; - } - - if (kex_guid == D3D11_KEY_EXCHANGE_HW_PROTECTION) - return true; - } - - DVLOG(1) << "Hardware key exchange is not supported."; - return false; -} - -class D3D11CdmProxyContext : public CdmProxyContext { - public: - explicit D3D11CdmProxyContext(const GUID& key_info_guid) - : key_info_guid_(key_info_guid) {} - ~D3D11CdmProxyContext() override = default; - - // The pointers are owned by the caller. - void SetKey(ID3D11CryptoSession* crypto_session, - const std::vector<uint8_t>& key_id, - CdmProxy::KeyType key_type, - const std::vector<uint8_t>& key_blob) { - std::string key_id_str(key_id.begin(), key_id.end()); - KeyInfo key_info(crypto_session, key_blob); - // Note that this would overwrite an entry but it is completely valid, e.g. - // updating the keyblob due to a configuration change. - key_info_map_[key_id_str][key_type] = std::move(key_info); - } - - void RemoveKey(ID3D11CryptoSession* crypto_session, - const std::vector<uint8_t>& key_id) { - // There's no need for a keytype for Remove() at the moment, because it's - // used for completely removing keys associated to |key_id|. - std::string key_id_str(key_id.begin(), key_id.end()); - key_info_map_.erase(key_id_str); - } - - // Removes all keys from the context. - void RemoveAllKeys() { key_info_map_.clear(); } - - // CdmProxyContext implementation. - base::Optional<D3D11DecryptContext> GetD3D11DecryptContext( - CdmProxy::KeyType key_type, - const std::string& key_id) override { - auto key_id_find_it = key_info_map_.find(key_id); - if (key_id_find_it == key_info_map_.end()) - return base::nullopt; - - auto& key_type_to_key_info = key_id_find_it->second; - auto key_type_find_it = key_type_to_key_info.find(key_type); - if (key_type_find_it == key_type_to_key_info.end()) - return base::nullopt; - - auto& key_info = key_type_find_it->second; - D3D11DecryptContext context = {}; - context.crypto_session = key_info.crypto_session; - context.key_blob = key_info.key_blob.data(); - context.key_blob_size = key_info.key_blob.size(); - context.key_info_guid = key_info_guid_; - return context; - } - - private: - // A structure to keep the data passed to SetKey(). See documentation for - // SetKey() for what the fields mean. - struct KeyInfo { - KeyInfo() = default; - KeyInfo(ID3D11CryptoSession* crypto_session, std::vector<uint8_t> key_blob) - : crypto_session(crypto_session), key_blob(std::move(key_blob)) {} - KeyInfo(const KeyInfo&) = default; - ~KeyInfo() = default; - - ID3D11CryptoSession* crypto_session; - std::vector<uint8_t> key_blob; - }; - - // Maps key ID -> key type -> KeyInfo. - // The key ID's type is string, which is converted from |key_id| in - // SetKey(). It's better to use string here rather than convert - // vector<uint8_t> to string every time in GetD3D11DecryptContext() because - // in most cases it would be called more often than SetKey() and RemoveKey() - // combined. - std::map<std::string, std::map<CdmProxy::KeyType, KeyInfo>> key_info_map_; - - const GUID key_info_guid_; - - DISALLOW_COPY_AND_ASSIGN(D3D11CdmProxyContext); -}; - -} // namespace - -// Watches for any content protection teardown events. -// If the instance has been started for watching, the destructor will -// automatically stop watching. -class D3D11CdmProxy::HardwareEventWatcher - : public base::win::ObjectWatcher::Delegate, - public base::PowerObserver { - public: - ~HardwareEventWatcher() override; - - // |teardown_callback| is called on the current sequence. - // Returns an instance if it starts watching for events, otherwise returns - // nullptr. - static std::unique_ptr<HardwareEventWatcher> Create( - ComD3D11Device device, - base::RepeatingClosure teardown_callback); - - private: - HardwareEventWatcher(ComD3D11Device device, - base::RepeatingClosure teardown_callback); - - // Start watching for events. - bool StartWatching(); - - // Registers for hardware content protection teardown events. - // Return true on success. - bool RegisterHardwareContentProtectionTeardown(ComD3D11Device device); - - // Regiesters for power events, specifically power resume event. - // Returns true on success. - bool RegisterPowerEvents(); - - // base::win::ObjectWatcher::Delegate implementation. - void OnObjectSignaled(HANDLE object) override; - - // base::PowerObserver implementation. Other power events are not relevant to - // this class. - void OnResume() override; - - // Stops watching for events. Good for clean up. - void StopWatching(); - - // IDXGIAdapter3::RegisterHardwareContentProtectionTeardownStatusEvent - // allows watching for teardown events. It is queried thru the following - // Devices. - ComD3D11Device device_; - ComDXGIDevice2 dxgi_device_; - ComDXGIAdapter3 dxgi_adapter_; - - // Cookie, event, and watcher used for watching events from - // RegisterHardwareContentProtectionTeardownStatusEvent. - DWORD teardown_event_cookie_ = 0u; - base::WaitableEvent content_protection_teardown_event_; - base::RepeatingClosure teardown_callback_; - base::win::ObjectWatcher teardown_status_watcher_; -}; - -class D3D11CdmContext : public CdmContext { - public: - explicit D3D11CdmContext(const GUID& key_info_guid) - : cdm_proxy_context_(key_info_guid) {} - ~D3D11CdmContext() override = default; - - // The pointers are owned by the caller. - void SetKey(ID3D11CryptoSession* crypto_session, - const std::vector<uint8_t>& key_id, - CdmProxy::KeyType key_type, - const std::vector<uint8_t>& key_blob) { - cdm_proxy_context_.SetKey(crypto_session, key_id, key_type, key_blob); - event_callbacks_.Notify(Event::kHasAdditionalUsableKey); - } - void RemoveKey(ID3D11CryptoSession* crypto_session, - const std::vector<uint8_t>& key_id) { - cdm_proxy_context_.RemoveKey(crypto_session, key_id); - } - - // Notifies of hardware reset. - void OnHardwareReset() { - cdm_proxy_context_.RemoveAllKeys(); - event_callbacks_.Notify(Event::kHardwareContextLost); - } - - base::WeakPtr<D3D11CdmContext> GetWeakPtr() { - return weak_factory_.GetWeakPtr(); - } - - // CdmContext implementation. - std::unique_ptr<CallbackRegistration> RegisterEventCB( - EventCB event_cb) override { - return event_callbacks_.Register(std::move(event_cb)); - } - CdmProxyContext* GetCdmProxyContext() override { return &cdm_proxy_context_; } - - Decryptor* GetDecryptor() override { - if (!decryptor_) - decryptor_.reset(new D3D11Decryptor(&cdm_proxy_context_)); - - return decryptor_.get(); - } - - private: - D3D11CdmProxyContext cdm_proxy_context_; - - std::unique_ptr<D3D11Decryptor> decryptor_; - - CallbackRegistry<EventCB::RunType> event_callbacks_; - - base::WeakPtrFactory<D3D11CdmContext> weak_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(D3D11CdmContext); -}; - -D3D11CdmProxy::D3D11CdmProxy(const GUID& crypto_type, - CdmProxy::Protocol protocol, - const FunctionIdMap& function_id_map) - : crypto_type_(crypto_type), - protocol_(protocol), - function_id_map_(function_id_map), - cdm_context_(std::make_unique<D3D11CdmContext>(crypto_type)), - create_device_func_(base::BindRepeating(D3D11CreateDevice)) {} - -D3D11CdmProxy::~D3D11CdmProxy() {} - -base::WeakPtr<CdmContext> D3D11CdmProxy::GetCdmContext() { - return cdm_context_->GetWeakPtr(); -} - -void D3D11CdmProxy::Initialize(Client* client, InitializeCB init_cb) { - DCHECK(client); - - auto failed = [this, &init_cb]() { - // The value doesn't matter as it shouldn't be used on a failure. - const uint32_t kFailedCryptoSessionId = 0xFF; - std::move(init_cb).Run(Status::kFail, protocol_, kFailedCryptoSessionId); - }; - - if (initialized_) { - failed(); - NOTREACHED() << "CdmProxy should not be initialized more than once."; - return; - } - - client_ = client; - - const D3D_FEATURE_LEVEL feature_levels[] = {D3D_FEATURE_LEVEL_11_1}; - - HRESULT hresult = create_device_func_.Run( - nullptr, // No adapter. - D3D_DRIVER_TYPE_HARDWARE, nullptr, // No software rasterizer. - 0, // flags, none. - feature_levels, base::size(feature_levels), D3D11_SDK_VERSION, &device_, - nullptr, &device_context_); - if (FAILED(hresult)) { - DLOG(ERROR) << "Failed to create the D3D11Device:" << hresult; - failed(); - return; - } - - // TODO(rkuroiwa): This should be registered iff - // D3D11_CONTENT_PROTECTION_CAPS_HARDWARE_TEARDOWN is set in the capabilities. - hardware_event_watcher_ = HardwareEventWatcher::Create( - device_, base::BindRepeating( - &D3D11CdmProxy::NotifyHardwareContentProtectionTeardown, - weak_factory_.GetWeakPtr())); - if (!hardware_event_watcher_) { - DLOG(ERROR) - << "Failed to start waching for content protection teardown events."; - failed(); - return; - } - - hresult = device_.As(&video_device_); - if (FAILED(hresult)) { - DLOG(ERROR) << "Failed to get ID3D11VideoDevice: " << hresult; - failed(); - return; - } - - if (!CanDoHardwareProtectedKeyExchange(video_device_, crypto_type_)) { - DLOG(ERROR) << "Cannot do hardware protected key exchange."; - failed(); - return; - } - - hresult = device_context_.As(&video_context_); - if (FAILED(hresult)) { - DLOG(ERROR) << "Failed to get ID3D11VideoContext: " << hresult; - failed(); - return; - } - - hresult = device_.As(&video_device1_); - if (FAILED(hresult)) { - DLOG(ERROR) << "Failed to get ID3D11VideoDevice1: " << hresult; - failed(); - return; - } - - hresult = device_context_.As(&video_context1_); - if (FAILED(hresult)) { - DLOG(ERROR) << "Failed to get ID3D11VideoContext1: " << hresult; - failed(); - return; - } - - ComD3D11CryptoSession csme_crypto_session; - hresult = video_device_->CreateCryptoSession( - &crypto_type_, &D3D11_DECODER_PROFILE_H264_VLD_NOFGT, - &D3D11_KEY_EXCHANGE_HW_PROTECTION, &csme_crypto_session); - if (FAILED(hresult)) { - DLOG(ERROR) << "Failed to Create CryptoSession: " << hresult; - failed(); - return; - } - - hresult = video_device1_->GetCryptoSessionPrivateDataSize( - &crypto_type_, &D3D11_DECODER_PROFILE_H264_VLD_NOFGT, - &D3D11_KEY_EXCHANGE_HW_PROTECTION, &private_input_size_, - &private_output_size_); - if (FAILED(hresult)) { - DLOG(ERROR) << "Failed to get private data sizes: " << hresult; - failed(); - return; - } - - const uint32_t crypto_session_id = next_crypto_session_id_++; - crypto_session_map_[crypto_session_id] = std::move(csme_crypto_session); - initialized_ = true; - std::move(init_cb).Run(Status::kOk, protocol_, crypto_session_id); -} - -void D3D11CdmProxy::Process(Function function, - uint32_t crypto_session_id, - const std::vector<uint8_t>& input_data_vec, - uint32_t expected_output_data_size, - ProcessCB process_cb) { - auto failed = [&process_cb]() { - std::move(process_cb).Run(Status::kFail, std::vector<uint8_t>()); - }; - - if (!initialized_) { - DLOG(ERROR) << "Not initialied."; - failed(); - return; - } - - auto function_id_it = function_id_map_.find(function); - if (function_id_it == function_id_map_.end()) { - DLOG(ERROR) << "Unrecognized function: " << static_cast<int>(function); - failed(); - return; - } - - auto crypto_session_it = crypto_session_map_.find(crypto_session_id); - if (crypto_session_it == crypto_session_map_.end()) { - DLOG(ERROR) << "Cannot find crypto session with ID " << crypto_session_id; - failed(); - return; - } - - ComD3D11CryptoSession& crypto_session = crypto_session_it->second; - - D3D11_KEY_EXCHANGE_HW_PROTECTION_DATA key_exchange_data = {}; - key_exchange_data.HWProtectionFunctionID = function_id_it->second; - - // Because D3D11_KEY_EXCHANGE_HW_PROTECTION_INPUT_DATA and - // D3D11_KEY_EXCHANGE_HW_PROTECTION_OUTPUT_DATA are variable size structures, - // uint8 array are allocated and casted to each type. - // -4 for the "BYTE pbInput[4]" field. - std::unique_ptr<uint8_t[]> input_data_raw( - new uint8_t[sizeof(D3D11_KEY_EXCHANGE_HW_PROTECTION_INPUT_DATA) - 4 + - input_data_vec.size()]); - std::unique_ptr<uint8_t[]> output_data_raw( - new uint8_t[sizeof(D3D11_KEY_EXCHANGE_HW_PROTECTION_OUTPUT_DATA) - 4 + - expected_output_data_size]); - - D3D11_KEY_EXCHANGE_HW_PROTECTION_INPUT_DATA* input_data = - reinterpret_cast<D3D11_KEY_EXCHANGE_HW_PROTECTION_INPUT_DATA*>( - input_data_raw.get()); - D3D11_KEY_EXCHANGE_HW_PROTECTION_OUTPUT_DATA* output_data = - reinterpret_cast<D3D11_KEY_EXCHANGE_HW_PROTECTION_OUTPUT_DATA*>( - output_data_raw.get()); - - key_exchange_data.pInputData = input_data; - key_exchange_data.pOutputData = output_data; - input_data->PrivateDataSize = private_input_size_; - input_data->HWProtectionDataSize = 0; - memcpy(input_data->pbInput, input_data_vec.data(), input_data_vec.size()); - - output_data->PrivateDataSize = private_output_size_; - output_data->HWProtectionDataSize = 0; - output_data->TransportTime = 0; - output_data->ExecutionTime = 0; - output_data->MaxHWProtectionDataSize = expected_output_data_size; - - HRESULT hresult = video_context_->NegotiateCryptoSessionKeyExchange( - crypto_session.Get(), sizeof(key_exchange_data), &key_exchange_data); - if (FAILED(hresult)) { - failed(); - return; - } - - std::move(process_cb) - .Run(Status::kOk, std::vector<uint8_t>( - output_data->pbOutput, - output_data->pbOutput + expected_output_data_size)); - return; -} - -void D3D11CdmProxy::CreateMediaCryptoSession( - const std::vector<uint8_t>& input_data, - CreateMediaCryptoSessionCB create_media_crypto_session_cb) { - auto failed = [&create_media_crypto_session_cb]() { - const uint32_t kInvalidSessionId = 0; - const uint64_t kNoOutputData = 0; - std::move(create_media_crypto_session_cb) - .Run(Status::kFail, kInvalidSessionId, kNoOutputData); - }; - if (!initialized_) { - DLOG(ERROR) << "Not initialized."; - failed(); - return; - } - - ComD3D11CryptoSession media_crypto_session; - HRESULT hresult = video_device_->CreateCryptoSession( - &crypto_type_, &D3D11_DECODER_PROFILE_H264_VLD_NOFGT, &crypto_type_, - &media_crypto_session); - if (FAILED(hresult)) { - DLOG(ERROR) << "Failed to create a crypto session: " << hresult; - failed(); - return; - } - - // Don't do CheckCryptoSessionStatus() yet. The status may be something like - // CONTEXT_LOST because GetDataForNewHardwareKey() is not called yet. - uint64_t output_data = 0; - if (!input_data.empty()) { - hresult = video_context1_->GetDataForNewHardwareKey( - media_crypto_session.Get(), input_data.size(), input_data.data(), - &output_data); - if (FAILED(hresult)) { - DLOG(ERROR) << "Failed to establish hardware session: " << hresult; - failed(); - return; - } - } - - D3D11_CRYPTO_SESSION_STATUS crypto_session_status = {}; - hresult = video_context1_->CheckCryptoSessionStatus( - media_crypto_session.Get(), &crypto_session_status); - if (FAILED(hresult) || - crypto_session_status != D3D11_CRYPTO_SESSION_STATUS_OK) { - DLOG(ERROR) << "Crypto session is not OK. Crypto session status " - << crypto_session_status << ". HRESULT " << hresult; - failed(); - return; - } - - const uint32_t media_crypto_session_id = next_crypto_session_id_++; - crypto_session_map_[media_crypto_session_id] = - std::move(media_crypto_session); - std::move(create_media_crypto_session_cb) - .Run(Status::kOk, media_crypto_session_id, output_data); -} - -void D3D11CdmProxy::SetKey(uint32_t crypto_session_id, - const std::vector<uint8_t>& key_id, - KeyType key_type, - const std::vector<uint8_t>& key_blob, - SetKeyCB set_key_cb) { - auto crypto_session_it = crypto_session_map_.find(crypto_session_id); - if (crypto_session_it == crypto_session_map_.end()) { - DLOG(WARNING) << crypto_session_id - << " did not map to a crypto session instance."; - std::move(set_key_cb).Run(Status::kFail); - return; - } - - cdm_context_->SetKey(crypto_session_it->second.Get(), key_id, key_type, - key_blob); - std::move(set_key_cb).Run(Status::kOk); -} - -void D3D11CdmProxy::RemoveKey(uint32_t crypto_session_id, - const std::vector<uint8_t>& key_id, - RemoveKeyCB remove_key_cb) { - auto crypto_session_it = crypto_session_map_.find(crypto_session_id); - if (crypto_session_it == crypto_session_map_.end()) { - DLOG(WARNING) << crypto_session_id - << " did not map to a crypto session instance."; - std::move(remove_key_cb).Run(Status::kFail); - return; - } - - cdm_context_->RemoveKey(crypto_session_it->second.Get(), key_id); - std::move(remove_key_cb).Run(Status::kOk); -} - -void D3D11CdmProxy::SetCreateDeviceCallbackForTesting( - D3D11CreateDeviceCB callback) { - create_device_func_ = std::move(callback); -} - -void D3D11CdmProxy::NotifyHardwareContentProtectionTeardown() { - cdm_context_->OnHardwareReset(); - client_->NotifyHardwareReset(); - Reset(); -} - -void D3D11CdmProxy::Reset() { - client_ = nullptr; - initialized_ = false; - crypto_session_map_.clear(); - device_.Reset(); - device_context_.Reset(); - video_device_.Reset(); - video_device1_.Reset(); - video_context_.Reset(); - video_context1_.Reset(); - // Note that this deregisters hardware reset event watcher. It shouldn't - // notify the clients until this is reinitialized. Also the client is set to - // null in this method. - hardware_event_watcher_ = nullptr; - crypto_session_map_.clear(); - private_input_size_ = 0; - private_output_size_ = 0; -} - -D3D11CdmProxy::HardwareEventWatcher::~HardwareEventWatcher() { - StopWatching(); -} - -std::unique_ptr<D3D11CdmProxy::HardwareEventWatcher> -D3D11CdmProxy::HardwareEventWatcher::Create( - ComD3D11Device device, - base::RepeatingClosure teardown_callback) { - std::unique_ptr<HardwareEventWatcher> event_watcher = base::WrapUnique( - new HardwareEventWatcher(device, std::move(teardown_callback))); - if (!event_watcher->StartWatching()) - return nullptr; - return event_watcher; -} - -D3D11CdmProxy::HardwareEventWatcher::HardwareEventWatcher( - ComD3D11Device device, - base::RepeatingClosure teardown_callback) - : device_(device), teardown_callback_(std::move(teardown_callback)) {} - -bool D3D11CdmProxy::HardwareEventWatcher::StartWatching() { - if (!RegisterPowerEvents() || - !RegisterHardwareContentProtectionTeardown(device_)) { - StopWatching(); - return false; - } - - return true; -} - -bool D3D11CdmProxy::HardwareEventWatcher:: - RegisterHardwareContentProtectionTeardown(ComD3D11Device device) { - device_ = device; - HRESULT hresult = device_.As(&dxgi_device_); - if (FAILED(hresult)) { - DVLOG(1) << "Failed to get dxgi device from device: " - << logging::SystemErrorCodeToString(hresult); - return false; - } - - hresult = dxgi_device_->GetParent(IID_PPV_ARGS(&dxgi_adapter_)); - if (FAILED(hresult)) { - DVLOG(1) << "Failed to get dxgi adapter from dxgi device: " - << logging::SystemErrorCodeToString(hresult); - return false; - } - - if (!teardown_status_watcher_.StartWatchingOnce( - content_protection_teardown_event_.handle(), this)) { - DVLOG(1) << "Failed to watch tear down event."; - return false; - } - - hresult = dxgi_adapter_->RegisterHardwareContentProtectionTeardownStatusEvent( - content_protection_teardown_event_.handle(), &teardown_event_cookie_); - if (FAILED(hresult)) { - DVLOG(1) - << "Failed to register for HardwareContentProtectionTeardownStatus: " - << logging::SystemErrorCodeToString(hresult); - return false; - } - - return true; -} - -bool D3D11CdmProxy::HardwareEventWatcher::RegisterPowerEvents() { - if (!base::PowerMonitor::AddObserver(this)) { - DVLOG(1) << "Power monitor not available."; - return false; - } - return true; -} - -void D3D11CdmProxy::HardwareEventWatcher::OnObjectSignaled(HANDLE object) { - DCHECK_EQ(object, content_protection_teardown_event_.handle()); - teardown_callback_.Run(); -} - -void D3D11CdmProxy::HardwareEventWatcher::OnResume() { - teardown_callback_.Run(); -} - -void D3D11CdmProxy::HardwareEventWatcher::StopWatching() { - if (dxgi_adapter_) { - dxgi_adapter_->UnregisterHardwareContentProtectionTeardownStatus( - teardown_event_cookie_); - } - teardown_status_watcher_.StopWatching(); - base::PowerMonitor::RemoveObserver(this); -} - -} // namespace media diff --git a/chromium/media/gpu/windows/d3d11_cdm_proxy.h b/chromium/media/gpu/windows/d3d11_cdm_proxy.h deleted file mode 100644 index 572ed2eab0b..00000000000 --- a/chromium/media/gpu/windows/d3d11_cdm_proxy.h +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MEDIA_GPU_WINDOWS_D3D11_CDM_PROXY_H_ -#define MEDIA_GPU_WINDOWS_D3D11_CDM_PROXY_H_ - -#include "media/cdm/cdm_proxy.h" - -#include <d3d11_1.h> -#include <dxgi1_4.h> -#include <wrl/client.h> - -#include <map> -#include <vector> - -#include "base/callback.h" -#include "base/memory/weak_ptr.h" -#include "media/base/win/d3d11_create_device_cb.h" -#include "media/gpu/media_gpu_export.h" -#include "media/gpu/windows/d3d11_com_defs.h" - -namespace media { - -class D3D11CdmContext; - -// This is a CdmProxy implementation that uses D3D11. -class MEDIA_GPU_EXPORT D3D11CdmProxy : public CdmProxy { - public: - using FunctionIdMap = std::map<Function, uint32_t>; - - // |crypto_type| is the ID that is used to do crypto session operations. This - // includes creating a crypto session with - // ID3D11VideoDevice::CreateCryptoSession(). This is "a GUID that specifies - // the type of encryption to use". - // https://msdn.microsoft.com/en-us/library/windows/desktop/hh447785(v=vs.85).aspx - // This is also used ot call - // ID3D11VideoDevice1::GetCryptoSessionPrivateDataSize(). It "Indicates the - // crypto type for which the private input and output size is queried." - // https://msdn.microsoft.com/en-us/library/windows/desktop/dn894143(v=vs.85).aspx - // |protocol| determines what protocol this is operating in. This - // value is passed to callbacks that require a protocol enum value. - // |function_id_map| maps Function enum to an integer. - D3D11CdmProxy(const GUID& crypto_type, - CdmProxy::Protocol protocol, - const FunctionIdMap& function_id_map); - ~D3D11CdmProxy() override; - - // CdmProxy implementation. - base::WeakPtr<CdmContext> GetCdmContext() override; - void Initialize(Client* client, InitializeCB init_cb) override; - void Process(Function function, - uint32_t crypto_session_id, - const std::vector<uint8_t>& input_data, - uint32_t expected_output_data_size, - ProcessCB process_cb) override; - void CreateMediaCryptoSession( - const std::vector<uint8_t>& input_data, - CreateMediaCryptoSessionCB create_media_crypto_session_cb) override; - void SetKey(uint32_t crypto_session_id, - const std::vector<uint8_t>& key_id, - KeyType key_type, - const std::vector<uint8_t>& key_blob, - SetKeyCB set_key_cb) override; - void RemoveKey(uint32_t crypto_session_id, - const std::vector<uint8_t>& key_id, - RemoveKeyCB remove_key_cb) override; - - void SetCreateDeviceCallbackForTesting(D3D11CreateDeviceCB callback); - - private: - - class HardwareEventWatcher; - - void NotifyHardwareContentProtectionTeardown(); - - // Reset the state of this instance to be reinitializable. - void Reset(); - - const GUID crypto_type_; - const CdmProxy::Protocol protocol_; - const FunctionIdMap function_id_map_; - - std::unique_ptr<D3D11CdmContext> cdm_context_; - - // Implmenenting this class does not require this to be a callback. But in - // order to inject D3D11CreateDevice() function for testing, this member is - // required. The test will replace this with a function that returns a mock - // devices. - D3D11CreateDeviceCB create_device_func_; - - // Counter for assigning IDs to crypto sessions. - uint32_t next_crypto_session_id_ = 1; - - // Everything from here until weak ptr factory (which must be at the end) - // should be reset in Reset(). - Client* client_ = nullptr; - bool initialized_ = false; - - ComD3D11Device device_; - ComD3D11DeviceContext device_context_; - // TODO(crbug.com/788880): Remove ID3D11VideoDevice and ID3D11VideoContext if - // they are not required. - ComD3D11VideoDevice video_device_; - ComD3D11VideoDevice1 video_device1_; - ComD3D11VideoContext video_context_; - ComD3D11VideoContext1 video_context1_; - - std::unique_ptr<HardwareEventWatcher> hardware_event_watcher_; - - // Crypto session ID -> actual crypto session. - std::map<uint32_t, ComD3D11CryptoSession> crypto_session_map_; - - // The values output from ID3D11VideoDevice1::GetCryptoSessionPrivateDataSize. - // Used when calling NegotiateCryptoSessionKeyExchange. - UINT private_input_size_ = 0; - UINT private_output_size_ = 0; - - base::WeakPtrFactory<D3D11CdmProxy> weak_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(D3D11CdmProxy); -}; - -} // namespace media - -#endif // MEDIA_GPU_WINDOWS_D3D11_CDM_PROXY_H_ diff --git a/chromium/media/gpu/windows/d3d11_cdm_proxy_unittest.cc b/chromium/media/gpu/windows/d3d11_cdm_proxy_unittest.cc deleted file mode 100644 index a711e8c6b18..00000000000 --- a/chromium/media/gpu/windows/d3d11_cdm_proxy_unittest.cc +++ /dev/null @@ -1,896 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/gpu/windows/d3d11_cdm_proxy.h" - -#include <d3d11.h> -#include <d3d11_1.h> -#include <initguid.h> - -#include "base/bind.h" -#include "base/power_monitor/power_monitor.h" -#include "base/power_monitor/power_monitor_source.h" -#include "base/run_loop.h" -#include "base/test/mock_callback.h" -#include "base/test/task_environment.h" -#include "media/base/callback_registry.h" -#include "media/base/win/d3d11_mocks.h" -#include "media/cdm/cdm_proxy_context.h" -#include "testing/gtest/include/gtest/gtest.h" - -using Microsoft::WRL::ComPtr; - -using ::testing::_; -using ::testing::AllOf; -using ::testing::AtLeast; -using ::testing::AtMost; -using ::testing::DoAll; -using ::testing::Invoke; -using ::testing::InvokeWithoutArgs; -using ::testing::Lt; -using ::testing::Mock; -using ::testing::Ne; -using ::testing::NiceMock; -using ::testing::Pointee; -using ::testing::Return; -using ::testing::SaveArg; -using ::testing::SetArgPointee; -using ::testing::WithArgs; - -namespace media { - -namespace { - -// TODO(rkuroiwa): Although inheriting from different classes, there are several -// mock CdmProxy clients already. They all have NotifyHardwareReset(), so share -// a single mock class that inherits from all the CdmProxy client classes. -class MockProxyClient : public CdmProxy::Client { - public: - MOCK_METHOD0(NotifyHardwareReset, void()); -}; - -class MockPowerMonitorSource : public base::PowerMonitorSource { - public: - // Use this method to send a power resume event. - void Resume() { - // Due to how ProcessPowerEvent() works, it has to be suspended first to - // resume. - ProcessPowerEvent(SUSPEND_EVENT); - ProcessPowerEvent(RESUME_EVENT); - } - - MOCK_METHOD0(IsOnBatteryPowerImpl, bool()); -}; - -// The values doesn't matter as long as this is consistently used thruout the -// test. -const CdmProxy::Protocol kTestProtocol = CdmProxy::Protocol::kIntel; -const CdmProxy::Function kTestFunction = - CdmProxy::Function::kIntelNegotiateCryptoSessionKeyExchange; -// TODO(rkuroiwa): Add test cases for KeyType. -const CdmProxy::KeyType kTestKeyType = CdmProxy::KeyType::kDecryptOnly; -const uint32_t kTestFunctionId = 123; -// clang-format off -DEFINE_GUID(CRYPTO_TYPE_GUID, - 0x01020304, 0xffee, 0xefba, - 0x93, 0xaa, 0x47, 0x77, 0x43, 0xb1, 0x22, 0x98); -// clang-format on - -} // namespace - -// Class for mocking the callbacks that get passed to the proxy methods. -class CallbackMock { - public: - MOCK_METHOD3(InitializeCallback, CdmProxy::InitializeCB::RunType); - MOCK_METHOD2(ProcessCallback, CdmProxy::ProcessCB::RunType); - MOCK_METHOD3(CreateMediaCryptoSessionCallback, - CdmProxy::CreateMediaCryptoSessionCB::RunType); - MOCK_METHOD1(SetKeyCallback, CdmProxy::SetKeyCB::RunType); - MOCK_METHOD1(RemoveKeyCallback, CdmProxy::RemoveKeyCB::RunType); -}; - -class D3D11CdmProxyTest : public ::testing::Test { - protected: - void SetUp() override { - std::map<CdmProxy::Function, uint32_t> function_id_map; - function_id_map[kTestFunction] = kTestFunctionId; - - // Use NiceMock because we don't care about base::PowerMonitorSource events - // other than calling Resume() directly. - auto mock_power_monitor_source = - std::make_unique<NiceMock<MockPowerMonitorSource>>(); - mock_power_monitor_source_ = mock_power_monitor_source.get(); - base::PowerMonitor::Initialize(std::move(mock_power_monitor_source)); - - proxy_ = std::make_unique<D3D11CdmProxy>(CRYPTO_TYPE_GUID, kTestProtocol, - function_id_map); - - device_mock_ = CreateD3D11Mock<D3D11DeviceMock>(); - video_device_mock_ = CreateD3D11Mock<D3D11VideoDeviceMock>(); - video_device1_mock_ = CreateD3D11Mock<D3D11VideoDevice1Mock>(); - crypto_session_mock_ = CreateD3D11Mock<D3D11CryptoSessionMock>(); - device_context_mock_ = CreateD3D11Mock<D3D11DeviceContextMock>(); - video_context_mock_ = CreateD3D11Mock<D3D11VideoContextMock>(); - video_context1_mock_ = CreateD3D11Mock<D3D11VideoContext1Mock>(); - dxgi_device_ = CreateD3D11Mock<DXGIDevice2Mock>(); - dxgi_adapter_ = CreateD3D11Mock<NiceMock<DXGIAdapter3Mock>>(); - - // These flags are a reasonable subset of flags to get HARDWARE protected - // playback. - content_protection_caps_.Caps = - D3D11_CONTENT_PROTECTION_CAPS_HARDWARE | - D3D11_CONTENT_PROTECTION_CAPS_HARDWARE_PROTECT_UNCOMPRESSED | - D3D11_CONTENT_PROTECTION_CAPS_HARDWARE_PROTECTED_MEMORY_PAGEABLE | - D3D11_CONTENT_PROTECTION_CAPS_HARDWARE_TEARDOWN | - D3D11_CONTENT_PROTECTION_CAPS_HARDWARE_DRM_COMMUNICATION; - // 1 for the mock behavior below for CheckCryptoKeyExchange(). - content_protection_caps_.KeyExchangeTypeCount = 1; - // This is arbitrary but 1 is reasonable, meaning doesn't need to be - // aligned. - content_protection_caps_.BlockAlignmentSize = 1; - // This value is arbitrary. - content_protection_caps_.ProtectedMemorySize = 10000000; - - OnCallsForInitialize(); - - proxy_->SetCreateDeviceCallbackForTesting( - base::BindRepeating(&D3D11CreateDeviceMock::Create, - base::Unretained(&create_device_mock_))); - } - - void TearDown() override { - proxy_.reset(); - base::PowerMonitor::ShutdownForTesting(); - } - - // Sets up ON_CALLs for the mock objects. These can be overriden with - // EXPECT_CALLs. - // |content_protection_caps_| should be set. - void OnCallsForInitialize() { - ON_CALL(create_device_mock_, - Create(_, D3D_DRIVER_TYPE_HARDWARE, _, _, _, _, _, _, _, _)) - .WillByDefault( - DoAll(SetComPointee<7>(device_mock_.Get()), - SetComPointeeAndReturnOk<9>(device_context_mock_.Get()))); - - COM_ON_CALL(device_mock_, QueryInterface(IID_ID3D11VideoDevice, _)) - .WillByDefault(SetComPointeeAndReturnOk<1>(video_device_mock_.Get())); - - COM_ON_CALL(device_mock_, QueryInterface(IID_ID3D11VideoDevice1, _)) - .WillByDefault(SetComPointeeAndReturnOk<1>(video_device1_mock_.Get())); - - COM_ON_CALL(device_mock_, QueryInterface(IID_IDXGIDevice2, _)) - .WillByDefault(SetComPointeeAndReturnOk<1>(dxgi_device_.Get())); - - COM_ON_CALL(dxgi_device_, GetParent(IID_IDXGIAdapter3, _)) - .WillByDefault(SetComPointeeAndReturnOk<1>(dxgi_adapter_.Get())); - - COM_ON_CALL(dxgi_adapter_, - RegisterHardwareContentProtectionTeardownStatusEvent(_, _)) - .WillByDefault(DoAll(SaveArg<0>(&teardown_event_), Return(S_OK))); - - COM_ON_CALL(device_context_mock_, QueryInterface(IID_ID3D11VideoContext, _)) - .WillByDefault(SetComPointeeAndReturnOk<1>(video_context_mock_.Get())); - - COM_ON_CALL(device_context_mock_, - QueryInterface(IID_ID3D11VideoContext1, _)) - .WillByDefault(SetComPointeeAndReturnOk<1>(video_context1_mock_.Get())); - - COM_ON_CALL( - video_device_mock_, - CreateCryptoSession(Pointee(CRYPTO_TYPE_GUID), _, - Pointee(D3D11_KEY_EXCHANGE_HW_PROTECTION), _)) - .WillByDefault(SetComPointeeAndReturnOk<3>(crypto_session_mock_.Get())); - - COM_ON_CALL(video_device1_mock_, GetCryptoSessionPrivateDataSize( - Pointee(CRYPTO_TYPE_GUID), _, _, _, _)) - .WillByDefault(DoAll(SetArgPointee<3>(kPrivateInputSize), - SetArgPointee<4>(kPrivateOutputSize), - Return(S_OK))); - - COM_ON_CALL(video_device_mock_, GetContentProtectionCaps(_, _, _)) - .WillByDefault( - DoAll(SetArgPointee<2>(content_protection_caps_), Return(S_OK))); - - COM_ON_CALL(video_device_mock_, CheckCryptoKeyExchange(_, _, Lt(1u), _)) - .WillByDefault(DoAll(SetArgPointee<3>(D3D11_KEY_EXCHANGE_HW_PROTECTION), - Return(S_OK))); - } - - // Helper method to do Initialize(). The returned mock objects are accessible - // thru member variables. - void Initialize(CdmProxy::Client* client, CdmProxy::InitializeCB callback) { - EXPECT_CALL(create_device_mock_, - Create(_, D3D_DRIVER_TYPE_HARDWARE, _, _, _, _, _, _, _, _)); - COM_EXPECT_CALL(device_mock_, QueryInterface(IID_ID3D11VideoDevice, _)) - .Times(AtLeast(1)); - COM_EXPECT_CALL(device_mock_, QueryInterface(IID_IDXGIDevice2, _)) - .Times(AtLeast(1)); - COM_EXPECT_CALL(dxgi_device_, GetParent(IID_IDXGIAdapter3, _)) - .Times(AtLeast(1)); - COM_EXPECT_CALL(dxgi_adapter_, - RegisterHardwareContentProtectionTeardownStatusEvent(_, _)) - .Times(AtLeast(1)); - COM_EXPECT_CALL(device_mock_, QueryInterface(IID_ID3D11VideoDevice1, _)) - .Times(AtLeast(1)); - COM_EXPECT_CALL(device_context_mock_, - QueryInterface(IID_ID3D11VideoContext, _)) - .Times(AtLeast(1)); - COM_EXPECT_CALL(device_context_mock_, - QueryInterface(IID_ID3D11VideoContext1, _)) - .Times(AtLeast(1)); - COM_EXPECT_CALL( - video_device_mock_, - CreateCryptoSession(Pointee(CRYPTO_TYPE_GUID), _, - Pointee(D3D11_KEY_EXCHANGE_HW_PROTECTION), _)); - COM_EXPECT_CALL( - video_device1_mock_, - GetCryptoSessionPrivateDataSize(Pointee(CRYPTO_TYPE_GUID), _, _, _, _)); - - COM_EXPECT_CALL(video_device_mock_, GetContentProtectionCaps(_, _, _)); - - COM_EXPECT_CALL(video_device_mock_, - CheckCryptoKeyExchange(_, _, Lt(1u), _)); - - proxy_->Initialize(client, std::move(callback)); - - Mock::VerifyAndClearExpectations(device_mock_.Get()); - Mock::VerifyAndClearExpectations(video_device_mock_.Get()); - Mock::VerifyAndClearExpectations(video_device1_mock_.Get()); - Mock::VerifyAndClearExpectations(crypto_session_mock_.Get()); - Mock::VerifyAndClearExpectations(device_context_mock_.Get()); - Mock::VerifyAndClearExpectations(video_context_mock_.Get()); - Mock::VerifyAndClearExpectations(video_context1_mock_.Get()); - } - - // Test case where the proxy is initialized and then hardware content - // protection teardown is notified. - void HardwareContentProtectionTeardown() { - base::RunLoop run_loop; - - EXPECT_CALL(callback_mock_, - InitializeCallback(CdmProxy::Status::kOk, _, _)); - ASSERT_NO_FATAL_FAILURE(Initialize( - &client_, base::BindOnce(&CallbackMock::InitializeCallback, - base::Unretained(&callback_mock_)))); - - EXPECT_CALL(client_, NotifyHardwareReset()); - - base::MockCallback<CdmContext::EventCB> event_cb; - auto callback_registration = - proxy_->GetCdmContext()->RegisterEventCB(event_cb.Get()); - EXPECT_CALL(event_cb, Run(CdmContext::Event::kHardwareContextLost)) - .WillOnce(InvokeWithoutArgs([&run_loop]() { run_loop.Quit(); })); - - SetEvent(teardown_event_); - run_loop.Run(); - } - - MockProxyClient client_; - std::unique_ptr<D3D11CdmProxy> proxy_; - // Owned by PowerMonitor. Use this to simulate a power-resume. - MockPowerMonitorSource* mock_power_monitor_source_; - - D3D11CreateDeviceMock create_device_mock_; - CallbackMock callback_mock_; - - ComPtr<D3D11DeviceMock> device_mock_; - ComPtr<D3D11VideoDeviceMock> video_device_mock_; - ComPtr<D3D11VideoDevice1Mock> video_device1_mock_; - ComPtr<D3D11CryptoSessionMock> crypto_session_mock_; - ComPtr<D3D11DeviceContextMock> device_context_mock_; - ComPtr<D3D11VideoContextMock> video_context_mock_; - ComPtr<D3D11VideoContext1Mock> video_context1_mock_; - ComPtr<DXGIDevice2Mock> dxgi_device_; - ComPtr<NiceMock<DXGIAdapter3Mock>> dxgi_adapter_; - - D3D11_VIDEO_CONTENT_PROTECTION_CAPS content_protection_caps_ = {}; - - // Event captured in Initialize(). Used in tests to notify hardware content - // protection teardown. - HANDLE teardown_event_; - - // These size values are arbitrary. Used for mocking - // GetCryptoSessionPrivateDataSize(). - const UINT kPrivateInputSize = 10; - const UINT kPrivateOutputSize = 40; - - // ObjectWatcher uses SequencedTaskRunnerHandle. - base::test::TaskEnvironment task_environment_; -}; - -// Verifies that if device creation fails, then the call fails. -TEST_F(D3D11CdmProxyTest, FailedToCreateDevice) { - EXPECT_CALL(create_device_mock_, Create(_, _, _, _, _, _, _, _, _, _)) - .WillOnce(Return(E_FAIL)); - EXPECT_CALL(callback_mock_, - InitializeCallback(CdmProxy::Status::kFail, _, _)); - proxy_->Initialize(&client_, - base::BindOnce(&CallbackMock::InitializeCallback, - base::Unretained(&callback_mock_))); -} - -// Initialize() success case. -TEST_F(D3D11CdmProxyTest, Initialize) { - EXPECT_CALL(callback_mock_, InitializeCallback(CdmProxy::Status::kOk, _, _)); - ASSERT_NO_FATAL_FAILURE( - Initialize(&client_, base::BindOnce(&CallbackMock::InitializeCallback, - base::Unretained(&callback_mock_)))); -} - -// Hardware content protection teardown is notified to the proxy. -// Verify that the client is notified. -TEST_F(D3D11CdmProxyTest, HardwareContentProtectionTeardown) { - EXPECT_NO_FATAL_FAILURE(HardwareContentProtectionTeardown()); -} - -// Verify that initialization after hardware content protection teardown works.. -TEST_F(D3D11CdmProxyTest, HardwareContentProtectionTeardownThenInitialize) { - ASSERT_NO_FATAL_FAILURE(HardwareContentProtectionTeardown()); - EXPECT_CALL(callback_mock_, InitializeCallback(CdmProxy::Status::kOk, _, _)); - ASSERT_NO_FATAL_FAILURE( - Initialize(&client_, base::BindOnce(&CallbackMock::InitializeCallback, - base::Unretained(&callback_mock_)))); -} - -// Verify that failing to register to hardware content protection teardown -// status event results in initialization failure. -TEST_F(D3D11CdmProxyTest, FailedToRegisterForContentProtectionTeardown) { - EXPECT_CALL(callback_mock_, - InitializeCallback(CdmProxy::Status::kFail, _, _)); - - COM_EXPECT_CALL(dxgi_adapter_, - RegisterHardwareContentProtectionTeardownStatusEvent(_, _)) - .Times(AtLeast(1)) - .WillRepeatedly(Return(E_FAIL)); - - proxy_->Initialize(&client_, - base::BindOnce(&CallbackMock::InitializeCallback, - base::Unretained(&callback_mock_))); -} - -// Verify that the client is notified on power suspend. -TEST_F(D3D11CdmProxyTest, PowerResume) { - base::RunLoop run_loop; - - EXPECT_CALL(callback_mock_, InitializeCallback(CdmProxy::Status::kOk, _, _)); - ASSERT_NO_FATAL_FAILURE( - Initialize(&client_, base::BindOnce(&CallbackMock::InitializeCallback, - base::Unretained(&callback_mock_)))); - - EXPECT_CALL(client_, NotifyHardwareReset()).WillOnce(Invoke([&run_loop]() { - run_loop.Quit(); - })); - - mock_power_monitor_source_->Resume(); - run_loop.Run(); -} - -// IRL power resume is notified and then hardware content protection teardown -// is notified. Make sure that the two notifications don't signal the clients -// more than once (without being reinitialized in between the notifications). -// Note that this test uses QuitWhenIdle(). If both notifications are processed -// this test will run forever. -TEST_F(D3D11CdmProxyTest, PowerResumeAndHardwareContentProtectionTeardown) { - base::RunLoop run_loop; - - EXPECT_CALL(callback_mock_, InitializeCallback(CdmProxy::Status::kOk, _, _)); - ASSERT_NO_FATAL_FAILURE( - Initialize(&client_, base::BindOnce(&CallbackMock::InitializeCallback, - base::Unretained(&callback_mock_)))); - - EXPECT_CALL(client_, NotifyHardwareReset()) - .Times(1) - .WillOnce(Invoke([&run_loop]() { run_loop.QuitWhenIdle(); })); - - mock_power_monitor_source_->Resume(); - SetEvent(teardown_event_); - run_loop.Run(); -} - -// Verify that if there isn't a power monitor, initialization fails. -TEST_F(D3D11CdmProxyTest, NoPowerMonitor) { - base::PowerMonitor::ShutdownForTesting(); - EXPECT_CALL(callback_mock_, - InitializeCallback(CdmProxy::Status::kFail, _, _)); - - proxy_->Initialize(&client_, - base::BindOnce(&CallbackMock::InitializeCallback, - base::Unretained(&callback_mock_))); -} - -// Initialization failure because HW key exchange is not available. -TEST_F(D3D11CdmProxyTest, NoHwKeyExchange) { - EXPECT_CALL(callback_mock_, - InitializeCallback(CdmProxy::Status::kFail, _, _)); - // GUID is set to non-D3D11_KEY_EXCHANGE_HW_PROTECTION, which means no HW key - // exchange. - COM_EXPECT_CALL(video_device_mock_, CheckCryptoKeyExchange(_, _, Lt(1u), _)) - .WillOnce( - DoAll(SetArgPointee<3>(D3D11_CRYPTO_TYPE_AES128_CTR), Return(S_OK))); - - proxy_->Initialize(&client_, - base::BindOnce(&CallbackMock::InitializeCallback, - base::Unretained(&callback_mock_))); -} - -// Verifies that Process() won't work if not initialized. -TEST_F(D3D11CdmProxyTest, ProcessUninitialized) { - // The size nor value here matter, so making non empty non zero vector. - const std::vector<uint8_t> kAnyInput(16, 0xFF); - // Output size is also arbitrary, just has to match with the mock. - const uint32_t kExpectedOutputDataSize = 20; - EXPECT_CALL(callback_mock_, ProcessCallback(CdmProxy::Status::kFail, _)); - proxy_->Process(kTestFunction, 0, kAnyInput, kExpectedOutputDataSize, - base::BindOnce(&CallbackMock::ProcessCallback, - base::Unretained(&callback_mock_))); -} - -// Verifies that using a crypto session that is not reported will fail. -TEST_F(D3D11CdmProxyTest, ProcessInvalidCryptoSessionID) { - uint32_t crypto_session_id = 0; - EXPECT_CALL(callback_mock_, InitializeCallback(CdmProxy::Status::kOk, _, _)) - .WillOnce(SaveArg<2>(&crypto_session_id)); - ASSERT_NO_FATAL_FAILURE( - Initialize(&client_, base::BindOnce(&CallbackMock::InitializeCallback, - base::Unretained(&callback_mock_)))); - Mock::VerifyAndClearExpectations(&callback_mock_); - - // The size nor value here matter, so making non empty non zero vector. - const std::vector<uint8_t> kAnyInput(16, 0xFF); - // Output size is also arbitrary, just has to match with the mock. - const uint32_t kExpectedOutputDataSize = 20; - EXPECT_CALL(callback_mock_, ProcessCallback(CdmProxy::Status::kFail, _)); - - // Use a crypto session ID that hasn't been reported. - proxy_->Process(kTestFunction, crypto_session_id + 1, kAnyInput, - kExpectedOutputDataSize, - base::BindOnce(&CallbackMock::ProcessCallback, - base::Unretained(&callback_mock_))); -} - -// Matcher for checking whether the structure passed to -// NegotiateCryptoSessionKeyExchange has the expected values. -MATCHER_P2(MatchesKeyExchangeStructure, expected, input_struct_size, "") { - D3D11_KEY_EXCHANGE_HW_PROTECTION_DATA* actual = - static_cast<D3D11_KEY_EXCHANGE_HW_PROTECTION_DATA*>(arg); - if (expected->HWProtectionFunctionID != actual->HWProtectionFunctionID) { - *result_listener << "function IDs mismatch. Expected " - << expected->HWProtectionFunctionID << " actual " - << actual->HWProtectionFunctionID; - return false; - } - D3D11_KEY_EXCHANGE_HW_PROTECTION_INPUT_DATA* expected_input_data = - expected->pInputData; - D3D11_KEY_EXCHANGE_HW_PROTECTION_INPUT_DATA* actual_input_data = - actual->pInputData; - if (memcmp(expected_input_data, actual_input_data, input_struct_size) != 0) { - *result_listener - << "D3D11_KEY_EXCHANGE_HW_PROTECTION_INPUT_DATA don't match."; - return false; - } - D3D11_KEY_EXCHANGE_HW_PROTECTION_OUTPUT_DATA* expected_output_data = - expected->pOutputData; - D3D11_KEY_EXCHANGE_HW_PROTECTION_OUTPUT_DATA* actual_output_data = - actual->pOutputData; - // Don't check that pbOutput field. It's filled by the callee. - if (expected_output_data->PrivateDataSize != - actual_output_data->PrivateDataSize) { - *result_listener << "D3D11_KEY_EXCHANGE_HW_PROTECTION_OUTPUT_DATA::" - "PrivateDataSize don't match. Expected " - << expected_output_data->PrivateDataSize << " actual " - << actual_output_data->PrivateDataSize; - return false; - } - if (expected_output_data->HWProtectionDataSize != - actual_output_data->HWProtectionDataSize) { - *result_listener << "D3D11_KEY_EXCHANGE_HW_PROTECTION_OUTPUT_DATA::" - "HWProtectionDataSize don't match. Expected " - << expected_output_data->HWProtectionDataSize << " actual " - << actual_output_data->HWProtectionDataSize; - return false; - } - if (expected_output_data->TransportTime != - actual_output_data->TransportTime) { - *result_listener << "D3D11_KEY_EXCHANGE_HW_PROTECTION_OUTPUT_DATA::" - "TransportTime don't match. Expected " - << expected_output_data->TransportTime << " actual " - << actual_output_data->TransportTime; - return false; - } - if (expected_output_data->ExecutionTime != - actual_output_data->ExecutionTime) { - *result_listener << "D3D11_KEY_EXCHANGE_HW_PROTECTION_OUTPUT_DATA::" - "ExecutionTime don't match. Expected " - << expected_output_data->ExecutionTime << " actual " - << actual_output_data->ExecutionTime; - return false; - } - if (expected_output_data->MaxHWProtectionDataSize != - actual_output_data->MaxHWProtectionDataSize) { - *result_listener << "D3D11_KEY_EXCHANGE_HW_PROTECTION_OUTPUT_DATA::" - "MaxHWProtectionDataSize don't match. Expected " - << expected_output_data->MaxHWProtectionDataSize - << " actual " - << actual_output_data->MaxHWProtectionDataSize; - return false; - } - return true; -} - -// Verifies that Process() works. -TEST_F(D3D11CdmProxyTest, Process) { - uint32_t crypto_session_id = 0; - EXPECT_CALL(callback_mock_, - InitializeCallback(CdmProxy::Status::kOk, kTestProtocol, _)) - .WillOnce(SaveArg<2>(&crypto_session_id)); - ASSERT_NO_FATAL_FAILURE( - Initialize(&client_, base::BindOnce(&CallbackMock::InitializeCallback, - base::Unretained(&callback_mock_)))); - Mock::VerifyAndClearExpectations(&callback_mock_); - - // The size nor value here matter, so making non empty non zero vector. - const std::vector<uint8_t> kAnyInput(16, 0xFF); - // Output size is also arbitrary, just has to match with the mock. - const uint32_t kExpectedOutputDataSize = 20; - - const uint32_t input_structure_size = - sizeof(D3D11_KEY_EXCHANGE_HW_PROTECTION_INPUT_DATA) - 4 + - kAnyInput.size(); - const uint32_t output_structure_size = - sizeof(D3D11_KEY_EXCHANGE_HW_PROTECTION_OUTPUT_DATA) - 4 + - kExpectedOutputDataSize; - std::unique_ptr<uint8_t[]> input_data_raw(new uint8_t[input_structure_size]); - std::unique_ptr<uint8_t[]> output_data_raw( - new uint8_t[output_structure_size]); - - D3D11_KEY_EXCHANGE_HW_PROTECTION_INPUT_DATA* input_data = - reinterpret_cast<D3D11_KEY_EXCHANGE_HW_PROTECTION_INPUT_DATA*>( - input_data_raw.get()); - D3D11_KEY_EXCHANGE_HW_PROTECTION_OUTPUT_DATA* output_data = - reinterpret_cast<D3D11_KEY_EXCHANGE_HW_PROTECTION_OUTPUT_DATA*>( - output_data_raw.get()); - - D3D11_KEY_EXCHANGE_HW_PROTECTION_DATA expected_key_exchange_data = {}; - expected_key_exchange_data.HWProtectionFunctionID = kTestFunctionId; - expected_key_exchange_data.pInputData = input_data; - expected_key_exchange_data.pOutputData = output_data; - input_data->PrivateDataSize = kPrivateInputSize; - input_data->HWProtectionDataSize = 0; - memcpy(input_data->pbInput, kAnyInput.data(), kAnyInput.size()); - - output_data->PrivateDataSize = kPrivateOutputSize; - output_data->HWProtectionDataSize = 0; - output_data->TransportTime = 0; - output_data->ExecutionTime = 0; - output_data->MaxHWProtectionDataSize = kExpectedOutputDataSize; - - // The value does not matter, so making non zero vector. - std::vector<uint8_t> test_output_data(kExpectedOutputDataSize, 0xAA); - EXPECT_CALL(callback_mock_, - ProcessCallback(CdmProxy::Status::kOk, test_output_data)); - - auto set_test_output_data = [&test_output_data](void* output) { - D3D11_KEY_EXCHANGE_HW_PROTECTION_DATA* kex_struct = - static_cast<D3D11_KEY_EXCHANGE_HW_PROTECTION_DATA*>(output); - memcpy(kex_struct->pOutputData->pbOutput, test_output_data.data(), - test_output_data.size()); - }; - - COM_EXPECT_CALL(video_context_mock_, - NegotiateCryptoSessionKeyExchange( - _, sizeof(expected_key_exchange_data), - MatchesKeyExchangeStructure(&expected_key_exchange_data, - input_structure_size))) - .WillOnce(DoAll(WithArgs<2>(Invoke(set_test_output_data)), Return(S_OK))); - - proxy_->Process(kTestFunction, crypto_session_id, kAnyInput, - kExpectedOutputDataSize, - base::BindOnce(&CallbackMock::ProcessCallback, - base::Unretained(&callback_mock_))); -} - -TEST_F(D3D11CdmProxyTest, CreateMediaCryptoSessionUninitialized) { - // The size nor value here matter, so making non empty non zero vector. - const std::vector<uint8_t> kAnyInput(16, 0xFF); - EXPECT_CALL(callback_mock_, - CreateMediaCryptoSessionCallback(CdmProxy::Status::kFail, _, _)); - proxy_->CreateMediaCryptoSession( - kAnyInput, base::BindOnce(&CallbackMock::CreateMediaCryptoSessionCallback, - base::Unretained(&callback_mock_))); -} - -// Tests the case where no extra data is specified. This is a success case. -TEST_F(D3D11CdmProxyTest, CreateMediaCryptoSessionNoExtraData) { - uint32_t crypto_session_id_from_initialize = 0; - EXPECT_CALL(callback_mock_, - InitializeCallback(CdmProxy::Status::kOk, kTestProtocol, _)) - .WillOnce(SaveArg<2>(&crypto_session_id_from_initialize)); - ASSERT_NO_FATAL_FAILURE( - Initialize(&client_, base::BindOnce(&CallbackMock::InitializeCallback, - base::Unretained(&callback_mock_)))); - Mock::VerifyAndClearExpectations(&callback_mock_); - - // Expect a new crypto session. - EXPECT_CALL(callback_mock_, CreateMediaCryptoSessionCallback( - CdmProxy::Status::kOk, - Ne(crypto_session_id_from_initialize), _)); - auto media_crypto_session_mock = CreateD3D11Mock<D3D11CryptoSessionMock>(); - COM_EXPECT_CALL(video_device_mock_, - CreateCryptoSession(Pointee(CRYPTO_TYPE_GUID), _, - Pointee(CRYPTO_TYPE_GUID), _)) - .WillOnce(SetComPointeeAndReturnOk<3>(media_crypto_session_mock.Get())); - - COM_EXPECT_CALL(video_context1_mock_, GetDataForNewHardwareKey(_, _, _, _)) - .Times(0); - - COM_EXPECT_CALL(video_context1_mock_, - CheckCryptoSessionStatus(media_crypto_session_mock.Get(), _)) - .WillOnce(DoAll(SetArgPointee<1>(D3D11_CRYPTO_SESSION_STATUS_OK), - Return(S_OK))); - proxy_->CreateMediaCryptoSession( - std::vector<uint8_t>(), - base::BindOnce(&CallbackMock::CreateMediaCryptoSessionCallback, - base::Unretained(&callback_mock_))); -} - -// |arg| is void*. This casts the pointer to uint8_t* and checks whether they -// match. -MATCHER_P(CastedToUint8Are, expected, "") { - const uint8_t* actual = static_cast<const uint8_t*>(arg); - for (size_t i = 0; i < expected.size(); ++i) { - if (actual[i] != expected[i]) { - *result_listener << "Mismatch at element " << i; - return false; - } - } - return true; -} - -// Verifies that extra data is used when creating a media crypto session. -TEST_F(D3D11CdmProxyTest, CreateMediaCryptoSessionWithExtraData) { - uint32_t crypto_session_id_from_initialize = 0; - EXPECT_CALL(callback_mock_, - InitializeCallback(CdmProxy::Status::kOk, kTestProtocol, _)) - .WillOnce(SaveArg<2>(&crypto_session_id_from_initialize)); - ASSERT_NO_FATAL_FAILURE( - Initialize(&client_, base::BindOnce(&CallbackMock::InitializeCallback, - base::Unretained(&callback_mock_)))); - Mock::VerifyAndClearExpectations(&callback_mock_); - - // Expect a new crypto session. - EXPECT_CALL(callback_mock_, CreateMediaCryptoSessionCallback( - CdmProxy::Status::kOk, - Ne(crypto_session_id_from_initialize), _)); - - auto media_crypto_session_mock = CreateD3D11Mock<D3D11CryptoSessionMock>(); - COM_EXPECT_CALL(video_device_mock_, - CreateCryptoSession(Pointee(CRYPTO_TYPE_GUID), _, - Pointee(CRYPTO_TYPE_GUID), _)) - .WillOnce(SetComPointeeAndReturnOk<3>(media_crypto_session_mock.Get())); - // The size nor value here matter, so making non empty non zero vector. - const std::vector<uint8_t> kAnyInput(16, 0xFF); - const uint64_t kAnyOutputData = 23298u; - COM_EXPECT_CALL(video_context1_mock_, - GetDataForNewHardwareKey(media_crypto_session_mock.Get(), - kAnyInput.size(), - CastedToUint8Are(kAnyInput), _)) - .WillOnce(DoAll(SetArgPointee<3>(kAnyOutputData), Return(S_OK))); - - COM_EXPECT_CALL(video_context1_mock_, - CheckCryptoSessionStatus(media_crypto_session_mock.Get(), _)) - .WillOnce(DoAll(SetArgPointee<1>(D3D11_CRYPTO_SESSION_STATUS_OK), - Return(S_OK))); - proxy_->CreateMediaCryptoSession( - kAnyInput, base::BindOnce(&CallbackMock::CreateMediaCryptoSessionCallback, - base::Unretained(&callback_mock_))); -} - -// Verify that GetCdmContext() is implemented and does not return null. -TEST_F(D3D11CdmProxyTest, GetCdmContext) { - base::WeakPtr<CdmContext> context = proxy_->GetCdmContext(); - ASSERT_TRUE(context); -} - -TEST_F(D3D11CdmProxyTest, GetCdmProxyContext) { - base::WeakPtr<CdmContext> context = proxy_->GetCdmContext(); - ASSERT_TRUE(context); - ASSERT_TRUE(context->GetCdmProxyContext()); -} - -// No keys are set. -TEST_F(D3D11CdmProxyTest, GetD3D11DecryptContextNoKey) { - base::WeakPtr<CdmContext> context = proxy_->GetCdmContext(); - ASSERT_TRUE(context); - CdmProxyContext* proxy_context = context->GetCdmProxyContext(); - auto decrypt_context = - proxy_context->GetD3D11DecryptContext(kTestKeyType, ""); - EXPECT_FALSE(decrypt_context); -} - -// A key is set but no keys for the key type requested. -TEST_F(D3D11CdmProxyTest, GetD3D11DecryptContextNoKeyForKeyType) { - uint32_t crypto_session_id_from_initialize = 0; - EXPECT_CALL(callback_mock_, - InitializeCallback(CdmProxy::Status::kOk, kTestProtocol, _)) - .WillOnce(SaveArg<2>(&crypto_session_id_from_initialize)); - ASSERT_NO_FATAL_FAILURE( - Initialize(&client_, base::BindOnce(&CallbackMock::InitializeCallback, - base::Unretained(&callback_mock_)))); - Mock::VerifyAndClearExpectations(&callback_mock_); - - const std::vector<uint8_t> kAnyBlob = {0x01, 0x4f, 0x83}; - - EXPECT_CALL(callback_mock_, SetKeyCallback(CdmProxy::Status::kOk)); - proxy_->SetKey(crypto_session_id_from_initialize, kAnyBlob, - CdmProxy::KeyType::kDecryptAndDecode, kAnyBlob, - base::BindOnce(&CallbackMock::SetKeyCallback, - base::Unretained(&callback_mock_))); - - base::WeakPtr<CdmContext> context = proxy_->GetCdmContext(); - CdmProxyContext* proxy_context = context->GetCdmProxyContext(); - auto decrypt_context = proxy_context->GetD3D11DecryptContext( - CdmProxy::KeyType::kDecryptOnly, - std::string(kAnyBlob.begin(), kAnyBlob.end())); - EXPECT_FALSE(decrypt_context); -} - -// Verifies that keys are set and is accessible with a getter. -TEST_F(D3D11CdmProxyTest, SetKeyAndGetDecryptContext) { - base::WeakPtr<CdmContext> context = proxy_->GetCdmContext(); - ASSERT_TRUE(context); - CdmProxyContext* proxy_context = context->GetCdmProxyContext(); - - uint32_t crypto_session_id_from_initialize = 0; - EXPECT_CALL(callback_mock_, - InitializeCallback(CdmProxy::Status::kOk, kTestProtocol, _)) - .WillOnce(SaveArg<2>(&crypto_session_id_from_initialize)); - ASSERT_NO_FATAL_FAILURE( - Initialize(&client_, base::BindOnce(&CallbackMock::InitializeCallback, - base::Unretained(&callback_mock_)))); - Mock::VerifyAndClearExpectations(&callback_mock_); - - std::vector<uint8_t> kKeyId = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - }; - std::vector<uint8_t> kKeyBlob = { - 0xab, 0x01, 0x20, 0xd3, 0xee, 0x05, 0x99, 0x87, - 0xff, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x7F, - }; - - base::MockCallback<CdmContext::EventCB> event_cb; - auto callback_registration = context->RegisterEventCB(event_cb.Get()); - EXPECT_CALL(event_cb, Run(CdmContext::Event::kHasAdditionalUsableKey)); - - EXPECT_CALL(callback_mock_, SetKeyCallback(CdmProxy::Status::kOk)); - proxy_->SetKey(crypto_session_id_from_initialize, kKeyId, kTestKeyType, - kKeyBlob, - base::BindOnce(&CallbackMock::SetKeyCallback, - base::Unretained(&callback_mock_))); - - // |event_cb| is posted. Run the loop to make sure it's fired. - base::RunLoop().RunUntilIdle(); - - std::string key_id_str(kKeyId.begin(), kKeyId.end()); - auto decrypt_context = - proxy_context->GetD3D11DecryptContext(kTestKeyType, key_id_str); - ASSERT_TRUE(decrypt_context); - - EXPECT_TRUE(decrypt_context->crypto_session) - << "Crypto session should not be null."; - const uint8_t* key_blob = - reinterpret_cast<const uint8_t*>(decrypt_context->key_blob); - EXPECT_EQ(kKeyBlob, std::vector<uint8_t>( - key_blob, key_blob + decrypt_context->key_blob_size)); - EXPECT_EQ(CRYPTO_TYPE_GUID, decrypt_context->key_info_guid); -} - -// Verify that the keys are not accessible via CdmProxyContext, after a -// teardown.. -TEST_F(D3D11CdmProxyTest, ClearKeysAfterHardwareContentProtectionTeardown) { - base::RunLoop run_loop; - - uint32_t crypto_session_id_from_initialize = 0; - EXPECT_CALL(callback_mock_, - InitializeCallback(CdmProxy::Status::kOk, kTestProtocol, _)) - .WillOnce(SaveArg<2>(&crypto_session_id_from_initialize)); - ASSERT_NO_FATAL_FAILURE( - Initialize(&client_, base::BindOnce(&CallbackMock::InitializeCallback, - base::Unretained(&callback_mock_)))); - Mock::VerifyAndClearExpectations(&callback_mock_); - - std::vector<uint8_t> kKeyId = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - }; - std::vector<uint8_t> kKeyBlob = { - 0xab, 0x01, 0x20, 0xd3, 0xee, 0x05, 0x99, 0x87, - 0xff, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x7F, - }; - EXPECT_CALL(callback_mock_, SetKeyCallback(CdmProxy::Status::kOk)); - proxy_->SetKey(crypto_session_id_from_initialize, kKeyId, kTestKeyType, - kKeyBlob, - base::BindOnce(&CallbackMock::SetKeyCallback, - base::Unretained(&callback_mock_))); - - EXPECT_CALL(client_, NotifyHardwareReset()).WillOnce(Invoke([&run_loop]() { - run_loop.Quit(); - })); - - SetEvent(teardown_event_); - run_loop.Run(); - - base::WeakPtr<CdmContext> context = proxy_->GetCdmContext(); - ASSERT_TRUE(context); - CdmProxyContext* proxy_context = context->GetCdmProxyContext(); - - std::string key_id_str(kKeyId.begin(), kKeyId.end()); - auto decrypt_context = - proxy_context->GetD3D11DecryptContext(kTestKeyType, key_id_str); - ASSERT_FALSE(decrypt_context); -} - -// Verify that removing a key works. -TEST_F(D3D11CdmProxyTest, RemoveKey) { - base::WeakPtr<CdmContext> context = proxy_->GetCdmContext(); - ASSERT_TRUE(context); - CdmProxyContext* proxy_context = context->GetCdmProxyContext(); - - uint32_t crypto_session_id_from_initialize = 0; - EXPECT_CALL(callback_mock_, - InitializeCallback(CdmProxy::Status::kOk, kTestProtocol, _)) - .WillOnce(SaveArg<2>(&crypto_session_id_from_initialize)); - ASSERT_NO_FATAL_FAILURE( - Initialize(&client_, base::BindOnce(&CallbackMock::InitializeCallback, - base::Unretained(&callback_mock_)))); - Mock::VerifyAndClearExpectations(&callback_mock_); - - std::vector<uint8_t> kKeyId = { - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, - }; - std::vector<uint8_t> kKeyBlob = { - 0xab, 0x01, 0x20, 0xd3, 0xee, 0x05, 0x99, 0x87, - 0xff, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x7F, - }; - EXPECT_CALL(callback_mock_, SetKeyCallback(CdmProxy::Status::kOk)); - EXPECT_CALL(callback_mock_, RemoveKeyCallback(CdmProxy::Status::kOk)); - proxy_->SetKey(crypto_session_id_from_initialize, kKeyId, kTestKeyType, - kKeyBlob, - base::BindOnce(&CallbackMock::SetKeyCallback, - base::Unretained(&callback_mock_))); - proxy_->RemoveKey(crypto_session_id_from_initialize, kKeyId, - base::BindOnce(&CallbackMock::RemoveKeyCallback, - base::Unretained(&callback_mock_))); - - std::string keyblob_str(kKeyId.begin(), kKeyId.end()); - auto decrypt_context = - proxy_context->GetD3D11DecryptContext(kTestKeyType, keyblob_str); - EXPECT_FALSE(decrypt_context); -} - -// Calling SetKey() and RemoveKey() for non-existent crypto session should -// fail but not crash. -TEST_F(D3D11CdmProxyTest, SetRemoveKeyWrongCryptoSessionId) { - const uint32_t kAnyCryptoSessionId = 0x9238; - const std::vector<uint8_t> kEmpty; - EXPECT_CALL(callback_mock_, RemoveKeyCallback(CdmProxy::Status::kFail)); - EXPECT_CALL(callback_mock_, SetKeyCallback(CdmProxy::Status::kFail)); - proxy_->RemoveKey(kAnyCryptoSessionId, kEmpty, - base::BindOnce(&CallbackMock::RemoveKeyCallback, - base::Unretained(&callback_mock_))); - proxy_->SetKey(kAnyCryptoSessionId, kEmpty, kTestKeyType, kEmpty, - base::BindOnce(&CallbackMock::SetKeyCallback, - base::Unretained(&callback_mock_))); -} - -TEST_F(D3D11CdmProxyTest, ProxyInvalidationInvalidatesCdmContext) { - base::WeakPtr<CdmContext> context = proxy_->GetCdmContext(); - EXPECT_TRUE(context); - proxy_.reset(); - EXPECT_FALSE(context); -} - -} // namespace media diff --git a/chromium/media/gpu/windows/d3d11_copying_texture_wrapper.cc b/chromium/media/gpu/windows/d3d11_copying_texture_wrapper.cc index 6b739a87e0c..039062e324c 100644 --- a/chromium/media/gpu/windows/d3d11_copying_texture_wrapper.cc +++ b/chromium/media/gpu/windows/d3d11_copying_texture_wrapper.cc @@ -8,6 +8,7 @@ #include "gpu/command_buffer/service/mailbox_manager.h" #include "media/gpu/windows/d3d11_com_defs.h" +#include "media/gpu/windows/display_helper.h" namespace media { @@ -81,11 +82,25 @@ bool CopyingTexture2DWrapper::ProcessTexture( output_texture_, 0, copy_color_space, mailbox_dest, output_color_space); } -bool CopyingTexture2DWrapper::Init(GetCommandBufferHelperCB get_helper_cb) { +bool CopyingTexture2DWrapper::Init( + scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner, + GetCommandBufferHelperCB get_helper_cb) { if (!video_processor_->Init(size_.width(), size_.height())) return false; - return output_texture_wrapper_->Init(std::move(get_helper_cb)); + return output_texture_wrapper_->Init(std::move(gpu_task_runner), + std::move(get_helper_cb)); +} + +void CopyingTexture2DWrapper::SetStreamHDRMetadata( + const HDRMetadata& stream_metadata) { + auto dxgi_stream_metadata = DisplayHelper::HdrMetadataToDXGI(stream_metadata); + video_processor_->SetStreamHDRMetadata(dxgi_stream_metadata); +} + +void CopyingTexture2DWrapper::SetDisplayHDRMetadata( + const DXGI_HDR_METADATA_HDR10& dxgi_display_metadata) { + video_processor_->SetDisplayHDRMetadata(dxgi_display_metadata); } } // namespace media diff --git a/chromium/media/gpu/windows/d3d11_copying_texture_wrapper.h b/chromium/media/gpu/windows/d3d11_copying_texture_wrapper.h index 139f318ac11..0768f351a7e 100644 --- a/chromium/media/gpu/windows/d3d11_copying_texture_wrapper.h +++ b/chromium/media/gpu/windows/d3d11_copying_texture_wrapper.h @@ -37,7 +37,12 @@ class MEDIA_GPU_EXPORT CopyingTexture2DWrapper : public Texture2DWrapper { MailboxHolderArray* mailbox_dest, gfx::ColorSpace* output_color_space) override; - bool Init(GetCommandBufferHelperCB get_helper_cb) override; + bool Init(scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner, + GetCommandBufferHelperCB get_helper_cb) override; + + void SetStreamHDRMetadata(const HDRMetadata& stream_metadata) override; + void SetDisplayHDRMetadata( + const DXGI_HDR_METADATA_HDR10& dxgi_display_metadata) override; private: gfx::Size size_; diff --git a/chromium/media/gpu/windows/d3d11_copying_texture_wrapper_unittest.cc b/chromium/media/gpu/windows/d3d11_copying_texture_wrapper_unittest.cc index da1adea3e03..ed36baac7e9 100644 --- a/chromium/media/gpu/windows/d3d11_copying_texture_wrapper_unittest.cc +++ b/chromium/media/gpu/windows/d3d11_copying_texture_wrapper_unittest.cc @@ -2,12 +2,16 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +#include <string.h> + #include <utility> #include "base/bind_helpers.h" +#include "base/test/task_environment.h" #include "media/gpu/windows/d3d11_copying_texture_wrapper.h" #include "media/gpu/windows/d3d11_texture_wrapper.h" #include "media/gpu/windows/d3d11_video_processor_proxy.h" +#include "media/gpu/windows/display_helper.h" #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" @@ -49,6 +53,16 @@ class MockVideoProcessorProxy : public VideoProcessorProxy { last_output_color_space_ = color_space; } + void SetStreamHDRMetadata( + const DXGI_HDR_METADATA_HDR10& stream_metadata) override { + last_stream_metadata_ = stream_metadata; + } + + void SetDisplayHDRMetadata( + const DXGI_HDR_METADATA_HDR10& display_metadata) override { + last_display_metadata_ = display_metadata; + } + HRESULT VideoProcessorBlt(ID3D11VideoProcessorOutputView* output_view, UINT output_frameno, UINT stream_count, @@ -61,9 +75,11 @@ class MockVideoProcessorProxy : public VideoProcessorProxy { MOCK_METHOD0(MockCreateVideoProcessorInputView, HRESULT()); MOCK_METHOD0(MockVideoProcessorBlt, HRESULT()); - // Most recent arguments to SetStream/OutputColorSpace(). + // Most recent arguments to SetStream/OutputColorSpace()/etc. base::Optional<gfx::ColorSpace> last_stream_color_space_; base::Optional<gfx::ColorSpace> last_output_color_space_; + base::Optional<DXGI_HDR_METADATA_HDR10> last_stream_metadata_; + base::Optional<DXGI_HDR_METADATA_HDR10> last_display_metadata_; }; class MockTexture2DWrapper : public Texture2DWrapper { @@ -81,12 +97,19 @@ class MockTexture2DWrapper : public Texture2DWrapper { return MockProcessTexture(); } - bool Init(GetCommandBufferHelperCB get_helper_cb) override { + bool Init(scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner, + GetCommandBufferHelperCB get_helper_cb) override { + gpu_task_runner_ = std::move(gpu_task_runner); return MockInit(); } MOCK_METHOD0(MockInit, bool()); MOCK_METHOD0(MockProcessTexture, bool()); + MOCK_METHOD1(SetStreamHDRMetadata, void(const HDRMetadata& stream_metadata)); + MOCK_METHOD1(SetDisplayHDRMetadata, + void(const DXGI_HDR_METADATA_HDR10& dxgi_display_metadata)); + + scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner_; }; CommandBufferHelperPtr UselessHelper() { @@ -108,6 +131,10 @@ class D3D11CopyingTexture2DWrapperTest FIELD(bool, PassthroughColorSpace, 6) #undef FIELD + void SetUp() override { + gpu_task_runner_ = task_environment_.GetMainThreadTaskRunner(); + } + std::unique_ptr<MockVideoProcessorProxy> ExpectProcessorProxy() { auto result = std::make_unique<MockVideoProcessorProxy>(); ON_CALL(*result.get(), MockInit(_, _)) @@ -125,7 +152,7 @@ class D3D11CopyingTexture2DWrapperTest return result; } - std::unique_ptr<Texture2DWrapper> ExpectTextureWrapper() { + std::unique_ptr<MockTexture2DWrapper> ExpectTextureWrapper() { auto result = std::make_unique<MockTexture2DWrapper>(); ON_CALL(*result.get(), MockInit()) @@ -134,7 +161,7 @@ class D3D11CopyingTexture2DWrapperTest ON_CALL(*result.get(), MockProcessTexture()) .WillByDefault(Return(GetProcessTexture())); - return std::move(result); + return result; } GetCommandBufferHelperCB CreateMockHelperCB() { @@ -151,6 +178,9 @@ class D3D11CopyingTexture2DWrapperTest SUCCEEDED(GetCreateVideoProcessorInputView()) && SUCCEEDED(GetVideoProcessorBlt()); } + + base::test::TaskEnvironment task_environment_; + scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner_; }; INSTANTIATE_TEST_CASE_P(CopyingTexture2DWrapperTest, @@ -175,14 +205,22 @@ TEST_P(D3D11CopyingTexture2DWrapperTest, base::Optional<gfx::ColorSpace> copy_color_space; if (!GetPassthroughColorSpace()) copy_color_space = gfx::ColorSpace::CreateDisplayP3D65(); + auto texture_wrapper = ExpectTextureWrapper(); + MockTexture2DWrapper* texture_wrapper_raw = texture_wrapper.get(); auto wrapper = std::make_unique<CopyingTexture2DWrapper>( - size, ExpectTextureWrapper(), std::move(processor), nullptr, + size, std::move(texture_wrapper), std::move(processor), nullptr, copy_color_space); + // TODO: check |gpu_task_runner_|. + MailboxHolderArray mailboxes; gfx::ColorSpace input_color_space = gfx::ColorSpace::CreateSCRGBLinear(); gfx::ColorSpace output_color_space; - EXPECT_EQ(wrapper->Init(CreateMockHelperCB()), InitSucceeds()); + EXPECT_EQ(wrapper->Init(gpu_task_runner_, CreateMockHelperCB()), + InitSucceeds()); + task_environment_.RunUntilIdle(); + if (GetProcessorProxyInit()) + EXPECT_EQ(texture_wrapper_raw->gpu_task_runner_, gpu_task_runner_); EXPECT_EQ(wrapper->ProcessTexture(nullptr, 0, input_color_space, &mailboxes, &output_color_space), ProcessTextureSucceeds()); @@ -205,4 +243,44 @@ TEST_P(D3D11CopyingTexture2DWrapperTest, // TODO: verify that these aren't sent multiple times, unless they change. } +TEST_P(D3D11CopyingTexture2DWrapperTest, HDRMetadataIsSentToVideoProcessor) { + HDRMetadata metadata; + metadata.mastering_metadata.primary_r = + MasteringMetadata::Chromaticity(0.1, 0.2); + metadata.mastering_metadata.primary_g = + MasteringMetadata::Chromaticity(0.3, 0.4); + metadata.mastering_metadata.primary_b = + MasteringMetadata::Chromaticity(0.5, 0.6); + metadata.mastering_metadata.white_point = + MasteringMetadata::Chromaticity(0.7, 0.8); + metadata.mastering_metadata.luminance_max = 0.9; + metadata.mastering_metadata.luminance_min = 0.05; + metadata.max_content_light_level = 1000; + metadata.max_frame_average_light_level = 10000; + + auto processor = ExpectProcessorProxy(); + MockVideoProcessorProxy* processor_raw = processor.get(); + auto wrapper = std::make_unique<CopyingTexture2DWrapper>( + gfx::Size(100, 200), ExpectTextureWrapper(), std::move(processor), + nullptr, gfx::ColorSpace::CreateSCRGBLinear()); + + const DXGI_HDR_METADATA_HDR10 dxgi_metadata = + DisplayHelper::HdrMetadataToDXGI(metadata); + + wrapper->SetStreamHDRMetadata(metadata); + EXPECT_TRUE(processor_raw->last_stream_metadata_); + EXPECT_FALSE(processor_raw->last_display_metadata_); + EXPECT_EQ(memcmp(&dxgi_metadata, &(*processor_raw->last_stream_metadata_), + sizeof(dxgi_metadata)), + 0); + processor_raw->last_stream_metadata_.reset(); + + wrapper->SetDisplayHDRMetadata(dxgi_metadata); + EXPECT_FALSE(processor_raw->last_stream_metadata_); + EXPECT_TRUE(processor_raw->last_display_metadata_); + EXPECT_EQ(memcmp(&dxgi_metadata, &(*processor_raw->last_display_metadata_), + sizeof(dxgi_metadata)), + 0); +} + } // namespace media diff --git a/chromium/media/gpu/windows/d3d11_decryptor.cc b/chromium/media/gpu/windows/d3d11_decryptor.cc deleted file mode 100644 index 9f64e0cba0b..00000000000 --- a/chromium/media/gpu/windows/d3d11_decryptor.cc +++ /dev/null @@ -1,381 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/gpu/windows/d3d11_decryptor.h" - -#include "base/bind.h" -#include "base/logging.h" -#include "base/stl_util.h" -#include "media/base/decoder_buffer.h" -#include "media/gpu/windows/d3d11_com_defs.h" - -namespace media { - -namespace { - -// "A buffer is defined as a single subresource." -// https://msdn.microsoft.com/en-us/library/windows/desktop/ff476901(v=vs.85).aspx -const UINT kSubresourceIndex = 0; -const UINT kWaitIfGPUBusy = 0; - -// This value is somewhat arbitrary but is a multiple of 16 and 4K and is -// equal to D3D11_REQ_TEXTURE2D_U_OR_V_DIMENSION. Since the buffers are cast -// to ID3D11Texture2D, setting it as its size should make sense. -const UINT kBufferSize = 16384; - -// Creates ID3D11Buffer using the values. Return true on success. -bool CreateBuffer(ID3D11Device* device, - D3D11_USAGE usage, - UINT bind_flags, - UINT cpu_access, - ID3D11Buffer** out) { - D3D11_BUFFER_DESC buf_desc = {}; - - buf_desc.ByteWidth = kBufferSize; - buf_desc.BindFlags = bind_flags; - buf_desc.Usage = usage; - buf_desc.CPUAccessFlags = cpu_access; - - HRESULT hresult = device->CreateBuffer(&buf_desc, nullptr, out); - return SUCCEEDED(hresult); -} - -// Copies |input| into |output|, the output buffer should be a staging buffer -// that is CPU writable. -bool CopyDataToBuffer(base::span<const uint8_t> input, - ID3D11DeviceContext* device_context, - ID3D11Buffer* output) { - D3D11_BUFFER_DESC output_buffer_desc = {}; - output->GetDesc(&output_buffer_desc); - - if (output_buffer_desc.ByteWidth < input.size()) { - DVLOG(1) << input.size() << " does not fit in " - << output_buffer_desc.ByteWidth; - return false; - } - - D3D11_MAPPED_SUBRESOURCE map_resource = {}; - HRESULT hresult = - device_context->Map(output, kSubresourceIndex, D3D11_MAP_WRITE, - kWaitIfGPUBusy, &map_resource); - if (FAILED(hresult)) { - DVLOG(3) << "Failed to map buffer for writing."; - return false; - } - memcpy(map_resource.pData, input.data(), input.size_bytes()); - device_context->Unmap(output, kSubresourceIndex); - return true; -} - -// Copies |input| into |output|. The input buffer is should be a staging buffer -// that is CPU readable. -bool CopyDataOutFromBuffer(ID3D11Buffer* input, - size_t input_size, - ID3D11DeviceContext* device_context, - std::vector<uint8_t>* output) { - D3D11_MAPPED_SUBRESOURCE map_resource = {}; - HRESULT hresult = device_context->Map( - input, kSubresourceIndex, D3D11_MAP_READ, kWaitIfGPUBusy, &map_resource); - if (FAILED(hresult)) { - DVLOG(3) << "Failed to map buffer for reading."; - return false; - } - output->resize(input_size); - memcpy(output->data(), map_resource.pData, input_size); - device_context->Unmap(input, kSubresourceIndex); - return true; -} - -D3D11_AES_CTR_IV StringIvToD3D11Iv(const std::string& iv) { - D3D11_AES_CTR_IV d3d11_iv = {}; - DCHECK_LE(iv.size(), 16u); - memcpy(&d3d11_iv, iv.data(), iv.size()); - return d3d11_iv; -} - -// Returns true if the entire sample is encrypted. -bool IsWholeSampleEncrypted(const DecryptConfig& decrypt_config, - size_t sample_size) { - const auto& subsamples = decrypt_config.subsamples(); - if (subsamples.size() != 1) - return false; - - return subsamples.front().clear_bytes == 0 && - subsamples.front().cypher_bytes == sample_size; -} - -// Checks whether |device1| is the same component as |device2|. -// Note that comparing COM pointers require using their IUnknowns. -// https://docs.microsoft.com/en-us/windows/desktop/api/unknwn/nf-unknwn-iunknown-queryinterface(q_) -bool SameDevices(ComD3D11Device device1, ComD3D11Device device2) { - // For the case where both are nullptrs, they aren't devices, so returning - // false here. - if (!device1 || !device2) - return false; - Microsoft::WRL::ComPtr<IUnknown> device1_iunknown; - Microsoft::WRL::ComPtr<IUnknown> device2_iunknown; - HRESULT hr = device1.CopyTo(device1_iunknown.ReleaseAndGetAddressOf()); - if (FAILED(hr)) - return false; - hr = device2.CopyTo(device2_iunknown.ReleaseAndGetAddressOf()); - if (FAILED(hr)) - return false; - return device1_iunknown == device2_iunknown; -} - -// Returns a value that is bigger than or equal to |num| that is a -// multiple of 16. -// E.g. num = 15 returns 16, 17 returns 32. -UINT To16Multiple(size_t num) { - return ((num + 15) >> 4) << 4; -} - -} // namespace - -D3D11Decryptor::D3D11Decryptor(CdmProxyContext* cdm_proxy_context) - : cdm_proxy_context_(cdm_proxy_context) { - DCHECK(cdm_proxy_context_); -} - -D3D11Decryptor::~D3D11Decryptor() {} - -void D3D11Decryptor::RegisterNewKeyCB(StreamType stream_type, - NewKeyCB new_key_cb) { - // TODO(crbug.com/821288): Use RegisterNewKeyCB() on CdmContext, and remove - // RegisterNewKeyCB from Decryptor interface. - NOTREACHED(); -} - -void D3D11Decryptor::Decrypt(StreamType stream_type, - scoped_refptr<DecoderBuffer> encrypted, - DecryptCB decrypt_cb) { - if (encrypted->end_of_stream()) { - std::move(decrypt_cb).Run(kSuccess, encrypted); - return; - } - - const auto* decrypt_config = encrypted->decrypt_config(); - if (!decrypt_config) { - // Not encrypted, nothing to do. - std::move(decrypt_cb).Run(kSuccess, encrypted); - return; - } - - if (decrypt_config->HasPattern()) { - DVLOG(3) << "Cannot handle pattern decryption."; - std::move(decrypt_cb).Run(kError, nullptr); - return; - } - - auto context = cdm_proxy_context_->GetD3D11DecryptContext( - CdmProxy::KeyType::kDecryptOnly, decrypt_config->key_id()); - if (!context) { - std::move(decrypt_cb).Run(kNoKey, nullptr); - return; - } - - // Because DecryptionBlt() implementation checks whether the device, buffers, - // and the crypto session are from the same device, the buffers have to be - // recreated. - if (!InitializeDecryptionBuffer(*context)) { - std::move(decrypt_cb).Run(kError, nullptr); - return; - } - - std::vector<uint8_t> output; - if (IsWholeSampleEncrypted(*encrypted->decrypt_config(), - encrypted->data_size())) { - if (!CtrDecrypt(base::make_span(encrypted->data(), encrypted->data_size()), - encrypted->decrypt_config()->iv(), *context, &output)) { - std::move(decrypt_cb).Run(kError, nullptr); - return; - } - } else { - if (!SubsampleCtrDecrypt(encrypted, *context, &output)) { - std::move(decrypt_cb).Run(kError, nullptr); - return; - } - } - - auto decoder_buffer = DecoderBuffer::CopyFrom(output.data(), output.size()); - decoder_buffer->set_timestamp(encrypted->timestamp()); - decoder_buffer->set_duration(encrypted->duration()); - decoder_buffer->set_is_key_frame(encrypted->is_key_frame()); - decoder_buffer->CopySideDataFrom(encrypted->side_data(), - encrypted->side_data_size()); - std::move(decrypt_cb).Run(kSuccess, decoder_buffer); -} - -void D3D11Decryptor::CancelDecrypt(StreamType stream_type) { - // Decrypt() calls the DecryptCB synchronously so there's nothing to cancel. -} - -void D3D11Decryptor::InitializeAudioDecoder(const AudioDecoderConfig& config, - DecoderInitCB init_cb) { - // D3D11Decryptor does not support audio decoding. - std::move(init_cb).Run(false); -} - -void D3D11Decryptor::InitializeVideoDecoder(const VideoDecoderConfig& config, - DecoderInitCB init_cb) { - // D3D11Decryptor does not support video decoding. - std::move(init_cb).Run(false); -} - -void D3D11Decryptor::DecryptAndDecodeAudio( - scoped_refptr<DecoderBuffer> encrypted, - const AudioDecodeCB& audio_decode_cb) { - NOTREACHED() << "D3D11Decryptor does not support audio decoding"; -} - -void D3D11Decryptor::DecryptAndDecodeVideo( - scoped_refptr<DecoderBuffer> encrypted, - const VideoDecodeCB& video_decode_cb) { - NOTREACHED() << "D3D11Decryptor does not support video decoding"; -} - -void D3D11Decryptor::ResetDecoder(StreamType stream_type) { - NOTREACHED() << "D3D11Decryptor does not support audio/video decoding"; -} - -void D3D11Decryptor::DeinitializeDecoder(StreamType stream_type) { - // D3D11Decryptor does not support audio/video decoding, but since this can be - // called any time after InitializeAudioDecoder/InitializeVideoDecoder, - // nothing to be done here. -} - -bool D3D11Decryptor::InitializeDecryptionBuffer( - const CdmProxyContext::D3D11DecryptContext& decrypt_context) { - ComPtr<ID3D11Device> crypto_session_device; - decrypt_context.crypto_session->GetDevice( - crypto_session_device.ReleaseAndGetAddressOf()); - - // If they are the same devices, then there is no reason to reinitialize the - // buffers. - if (SameDevices(crypto_session_device, device_)) - return true; - - device_ = crypto_session_device; - device_->GetImmediateContext(device_context_.ReleaseAndGetAddressOf()); - - HRESULT hresult = - device_context_.CopyTo(video_context_.ReleaseAndGetAddressOf()); - if (FAILED(hresult)) { - DVLOG(2) << "Failed to get video context."; - return false; - } - - // The buffer is staging so that the data can be accessed by the CPU and HW. - if (!CreateBuffer(device_.Get(), D3D11_USAGE_STAGING, 0, // no binding. - D3D11_CPU_ACCESS_READ | D3D11_CPU_ACCESS_WRITE, - encrypted_sample_buffer_.ReleaseAndGetAddressOf())) { - DVLOG(2) << "Failed to create buffer for encrypted sample."; - return false; - } - - // Note that the cpu access flag is 0 because this buffer is used to write the - // decrypted buffer in HW. - if (!CreateBuffer(device_.Get(), D3D11_USAGE_DEFAULT, - D3D11_BIND_RENDER_TARGET, - 0, // no cpu access. - decrypted_sample_buffer_.ReleaseAndGetAddressOf())) { - DVLOG(2) << "Failed to create buffer for decrypted sample."; - return false; - } - - if (!CreateBuffer(device_.Get(), D3D11_USAGE_STAGING, 0, // no binding. - D3D11_CPU_ACCESS_READ | D3D11_CPU_ACCESS_WRITE, - cpu_accessible_buffer_.ReleaseAndGetAddressOf())) { - DVLOG(2) << "Failed to create cpu accessible buffer."; - return false; - } - - return true; -} - -bool D3D11Decryptor::CtrDecrypt( - base::span<const uint8_t> input, - const std::string& iv, - const CdmProxyContext::D3D11DecryptContext& context, - std::vector<uint8_t>* output) { - output->clear(); - if (input.empty()) - return true; - - if (!CopyDataToBuffer(input, device_context_.Get(), - encrypted_sample_buffer_.Get())) { - return false; - } - - D3D11_AES_CTR_IV aes_ctr_iv = StringIvToD3D11Iv(iv); - - // The size of the encrypted bytes must be a multiple of 16. See more at - // https://crbug.com/849466. - D3D11_ENCRYPTED_BLOCK_INFO block_info = {}; - block_info.NumEncryptedBytesAtBeginning = To16Multiple(input.size()); - DCHECK_LE(block_info.NumEncryptedBytesAtBeginning, kBufferSize); - block_info.NumBytesInSkipPattern = - kBufferSize - block_info.NumEncryptedBytesAtBeginning; - - // ID3D11Buffers should be used but since the interface takes ID3D11Texture2D, - // it is reinterpret cast. See more at https://crbug.com/849466. - video_context_->DecryptionBlt( - context.crypto_session, - reinterpret_cast<ID3D11Texture2D*>(encrypted_sample_buffer_.Get()), - reinterpret_cast<ID3D11Texture2D*>(decrypted_sample_buffer_.Get()), - &block_info, context.key_blob_size, context.key_blob, sizeof(aes_ctr_iv), - &aes_ctr_iv); - - // Because DecryptionBlt() doesn't have a return value, this is a hack to - // check for decryption operation status. If it has been modified, then there - // was an error. See more at https://crbug.com/849466. - HRESULT result = static_cast<HRESULT>(block_info.NumBytesInEncryptPattern); - if (FAILED(result)) { - DVLOG(3) << "Decryption error :" - << logging::SystemErrorCodeToString(result); - return false; - } - - device_context_->CopyResource(cpu_accessible_buffer_.Get(), - decrypted_sample_buffer_.Get()); - return CopyDataOutFromBuffer(cpu_accessible_buffer_.Get(), input.size(), - device_context_.Get(), output); -} - -// TODO(crbug.com/845631): This is the same as DecryptCencBuffer(), so it should -// be deduped. -bool D3D11Decryptor::SubsampleCtrDecrypt( - scoped_refptr<DecoderBuffer> encrypted, - const CdmProxyContext::D3D11DecryptContext& context, - std::vector<uint8_t>* output) { - const auto& subsamples = encrypted->decrypt_config()->subsamples(); - std::vector<uint8_t> encrypted_data; - const uint8_t* data = encrypted->data(); - for (const auto& subsample : subsamples) { - data += subsample.clear_bytes; - encrypted_data.insert(encrypted_data.end(), data, - data + subsample.cypher_bytes); - data += subsample.cypher_bytes; - } - - std::vector<uint8_t> decrypted_data; - if (!CtrDecrypt(encrypted_data, encrypted->decrypt_config()->iv(), context, - &decrypted_data)) { - return false; - } - - data = encrypted->data(); - const uint8_t* decrypted_data_ptr = decrypted_data.data(); - for (const auto& subsample : subsamples) { - output->insert(output->end(), data, data + subsample.clear_bytes); - data += subsample.clear_bytes; - output->insert(output->end(), decrypted_data_ptr, - decrypted_data_ptr + subsample.cypher_bytes); - decrypted_data_ptr += subsample.cypher_bytes; - data += subsample.cypher_bytes; - } - return true; -} - -} // namespace media diff --git a/chromium/media/gpu/windows/d3d11_decryptor.h b/chromium/media/gpu/windows/d3d11_decryptor.h deleted file mode 100644 index 2f49a28c13a..00000000000 --- a/chromium/media/gpu/windows/d3d11_decryptor.h +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MEDIA_GPU_WINDOWS_D3D11_DECRYPTOR_H_ -#define MEDIA_GPU_WINDOWS_D3D11_DECRYPTOR_H_ - -#include <wrl/client.h> - -#include "base/containers/span.h" -#include "base/macros.h" -#include "base/memory/weak_ptr.h" -#include "media/base/decryptor.h" -#include "media/base/win/d3d11_create_device_cb.h" -#include "media/cdm/cdm_proxy_context.h" -#include "media/gpu/media_gpu_export.h" - -namespace media { - -class MEDIA_GPU_EXPORT D3D11Decryptor : public Decryptor { - public: - explicit D3D11Decryptor(CdmProxyContext* cdm_proxy_context); - ~D3D11Decryptor() final; - - // Decryptor implementation. - void RegisterNewKeyCB(StreamType stream_type, NewKeyCB key_added_cb) final; - void Decrypt(StreamType stream_type, - scoped_refptr<DecoderBuffer> encrypted, - DecryptCB decrypt_cb) final; - void CancelDecrypt(StreamType stream_type) final; - void InitializeAudioDecoder(const AudioDecoderConfig& config, - DecoderInitCB init_cb) final; - void InitializeVideoDecoder(const VideoDecoderConfig& config, - DecoderInitCB init_cb) final; - void DecryptAndDecodeAudio(scoped_refptr<DecoderBuffer> encrypted, - const AudioDecodeCB& audio_decode_cb) final; - void DecryptAndDecodeVideo(scoped_refptr<DecoderBuffer> encrypted, - const VideoDecodeCB& video_decode_cb) final; - void ResetDecoder(StreamType stream_type) final; - void DeinitializeDecoder(StreamType stream_type) final; - - private: - // Initialize the buffers for decryption from decryption context. - bool InitializeDecryptionBuffer( - const CdmProxyContext::D3D11DecryptContext& decrypt_context); - - // CTR mode decrypts |encrypted| data into |output|. |output| is always - // cleared. Returns true on success. - bool CtrDecrypt(base::span<const uint8_t> input, - const std::string& iv, - const CdmProxyContext::D3D11DecryptContext& context, - std::vector<uint8_t>* output); - - // CTR mode decryption method, aware of subsamples. |output| is always - // cleared. Returns true and populates |output| on success. - bool SubsampleCtrDecrypt(scoped_refptr<DecoderBuffer> encrypted, - const CdmProxyContext::D3D11DecryptContext& context, - std::vector<uint8_t>* output); - - CdmProxyContext* cdm_proxy_context_; - - template <class T> - using ComPtr = Microsoft::WRL::ComPtr<T>; - - // After a successful InitializeDecryptionBuffer() call, these are set for the - // current Decrypt() call. - ComPtr<ID3D11Device> device_; - ComPtr<ID3D11DeviceContext> device_context_; - ComPtr<ID3D11VideoContext> video_context_; - - // Due to how D3D11 resource permissons work, there are differences between - // CPU (user) and HW accessible buffers. And things get more complicated with - // what can read or write from/to it, what combinations are valid, and - // performance tradeoffs in giving different permissions. The most straight - // forward way is to use three buffers as described below. - - // A buffer where encrypted data is written by the CPU and is readable by the - // HW. - ComPtr<ID3D11Buffer> encrypted_sample_buffer_; - - // A buffer where the decrypted buffer is written by the HW that is not CPU - // accessible. - ComPtr<ID3D11Buffer> decrypted_sample_buffer_; - - // A CPU accessible buffer where the content of |decrypted_buffer_| is copied - // to. - ComPtr<ID3D11Buffer> cpu_accessible_buffer_; - - base::WeakPtrFactory<D3D11Decryptor> weak_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(D3D11Decryptor); -}; - -} // namespace media - -#endif // MEDIA_GPU_WINDOWS_D3D11_DECRYPTOR_H_ diff --git a/chromium/media/gpu/windows/d3d11_decryptor_unittest.cc b/chromium/media/gpu/windows/d3d11_decryptor_unittest.cc deleted file mode 100644 index f00dbec95e7..00000000000 --- a/chromium/media/gpu/windows/d3d11_decryptor_unittest.cc +++ /dev/null @@ -1,519 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/gpu/windows/d3d11_decryptor.h" - -#include <initguid.h> - -#include <array> - -#include "base/bind.h" -#include "base/containers/span.h" -#include "base/stl_util.h" -#include "media/base/decoder_buffer.h" -#include "media/base/subsample_entry.h" -#include "media/base/win/d3d11_mocks.h" -#include "media/cdm/cdm_proxy_context.h" - -using ::testing::_; -using ::testing::AtLeast; -using ::testing::DoAll; -using ::testing::ElementsAreArray; -using ::testing::Invoke; -using ::testing::IsNull; -using ::testing::Mock; -using ::testing::Pointee; -using ::testing::Return; -using ::testing::SetArgPointee; - -template <class T> -using ComPtr = Microsoft::WRL::ComPtr<T>; - -namespace media { - -namespace { -// clang-format off -// The value doesn't matter this is just a GUID. -DEFINE_GUID(TEST_GUID, - 0x01020304, 0xffee, 0xefba, - 0x93, 0xaa, 0x47, 0x77, 0x43, 0xb1, 0x22, 0x98); -// clang-format on - -// Should be non-0 so that it's different from the default TimeDelta. -constexpr base::TimeDelta kTestTimestamp = - base::TimeDelta::FromMilliseconds(33); - -const uint8_t kAnyKeyBlob[] = {3, 5, 38, 19}; -const char kKeyId[] = "some 16 byte id."; -const char kIv[] = "some 16 byte iv."; - -// For tests where the input doesn't matter. -const uint8_t kAnyInput[] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, -}; - -const SubsampleEntry kAnyInputSubsample(0, base::size(kAnyInput)); - -scoped_refptr<DecoderBuffer> TestDecoderBuffer( - const uint8_t* input, - size_t data_size, - const std::string& key_id, - const std::string& iv, - const std::vector<SubsampleEntry>& subsamples) { - scoped_refptr<DecoderBuffer> encrypted_buffer = - DecoderBuffer::CopyFrom(input, data_size); - - encrypted_buffer->set_decrypt_config( - DecryptConfig::CreateCencConfig(key_id, iv, subsamples)); - encrypted_buffer->set_timestamp(kTestTimestamp); - return encrypted_buffer; -} - -CdmProxyContext::D3D11DecryptContext TestDecryptContext( - ComPtr<D3D11CryptoSessionMock> crypto_session_mock) { - CdmProxyContext::D3D11DecryptContext decrypt_context = {}; - decrypt_context.crypto_session = crypto_session_mock.Get(); - decrypt_context.key_blob = kAnyKeyBlob; - decrypt_context.key_blob_size = base::size(kAnyKeyBlob); - decrypt_context.key_info_guid = TEST_GUID; - return decrypt_context; -} - -class CallbackMock { - public: - MOCK_METHOD2(DecryptCallback, Decryptor::DecryptCB::RunType); -}; - -class CdmProxyContextMock : public CdmProxyContext { - public: - MOCK_METHOD2(GetD3D11DecryptContext, - base::Optional<D3D11DecryptContext>(CdmProxy::KeyType key_type, - const std::string& key_id)); -}; - -// Checks that BUFFER_DESC has these fields match. -// Flags are ORed values, so this only checks that the expected flags are set. -// The other fields are ignored. -MATCHER_P3(BufferDescHas, usage, bind_flags, cpu_access, "") { - const D3D11_BUFFER_DESC& buffer_desc = *arg; - if (buffer_desc.Usage != usage) - return false; - - // Because the flags are enums the compiler infers that the input flags are - // signed ints. And the compiler rejects comparing signed int and unsigned - // int, so they are cast here. - const UINT unsigned_bind_flags = bind_flags; - const UINT unsigned_cpu_access_flags = cpu_access; - - if ((buffer_desc.BindFlags & unsigned_bind_flags) != unsigned_bind_flags) - return false; - - return (buffer_desc.CPUAccessFlags & unsigned_cpu_access_flags) == - unsigned_cpu_access_flags; -} - -// NumEncryptedBytesAtBeginning must be greater than or equal to the size of the -// encrypted data, and also a multiple of 16. -MATCHER_P(NumEncryptedBytesAtBeginningGreaterOrEq, value, "") { - const D3D11_ENCRYPTED_BLOCK_INFO& block_info = *arg; - if (block_info.NumEncryptedBytesAtBeginning < value) { - *result_listener << block_info.NumEncryptedBytesAtBeginning - << " is not less than " << value; - return false; - } - return block_info.NumEncryptedBytesAtBeginning % 16 == 0; -} - -ACTION_P(SetBufferDescSize, size) { - arg0->ByteWidth = size; -} - -MATCHER_P2(OutputDataEquals, data, size, "") { - scoped_refptr<DecoderBuffer> buffer = arg; - if (size != buffer->data_size()) { - return false; - } - if (buffer->timestamp() != kTestTimestamp) { - return false; - } - - std::vector<uint8_t> expected(data, data + size); - std::vector<uint8_t> actual(buffer->data(), - buffer->data() + buffer->data_size()); - return actual == expected; -} - -} // namespace - -class D3D11DecryptorTest : public ::testing::Test { - protected: - void SetUp() override { - decryptor_ = std::make_unique<D3D11Decryptor>(&mock_proxy_); - - device_mock_ = CreateD3D11Mock<D3D11DeviceMock>(); - device_context_mock_ = CreateD3D11Mock<D3D11DeviceContextMock>(); - video_context_mock_ = CreateD3D11Mock<D3D11VideoContextMock>(); - staging_buffer1_ = CreateD3D11Mock<D3D11BufferMock>(); - staging_buffer2_ = CreateD3D11Mock<D3D11BufferMock>(); - gpu_buffer_ = CreateD3D11Mock<D3D11BufferMock>(); - } - - // Only sets mock expectations to the objects. Use this for the case where the - // buffers are expected to be created from |device_mock_|, that's accessible - // through |crypto_session_mock|'s GetDevice() function. - void SetExpectationsForSuccessfulBufferInitialization( - D3D11CryptoSessionMock* crypto_session_mock, - CdmProxyContext::D3D11DecryptContext* decrypt_context) { - // As noted in the function comment, the device is accessible from the - // crypto session. - EXPECT_CALL(*crypto_session_mock, GetDevice(_)) - .Times(AtLeast(1)) - .WillRepeatedly(SetComPointee<0>(device_mock_.Get())); - - // The other components accessible (directly or indirectly) from the device. - COM_EXPECT_CALL(device_mock_, GetImmediateContext(_)) - .Times(AtLeast(1)) - .WillRepeatedly(SetComPointee<0>(device_context_mock_.Get())); - COM_EXPECT_CALL(device_context_mock_, - QueryInterface(IID_ID3D11VideoContext, _)) - .Times(AtLeast(1)) - .WillRepeatedly(SetComPointeeAndReturnOk<1>(video_context_mock_.Get())); - - EXPECT_CALL(mock_proxy_, - GetD3D11DecryptContext(CdmProxy::KeyType::kDecryptOnly, kKeyId)) - .WillOnce(Return(*decrypt_context)); - - // These return big enough size. - COM_ON_CALL(staging_buffer1_, GetDesc(_)) - .WillByDefault(SetBufferDescSize(20000)); - COM_ON_CALL(staging_buffer2_, GetDesc(_)) - .WillByDefault(SetBufferDescSize(20000)); - COM_ON_CALL(gpu_buffer_, GetDesc(_)) - .WillByDefault(SetBufferDescSize(20000)); - - // It should be requesting for 2 staging buffers one for writing the data to - // a GPU buffer and one for reading from the a GPU buffer. - COM_EXPECT_CALL(device_mock_, - CreateBuffer(BufferDescHas(D3D11_USAGE_STAGING, 0u, - D3D11_CPU_ACCESS_READ | - D3D11_CPU_ACCESS_WRITE), - nullptr, _)) - .WillOnce(SetComPointeeAndReturnOk<2>(staging_buffer1_.Get())) - .WillOnce(SetComPointeeAndReturnOk<2>(staging_buffer2_.Get())); - - // It should be requesting a GPU only accessible buffer to the decrypted - // output. - COM_EXPECT_CALL(device_mock_, - CreateBuffer(BufferDescHas(D3D11_USAGE_DEFAULT, - D3D11_BIND_RENDER_TARGET, 0u), - nullptr, _)) - .WillOnce(SetComPointeeAndReturnOk<2>(gpu_buffer_.Get())); - } - - // |input| is the input to the Decrypt() function, the subsample information - // is in |subsamples| therefore |input| may not be entirely encrypted. The - // data that is encrypted in |input| should be |encrypted_input|. - // |whole_output| is the expected output from the Decrypt() call, reported by - // the callback. The decrypted result of |encrypted_input| should be - // |decrypted_output|. - void ExpectSuccessfulDecryption(D3D11CryptoSessionMock* crypto_session_mock, - base::span<const uint8_t> input, - base::span<const uint8_t> encrypted_input, - base::span<const uint8_t> whole_output, - base::span<const uint8_t> decrypted_output, - const std::vector<SubsampleEntry>& subsamples, - D3D11Decryptor* decryptor) { - D3D11_MAPPED_SUBRESOURCE staging_buffer1_subresource = {}; - auto staging_buffer1_subresource_buffer = - std::make_unique<uint8_t[]>(20000); - staging_buffer1_subresource.pData = - staging_buffer1_subresource_buffer.get(); - - // It should be requesting for a memory mapped buffer, from the staging - // buffer, to pass the encrypted data to the GPU. - COM_EXPECT_CALL(device_context_mock_, - Map(staging_buffer1_.Get(), 0, D3D11_MAP_WRITE, _, _)) - .WillOnce( - DoAll(SetArgPointee<4>(staging_buffer1_subresource), Return(S_OK))); - COM_EXPECT_CALL(device_context_mock_, Unmap(staging_buffer1_.Get(), 0)); - - COM_EXPECT_CALL( - video_context_mock_, - DecryptionBlt( - crypto_session_mock, - reinterpret_cast<ID3D11Texture2D*>(staging_buffer1_.Get()), - reinterpret_cast<ID3D11Texture2D*>(gpu_buffer_.Get()), - NumEncryptedBytesAtBeginningGreaterOrEq(encrypted_input.size()), - sizeof(kAnyKeyBlob), kAnyKeyBlob, _, _)); - COM_EXPECT_CALL(device_context_mock_, - CopyResource(staging_buffer2_.Get(), gpu_buffer_.Get())); - - D3D11_MAPPED_SUBRESOURCE staging_buffer2_subresource = {}; - - // pData field is non-const void* so make a copy of kFakeDecryptedData that - // can be cast to void*. - std::unique_ptr<uint8_t[]> decrypted_data = - std::make_unique<uint8_t[]>(decrypted_output.size()); - memcpy(decrypted_data.get(), decrypted_output.data(), - decrypted_output.size()); - staging_buffer2_subresource.pData = decrypted_data.get(); - - // Tt should be requesting for a memory mapped buffer, from the staging - // buffer, to read the decrypted data out from the GPU buffer. - COM_EXPECT_CALL(device_context_mock_, - Map(staging_buffer2_.Get(), 0, D3D11_MAP_READ, _, _)) - .WillOnce( - DoAll(SetArgPointee<4>(staging_buffer2_subresource), Return(S_OK))); - COM_EXPECT_CALL(device_context_mock_, Unmap(staging_buffer2_.Get(), 0)); - - CallbackMock callbacks; - EXPECT_CALL(callbacks, - DecryptCallback(Decryptor::kSuccess, - OutputDataEquals(whole_output.data(), - whole_output.size()))); - - scoped_refptr<DecoderBuffer> encrypted_buffer = - TestDecoderBuffer(input.data(), input.size(), kKeyId, kIv, subsamples); - decryptor->Decrypt(Decryptor::kAudio, encrypted_buffer, - base::BindRepeating(&CallbackMock::DecryptCallback, - base::Unretained(&callbacks))); - - // Verify that the data copied to the GPU buffer is the encrypted portion. - EXPECT_TRUE(std::equal(encrypted_input.begin(), encrypted_input.end(), - staging_buffer1_subresource_buffer.get())); - } - - std::unique_ptr<D3D11Decryptor> decryptor_; - CdmProxyContextMock mock_proxy_; - - ComPtr<D3D11DeviceMock> device_mock_; - ComPtr<D3D11DeviceContextMock> device_context_mock_; - ComPtr<D3D11VideoContextMock> video_context_mock_; - - private: - ComPtr<D3D11BufferMock> staging_buffer1_; - ComPtr<D3D11BufferMock> staging_buffer2_; - ComPtr<D3D11BufferMock> gpu_buffer_; -}; - -// Verify that full sample encrypted sample works. -TEST_F(D3D11DecryptorTest, FullSampleCtrDecrypt) { - const uint8_t kInput[] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - }; - const SubsampleEntry kSubsample(0, base::size(kInput)); - // This is arbitrary. Just used to check that this value is output from the - // method. - const uint8_t kFakeDecryptedData[] = { - 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, - }; - static_assert(base::size(kFakeDecryptedData) == base::size(kInput), - "Fake input and output data size must match."); - - ComPtr<D3D11CryptoSessionMock> crypto_session_mock = - CreateD3D11Mock<D3D11CryptoSessionMock>(); - CdmProxyContext::D3D11DecryptContext decrypt_context = - TestDecryptContext(crypto_session_mock); - SetExpectationsForSuccessfulBufferInitialization(crypto_session_mock.Get(), - &decrypt_context); - - // The entire sample is encrypted so the encrypted/decrypted portions are the - // input/output. - ExpectSuccessfulDecryption(crypto_session_mock.Get(), kInput, kInput, - kFakeDecryptedData, kFakeDecryptedData, - {kSubsample}, decryptor_.get()); -} - -// Verify that it works for encrypted input that's not a multiple of 16. -TEST_F(D3D11DecryptorTest, InputSizeNotMultipleOf16) { - const uint8_t kInput[] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, - }; - const SubsampleEntry kSubsample(0, base::size(kInput)); - // This is arbitrary. Just used to check that this value is output from the - // method. - const uint8_t kFakeDecryptedData[] = { - 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, - }; - static_assert(base::size(kFakeDecryptedData) == base::size(kInput), - "Fake input and output data size must match."); - - ComPtr<D3D11CryptoSessionMock> crypto_session_mock = - CreateD3D11Mock<D3D11CryptoSessionMock>(); - CdmProxyContext::D3D11DecryptContext decrypt_context = - TestDecryptContext(crypto_session_mock); - - SetExpectationsForSuccessfulBufferInitialization(crypto_session_mock.Get(), - &decrypt_context); - - // The entire sample is encrypted so the encrypted/decrypted portions are the - // input/output. - ExpectSuccessfulDecryption(crypto_session_mock.Get(), kInput, kInput, - kFakeDecryptedData, kFakeDecryptedData, - {kSubsample}, decryptor_.get()); -} - -// Verify subsample decryption works. -TEST_F(D3D11DecryptorTest, SubsampleCtrDecrypt) { - // clang-format off - const uint8_t kInput[] = { - // clear 16 bytes. - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - // encrypted 16 bytes. - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - // clear 5 bytes. - 0, 1, 2, 3, 4, - // encrypted 16 bytes. - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - }; - // Encrypted parts of the input - const uint8_t kInputEncrypted[] = { - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - }; - // This is arbitrary. Just used to check that this value is output from the - // method. - const uint8_t kFakeOutputData[] = { - // clear 16 bytes. - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, - // decrypted 16 bytes. - 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, - // clear 5 bytes. - 0, 1, 2, 3, 4, - // decrypted 16 bytes. - 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, - }; - const uint8_t kFakeDecryptedData[] = { - 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, - 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, - }; - // clang-format on - static_assert(base::size(kFakeOutputData) == base::size(kInput), - "Fake input and output data size must match."); - const std::vector<SubsampleEntry> subsamples = {SubsampleEntry(16, 16), - SubsampleEntry(5, 16)}; - - ComPtr<D3D11CryptoSessionMock> crypto_session_mock = - CreateD3D11Mock<D3D11CryptoSessionMock>(); - CdmProxyContext::D3D11DecryptContext decrypt_context = - TestDecryptContext(crypto_session_mock); - - SetExpectationsForSuccessfulBufferInitialization(crypto_session_mock.Get(), - &decrypt_context); - ExpectSuccessfulDecryption(crypto_session_mock.Get(), kInput, kInputEncrypted, - kFakeOutputData, kFakeDecryptedData, subsamples, - decryptor_.get()); -} - -// Verify that if the input is too big, it fails. This may be removed if the -// implementation supports big input. -TEST_F(D3D11DecryptorTest, DecryptInputTooBig) { - // Something pretty big to be an audio frame. The actual data size doesn't - // matter. - std::array<uint8_t, 1000000> kInput; - const SubsampleEntry kSubsample(0, base::size(kInput)); - - ComPtr<D3D11CryptoSessionMock> crypto_session_mock = - CreateD3D11Mock<D3D11CryptoSessionMock>(); - CdmProxyContext::D3D11DecryptContext decrypt_context = - TestDecryptContext(crypto_session_mock); - - SetExpectationsForSuccessfulBufferInitialization(crypto_session_mock.Get(), - &decrypt_context); - CallbackMock callbacks; - EXPECT_CALL(callbacks, DecryptCallback(Decryptor::kError, IsNull())); - - scoped_refptr<DecoderBuffer> encrypted_buffer = TestDecoderBuffer( - kInput.data(), base::size(kInput), kKeyId, kIv, {kSubsample}); - decryptor_->Decrypt(Decryptor::kAudio, encrypted_buffer, - base::BindRepeating(&CallbackMock::DecryptCallback, - base::Unretained(&callbacks))); -} - -// If there is no decrypt config, it must be in the clear, so it shouldn't -// change the output. -TEST_F(D3D11DecryptorTest, NoDecryptConfig) { - scoped_refptr<DecoderBuffer> clear_buffer = - DecoderBuffer::CopyFrom(kAnyInput, base::size(kAnyInput)); - clear_buffer->set_timestamp(kTestTimestamp); - CallbackMock callbacks; - EXPECT_CALL( - callbacks, - DecryptCallback(Decryptor::kSuccess, - OutputDataEquals(kAnyInput, base::size(kAnyInput)))); - decryptor_->Decrypt(Decryptor::kAudio, clear_buffer, - base::BindRepeating(&CallbackMock::DecryptCallback, - base::Unretained(&callbacks))); -} - -// The current decryptor cannot deal with pattern encryption. -TEST_F(D3D11DecryptorTest, PatternDecryption) { - scoped_refptr<DecoderBuffer> encrypted_buffer = - DecoderBuffer::CopyFrom(kAnyInput, base::size(kAnyInput)); - encrypted_buffer->set_decrypt_config(DecryptConfig::CreateCbcsConfig( - kKeyId, kIv, {kAnyInputSubsample}, EncryptionPattern(1, 9))); - - CallbackMock callbacks; - EXPECT_CALL(callbacks, DecryptCallback(Decryptor::kError, IsNull())); - decryptor_->Decrypt(Decryptor::kAudio, encrypted_buffer, - base::BindRepeating(&CallbackMock::DecryptCallback, - base::Unretained(&callbacks))); -} - -// If there is no decrypt context, it's missing a key. -TEST_F(D3D11DecryptorTest, NoDecryptContext) { - scoped_refptr<DecoderBuffer> encrypted_buffer = TestDecoderBuffer( - kAnyInput, base::size(kAnyInput), kKeyId, kIv, {kAnyInputSubsample}); - - EXPECT_CALL(mock_proxy_, - GetD3D11DecryptContext(CdmProxy::KeyType::kDecryptOnly, kKeyId)) - .WillOnce(Return(base::nullopt)); - - CallbackMock callbacks; - EXPECT_CALL(callbacks, DecryptCallback(Decryptor::kNoKey, IsNull())); - decryptor_->Decrypt(Decryptor::kAudio, encrypted_buffer, - base::BindRepeating(&CallbackMock::DecryptCallback, - base::Unretained(&callbacks))); -} - -// Verify that if the crypto session's device is the same as the previous call, -// the buffers aren't recreated. -TEST_F(D3D11DecryptorTest, ReuseBuffers) { - ComPtr<D3D11CryptoSessionMock> crypto_session_mock = - CreateD3D11Mock<D3D11CryptoSessionMock>(); - CdmProxyContext::D3D11DecryptContext decrypt_context = - TestDecryptContext(crypto_session_mock); - - SetExpectationsForSuccessfulBufferInitialization(crypto_session_mock.Get(), - &decrypt_context); - - // This test doesn't require checking the output or the correctness of the - // decyrption, so just pass the input buffer for output. - ExpectSuccessfulDecryption(crypto_session_mock.Get(), kAnyInput, kAnyInput, - kAnyInput, kAnyInput, {kAnyInputSubsample}, - decryptor_.get()); - Mock::VerifyAndClearExpectations(crypto_session_mock.Get()); - Mock::VerifyAndClearExpectations(device_mock_.Get()); - Mock::VerifyAndClearExpectations(video_context_mock_.Get()); - Mock::VerifyAndClearExpectations(device_context_mock_.Get()); - Mock::VerifyAndClearExpectations(&mock_proxy_); - - COM_EXPECT_CALL(crypto_session_mock, GetDevice(_)) - .Times(AtLeast(1)) - .WillRepeatedly(SetComPointee<0>(device_mock_.Get())); - EXPECT_CALL(mock_proxy_, - GetD3D11DecryptContext(CdmProxy::KeyType::kDecryptOnly, kKeyId)) - .WillOnce(Return(decrypt_context)); - - // Buffers should not be (re)initialized on the next call to decrypt because - // it's the same device as the previous call. - COM_EXPECT_CALL(device_mock_, CreateBuffer(_, _, _)).Times(0); - - // This calls Decrypt() so that the expectations above are triggered. - ExpectSuccessfulDecryption(crypto_session_mock.Get(), kAnyInput, kAnyInput, - kAnyInput, kAnyInput, {kAnyInputSubsample}, - decryptor_.get()); -} - -} // namespace media diff --git a/chromium/media/gpu/windows/d3d11_h264_accelerator.cc b/chromium/media/gpu/windows/d3d11_h264_accelerator.cc index b1e6e5edd7d..df549d3a380 100644 --- a/chromium/media/gpu/windows/d3d11_h264_accelerator.cc +++ b/chromium/media/gpu/windows/d3d11_h264_accelerator.cc @@ -10,7 +10,6 @@ #include "base/trace_event/trace_event.h" #include "media/base/media_log.h" #include "media/base/win/mf_helpers.h" -#include "media/cdm/cdm_proxy_context.h" #include "media/gpu/h264_decoder.h" #include "media/gpu/h264_dpb.h" #include "media/gpu/windows/d3d11_picture_buffer.h" @@ -64,20 +63,16 @@ D3D11H264Picture::~D3D11H264Picture() { D3D11H264Accelerator::D3D11H264Accelerator( D3D11VideoDecoderClient* client, MediaLog* media_log, - CdmProxyContext* cdm_proxy_context, ComD3D11VideoDecoder video_decoder, ComD3D11VideoDevice video_device, std::unique_ptr<VideoContextWrapper> video_context) : client_(client), media_log_(media_log), - cdm_proxy_context_(cdm_proxy_context), video_decoder_(video_decoder), video_device_(video_device), video_context_(std::move(video_context)) { DCHECK(client); DCHECK(media_log_); - // |cdm_proxy_context_| is non-null for encrypted content but can be null for - // clear content. } D3D11H264Accelerator::~D3D11H264Accelerator() {} @@ -99,28 +94,9 @@ DecoderStatus D3D11H264Accelerator::SubmitFrameMetadata( const H264Picture::Vector& ref_pic_listb1, scoped_refptr<H264Picture> pic) { const bool is_encrypted = pic->decrypt_config(); - - std::unique_ptr<D3D11_VIDEO_DECODER_BEGIN_FRAME_CRYPTO_SESSION> content_key; - // This decrypt context has to be outside the if block because pKeyInfo in - // D3D11_VIDEO_DECODER_BEGIN_FRAME_CRYPTO_SESSION is a pointer (to a GUID). - base::Optional<CdmProxyContext::D3D11DecryptContext> decrypt_context; if (is_encrypted) { - DCHECK(cdm_proxy_context_) << "No CdmProxyContext but picture is encrypted"; - decrypt_context = cdm_proxy_context_->GetD3D11DecryptContext( - CdmProxy::KeyType::kDecryptAndDecode, pic->decrypt_config()->key_id()); - if (!decrypt_context) { - RecordFailure("Cannot find decrypt context for the frame."); - return DecoderStatus::kTryAgain; - } - - content_key = - std::make_unique<D3D11_VIDEO_DECODER_BEGIN_FRAME_CRYPTO_SESSION>(); - content_key->pCryptoSession = decrypt_context->crypto_session; - content_key->pBlob = const_cast<void*>(decrypt_context->key_blob); - content_key->BlobSize = decrypt_context->key_blob_size; - content_key->pKeyInfoId = &decrypt_context->key_info_guid; - frame_iv_.assign(pic->decrypt_config()->iv().begin(), - pic->decrypt_config()->iv().end()); + RecordFailure("Cannot find decrypt context for the frame."); + return DecoderStatus::kFail; } HRESULT hr; @@ -128,7 +104,7 @@ DecoderStatus D3D11H264Accelerator::SubmitFrameMetadata( hr = video_context_->DecoderBeginFrame( video_decoder_.Get(), static_cast<D3D11H264Picture*>(pic.get())->picture->output_view().Get(), - content_key ? sizeof(*content_key) : 0, content_key.get()); + 0, nullptr); if (hr == E_PENDING || hr == D3DERR_WASSTILLDRAWING) { // Hardware is busy. We should make the call again. diff --git a/chromium/media/gpu/windows/d3d11_h264_accelerator.h b/chromium/media/gpu/windows/d3d11_h264_accelerator.h index 7022dc8c395..cd9dd468755 100644 --- a/chromium/media/gpu/windows/d3d11_h264_accelerator.h +++ b/chromium/media/gpu/windows/d3d11_h264_accelerator.h @@ -26,17 +26,14 @@ #include "ui/gl/gl_image.h" namespace media { -class CdmProxyContext; + class D3D11H264Accelerator; class MediaLog; - class D3D11H264Accelerator : public H264Decoder::H264Accelerator { public: - // |cdm_proxy_context| may be null for clear content. D3D11H264Accelerator(D3D11VideoDecoderClient* client, MediaLog* media_log, - CdmProxyContext* cdm_proxy_context, ComD3D11VideoDecoder video_decoder, ComD3D11VideoDevice video_device, std::unique_ptr<VideoContextWrapper> video_context); @@ -90,7 +87,6 @@ class D3D11H264Accelerator : public H264Decoder::H264Accelerator { D3D11VideoDecoderClient* client_; MediaLog* media_log_ = nullptr; - CdmProxyContext* const cdm_proxy_context_; ComD3D11VideoDecoder video_decoder_; ComD3D11VideoDevice video_device_; diff --git a/chromium/media/gpu/windows/d3d11_picture_buffer.cc b/chromium/media/gpu/windows/d3d11_picture_buffer.cc index ae193f7c4a1..7c0278b690e 100644 --- a/chromium/media/gpu/windows/d3d11_picture_buffer.cc +++ b/chromium/media/gpu/windows/d3d11_picture_buffer.cc @@ -22,11 +22,14 @@ namespace media { D3D11PictureBuffer::D3D11PictureBuffer( + scoped_refptr<base::SequencedTaskRunner> delete_task_runner, ComD3D11Texture2D texture, std::unique_ptr<Texture2DWrapper> texture_wrapper, gfx::Size size, size_t level) - : texture_(std::move(texture)), + : RefCountedDeleteOnSequence<D3D11PictureBuffer>( + std::move(delete_task_runner)), + texture_(std::move(texture)), texture_wrapper_(std::move(texture_wrapper)), size_(size), level_(level) {} @@ -34,16 +37,19 @@ D3D11PictureBuffer::D3D11PictureBuffer( D3D11PictureBuffer::~D3D11PictureBuffer() { } -bool D3D11PictureBuffer::Init(GetCommandBufferHelperCB get_helper_cb, - ComD3D11VideoDevice video_device, - const GUID& decoder_guid, - std::unique_ptr<MediaLog> media_log) { +bool D3D11PictureBuffer::Init( + scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner, + GetCommandBufferHelperCB get_helper_cb, + ComD3D11VideoDevice video_device, + const GUID& decoder_guid, + std::unique_ptr<MediaLog> media_log) { D3D11_VIDEO_DECODER_OUTPUT_VIEW_DESC view_desc = {}; view_desc.DecodeProfile = decoder_guid; view_desc.ViewDimension = D3D11_VDOV_DIMENSION_TEXTURE2D; view_desc.Texture2D.ArraySlice = (UINT)level_; - if (!texture_wrapper_->Init(std::move(get_helper_cb))) { + if (!texture_wrapper_->Init(std::move(gpu_task_runner), + std::move(get_helper_cb))) { MEDIA_LOG(ERROR, media_log) << "Failed to Initialize the wrapper"; return false; } diff --git a/chromium/media/gpu/windows/d3d11_picture_buffer.h b/chromium/media/gpu/windows/d3d11_picture_buffer.h index 77c14800319..d605772d147 100644 --- a/chromium/media/gpu/windows/d3d11_picture_buffer.h +++ b/chromium/media/gpu/windows/d3d11_picture_buffer.h @@ -11,8 +11,7 @@ #include <memory> #include <vector> -#include "base/memory/ref_counted.h" - +#include "base/memory/ref_counted_delete_on_sequence.h" #include "gpu/command_buffer/service/mailbox_manager.h" #include "gpu/command_buffer/service/texture_manager.h" #include "gpu/ipc/service/command_buffer_stub.h" @@ -44,17 +43,20 @@ class Texture2DWrapper; // GpuResources have to be retained until the mailbox is used, but we just // retain the whole thing. class MEDIA_GPU_EXPORT D3D11PictureBuffer - : public base::RefCountedThreadSafe<D3D11PictureBuffer> { + : public base::RefCountedDeleteOnSequence<D3D11PictureBuffer> { public: // |texture_wrapper| is responsible for controlling mailbox access to // the ID3D11Texture2D, // |level| is the picturebuffer index inside the Array-type ID3D11Texture2D. - D3D11PictureBuffer(ComD3D11Texture2D texture, - std::unique_ptr<Texture2DWrapper> texture_wrapper, - gfx::Size size, - size_t level); - - bool Init(GetCommandBufferHelperCB get_helper_cb, + D3D11PictureBuffer( + scoped_refptr<base::SequencedTaskRunner> delete_task_runner, + ComD3D11Texture2D texture, + std::unique_ptr<Texture2DWrapper> texture_wrapper, + gfx::Size size, + size_t level); + + bool Init(scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner, + GetCommandBufferHelperCB get_helper_cb, ComD3D11VideoDevice video_device, const GUID& decoder_guid, std::unique_ptr<MediaLog> media_log); @@ -84,12 +86,15 @@ class MEDIA_GPU_EXPORT D3D11PictureBuffer return output_view_; } + Texture2DWrapper* texture_wrapper() const { return texture_wrapper_.get(); } + // Shouldn't be here, but simpler for now. base::TimeDelta timestamp_; private: ~D3D11PictureBuffer(); - friend class base::RefCountedThreadSafe<D3D11PictureBuffer>; + friend class base::RefCountedDeleteOnSequence<D3D11PictureBuffer>; + friend class base::DeleteHelper<D3D11PictureBuffer>; ComD3D11Texture2D texture_; std::unique_ptr<Texture2DWrapper> texture_wrapper_; diff --git a/chromium/media/gpu/windows/d3d11_texture_wrapper.cc b/chromium/media/gpu/windows/d3d11_texture_wrapper.cc index e777c3cd341..ab7ea22a87f 100644 --- a/chromium/media/gpu/windows/d3d11_texture_wrapper.cc +++ b/chromium/media/gpu/windows/d3d11_texture_wrapper.cc @@ -10,6 +10,7 @@ #include <vector> #include "gpu/command_buffer/service/mailbox_manager.h" +#include "media/base/bind_to_current_loop.h" #include "media/base/win/mf_helpers.h" #include "ui/gl/gl_image.h" @@ -67,15 +68,17 @@ bool DefaultTexture2DWrapper::ProcessTexture( const gfx::ColorSpace& input_color_space, MailboxHolderArray* mailbox_dest, gfx::ColorSpace* output_color_space) { - // TODO(liberato): When |gpu_resources_| is a SB<>, it's okay to post and - // forget this call. It will still be ordered properly with respect to any - // access on the gpu main thread. - // TODO(liberato): Would be nice if SB<> knew how to post and reply, so that - // we could get the error code back eventually, and fail later with it. - auto result = gpu_resources_->PushNewTexture(std::move(texture), array_slice); - if (!result.is_ok()) + // If we've received an error, then return it to our caller. This is probably + // from some previous operation. + // TODO(liberato): Return the error. + if (received_error_) return false; + // It's okay to post and forget this call, since it'll be ordered correctly + // with respect to any access on the gpu main thread. + gpu_resources_.Post(FROM_HERE, &GpuResources::PushNewTexture, + std::move(texture), array_slice); + // TODO(liberato): make sure that |mailbox_holders_| is zero-initialized in // case we don't use all the planes. for (size_t i = 0; i < VideoFrame::kMaxPlanes; i++) @@ -87,15 +90,19 @@ bool DefaultTexture2DWrapper::ProcessTexture( return true; } -bool DefaultTexture2DWrapper::Init(GetCommandBufferHelperCB get_helper_cb) { - gpu_resources_ = std::make_unique<GpuResources>(); - if (!gpu_resources_) - return false; +bool DefaultTexture2DWrapper::Init( + scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner, + GetCommandBufferHelperCB get_helper_cb) { + gpu_resources_ = base::SequenceBound<GpuResources>( + std::move(gpu_task_runner), + BindToCurrentLoop(base::BindOnce(&DefaultTexture2DWrapper::OnError, + weak_factory_.GetWeakPtr()))); // YUV textures are mapped onto two GL textures, while RGB use one. int textures_per_picture = 0; switch (dxgi_format_) { case DXGI_FORMAT_NV12: + case DXGI_FORMAT_P010: textures_per_picture = 2; break; case DXGI_FORMAT_B8G8R8A8_UNORM: @@ -107,6 +114,8 @@ bool DefaultTexture2DWrapper::Init(GetCommandBufferHelperCB get_helper_cb) { } // Generate mailboxes and holders. + // TODO(liberato): Verify that this is really okay off the GPU main thread. + // The current implementation is. std::vector<gpu::Mailbox> mailboxes; for (int texture_idx = 0; texture_idx < textures_per_picture; texture_idx++) { mailboxes.push_back(gpu::Mailbox::Generate()); @@ -119,15 +128,25 @@ bool DefaultTexture2DWrapper::Init(GetCommandBufferHelperCB get_helper_cb) { // device for decoding. Sharing seems not to work very well. Otherwise, we // would create the texture with KEYED_MUTEX and NTHANDLE, then send along // a handle that we get from |texture| as an IDXGIResource1. - // TODO(liberato): this should happen on the gpu thread. - // TODO(liberato): the out param would be handled similarly to - // CodecImageHolder when we add a pool. - return gpu_resources_->Init(std::move(get_helper_cb), std::move(mailboxes), - GL_TEXTURE_EXTERNAL_OES, size_, - textures_per_picture); + gpu_resources_.Post(FROM_HERE, &GpuResources::Init, std::move(get_helper_cb), + std::move(mailboxes), GL_TEXTURE_EXTERNAL_OES, size_, + textures_per_picture); + return true; +} + +void DefaultTexture2DWrapper::OnError(Status status) { + if (!received_error_) + received_error_ = status; } -DefaultTexture2DWrapper::GpuResources::GpuResources() {} +void DefaultTexture2DWrapper::SetStreamHDRMetadata( + const HDRMetadata& stream_metadata) {} + +void DefaultTexture2DWrapper::SetDisplayHDRMetadata( + const DXGI_HDR_METADATA_HDR10& dxgi_display_metadata) {} + +DefaultTexture2DWrapper::GpuResources::GpuResources(OnErrorCB on_error_cb) + : on_error_cb_(std::move(on_error_cb)) {} DefaultTexture2DWrapper::GpuResources::~GpuResources() { if (helper_ && helper_->MakeContextCurrent()) { @@ -136,7 +155,7 @@ DefaultTexture2DWrapper::GpuResources::~GpuResources() { } } -bool DefaultTexture2DWrapper::GpuResources::Init( +void DefaultTexture2DWrapper::GpuResources::Init( GetCommandBufferHelperCB get_helper_cb, const std::vector<gpu::Mailbox> mailboxes, GLenum target, @@ -144,8 +163,10 @@ bool DefaultTexture2DWrapper::GpuResources::Init( int textures_per_picture) { helper_ = get_helper_cb.Run(); - if (!helper_ || !helper_->MakeContextCurrent()) - return false; + if (!helper_ || !helper_->MakeContextCurrent()) { + NotifyError(StatusCode::kCantMakeContextCurrent); + return; + } // Create the textures and attach them to the mailboxes. // TODO(liberato): Should we use GL_FLOAT for an fp16 texture? It doesn't @@ -168,7 +189,10 @@ bool DefaultTexture2DWrapper::GpuResources::Init( // clang-format on }; EGLStreamKHR stream = eglCreateStreamKHR(egl_display, stream_attributes); - RETURN_ON_FAILURE(!!stream, "Could not create stream", false); + if (!stream) { + NotifyError(StatusCode::kCantCreateEglStream); + return; + } // |stream| will be destroyed when the GLImage is. // TODO(liberato): for tests, it will be destroyed pretty much at the end of @@ -204,7 +228,10 @@ bool DefaultTexture2DWrapper::GpuResources::Init( } EGLBoolean result = eglStreamConsumerGLTextureExternalAttribsNV( egl_display, stream, consumer_attributes.data()); - RETURN_ON_FAILURE(result, "Could not set stream consumer", false); + if (!result) { + NotifyError(StatusCode::kCantCreateEglStreamConsumer); + return; + } EGLAttrib producer_attributes[] = { EGL_NONE, @@ -212,7 +239,10 @@ bool DefaultTexture2DWrapper::GpuResources::Init( result = eglCreateStreamProducerD3DTextureANGLE(egl_display, stream, producer_attributes); - RETURN_ON_FAILURE(result, "Could not create stream", false); + if (!result) { + NotifyError(StatusCode::kCantCreateEglStreamProducer); + return; + } // Note that this is valid as long as |gl_image_| is valid; it is // what deletes the stream. @@ -224,15 +254,15 @@ bool DefaultTexture2DWrapper::GpuResources::Init( helper_->BindImage(service_ids_[texture_idx], gl_image_.get(), false /* client_managed */); } - - return true; } -Status DefaultTexture2DWrapper::GpuResources::PushNewTexture( +void DefaultTexture2DWrapper::GpuResources::PushNewTexture( ComD3D11Texture2D texture, size_t array_slice) { - if (!helper_ || !helper_->MakeContextCurrent()) - return Status(StatusCode::kCantMakeContextCurrent); + if (!helper_ || !helper_->MakeContextCurrent()) { + NotifyError(StatusCode::kCantMakeContextCurrent); + return; + } // Notify |gl_image_| that it has a new texture. gl_image_->SetTexture(texture, array_slice); @@ -248,13 +278,20 @@ Status DefaultTexture2DWrapper::GpuResources::PushNewTexture( if (!eglStreamPostD3DTextureANGLE(egl_display, stream_, static_cast<void*>(texture.Get()), frame_attributes)) { - return Status(StatusCode::kCantPostTexture); + NotifyError(StatusCode::kCantPostTexture); + return; } - if (!eglStreamConsumerAcquireKHR(egl_display, stream_)) - return Status(StatusCode::kCantPostAcquireStream); + if (!eglStreamConsumerAcquireKHR(egl_display, stream_)) { + NotifyError(StatusCode::kCantPostAcquireStream); + return; + } +} - return OkStatus(); +void DefaultTexture2DWrapper::GpuResources::NotifyError(Status status) { + if (on_error_cb_) + std::move(on_error_cb_).Run(std::move(status)); + // else this isn't the first error, so skip it. } } // namespace media diff --git a/chromium/media/gpu/windows/d3d11_texture_wrapper.h b/chromium/media/gpu/windows/d3d11_texture_wrapper.h index f7fe065285f..5d71e209809 100644 --- a/chromium/media/gpu/windows/d3d11_texture_wrapper.h +++ b/chromium/media/gpu/windows/d3d11_texture_wrapper.h @@ -10,8 +10,12 @@ #include <memory> #include <vector> +#include "base/memory/weak_ptr.h" +#include "base/optional.h" +#include "base/threading/sequence_bound.h" #include "gpu/command_buffer/service/mailbox_manager.h" #include "gpu/command_buffer/service/texture_manager.h" +#include "media/base/hdr_metadata.h" #include "media/base/status.h" #include "media/base/video_frame.h" #include "media/gpu/command_buffer_helper.h" @@ -41,7 +45,8 @@ class MEDIA_GPU_EXPORT Texture2DWrapper { virtual ~Texture2DWrapper(); // Initialize the wrapper. - virtual bool Init(GetCommandBufferHelperCB get_helper_cb) = 0; + virtual bool Init(scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner, + GetCommandBufferHelperCB get_helper_cb) = 0; // Import |texture|, |array_slice| and return the mailbox(es) that can be // used to refer to it. @@ -50,6 +55,10 @@ class MEDIA_GPU_EXPORT Texture2DWrapper { const gfx::ColorSpace& input_color_space, MailboxHolderArray* mailbox_dest_out, gfx::ColorSpace* output_color_space) = 0; + + virtual void SetStreamHDRMetadata(const HDRMetadata& stream_metadata) = 0; + virtual void SetDisplayHDRMetadata( + const DXGI_HDR_METADATA_HDR10& dxgi_display_metadata) = 0; }; // The default texture wrapper that uses GPUResources to talk to hardware @@ -58,12 +67,16 @@ class MEDIA_GPU_EXPORT Texture2DWrapper { // instance for each concurrently outstanding texture. class MEDIA_GPU_EXPORT DefaultTexture2DWrapper : public Texture2DWrapper { public: + // Error callback for GpuResource to notify us of errors. + using OnErrorCB = base::OnceCallback<void(Status)>; + // While the specific texture instance can change on every call to // ProcessTexture, the dxgi format must be the same for all of them. DefaultTexture2DWrapper(const gfx::Size& size, DXGI_FORMAT dxgi_format); ~DefaultTexture2DWrapper() override; - bool Init(GetCommandBufferHelperCB get_helper_cb) override; + bool Init(scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner, + GetCommandBufferHelperCB get_helper_cb) override; bool ProcessTexture(ComD3D11Texture2D texture, size_t array_slice, @@ -71,6 +84,10 @@ class MEDIA_GPU_EXPORT DefaultTexture2DWrapper : public Texture2DWrapper { MailboxHolderArray* mailbox_dest, gfx::ColorSpace* output_color_space) override; + void SetStreamHDRMetadata(const HDRMetadata& stream_metadata) override; + void SetDisplayHDRMetadata( + const DXGI_HDR_METADATA_HDR10& dxgi_display_metadata) override; + private: // Things that are to be accessed / freed only on the main thread. In // addition to setting up the textures to render from a D3D11 texture, @@ -78,21 +95,27 @@ class MEDIA_GPU_EXPORT DefaultTexture2DWrapper : public Texture2DWrapper { // can use the mailbox. class GpuResources { public: - GpuResources(); + GpuResources(OnErrorCB on_error_cb); ~GpuResources(); - bool Init(GetCommandBufferHelperCB get_helper_cb, + void Init(GetCommandBufferHelperCB get_helper_cb, const std::vector<gpu::Mailbox> mailboxes, GLenum target, gfx::Size size, int textures_per_picture); // Push a new |texture|, |array_slice| to |gl_image_|. - Status PushNewTexture(ComD3D11Texture2D texture, size_t array_slice); + void PushNewTexture(ComD3D11Texture2D texture, size_t array_slice); std::vector<uint32_t> service_ids_; private: + // Notify our wrapper about |status|, if we haven't before. + void NotifyError(Status status); + + // May be empty if we've already sent an error. + OnErrorCB on_error_cb_; + scoped_refptr<CommandBufferHelper> helper_; scoped_refptr<gl::GLImageDXGI> gl_image_; EGLStreamKHR stream_; @@ -100,10 +123,18 @@ class MEDIA_GPU_EXPORT DefaultTexture2DWrapper : public Texture2DWrapper { DISALLOW_COPY_AND_ASSIGN(GpuResources); }; + // Receive an error from |gpu_resources_| and store it in |received_error_|. + void OnError(Status status); + + // The first error status that we've received from |gpu_resources_|, if any. + base::Optional<Status> received_error_; + gfx::Size size_; - std::unique_ptr<GpuResources> gpu_resources_; + base::SequenceBound<GpuResources> gpu_resources_; MailboxHolderArray mailbox_holders_; DXGI_FORMAT dxgi_format_; + + base::WeakPtrFactory<DefaultTexture2DWrapper> weak_factory_{this}; }; } // namespace media diff --git a/chromium/media/gpu/windows/d3d11_texture_wrapper_unittest.cc b/chromium/media/gpu/windows/d3d11_texture_wrapper_unittest.cc new file mode 100644 index 00000000000..4b355a17dcd --- /dev/null +++ b/chromium/media/gpu/windows/d3d11_texture_wrapper_unittest.cc @@ -0,0 +1,126 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/gpu/windows/d3d11_texture_wrapper.h" + +#include <utility> + +#include "base/bind.h" +#include "base/bind_helpers.h" +#include "base/single_thread_task_runner.h" +#include "base/test/task_environment.h" +#include "base/win/windows_version.h" +#include "media/gpu/test/fake_command_buffer_helper.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "ui/gl/gl_context_egl.h" +#include "ui/gl/gl_egl_api_implementation.h" +#include "ui/gl/gl_image.h" +#include "ui/gl/gl_share_group.h" +#include "ui/gl/gl_surface.h" +#include "ui/gl/init/gl_factory.h" +#include "ui/gl/test/gl_surface_test_support.h" + +using ::testing::_; +using ::testing::Bool; +using ::testing::Combine; +using ::testing::Return; +using ::testing::Values; + +namespace media { + +#define STOP_IF_WIN7() \ + do { \ + if (base::win::GetVersion() <= base::win::Version::WIN7) \ + return; \ + } while (0); + +class D3D11TextureWrapperUnittest : public ::testing::Test { + public: + void SetUp() override { + // Surface creation fails sometimes on win7, mostly. Just skip the test. + STOP_IF_WIN7(); + + task_runner_ = task_environment_.GetMainThreadTaskRunner(); + + gl::GLSurfaceTestSupport::InitializeOneOffImplementation( + gl::kGLImplementationEGLANGLE, false); + surface_ = gl::init::CreateOffscreenGLSurface(gfx::Size()); + share_group_ = new gl::GLShareGroup(); + context_ = gl::init::CreateGLContext(share_group_.get(), surface_.get(), + gl::GLContextAttribs()); + context_->MakeCurrent(surface_.get()); + + // Create some objects that most tests want. + fake_command_buffer_helper_ = + base::MakeRefCounted<FakeCommandBufferHelper>(task_runner_); + get_helper_cb_ = base::BindRepeating( + [](scoped_refptr<CommandBufferHelper> helper) { return helper; }, + fake_command_buffer_helper_); + } + + void TearDown() override { + STOP_IF_WIN7(); + context_->ReleaseCurrent(surface_.get()); + context_ = nullptr; + share_group_ = nullptr; + surface_ = nullptr; + gl::init::ShutdownGL(false); + } + + base::test::TaskEnvironment task_environment_; + scoped_refptr<base::SingleThreadTaskRunner> task_runner_; + + scoped_refptr<gl::GLSurface> surface_; + scoped_refptr<gl::GLShareGroup> share_group_; + scoped_refptr<gl::GLContext> context_; + + // Made-up size for the images. + const gfx::Size size_{100, 200}; + + // CommandBufferHelper, and a callback that returns it. Useful to initialize + // a wrapper. + scoped_refptr<FakeCommandBufferHelper> fake_command_buffer_helper_; + GetCommandBufferHelperCB get_helper_cb_; +}; + +TEST_F(D3D11TextureWrapperUnittest, NV12InitSucceeds) { + STOP_IF_WIN7(); + const DXGI_FORMAT dxgi_format = DXGI_FORMAT_NV12; + + auto wrapper = std::make_unique<DefaultTexture2DWrapper>(size_, dxgi_format); + const bool init_result = wrapper->Init(task_runner_, get_helper_cb_); + EXPECT_TRUE(init_result); + + // TODO: verify that ProcessTexture processes both textures. +} + +TEST_F(D3D11TextureWrapperUnittest, BGRA8InitSucceeds) { + STOP_IF_WIN7(); + const DXGI_FORMAT dxgi_format = DXGI_FORMAT_B8G8R8A8_UNORM; + + auto wrapper = std::make_unique<DefaultTexture2DWrapper>(size_, dxgi_format); + const bool init_result = wrapper->Init(task_runner_, get_helper_cb_); + EXPECT_TRUE(init_result); +} + +TEST_F(D3D11TextureWrapperUnittest, FP16InitSucceeds) { + STOP_IF_WIN7(); + const DXGI_FORMAT dxgi_format = DXGI_FORMAT_R16G16B16A16_FLOAT; + + auto wrapper = std::make_unique<DefaultTexture2DWrapper>(size_, dxgi_format); + const bool init_result = wrapper->Init(task_runner_, get_helper_cb_); + EXPECT_TRUE(init_result); +} + +TEST_F(D3D11TextureWrapperUnittest, P010InitSucceeds) { + STOP_IF_WIN7(); + const DXGI_FORMAT dxgi_format = DXGI_FORMAT_P010; + + auto wrapper = std::make_unique<DefaultTexture2DWrapper>(size_, dxgi_format); + const bool init_result = wrapper->Init(task_runner_, get_helper_cb_); + EXPECT_TRUE(init_result); +} + +} // namespace media
\ No newline at end of file diff --git a/chromium/media/gpu/windows/d3d11_video_decoder.cc b/chromium/media/gpu/windows/d3d11_video_decoder.cc index eae07b3c08e..3ba6d9b3225 100644 --- a/chromium/media/gpu/windows/d3d11_video_decoder.cc +++ b/chromium/media/gpu/windows/d3d11_video_decoder.cc @@ -16,9 +16,9 @@ #include "base/memory/ref_counted_delete_on_sequence.h" #include "base/metrics/histogram_functions.h" #include "base/metrics/histogram_macros.h" +#include "base/threading/sequenced_task_runner_handle.h" #include "base/trace_event/trace_event.h" #include "media/base/bind_to_current_loop.h" -#include "media/base/cdm_context.h" #include "media/base/decoder_buffer.h" #include "media/base/media_log.h" #include "media/base/media_switches.h" @@ -30,9 +30,9 @@ #include "media/gpu/windows/d3d11_video_context_wrapper.h" #include "media/gpu/windows/d3d11_video_decoder_impl.h" #include "media/gpu/windows/d3d11_video_device_format_support.h" +#include "media/gpu/windows/display_helper.h" #include "media/gpu/windows/supported_profile_helpers.h" #include "media/media_buildflags.h" -#include "ui/gl/direct_composition_surface_win.h" #include "ui/gl/gl_angle_util_win.h" #include "ui/gl/gl_switches.h" @@ -82,7 +82,8 @@ std::unique_ptr<VideoDecoder> D3D11VideoDecoder::Create( const gpu::GpuDriverBugWorkarounds& gpu_workarounds, base::RepeatingCallback<gpu::CommandBufferStub*()> get_stub_cb, D3D11VideoDecoder::GetD3D11DeviceCB get_d3d11_device_cb, - SupportedConfigs supported_configs) { + SupportedConfigs supported_configs, + bool is_hdr_supported) { // We create |impl_| on the wrong thread, but we never use it here. // Note that the output callback will hop to our thread, post the video // frame, and along with a callback that will hop back to the impl thread @@ -94,13 +95,12 @@ std::unique_ptr<VideoDecoder> D3D11VideoDecoder::Create( base::BindRepeating(CreateCommandBufferHelper, std::move(get_stub_cb), scoped_refptr<CommandBufferHelperHolder>( new CommandBufferHelperHolder(gpu_task_runner))); - return base::WrapUnique<VideoDecoder>( - new D3D11VideoDecoder(std::move(gpu_task_runner), std::move(media_log), - gpu_preferences, gpu_workarounds, - std::make_unique<D3D11VideoDecoderImpl>( - std::move(cloned_media_log), get_helper_cb), - get_helper_cb, std::move(get_d3d11_device_cb), - std::move(supported_configs))); + return base::WrapUnique<VideoDecoder>(new D3D11VideoDecoder( + gpu_task_runner, std::move(media_log), gpu_preferences, gpu_workarounds, + base::SequenceBound<D3D11VideoDecoderImpl>( + gpu_task_runner, std::move(cloned_media_log), get_helper_cb), + get_helper_cb, std::move(get_d3d11_device_cb), + std::move(supported_configs), is_hdr_supported)); } D3D11VideoDecoder::D3D11VideoDecoder( @@ -108,23 +108,24 @@ D3D11VideoDecoder::D3D11VideoDecoder( std::unique_ptr<MediaLog> media_log, const gpu::GpuPreferences& gpu_preferences, const gpu::GpuDriverBugWorkarounds& gpu_workarounds, - std::unique_ptr<D3D11VideoDecoderImpl> impl, + base::SequenceBound<D3D11VideoDecoderImpl> impl, base::RepeatingCallback<scoped_refptr<CommandBufferHelper>()> get_helper_cb, GetD3D11DeviceCB get_d3d11_device_cb, - SupportedConfigs supported_configs) + SupportedConfigs supported_configs, + bool is_hdr_supported) : media_log_(std::move(media_log)), impl_(std::move(impl)), - impl_task_runner_(std::move(gpu_task_runner)), + gpu_task_runner_(std::move(gpu_task_runner)), + decoder_task_runner_(base::SequencedTaskRunnerHandle::Get()), already_initialized_(false), gpu_preferences_(gpu_preferences), gpu_workarounds_(gpu_workarounds), get_d3d11_device_cb_(std::move(get_d3d11_device_cb)), get_helper_cb_(std::move(get_helper_cb)), - supported_configs_(std::move(supported_configs)) { + supported_configs_(std::move(supported_configs)), + is_hdr_supported_(is_hdr_supported) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DCHECK(media_log_); - - impl_weak_ = impl_->GetWeakPtr(); } D3D11VideoDecoder::~D3D11VideoDecoder() { @@ -133,10 +134,7 @@ D3D11VideoDecoder::~D3D11VideoDecoder() { // from |impl_| will be cancelled by |weak_factory_| when we return. DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - if (impl_task_runner_->RunsTasksInCurrentSequence()) - impl_.reset(); - else - impl_task_runner_->DeleteSoon(FROM_HERE, std::move(impl_)); + impl_.Reset(); // Explicitly destroy the decoder, since it can reference picture buffers. accelerated_video_decoder_.reset(); @@ -151,7 +149,6 @@ std::string D3D11VideoDecoder::GetDisplayName() const { HRESULT D3D11VideoDecoder::InitializeAcceleratedDecoder( const VideoDecoderConfig& config, - CdmProxyContext* proxy_context, ComD3D11VideoDecoder video_decoder) { TRACE_EVENT0("gpu", "D3D11VideoDecoder::InitializeAcceleratedDecoder"); // If we got an 11.1 D3D11 Device, we can use a |ID3D11VideoContext1|, @@ -169,18 +166,18 @@ HRESULT D3D11VideoDecoder::InitializeAcceleratedDecoder( profile_ = config.profile(); if (config.codec() == kCodecVP9) { accelerated_video_decoder_ = std::make_unique<VP9Decoder>( - std::make_unique<D3D11VP9Accelerator>( - this, media_log_.get(), proxy_context, video_decoder, video_device_, - std::move(video_context)), + std::make_unique<D3D11VP9Accelerator>(this, media_log_.get(), + video_decoder, video_device_, + std::move(video_context)), profile_, config.color_space_info()); return hr; } if (config.codec() == kCodecH264) { accelerated_video_decoder_ = std::make_unique<H264Decoder>( - std::make_unique<D3D11H264Accelerator>( - this, media_log_.get(), proxy_context, video_decoder, video_device_, - std::move(video_context)), + std::make_unique<D3D11H264Accelerator>(this, media_log_.get(), + video_decoder, video_device_, + std::move(video_context)), profile_, config.color_space_info()); return hr; } @@ -190,10 +187,10 @@ HRESULT D3D11VideoDecoder::InitializeAcceleratedDecoder( void D3D11VideoDecoder::Initialize(const VideoDecoderConfig& config, bool low_delay, - CdmContext* cdm_context, + CdmContext* /* cdm_context */, InitCB init_cb, const OutputCB& output_cb, - const WaitingCB& waiting_cb) { + const WaitingCB& /* waiting_cb */) { TRACE_EVENT0("gpu", "D3D11VideoDecoder::Initialize"); if (already_initialized_) AddLifetimeProgressionStage(D3D11LifetimeProgression::kPlaybackSucceeded); @@ -201,14 +198,12 @@ void D3D11VideoDecoder::Initialize(const VideoDecoderConfig& config, DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); DCHECK(output_cb); - DCHECK(waiting_cb); state_ = State::kInitializing; config_ = config; init_cb_ = std::move(init_cb); output_cb_ = output_cb; - waiting_cb_ = waiting_cb; // Verify that |config| matches one of the supported configurations. This // helps us skip configs that are supported by the VDA but not us, since @@ -227,6 +222,11 @@ void D3D11VideoDecoder::Initialize(const VideoDecoderConfig& config, return; } + if (config.is_encrypted()) { + NotifyError("D3D11VideoDecoder does not support encrypted stream"); + return; + } + // Initialize the video decoder. // Note that we assume that this is the ANGLE device, since we don't implement @@ -240,6 +240,9 @@ void D3D11VideoDecoder::Initialize(const VideoDecoderConfig& config, // it's been specifically configured via switch to avoid d3d11. // // TODO(liberato): On re-init, we can probably re-use the device. + // TODO(liberato): This isn't allowed off the main thread, since the callback + // does who-knows-what. Either we should be given the angle device, or we + // should thread-hop to get it. device_ = get_d3d11_device_cb_.Run(); if (!device_) { // This happens if, for example, if chrome is configured to use @@ -288,13 +291,12 @@ void D3D11VideoDecoder::Initialize(const VideoDecoderConfig& config, // Use IsHDRSupported to guess whether the compositor can output HDR textures. // See TextureSelector for notes about why the decoder should not care. - texture_selector_ = - TextureSelector::Create(gpu_preferences_, gpu_workarounds_, - decoder_configurator_->TextureFormat(), - gl::DirectCompositionSurfaceWin::IsHDRSupported() - ? TextureSelector::HDRMode::kSDROrHDR - : TextureSelector::HDRMode::kSDROnly, - &format_checker, media_log_.get()); + texture_selector_ = TextureSelector::Create( + gpu_preferences_, gpu_workarounds_, + decoder_configurator_->TextureFormat(), + is_hdr_supported_ ? TextureSelector::HDRMode::kSDROrHDR + : TextureSelector::HDRMode::kSDROnly, + &format_checker, media_log_.get()); if (!texture_selector_) { NotifyError("D3DD11: Cannot get TextureSelector for format"); return; @@ -333,12 +335,6 @@ void D3D11VideoDecoder::Initialize(const VideoDecoderConfig& config, return; } - if (config.is_encrypted() && dec_config.guidConfigBitstreamEncryption != - D3D11_DECODER_ENCRYPTION_HW_CENC) { - // For encrypted media, it has to use HW CENC decoder config. - continue; - } - if (config.codec() == kCodecVP9 && dec_config.ConfigBitstreamRaw == 1) { // DXVA VP9 specification mentions ConfigBitstreamRaw "shall be 1". found = true; @@ -364,19 +360,7 @@ void D3D11VideoDecoder::Initialize(const VideoDecoderConfig& config, return; } - CdmProxyContext* proxy_context = nullptr; -#if BUILDFLAG(ENABLE_CDM_PROXY) - if (cdm_context) - proxy_context = cdm_context->GetCdmProxyContext(); -#endif - - // Ensure that if we are encrypted, that we have a CDM. - if (config_.is_encrypted() && !proxy_context) { - NotifyError("Video stream is encrypted, but no cdm was found"); - return; - } - - hr = InitializeAcceleratedDecoder(config, proxy_context, video_decoder); + hr = InitializeAcceleratedDecoder(config, video_decoder); if (!SUCCEEDED(hr)) { NotifyError("Failed to get device context"); @@ -395,14 +379,6 @@ void D3D11VideoDecoder::Initialize(const VideoDecoderConfig& config, config.GetHumanReadableCodecName()); } - // |cdm_context| could be null for clear playback. - // TODO(liberato): On re-init, should this still happen? - if (cdm_context) { - new_key_callback_registration_ = - cdm_context->RegisterEventCB(base::BindRepeating( - &D3D11VideoDecoder::OnCdmContextEvent, weak_factory_.GetWeakPtr())); - } - auto impl_init_cb = base::BindOnce(&D3D11VideoDecoder::OnGpuInitComplete, weak_factory_.GetWeakPtr()); @@ -412,25 +388,14 @@ void D3D11VideoDecoder::Initialize(const VideoDecoderConfig& config, AddLifetimeProgressionStage(D3D11LifetimeProgression::kInitializeSucceeded); - // Initialize the gpu side. We wait until everything else is initialized, - // since we allow it to call us back re-entrantly to reduce latency. Note - // that if we're not on the same thread, then we should probably post the - // call earlier, since re-entrancy won't be an issue. - if (impl_task_runner_->RunsTasksInCurrentSequence()) { - impl_->Initialize(std::move(impl_init_cb), - std::move(get_picture_buffer_cb)); - return; - } - + // Initialize the gpu side. It would be nice if we could ask SB<> to elide + // the post if we're already on that thread, but it can't. // Bind our own init / output cb that hop to this thread, so we don't call // the originals on some other thread. // Important but subtle note: base::Bind will copy |config_| since it's a // const ref. - impl_task_runner_->PostTask( - FROM_HERE, - base::BindOnce(&D3D11VideoDecoderImpl::Initialize, impl_weak_, - BindToCurrentLoop(std::move(impl_init_cb)), - BindToCurrentLoop(std::move(get_picture_buffer_cb)))); + impl_.Post(FROM_HERE, &D3D11VideoDecoderImpl::Initialize, + BindToCurrentLoop(std::move(impl_init_cb))); } void D3D11VideoDecoder::AddLifetimeProgressionStage( @@ -455,7 +420,9 @@ void D3D11VideoDecoder::ReceivePictureBufferFromClient( DoDecode(); } -void D3D11VideoDecoder::OnGpuInitComplete(bool success) { +void D3D11VideoDecoder::OnGpuInitComplete( + bool success, + D3D11VideoDecoderImpl::ReleaseMailboxCB release_mailbox_cb) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); TRACE_EVENT0("gpu", "D3D11VideoDecoder::OnGpuInitComplete"); @@ -472,6 +439,8 @@ void D3D11VideoDecoder::OnGpuInitComplete(bool success) { return; } + release_mailbox_cb_ = std::move(release_mailbox_cb); + state_ = State::kRunning; std::move(init_cb_).Run(OkStatus()); } @@ -595,9 +564,8 @@ void D3D11VideoDecoder::DoDecode() { } CreatePictureBuffers(); } else if (result == media::AcceleratedVideoDecoder::kTryAgain) { - state_ = State::kWaitingForNewKey; - waiting_cb_.Run(WaitingReason::kNoDecryptionKey); - // Another DoDecode() task would be posted in OnCdmContextEvent(). + LOG(ERROR) << "Try again is not supported"; + NotifyError("Try again is not supported"); return; } else { LOG(ERROR) << "VDA Error " << result; @@ -627,21 +595,6 @@ void D3D11VideoDecoder::Reset(base::OnceClosure closure) { // TODO(liberato): how do we signal an error? accelerated_video_decoder_->Reset(); - if (state_ == State::kWaitingForReset && config_.is_encrypted()) { - // On a hardware context loss event, a new swap chain has to be created (in - // the compositor). By clearing the picture buffers, next DoDecode() call - // will create a new texture. This makes the compositor to create a new swap - // chain. - // More detailed explanation at crbug.com/858286 - picture_buffers_.clear(); - } - - // Transition out of kWaitingForNewKey since the new buffer could be clear or - // have a different key ID. Transition out of kWaitingForReset since reset - // just happened. - if (state_ == State::kWaitingForNewKey || state_ == State::kWaitingForReset) - state_ = State::kRunning; - std::move(closure).Run(); } @@ -682,6 +635,19 @@ void D3D11VideoDecoder::CreatePictureBuffers() { return; } + HDRMetadata stream_metadata; + if (config_.hdr_metadata()) + stream_metadata = *config_.hdr_metadata(); + // else leave |stream_metadata| default-initialized. We might use it anyway. + + base::Optional<DXGI_HDR_METADATA_HDR10> display_metadata; + if (decoder_configurator_->TextureFormat() == DXGI_FORMAT_P010) { + // For HDR formats, try to get the display metadata. This may fail, which + // is okay. We'll just skip sending the metadata. + DisplayHelper display_helper(device_); + display_metadata = display_helper.GetDisplayMetadata(); + } + // Drop any old pictures. for (auto& buffer : picture_buffers_) DCHECK(!buffer->in_picture_use()); @@ -696,14 +662,28 @@ void D3D11VideoDecoder::CreatePictureBuffers() { return; } - picture_buffers_.push_back( - new D3D11PictureBuffer(in_texture, std::move(tex_wrapper), size, i)); - if (!picture_buffers_[i]->Init(get_helper_cb_, video_device_, - decoder_configurator_->DecoderGuid(), - media_log_->Clone())) { + picture_buffers_.push_back(new D3D11PictureBuffer( + decoder_task_runner_, in_texture, std::move(tex_wrapper), size, i)); + if (!picture_buffers_[i]->Init( + gpu_task_runner_, get_helper_cb_, video_device_, + decoder_configurator_->DecoderGuid(), media_log_->Clone())) { NotifyError("Unable to allocate PictureBuffer"); return; } + + // If we have display metadata, then tell the processor. Note that the + // order of these calls is important, and we must set the display metadata + // if we set the stream metadata, else it can crash on some AMD cards. + if (display_metadata) { + if (config_.hdr_metadata() || + gpu_workarounds_.use_empty_video_hdr_metadata) { + // It's okay if this has an empty-initialized metadata. + picture_buffers_[i]->texture_wrapper()->SetStreamHDRMetadata( + stream_metadata); + } + picture_buffers_[i]->texture_wrapper()->SetDisplayHDRMetadata( + *display_metadata); + } } } @@ -760,10 +740,16 @@ bool D3D11VideoDecoder::OutputResult(const CodecPicture* picture, return false; } - // TODO(liberato): bind this to the gpu main thread. - frame->SetReleaseMailboxCB(media::BindToCurrentLoop( - base::BindOnce(&D3D11VideoDecoderImpl::OnMailboxReleased, impl_weak_, - scoped_refptr<D3D11PictureBuffer>(picture_buffer)))); + // Remember that this will likely thread-hop to the GPU main thread. Note + // that |picture_buffer| will delete on sequence, so it's okay even if + // |wait_complete_cb| doesn't ever run. + auto wait_complete_cb = BindToCurrentLoop( + base::BindOnce(&D3D11VideoDecoder::ReceivePictureBufferFromClient, + weak_factory_.GetWeakPtr(), + scoped_refptr<D3D11PictureBuffer>(picture_buffer))); + frame->SetReleaseMailboxCB( + base::BindOnce(release_mailbox_cb_, std::move(wait_complete_cb))); + frame->metadata()->SetBoolean(VideoFrameMetadata::POWER_EFFICIENT, true); // For NV12, overlay is allowed by default. If the decoder is going to support // non-NV12 textures, then this may have to be conditionally set. Also note @@ -782,47 +768,11 @@ bool D3D11VideoDecoder::OutputResult(const CodecPicture* picture, frame->metadata()->SetBoolean(VideoFrameMetadata::ALLOW_OVERLAY, allow_overlay); - if (config_.is_encrypted()) { - frame->metadata()->SetBoolean(VideoFrameMetadata::PROTECTED_VIDEO, true); - frame->metadata()->SetBoolean(VideoFrameMetadata::HW_PROTECTED, true); - } - frame->set_color_space(output_color_space); output_cb_.Run(frame); return true; } -void D3D11VideoDecoder::OnCdmContextEvent(CdmContext::Event event) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DVLOG(1) << __func__ << ": event = " << static_cast<int>(event); - - if (state_ == State::kInitializing || state_ == State::kError) { - DVLOG(1) << "Do nothing in " << static_cast<int>(state_) << " state."; - return; - } - - switch (event) { - case CdmContext::Event::kHasAdditionalUsableKey: - // Note that this event may happen before DoDecode() because the key - // acquisition stack runs independently of the media decoding stack. - // So if this isn't in kWaitingForNewKey state no "resuming" is - // required therefore no special action taken here. - if (state_ != State::kWaitingForNewKey) - return; - - state_ = State::kRunning; - base::ThreadTaskRunnerHandle::Get()->PostTask( - FROM_HERE, base::BindOnce(&D3D11VideoDecoder::DoDecode, - weak_factory_.GetWeakPtr())); - return; - - case CdmContext::Event::kHardwareContextLost: - state_ = State::kWaitingForReset; - waiting_cb_.Run(WaitingReason::kDecoderStateLost); - return; - } -} - // TODO(tmathmeyer) eventually have this take a Status and pass it through // to each of the callbacks. void D3D11VideoDecoder::NotifyError(const char* reason) { @@ -926,10 +876,6 @@ D3D11VideoDecoder::GetSupportedVideoDecoderConfigs( return {}; } - const bool allow_encrypted = - (usable_feature_level > D3D_FEATURE_LEVEL_11_0) && - base::FeatureList::IsEnabled(kHardwareSecureDecryption); - std::vector<SupportedVideoDecoderConfig> configs; // VP9 has no default resolutions since it may not even be supported. ResolutionPair max_h264_resolutions(gfx::Size(1920, 1088), gfx::Size()); @@ -952,7 +898,7 @@ D3D11VideoDecoder::GetSupportedVideoDecoderConfigs( 1), // profile_max min_resolution, // coded_size_min max_h264_resolutions.first, // coded_size_max - allow_encrypted, // allow_encrypted + false, // allow_encrypted false)); // require_encrypted configs.push_back(SupportedVideoDecoderConfig( static_cast<VideoCodecProfile>(H264PROFILE_HIGH10PROFILE + @@ -960,7 +906,7 @@ D3D11VideoDecoder::GetSupportedVideoDecoderConfigs( H264PROFILE_MAX, // profile_max min_resolution, // coded_size_min max_h264_resolutions.first, // coded_size_max - allow_encrypted, // allow_encrypted + false, // allow_encrypted false)); // require_encrypted // portrait @@ -970,7 +916,7 @@ D3D11VideoDecoder::GetSupportedVideoDecoderConfigs( 1), // profile_max min_resolution, // coded_size_min max_h264_resolutions.second, // coded_size_max - allow_encrypted, // allow_encrypted + false, // allow_encrypted false)); // require_encrypted configs.push_back(SupportedVideoDecoderConfig( static_cast<VideoCodecProfile>(H264PROFILE_HIGH10PROFILE + @@ -978,7 +924,7 @@ D3D11VideoDecoder::GetSupportedVideoDecoderConfigs( H264PROFILE_MAX, // profile_max min_resolution, // coded_size_min max_h264_resolutions.second, // coded_size_max - allow_encrypted, // allow_encrypted + false, // allow_encrypted false)); // require_encrypted } @@ -991,7 +937,7 @@ D3D11VideoDecoder::GetSupportedVideoDecoderConfigs( VP9PROFILE_PROFILE0, // profile_max min_resolution, // coded_size_min max_vp9_profile0_resolutions.first, // coded_size_max - allow_encrypted, // allow_encrypted + false, // allow_encrypted false)); // require_encrypted // portrait configs.push_back(SupportedVideoDecoderConfig( @@ -999,7 +945,7 @@ D3D11VideoDecoder::GetSupportedVideoDecoderConfigs( VP9PROFILE_PROFILE0, // profile_max min_resolution, // coded_size_min max_vp9_profile0_resolutions.second, // coded_size_max - allow_encrypted, // allow_encrypted + false, // allow_encrypted false)); // require_encrypted } @@ -1011,7 +957,7 @@ D3D11VideoDecoder::GetSupportedVideoDecoderConfigs( VP9PROFILE_PROFILE2, // profile_max min_resolution, // coded_size_min max_vp9_profile2_resolutions.first, // coded_size_max - allow_encrypted, // allow_encrypted + false, // allow_encrypted false)); // require_encrypted // portrait configs.push_back(SupportedVideoDecoderConfig( @@ -1019,12 +965,12 @@ D3D11VideoDecoder::GetSupportedVideoDecoderConfigs( VP9PROFILE_PROFILE2, // profile_max min_resolution, // coded_size_min max_vp9_profile2_resolutions.second, // coded_size_max - allow_encrypted, // allow_encrypted + false, // allow_encrypted false)); // require_encrypted } } - // TODO(liberato): Should we separate out h264, vp9, and encrypted? + // TODO(liberato): Should we separate out h264 and vp9? UMA_HISTOGRAM_ENUMERATION(uma_name, NotSupportedReason::kVideoIsSupported); return configs; diff --git a/chromium/media/gpu/windows/d3d11_video_decoder.h b/chromium/media/gpu/windows/d3d11_video_decoder.h index 94c609fd976..d9ba26ab254 100644 --- a/chromium/media/gpu/windows/d3d11_video_decoder.h +++ b/chromium/media/gpu/windows/d3d11_video_decoder.h @@ -15,6 +15,7 @@ #include "base/sequence_checker.h" #include "base/sequenced_task_runner.h" #include "base/single_thread_task_runner.h" +#include "base/threading/sequence_bound.h" #include "gpu/config/gpu_driver_bug_workarounds.h" #include "gpu/config/gpu_preferences.h" #include "media/base/callback_registry.h" @@ -27,6 +28,7 @@ #include "media/gpu/windows/d3d11_h264_accelerator.h" #include "media/gpu/windows/d3d11_texture_selector.h" #include "media/gpu/windows/d3d11_video_decoder_client.h" +#include "media/gpu/windows/d3d11_video_decoder_impl.h" #include "media/gpu/windows/d3d11_vp9_accelerator.h" #include "media/video/supported_video_decoder_config.h" @@ -37,7 +39,6 @@ class CommandBufferStub; namespace media { class D3D11PictureBuffer; -class D3D11VideoDecoderImpl; class D3D11VideoDecoderTest; class MediaLog; @@ -63,7 +64,8 @@ class MEDIA_GPU_EXPORT D3D11VideoDecoder : public VideoDecoder, const gpu::GpuDriverBugWorkarounds& gpu_workarounds, base::RepeatingCallback<gpu::CommandBufferStub*()> get_stub_cb, GetD3D11DeviceCB get_d3d11_device_cb, - SupportedConfigs supported_configs); + SupportedConfigs supported_configs, + bool is_hdr_supported); // VideoDecoder implementation: std::string GetDisplayName() const override; @@ -110,24 +112,26 @@ class MEDIA_GPU_EXPORT D3D11VideoDecoder : public VideoDecoder, std::unique_ptr<MediaLog> media_log, const gpu::GpuPreferences& gpu_preferences, const gpu::GpuDriverBugWorkarounds& gpu_workarounds, - std::unique_ptr<D3D11VideoDecoderImpl> impl, + base::SequenceBound<D3D11VideoDecoderImpl> impl, base::RepeatingCallback<scoped_refptr<CommandBufferHelper>()> get_helper_cb, GetD3D11DeviceCB get_d3d11_device_cb, - SupportedConfigs supported_configs); + SupportedConfigs supported_configs, + bool is_hdr_supported); // Receive |buffer|, that is now unused by the client. void ReceivePictureBufferFromClient(scoped_refptr<D3D11PictureBuffer> buffer); // Called when the gpu side of initialization is complete. - void OnGpuInitComplete(bool success); + void OnGpuInitComplete( + bool success, + D3D11VideoDecoderImpl::ReleaseMailboxCB release_mailbox_cb); // Run the decoder loop. void DoDecode(); // instantiate |accelerated_video_decoder_| based on the video profile HRESULT InitializeAcceleratedDecoder(const VideoDecoderConfig& config, - CdmProxyContext* proxy_context, ComD3D11VideoDecoder video_decoder); // Query the video device for a specific decoder ID. @@ -196,38 +200,22 @@ class MEDIA_GPU_EXPORT D3D11VideoDecoder : public VideoDecoder, // creation is pending. kRunning, - // The decoder cannot make progress because it doesn't have the key to - // decrypt the buffer. Waiting for a new key to be available. - // This should only be transitioned from kRunning, and should only - // transition to kRunning or kWaitingForReset. - kWaitingForNewKey, - - // The decoder cannot make progress because it's waiting for a Reset(). This - // could happen as a result of CdmContext hardware context loss. This should - // only be transitioned from kRunning or kWaitingForNewKey, and should only - // transition to kRunning. - kWaitingForReset, - // A fatal error occurred. A terminal state. kError, }; - // Callback to notify that new CdmContext event is available. - void OnCdmContextEvent(CdmContext::Event event); - // Enter the kError state. This will fail any pending |init_cb_| and / or // pending decode as well. void NotifyError(const char* reason); - // The implementation, which we trampoline to the impl thread. - // This must be freed on the impl thread. - std::unique_ptr<D3D11VideoDecoderImpl> impl_; + // The implementation, which lives on the GPU main thread. + base::SequenceBound<D3D11VideoDecoderImpl> impl_; - // Weak ptr to |impl_|, which we use for callbacks. - base::WeakPtr<D3D11VideoDecoderImpl> impl_weak_; + // GPU main thread task runner. + scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner_; - // Task runner for |impl_|. This must be the GPU main thread. - scoped_refptr<base::SequencedTaskRunner> impl_task_runner_; + // Task runner on which |this| lives. + scoped_refptr<base::SequencedTaskRunner> decoder_task_runner_; // Set in initialize, and used to determine reinitializations. bool already_initialized_; @@ -239,13 +227,18 @@ class MEDIA_GPU_EXPORT D3D11VideoDecoder : public VideoDecoder, VideoDecoderConfig config_; InitCB init_cb_; OutputCB output_cb_; - WaitingCB waiting_cb_; + + // Callback to be used as a release CB for VideoFrames. Be sure to + // BindToCurrentLoop the closure that it takes. + D3D11VideoDecoderImpl::ReleaseMailboxCB release_mailbox_cb_; // Right now, this is used both for the video decoder and for display. In // the future, this should only be for the video decoder. We should use // the ANGLE device for display (plus texture sharing, if needed). GetD3D11DeviceCB get_d3d11_device_cb_; + // These may be accessed from |decoder_task_runner_|, since the angle device + // is in multi-threaded mode. Just be sure not to set any global state. ComD3D11Device device_; ComD3D11DeviceContext device_context_; ComD3D11VideoDevice video_device_; @@ -265,9 +258,6 @@ class MEDIA_GPU_EXPORT D3D11VideoDecoder : public VideoDecoder, DecodeCB current_decode_cb_; base::TimeDelta current_timestamp_; - // Callback registration to keep the new key callback registered. - std::unique_ptr<CallbackRegistration> new_key_callback_registration_; - // Must be called on the gpu main thread. So, don't call it from here, // since we don't know what thread we're on. base::RepeatingCallback<gpu::CommandBufferStub*()> get_stub_cb_; @@ -290,6 +280,9 @@ class MEDIA_GPU_EXPORT D3D11VideoDecoder : public VideoDecoder, SupportedConfigs supported_configs_; + // Should we assume that we're outputting to an HDR display? + bool is_hdr_supported_; + base::WeakPtrFactory<D3D11VideoDecoder> weak_factory_{this}; DISALLOW_COPY_AND_ASSIGN(D3D11VideoDecoder); diff --git a/chromium/media/gpu/windows/d3d11_video_decoder_impl.cc b/chromium/media/gpu/windows/d3d11_video_decoder_impl.cc index 89c88e56c6f..719e7b516fe 100644 --- a/chromium/media/gpu/windows/d3d11_video_decoder_impl.cc +++ b/chromium/media/gpu/windows/d3d11_video_decoder_impl.cc @@ -8,6 +8,7 @@ #include "gpu/command_buffer/common/sync_token.h" #include "gpu/command_buffer/service/scheduler.h" #include "gpu/ipc/service/gpu_channel.h" +#include "media/base/bind_to_current_loop.h" #include "media/base/media_log.h" #include "media/gpu/windows/d3d11_picture_buffer.h" @@ -25,16 +26,12 @@ D3D11VideoDecoderImpl::~D3D11VideoDecoderImpl() { DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); } -void D3D11VideoDecoderImpl::Initialize( - InitCB init_cb, - ReturnPictureBufferCB return_picture_buffer_cb) { +void D3D11VideoDecoderImpl::Initialize(InitCB init_cb) { DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); - return_picture_buffer_cb_ = std::move(return_picture_buffer_cb); - // If have a helper, then we're as initialized as we need to be. if (helper_) { - std::move(init_cb).Run(true); + std::move(init_cb).Run(true, release_mailbox_cb_); return; } helper_ = get_helper_cb_.Run(); @@ -46,31 +43,36 @@ void D3D11VideoDecoderImpl::Initialize( if (media_log_) MEDIA_LOG(ERROR, media_log_) << reason; - std::move(init_cb).Run(false); + std::move(init_cb).Run(false, ReleaseMailboxCB()); return; } - std::move(init_cb).Run(true); + release_mailbox_cb_ = BindToCurrentLoop(base::BindRepeating( + &D3D11VideoDecoderImpl::OnMailboxReleased, GetWeakPtr())); + + std::move(init_cb).Run(true, release_mailbox_cb_); } void D3D11VideoDecoderImpl::OnMailboxReleased( - scoped_refptr<D3D11PictureBuffer> buffer, + base::OnceClosure wait_complete_cb, const gpu::SyncToken& sync_token) { DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); - if (!helper_) + if (!helper_) { + std::move(wait_complete_cb).Run(); return; + } helper_->WaitForSyncToken( sync_token, base::BindOnce(&D3D11VideoDecoderImpl::OnSyncTokenReleased, - GetWeakPtr(), std::move(buffer))); + GetWeakPtr(), std::move(wait_complete_cb))); } void D3D11VideoDecoderImpl::OnSyncTokenReleased( - scoped_refptr<D3D11PictureBuffer> buffer) { + base::OnceClosure wait_complete_cb) { DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); - return_picture_buffer_cb_.Run(std::move(buffer)); + std::move(wait_complete_cb).Run(); } base::WeakPtr<D3D11VideoDecoderImpl> D3D11VideoDecoderImpl::GetWeakPtr() { diff --git a/chromium/media/gpu/windows/d3d11_video_decoder_impl.h b/chromium/media/gpu/windows/d3d11_video_decoder_impl.h index c6549268c2c..770d81cf88f 100644 --- a/chromium/media/gpu/windows/d3d11_video_decoder_impl.h +++ b/chromium/media/gpu/windows/d3d11_video_decoder_impl.h @@ -34,6 +34,10 @@ class D3D11PictureBuffer; // TODO(liberato): Rename this class as a follow-on to this refactor. class MEDIA_GPU_EXPORT D3D11VideoDecoderImpl { public: + // Callback to us to wait for a sync token, then call a closure. + using ReleaseMailboxCB = + base::RepeatingCallback<void(base::OnceClosure, const gpu::SyncToken&)>; + // May be constructed on any thread. explicit D3D11VideoDecoderImpl( std::unique_ptr<MediaLog> media_log, @@ -41,20 +45,17 @@ class MEDIA_GPU_EXPORT D3D11VideoDecoderImpl { get_helper_cb); virtual ~D3D11VideoDecoderImpl(); - using InitCB = base::OnceCallback<void(bool success)>; + using InitCB = base::OnceCallback<void(bool success, ReleaseMailboxCB)>; // Returns a picture buffer that's no longer in use by the client. using ReturnPictureBufferCB = base::RepeatingCallback<void(scoped_refptr<D3D11PictureBuffer>)>; - // We will call back |init_cb| with the init status. |try_decoding_cb| should - // try to re-start decoding. We'll call this when we do something that might - // allow decoding to make progress, such as reclaim a picture buffer. - virtual void Initialize(InitCB init_cb, - ReturnPictureBufferCB return_picture_buffer_cb); + // We will call back |init_cb| with the init status. + virtual void Initialize(InitCB init_cb); - // Called when the VideoFrame that uses |buffer| is freed. - void OnMailboxReleased(scoped_refptr<D3D11PictureBuffer> buffer, + // Called to wait on |sync_token|, and call |wait_complete_cb| when done. + void OnMailboxReleased(base::OnceClosure wait_complete_cb, const gpu::SyncToken& sync_token); // Return a weak ptr, since D3D11VideoDecoder constructs callbacks for us. @@ -62,12 +63,15 @@ class MEDIA_GPU_EXPORT D3D11VideoDecoderImpl { base::WeakPtr<D3D11VideoDecoderImpl> GetWeakPtr(); private: - void OnSyncTokenReleased(scoped_refptr<D3D11PictureBuffer> buffer); + void OnSyncTokenReleased(base::OnceClosure); std::unique_ptr<MediaLog> media_log_; + // Cached copy of the callback to OnMailboxReleased. + ReleaseMailboxCB release_mailbox_cb_; + // Called when we get a picture buffer back from the client. - ReturnPictureBufferCB return_picture_buffer_cb_; + // ReturnPictureBufferCB return_picture_buffer_cb_; base::RepeatingCallback<scoped_refptr<CommandBufferHelper>()> get_helper_cb_; scoped_refptr<CommandBufferHelper> helper_; diff --git a/chromium/media/gpu/windows/d3d11_video_decoder_unittest.cc b/chromium/media/gpu/windows/d3d11_video_decoder_unittest.cc index 1eb58153b01..91908445262 100644 --- a/chromium/media/gpu/windows/d3d11_video_decoder_unittest.cc +++ b/chromium/media/gpu/windows/d3d11_video_decoder_unittest.cc @@ -40,16 +40,15 @@ namespace media { class MockD3D11VideoDecoderImpl : public D3D11VideoDecoderImpl { public: - MockD3D11VideoDecoderImpl() + MockD3D11VideoDecoderImpl(MockD3D11VideoDecoderImpl** thiz) : D3D11VideoDecoderImpl( nullptr, - base::RepeatingCallback<scoped_refptr<CommandBufferHelper>()>()) {} - - void Initialize(InitCB init_cb, - ReturnPictureBufferCB return_picture_buffer_cb) override { - MockInitialize(); + base::RepeatingCallback<scoped_refptr<CommandBufferHelper>()>()) { + *thiz = this; } + void Initialize(InitCB init_cb) override { MockInitialize(); } + MOCK_METHOD0(MockInitialize, void()); }; @@ -65,6 +64,7 @@ class D3D11VideoDecoderTest : public ::testing::Test { gpu_preferences_.enable_zero_copy_dxgi_video = true; gpu_preferences_.use_passthrough_cmd_decoder = false; gpu_workarounds_.disable_dxgi_zero_copy_video = false; + gpu_task_runner_ = task_environment_.GetMainThreadTaskRunner(); // Create a mock D3D11 device that supports 11.0. Note that if you change // this, then you probably also want VideoDevice1 and friends, below. @@ -189,11 +189,9 @@ class D3D11VideoDecoderTest : public ::testing::Test { supported_configs = D3D11VideoDecoder::GetSupportedVideoDecoderConfigs( gpu_preferences_, gpu_workarounds_, get_device_cb); } - std::unique_ptr<MockD3D11VideoDecoderImpl> impl = - std::make_unique<NiceMock<MockD3D11VideoDecoderImpl>>(); - impl_ = impl.get(); - - gpu_task_runner_ = base::ThreadTaskRunnerHandle::Get(); + base::SequenceBound<MockD3D11VideoDecoderImpl> impl(gpu_task_runner_, + &impl_); + task_environment_.RunUntilIdle(); // We store it in a std::unique_ptr<VideoDecoder> so that the default // deleter works. The dtor is protected. @@ -202,7 +200,7 @@ class D3D11VideoDecoderTest : public ::testing::Test { gpu_task_runner_, std::make_unique<NullMediaLog>(), gpu_preferences_, gpu_workarounds_, std::move(impl), base::RepeatingCallback<scoped_refptr<CommandBufferHelper>()>(), - get_device_cb, *supported_configs)); + get_device_cb, *supported_configs, is_hdr_supported_)); } void InitializeDecoder(const VideoDecoderConfig& config, @@ -218,8 +216,8 @@ class D3D11VideoDecoderTest : public ::testing::Test { } decoder_->Initialize( config, low_delay, cdm_context, - base::BindRepeating(&D3D11VideoDecoderTest::CheckExpectedStatus, - base::Unretained(this), expectation), + base::BindOnce(&D3D11VideoDecoderTest::CheckExpectedStatus, + base::Unretained(this), expectation), base::DoNothing(), base::DoNothing()); base::RunLoop().RunUntilIdle(); } @@ -231,7 +229,7 @@ class D3D11VideoDecoderTest : public ::testing::Test { MOCK_METHOD1(MockInitCB, void(Status)); - base::test::TaskEnvironment env_; + base::test::TaskEnvironment task_environment_; scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner_; @@ -249,6 +247,9 @@ class D3D11VideoDecoderTest : public ::testing::Test { Microsoft::WRL::ComPtr<DXGIDeviceMock> mock_dxgi_device_; Microsoft::WRL::ComPtr<DXGIAdapterMock> mock_dxgi_adapter_; + // Used by CreateDecoder() to tell D3D11VideoDecoder about HDR support. + bool is_hdr_supported_ = true; + DXGI_ADAPTER_DESC mock_adapter_desc_; base::Optional<base::test::ScopedFeatureList> scoped_feature_list_; diff --git a/chromium/media/gpu/windows/d3d11_video_processor_proxy.cc b/chromium/media/gpu/windows/d3d11_video_processor_proxy.cc index 74341290230..2799986e5a9 100644 --- a/chromium/media/gpu/windows/d3d11_video_processor_proxy.cc +++ b/chromium/media/gpu/windows/d3d11_video_processor_proxy.cc @@ -98,6 +98,29 @@ void VideoProcessorProxy::SetOutputColorSpace( } } +void VideoProcessorProxy::SetStreamHDRMetadata( + const DXGI_HDR_METADATA_HDR10& stream_metadata) { + ComD3D11VideoContext2 video_context2; + if (FAILED(video_context_.As(&video_context2))) + return; + + // TODO: we shouldn't do this unless we also set the display metadata. + video_context2->VideoProcessorSetOutputHDRMetaData( + video_processor_.Get(), DXGI_HDR_METADATA_TYPE_HDR10, + sizeof(stream_metadata), &stream_metadata); +} + +void VideoProcessorProxy::SetDisplayHDRMetadata( + const DXGI_HDR_METADATA_HDR10& display_metadata) { + ComD3D11VideoContext2 video_context2; + if (FAILED(video_context_.As(&video_context2))) + return; + + video_context2->VideoProcessorSetOutputHDRMetaData( + video_processor_.Get(), DXGI_HDR_METADATA_TYPE_HDR10, + sizeof(display_metadata), &display_metadata); +} + HRESULT VideoProcessorProxy::VideoProcessorBlt( ID3D11VideoProcessorOutputView* output_view, UINT output_frameno, diff --git a/chromium/media/gpu/windows/d3d11_video_processor_proxy.h b/chromium/media/gpu/windows/d3d11_video_processor_proxy.h index a6eece036f3..1e5679a2b74 100644 --- a/chromium/media/gpu/windows/d3d11_video_processor_proxy.h +++ b/chromium/media/gpu/windows/d3d11_video_processor_proxy.h @@ -9,6 +9,7 @@ #include <wrl/client.h> #include <cstdint> +#include "media/base/hdr_metadata.h" #include "media/gpu/media_gpu_export.h" #include "media/gpu/windows/d3d11_com_defs.h" #include "ui/gfx/color_space.h" @@ -43,6 +44,13 @@ class MEDIA_GPU_EXPORT VideoProcessorProxy { // Configure the output color space on the video context. virtual void SetOutputColorSpace(const gfx::ColorSpace& color_space); + // Set the stream / display metadata. Optional, and may silently do nothing + // if it's not supported. + virtual void SetStreamHDRMetadata( + const DXGI_HDR_METADATA_HDR10& stream_metadata); + virtual void SetDisplayHDRMetadata( + const DXGI_HDR_METADATA_HDR10& display_metadata); + virtual HRESULT VideoProcessorBlt(ID3D11VideoProcessorOutputView* output_view, UINT output_frameno, UINT stream_count, diff --git a/chromium/media/gpu/windows/d3d11_vp9_accelerator.cc b/chromium/media/gpu/windows/d3d11_vp9_accelerator.cc index 267f09a4d17..eeec7896bde 100644 --- a/chromium/media/gpu/windows/d3d11_vp9_accelerator.cc +++ b/chromium/media/gpu/windows/d3d11_vp9_accelerator.cc @@ -9,7 +9,6 @@ #include <utility> #include "base/memory/ptr_util.h" -#include "media/cdm/cdm_proxy_context.h" #include "media/gpu/windows/d3d11_vp9_picture.h" namespace media { @@ -37,21 +36,17 @@ CreateSubsampleMappingBlock(const std::vector<SubsampleEntry>& from) { D3D11VP9Accelerator::D3D11VP9Accelerator( D3D11VideoDecoderClient* client, MediaLog* media_log, - CdmProxyContext* cdm_proxy_context, ComD3D11VideoDecoder video_decoder, ComD3D11VideoDevice video_device, std::unique_ptr<VideoContextWrapper> video_context) : client_(client), media_log_(media_log), - cdm_proxy_context_(cdm_proxy_context), status_feedback_(0), video_decoder_(std::move(video_decoder)), video_device_(std::move(video_device)), video_context_(std::move(video_context)) { DCHECK(client); DCHECK(media_log_); - // |cdm_proxy_context_| is non-null for encrypted content but can be null for - // clear content. } D3D11VP9Accelerator::~D3D11VP9Accelerator() {} @@ -70,33 +65,18 @@ scoped_refptr<VP9Picture> D3D11VP9Accelerator::CreateVP9Picture() { } bool D3D11VP9Accelerator::BeginFrame(const D3D11VP9Picture& pic) { - // This |decrypt_context| has to be outside the if block because pKeyInfo in - // D3D11_VIDEO_DECODER_BEGIN_FRAME_CRYPTO_SESSION is a pointer (to a GUID). - base::Optional<CdmProxyContext::D3D11DecryptContext> decrypt_context; - std::unique_ptr<D3D11_VIDEO_DECODER_BEGIN_FRAME_CRYPTO_SESSION> content_key; - if (const DecryptConfig* config = pic.decrypt_config()) { - DCHECK(cdm_proxy_context_) << "No CdmProxyContext but picture is encrypted"; - decrypt_context = cdm_proxy_context_->GetD3D11DecryptContext( - CdmProxy::KeyType::kDecryptAndDecode, config->key_id()); - if (!decrypt_context) { - RecordFailure("crypto_config", - "Cannot find the decrypt context for the frame."); - return false; // TODO(crbug.com/894573): support kTryAgain. - } - - content_key = - std::make_unique<D3D11_VIDEO_DECODER_BEGIN_FRAME_CRYPTO_SESSION>(); - content_key->pCryptoSession = decrypt_context->crypto_session; - content_key->pBlob = const_cast<void*>(decrypt_context->key_blob); - content_key->BlobSize = decrypt_context->key_blob_size; - content_key->pKeyInfoId = &decrypt_context->key_info_guid; + const bool is_encrypted = pic.decrypt_config(); + if (is_encrypted) { + RecordFailure("crypto_config", + "Cannot find the decrypt context for the frame."); + return false; } HRESULT hr; do { hr = video_context_->DecoderBeginFrame( - video_decoder_.Get(), pic.picture_buffer()->output_view().Get(), - content_key ? sizeof(*content_key) : 0, content_key.get()); + video_decoder_.Get(), pic.picture_buffer()->output_view().Get(), 0, + nullptr); } while (hr == E_PENDING || hr == D3DERR_WASSTILLDRAWING); if (FAILED(hr)) { @@ -121,7 +101,6 @@ void D3D11VP9Accelerator::CopyFrameParams(const D3D11VP9Picture& pic, COPY_PARAM(frame_context_idx); COPY_PARAM(reset_frame_context); COPY_PARAM(allow_high_precision_mv); - COPY_PARAM(refresh_frame_context); COPY_PARAM(frame_parallel_decoding_mode); COPY_PARAM(intra_only); COPY_PARAM(frame_context_idx); @@ -145,6 +124,18 @@ void D3D11VP9Accelerator::CopyFrameParams(const D3D11VP9Picture& pic, SET_PARAM(log2_tile_rows, tile_rows_log2); #undef COPY_PARAM #undef SET_PARAM + + // This is taken, approximately, from libvpx. + gfx::Size this_frame_size(pic.frame_hdr->frame_width, + pic.frame_hdr->frame_height); + pic_params->use_prev_in_find_mv_refs = last_frame_size_ == this_frame_size && + !pic.frame_hdr->error_resilient_mode && + !pic.frame_hdr->intra_only && + last_show_frame_; + + // TODO(liberato): So, uh, do we ever need to reset this? + last_frame_size_ = this_frame_size; + last_show_frame_ = pic.frame_hdr->show_frame; } void D3D11VP9Accelerator::CopyReferenceFrames( diff --git a/chromium/media/gpu/windows/d3d11_vp9_accelerator.h b/chromium/media/gpu/windows/d3d11_vp9_accelerator.h index e31d7a33d9a..dc262d68d26 100644 --- a/chromium/media/gpu/windows/d3d11_vp9_accelerator.h +++ b/chromium/media/gpu/windows/d3d11_vp9_accelerator.h @@ -19,13 +19,11 @@ #include "media/gpu/windows/d3d11_vp9_picture.h" namespace media { -class CdmProxyContext; class D3D11VP9Accelerator : public VP9Decoder::VP9Accelerator { public: D3D11VP9Accelerator(D3D11VideoDecoderClient* client, MediaLog* media_log, - CdmProxyContext* cdm_proxy_context, ComD3D11VideoDecoder video_decoder, ComD3D11VideoDevice video_device, std::unique_ptr<VideoContextWrapper> video_context); @@ -73,12 +71,15 @@ class D3D11VP9Accelerator : public VP9Decoder::VP9Accelerator { D3D11VideoDecoderClient* client_; MediaLog* const media_log_; - CdmProxyContext* cdm_proxy_context_; UINT status_feedback_; ComD3D11VideoDecoder video_decoder_; ComD3D11VideoDevice video_device_; std::unique_ptr<VideoContextWrapper> video_context_; + // Used to set |use_prev_in_find_mv_refs| properly. + gfx::Size last_frame_size_; + bool last_show_frame_ = false; + DISALLOW_COPY_AND_ASSIGN(D3D11VP9Accelerator); }; diff --git a/chromium/media/gpu/windows/display_helper.h b/chromium/media/gpu/windows/display_helper.h index baf45eec5f0..2dc77aa1174 100644 --- a/chromium/media/gpu/windows/display_helper.h +++ b/chromium/media/gpu/windows/display_helper.h @@ -5,6 +5,7 @@ #ifndef MEDIA_GPU_WINDOWS_DISPLAY_HELPER_H_ #define MEDIA_GPU_WINDOWS_DISPLAY_HELPER_H_ +#include "base/macros.h" #include "base/optional.h" #include "media/base/hdr_metadata.h" #include "media/gpu/media_gpu_export.h" diff --git a/chromium/media/gpu/windows/dxva_video_decode_accelerator_win.cc b/chromium/media/gpu/windows/dxva_video_decode_accelerator_win.cc index 30fc6d3977b..ff451f0bb17 100644 --- a/chromium/media/gpu/windows/dxva_video_decode_accelerator_win.cc +++ b/chromium/media/gpu/windows/dxva_video_decode_accelerator_win.cc @@ -128,6 +128,12 @@ static const GUID DXVA2_Intel_ModeH264_E = { 0x4c54, {0x88, 0xFE, 0xAB, 0xD2, 0x5C, 0x15, 0xB3, 0xD6}}; +static const CLSID CLSID_CAV1DecoderMFT = { + 0xC843981A, + 0x3359, + 0x4721, + {0xB3, 0xF5, 0xD4, 0x84, 0xD8, 0x56, 0x1E, 0x47}}; + constexpr const wchar_t* const kMediaFoundationVideoDecoderDLLs[] = { L"mf.dll", L"mfplat.dll", L"msmpeg2vdec.dll", }; @@ -140,6 +146,23 @@ uint64_t GetCurrentQPC() { return perf_counter_now.QuadPart; } +HRESULT CreateAV1Decoder(const IID& iid, void** object) { + MFT_REGISTER_TYPE_INFO type_info = {MFMediaType_Video, MFVideoFormat_AV1}; + + base::win::ScopedCoMem<IMFActivate*> acts; + UINT32 acts_num; + HRESULT hr = + ::MFTEnumEx(MFT_CATEGORY_VIDEO_DECODER, MFT_ENUM_FLAG_SORTANDFILTER, + &type_info, nullptr, &acts, &acts_num); + if (FAILED(hr)) + return hr; + + if (acts_num < 1) + return E_FAIL; + + return acts[0]->ActivateObject(iid, object); +} + uint64_t g_last_process_output_time; HRESULT g_last_device_removed_reason; @@ -162,8 +185,9 @@ HRESULT g_last_device_removed_reason; namespace media { static const VideoCodecProfile kSupportedProfiles[] = { - H264PROFILE_BASELINE, H264PROFILE_MAIN, H264PROFILE_HIGH, - VP8PROFILE_ANY, VP9PROFILE_PROFILE0, VP9PROFILE_PROFILE2}; + H264PROFILE_BASELINE, H264PROFILE_MAIN, H264PROFILE_HIGH, + VP8PROFILE_ANY, VP9PROFILE_PROFILE0, VP9PROFILE_PROFILE2, + AV1PROFILE_PROFILE_MAIN, AV1PROFILE_PROFILE_HIGH, AV1PROFILE_PROFILE_PRO}; CreateDXGIDeviceManager DXVAVideoDecodeAccelerator::create_dxgi_device_manager_ = NULL; @@ -437,6 +461,8 @@ class VP9ConfigChangeDetector : public ConfigChangeDetector { // Returns false on failure. bool DetectConfig(const uint8_t* stream, unsigned int size) override { parser_.SetStream(stream, size, nullptr); + config_changed_ = false; + Vp9FrameHeader fhdr; gfx::Size allocate_size; std::unique_ptr<DecryptConfig> null_config; @@ -494,11 +520,17 @@ class VP8ConfigChangeDetector : public ConfigChangeDetector { // Detects stream configuration changes. // Returns false on failure. bool DetectConfig(const uint8_t* stream, unsigned int size) override { + config_changed_ = false; + Vp8FrameHeader fhdr; if (!parser_.ParseFrame(stream, size, &fhdr)) return false; - if (fhdr.IsKeyframe() && fhdr.is_full_range) { + // VP8 parser only return resolution and range fields on key frames. + if (!fhdr.IsKeyframe()) + return true; + + if (fhdr.is_full_range) { // VP8 has no color space information, only the range. We will always // prefer the config color space if set, but indicate JPEG when the full // range flag is set on the frame header. @@ -507,22 +539,12 @@ class VP8ConfigChangeDetector : public ConfigChangeDetector { // Does VP8 need a separate visible rect? gfx::Size new_size(fhdr.width, fhdr.height); - if (!size_.IsEmpty() && !pending_config_changed_ && !config_changed_ && - size_ != new_size) { - pending_config_changed_ = true; + if (!size_.IsEmpty() && size_ != new_size) { + config_changed_ = true; DVLOG(1) << "Configuration changed from " << size_.ToString() << " to " << new_size.ToString(); } size_ = new_size; - - // Resolution changes can happen on any frame technically, so wait for a - // keyframe before signaling the config change. - if (fhdr.IsKeyframe() && pending_config_changed_) { - config_changed_ = true; - pending_config_changed_ = false; - } - if (pending_config_changed_) - DVLOG(3) << "Deferring config change until next keyframe..."; return true; } @@ -539,7 +561,6 @@ class VP8ConfigChangeDetector : public ConfigChangeDetector { private: gfx::Size size_; - bool pending_config_changed_ = false; VideoColorSpace color_space_; Vp8Parser parser_; }; @@ -1343,7 +1364,9 @@ DXVAVideoDecodeAccelerator::GetSupportedProfiles( const bool is_vp9 = supported_profile >= VP9PROFILE_MIN && supported_profile <= VP9PROFILE_MAX; const bool is_vp8 = supported_profile == VP8PROFILE_ANY; - DCHECK(is_h264 || is_vp9 || is_vp8); + const bool is_av1 = supported_profile >= AV1PROFILE_MIN && + supported_profile <= AV1PROFILE_MAX; + DCHECK(is_h264 || is_vp9 || is_vp8 || is_av1); ResolutionPair max_resolutions; if (is_h264) { @@ -1354,6 +1377,17 @@ DXVAVideoDecodeAccelerator::GetSupportedProfiles( max_resolutions = max_vp9_profile2_resolutions; } else if (is_vp8) { max_resolutions = max_vp8_resolutions; + } else if (is_av1) { + if (!base::FeatureList::IsEnabled(kMediaFoundationAV1Decoding)) + continue; + + // TODO(dalecurtis): Update GetResolutionsForDecoders() to support AV1. + SupportedProfile profile; + profile.profile = supported_profile; + profile.min_resolution = gfx::Size(); + profile.max_resolution = gfx::Size(8192, 8192); + profiles.push_back(profile); + continue; } // Skip adding VPx profiles if it's not supported or disabled. @@ -1452,13 +1486,23 @@ bool DXVAVideoDecodeAccelerator::InitDecoder(VideoCodecProfile profile) { using_ms_vpx_mft_ = true; } - if (!decoder_dll) { - RETURN_ON_FAILURE(false, "Unsupported codec.", false); - } + if (base::FeatureList::IsEnabled(kMediaFoundationAV1Decoding) && + (profile >= AV1PROFILE_MIN && profile <= AV1PROFILE_MAX)) { + codec_ = kCodecAV1; + clsid = CLSID_CAV1DecoderMFT; - HRESULT hr = - CreateCOMObjectFromDll(decoder_dll, clsid, IID_PPV_ARGS(&decoder_)); - RETURN_ON_HR_FAILURE(hr, "Failed to create decoder instance", false); + // Since the AV1 decoder is a Windows Store package, it can't be created + // from a DLL file like the other codecs. Microsoft baked helper code into + // the OS for VP9 which is why it works and AV1 doesn't. + HRESULT hr = CreateAV1Decoder(IID_PPV_ARGS(&decoder_)); + RETURN_ON_HR_FAILURE(hr, "Failed to create decoder instance", false); + } else { + if (!decoder_dll) + RETURN_ON_FAILURE(false, "Unsupported codec.", false); + HRESULT hr = + CreateCOMObjectFromDll(decoder_dll, clsid, IID_PPV_ARGS(&decoder_)); + RETURN_ON_HR_FAILURE(hr, "Failed to create decoder instance", false); + } RETURN_ON_FAILURE(CheckDecoderDxvaSupport(), "Failed to check decoder DXVA support", false); @@ -1482,8 +1526,8 @@ bool DXVAVideoDecodeAccelerator::InitDecoder(VideoCodecProfile profile) { device_manager_to_use = reinterpret_cast<ULONG_PTR>(device_manager_.Get()); } - hr = decoder_->ProcessMessage(MFT_MESSAGE_SET_D3D_MANAGER, - device_manager_to_use); + HRESULT hr = decoder_->ProcessMessage(MFT_MESSAGE_SET_D3D_MANAGER, + device_manager_to_use); if (use_dx11_) { RETURN_ON_HR_FAILURE(hr, "Failed to pass DX11 manager to decoder", false); } else { @@ -1635,6 +1679,8 @@ bool DXVAVideoDecodeAccelerator::SetDecoderInputMediaType() { hr = media_type->SetGUID(MF_MT_SUBTYPE, MEDIASUBTYPE_VP90); } else if (codec_ == kCodecVP8) { hr = media_type->SetGUID(MF_MT_SUBTYPE, MEDIASUBTYPE_VP80); + } else if (codec_ == kCodecAV1) { + hr = media_type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_AV1); } else { NOTREACHED(); RETURN_ON_FAILURE(false, "Unsupported codec on input media type.", false); @@ -1708,29 +1754,35 @@ bool DXVAVideoDecodeAccelerator::GetStreamsInfoAndBufferReqs() { DVLOG(1) << "Input stream info: "; DVLOG(1) << "Max latency: " << input_stream_info_.hnsMaxLatency; - if (codec_ == kCodecH264) { - // There should be three flags, one for requiring a whole frame be in a - // single sample, one for requiring there be one buffer only in a single - // sample, and one that specifies a fixed sample size. (as in cbSize) - CHECK_EQ(input_stream_info_.dwFlags, 0x7u); - } + + // There should be three flags, one for requiring a whole frame be in a + // single sample, one for requiring there be one buffer only in a single + // sample, and one that specifies a fixed sample size. (as in cbSize) + if (codec_ == kCodecH264 && input_stream_info_.dwFlags != 0x7u) + return false; DVLOG(1) << "Min buffer size: " << input_stream_info_.cbSize; DVLOG(1) << "Max lookahead: " << input_stream_info_.cbMaxLookahead; DVLOG(1) << "Alignment: " << input_stream_info_.cbAlignment; DVLOG(1) << "Output stream info: "; - // The flags here should be the same and mean the same thing, except when - // DXVA is enabled, there is an extra 0x100 flag meaning decoder will - // allocate its own sample. DVLOG(1) << "Flags: " << std::hex << std::showbase << output_stream_info_.dwFlags; - if (codec_ == kCodecH264) { - CHECK_EQ(output_stream_info_.dwFlags, 0x107u); - } DVLOG(1) << "Min buffer size: " << output_stream_info_.cbSize; DVLOG(1) << "Alignment: " << output_stream_info_.cbAlignment; - return true; + + // The flags here should be the same and mean the same thing, except when + // DXVA is enabled, there is an extra 0x100 flag meaning decoder will + // allocate its own sample. + if (codec_ == kCodecH264 && output_stream_info_.dwFlags != 0x107u) + return false; + + // We should fail above during MFT_MESSAGE_SET_D3D_MANAGER if the decoder + // doesn't allocate its own samples, but some MFTs are broken. + if (output_stream_info_.dwFlags & MFT_OUTPUT_STREAM_PROVIDES_SAMPLES) + return true; + + return false; } void DXVAVideoDecodeAccelerator::DoDecode(const gfx::Rect& visible_rect, diff --git a/chromium/media/gpu/windows/hresult_status_debug_device.cc b/chromium/media/gpu/windows/hresult_status_debug_device.cc new file mode 100644 index 00000000000..25349655d05 --- /dev/null +++ b/chromium/media/gpu/windows/hresult_status_debug_device.cc @@ -0,0 +1,63 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/gpu/windows/hresult_status_debug_device.h" +#include "media/base/media_serializers.h" +#include "media/base/win/hresult_status_helper.h" + +namespace media { + +// Only define this method in debug mode. +#if !defined(NDEBUG) + +Status AddDebugMessages(Status error, ComD3D11Device device) { + // MSDN says that this needs to be casted twice, then GetMessage should + // be called with a malloc. + Microsoft::WRL::ComPtr<ID3D11Debug> debug_layer; + if (!SUCCEEDED(device.As(&debug_layer))) + return error; + + Microsoft::WRL::ComPtr<ID3D11InfoQueue> message_layer; + if (!SUCCEEDED(debug_layer.As(&message_layer))) + return error; + + uint64_t messages_count = message_layer->GetNumStoredMessages(); + if (messages_count == 0) + return error; + + std::vector<std::string> messages(messages_count, ""); + for (uint64_t i = 0; i < messages_count; i++) { + SIZE_T size; + message_layer->GetMessage(i, nullptr, &size); + D3D11_MESSAGE* message = reinterpret_cast<D3D11_MESSAGE*>(malloc(size)); + if (!message) // probably OOM - so just stop trying to get more. + return error; + message_layer->GetMessage(i, message, &size); + messages.emplace_back(message->pDescription); + free(message); + } + + return std::move(error).WithData("debug_info", messages); +} + +#endif // !defined(NDEBUG) + +Status D3D11HresultToStatus(HRESULT hresult, + ComD3D11Device device, + const char* message, + const base::Location& location) { + if (SUCCEEDED(hresult)) + return OkStatus(); +#if !defined(NDEBUG) + return AddDebugMessages( + HresultToStatus(hresult, message, StatusCode::kWindowsD3D11Error, + location), + device); +#else // !defined(NDEBUG) + return HresultToStatus(hresult, message, StatusCode::kWindowsD3D11Error, + location); +#endif // !defined(NDEBUG) +} + +} // namespace media diff --git a/chromium/media/gpu/windows/hresult_status_debug_device.h b/chromium/media/gpu/windows/hresult_status_debug_device.h new file mode 100644 index 00000000000..b07fdc5714c --- /dev/null +++ b/chromium/media/gpu/windows/hresult_status_debug_device.h @@ -0,0 +1,27 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_GPU_WINDOWS_HRESULT_STATUS_DEBUG_DEVICE_H_ +#define MEDIA_GPU_WINDOWS_HRESULT_STATUS_DEBUG_DEVICE_H_ + +#include <d3d11.h> +#include <wrl/client.h> + +#include "media/base/status.h" +#include "media/gpu/windows/d3d11_com_defs.h" + +namespace media { + +// In debug mode, this uses |AddDebugMessages()| to give us a detailed error +// trace from the d3d11 stack. Otherwise, it just generates a Status with the +// kWindowsD3D11Error code. +Status D3D11HresultToStatus( + HRESULT hresult, + ComD3D11Device device, + const char* message = nullptr, + const base::Location& location = base::Location::Current()); + +} // namespace media + +#endif // MEDIA_GPU_WINDOWS_HRESULT_STATUS_DEBUG_DEVICE_H_
\ No newline at end of file diff --git a/chromium/media/gpu/windows/media_foundation_video_encode_accelerator_win.cc b/chromium/media/gpu/windows/media_foundation_video_encode_accelerator_win.cc index 910f3784ee7..e7a30122177 100644 --- a/chromium/media/gpu/windows/media_foundation_video_encode_accelerator_win.cc +++ b/chromium/media/gpu/windows/media_foundation_video_encode_accelerator_win.cc @@ -40,7 +40,7 @@ const size_t kMaxResolutionWidth = 1920; const size_t kMaxResolutionHeight = 1088; const size_t kNumInputBuffers = 3; // Media Foundation uses 100 nanosecond units for time, see -// https://msdn.microsoft.com/en-us/library/windows/desktop/ms697282(v=vs.85).aspx +// https://msdn.microsoft.com/en-us/library/windows/desktop/ms697282(v=vs.85).aspx. const size_t kOneMicrosecondInMFSampleTimeUnits = 10; const size_t kOutputSampleBufferSizeRatio = 4; @@ -48,9 +48,6 @@ constexpr const wchar_t* const kMediaFoundationVideoEncoderDLLs[] = { L"mf.dll", L"mfplat.dll", }; -// Resolutions that some platforms support, should be listed in ascending order. -constexpr const gfx::Size kOptionalMaxResolutions[] = {gfx::Size(3840, 2176)}; - eAVEncH264VProfile GetH264VProfile(VideoCodecProfile profile) { switch (profile) { case H264PROFILE_BASELINE: @@ -106,8 +103,12 @@ struct MediaFoundationVideoEncodeAccelerator::BitstreamBufferRef { // attributes are not supported by Windows 7, setting them will return errors. // See bug: http://crbug.com/777659. MediaFoundationVideoEncodeAccelerator::MediaFoundationVideoEncodeAccelerator( - bool compatible_with_win7) + bool compatible_with_win7, + bool enable_async_mft) : compatible_with_win7_(compatible_with_win7), + enable_async_mft_(enable_async_mft), + is_async_mft_(false), + input_required_(false), main_client_task_runner_(base::ThreadTaskRunnerHandle::Get()), encoder_thread_("MFEncoderThread") {} @@ -130,21 +131,28 @@ MediaFoundationVideoEncodeAccelerator::GetSupportedProfiles() { SupportedProfiles profiles; target_bitrate_ = kDefaultTargetBitrate; frame_rate_ = kMaxFrameRateNumerator / kMaxFrameRateDenominator; - input_visible_size_ = gfx::Size(kMaxResolutionWidth, kMaxResolutionHeight); - if (!CreateHardwareEncoderMFT() || !SetEncoderModes() || - !InitializeInputOutputSamples(H264PROFILE_BASELINE)) { + IMFActivate** pp_activate = nullptr; + uint32_t encoder_count = EnumerateHardwareEncoders(&pp_activate); + if (!encoder_count) { ReleaseEncoderResources(); DVLOG(1) << "Hardware encode acceleration is not available on this platform."; + return profiles; } - gfx::Size highest_supported_resolution = input_visible_size_; - for (const auto& resolution : kOptionalMaxResolutions) { - DCHECK_GT(resolution.GetArea(), highest_supported_resolution.GetArea()); - if (!IsResolutionSupported(resolution)) - break; - highest_supported_resolution = resolution; + if (pp_activate) { + // Release the enumerated instances if any. + // According to Windows Dev Center, + // https://docs.microsoft.com/en-us/windows/win32/api/mfapi/nf-mfapi-mftenumex + // The caller must release the pointers. + for (UINT32 i = 0; i < encoder_count; i++) { + if (pp_activate[i]) { + pp_activate[i]->Release(); + pp_activate[i] = nullptr; + } + } + CoTaskMemFree(pp_activate); } ReleaseEncoderResources(); @@ -154,7 +162,7 @@ MediaFoundationVideoEncodeAccelerator::GetSupportedProfiles() { profile.profile = H264PROFILE_BASELINE; profile.max_framerate_numerator = kMaxFrameRateNumerator; profile.max_framerate_denominator = kMaxFrameRateDenominator; - profile.max_resolution = highest_supported_resolution; + profile.max_resolution = gfx::Size(kMaxResolutionWidth, kMaxResolutionHeight); profiles.push_back(profile); profile.profile = H264PROFILE_MAIN; @@ -188,39 +196,61 @@ bool MediaFoundationVideoEncodeAccelerator::Initialize(const Config& config, return false; } encoder_thread_task_runner_ = encoder_thread_.task_runner(); - - if (!CreateHardwareEncoderMFT()) { - DLOG(ERROR) << "Failed creating a hardware encoder MFT."; + IMFActivate** pp_activate = nullptr; + uint32_t encoder_count = EnumerateHardwareEncoders(&pp_activate); + if (!encoder_count) { + DLOG(ERROR) << "Failed finding a hardware encoder MFT."; return false; } + if (is_async_mft_) { + if (!ActivateAsyncEncoder(pp_activate, encoder_count)) { + DLOG(ERROR) << "Failed activating an async hardware encoder MFT."; + + if (pp_activate) { + // Release the enumerated instances if any. + // According to Windows Dev Center, + // https://docs.microsoft.com/en-us/windows/win32/api/mfapi/nf-mfapi-mftenumex + // The caller must release the pointers. + for (UINT32 i = 0; i < encoder_count; i++) { + if (pp_activate[i]) { + pp_activate[i]->Release(); + pp_activate[i] = nullptr; + } + } + CoTaskMemFree(pp_activate); + } + return false; + } + + if (pp_activate) { + // Release the enumerated instances if any. + // According to Windows Dev Center, + // https://docs.microsoft.com/en-us/windows/win32/api/mfapi/nf-mfapi-mftenumex + // The caller must release the pointers. + for (UINT32 i = 0; i < encoder_count; i++) { + if (pp_activate[i]) { + pp_activate[i]->Release(); + pp_activate[i] = nullptr; + } + } + CoTaskMemFree(pp_activate); + } + } + main_client_weak_factory_.reset(new base::WeakPtrFactory<Client>(client)); main_client_ = main_client_weak_factory_->GetWeakPtr(); input_visible_size_ = config.input_visible_size; frame_rate_ = kMaxFrameRateNumerator / kMaxFrameRateDenominator; target_bitrate_ = config.initial_bitrate; bitstream_buffer_size_ = config.input_visible_size.GetArea(); - u_plane_offset_ = - VideoFrame::PlaneSize(PIXEL_FORMAT_I420, VideoFrame::kYPlane, - input_visible_size_) - .GetArea(); - v_plane_offset_ = u_plane_offset_ + VideoFrame::PlaneSize(PIXEL_FORMAT_I420, - VideoFrame::kUPlane, - input_visible_size_) - .GetArea(); - y_stride_ = VideoFrame::RowBytes(VideoFrame::kYPlane, PIXEL_FORMAT_I420, - input_visible_size_.width()); - u_stride_ = VideoFrame::RowBytes(VideoFrame::kUPlane, PIXEL_FORMAT_I420, - input_visible_size_.width()); - v_stride_ = VideoFrame::RowBytes(VideoFrame::kVPlane, PIXEL_FORMAT_I420, - input_visible_size_.width()); if (!SetEncoderModes()) { DLOG(ERROR) << "Failed setting encoder parameters."; return false; } - if (!InitializeInputOutputSamples(config.output_profile)) { + if (!InitializeInputOutputParameters(config.output_profile)) { DLOG(ERROR) << "Failed initializing input-output samples."; return false; } @@ -232,20 +262,40 @@ bool MediaFoundationVideoEncodeAccelerator::Initialize(const Config& config, input_sample_ = CreateEmptySampleWithBuffer( input_stream_info.cbSize ? input_stream_info.cbSize - : VideoFrame::AllocationSize(PIXEL_FORMAT_I420, input_visible_size_), + : VideoFrame::AllocationSize(PIXEL_FORMAT_NV12, input_visible_size_), input_stream_info.cbAlignment); - MFT_OUTPUT_STREAM_INFO output_stream_info; - hr = encoder_->GetOutputStreamInfo(output_stream_id_, &output_stream_info); - RETURN_ON_HR_FAILURE(hr, "Couldn't get output stream info", false); - output_sample_ = CreateEmptySampleWithBuffer( - output_stream_info.cbSize - ? output_stream_info.cbSize - : bitstream_buffer_size_ * kOutputSampleBufferSizeRatio, - output_stream_info.cbAlignment); - - hr = encoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, NULL); - RETURN_ON_HR_FAILURE(hr, "Couldn't set ProcessMessage", false); + if (is_async_mft_) { + // Start the asynchronous processing model + hr = encoder_->ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0); + RETURN_ON_HR_FAILURE( + hr, "Couldn't set ProcessMessage MFT_MESSAGE_COMMAND_FLUSH", false); + hr = encoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0); + RETURN_ON_HR_FAILURE( + hr, "Couldn't set ProcessMessage MFT_MESSAGE_NOTIFY_BEGIN_STREAMING", + false); + hr = encoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0); + RETURN_ON_HR_FAILURE( + hr, "Couldn't set ProcessMessage MFT_MESSAGE_NOTIFY_START_OF_STREAM", + false); + hr = encoder_->QueryInterface(IID_PPV_ARGS(&event_generator_)); + RETURN_ON_HR_FAILURE(hr, "Couldn't get event generator", false); + } else { + // Create output sample for synchronous processing model + MFT_OUTPUT_STREAM_INFO output_stream_info; + hr = encoder_->GetOutputStreamInfo(output_stream_id_, &output_stream_info); + RETURN_ON_HR_FAILURE(hr, "Couldn't get output stream info", false); + output_sample_ = CreateEmptySampleWithBuffer( + output_stream_info.cbSize + ? output_stream_info.cbSize + : bitstream_buffer_size_ * kOutputSampleBufferSizeRatio, + output_stream_info.cbAlignment); + + hr = encoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0); + RETURN_ON_HR_FAILURE( + hr, "Couldn't set ProcessMessage MFT_MESSAGE_NOTIFY_BEGIN_STREAMING", + false); + } main_client_task_runner_->PostTask( FROM_HERE, base::BindOnce(&Client::RequireBitstreamBuffers, main_client_, @@ -342,25 +392,27 @@ bool MediaFoundationVideoEncodeAccelerator::PreSandboxInitialization() { return result; } -bool MediaFoundationVideoEncodeAccelerator::CreateHardwareEncoderMFT() { +uint32_t MediaFoundationVideoEncodeAccelerator::EnumerateHardwareEncoders( + IMFActivate*** pp_activate) { DVLOG(3) << __func__; DCHECK(main_client_task_runner_->BelongsToCurrentThread()); if (!compatible_with_win7_ && base::win::GetVersion() < base::win::Version::WIN8) { DVLOG(ERROR) << "Windows versions earlier than 8 are not supported."; - return false; + return 0; } for (const wchar_t* mfdll : kMediaFoundationVideoEncoderDLLs) { if (!::GetModuleHandle(mfdll)) { DVLOG(ERROR) << mfdll << " is required for encoding"; - return false; + return 0; } } - if (!(session_ = InitializeMediaFoundation())) - return false; + if (!(session_ = InitializeMediaFoundation())) { + return 0; + } uint32_t flags = MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER; MFT_REGISTER_TYPE_INFO input_info; @@ -370,32 +422,119 @@ bool MediaFoundationVideoEncodeAccelerator::CreateHardwareEncoderMFT() { output_info.guidMajorType = MFMediaType_Video; output_info.guidSubtype = MFVideoFormat_H264; - base::win::ScopedCoMem<CLSID> CLSIDs; uint32_t count = 0; - HRESULT hr = MFTEnum(MFT_CATEGORY_VIDEO_ENCODER, flags, &input_info, - &output_info, NULL, &CLSIDs, &count); - RETURN_ON_HR_FAILURE(hr, "Couldn't enumerate hardware encoder", false); - RETURN_ON_FAILURE((count > 0), "No HW encoder found", false); - DVLOG(3) << "HW encoder(s) found: " << count; - hr = ::CoCreateInstance(CLSIDs[0], nullptr, CLSCTX_ALL, - IID_PPV_ARGS(&encoder_)); - RETURN_ON_HR_FAILURE(hr, "Couldn't activate hardware encoder", false); + HRESULT hr = E_FAIL; + if (enable_async_mft_) { + // Use MFTEnumEx to find hardware encoder. + hr = MFTEnumEx(MFT_CATEGORY_VIDEO_ENCODER, flags, &input_info, &output_info, + pp_activate, &count); + RETURN_ON_HR_FAILURE( + hr, "Couldn't enumerate hardware encoder from MFTEnumEx", 0); + RETURN_ON_FAILURE((count > 0), "No asynchronous MFT encoder found", 0); + DVLOG(3) << "Hardware encoder(s) available found from MFTEnumEx: " << count; + is_async_mft_ = true; + } else { + // Use MFTEnum to find hardware encoder. + base::win::ScopedCoMem<CLSID> CLSIDs; + hr = MFTEnum(MFT_CATEGORY_VIDEO_ENCODER, flags, &input_info, &output_info, + nullptr, &CLSIDs, &count); + RETURN_ON_HR_FAILURE(hr, "Couldn't enumerate hardware encoder from MFTEnum", + 0); + RETURN_ON_FAILURE((count > 0), "No legacy MFT encoder found", 0); + DVLOG(3) << "Hardware encoder(s) available found from MFTEnum: " << count; + hr = ::CoCreateInstance(CLSIDs[0], nullptr, CLSCTX_ALL, + IID_PPV_ARGS(&encoder_)); + RETURN_ON_HR_FAILURE(hr, "Couldn't create legacy MFT encoder", 0); + RETURN_ON_FAILURE((encoder_.Get() != nullptr), + "No legacy MFT encoder instance created", 0); + is_async_mft_ = false; + } + + return count; +} + +bool MediaFoundationVideoEncodeAccelerator::ActivateAsyncEncoder( + IMFActivate** pp_activate, + uint32_t encoder_count) { + DVLOG(3) << __func__; + DCHECK(main_client_task_runner_->BelongsToCurrentThread()); + + // Try to create the encoder with priority according to merit value. + HRESULT hr = E_FAIL; + for (UINT32 i = 0; i < encoder_count; i++) { + if (FAILED(hr)) { + DCHECK(!encoder_); + DCHECK(!activate_); + hr = pp_activate[i]->ActivateObject(IID_PPV_ARGS(&encoder_)); + if (encoder_.Get() != nullptr) { + DCHECK(SUCCEEDED(hr)); + + activate_ = pp_activate[i]; + pp_activate[i] = nullptr; + + // Print the friendly name. + base::win::ScopedCoMem<WCHAR> friendly_name; + UINT32 name_length; + activate_->GetAllocatedString(MFT_FRIENDLY_NAME_Attribute, + &friendly_name, &name_length); + DVLOG(3) << "Selected asynchronous hardware encoder's friendly name: " + << friendly_name; + } else { + DCHECK(FAILED(hr)); + + // The component that calls ActivateObject is + // responsible for calling ShutdownObject, + // https://docs.microsoft.com/en-us/windows/win32/api/mfobjects/nf-mfobjects-imfactivate-shutdownobject. + pp_activate[i]->ShutdownObject(); + } + } + } + + RETURN_ON_HR_FAILURE(hr, "Couldn't activate asynchronous hardware encoder", + false); RETURN_ON_FAILURE((encoder_.Get() != nullptr), - "No HW encoder instance created", false); + "No asynchronous hardware encoder instance created", false); + + Microsoft::WRL::ComPtr<IMFAttributes> all_attributes; + hr = encoder_->GetAttributes(&all_attributes); + if (SUCCEEDED(hr)) { + // An asynchronous MFT must support dynamic format changes, + // https://docs.microsoft.com/en-us/windows/win32/medfound/asynchronous-mfts#format-changes. + UINT32 dynamic = FALSE; + hr = all_attributes->GetUINT32(MFT_SUPPORT_DYNAMIC_FORMAT_CHANGE, &dynamic); + if (!dynamic) { + DLOG(ERROR) << "Couldn't support dynamic format change."; + return false; + } + + // Unlock the selected asynchronous MFTs, + // https://docs.microsoft.com/en-us/windows/win32/medfound/asynchronous-mfts#unlocking-asynchronous-mfts. + UINT32 async = FALSE; + hr = all_attributes->GetUINT32(MF_TRANSFORM_ASYNC, &async); + if (!async) { + DLOG(ERROR) << "MFT encoder is not asynchronous."; + return false; + } + + hr = all_attributes->SetUINT32(MF_TRANSFORM_ASYNC_UNLOCK, TRUE); + RETURN_ON_HR_FAILURE(hr, "Couldn't unlock transform async", false); + } + return true; } -bool MediaFoundationVideoEncodeAccelerator::InitializeInputOutputSamples( +bool MediaFoundationVideoEncodeAccelerator::InitializeInputOutputParameters( VideoCodecProfile output_profile) { DCHECK(main_client_task_runner_->BelongsToCurrentThread()); + DCHECK(encoder_); DWORD input_count = 0; DWORD output_count = 0; HRESULT hr = encoder_->GetStreamCount(&input_count, &output_count); RETURN_ON_HR_FAILURE(hr, "Couldn't get stream count", false); if (input_count < 1 || output_count < 1) { - LOG(ERROR) << "Stream count too few: input " << input_count << ", output " - << output_count; + DLOG(ERROR) << "Stream count too few: input " << input_count << ", output " + << output_count; return false; } @@ -410,13 +549,13 @@ bool MediaFoundationVideoEncodeAccelerator::InitializeInputOutputSamples( input_stream_id_ = 0; output_stream_id_ = 0; } else { - LOG(ERROR) << "Couldn't find stream ids."; + DLOG(ERROR) << "Couldn't find stream ids from hardware encoder."; return false; } // Initialize output parameters. hr = MFCreateMediaType(&imf_output_media_type_); - RETURN_ON_HR_FAILURE(hr, "Couldn't create media type", false); + RETURN_ON_HR_FAILURE(hr, "Couldn't create output media type", false); hr = imf_output_media_type_->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); RETURN_ON_HR_FAILURE(hr, "Couldn't set media type", false); hr = imf_output_media_type_->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264); @@ -442,10 +581,10 @@ bool MediaFoundationVideoEncodeAccelerator::InitializeInputOutputSamples( // Initialize input parameters. hr = MFCreateMediaType(&imf_input_media_type_); - RETURN_ON_HR_FAILURE(hr, "Couldn't create media type", false); + RETURN_ON_HR_FAILURE(hr, "Couldn't create input media type", false); hr = imf_input_media_type_->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); RETURN_ON_HR_FAILURE(hr, "Couldn't set media type", false); - hr = imf_input_media_type_->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_YV12); + hr = imf_input_media_type_->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12); RETURN_ON_HR_FAILURE(hr, "Couldn't set video format", false); hr = MFSetAttributeRatio(imf_input_media_type_.Get(), MF_MT_FRAME_RATE, frame_rate_, 1); @@ -465,11 +604,11 @@ bool MediaFoundationVideoEncodeAccelerator::InitializeInputOutputSamples( bool MediaFoundationVideoEncodeAccelerator::SetEncoderModes() { DCHECK(main_client_task_runner_->BelongsToCurrentThread()); - RETURN_ON_FAILURE((encoder_.Get() != nullptr), - "No HW encoder instance created", false); + DCHECK(encoder_); HRESULT hr = encoder_.As(&codec_api_); RETURN_ON_HR_FAILURE(hr, "Couldn't get ICodecAPI", false); + VARIANT var; var.vt = VT_UI4; var.ulVal = eAVEncCommonRateControlMode_CBR; @@ -481,44 +620,42 @@ bool MediaFoundationVideoEncodeAccelerator::SetEncoderModes() { // setting it on Windows 7 returns error. RETURN_ON_HR_FAILURE(hr, "Couldn't set CommonRateControlMode", false); } + + if (is_async_mft_ && S_OK == codec_api_->IsModifiable( + &CODECAPI_AVEncVideoTemporalLayerCount)) { + var.ulVal = 1; + hr = codec_api_->SetValue(&CODECAPI_AVEncVideoTemporalLayerCount, &var); + if (!compatible_with_win7_) { + RETURN_ON_HR_FAILURE(hr, "Couldn't set temporal layer count", false); + } + } + var.ulVal = target_bitrate_; hr = codec_api_->SetValue(&CODECAPI_AVEncCommonMeanBitRate, &var); if (!compatible_with_win7_) { RETURN_ON_HR_FAILURE(hr, "Couldn't set bitrate", false); } - var.ulVal = eAVEncAdaptiveMode_Resolution; - hr = codec_api_->SetValue(&CODECAPI_AVEncAdaptiveMode, &var); - if (!compatible_with_win7_) { - RETURN_ON_HR_FAILURE(hr, "Couldn't set FrameRate", false); - } - var.vt = VT_BOOL; - var.boolVal = VARIANT_TRUE; - hr = codec_api_->SetValue(&CODECAPI_AVLowLatencyMode, &var); - if (!compatible_with_win7_) { - RETURN_ON_HR_FAILURE(hr, "Couldn't set LowLatencyMode", false); - } - return true; -} - -bool MediaFoundationVideoEncodeAccelerator::IsResolutionSupported( - const gfx::Size& resolution) { - DCHECK(main_client_task_runner_->BelongsToCurrentThread()); - DCHECK(encoder_); - - HRESULT hr = - MFSetAttributeSize(imf_output_media_type_.Get(), MF_MT_FRAME_SIZE, - resolution.width(), resolution.height()); - RETURN_ON_HR_FAILURE(hr, "Couldn't set frame size", false); - hr = encoder_->SetOutputType(output_stream_id_, imf_output_media_type_.Get(), - 0); - RETURN_ON_HR_FAILURE(hr, "Couldn't set output media type", false); + if (!is_async_mft_ || + (is_async_mft_ && + S_OK == codec_api_->IsModifiable(&CODECAPI_AVEncAdaptiveMode))) { + var.ulVal = eAVEncAdaptiveMode_Resolution; + hr = codec_api_->SetValue(&CODECAPI_AVEncAdaptiveMode, &var); + if (!compatible_with_win7_) { + RETURN_ON_HR_FAILURE(hr, "Couldn't set adaptive mode", false); + } + } - hr = MFSetAttributeSize(imf_input_media_type_.Get(), MF_MT_FRAME_SIZE, - resolution.width(), resolution.height()); - RETURN_ON_HR_FAILURE(hr, "Couldn't set frame size", false); - hr = encoder_->SetInputType(input_stream_id_, imf_input_media_type_.Get(), 0); - RETURN_ON_HR_FAILURE(hr, "Couldn't set input media type", false); + if (!is_async_mft_ || + (is_async_mft_ && + S_OK == codec_api_->IsModifiable(&CODECAPI_AVLowLatencyMode))) { + var.vt = VT_BOOL; + var.boolVal = VARIANT_TRUE; + hr = codec_api_->SetValue(&CODECAPI_AVLowLatencyMode, &var); + if (!compatible_with_win7_) { + RETURN_ON_HR_FAILURE(hr, "Couldn't set low latency mode", false); + } + } return true; } @@ -538,30 +675,131 @@ void MediaFoundationVideoEncodeAccelerator::EncodeTask( DVLOG(3) << __func__; DCHECK(encoder_thread_task_runner_->BelongsToCurrentThread()); + if (is_async_mft_) { + AsyncEncodeTask(frame, force_keyframe); + } else { + SyncEncodeTask(frame, force_keyframe); + } +} + +void MediaFoundationVideoEncodeAccelerator::AsyncEncodeTask( + scoped_refptr<VideoFrame> frame, + bool force_keyframe) { + bool input_delivered = false; + HRESULT hr = E_FAIL; + if (input_required_) { + // Hardware MFT is waiting for this coming input. + hr = ProcessInput(frame, force_keyframe); + if (FAILED(hr)) { + NotifyError(kPlatformFailureError); + RETURN_ON_HR_FAILURE(hr, "Couldn't encode", ); + } + + DVLOG(3) << "Sent for encode " << hr; + input_delivered = true; + input_required_ = false; + } else { + Microsoft::WRL::ComPtr<IMFMediaEvent> media_event; + hr = event_generator_->GetEvent(MF_EVENT_FLAG_NO_WAIT, &media_event); + if (FAILED(hr)) { + DLOG(WARNING) << "Abandoned input frame for video encoder."; + return; + } + + MediaEventType event_type; + hr = media_event->GetType(&event_type); + if (FAILED(hr)) { + DLOG(ERROR) << "Failed to get the type of media event."; + return; + } + + // Always deliver the current input into HMFT. + if (event_type == METransformNeedInput) { + hr = ProcessInput(frame, force_keyframe); + if (FAILED(hr)) { + NotifyError(kPlatformFailureError); + RETURN_ON_HR_FAILURE(hr, "Couldn't encode", ); + } + + DVLOG(3) << "Sent for encode " << hr; + input_delivered = true; + } else if (event_type == METransformHaveOutput) { + ProcessOutputAsync(); + input_delivered = + TryToDeliverInputFrame(std::move(frame), force_keyframe); + } + } + + if (!input_delivered) { + DLOG(ERROR) << "Failed to deliver input frame to video encoder"; + return; + } + + TryToReturnBitstreamBuffer(); +} + +void MediaFoundationVideoEncodeAccelerator::SyncEncodeTask( + scoped_refptr<VideoFrame> frame, + bool force_keyframe) { + HRESULT hr = E_FAIL; + hr = ProcessInput(frame, force_keyframe); + + // According to MSDN, if encoder returns MF_E_NOTACCEPTING, we need to try + // processing the output. This error indicates that encoder does not accept + // any more input data. + if (hr == MF_E_NOTACCEPTING) { + DVLOG(3) << "MF_E_NOTACCEPTING"; + ProcessOutputSync(); + hr = encoder_->ProcessInput(input_stream_id_, input_sample_.Get(), 0); + if (FAILED(hr)) { + NotifyError(kPlatformFailureError); + RETURN_ON_HR_FAILURE(hr, "Couldn't encode", ); + } + } else if (FAILED(hr)) { + NotifyError(kPlatformFailureError); + RETURN_ON_HR_FAILURE(hr, "Couldn't encode", ); + } + DVLOG(3) << "Sent for encode " << hr; + + ProcessOutputSync(); +} + +HRESULT MediaFoundationVideoEncodeAccelerator::ProcessInput( + scoped_refptr<VideoFrame> frame, + bool force_keyframe) { + DVLOG(3) << __func__; + DCHECK(encoder_thread_task_runner_->BelongsToCurrentThread()); + DCHECK_EQ(frame->format(), PIXEL_FORMAT_I420); + + // Convert I420 to NV12 as input. Microsoft::WRL::ComPtr<IMFMediaBuffer> input_buffer; input_sample_->GetBufferByIndex(0, &input_buffer); { MediaBufferScopedPointer scoped_buffer(input_buffer.Get()); DCHECK(scoped_buffer.get()); - libyuv::I420Copy(frame->visible_data(VideoFrame::kYPlane), - frame->stride(VideoFrame::kYPlane), - frame->visible_data(VideoFrame::kVPlane), - frame->stride(VideoFrame::kVPlane), - frame->visible_data(VideoFrame::kUPlane), - frame->stride(VideoFrame::kUPlane), scoped_buffer.get(), - y_stride_, scoped_buffer.get() + u_plane_offset_, - u_stride_, scoped_buffer.get() + v_plane_offset_, - v_stride_, input_visible_size_.width(), - input_visible_size_.height()); + int dst_stride_y = frame->stride(VideoFrame::kYPlane); + uint8_t* dst_uv = + scoped_buffer.get() + + frame->stride(VideoFrame::kYPlane) * frame->rows(VideoFrame::kYPlane); + int dst_stride_uv = frame->stride(VideoFrame::kUPlane) * 2; + libyuv::I420ToNV12(frame->visible_data(VideoFrame::kYPlane), + frame->stride(VideoFrame::kYPlane), + frame->visible_data(VideoFrame::kUPlane), + frame->stride(VideoFrame::kUPlane), + frame->visible_data(VideoFrame::kVPlane), + frame->stride(VideoFrame::kVPlane), scoped_buffer.get(), + dst_stride_y, dst_uv, dst_stride_uv, + input_visible_size_.width(), + input_visible_size_.height()); } input_sample_->SetSampleTime(frame->timestamp().InMicroseconds() * kOneMicrosecondInMFSampleTimeUnits); - UINT64 sample_duration = 1; + UINT64 sample_duration = 0; HRESULT hr = MFFrameRateToAverageTimePerFrame(frame_rate_, 1, &sample_duration); - RETURN_ON_HR_FAILURE(hr, "Couldn't calculate sample duration", ); + RETURN_ON_HR_FAILURE(hr, "Couldn't calculate sample duration", E_FAIL); input_sample_->SetSampleDuration(sample_duration); // Release frame after input is copied. @@ -572,34 +810,100 @@ void MediaFoundationVideoEncodeAccelerator::EncodeTask( var.vt = VT_UI4; var.ulVal = 1; hr = codec_api_->SetValue(&CODECAPI_AVEncVideoForceKeyFrame, &var); - if (!compatible_with_win7_ && !SUCCEEDED(hr)) { + if (!compatible_with_win7_ && FAILED(hr)) { LOG(WARNING) << "Failed to set CODECAPI_AVEncVideoForceKeyFrame, " "HRESULT: 0x" << std::hex << hr; } } - hr = encoder_->ProcessInput(input_stream_id_, input_sample_.Get(), 0); - // According to MSDN, if encoder returns MF_E_NOTACCEPTING, we need to try - // processing the output. This error indicates that encoder does not accept - // any more input data. - if (hr == MF_E_NOTACCEPTING) { - DVLOG(3) << "MF_E_NOTACCEPTING"; - ProcessOutput(); - hr = encoder_->ProcessInput(input_stream_id_, input_sample_.Get(), 0); - if (!SUCCEEDED(hr)) { - NotifyError(kPlatformFailureError); - RETURN_ON_HR_FAILURE(hr, "Couldn't encode", ); + return encoder_->ProcessInput(input_stream_id_, input_sample_.Get(), 0); +} + +void MediaFoundationVideoEncodeAccelerator::ProcessOutputAsync() { + DVLOG(3) << __func__; + DCHECK(encoder_thread_task_runner_->BelongsToCurrentThread()); + + MFT_OUTPUT_DATA_BUFFER output_data_buffer = {0}; + output_data_buffer.dwStreamID = output_stream_id_; + output_data_buffer.dwStatus = 0; + output_data_buffer.pEvents = nullptr; + output_data_buffer.pSample = nullptr; + DWORD status = 0; + HRESULT hr = encoder_->ProcessOutput(0, 1, &output_data_buffer, &status); + if (hr == MF_E_TRANSFORM_STREAM_CHANGE) { + hr = S_OK; + Microsoft::WRL::ComPtr<IMFMediaType> media_type; + for (DWORD type_index = 0; SUCCEEDED(hr); ++type_index) { + hr = encoder_->GetOutputAvailableType(output_stream_id_, type_index, + &media_type); + if (SUCCEEDED(hr)) { + break; + } } - } else if (!SUCCEEDED(hr)) { - NotifyError(kPlatformFailureError); - RETURN_ON_HR_FAILURE(hr, "Couldn't encode", ); + hr = encoder_->SetOutputType(output_stream_id_, media_type.Get(), 0); + return; } - DVLOG(3) << "Sent for encode " << hr; - ProcessOutput(); + RETURN_ON_HR_FAILURE(hr, "Couldn't get encoded data", ); + DVLOG(3) << "Got encoded data " << hr; + + Microsoft::WRL::ComPtr<IMFMediaBuffer> output_buffer; + hr = output_data_buffer.pSample->GetBufferByIndex(0, &output_buffer); + RETURN_ON_HR_FAILURE(hr, "Couldn't get buffer by index", ); + + DWORD size = 0; + hr = output_buffer->GetCurrentLength(&size); + RETURN_ON_HR_FAILURE(hr, "Couldn't get buffer length", ); + + base::TimeDelta timestamp; + LONGLONG sample_time; + hr = output_data_buffer.pSample->GetSampleTime(&sample_time); + if (SUCCEEDED(hr)) { + timestamp = base::TimeDelta::FromMicroseconds( + sample_time / kOneMicrosecondInMFSampleTimeUnits); + } + + const bool keyframe = MFGetAttributeUINT32( + output_data_buffer.pSample, MFSampleExtension_CleanPoint, false); + DVLOG(3) << "Encoded data with size:" << size << " keyframe " << keyframe; + + // If no bit stream buffer presents, queue the output first. + if (bitstream_buffer_queue_.empty()) { + DVLOG(3) << "No bitstream buffers."; + // We need to copy the output so that encoding can continue. + std::unique_ptr<EncodeOutput> encode_output( + new EncodeOutput(size, keyframe, timestamp)); + { + MediaBufferScopedPointer scoped_buffer(output_buffer.Get()); + memcpy(encode_output->memory(), scoped_buffer.get(), size); + } + encoder_output_queue_.push_back(std::move(encode_output)); + output_data_buffer.pSample->Release(); + output_data_buffer.pSample = nullptr; + return; + } + + // Immediately return encoded buffer with BitstreamBuffer to client. + std::unique_ptr<MediaFoundationVideoEncodeAccelerator::BitstreamBufferRef> + buffer_ref = std::move(bitstream_buffer_queue_.front()); + bitstream_buffer_queue_.pop_front(); + + { + MediaBufferScopedPointer scoped_buffer(output_buffer.Get()); + memcpy(buffer_ref->mapping.memory(), scoped_buffer.get(), size); + } + + output_data_buffer.pSample->Release(); + output_data_buffer.pSample = nullptr; + + main_client_task_runner_->PostTask( + FROM_HERE, + base::BindOnce(&Client::BitstreamBufferReady, main_client_, + buffer_ref->id, + BitstreamBufferMetadata(size, keyframe, timestamp))); } -void MediaFoundationVideoEncodeAccelerator::ProcessOutput() { +void MediaFoundationVideoEncodeAccelerator::ProcessOutputSync() { DVLOG(3) << __func__; DCHECK(encoder_thread_task_runner_->BelongsToCurrentThread()); @@ -676,7 +980,87 @@ void MediaFoundationVideoEncodeAccelerator::ProcessOutput() { // Keep calling ProcessOutput recursively until MF_E_TRANSFORM_NEED_MORE_INPUT // is returned to flush out all the output. - ProcessOutput(); + ProcessOutputSync(); +} + +bool MediaFoundationVideoEncodeAccelerator::TryToDeliverInputFrame( + scoped_refptr<VideoFrame> frame, + bool force_keyframe) { + bool input_delivered = false; + Microsoft::WRL::ComPtr<IMFMediaEvent> media_event; + MediaEventType event_type; + do { + HRESULT hr = + event_generator_->GetEvent(MF_EVENT_FLAG_NO_WAIT, &media_event); + if (FAILED(hr)) { + break; + } + + hr = media_event->GetType(&event_type); + if (FAILED(hr)) { + DLOG(ERROR) << "Failed to get the type of media event."; + break; + } + + switch (event_type) { + case METransformHaveOutput: { + ProcessOutputAsync(); + continue; + } + case METransformNeedInput: { + hr = ProcessInput(frame, force_keyframe); + if (FAILED(hr)) { + NotifyError(kPlatformFailureError); + RETURN_ON_HR_FAILURE(hr, "Couldn't encode", false); + } + + DVLOG(3) << "Sent for encode " << hr; + return true; + } + default: + break; + } + } while (true); + + return input_delivered; +} + +void MediaFoundationVideoEncodeAccelerator::TryToReturnBitstreamBuffer() { + // Try to fetch the encoded frame in time. + bool output_processed = false; + do { + Microsoft::WRL::ComPtr<IMFMediaEvent> media_event; + MediaEventType event_type; + HRESULT hr = + event_generator_->GetEvent(MF_EVENT_FLAG_NO_WAIT, &media_event); + if (FAILED(hr)) { + if (!output_processed) { + continue; + } else { + break; + } + } + + hr = media_event->GetType(&event_type); + if (FAILED(hr)) { + DLOG(ERROR) << "Failed to get the type of media event."; + break; + } + + switch (event_type) { + case METransformHaveOutput: { + ProcessOutputAsync(); + output_processed = true; + break; + } + case METransformNeedInput: { + input_required_ = true; + continue; + } + default: + break; + } + } while (true); } void MediaFoundationVideoEncodeAccelerator::UseOutputBitstreamBufferTask( @@ -689,31 +1073,22 @@ void MediaFoundationVideoEncodeAccelerator::UseOutputBitstreamBufferTask( std::unique_ptr<MediaFoundationVideoEncodeAccelerator::EncodeOutput> encode_output = std::move(encoder_output_queue_.front()); encoder_output_queue_.pop_front(); - ReturnBitstreamBuffer(std::move(encode_output), std::move(buffer_ref)); + memcpy(buffer_ref->mapping.memory(), encode_output->memory(), + encode_output->size()); + + main_client_task_runner_->PostTask( + FROM_HERE, + base::BindOnce(&Client::BitstreamBufferReady, main_client_, + buffer_ref->id, + BitstreamBufferMetadata( + encode_output->size(), encode_output->keyframe, + encode_output->capture_timestamp))); return; } bitstream_buffer_queue_.push_back(std::move(buffer_ref)); } -void MediaFoundationVideoEncodeAccelerator::ReturnBitstreamBuffer( - std::unique_ptr<EncodeOutput> encode_output, - std::unique_ptr<MediaFoundationVideoEncodeAccelerator::BitstreamBufferRef> - buffer_ref) { - DVLOG(3) << __func__; - DCHECK(encoder_thread_task_runner_->BelongsToCurrentThread()); - - memcpy(buffer_ref->mapping.memory(), encode_output->memory(), - encode_output->size()); - main_client_task_runner_->PostTask( - FROM_HERE, - base::BindOnce(&Client::BitstreamBufferReady, main_client_, - buffer_ref->id, - BitstreamBufferMetadata( - encode_output->size(), encode_output->keyframe, - encode_output->capture_timestamp))); -} - void MediaFoundationVideoEncodeAccelerator::RequestEncodingParametersChangeTask( uint32_t bitrate, uint32_t framerate) { @@ -732,7 +1107,7 @@ void MediaFoundationVideoEncodeAccelerator::RequestEncodingParametersChangeTask( var.ulVal = target_bitrate_; HRESULT hr = codec_api_->SetValue(&CODECAPI_AVEncCommonMeanBitRate, &var); if (!compatible_with_win7_) { - RETURN_ON_HR_FAILURE(hr, "Couldn't set bitrate", ); + RETURN_ON_HR_FAILURE(hr, "Couldn't update bitrate", ); } } } @@ -748,12 +1123,23 @@ void MediaFoundationVideoEncodeAccelerator::DestroyTask() { } void MediaFoundationVideoEncodeAccelerator::ReleaseEncoderResources() { + while (!bitstream_buffer_queue_.empty()) + bitstream_buffer_queue_.pop_front(); + while (!encoder_output_queue_.empty()) + encoder_output_queue_.pop_front(); + + if (activate_.Get() != nullptr) { + activate_->ShutdownObject(); + activate_->Release(); + activate_.Reset(); + } encoder_.Reset(); codec_api_.Reset(); + event_generator_.Reset(); imf_input_media_type_.Reset(); imf_output_media_type_.Reset(); input_sample_.Reset(); output_sample_.Reset(); } -} // namespace content +} // namespace media diff --git a/chromium/media/gpu/windows/media_foundation_video_encode_accelerator_win.h b/chromium/media/gpu/windows/media_foundation_video_encode_accelerator_win.h index e7f5cff666e..274b64e9e88 100644 --- a/chromium/media/gpu/windows/media_foundation_video_encode_accelerator_win.h +++ b/chromium/media/gpu/windows/media_foundation_video_encode_accelerator_win.h @@ -37,7 +37,8 @@ class MEDIA_GPU_EXPORT MediaFoundationVideoEncodeAccelerator // If |compatible_with_win7| is true, MediaFoundationVideoEncoderAccelerator // works on Windows 7. Some attributes of the encoder are not supported on old // systems, which may impact the performance or quality of the output. - explicit MediaFoundationVideoEncodeAccelerator(bool compatible_with_win7); + explicit MediaFoundationVideoEncodeAccelerator(bool compatible_with_win7, + bool enable_async_mft); // VideoEncodeAccelerator implementation. VideoEncodeAccelerator::SupportedProfiles GetSupportedProfiles() override; @@ -48,7 +49,7 @@ class MEDIA_GPU_EXPORT MediaFoundationVideoEncodeAccelerator uint32_t framerate) override; void Destroy() override; - // Preload dlls required for encoding. Returns true if all required dlls are + // Preloads dlls required for encoding. Returns true if all required dlls are // correctly loaded. static bool PreSandboxInitialization(); @@ -62,40 +63,46 @@ class MEDIA_GPU_EXPORT MediaFoundationVideoEncodeAccelerator // Holds output buffers coming from the encoder. class EncodeOutput; - // Creates an hardware encoder backed IMFTransform instance on |encoder_|. - bool CreateHardwareEncoderMFT(); + // Enumerates all hardware encoder backed IMFTransform instances. + uint32_t EnumerateHardwareEncoders(IMFActivate*** pp_activate); - // Initializes and allocates memory for input and output samples. - bool InitializeInputOutputSamples(VideoCodecProfile output_profile); + // Activates the asynchronous encoder instance |encoder_| according to codec + // merit. + bool ActivateAsyncEncoder(IMFActivate** pp_activate, uint32_t activate_count); + + // Initializes and allocates memory for input and output parameters. + bool InitializeInputOutputParameters(VideoCodecProfile output_profile); // Initializes encoder parameters for real-time use. bool SetEncoderModes(); - // Returns true if we can initialize input and output samples with the given - // frame size, otherwise false. - bool IsResolutionSupported(const gfx::Size& size); - // Helper function to notify the client of an error on // |main_client_task_runner_|. void NotifyError(VideoEncodeAccelerator::Error error); // Encoding tasks to be run on |encoder_thread_|. void EncodeTask(scoped_refptr<VideoFrame> frame, bool force_keyframe); + void AsyncEncodeTask(scoped_refptr<VideoFrame> frame, bool force_keyframe); + void SyncEncodeTask(scoped_refptr<VideoFrame> frame, bool force_keyframe); + + // Processes the input video frame for the encoder. + HRESULT ProcessInput(scoped_refptr<VideoFrame> frame, bool force_keyframe); // Checks for and copies encoded output on |encoder_thread_|. - void ProcessOutput(); + void ProcessOutputAsync(); + void ProcessOutputSync(); + + // Tries to deliver the input frame to the encoder. + bool TryToDeliverInputFrame(scoped_refptr<VideoFrame> frame, + bool force_keyframe); + + // Tries to return a bitstream buffer to the client. + void TryToReturnBitstreamBuffer(); // Inserts the output buffers for reuse on |encoder_thread_|. void UseOutputBitstreamBufferTask( std::unique_ptr<BitstreamBufferRef> buffer_ref); - // Copies EncodeOutput into a BitstreamBuffer and returns it to the - // |main_client_|. - void ReturnBitstreamBuffer( - std::unique_ptr<EncodeOutput> encode_output, - std::unique_ptr<MediaFoundationVideoEncodeAccelerator::BitstreamBufferRef> - buffer_ref); - // Changes encode parameters on |encoder_thread_|. void RequestEncodingParametersChangeTask(uint32_t bitrate, uint32_t framerate); @@ -108,6 +115,12 @@ class MEDIA_GPU_EXPORT MediaFoundationVideoEncodeAccelerator const bool compatible_with_win7_; + // Flag to enable the usage of MFTEnumEx. + const bool enable_async_mft_; + + // Whether asynchronous hardware encoder enabled or not. + bool is_async_mft_; + // Bitstream buffers ready to be used to return encoded output as a FIFO. base::circular_deque<std::unique_ptr<BitstreamBufferRef>> bitstream_buffer_queue_; @@ -119,14 +132,11 @@ class MEDIA_GPU_EXPORT MediaFoundationVideoEncodeAccelerator size_t bitstream_buffer_size_; uint32_t frame_rate_; uint32_t target_bitrate_; - size_t u_plane_offset_; - size_t v_plane_offset_; - size_t y_stride_; - size_t u_stride_; - size_t v_stride_; + Microsoft::WRL::ComPtr<IMFActivate> activate_; Microsoft::WRL::ComPtr<IMFTransform> encoder_; Microsoft::WRL::ComPtr<ICodecAPI> codec_api_; + Microsoft::WRL::ComPtr<IMFMediaEventGenerator> event_generator_; DWORD input_stream_id_; DWORD output_stream_id_; @@ -134,6 +144,7 @@ class MEDIA_GPU_EXPORT MediaFoundationVideoEncodeAccelerator Microsoft::WRL::ComPtr<IMFMediaType> imf_input_media_type_; Microsoft::WRL::ComPtr<IMFMediaType> imf_output_media_type_; + bool input_required_; Microsoft::WRL::ComPtr<IMFSample> input_sample_; Microsoft::WRL::ComPtr<IMFSample> output_sample_; diff --git a/chromium/media/learning/common/media_learning_tasks.cc b/chromium/media/learning/common/media_learning_tasks.cc index 4af7bc2e3b1..e579028e441 100644 --- a/chromium/media/learning/common/media_learning_tasks.cc +++ b/chromium/media/learning/common/media_learning_tasks.cc @@ -4,6 +4,8 @@ #include "media/learning/common/media_learning_tasks.h" +#include "base/notreached.h" + namespace media { namespace learning { diff --git a/chromium/media/learning/common/value.cc b/chromium/media/learning/common/value.cc index 757a8b3d33d..b3fedfb7f68 100644 --- a/chromium/media/learning/common/value.cc +++ b/chromium/media/learning/common/value.cc @@ -4,6 +4,8 @@ #include "media/learning/common/value.h" +#include <cstring> + #include "base/hash/hash.h" namespace media { diff --git a/chromium/media/learning/impl/extra_trees_trainer.cc b/chromium/media/learning/impl/extra_trees_trainer.cc index b820c6a64d8..572c0f1f56f 100644 --- a/chromium/media/learning/impl/extra_trees_trainer.cc +++ b/chromium/media/learning/impl/extra_trees_trainer.cc @@ -7,7 +7,7 @@ #include <set> #include "base/bind.h" -#include "base/logging.h" +#include "base/check_op.h" #include "media/learning/impl/voting_ensemble.h" namespace media { diff --git a/chromium/media/learning/impl/learning_session_impl.cc b/chromium/media/learning/impl/learning_session_impl.cc index 39fe3da2a20..2680a881dd4 100644 --- a/chromium/media/learning/impl/learning_session_impl.cc +++ b/chromium/media/learning/impl/learning_session_impl.cc @@ -8,7 +8,7 @@ #include <utility> #include "base/bind.h" -#include "base/logging.h" +#include "base/check.h" #include "media/learning/impl/distribution_reporter.h" #include "media/learning/impl/learning_task_controller_impl.h" diff --git a/chromium/media/learning/impl/learning_task_controller_impl.cc b/chromium/media/learning/impl/learning_task_controller_impl.cc index 1ec060c00c4..45c812d2489 100644 --- a/chromium/media/learning/impl/learning_task_controller_impl.cc +++ b/chromium/media/learning/impl/learning_task_controller_impl.cc @@ -9,6 +9,8 @@ #include <vector> #include "base/bind.h" +#include "base/check_op.h" +#include "base/notreached.h" #include "media/learning/impl/distribution_reporter.h" #include "media/learning/impl/extra_trees_trainer.h" #include "media/learning/impl/lookup_table_trainer.h" diff --git a/chromium/media/learning/impl/lookup_table_trainer.cc b/chromium/media/learning/impl/lookup_table_trainer.cc index 57ebdbbc0f6..f858e68756e 100644 --- a/chromium/media/learning/impl/lookup_table_trainer.cc +++ b/chromium/media/learning/impl/lookup_table_trainer.cc @@ -6,7 +6,6 @@ #include <map> -#include "base/logging.h" namespace media { namespace learning { diff --git a/chromium/media/learning/impl/model.h b/chromium/media/learning/impl/model.h index 361cb241c53..673b61e4c43 100644 --- a/chromium/media/learning/impl/model.h +++ b/chromium/media/learning/impl/model.h @@ -5,6 +5,7 @@ #ifndef MEDIA_LEARNING_IMPL_MODEL_H_ #define MEDIA_LEARNING_IMPL_MODEL_H_ +#include "base/callback.h" #include "base/component_export.h" #include "media/learning/common/labelled_example.h" #include "media/learning/common/target_histogram.h" diff --git a/chromium/media/learning/impl/random_number_generator.cc b/chromium/media/learning/impl/random_number_generator.cc index fb503bf2d52..6fcff2f5ebc 100644 --- a/chromium/media/learning/impl/random_number_generator.cc +++ b/chromium/media/learning/impl/random_number_generator.cc @@ -6,7 +6,6 @@ #include <limits> -#include "base/logging.h" #include "base/rand_util.h" namespace media { diff --git a/chromium/media/learning/impl/random_tree_trainer.cc b/chromium/media/learning/impl/random_tree_trainer.cc index c45b757716e..b75deafec04 100644 --- a/chromium/media/learning/impl/random_tree_trainer.cc +++ b/chromium/media/learning/impl/random_tree_trainer.cc @@ -7,7 +7,7 @@ #include <math.h> #include "base/bind.h" -#include "base/logging.h" +#include "base/check_op.h" #include "base/optional.h" #include "base/threading/sequenced_task_runner_handle.h" diff --git a/chromium/media/media_options.gni b/chromium/media/media_options.gni index 1d62f7f714a..ef8fbe4de4a 100644 --- a/chromium/media/media_options.gni +++ b/chromium/media/media_options.gni @@ -29,6 +29,7 @@ media_subcomponent_deps = [ "//media/muxers", "//media/renderers", "//media/video", + "//media/webcodecs", ] if (is_fuchsia) { @@ -158,20 +159,6 @@ enable_library_cdms = (is_linux && !is_chromecast) || is_mac || is_win || is_fuchsia declare_args() { - # Experiment to enable mojo media services (e.g. "renderer", "cdm", see - # |mojo_media_services|). When enabled, selected mojo paths will be enabled in - # the media pipeline and corresponding services will hosted in the selected - # remote process (e.g. "utility" process, see |mojo_media_host|). - # This is explicitly disabled for Fuchsia. - enable_mojo_media = - !is_fuchsia && - (is_android || is_chromeos || is_mac || is_win || enable_library_cdms || - (is_desktop_linux && use_vaapi) || is_chromecast) - - # Enable the TestMojoMediaClient to be used in mojo MediaService. This is for - # testing only and will override the default platform MojoMediaClient, if any. - enable_test_mojo_media_client = false - # When enabled, this feature allows developers to use a runtime flag to # choose the implementation of the renderer that is used. On a build which # enables the mojo renderer, if --disable-mojo-renderer is passed at start-up, @@ -195,65 +182,43 @@ declare_args() { # (which takes precedence), or by setting |alternate_cdm_storage_id_key|. # The key must be a string of at least 32 characters. alternate_cdm_storage_id_key = "" - - # This feature is still under development. See crbug.com/641132. - enable_cdm_proxy = false } -# This feature can only be enabled when using Mojo media. Furthermore, the -# "renderer" service must be enabled, which is asserted in -# //media/mojo/services/BUILD.gn -assert( - !enable_runtime_media_renderer_selection || enable_mojo_media, - "Runtime media renderer selection only applies when mojo media is enabled.") - assert(!enable_cdm_host_verification || is_mac || is_win, "CDM host verification is only supported on Mac and Windows.") -assert(enable_library_cdms || !enable_cdm_proxy, - "CDM Proxy should only be enabled when library CDM is enabled.") - +# Default |mojo_media_services| and |mojo_media_host| on various platforms. See +# comments below for valid values. Can be overridden by gn build arguments from +# the --args command line flag. _default_mojo_media_services = [] -_default_mojo_media_host = "none" - -# Default mojo_media_services and mojo_media_host on various platforms. -# Can be overridden by gn build arguments from the --args command line flag -# for local testing. -if (enable_mojo_media) { - if (is_chromecast) { - _default_mojo_media_services = cast_mojo_media_services - _default_mojo_media_host = cast_mojo_media_host - } else if (is_android) { - _default_mojo_media_services = [ - "cdm", - "audio_decoder", - "video_decoder", - ] - _default_mojo_media_host = "gpu" - } else if (is_chromeos || is_mac || is_win || - (is_desktop_linux && use_vaapi)) { - _default_mojo_media_services = [ "video_decoder" ] - _default_mojo_media_host = "gpu" - } - - if (enable_library_cdms) { - _default_mojo_media_services += [ "cdm" ] +_default_mojo_media_host = "" + +if (is_chromecast) { + _default_mojo_media_services = cast_mojo_media_services + _default_mojo_media_host = cast_mojo_media_host +} else if (is_android) { + _default_mojo_media_services = [ + "cdm", + "audio_decoder", + "video_decoder", + ] + _default_mojo_media_host = "gpu" +} else if (is_chromeos || is_mac || is_win || (is_desktop_linux && use_vaapi)) { + _default_mojo_media_services = [ "video_decoder" ] + _default_mojo_media_host = "gpu" +} - # Having a CDM running means it could require a CdmProxy running in the GPU - # process. - assert( - _default_mojo_media_host == "none" || _default_mojo_media_host == "gpu", - "For now, mojo_media_host should not overwrite it with a different " + - "value if it has been set.") - _default_mojo_media_host = "gpu" - } +# When |enable_library_cdms| is true, the "cdm" service will run in a separate +# CdmService in the CDM (utility) process. Therefor there's no need to specify +# |_default_mojo_media_host| which controls where the MediaService runs in. +# On Fuchsia, |enable_library_cdms| is only enabled to build libclearkeycdm.so, +# the mojo CDM service is not used. +if (enable_library_cdms && !is_fuchsia) { + _default_mojo_media_services += [ "cdm" ] } -# Use another declare_args() to pick up possible overrides of enable_mojo_media -# from --args command line flags. See "gn help declare_args". declare_args() { # A list of mojo media services that should be used in the media pipeline. - # Must not be empty if |enable_mojo_media| is true. # Valid entries in the list are: # - "renderer": Use mojo-based media Renderer service. # - "cdm": Use mojo-based Content Decryption Module. @@ -270,10 +235,10 @@ declare_args() { # |mojo_media_services| still run in the MediaService in the process specified # by "mojo_media_host". # Valid options are: - # - "none": Do not use mojo media service. # - "browser": Use mojo media service hosted in the browser process. # - "gpu": Use mojo media service hosted in the gpu process. # - "utility": Use mojo media service hosted in the utility process. + # - "": Do not use mojo media service. mojo_media_host = _default_mojo_media_host } diff --git a/chromium/media/midi/message_util.cc b/chromium/media/midi/message_util.cc index 539d62dcd3c..350932e9f23 100644 --- a/chromium/media/midi/message_util.cc +++ b/chromium/media/midi/message_util.cc @@ -4,8 +4,8 @@ #include "media/midi/message_util.h" -#include "base/logging.h" #include "base/metrics/histogram_macros.h" +#include "base/notreached.h" namespace midi { diff --git a/chromium/media/midi/midi_manager_mac_unittest.cc b/chromium/media/midi/midi_manager_mac_unittest.cc index d3f54ef9e53..30771290c4f 100644 --- a/chromium/media/midi/midi_manager_mac_unittest.cc +++ b/chromium/media/midi/midi_manager_mac_unittest.cc @@ -10,7 +10,6 @@ #include <memory> -#include "base/logging.h" #include "base/macros.h" #include "base/run_loop.h" #include "base/synchronization/lock.h" diff --git a/chromium/media/midi/midi_manager_unittest.cc b/chromium/media/midi/midi_manager_unittest.cc index 4a3e4abb6e2..b629c7df468 100644 --- a/chromium/media/midi/midi_manager_unittest.cc +++ b/chromium/media/midi/midi_manager_unittest.cc @@ -11,7 +11,7 @@ #include <vector> #include "base/bind.h" -#include "base/logging.h" +#include "base/check_op.h" #include "base/macros.h" #include "base/memory/weak_ptr.h" #include "base/run_loop.h" diff --git a/chromium/media/midi/midi_manager_usb.cc b/chromium/media/midi/midi_manager_usb.cc index d57f374d8b0..b3bce71d379 100644 --- a/chromium/media/midi/midi_manager_usb.cc +++ b/chromium/media/midi/midi_manager_usb.cc @@ -8,7 +8,7 @@ #include <utility> #include "base/bind.h" -#include "base/logging.h" +#include "base/check_op.h" #include "base/strings/stringprintf.h" #include "media/midi/midi_service.h" #include "media/midi/task_service.h" diff --git a/chromium/media/midi/midi_message_queue.cc b/chromium/media/midi/midi_message_queue.cc index 08501ec772c..1beb682c1cb 100644 --- a/chromium/media/midi/midi_message_queue.cc +++ b/chromium/media/midi/midi_message_queue.cc @@ -6,7 +6,8 @@ #include <algorithm> -#include "base/logging.h" +#include "base/check_op.h" +#include "base/notreached.h" #include "media/midi/message_util.h" namespace midi { diff --git a/chromium/media/mojo/BUILD.gn b/chromium/media/mojo/BUILD.gn index 4401902c5b4..7005cfbbb78 100644 --- a/chromium/media/mojo/BUILD.gn +++ b/chromium/media/mojo/BUILD.gn @@ -18,41 +18,32 @@ buildflag_header("buildflags") { enable_mojo_media_in_gpu_process = false enable_mojo_media_in_utility_process = false - if (!enable_mojo_media) { - assert(mojo_media_services == [], "Mojo media is not enabled") - assert(mojo_media_host == "none", "Mojo media is not enabled") - assert(!enable_test_mojo_media_client, "Mojo media is not enabled") - } else { - assert(mojo_media_services != [], "No mojo media service specified") - foreach(service, mojo_media_services) { - if (service == "renderer") { - enable_mojo_renderer = true - } else if (service == "cdm") { - enable_mojo_cdm = true - } else if (service == "audio_decoder") { - enable_mojo_audio_decoder = true - } else if (service == "video_decoder") { - enable_mojo_video_decoder = true - } else { - assert(false, "Invalid mojo media service: $service") - } - } - - if (mojo_media_host == "browser") { - enable_mojo_media_in_browser_process = true - } else if (mojo_media_host == "gpu") { - enable_mojo_media_in_gpu_process = true - } else if (mojo_media_host == "utility") { - enable_mojo_media_in_utility_process = true + foreach(service, mojo_media_services) { + if (service == "renderer") { + enable_mojo_renderer = true + } else if (service == "cdm") { + enable_mojo_cdm = true + } else if (service == "audio_decoder") { + enable_mojo_audio_decoder = true + } else if (service == "video_decoder") { + enable_mojo_video_decoder = true } else { - assert(false, "Invalid mojo media host: $mojo_media_host") + assert(false, "Invalid mojo media service: $service") } } + if (mojo_media_host == "browser") { + enable_mojo_media_in_browser_process = true + } else if (mojo_media_host == "gpu") { + enable_mojo_media_in_gpu_process = true + } else if (mojo_media_host == "utility") { + enable_mojo_media_in_utility_process = true + } else if (mojo_media_host != "") { + assert(false, "Invalid mojo media host: $mojo_media_host") + } + flags = [ "ENABLE_CAST_RENDERER=$enable_cast_renderer", - "ENABLE_MOJO_MEDIA=$enable_mojo_media", - "ENABLE_TEST_MOJO_MEDIA_CLIENT=$enable_test_mojo_media_client", "ENABLE_MOJO_RENDERER=$enable_mojo_renderer", "ENABLE_MOJO_CDM=$enable_mojo_cdm", "ENABLE_MOJO_AUDIO_DECODER=$enable_mojo_audio_decoder", diff --git a/chromium/media/mojo/README.md b/chromium/media/mojo/README.md index 8ed718d7198..d13dc2c607e 100644 --- a/chromium/media/mojo/README.md +++ b/chromium/media/mojo/README.md @@ -62,10 +62,8 @@ specify which remote media components you want to enable. For example, with the following gn arguments, the media pipeline will enable `MojoRenderer` and `MojoCdm`: ``` -enable_mojo_media = true mojo_media_services = ["renderer", "cdm"] ``` -Note that you must set `enable_mojo_media` first. ### Media Mojo Interface Factory @@ -320,11 +318,6 @@ local media components get services from content layer through the `MediaClient` interface. In `MediaService` and `CdmService`, remote media components get services from the through **secure auxiliary services**. -Note that as a `service_manager::Service`, `MediaService` and `CdmService` can -always connect to other `service_manager::Service` hosted by the service_manager -through the `Connector` interface. However, these are generic services that -doesn’t belong to any individual `RenderFrame`, or even user profile. - Some services do require `RenderFrame` or user profile identity, e.g. file system. Since media components all belong to a given `RenderFrame`, we must maintain the frame identity when accessing these services for security reasons. @@ -332,10 +325,12 @@ These services are called secure auxiliary services. `FrameServiceBase` is a base class for all secure auxiliary services to help manage the lifetime of these services (e.g. to handle navigation). -In `MediaInterfaceProxy`, when we request `media::mojom::InterfaceFactory` in -the `MediaService` or `CdmService`, we call `GetFrameServices()` to configure -which secure auxiliary services are exposed to the remote components over the -separate `blink::mojom::BrowserInterfaceBroker`. +When a `MediaInterfaceProxy` is created, in addition to providing the +`media::mojom::InterfaceFactory`, the `RenderFrame` is provisioned with a +`media::mojom::FrameInterfaceFactory` that exposes these secure auxiliary +services on a per-frame basis. The `FrameInterfaceFactory` directly provides +services from //content, and it provides a way for //content embedders to +register additional auxiliary services via the `BindEmbedderReceiver()` method. Currently only the remote CDM needs secure auxiliary services. This is a list of currently supported services: @@ -344,7 +339,6 @@ currently supported services: * `PlatformVerification`: to check whether the platform is secure * `CdmFileIO`: for the CDM to store persistent data * `ProvisionFetcher`: for Android MediaDrm device provisioning -* `CdmProxy`: (in progress) ### Security @@ -363,8 +357,7 @@ which process in production, see [Adoption](#Adoption) below. Also note that all the [Secure Auxiliary Services](#Secure-Auxiliary-Services) are running in a more privileged process than the process where the media components that use them run in. For example, all of the existing services run -in the browser process except for the `CdmProxy`, which runs in the GPU process. -They must defend against compromised media components. +in the browser process. They must defend against compromised media components. ### Adoption @@ -405,7 +398,6 @@ They must defend against compromised media components. * `MediaService` in the GPU process (registered in `GpuServiceFactory` with `GpuMojoMediaClient`) * `MojoVideoDecoder` + hardware video decoders such as D3D11VideoDecoder - * Provides `CdmProxy` to the `CdmService` ## Other Services diff --git a/chromium/media/mojo/clients/mojo_cdm.cc b/chromium/media/mojo/clients/mojo_cdm.cc index 1bd9ba9b9e9..acc9d1ca9de 100644 --- a/chromium/media/mojo/clients/mojo_cdm.cc +++ b/chromium/media/mojo/clients/mojo_cdm.cc @@ -21,7 +21,6 @@ #include "media/mojo/clients/mojo_decryptor.h" #include "media/mojo/common/media_type_converters.h" #include "media/mojo/mojom/decryptor.mojom.h" -#include "media/mojo/mojom/interface_factory.mojom.h" #include "services/service_manager/public/cpp/connect.h" #include "services/service_manager/public/mojom/interface_provider.mojom.h" #include "url/origin.h" @@ -43,15 +42,14 @@ void MojoCdm::Create( const url::Origin& security_origin, const CdmConfig& cdm_config, mojo::PendingRemote<mojom::ContentDecryptionModule> remote_cdm, - mojom::InterfaceFactory* interface_factory, const SessionMessageCB& session_message_cb, const SessionClosedCB& session_closed_cb, const SessionKeysChangeCB& session_keys_change_cb, const SessionExpirationUpdateCB& session_expiration_update_cb, CdmCreatedCB cdm_created_cb) { - scoped_refptr<MojoCdm> mojo_cdm(new MojoCdm( - std::move(remote_cdm), interface_factory, session_message_cb, - session_closed_cb, session_keys_change_cb, session_expiration_update_cb)); + scoped_refptr<MojoCdm> mojo_cdm( + new MojoCdm(std::move(remote_cdm), session_message_cb, session_closed_cb, + session_keys_change_cb, session_expiration_update_cb)); // |mojo_cdm| ownership is passed to the promise. auto promise = std::make_unique<CdmInitializedPromise>( @@ -62,13 +60,11 @@ void MojoCdm::Create( } MojoCdm::MojoCdm(mojo::PendingRemote<mojom::ContentDecryptionModule> remote_cdm, - mojom::InterfaceFactory* interface_factory, const SessionMessageCB& session_message_cb, const SessionClosedCB& session_closed_cb, const SessionKeysChangeCB& session_keys_change_cb, const SessionExpirationUpdateCB& session_expiration_update_cb) : remote_cdm_(std::move(remote_cdm)), - interface_factory_(interface_factory), cdm_id_(CdmContext::kInvalidCdmId), session_message_cb_(session_message_cb), session_closed_cb_(session_closed_cb), @@ -305,26 +301,12 @@ Decryptor* MojoCdm::GetDecryptor() { if (decryptor_) return decryptor_.get(); - mojo::PendingRemote<mojom::Decryptor> decryptor_remote; - // Can be called on a different thread. if (decryptor_remote_.is_valid()) { DVLOG(1) << __func__ << ": Using Decryptor exposed by the CDM directly"; - decryptor_remote = std::move(decryptor_remote_); - } else if (interface_factory_ && cdm_id_ != CdmContext::kInvalidCdmId) { -#if BUILDFLAG(ENABLE_CDM_PROXY) - // TODO(xhwang): Pass back info on whether Decryptor is supported by the - // remote CDM. - DVLOG(1) << __func__ << ": Using Decryptor associated with CDM ID " - << cdm_id_ << ", typically hosted by CdmProxy in MediaService"; - interface_factory_->CreateDecryptor( - cdm_id_, decryptor_remote.InitWithNewPipeAndPassReceiver()); -#endif // BUILDFLAG(ENABLE_CDM_PROXY) + decryptor_ = std::make_unique<MojoDecryptor>(std::move(decryptor_remote_)); } - if (decryptor_remote) - decryptor_ = std::make_unique<MojoDecryptor>(std::move(decryptor_remote)); - return decryptor_.get(); } diff --git a/chromium/media/mojo/clients/mojo_cdm.h b/chromium/media/mojo/clients/mojo_cdm.h index 4c1a0ffbeff..45f0fcb09f1 100644 --- a/chromium/media/mojo/clients/mojo_cdm.h +++ b/chromium/media/mojo/clients/mojo_cdm.h @@ -35,10 +35,6 @@ class Origin; namespace media { -namespace mojom { -class InterfaceFactory; -} - class MojoDecryptor; // A ContentDecryptionModule that proxies to a mojom::ContentDecryptionModule. @@ -55,7 +51,6 @@ class MojoCdm : public ContentDecryptionModule, const url::Origin& security_origin, const CdmConfig& cdm_config, mojo::PendingRemote<mojom::ContentDecryptionModule> remote_cdm, - mojom::InterfaceFactory* interface_factory, const SessionMessageCB& session_message_cb, const SessionClosedCB& session_closed_cb, const SessionKeysChangeCB& session_keys_change_cb, @@ -91,7 +86,6 @@ class MojoCdm : public ContentDecryptionModule, private: MojoCdm(mojo::PendingRemote<mojom::ContentDecryptionModule> remote_cdm, - mojom::InterfaceFactory* interface_factory, const SessionMessageCB& session_message_cb, const SessionClosedCB& session_closed_cb, const SessionKeysChangeCB& session_keys_change_cb, @@ -140,7 +134,6 @@ class MojoCdm : public ContentDecryptionModule, THREAD_CHECKER(thread_checker_); mojo::Remote<mojom::ContentDecryptionModule> remote_cdm_; - mojom::InterfaceFactory* interface_factory_; mojo::AssociatedReceiver<ContentDecryptionModuleClient> client_receiver_{ this}; diff --git a/chromium/media/mojo/clients/mojo_cdm_factory.cc b/chromium/media/mojo/clients/mojo_cdm_factory.cc index 118f0abd01f..d606e735eb1 100644 --- a/chromium/media/mojo/clients/mojo_cdm_factory.cc +++ b/chromium/media/mojo/clients/mojo_cdm_factory.cc @@ -64,8 +64,8 @@ void MojoCdmFactory::Create( key_system, cdm_pending_remote.InitWithNewPipeAndPassReceiver()); MojoCdm::Create(key_system, security_origin, cdm_config, - std::move(cdm_pending_remote), interface_factory_, - session_message_cb, session_closed_cb, session_keys_change_cb, + std::move(cdm_pending_remote), session_message_cb, + session_closed_cb, session_keys_change_cb, session_expiration_update_cb, std::move(cdm_created_cb)); } diff --git a/chromium/media/mojo/clients/mojo_cdm_unittest.cc b/chromium/media/mojo/clients/mojo_cdm_unittest.cc index ab0eb09b91d..f7b402b4f54 100644 --- a/chromium/media/mojo/clients/mojo_cdm_unittest.cc +++ b/chromium/media/mojo/clients/mojo_cdm_unittest.cc @@ -99,7 +99,6 @@ class MojoCdmTest : public ::testing::Test { MojoCdm::Create(key_system, url::Origin::Create(GURL(kTestSecurityOrigin)), CdmConfig(), cdm_receiver_.BindNewPipeAndPassRemote(), - nullptr, base::Bind(&MockCdmClient::OnSessionMessage, base::Unretained(&cdm_client_)), base::Bind(&MockCdmClient::OnSessionClosed, diff --git a/chromium/media/mojo/mojom/BUILD.gn b/chromium/media/mojo/mojom/BUILD.gn index 10940dcef80..f86298e18d2 100644 --- a/chromium/media/mojo/mojom/BUILD.gn +++ b/chromium/media/mojo/mojom/BUILD.gn @@ -23,6 +23,7 @@ mojom("mojom") { "decryptor.mojom", "demuxer_stream.mojom", "display_media_information.mojom", + "frame_interface_factory.mojom", "interface_factory.mojom", "key_system_support.mojom", "media_log.mojom", @@ -31,10 +32,10 @@ mojom("mojom") { "media_types.mojom", "output_protection.mojom", "platform_verification.mojom", + "playback_events_recorder.mojom", "provision_fetcher.mojom", "renderer.mojom", "renderer_extensions.mojom", - "soda_service.mojom", "video_decode_perf_history.mojom", "video_decode_stats_recorder.mojom", "video_decoder.mojom", @@ -49,6 +50,8 @@ mojom("mojom") { if (is_android) { sources += [ "android_overlay.mojom" ] + } else { + sources += [ "speech_recognition_service.mojom" ] } if (is_chromecast) { @@ -76,11 +79,6 @@ mojom("mojom") { enabled_features = [ "enable_cast_renderer" ] } - if (enable_cdm_proxy) { - enabled_features = [ "enable_cdm_proxy" ] - sources += [ "cdm_proxy.mojom" ] - } - export_class_attribute_blink = "BLINK_PLATFORM_EXPORT" export_define_blink = "BLINK_PLATFORM_IMPLEMENTATION=1" export_header_blink = "third_party/blink/public/platform/web_common.h" @@ -115,7 +113,7 @@ source_set("unit_tests") { "audio_decoder_config_mojom_traits_unittest.cc", "cdm_key_information_mojom_traits_unittest.cc", "video_decoder_config_mojom_traits_unittest.cc", - "video_encoder_info_mojom_traits_unittest.cc", + "video_encode_accelerator_mojom_traits_unittest.cc", "video_frame_mojom_traits_unittest.cc", ] diff --git a/chromium/media/mojo/mojom/audio_output_stream.mojom b/chromium/media/mojo/mojom/audio_output_stream.mojom index 8e7077e9401..68ed6e7db47 100644 --- a/chromium/media/mojo/mojom/audio_output_stream.mojom +++ b/chromium/media/mojo/mojom/audio_output_stream.mojom @@ -75,8 +75,7 @@ interface AudioOutputStreamProvider { // output streams that are related during audio processing. // This method fails if it is called more than once. Acquire(AudioParameters params, - pending_remote<AudioOutputStreamProviderClient> client, - mojo_base.mojom.UnguessableToken? processing_id); + pending_remote<AudioOutputStreamProviderClient> client); }; interface AudioOutputStreamProviderClient { diff --git a/chromium/media/mojo/mojom/cdm_key_information_mojom_traits.cc b/chromium/media/mojo/mojom/cdm_key_information_mojom_traits.cc index 94a09fef5b2..e3b2182b4e9 100644 --- a/chromium/media/mojo/mojom/cdm_key_information_mojom_traits.cc +++ b/chromium/media/mojo/mojom/cdm_key_information_mojom_traits.cc @@ -4,7 +4,7 @@ #include "media/mojo/mojom/cdm_key_information_mojom_traits.h" -#include "base/logging.h" +#include "base/notreached.h" namespace mojo { diff --git a/chromium/media/mojo/mojom/cdm_proxy.mojom b/chromium/media/mojo/mojom/cdm_proxy.mojom deleted file mode 100644 index d5037632b77..00000000000 --- a/chromium/media/mojo/mojom/cdm_proxy.mojom +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -module media.mojom; - -// An interface that helps proxy part of ContentDecryptionModule (CDM) -// functionalities to a different entity, e.g. hardware CDM modules. -// In general, the interpretation of the method and callback parameters are -// protocol dependent. -// CdmProxy implementation is hosted in the GPU process. -interface CdmProxy { - // See media/cdm/cdm_proxy.h for the following native enums. - [Native] - enum Status; - - [Native] - enum Protocol; - - [Native] - enum Function; - - [Native] - enum KeyType; - - // Initializes the proxy. - // If the proxy created a crypto session, then the ID for the crypto session - // is |crypto_session_id|. - // |cdm_id| can be used to connect the remote media pipeline and CdmProxy. - Initialize(pending_associated_remote<CdmProxyClient> client) - => (Status status, - Protocol protocol, - uint32 crypto_session_id, - int32 cdm_id); - - // Processes and updates the state of the proxy. - // |func| specifies what type of function to use. - // |crypto_session_id| is a value returned from Initialize() or - // CreateMediaCryptoSessions(). - // |input_data| is the input data to be processed. - // |output_data_size| is the expected size of |output_data|. Some protocols - // require this field in order to determine the size of the output, but some - // may completely ignore it. - // The output data is passed back in |output_data|. - Process(Function func, - uint32 crypto_session_id, - array<uint8> input_data, - uint32 output_data_size) => (Status status, - array<uint8> output_data); - - // Creates a crypto session for handling media. - // If extra data has to be passed to further setup the media crypto session, - // pass the data as |input_data|. - // |crypto_session_id| is the ID for the crypto session. - // |output_data| is extra value, if any. - CreateMediaCryptoSession(array<uint8> input_data) => ( - Status status, uint32 crypto_session_id, uint64 output_data); - - // Sets a key in the proxy. - // |crypto_session_id| is the crypto session for decryption. - // |key_id| is the ID of the key. - // |key_blob| is the opaque key blob for decrypting or decoding. - SetKey(uint32 crypto_session_id, array<uint8> key_id, KeyType key_type, - array<uint8> key_blob) => (Status status); - - // Removes a key from the proxy. - // |crypto_session_id| is the crypto session for decryption. - // |key_id| is the ID of the key. - RemoveKey(uint32 crypto_session_id, array<uint8> key_id) => (Status status); -}; - -// Client of CdmProxy. -// CdmProxyClient is running in the fully sandboxed CDM (e.g. utility) process. -interface CdmProxyClient { - // Notifies the client that there has been a hardware reset. - NotifyHardwareReset(); -}; diff --git a/chromium/media/mojo/mojom/cdm_proxy.typemap b/chromium/media/mojo/mojom/cdm_proxy.typemap deleted file mode 100644 index d2f05392218..00000000000 --- a/chromium/media/mojo/mojom/cdm_proxy.typemap +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright 2017 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -mojom = "//media/mojo/mojom/cdm_proxy.mojom" - -public_headers = [ "//media/cdm/cdm_proxy.h" ] - -traits_headers = [ "//media/base/ipc/media_param_traits_macros.h" ] - -deps = [ - "//media", - "//media/base/ipc", -] - -type_mappings = [ - "media.mojom.CdmProxy.Function=::media::CdmProxy::Function", - "media.mojom.CdmProxy.KeyType=::media::CdmProxy::KeyType", - "media.mojom.CdmProxy.Protocol=::media::CdmProxy::Protocol", - "media.mojom.CdmProxy.Status=::media::CdmProxy::Status", -] diff --git a/chromium/media/mojo/mojom/cdm_service.mojom b/chromium/media/mojo/mojom/cdm_service.mojom index a5402a788fa..e154033021c 100644 --- a/chromium/media/mojo/mojom/cdm_service.mojom +++ b/chromium/media/mojo/mojom/cdm_service.mojom @@ -5,6 +5,7 @@ module media.mojom; import "media/mojo/mojom/content_decryption_module.mojom"; +import "media/mojo/mojom/frame_interface_factory.mojom"; import "mojo/public/mojom/base/file_path.mojom"; import "services/service_manager/public/mojom/interface_provider.mojom"; @@ -28,11 +29,12 @@ interface CdmService { [EnableIf=is_mac] pending_remote<SeatbeltExtensionTokenProvider>? token_provider); - // Requests an CdmFactory. |host_interfaces| can optionally be used to provide - // interfaces hosted by the caller to the remote CdmFactory implementation. + // Requests an CdmFactory. |frame_interfaces| can optionally be used to + // provide interfaces hosted by the caller to the remote CdmFactory + // implementation. CreateCdmFactory( pending_receiver<CdmFactory> factory, - pending_remote<service_manager.mojom.InterfaceProvider> host_interfaces); + pending_remote<FrameInterfaceFactory> frame_interfaces); }; // An interface to provide a sandbox seatbelt extension token synchronously. diff --git a/chromium/media/mojo/mojom/frame_interface_factory.mojom b/chromium/media/mojo/mojom/frame_interface_factory.mojom new file mode 100644 index 00000000000..49a153bd05f --- /dev/null +++ b/chromium/media/mojo/mojom/frame_interface_factory.mojom @@ -0,0 +1,24 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +module media.mojom; + +import "media/mojo/mojom/cdm_storage.mojom"; +import "media/mojo/mojom/provision_fetcher.mojom"; +import "mojo/public/mojom/base/generic_pending_receiver.mojom"; + +// A factory for acquiring media mojo interfaces that are bound to a +// RenderFrameHost. +interface FrameInterfaceFactory { + // Binds the ProvisionFetcher for the frame. + CreateProvisionFetcher(pending_receiver<ProvisionFetcher> provision_fetcher); + + // Binds the CdmStorage for the frame. This requires that the frame have + // CDM storage available. + CreateCdmStorage(pending_receiver<CdmStorage> cdm_storage); + + // Binds a generic media frame-bound interface. This is to allow //content + // embedders to provide additional interfaces. + BindEmbedderReceiver(mojo_base.mojom.GenericPendingReceiver receiver); +}; diff --git a/chromium/media/mojo/mojom/interface_factory.mojom b/chromium/media/mojo/mojom/interface_factory.mojom index 8cea7ee2d35..7215013dfa0 100644 --- a/chromium/media/mojo/mojom/interface_factory.mojom +++ b/chromium/media/mojo/mojom/interface_factory.mojom @@ -5,8 +5,6 @@ module media.mojom; import "media/mojo/mojom/audio_decoder.mojom"; -[EnableIf=enable_cdm_proxy] -import "media/mojo/mojom/cdm_proxy.mojom"; import "media/mojo/mojom/decryptor.mojom"; import "media/mojo/mojom/content_decryption_module.mojom"; import "media/mojo/mojom/renderer.mojom"; @@ -63,14 +61,4 @@ interface InterfaceFactory { // this call may be initiated by an untrusted process (e.g. renderer), so the // implementation must fully validate |key_system| before creating the CDM. CreateCdm(string key_system, pending_receiver<ContentDecryptionModule> cdm); - - // Creates a Decryptor associated with the |cdm_id|. - CreateDecryptor(int32 cdm_id, pending_receiver<Decryptor> decryptor); - - // Creates a CdmProxy that proxies part of CDM functionalities to a different - // entity, e.g. hardware CDM modules. The created |cdm_proxy| must match the - // type of the CDM, identified by |cdm_guid|. - [EnableIf=enable_cdm_proxy] - CreateCdmProxy(mojo_base.mojom.Token cdm_guid, - pending_receiver<CdmProxy> cdm_proxy); }; diff --git a/chromium/media/mojo/mojom/key_system_support.mojom b/chromium/media/mojo/mojom/key_system_support.mojom index 06ede1b0ee1..69d608ca7d5 100644 --- a/chromium/media/mojo/mojom/key_system_support.mojom +++ b/chromium/media/mojo/mojom/key_system_support.mojom @@ -16,8 +16,7 @@ struct KeySystemCapability { bool supports_vp9_profile2; array<EncryptionScheme> encryption_schemes; - // Hardware secure codecs and encryption schemes supported by the CDM, - // directly or indirectly through CdmProxy. + // Hardware secure codecs and encryption schemes supported by the CDM. array<VideoCodec> hw_secure_video_codecs; array<EncryptionScheme> hw_secure_encryption_schemes; diff --git a/chromium/media/mojo/mojom/media_metrics_provider.mojom b/chromium/media/mojo/mojom/media_metrics_provider.mojom index bb9ad8db7e4..526e2b17018 100644 --- a/chromium/media/mojo/mojom/media_metrics_provider.mojom +++ b/chromium/media/mojo/mojom/media_metrics_provider.mojom @@ -7,6 +7,7 @@ module media.mojom; import "media/learning/mojo/public/mojom/learning_task_controller.mojom"; import "media/mojo/mojom/media_types.mojom"; import "media/mojo/mojom/video_decode_stats_recorder.mojom"; +import "media/mojo/mojom/playback_events_recorder.mojom"; import "media/mojo/mojom/watch_time_recorder.mojom"; import "mojo/public/mojom/base/time.mojom"; import "ui/gfx/geometry/mojom/geometry.mojom"; @@ -64,8 +65,13 @@ interface MediaMetricsProvider { // Returns a LearningTaskController for the given |taskName|. AcquireLearningTaskController( - string taskName, - pending_receiver<media.learning.mojom.LearningTaskController> controller); + string taskName, + pending_receiver<media.learning.mojom.LearningTaskController> controller); + + // Returns a PlaybackEventsRecorder instance. Implementation may drop the + // |receiver| if it's not interested in recording playback events. + AcquirePlaybackEventsRecorder( + pending_receiver<PlaybackEventsRecorder> receiver); // Can be called multiple times to set properties about a playback. SetHasAudio(AudioCodec codec); diff --git a/chromium/media/mojo/mojom/media_service.mojom b/chromium/media/mojo/mojom/media_service.mojom index 995e6c55d98..738b54a72c7 100644 --- a/chromium/media/mojo/mojom/media_service.mojom +++ b/chromium/media/mojo/mojom/media_service.mojom @@ -4,18 +4,18 @@ module media.mojom; +import "media/mojo/mojom/frame_interface_factory.mojom"; import "media/mojo/mojom/interface_factory.mojom"; -import "services/service_manager/public/mojom/interface_provider.mojom"; // A service to provide media InterfaceFactory, typically to the media pipeline // running in the renderer process. The service itself runs in the process // specified by the |mojo_media_host| gn build flag. The service is always // connected from the browser process. interface MediaService { - // Requests an InterfaceFactory. |host_interfaces| can optionally be used to + // Requests an InterfaceFactory. |frame_interfaces| can optionally be used to // provide interfaces hosted by the caller to the remote InterfaceFactory // implementation. CreateInterfaceFactory( pending_receiver<InterfaceFactory> factory, - pending_remote<service_manager.mojom.InterfaceProvider> host_interfaces); + pending_remote<FrameInterfaceFactory> frame_interfaces); }; diff --git a/chromium/media/mojo/mojom/media_types.mojom b/chromium/media/mojo/mojom/media_types.mojom index 06f95f16679..6b083bfe691 100644 --- a/chromium/media/mojo/mojom/media_types.mojom +++ b/chromium/media/mojo/mojom/media_types.mojom @@ -279,6 +279,7 @@ struct VideoFrame { mojo_base.mojom.DictionaryValue metadata; gfx.mojom.ColorSpace color_space; + HDRMetadata? hdr_metadata; }; // Possible choices for storing VideoFrame data. diff --git a/chromium/media/mojo/mojom/playback_events_recorder.mojom b/chromium/media/mojo/mojom/playback_events_recorder.mojom new file mode 100644 index 00000000000..4971c776c16 --- /dev/null +++ b/chromium/media/mojo/mojom/playback_events_recorder.mojom @@ -0,0 +1,45 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +module media.mojom; + +import "media/mojo/mojom/media_types.mojom"; +import "ui/gfx/geometry/mojom/geometry.mojom"; + +// PlaybackEventsRecorder allows to observe and record events occurring in the +// media pipeline. The interface is called by the renderer process (in +// blink::WebMediaPlayerImpl), while the implementation normally runs in the +// browser. +interface PlaybackEventsRecorder { + // Called when player has been started or resumed. + OnPlaying(); + + // Called when player has been paused. + OnPaused(); + + // Called when player position is being changed. + OnSeeking(); + + // Called when the player has reached the end of the current file. + OnEnded(); + + // Called when playback has failed due to the specified error. + OnError(PipelineStatus status); + + // Called when playback has been suspended while buffering the media. + OnBuffering(); + + // Called after media buffering has completed. The player is in buffering + // state after the following events until this method is called: + // 1. Player initialized (i.e. after PlaybackEventsRecorder is created). + // 2. Seek operation, see OnSeeking(). + // 3. Buffering started, see OnBuffering(). + OnBufferingComplete(); + + // Called when video playback starts and every time video resolution changes. + OnNaturalSizeChanged(gfx.mojom.Size size); + + // Called periodically (e.g. every second) while playback is active. + OnPipelineStatistics(PipelineStatistics stats); +}; diff --git a/chromium/media/mojo/mojom/soda_service.mojom b/chromium/media/mojo/mojom/soda_service.mojom deleted file mode 100644 index eb9057a32db..00000000000 --- a/chromium/media/mojo/mojom/soda_service.mojom +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2020 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -module media.mojom; - -import "media/mojo/mojom/media_types.mojom"; - -// The main interface a client uses to interact with a SODA service -// process. Every renderer can own one or more Remote<SodaContext>, -// with the receiver bound through the BrowserInterfaceBroker. -interface SodaContext { - // Bind the recognizers to the SODA service. - BindRecognizer(pending_receiver<SodaRecognizer> receiver, - pending_remote<SodaRecognizerClient> client); -}; - -// The main interface to a Speech On-Device API (SODA) service process. -// Used by the browser to issue top-level control requests to the service, -// acquired during process launch. -interface SodaService { - // Bind the SODA context to a new instance of SODA. - BindContext(pending_receiver<SodaContext> context); -}; - -// The interface used to pass raw audio from the renderer to the SODA -// service. The remote lives in the renderer process and the receiver -// lives in the SODA process. -interface SodaRecognizer { - // Initialize the SODA instance. SODA will use the SODA recognition client - // to return the recognition events containing the transcribed audio back - // to the originating media. - SendAudioToSoda(AudioDataS16 buffer); -}; - -// The interface used to return speech recognition events from the SODA -// service back to the originating media. The remote lives in the SODA -// process and the receiver lives in the renderer process. -interface SodaRecognizerClient { - // Triggered by SODA on a speech recognition event. - OnSodaRecognitionEvent(string transcription); -}; diff --git a/chromium/media/mojo/mojom/speech_recognition_service.mojom b/chromium/media/mojo/mojom/speech_recognition_service.mojom new file mode 100644 index 00000000000..6835c22c5ce --- /dev/null +++ b/chromium/media/mojo/mojom/speech_recognition_service.mojom @@ -0,0 +1,50 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +module media.mojom; + +import "media/mojo/mojom/media_types.mojom"; + +// The main interface a client uses to interact with a speech recognition +// service process. Every renderer can own one or more +// Remote<SpeechRecognitionContext>, with the receiver bound through the +// BrowserInterfaceBroker. +interface SpeechRecognitionContext { + // Bind the recognizers to the speech recognition service. + BindRecognizer(pending_receiver<SpeechRecognitionRecognizer> receiver, + pending_remote<SpeechRecognitionRecognizerClient> client); +}; + +// The main interface to a speech secognition service process. +// Used by the browser to issue top-level control requests to the service, +// acquired during process launch. +interface SpeechRecognitionService { + // Bind the context to a new instance of the speech recognition. + BindContext(pending_receiver<SpeechRecognitionContext> context); +}; + +// The interface used to pass raw audio from the renderer to the speech +// recognition service. The remote lives in the renderer process and the +// receiver lives in the speech recognition process. +interface SpeechRecognitionRecognizer { + // Initialize the speech recognition instance. The speech recognition client + // will return the recognition events containing the transcribed audio back + // to the originating media. + SendAudioToSpeechRecognitionService(AudioDataS16 buffer); +}; + +// The interface used to return speech recognition events from the speech +// recognition service back to the originating media. The remote lives in the +// speech recognition process and the receiver lives in the renderer process. +interface SpeechRecognitionRecognizerClient { + // Triggered by speech recognition process on a speech recognition event. + OnSpeechRecognitionRecognitionEvent(SpeechRecognitionResult result); +}; + +// A speech recognition result created by the speech service and passed to the +// renderer. +struct SpeechRecognitionResult { + string transcription; + bool is_final; +}; diff --git a/chromium/media/mojo/mojom/typemaps.gni b/chromium/media/mojo/mojom/typemaps.gni index 8e5ef00405d..44e4010102d 100644 --- a/chromium/media/mojo/mojom/typemaps.gni +++ b/chromium/media/mojo/mojom/typemaps.gni @@ -27,7 +27,3 @@ typemaps = [ if (enable_media_drm_storage) { typemaps += [ "//media/mojo/mojom/media_drm_storage.typemap" ] } - -if (enable_cdm_proxy) { - typemaps += [ "//media/mojo/mojom/cdm_proxy.typemap" ] -} diff --git a/chromium/media/mojo/mojom/video_encode_accelerator.mojom b/chromium/media/mojo/mojom/video_encode_accelerator.mojom index b1baa0fd9d4..2bab244c42d 100644 --- a/chromium/media/mojo/mojom/video_encode_accelerator.mojom +++ b/chromium/media/mojo/mojom/video_encode_accelerator.mojom @@ -61,6 +61,17 @@ struct VideoBitrateAllocation { }; // This defines a mojo transport format for +// media::VideoEncodeAccelerator::Config::SpatialLayer. +struct SpatialLayer { + int32 width; + int32 height; + uint32 bitrate_bps; + uint32 framerate; + uint8 max_qp; + uint8 num_of_temporal_layers; +}; + +// This defines a mojo transport format for // media::VideoEncodeAccelerator::Config. struct VideoEncodeAcceleratorConfig { // See media::VideoEncodeAccelerator::Config::ContentType @@ -88,6 +99,7 @@ struct VideoEncodeAcceleratorConfig { StorageType storage_type; bool has_storage_type; // Whether or not config has storage type config ContentType content_type; + array<SpatialLayer> spatial_layers; }; interface VideoEncodeAccelerator { diff --git a/chromium/media/mojo/mojom/video_encode_accelerator.typemap b/chromium/media/mojo/mojom/video_encode_accelerator.typemap index 0f1caea7bf8..be2bfda9985 100644 --- a/chromium/media/mojo/mojom/video_encode_accelerator.typemap +++ b/chromium/media/mojo/mojom/video_encode_accelerator.typemap @@ -31,6 +31,7 @@ type_mappings = [ "media.mojom.VideoBitrateAllocation=::media::VideoBitrateAllocation", "media.mojom.VideoEncodeAccelerator.Error=::media::VideoEncodeAccelerator::Error", "media.mojom.VideoEncodeAcceleratorConfig=::media::VideoEncodeAccelerator::Config", + "media.mojom.SpatialLayer=::media::VideoEncodeAccelerator::Config::SpatialLayer", "media.mojom.VideoEncodeAcceleratorSupportedProfile=::media::VideoEncodeAccelerator::SupportedProfile", "media.mojom.Vp8Metadata=::media::Vp8Metadata", ] diff --git a/chromium/media/mojo/mojom/video_encode_accelerator_mojom_traits.cc b/chromium/media/mojo/mojom/video_encode_accelerator_mojom_traits.cc index 1d3b11b22a3..1a64d95d376 100644 --- a/chromium/media/mojo/mojom/video_encode_accelerator_mojom_traits.cc +++ b/chromium/media/mojo/mojom/video_encode_accelerator_mojom_traits.cc @@ -4,7 +4,7 @@ #include "media/mojo/mojom/video_encode_accelerator_mojom_traits.h" -#include "base/logging.h" +#include "base/notreached.h" #include "base/optional.h" #include "media/base/video_bitrate_allocation.h" #include "mojo/public/cpp/base/time_mojom_traits.h" @@ -196,6 +196,20 @@ bool EnumTraits<media::mojom::VideoEncodeAcceleratorConfig::ContentType, } // static +bool StructTraits<media::mojom::SpatialLayerDataView, + media::VideoEncodeAccelerator::Config::SpatialLayer>:: + Read(media::mojom::SpatialLayerDataView input, + media::VideoEncodeAccelerator::Config::SpatialLayer* output) { + output->width = input.width(); + output->height = input.height(); + output->bitrate_bps = input.bitrate_bps(); + output->framerate = input.framerate(); + output->max_qp = input.max_qp(); + output->num_of_temporal_layers = input.num_of_temporal_layers(); + return true; +} + +// static bool StructTraits<media::mojom::VideoEncodeAcceleratorConfigDataView, media::VideoEncodeAccelerator::Config>:: Read(media::mojom::VideoEncodeAcceleratorConfigDataView input, @@ -235,10 +249,15 @@ bool StructTraits<media::mojom::VideoEncodeAcceleratorConfigDataView, if (!input.ReadContentType(&content_type)) return false; + std::vector<media::VideoEncodeAccelerator::Config::SpatialLayer> + spatial_layers; + if (!input.ReadSpatialLayers(&spatial_layers)) + return false; + *output = media::VideoEncodeAccelerator::Config( input_format, input_visible_size, output_profile, input.initial_bitrate(), initial_framerate, gop_length, h264_output_level, storage_type, - content_type); + content_type, spatial_layers); return true; } diff --git a/chromium/media/mojo/mojom/video_encode_accelerator_mojom_traits.h b/chromium/media/mojo/mojom/video_encode_accelerator_mojom_traits.h index 6b1a5546ec8..eebc8813801 100644 --- a/chromium/media/mojo/mojom/video_encode_accelerator_mojom_traits.h +++ b/chromium/media/mojo/mojom/video_encode_accelerator_mojom_traits.h @@ -132,6 +132,43 @@ struct EnumTraits<media::mojom::VideoEncodeAcceleratorConfig::ContentType, }; template <> +struct StructTraits<media::mojom::SpatialLayerDataView, + media::VideoEncodeAccelerator::Config::SpatialLayer> { + static int32_t width( + const media::VideoEncodeAccelerator::Config::SpatialLayer& input) { + return input.width; + } + + static int32_t height( + const media::VideoEncodeAccelerator::Config::SpatialLayer& input) { + return input.height; + } + + static uint32_t bitrate_bps( + const media::VideoEncodeAccelerator::Config::SpatialLayer& input) { + return input.bitrate_bps; + } + + static uint32_t framerate( + const media::VideoEncodeAccelerator::Config::SpatialLayer& input) { + return input.framerate; + } + + static uint8_t max_qp( + const media::VideoEncodeAccelerator::Config::SpatialLayer& input) { + return input.max_qp; + } + + static uint8_t num_of_temporal_layers( + const media::VideoEncodeAccelerator::Config::SpatialLayer& input) { + return input.num_of_temporal_layers; + } + + static bool Read(media::mojom::SpatialLayerDataView input, + media::VideoEncodeAccelerator::Config::SpatialLayer* output); +}; + +template <> struct StructTraits<media::mojom::VideoEncodeAcceleratorConfigDataView, media::VideoEncodeAccelerator::Config> { static media::VideoPixelFormat input_format( @@ -200,6 +237,11 @@ struct StructTraits<media::mojom::VideoEncodeAcceleratorConfigDataView, return input.content_type; } + static const std::vector<media::VideoEncodeAccelerator::Config::SpatialLayer>& + spatial_layers(const media::VideoEncodeAccelerator::Config& input) { + return input.spatial_layers; + } + static bool Read(media::mojom::VideoEncodeAcceleratorConfigDataView input, media::VideoEncodeAccelerator::Config* output); }; diff --git a/chromium/media/mojo/mojom/video_encode_accelerator_mojom_traits_unittest.cc b/chromium/media/mojo/mojom/video_encode_accelerator_mojom_traits_unittest.cc new file mode 100644 index 00000000000..cfd516058b6 --- /dev/null +++ b/chromium/media/mojo/mojom/video_encode_accelerator_mojom_traits_unittest.cc @@ -0,0 +1,156 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/mojo/mojom/video_encode_accelerator_mojom_traits.h" +#include "media/mojo/mojom/video_encoder_info_mojom_traits.h" + +#include "media/video/video_encode_accelerator.h" +#include "media/video/video_encoder_info.h" +#include "mojo/public/cpp/test_support/test_utils.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace media { + +// These binary operators are implemented here because they are used in this +// unittest. They cannot be enclosed by anonymous namespace, because they must +// be visible by gtest in linking. +bool operator==(const ::media::ScalingSettings& l, + const ::media::ScalingSettings& r) { + return l.min_qp == r.min_qp && l.max_qp == r.max_qp; +} + +bool operator!=(const ::media::ScalingSettings& l, + const ::media::ScalingSettings& r) { + return !(l == r); +} + +bool operator==(const ::media::ResolutionBitrateLimit& l, + const ::media::ResolutionBitrateLimit& r) { + return (l.frame_size == r.frame_size && + l.min_start_bitrate_bps == r.min_start_bitrate_bps && + l.min_bitrate_bps == r.min_bitrate_bps && + l.max_bitrate_bps == r.max_bitrate_bps); +} + +bool operator!=(const ::media::ResolutionBitrateLimit& l, + const ::media::ResolutionBitrateLimit& r) { + return !(l == r); +} + +bool operator==(const ::media::VideoEncoderInfo& l, + const ::media::VideoEncoderInfo& r) { + if (l.implementation_name != r.implementation_name) + return false; + if (l.supports_native_handle != r.supports_native_handle) + return false; + if (l.has_trusted_rate_controller != r.has_trusted_rate_controller) + return false; + if (l.is_hardware_accelerated != r.is_hardware_accelerated) + return false; + if (l.supports_simulcast != r.supports_simulcast) + return false; + if (l.scaling_settings != r.scaling_settings) + return false; + for (size_t i = 0; i < ::media::VideoEncoderInfo::kMaxSpatialLayers; ++i) { + if (l.fps_allocation[i] != r.fps_allocation[i]) + return false; + } + if (l.resolution_bitrate_limits != r.resolution_bitrate_limits) + return false; + return true; +} + +bool operator==( + const ::media::VideoEncodeAccelerator::Config::SpatialLayer& l, + const ::media::VideoEncodeAccelerator::Config::SpatialLayer& r) { + return (l.width == r.width && l.height == r.height && + l.bitrate_bps == r.bitrate_bps && l.framerate == r.framerate && + l.max_qp == r.max_qp && + l.num_of_temporal_layers == r.num_of_temporal_layers); +} + +bool operator==(const ::media::VideoEncodeAccelerator::Config& l, + const ::media::VideoEncodeAccelerator::Config& r) { + return l.input_format == r.input_format && + l.input_visible_size == r.input_visible_size && + l.output_profile == r.output_profile && + l.initial_bitrate == r.initial_bitrate && + l.initial_framerate == r.initial_framerate && + l.gop_length == r.gop_length && + l.h264_output_level == r.h264_output_level && + l.storage_type == r.storage_type && l.content_type == r.content_type && + l.spatial_layers == r.spatial_layers; +} + +TEST(VideoEncoderInfoStructTraitTest, RoundTrip) { + ::media::VideoEncoderInfo input; + input.implementation_name = "FakeVideoEncodeAccelerator"; + // Scaling settings. + input.scaling_settings = ::media::ScalingSettings(12, 123); + // FPS allocation. + for (size_t i = 0; i < ::media::VideoEncoderInfo::kMaxSpatialLayers; ++i) + input.fps_allocation[i] = {5, 5, 10}; + // Resolution bitrate limits. + input.resolution_bitrate_limits.push_back(::media::ResolutionBitrateLimit( + gfx::Size(123, 456), 123456, 123456, 789012)); + input.resolution_bitrate_limits.push_back(::media::ResolutionBitrateLimit( + gfx::Size(789, 1234), 1234567, 1234567, 7890123)); + // Other bool values. + input.supports_native_handle = true; + input.has_trusted_rate_controller = true; + input.is_hardware_accelerated = true; + input.supports_simulcast = true; + + ::media::VideoEncoderInfo output = input; + ASSERT_TRUE(mojo::test::SerializeAndDeserialize<mojom::VideoEncoderInfo>( + &input, &output)); + EXPECT_EQ(input, output); +} + +TEST(SpatialLayerStructTraitTest, RoundTrip) { + ::media::VideoEncodeAccelerator::Config::SpatialLayer input_spatial_layer; + input_spatial_layer.width = 320; + input_spatial_layer.width = 180; + input_spatial_layer.bitrate_bps = 12345678u; + input_spatial_layer.framerate = 24u; + input_spatial_layer.max_qp = 30u; + input_spatial_layer.num_of_temporal_layers = 3u; + ::media::VideoEncodeAccelerator::Config::SpatialLayer output_spatial_layer; + ASSERT_TRUE(mojo::test::SerializeAndDeserialize<mojom::SpatialLayer>( + &input_spatial_layer, &output_spatial_layer)); + EXPECT_EQ(input_spatial_layer, output_spatial_layer); +} + +TEST(VideoEncodeAcceleratorConfigStructTraitTest, RoundTrip) { + std::vector<::media::VideoEncodeAccelerator::Config::SpatialLayer> + input_spatial_layers(3); + gfx::Size kBaseSize(320, 180); + uint32_t kBaseBitrateBps = 123456u; + uint32_t kBaseFramerate = 24u; + for (size_t i = 0; i < input_spatial_layers.size(); ++i) { + input_spatial_layers[i].width = + static_cast<int32_t>(kBaseSize.width() * (i + 1)); + input_spatial_layers[i].height = + static_cast<int32_t>(kBaseSize.height() * (i + 1)); + input_spatial_layers[i].bitrate_bps = kBaseBitrateBps * (i + 1) / 2; + input_spatial_layers[i].framerate = kBaseFramerate * 2 / (i + 1); + input_spatial_layers[i].max_qp = 30 * (i + 1) / 2; + input_spatial_layers[i].num_of_temporal_layers = 3 - i; + } + ::media::VideoEncodeAccelerator::Config input_config( + ::media::PIXEL_FORMAT_NV12, kBaseSize, ::media::VP9PROFILE_PROFILE0, + kBaseBitrateBps, kBaseFramerate, base::nullopt, base::nullopt, + ::media::VideoEncodeAccelerator::Config::StorageType::kDmabuf, + ::media::VideoEncodeAccelerator::Config::ContentType::kCamera, + input_spatial_layers); + DVLOG(4) << input_config.AsHumanReadableString(); + + ::media::VideoEncodeAccelerator::Config output_config{}; + ASSERT_TRUE( + mojo::test::SerializeAndDeserialize<mojom::VideoEncodeAcceleratorConfig>( + &input_config, &output_config)); + DVLOG(4) << output_config.AsHumanReadableString(); + EXPECT_EQ(input_config, output_config); +} +} // namespace media diff --git a/chromium/media/mojo/mojom/video_encoder_info.mojom b/chromium/media/mojo/mojom/video_encoder_info.mojom index e02bead05ae..a299ca7f84e 100644 --- a/chromium/media/mojo/mojom/video_encoder_info.mojom +++ b/chromium/media/mojo/mojom/video_encoder_info.mojom @@ -26,7 +26,7 @@ struct VideoEncoderInfo { bool is_hardware_accelerated; bool supports_simulcast; - ScalingSettings scaling_settings; + ScalingSettings? scaling_settings; // This array size is equal to media::VideoEncoderInfo::kMaxSpatialLayers. array<array<uint8>, 5> fps_allocation; array<ResolutionBitrateLimit> resolution_bitrate_limits; diff --git a/chromium/media/mojo/mojom/video_encoder_info_mojom_traits.h b/chromium/media/mojo/mojom/video_encoder_info_mojom_traits.h index be13d828f3e..7489a2489c8 100644 --- a/chromium/media/mojo/mojom/video_encoder_info_mojom_traits.h +++ b/chromium/media/mojo/mojom/video_encoder_info_mojom_traits.h @@ -75,7 +75,7 @@ class StructTraits<media::mojom::VideoEncoderInfoDataView, const media::VideoEncoderInfo& video_encoder_info) { return video_encoder_info.supports_simulcast; } - static const media::ScalingSettings& scaling_settings( + static const base::Optional<media::ScalingSettings>& scaling_settings( const media::VideoEncoderInfo& video_encoder_info) { return video_encoder_info.scaling_settings; } diff --git a/chromium/media/mojo/mojom/video_encoder_info_mojom_traits_unittest.cc b/chromium/media/mojo/mojom/video_encoder_info_mojom_traits_unittest.cc deleted file mode 100644 index 455ac3bd893..00000000000 --- a/chromium/media/mojo/mojom/video_encoder_info_mojom_traits_unittest.cc +++ /dev/null @@ -1,88 +0,0 @@ -// Copyright 2019 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/mojo/mojom/video_encoder_info_mojom_traits.h" - -#include "media/video/video_encoder_info.h" - -#include "mojo/public/cpp/test_support/test_utils.h" -#include "testing/gtest/include/gtest/gtest.h" - -namespace media { - -// These binary operators are implemented here because they are used in this -// unittest. They cannot be enclosed by anonymous namespace, because they must -// be visible by gtest in linking. -bool operator==(const ::media::ScalingSettings& l, - const ::media::ScalingSettings& r) { - return l.min_qp == r.min_qp && l.max_qp == r.max_qp; -} - -bool operator!=(const ::media::ScalingSettings& l, - const ::media::ScalingSettings& r) { - return !(l == r); -} - -bool operator==(const ::media::ResolutionBitrateLimit& l, - const ::media::ResolutionBitrateLimit& r) { - return (l.frame_size == r.frame_size && - l.min_start_bitrate_bps == r.min_start_bitrate_bps && - l.min_bitrate_bps == r.min_bitrate_bps && - l.max_bitrate_bps == r.max_bitrate_bps); -} - -bool operator!=(const ::media::ResolutionBitrateLimit& l, - const ::media::ResolutionBitrateLimit& r) { - return !(l == r); -} - -bool operator==(const ::media::VideoEncoderInfo& l, - const ::media::VideoEncoderInfo& r) { - if (l.implementation_name != r.implementation_name) - return false; - if (l.supports_native_handle != r.supports_native_handle) - return false; - if (l.has_trusted_rate_controller != r.has_trusted_rate_controller) - return false; - if (l.is_hardware_accelerated != r.is_hardware_accelerated) - return false; - if (l.supports_simulcast != r.supports_simulcast) - return false; - if (l.scaling_settings != r.scaling_settings) - return false; - for (size_t i = 0; i < ::media::VideoEncoderInfo::kMaxSpatialLayers; ++i) { - if (l.fps_allocation[i] != r.fps_allocation[i]) - return false; - } - if (l.resolution_bitrate_limits != r.resolution_bitrate_limits) - return false; - return true; -} - -TEST(VideoEncoderInfoStructTraitTest, RoundTrip) { - ::media::VideoEncoderInfo input; - input.implementation_name = "FakeVideoEncodeAccelerator"; - // Scaling settings. - input.scaling_settings.min_qp = 12; - input.scaling_settings.max_qp = 123; - // FPS allocation. - for (size_t i = 0; i < ::media::VideoEncoderInfo::kMaxSpatialLayers; ++i) - input.fps_allocation[i] = {5, 5, 10}; - // Resolution bitrate limits. - input.resolution_bitrate_limits.push_back(::media::ResolutionBitrateLimit( - gfx::Size(123, 456), 123456, 123456, 789012)); - input.resolution_bitrate_limits.push_back(::media::ResolutionBitrateLimit( - gfx::Size(789, 1234), 1234567, 1234567, 7890123)); - // Other bool values. - input.supports_native_handle = true; - input.has_trusted_rate_controller = true; - input.is_hardware_accelerated = true; - input.supports_simulcast = true; - - ::media::VideoEncoderInfo output = input; - ASSERT_TRUE(mojo::test::SerializeAndDeserialize<mojom::VideoEncoderInfo>( - &input, &output)); - EXPECT_EQ(input, output); -} -} // namespace media diff --git a/chromium/media/mojo/mojom/video_frame_mojom_traits.cc b/chromium/media/mojo/mojom/video_frame_mojom_traits.cc index 464be5dd0cf..b2c93afe712 100644 --- a/chromium/media/mojo/mojom/video_frame_mojom_traits.cc +++ b/chromium/media/mojo/mojom/video_frame_mojom_traits.cc @@ -14,6 +14,7 @@ #include "media/base/color_plane_layout.h" #include "media/base/format_utils.h" #include "media/mojo/common/mojo_shared_buffer_video_frame.h" +#include "media/mojo/mojom/hdr_metadata_mojom_traits.h" #include "mojo/public/cpp/base/time_mojom_traits.h" #include "mojo/public/cpp/base/values_mojom_traits.h" #include "mojo/public/cpp/system/handle.h" @@ -285,6 +286,11 @@ bool StructTraits<media::mojom::VideoFrameDataView, return false; frame->set_color_space(color_space); + base::Optional<media::HDRMetadata> hdr_metadata; + if (!input.ReadHdrMetadata(&hdr_metadata)) + return false; + frame->set_hdr_metadata(std::move(hdr_metadata)); + *output = std::move(frame); return true; } diff --git a/chromium/media/mojo/mojom/video_frame_mojom_traits.h b/chromium/media/mojo/mojom/video_frame_mojom_traits.h index 196243b4817..c064e76b79a 100644 --- a/chromium/media/mojo/mojom/video_frame_mojom_traits.h +++ b/chromium/media/mojo/mojom/video_frame_mojom_traits.h @@ -62,6 +62,11 @@ struct StructTraits<media::mojom::VideoFrameDataView, return input->ColorSpace(); } + static const base::Optional<media::HDRMetadata>& hdr_metadata( + const scoped_refptr<media::VideoFrame>& input) { + return input->hdr_metadata(); + } + static const base::Optional<gpu::VulkanYCbCrInfo>& ycbcr_info( const scoped_refptr<media::VideoFrame>& input) { return input->ycbcr_info(); diff --git a/chromium/media/mojo/services/BUILD.gn b/chromium/media/mojo/services/BUILD.gn index 89d10b71f40..3f39a35e113 100644 --- a/chromium/media/mojo/services/BUILD.gn +++ b/chromium/media/mojo/services/BUILD.gn @@ -15,8 +15,6 @@ jumbo_component("services") { "gpu_mojo_media_client.h", "interface_factory_impl.cc", "interface_factory_impl.h", - "media_interface_provider.cc", - "media_interface_provider.h", "media_metrics_provider.cc", "media_metrics_provider.h", "media_mojo_export.h", @@ -111,6 +109,10 @@ jumbo_component("services") { ] } + if (is_fuchsia) { + deps += [ "//media/fuchsia/metrics" ] + } + if (enable_media_drm_storage) { sources += [ "mojo_media_drm_storage.cc", @@ -134,19 +136,6 @@ jumbo_component("services") { "//media/cdm:cdm_paths", ] - if (enable_cdm_proxy) { - sources += [ - "mojo_cdm_proxy.cc", - "mojo_cdm_proxy.h", - "mojo_cdm_proxy_service.cc", - "mojo_cdm_proxy_service.h", - ] - deps += [ - # Needed by test_mojo_media_client.cc to create ClearKeyCdmProxy. - "//media/cdm/library_cdm/clear_key_cdm:clear_key_cdm_proxy", - ] - } - # TODO(xhwang): Ideally media should not worry about sandbox. Find a way to # remove this dependency. if (is_mac) { @@ -202,10 +191,6 @@ source_set("unit_tests") { "mojo_cdm_helper_unittest.cc", ] - if (enable_cdm_proxy) { - sources += [ "mojo_cdm_proxy_unittest.cc" ] - } - deps += [ "//media/cdm:cdm_api" ] } diff --git a/chromium/media/mojo/services/android_mojo_media_client.cc b/chromium/media/mojo/services/android_mojo_media_client.cc index aae7227cafb..fee056b790f 100644 --- a/chromium/media/mojo/services/android_mojo_media_client.cc +++ b/chromium/media/mojo/services/android_mojo_media_client.cc @@ -34,16 +34,16 @@ std::unique_ptr<AudioDecoder> AndroidMojoMediaClient::CreateAudioDecoder( } std::unique_ptr<CdmFactory> AndroidMojoMediaClient::CreateCdmFactory( - service_manager::mojom::InterfaceProvider* host_interfaces) { - if (!host_interfaces) { + mojom::FrameInterfaceFactory* frame_interfaces) { + if (!frame_interfaces) { NOTREACHED() << "Host interfaces should be provided when using CDM with " << "AndroidMojoMediaClient"; return nullptr; } return std::make_unique<AndroidCdmFactory>( - base::BindRepeating(&CreateProvisionFetcher, host_interfaces), - base::BindRepeating(&CreateMediaDrmStorage, host_interfaces)); + base::BindRepeating(&CreateProvisionFetcher, frame_interfaces), + base::BindRepeating(&CreateMediaDrmStorage, frame_interfaces)); } } // namespace media diff --git a/chromium/media/mojo/services/android_mojo_media_client.h b/chromium/media/mojo/services/android_mojo_media_client.h index 346a7081648..4ab6dc31782 100644 --- a/chromium/media/mojo/services/android_mojo_media_client.h +++ b/chromium/media/mojo/services/android_mojo_media_client.h @@ -23,7 +23,7 @@ class AndroidMojoMediaClient : public MojoMediaClient { scoped_refptr<base::SingleThreadTaskRunner> task_runner) final; std::unique_ptr<CdmFactory> CreateCdmFactory( - service_manager::mojom::InterfaceProvider* host_interfaces) final; + mojom::FrameInterfaceFactory* frame_interfaces) final; private: DISALLOW_COPY_AND_ASSIGN(AndroidMojoMediaClient); diff --git a/chromium/media/mojo/services/android_mojo_util.cc b/chromium/media/mojo/services/android_mojo_util.cc index 6b5a31e43fb..f91415e62a8 100644 --- a/chromium/media/mojo/services/android_mojo_util.cc +++ b/chromium/media/mojo/services/android_mojo_util.cc @@ -6,28 +6,25 @@ #include "media/mojo/services/mojo_media_drm_storage.h" #include "mojo/public/cpp/bindings/pending_remote.h" -#include "services/service_manager/public/mojom/interface_provider.mojom.h" namespace media { namespace android_mojo_util { std::unique_ptr<ProvisionFetcher> CreateProvisionFetcher( - service_manager::mojom::InterfaceProvider* host_interfaces) { - DCHECK(host_interfaces); + media::mojom::FrameInterfaceFactory* frame_interfaces) { + DCHECK(frame_interfaces); mojo::PendingRemote<mojom::ProvisionFetcher> provision_fetcher; - host_interfaces->GetInterface( - mojom::ProvisionFetcher::Name_, - provision_fetcher.InitWithNewPipeAndPassReceiver().PassPipe()); + frame_interfaces->CreateProvisionFetcher( + provision_fetcher.InitWithNewPipeAndPassReceiver()); return std::make_unique<MojoProvisionFetcher>(std::move(provision_fetcher)); } std::unique_ptr<MediaDrmStorage> CreateMediaDrmStorage( - service_manager::mojom::InterfaceProvider* host_interfaces) { - DCHECK(host_interfaces); + media::mojom::FrameInterfaceFactory* frame_interfaces) { + DCHECK(frame_interfaces); mojo::PendingRemote<mojom::MediaDrmStorage> media_drm_storage; - host_interfaces->GetInterface( - mojom::MediaDrmStorage::Name_, - media_drm_storage.InitWithNewPipeAndPassReceiver().PassPipe()); + frame_interfaces->BindEmbedderReceiver(mojo::GenericPendingReceiver( + media_drm_storage.InitWithNewPipeAndPassReceiver())); return std::make_unique<MojoMediaDrmStorage>(std::move(media_drm_storage)); } diff --git a/chromium/media/mojo/services/android_mojo_util.h b/chromium/media/mojo/services/android_mojo_util.h index 62f17839c18..3a41f51e251 100644 --- a/chromium/media/mojo/services/android_mojo_util.h +++ b/chromium/media/mojo/services/android_mojo_util.h @@ -7,25 +7,18 @@ #include <memory> +#include "media/mojo/mojom/frame_interface_factory.mojom.h" #include "media/mojo/services/mojo_media_drm_storage.h" #include "media/mojo/services/mojo_provision_fetcher.h" -namespace service_manager { -namespace mojom { - -class InterfaceProvider; - -} // namespace mojom -} // namespace service_manager - namespace media { namespace android_mojo_util { std::unique_ptr<ProvisionFetcher> CreateProvisionFetcher( - service_manager::mojom::InterfaceProvider* host_interfaces); + media::mojom::FrameInterfaceFactory* frame_interfaces); std::unique_ptr<MediaDrmStorage> CreateMediaDrmStorage( - service_manager::mojom::InterfaceProvider* host_interfaces); + media::mojom::FrameInterfaceFactory* frame_interfaces); } // namespace android_mojo_util } // namespace media diff --git a/chromium/media/mojo/services/cdm_service.cc b/chromium/media/mojo/services/cdm_service.cc index 06a61b8a1e8..7601bfdfda4 100644 --- a/chromium/media/mojo/services/cdm_service.cc +++ b/chromium/media/mojo/services/cdm_service.cc @@ -41,9 +41,8 @@ namespace { // details. class CdmFactoryImpl : public DeferredDestroy<mojom::CdmFactory> { public: - CdmFactoryImpl( - CdmService::Client* client, - mojo::PendingRemote<service_manager::mojom::InterfaceProvider> interfaces) + CdmFactoryImpl(CdmService::Client* client, + mojo::PendingRemote<mojom::FrameInterfaceFactory> interfaces) : client_(client), interfaces_(std::move(interfaces)) { DVLOG(1) << __func__; @@ -99,7 +98,7 @@ class CdmFactoryImpl : public DeferredDestroy<mojom::CdmFactory> { MojoCdmServiceContext cdm_service_context_; CdmService::Client* client_; - mojo::Remote<service_manager::mojom::InterfaceProvider> interfaces_; + mojo::Remote<mojom::FrameInterfaceFactory> interfaces_; mojo::UniqueReceiverSet<mojom::ContentDecryptionModule> cdm_receivers_; std::unique_ptr<media::CdmFactory> cdm_factory_; base::OnceClosure destroy_cb_; @@ -183,15 +182,14 @@ void CdmService::LoadCdm(const base::FilePath& cdm_path) { void CdmService::CreateCdmFactory( mojo::PendingReceiver<mojom::CdmFactory> receiver, - mojo::PendingRemote<service_manager::mojom::InterfaceProvider> - host_interfaces) { + mojo::PendingRemote<mojom::FrameInterfaceFactory> frame_interfaces) { // Ignore receiver if service has already stopped. if (!client_) return; cdm_factory_receivers_.AddReceiver( std::make_unique<CdmFactoryImpl>(client_.get(), - std::move(host_interfaces)), + std::move(frame_interfaces)), std::move(receiver)); } diff --git a/chromium/media/mojo/services/cdm_service.h b/chromium/media/mojo/services/cdm_service.h index 23f683b33f2..f85cb1af521 100644 --- a/chromium/media/mojo/services/cdm_service.h +++ b/chromium/media/mojo/services/cdm_service.h @@ -12,12 +12,12 @@ #include "media/media_buildflags.h" #include "media/mojo/mojom/cdm_service.mojom.h" #include "media/mojo/mojom/content_decryption_module.mojom.h" +#include "media/mojo/mojom/frame_interface_factory.mojom.h" #include "media/mojo/services/deferred_destroy_unique_receiver_set.h" #include "media/mojo/services/media_mojo_export.h" #include "mojo/public/cpp/bindings/pending_receiver.h" #include "mojo/public/cpp/bindings/pending_remote.h" #include "mojo/public/cpp/bindings/receiver.h" -#include "services/service_manager/public/mojom/interface_provider.mojom.h" #if BUILDFLAG(ENABLE_CDM_HOST_VERIFICATION) #include "media/cdm/cdm_host_file.h" @@ -37,11 +37,11 @@ class MEDIA_MOJO_EXPORT CdmService : public mojom::CdmService { // be a no-op if the process is already sandboxed. virtual void EnsureSandboxed() = 0; - // Returns the CdmFactory to be used by MojoCdmService. |host_interfaces| + // Returns the CdmFactory to be used by MojoCdmService. |frame_interfaces| // can be used to request interfaces provided remotely by the host. It may // be a nullptr if the host chose not to bind the InterfacePtr. virtual std::unique_ptr<CdmFactory> CreateCdmFactory( - service_manager::mojom::InterfaceProvider* host_interfaces) = 0; + mojom::FrameInterfaceFactory* frame_interfaces) = 0; #if BUILDFLAG(ENABLE_CDM_HOST_VERIFICATION) // Gets a list of CDM host file paths and put them in |cdm_host_file_paths|. @@ -73,8 +73,7 @@ class MEDIA_MOJO_EXPORT CdmService : public mojom::CdmService { #endif // defined(OS_MACOSX) void CreateCdmFactory( mojo::PendingReceiver<mojom::CdmFactory> receiver, - mojo::PendingRemote<service_manager::mojom::InterfaceProvider> - host_interfaces) final; + mojo::PendingRemote<mojom::FrameInterfaceFactory> frame_interfaces) final; mojo::Receiver<mojom::CdmService> receiver_; std::unique_ptr<Client> client_; diff --git a/chromium/media/mojo/services/cdm_service_unittest.cc b/chromium/media/mojo/services/cdm_service_unittest.cc index 3190e5d3e1b..02aec3d22e6 100644 --- a/chromium/media/mojo/services/cdm_service_unittest.cc +++ b/chromium/media/mojo/services/cdm_service_unittest.cc @@ -12,7 +12,6 @@ #include "media/cdm/default_cdm_factory.h" #include "media/media_buildflags.h" #include "media/mojo/services/cdm_service.h" -#include "media/mojo/services/media_interface_provider.h" #include "mojo/public/cpp/bindings/pending_remote.h" #include "mojo/public/cpp/bindings/remote.h" #include "testing/gmock/include/gmock/gmock.h" @@ -45,7 +44,7 @@ class MockCdmServiceClient : public media::CdmService::Client { MOCK_METHOD0(EnsureSandboxed, void()); std::unique_ptr<media::CdmFactory> CreateCdmFactory( - service_manager::mojom::InterfaceProvider* host_interfaces) override { + mojom::FrameInterfaceFactory* frame_interfaces) override { return std::make_unique<media::DefaultCdmFactory>(); } @@ -74,9 +73,8 @@ class CdmServiceTest : public testing::Test { base::TimeDelta(), base::BindRepeating(&CdmServiceTest::CdmServiceIdle, base::Unretained(this))); - mojo::PendingRemote<service_manager::mojom::InterfaceProvider> interfaces; - auto provider = std::make_unique<MediaInterfaceProvider>( - interfaces.InitWithNewPipeAndPassReceiver()); + mojo::PendingRemote<mojom::FrameInterfaceFactory> interfaces; + ignore_result(interfaces.InitWithNewPipeAndPassReceiver()); ASSERT_FALSE(cdm_factory_remote_); cdm_service_remote_->CreateCdmFactory( diff --git a/chromium/media/mojo/services/gpu_mojo_media_client.cc b/chromium/media/mojo/services/gpu_mojo_media_client.cc index 5074b4350a0..293591767dc 100644 --- a/chromium/media/mojo/services/gpu_mojo_media_client.cc +++ b/chromium/media/mojo/services/gpu_mojo_media_client.cc @@ -42,6 +42,7 @@ #if defined(OS_WIN) #include "media/gpu/windows/d3d11_video_decoder.h" +#include "ui/gl/direct_composition_surface_win.h" #include "ui/gl/gl_angle_util_win.h" #endif // defined(OS_WIN) @@ -117,18 +118,18 @@ GpuMojoMediaClient::GpuMojoMediaClient( scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner, base::WeakPtr<MediaGpuChannelManager> media_gpu_channel_manager, gpu::GpuMemoryBufferFactory* gpu_memory_buffer_factory, - AndroidOverlayMojoFactoryCB android_overlay_factory_cb, - CdmProxyFactoryCB cdm_proxy_factory_cb) + AndroidOverlayMojoFactoryCB android_overlay_factory_cb) : gpu_preferences_(gpu_preferences), gpu_workarounds_(gpu_workarounds), gpu_feature_info_(gpu_feature_info), gpu_task_runner_(std::move(gpu_task_runner)), media_gpu_channel_manager_(std::move(media_gpu_channel_manager)), - android_overlay_factory_cb_(std::move(android_overlay_factory_cb)), + android_overlay_factory_cb_(std::move(android_overlay_factory_cb)) #if BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION) - gpu_memory_buffer_factory_(gpu_memory_buffer_factory), + , + gpu_memory_buffer_factory_(gpu_memory_buffer_factory) #endif // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION) - cdm_proxy_factory_cb_(std::move(cdm_proxy_factory_cb)) { +{ } GpuMojoMediaClient::~GpuMojoMediaClient() = default; @@ -240,8 +241,8 @@ std::unique_ptr<VideoDecoder> GpuMojoMediaClient::CreateVideoDecoder( // ignored. If we can tell that here, then VideoFrameFactory can use it // as a signal about whether it's supposed to get YCbCrInfo rather than // requiring the provider to set |is_vulkan| in the ImageRecord. - auto ycbcr_helper = - YCbCrHelper::Create(gpu_task_runner_, std::move(get_stub_cb)); + auto frame_info_helper = + FrameInfoHelper::Create(gpu_task_runner_, std::move(get_stub_cb)); video_decoder = std::make_unique<MediaCodecVideoDecoder>( gpu_preferences_, gpu_feature_info_, media_log->Clone(), DeviceInfo::GetInstance(), @@ -252,7 +253,7 @@ std::unique_ptr<VideoDecoder> GpuMojoMediaClient::CreateVideoDecoder( std::make_unique<VideoFrameFactoryImpl>( gpu_task_runner_, gpu_preferences_, std::move(image_provider), MaybeRenderEarlyManager::Create(gpu_task_runner_), - std::move(ycbcr_helper))); + std::move(frame_info_helper))); #elif BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION) if (IsNewAcceleratedVideoDecoderUsed(gpu_preferences_)) { @@ -309,7 +310,8 @@ std::unique_ptr<VideoDecoder> GpuMojoMediaClient::CreateVideoDecoder( media_gpu_channel_manager_, command_buffer_id->channel_token, command_buffer_id->route_id), - GetD3D11DeviceCallback(), *d3d11_supported_configs_); + GetD3D11DeviceCallback(), *d3d11_supported_configs_, + gl::DirectCompositionSurfaceWin::IsHDRSupported()); } #endif // defined(OS_WIN) break; @@ -320,7 +322,7 @@ std::unique_ptr<VideoDecoder> GpuMojoMediaClient::CreateVideoDecoder( } std::unique_ptr<CdmFactory> GpuMojoMediaClient::CreateCdmFactory( - service_manager::mojom::InterfaceProvider* interface_provider) { + mojom::FrameInterfaceFactory* interface_provider) { #if defined(OS_ANDROID) return std::make_unique<AndroidCdmFactory>( base::BindRepeating(&CreateProvisionFetcher, interface_provider), @@ -330,14 +332,4 @@ std::unique_ptr<CdmFactory> GpuMojoMediaClient::CreateCdmFactory( #endif // defined(OS_ANDROID) } -#if BUILDFLAG(ENABLE_CDM_PROXY) -std::unique_ptr<CdmProxy> GpuMojoMediaClient::CreateCdmProxy( - const base::Token& cdm_guid) { - if (cdm_proxy_factory_cb_) - return cdm_proxy_factory_cb_.Run(cdm_guid); - - return nullptr; -} -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - } // namespace media diff --git a/chromium/media/mojo/services/gpu_mojo_media_client.h b/chromium/media/mojo/services/gpu_mojo_media_client.h index 889fa7fd67d..e62511df65a 100644 --- a/chromium/media/mojo/services/gpu_mojo_media_client.h +++ b/chromium/media/mojo/services/gpu_mojo_media_client.h @@ -17,7 +17,6 @@ #include "gpu/config/gpu_feature_info.h" #include "gpu/config/gpu_preferences.h" #include "media/base/android_overlay_mojo_factory.h" -#include "media/cdm/cdm_proxy.h" #include "media/media_buildflags.h" #include "media/mojo/services/mojo_media_client.h" #include "media/video/supported_video_decoder_config.h" @@ -34,8 +33,6 @@ class GpuMojoMediaClient : public MojoMediaClient { public: // |media_gpu_channel_manager| must only be used on |gpu_task_runner|, which // is expected to be the GPU main thread task runner. - // |cdm_proxy_factory_cb| can be used to create a CdmProxy. May be null if - // CdmProxy is not supported on the platform. GpuMojoMediaClient( const gpu::GpuPreferences& gpu_preferences, const gpu::GpuDriverBugWorkarounds& gpu_workarounds, @@ -43,8 +40,7 @@ class GpuMojoMediaClient : public MojoMediaClient { scoped_refptr<base::SingleThreadTaskRunner> gpu_task_runner, base::WeakPtr<MediaGpuChannelManager> media_gpu_channel_manager, gpu::GpuMemoryBufferFactory* gpu_memory_buffer_factory, - AndroidOverlayMojoFactoryCB android_overlay_factory_cb, - CdmProxyFactoryCB cdm_proxy_factory_cb); + AndroidOverlayMojoFactoryCB android_overlay_factory_cb); ~GpuMojoMediaClient() final; // MojoMediaClient implementation. @@ -59,10 +55,7 @@ class GpuMojoMediaClient : public MojoMediaClient { RequestOverlayInfoCB request_overlay_info_cb, const gfx::ColorSpace& target_color_space) final; std::unique_ptr<CdmFactory> CreateCdmFactory( - service_manager::mojom::InterfaceProvider* interface_provider) final; -#if BUILDFLAG(ENABLE_CDM_PROXY) - std::unique_ptr<CdmProxy> CreateCdmProxy(const base::Token& cdm_guid) final; -#endif // BUILDFLAG(ENABLE_CDM_PROXY) + mojom::FrameInterfaceFactory* interface_provider) final; private: gpu::GpuPreferences gpu_preferences_; @@ -76,7 +69,6 @@ class GpuMojoMediaClient : public MojoMediaClient { gpu::GpuMemoryBufferFactory* const gpu_memory_buffer_factory_; base::Optional<SupportedVideoDecoderConfigs> cros_supported_configs_; #endif // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION) - CdmProxyFactoryCB cdm_proxy_factory_cb_; #if defined(OS_WIN) base::Optional<SupportedVideoDecoderConfigs> d3d11_supported_configs_; #endif // defined(OS_WIN) diff --git a/chromium/media/mojo/services/interface_factory_impl.cc b/chromium/media/mojo/services/interface_factory_impl.cc index 4b07b310208..d6a15e06943 100644 --- a/chromium/media/mojo/services/interface_factory_impl.cc +++ b/chromium/media/mojo/services/interface_factory_impl.cc @@ -14,7 +14,6 @@ #include "media/mojo/mojom/renderer_extensions.mojom.h" #include "media/mojo/services/mojo_decryptor_service.h" #include "media/mojo/services/mojo_media_client.h" -#include "services/service_manager/public/mojom/interface_provider.mojom.h" #if BUILDFLAG(ENABLE_MOJO_AUDIO_DECODER) #include "media/mojo/services/mojo_audio_decoder_service.h" @@ -35,17 +34,12 @@ #include "media/mojo/services/mojo_cdm_service.h" #endif // BUILDFLAG(ENABLE_MOJO_CDM) -#if BUILDFLAG(ENABLE_CDM_PROXY) -#include "media/mojo/services/mojo_cdm_proxy_service.h" -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - namespace media { InterfaceFactoryImpl::InterfaceFactoryImpl( - mojo::PendingRemote<service_manager::mojom::InterfaceProvider> - host_interfaces, + mojo::PendingRemote<mojom::FrameInterfaceFactory> frame_interfaces, MojoMediaClient* mojo_media_client) - : host_interfaces_(std::move(host_interfaces)), + : frame_interfaces_(std::move(frame_interfaces)), mojo_media_client_(mojo_media_client) { DVLOG(1) << __func__; DCHECK(mojo_media_client_); @@ -96,7 +90,7 @@ void InterfaceFactoryImpl::CreateDefaultRenderer( DVLOG(2) << __func__; #if BUILDFLAG(ENABLE_MOJO_RENDERER) auto renderer = mojo_media_client_->CreateRenderer( - host_interfaces_.get(), base::ThreadTaskRunnerHandle::Get(), &media_log_, + frame_interfaces_.get(), base::ThreadTaskRunnerHandle::Get(), &media_log_, audio_device_id); if (!renderer) { DLOG(ERROR) << "Renderer creation failed."; @@ -126,7 +120,7 @@ void InterfaceFactoryImpl::CreateCastRenderer( mojo::PendingReceiver<media::mojom::Renderer> receiver) { DVLOG(2) << __func__; auto renderer = mojo_media_client_->CreateCastRenderer( - host_interfaces_.get(), base::ThreadTaskRunnerHandle::Get(), &media_log_, + frame_interfaces_.get(), base::ThreadTaskRunnerHandle::Get(), &media_log_, overlay_plane_id); if (!renderer) { DLOG(ERROR) << "Renderer creation failed."; @@ -184,37 +178,6 @@ void InterfaceFactoryImpl::CreateCdm( #endif // BUILDFLAG(ENABLE_MOJO_CDM) } -void InterfaceFactoryImpl::CreateDecryptor( - int cdm_id, - mojo::PendingReceiver<mojom::Decryptor> receiver) { - DVLOG(2) << __func__; - auto mojo_decryptor_service = - MojoDecryptorService::Create(cdm_id, &cdm_service_context_); - if (!mojo_decryptor_service) { - DLOG(ERROR) << "MojoDecryptorService creation failed."; - return; - } - - decryptor_receivers_.Add(std::move(mojo_decryptor_service), - std::move(receiver)); -} -#if BUILDFLAG(ENABLE_CDM_PROXY) -void InterfaceFactoryImpl::CreateCdmProxy( - const base::Token& cdm_guid, - mojo::PendingReceiver<mojom::CdmProxy> receiver) { - DVLOG(2) << __func__; - auto cdm_proxy = mojo_media_client_->CreateCdmProxy(cdm_guid); - if (!cdm_proxy) { - DLOG(ERROR) << "CdmProxy creation failed."; - return; - } - - cdm_proxy_receivers_.Add(std::make_unique<MojoCdmProxyService>( - std::move(cdm_proxy), &cdm_service_context_), - std::move(receiver)); -} -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - void InterfaceFactoryImpl::OnDestroyPending(base::OnceClosure destroy_cb) { DVLOG(1) << __func__; destroy_cb_ = std::move(destroy_cb); @@ -244,11 +207,6 @@ bool InterfaceFactoryImpl::IsEmpty() { return false; #endif // BUILDFLAG(ENABLE_MOJO_CDM) -#if BUILDFLAG(ENABLE_CDM_PROXY) - if (!cdm_proxy_receivers_.empty()) - return false; -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - if (!decryptor_receivers_.empty()) return false; @@ -278,10 +236,6 @@ void InterfaceFactoryImpl::SetReceiverDisconnectHandler() { cdm_receivers_.set_disconnect_handler(disconnect_cb); #endif // BUILDFLAG(ENABLE_MOJO_CDM) -#if BUILDFLAG(ENABLE_CDM_PROXY) - cdm_proxy_receivers_.set_disconnect_handler(disconnect_cb); -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - decryptor_receivers_.set_disconnect_handler(disconnect_cb); } @@ -294,7 +248,8 @@ void InterfaceFactoryImpl::OnReceiverDisconnect() { #if BUILDFLAG(ENABLE_MOJO_CDM) CdmFactory* InterfaceFactoryImpl::GetCdmFactory() { if (!cdm_factory_) { - cdm_factory_ = mojo_media_client_->CreateCdmFactory(host_interfaces_.get()); + cdm_factory_ = + mojo_media_client_->CreateCdmFactory(frame_interfaces_.get()); LOG_IF(ERROR, !cdm_factory_) << "CdmFactory not available."; } return cdm_factory_.get(); diff --git a/chromium/media/mojo/services/interface_factory_impl.h b/chromium/media/mojo/services/interface_factory_impl.h index 79d71cb6e96..bf6d0e64b07 100644 --- a/chromium/media/mojo/services/interface_factory_impl.h +++ b/chromium/media/mojo/services/interface_factory_impl.h @@ -16,6 +16,7 @@ #include "media/mojo/mojom/audio_decoder.mojom.h" #include "media/mojo/mojom/content_decryption_module.mojom.h" #include "media/mojo/mojom/decryptor.mojom.h" +#include "media/mojo/mojom/frame_interface_factory.mojom.h" #include "media/mojo/mojom/interface_factory.mojom.h" #include "media/mojo/mojom/renderer.mojom.h" #include "media/mojo/mojom/video_decoder.mojom.h" @@ -26,11 +27,6 @@ #include "mojo/public/cpp/bindings/receiver.h" #include "mojo/public/cpp/bindings/remote.h" #include "mojo/public/cpp/bindings/unique_receiver_set.h" -#include "services/service_manager/public/mojom/interface_provider.mojom.h" - -#if BUILDFLAG(ENABLE_CDM_PROXY) -#include "media/mojo/mojom/cdm_proxy.mojom.h" -#endif // BUILDFLAG(ENABLE_CDM_PROXY) namespace media { @@ -40,8 +36,7 @@ class MojoMediaClient; class InterfaceFactoryImpl : public DeferredDestroy<mojom::InterfaceFactory> { public: InterfaceFactoryImpl( - mojo::PendingRemote<service_manager::mojom::InterfaceProvider> - host_interfaces, + mojo::PendingRemote<mojom::FrameInterfaceFactory> frame_interfaces, MojoMediaClient* mojo_media_client); ~InterfaceFactoryImpl() final; @@ -74,12 +69,6 @@ class InterfaceFactoryImpl : public DeferredDestroy<mojom::InterfaceFactory> { void CreateCdm( const std::string& key_system, mojo::PendingReceiver<mojom::ContentDecryptionModule> receiver) final; - void CreateDecryptor(int cdm_id, - mojo::PendingReceiver<mojom::Decryptor> receiver) final; -#if BUILDFLAG(ENABLE_CDM_PROXY) - void CreateCdmProxy(const base::Token& cdm_guid, - mojo::PendingReceiver<mojom::CdmProxy> receiver) final; -#endif // BUILDFLAG(ENABLE_CDM_PROXY) // DeferredDestroy<mojom::InterfaceFactory> implemenation. void OnDestroyPending(base::OnceClosure destroy_cb) final; @@ -120,11 +109,7 @@ class InterfaceFactoryImpl : public DeferredDestroy<mojom::InterfaceFactory> { mojo::UniqueReceiverSet<mojom::ContentDecryptionModule> cdm_receivers_; #endif // BUILDFLAG(ENABLE_MOJO_CDM) -#if BUILDFLAG(ENABLE_CDM_PROXY) - mojo::UniqueReceiverSet<mojom::CdmProxy> cdm_proxy_receivers_; -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - - mojo::Remote<service_manager::mojom::InterfaceProvider> host_interfaces_; + mojo::Remote<mojom::FrameInterfaceFactory> frame_interfaces_; mojo::UniqueReceiverSet<mojom::Decryptor> decryptor_receivers_; diff --git a/chromium/media/mojo/services/media_interface_provider.cc b/chromium/media/mojo/services/media_interface_provider.cc deleted file mode 100644 index 844078fca80..00000000000 --- a/chromium/media/mojo/services/media_interface_provider.cc +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/mojo/services/media_interface_provider.h" - -namespace media { - -MediaInterfaceProvider::MediaInterfaceProvider( - mojo::PendingReceiver<service_manager::mojom::InterfaceProvider> receiver) - : receiver_(this, std::move(receiver)) {} - -MediaInterfaceProvider::~MediaInterfaceProvider() = default; - -void MediaInterfaceProvider::GetInterface( - const std::string& interface_name, - mojo::ScopedMessagePipeHandle handle) { - registry_.BindInterface(interface_name, std::move(handle)); -} - -} // namespace media diff --git a/chromium/media/mojo/services/media_interface_provider.h b/chromium/media/mojo/services/media_interface_provider.h deleted file mode 100644 index 26c1e2a652a..00000000000 --- a/chromium/media/mojo/services/media_interface_provider.h +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MEDIA_MOJO_SERVICES_MEDIA_INTERFACE_PROVIDER_H_ -#define MEDIA_MOJO_SERVICES_MEDIA_INTERFACE_PROVIDER_H_ - -#include "media/mojo/services/media_mojo_export.h" -#include "mojo/public/cpp/bindings/pending_receiver.h" -#include "mojo/public/cpp/bindings/receiver.h" -#include "services/service_manager/public/cpp/binder_registry.h" -#include "services/service_manager/public/mojom/interface_provider.mojom.h" - -namespace media { - -class MEDIA_MOJO_EXPORT MediaInterfaceProvider - : public service_manager::mojom::InterfaceProvider { - public: - explicit MediaInterfaceProvider( - mojo::PendingReceiver<service_manager::mojom::InterfaceProvider> - receiver); - ~MediaInterfaceProvider() override; - - service_manager::BinderRegistry* registry() { return ®istry_; } - - private: - // service_manager::mojom::InterfaceProvider: - void GetInterface(const std::string& interface_name, - mojo::ScopedMessagePipeHandle handle) override; - - service_manager::BinderRegistry registry_; - - mojo::Receiver<service_manager::mojom::InterfaceProvider> receiver_; - - DISALLOW_COPY_AND_ASSIGN(MediaInterfaceProvider); -}; - -} // namespace media - -#endif // MEDIA_MOJO_SERVICES_MEDIA_BINDER_REGISTRY_H_ diff --git a/chromium/media/mojo/services/media_metrics_provider.cc b/chromium/media/mojo/services/media_metrics_provider.cc index f6cd6bedd49..e0920fce86b 100644 --- a/chromium/media/mojo/services/media_metrics_provider.cc +++ b/chromium/media/mojo/services/media_metrics_provider.cc @@ -20,7 +20,11 @@ #if !defined(OS_ANDROID) #include "media/filters/decrypting_video_decoder.h" -#endif +#endif // !defined(OS_ANDROID) + +#if defined(OS_FUCHSIA) +#include "media/fuchsia/metrics/fuchsia_playback_events_recorder.h" +#endif // defined(OS_FUCHSIA) namespace media { @@ -290,6 +294,13 @@ void MediaMetricsProvider::AcquireVideoDecodeStatsRecorder( std::move(receiver)); } +void MediaMetricsProvider::AcquirePlaybackEventsRecorder( + mojo::PendingReceiver<mojom::PlaybackEventsRecorder> receiver) { +#if defined(OS_FUCHSIA) + FuchsiaPlaybackEventsRecorder::Create(std::move(receiver)); +#endif +} + void MediaMetricsProvider::AcquireLearningTaskController( const std::string& taskName, mojo::PendingReceiver<media::learning::mojom::LearningTaskController> diff --git a/chromium/media/mojo/services/media_metrics_provider.h b/chromium/media/mojo/services/media_metrics_provider.h index 4141ad2934a..c9857f2c771 100644 --- a/chromium/media/mojo/services/media_metrics_provider.h +++ b/chromium/media/mojo/services/media_metrics_provider.h @@ -121,6 +121,8 @@ class MEDIA_MOJO_EXPORT MediaMetricsProvider mojo::PendingReceiver<mojom::WatchTimeRecorder> receiver) override; void AcquireVideoDecodeStatsRecorder( mojo::PendingReceiver<mojom::VideoDecodeStatsRecorder> receiver) override; + void AcquirePlaybackEventsRecorder( + mojo::PendingReceiver<mojom::PlaybackEventsRecorder> receiver) override; void AcquireLearningTaskController( const std::string& taskName, mojo::PendingReceiver<media::learning::mojom::LearningTaskController> diff --git a/chromium/media/mojo/services/media_service.cc b/chromium/media/mojo/services/media_service.cc index c669262a3c8..2ff11ac5139 100644 --- a/chromium/media/mojo/services/media_service.cc +++ b/chromium/media/mojo/services/media_service.cc @@ -5,7 +5,7 @@ #include "media/mojo/services/media_service.h" #include "base/bind.h" -#include "base/logging.h" +#include "base/check.h" #include "media/media_buildflags.h" #include "media/mojo/mojom/interface_factory.mojom.h" #include "media/mojo/services/interface_factory_impl.h" @@ -25,14 +25,13 @@ MediaService::~MediaService() = default; void MediaService::CreateInterfaceFactory( mojo::PendingReceiver<mojom::InterfaceFactory> receiver, - mojo::PendingRemote<service_manager::mojom::InterfaceProvider> - host_interfaces) { + mojo::PendingRemote<mojom::FrameInterfaceFactory> frame_interfaces) { // Ignore request if service has already stopped. if (!mojo_media_client_) return; interface_factory_receivers_.Add( - std::make_unique<InterfaceFactoryImpl>(std::move(host_interfaces), + std::make_unique<InterfaceFactoryImpl>(std::move(frame_interfaces), mojo_media_client_.get()), std::move(receiver)); } diff --git a/chromium/media/mojo/services/media_service.h b/chromium/media/mojo/services/media_service.h index 9656a8a1064..01f72aec1eb 100644 --- a/chromium/media/mojo/services/media_service.h +++ b/chromium/media/mojo/services/media_service.h @@ -9,6 +9,7 @@ #include "base/macros.h" #include "build/build_config.h" +#include "media/mojo/mojom/frame_interface_factory.mojom.h" #include "media/mojo/mojom/interface_factory.mojom.h" #include "media/mojo/mojom/media_service.mojom.h" #include "media/mojo/services/media_mojo_export.h" @@ -16,7 +17,6 @@ #include "mojo/public/cpp/bindings/pending_remote.h" #include "mojo/public/cpp/bindings/receiver.h" #include "mojo/public/cpp/bindings/unique_receiver_set.h" -#include "services/service_manager/public/mojom/interface_provider.mojom.h" namespace media { @@ -34,8 +34,7 @@ class MEDIA_MOJO_EXPORT MediaService : public mojom::MediaService { // mojom::MediaService implementation: void CreateInterfaceFactory( mojo::PendingReceiver<mojom::InterfaceFactory> receiver, - mojo::PendingRemote<service_manager::mojom::InterfaceProvider> - host_interfaces) final; + mojo::PendingRemote<mojom::FrameInterfaceFactory> frame_interfaces) final; mojo::Receiver<mojom::MediaService> receiver_; diff --git a/chromium/media/mojo/services/media_service_factory.cc b/chromium/media/mojo/services/media_service_factory.cc index 249632aada1..af044a25cf7 100644 --- a/chromium/media/mojo/services/media_service_factory.cc +++ b/chromium/media/mojo/services/media_service_factory.cc @@ -6,7 +6,8 @@ #include <memory> -#include "base/logging.h" +#include "base/notreached.h" +#include "build/build_config.h" #include "media/mojo/buildflags.h" #include "media/mojo/services/gpu_mojo_media_client.h" #include "media/mojo/services/media_service.h" @@ -20,9 +21,7 @@ namespace media { std::unique_ptr<MediaService> CreateMediaService( mojo::PendingReceiver<mojom::MediaService> receiver) { -#if BUILDFLAG(ENABLE_TEST_MOJO_MEDIA_CLIENT) - return CreateMediaServiceForTesting(std::move(receiver)); -#elif defined(OS_ANDROID) +#if defined(OS_ANDROID) return std::make_unique<MediaService>( std::make_unique<AndroidMojoMediaClient>(), std::move(receiver)); #else @@ -39,14 +38,12 @@ std::unique_ptr<MediaService> CreateGpuMediaService( scoped_refptr<base::SingleThreadTaskRunner> task_runner, base::WeakPtr<MediaGpuChannelManager> media_gpu_channel_manager, gpu::GpuMemoryBufferFactory* gpu_memory_buffer_factory, - AndroidOverlayMojoFactoryCB android_overlay_factory_cb, - CdmProxyFactoryCB cdm_proxy_factory_cb) { + AndroidOverlayMojoFactoryCB android_overlay_factory_cb) { return std::make_unique<MediaService>( std::make_unique<GpuMojoMediaClient>( gpu_preferences, gpu_workarounds, gpu_feature_info, task_runner, media_gpu_channel_manager, gpu_memory_buffer_factory, - std::move(android_overlay_factory_cb), - std::move(cdm_proxy_factory_cb)), + std::move(android_overlay_factory_cb)), std::move(receiver)); } diff --git a/chromium/media/mojo/services/media_service_factory.h b/chromium/media/mojo/services/media_service_factory.h index cc55902e763..d0052bf2acf 100644 --- a/chromium/media/mojo/services/media_service_factory.h +++ b/chromium/media/mojo/services/media_service_factory.h @@ -14,7 +14,6 @@ #include "gpu/config/gpu_feature_info.h" #include "gpu/config/gpu_preferences.h" #include "media/base/android_overlay_mojo_factory.h" -#include "media/cdm/cdm_proxy.h" #include "media/mojo/mojom/media_service.mojom.h" #include "media/mojo/services/media_mojo_export.h" #include "media/mojo/services/media_service.h" @@ -29,16 +28,13 @@ namespace media { class MediaGpuChannelManager; // Creates a MediaService instance using the default MojoMediaClient on each -// platform. Uses the TestMojoMediaClient if |enable_test_mojo_media_client| is -// true. +// platform. std::unique_ptr<MediaService> MEDIA_MOJO_EXPORT CreateMediaService(mojo::PendingReceiver<mojom::MediaService> receiver); // Creates a MediaService instance using the GpuMojoMediaClient. // |media_gpu_channel_manager| must only be used on |task_runner|, which is // expected to be the GPU main thread task runner. -// |cdm_proxy_factory_cb| can be used to create a CdmProxy. May be null if -// CdmProxy is not supported on the platform. std::unique_ptr<MediaService> MEDIA_MOJO_EXPORT CreateGpuMediaService( mojo::PendingReceiver<mojom::MediaService> receiver, const gpu::GpuPreferences& gpu_preferences, @@ -47,8 +43,7 @@ std::unique_ptr<MediaService> MEDIA_MOJO_EXPORT CreateGpuMediaService( scoped_refptr<base::SingleThreadTaskRunner> task_runner, base::WeakPtr<MediaGpuChannelManager> media_gpu_channel_manager, gpu::GpuMemoryBufferFactory* gpu_memory_buffer_factory, - AndroidOverlayMojoFactoryCB android_overlay_factory_cb, - CdmProxyFactoryCB cdm_proxy_factory_cb); + AndroidOverlayMojoFactoryCB android_overlay_factory_cb); // Creates a MediaService instance using the TestMojoMediaClient. std::unique_ptr<MediaService> MEDIA_MOJO_EXPORT CreateMediaServiceForTesting( diff --git a/chromium/media/mojo/services/media_service_unittest.cc b/chromium/media/mojo/services/media_service_unittest.cc index a0852d6b6c5..ef5f39687a0 100644 --- a/chromium/media/mojo/services/media_service_unittest.cc +++ b/chromium/media/mojo/services/media_service_unittest.cc @@ -27,7 +27,6 @@ #include "media/mojo/mojom/interface_factory.mojom.h" #include "media/mojo/mojom/media_service.mojom.h" #include "media/mojo/mojom/renderer.mojom.h" -#include "media/mojo/services/media_interface_provider.h" #include "media/mojo/services/media_service_factory.h" #include "mojo/public/cpp/bindings/associated_receiver.h" #include "mojo/public/cpp/bindings/pending_associated_receiver.h" @@ -38,11 +37,6 @@ #include "url/gurl.h" #include "url/origin.h" -#if BUILDFLAG(ENABLE_CDM_PROXY) -#include "media/cdm/cdm_paths.h" // nogncheck -#include "media/mojo/mojom/cdm_proxy.mojom.h" -#endif - namespace media { namespace { @@ -67,28 +61,6 @@ const char kInvalidKeySystem[] = "invalid.key.system"; const char kSecurityOrigin[] = "https://foo.com"; -// Returns a trivial encrypted DecoderBuffer. -scoped_refptr<DecoderBuffer> CreateEncryptedBuffer() { - scoped_refptr<DecoderBuffer> encrypted_buffer(new DecoderBuffer(100)); - encrypted_buffer->set_decrypt_config( - DecryptConfig::CreateCencConfig("dummy_key_id", "0123456789ABCDEF", {})); - return encrypted_buffer; -} - -#if BUILDFLAG(ENABLE_CDM_PROXY) -class MockCdmProxyClient : public mojom::CdmProxyClient { - public: - MockCdmProxyClient() = default; - ~MockCdmProxyClient() override = default; - - // mojom::CdmProxyClient implementation. - MOCK_METHOD0(NotifyHardwareReset, void()); - - private: - DISALLOW_COPY_AND_ASSIGN(MockCdmProxyClient); -}; -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - class MockRendererClient : public mojom::RendererClient { public: MockRendererClient() = default; @@ -127,20 +99,13 @@ ACTION_P(QuitLoop, run_loop) { class MediaServiceTest : public testing::Test { public: MediaServiceTest() - : -#if BUILDFLAG(ENABLE_CDM_PROXY) - cdm_proxy_client_receiver_(&cdm_proxy_client_), -#endif - renderer_client_receiver_(&renderer_client_), - video_stream_(DemuxerStream::VIDEO) { - } + : renderer_client_receiver_(&renderer_client_), + video_stream_(DemuxerStream::VIDEO) {} ~MediaServiceTest() override = default; void SetUp() override { - mojo::PendingRemote<service_manager::mojom::InterfaceProvider> - host_interfaces; - auto provider = std::make_unique<MediaInterfaceProvider>( - host_interfaces.InitWithNewPipeAndPassReceiver()); + mojo::PendingRemote<mojom::FrameInterfaceFactory> frame_interfaces; + ignore_result(frame_interfaces.InitWithNewPipeAndPassReceiver()); media_service_impl_ = CreateMediaServiceForTesting( media_service_.BindNewPipeAndPassReceiver()); @@ -150,7 +115,7 @@ class MediaServiceTest : public testing::Test { base::Unretained(this))); media_service_->CreateInterfaceFactory( interface_factory_.BindNewPipeAndPassReceiver(), - std::move(host_interfaces)); + std::move(frame_interfaces)); } MOCK_METHOD3(OnCdmInitialized, @@ -182,58 +147,6 @@ class MediaServiceTest : public testing::Test { return cdm_id; } -#if BUILDFLAG(ENABLE_CDM_PROXY) - MOCK_METHOD4(OnCdmProxyInitialized, - void(CdmProxy::Status status, - CdmProxy::Protocol protocol, - uint32_t crypto_session_id, - int cdm_id)); - - // Returns the CDM ID associated with the CdmProxy. - int InitializeCdmProxy(const base::Token& cdm_guid) { - base::RunLoop run_loop; - interface_factory_->CreateCdmProxy(cdm_guid, - cdm_proxy_.BindNewPipeAndPassReceiver()); - - mojo::PendingAssociatedRemote<mojom::CdmProxyClient> client_remote; - cdm_proxy_client_receiver_.Bind( - client_remote.InitWithNewEndpointAndPassReceiver()); - int cdm_id = CdmContext::kInvalidCdmId; - - EXPECT_CALL(*this, OnCdmProxyInitialized(CdmProxy::Status::kOk, _, _, _)) - .WillOnce(DoAll(SaveArg<3>(&cdm_id), QuitLoop(&run_loop))); - cdm_proxy_->Initialize( - std::move(client_remote), - base::BindOnce(&MediaServiceTest::OnCdmProxyInitialized, - base::Unretained(this))); - run_loop.Run(); - return cdm_id; - } -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - - MOCK_METHOD2(OnDecrypted, - void(Decryptor::Status, scoped_refptr<DecoderBuffer>)); - - void CreateDecryptor(int cdm_id, bool expected_result) { - base::RunLoop run_loop; - mojo::PendingRemote<mojom::Decryptor> decryptor_remote; - interface_factory_->CreateDecryptor( - cdm_id, decryptor_remote.InitWithNewPipeAndPassReceiver()); - MojoDecryptor mojo_decryptor(std::move(decryptor_remote)); - - // In the success case, there's no decryption key to decrypt the buffer so - // we would expect no-key. - auto expected_status = - expected_result ? Decryptor::kNoKey : Decryptor::kError; - - EXPECT_CALL(*this, OnDecrypted(expected_status, _)) - .WillOnce(QuitLoop(&run_loop)); - mojo_decryptor.Decrypt( - Decryptor::kVideo, CreateEncryptedBuffer(), - base::BindOnce(&MediaServiceTest::OnDecrypted, base::Unretained(this))); - run_loop.Run(); - } - MOCK_METHOD1(OnRendererInitialized, void(bool)); void InitializeRenderer(const VideoDecoderConfig& video_config, @@ -276,12 +189,6 @@ class MediaServiceTest : public testing::Test { std::unique_ptr<MediaService> media_service_impl_; -#if BUILDFLAG(ENABLE_CDM_PROXY) - mojo::Remote<mojom::CdmProxy> cdm_proxy_; - NiceMock<MockCdmProxyClient> cdm_proxy_client_; - mojo::AssociatedReceiver<mojom::CdmProxyClient> cdm_proxy_client_receiver_; -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - NiceMock<MockRendererClient> renderer_client_; mojo::AssociatedReceiver<mojom::RendererClient> renderer_client_receiver_; @@ -311,11 +218,6 @@ TEST_F(MediaServiceTest, InitializeCdm_Success) { TEST_F(MediaServiceTest, InitializeCdm_InvalidKeySystem) { InitializeCdm(kInvalidKeySystem, false); } - -TEST_F(MediaServiceTest, Decryptor_WithCdm) { - int cdm_id = InitializeCdm(kClearKeyKeySystem, true); - CreateDecryptor(cdm_id, true); -} #endif // BUILDFLAG(ENABLE_MOJO_CDM) && !defined(OS_ANDROID) #if BUILDFLAG(ENABLE_MOJO_RENDERER) @@ -324,43 +226,6 @@ TEST_F(MediaServiceTest, InitializeRenderer) { } #endif // BUILDFLAG(ENABLE_MOJO_RENDERER) -#if BUILDFLAG(ENABLE_CDM_PROXY) -TEST_F(MediaServiceTest, CdmProxy) { - InitializeCdmProxy(kClearKeyCdmGuid); -} - -TEST_F(MediaServiceTest, Decryptor_WithCdmProxy) { - int cdm_id = InitializeCdmProxy(kClearKeyCdmGuid); - CreateDecryptor(cdm_id, true); -} - -TEST_F(MediaServiceTest, Decryptor_WrongCdmId) { - int cdm_id = InitializeCdmProxy(kClearKeyCdmGuid); - CreateDecryptor(cdm_id + 1, false); -} - -TEST_F(MediaServiceTest, CdmProxyPreventsIdling) { - InitializeCdmProxy(kClearKeyCdmGuid); - - // Disconnecting InterfaceFactory should not terminate the MediaService since - // there is still a CdmProxy hosted. - interface_factory_.reset(); - cdm_proxy_.FlushForTesting(); - - // Disconnecting CdmProxy will cause the service to idle since no other - // connections should be active. - base::RunLoop run_loop; - EXPECT_CALL(*this, OnMediaServiceIdle()).WillOnce(QuitLoop(&run_loop)); - cdm_proxy_.reset(); - run_loop.Run(); -} -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - -TEST_F(MediaServiceTest, Decryptor_WithoutCdmOrCdmProxy) { - // Creating decryptor without creating CDM or CdmProxy. - CreateDecryptor(1, false); -} - TEST_F(MediaServiceTest, InterfaceFactoryPreventsIdling) { // The service should not idle during this operation. interface_factory_.FlushForTesting(); diff --git a/chromium/media/mojo/services/mojo_audio_output_stream_provider.cc b/chromium/media/mojo/services/mojo_audio_output_stream_provider.cc index 111ffb0cee0..7a1ab08dfbd 100644 --- a/chromium/media/mojo/services/mojo_audio_output_stream_provider.cc +++ b/chromium/media/mojo/services/mojo_audio_output_stream_provider.cc @@ -38,8 +38,8 @@ MojoAudioOutputStreamProvider::~MojoAudioOutputStreamProvider() { void MojoAudioOutputStreamProvider::Acquire( const AudioParameters& params, - mojo::PendingRemote<mojom::AudioOutputStreamProviderClient> provider_client, - const base::Optional<base::UnguessableToken>& processing_id) { + mojo::PendingRemote<mojom::AudioOutputStreamProviderClient> + provider_client) { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); // |processing_id| gets dropped here. It's not supported outside of the audio // service. As this class is slated for removal, it will not be updated to diff --git a/chromium/media/mojo/services/mojo_audio_output_stream_provider.h b/chromium/media/mojo/services/mojo_audio_output_stream_provider.h index a787e455853..52c23ee60ee 100644 --- a/chromium/media/mojo/services/mojo_audio_output_stream_provider.h +++ b/chromium/media/mojo/services/mojo_audio_output_stream_provider.h @@ -45,11 +45,9 @@ class MEDIA_MOJO_EXPORT MojoAudioOutputStreamProvider private: // mojom::AudioOutputStreamProvider implementation. - void Acquire( - const AudioParameters& params, - mojo::PendingRemote<mojom::AudioOutputStreamProviderClient> - provider_client, - const base::Optional<base::UnguessableToken>& processing_id) override; + void Acquire(const AudioParameters& params, + mojo::PendingRemote<mojom::AudioOutputStreamProviderClient> + provider_client) override; // Called when |audio_output_| had an error. void CleanUp(bool had_error); diff --git a/chromium/media/mojo/services/mojo_audio_output_stream_provider_unittest.cc b/chromium/media/mojo/services/mojo_audio_output_stream_provider_unittest.cc index 6484b81d97b..04fa66ef823 100644 --- a/chromium/media/mojo/services/mojo_audio_output_stream_provider_unittest.cc +++ b/chromium/media/mojo/services/mojo_audio_output_stream_provider_unittest.cc @@ -92,12 +92,12 @@ TEST(MojoAudioOutputStreamProviderTest, AcquireTwice_BadMessage) { mojo::PendingRemote<mojom::AudioOutputStreamProviderClient> client_1; ignore_result(client_1.InitWithNewPipeAndPassReceiver()); provider_remote->Acquire(media::AudioParameters::UnavailableDeviceParams(), - std::move(client_1), base::nullopt); + std::move(client_1)); mojo::PendingRemote<mojom::AudioOutputStreamProviderClient> client_2; ignore_result(client_2.InitWithNewPipeAndPassReceiver()); provider_remote->Acquire(media::AudioParameters::UnavailableDeviceParams(), - std::move(client_2), base::nullopt); + std::move(client_2)); EXPECT_CALL(deleter, Run(provider)).WillOnce(DeleteArg<0>()); base::RunLoop().RunUntilIdle(); @@ -130,7 +130,7 @@ TEST(MojoAudioOutputStreamProviderTest, mojo::PendingRemote<mojom::AudioOutputStreamProviderClient> client; ignore_result(client.InitWithNewPipeAndPassReceiver()); - provider_remote->Acquire(params, std::move(client), base::nullopt); + provider_remote->Acquire(params, std::move(client)); #if defined(OS_ANDROID) base::RunLoop().RunUntilIdle(); diff --git a/chromium/media/mojo/services/mojo_cdm_helper.cc b/chromium/media/mojo/services/mojo_cdm_helper.cc index ae74069d828..2d5761e44e5 100644 --- a/chromium/media/mojo/services/mojo_cdm_helper.cc +++ b/chromium/media/mojo/services/mojo_cdm_helper.cc @@ -11,13 +11,11 @@ #include "media/mojo/services/mojo_cdm_file_io.h" #include "mojo/public/cpp/bindings/callback_helpers.h" #include "mojo/public/cpp/bindings/pending_remote.h" -#include "services/service_manager/public/cpp/connect.h" namespace media { -MojoCdmHelper::MojoCdmHelper( - service_manager::mojom::InterfaceProvider* interface_provider) - : interface_provider_(interface_provider) {} +MojoCdmHelper::MojoCdmHelper(mojom::FrameInterfaceFactory* frame_interfaces) + : frame_interfaces_(frame_interfaces) {} MojoCdmHelper::~MojoCdmHelper() = default; @@ -39,27 +37,6 @@ cdm::FileIO* MojoCdmHelper::CreateCdmFileIO(cdm::FileIOClient* client) { return cdm_file_io; } -#if BUILDFLAG(ENABLE_CDM_PROXY) -cdm::CdmProxy* MojoCdmHelper::CreateCdmProxy(cdm::CdmProxyClient* client) { - DVLOG(3) << __func__; - if (cdm_proxy_) { - DVLOG(1) << __func__ << ": Only one outstanding CdmProxy allowed."; - return nullptr; - } - - mojo::PendingRemote<mojom::CdmProxy> cdm_proxy_remote; - service_manager::GetInterface<mojom::CdmProxy>( - interface_provider_, cdm_proxy_remote.InitWithNewPipeAndPassReceiver()); - cdm_proxy_ = - std::make_unique<MojoCdmProxy>(std::move(cdm_proxy_remote), client); - return cdm_proxy_.get(); -} - -int MojoCdmHelper::GetCdmProxyCdmId() { - return cdm_proxy_ ? cdm_proxy_->GetCdmId() : CdmContext::kInvalidCdmId; -} -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - cdm::Buffer* MojoCdmHelper::CreateCdmBuffer(size_t capacity) { return GetAllocator()->CreateCdmBuffer(capacity); } @@ -118,8 +95,8 @@ void MojoCdmHelper::ReportFileReadSize(int file_size_bytes) { void MojoCdmHelper::ConnectToCdmStorage() { if (!cdm_storage_remote_) { - service_manager::GetInterface<mojom::CdmStorage>( - interface_provider_, cdm_storage_remote_.BindNewPipeAndPassReceiver()); + frame_interfaces_->CreateCdmStorage( + cdm_storage_remote_.BindNewPipeAndPassReceiver()); } } @@ -131,16 +108,15 @@ CdmAllocator* MojoCdmHelper::GetAllocator() { void MojoCdmHelper::ConnectToOutputProtection() { if (!output_protection_) { - service_manager::GetInterface<mojom::OutputProtection>( - interface_provider_, output_protection_.BindNewPipeAndPassReceiver()); + frame_interfaces_->BindEmbedderReceiver(mojo::GenericPendingReceiver( + output_protection_.BindNewPipeAndPassReceiver())); } } void MojoCdmHelper::ConnectToPlatformVerification() { if (!platform_verification_) { - interface_provider_->GetInterface( - mojom::PlatformVerification::Name_, - platform_verification_.BindNewPipeAndPassReceiver().PassPipe()); + frame_interfaces_->BindEmbedderReceiver(mojo::GenericPendingReceiver( + platform_verification_.BindNewPipeAndPassReceiver())); } } diff --git a/chromium/media/mojo/services/mojo_cdm_helper.h b/chromium/media/mojo/services/mojo_cdm_helper.h index 4a875028470..25bf7d13790 100644 --- a/chromium/media/mojo/services/mojo_cdm_helper.h +++ b/chromium/media/mojo/services/mojo_cdm_helper.h @@ -14,23 +14,13 @@ #include "media/cdm/cdm_auxiliary_helper.h" #include "media/media_buildflags.h" #include "media/mojo/mojom/cdm_storage.mojom.h" +#include "media/mojo/mojom/frame_interface_factory.mojom.h" #include "media/mojo/mojom/output_protection.mojom.h" #include "media/mojo/mojom/platform_verification.mojom.h" #include "media/mojo/services/media_mojo_export.h" #include "media/mojo/services/mojo_cdm_file_io.h" #include "mojo/public/cpp/bindings/remote.h" -#if BUILDFLAG(ENABLE_CDM_PROXY) -#include "media/mojo/mojom/cdm_proxy.mojom.h" -#include "media/mojo/services/mojo_cdm_proxy.h" -#endif - -namespace service_manager { -namespace mojom { -class InterfaceProvider; -} -} // namespace service_manager - namespace media { // Helper class that connects the CDM to various auxiliary services. All @@ -39,17 +29,12 @@ namespace media { class MEDIA_MOJO_EXPORT MojoCdmHelper final : public CdmAuxiliaryHelper, public MojoCdmFileIO::Delegate { public: - explicit MojoCdmHelper( - service_manager::mojom::InterfaceProvider* interface_provider); + explicit MojoCdmHelper(mojom::FrameInterfaceFactory* frame_interfaces); ~MojoCdmHelper() final; // CdmAuxiliaryHelper implementation. void SetFileReadCB(FileReadCB file_read_cb) final; cdm::FileIO* CreateCdmFileIO(cdm::FileIOClient* client) final; -#if BUILDFLAG(ENABLE_CDM_PROXY) - cdm::CdmProxy* CreateCdmProxy(cdm::CdmProxyClient* client) final; - int GetCdmProxyCdmId() final; -#endif cdm::Buffer* CreateCdmBuffer(size_t capacity) final; std::unique_ptr<VideoFrameImpl> CreateCdmVideoFrame() final; void QueryStatus(QueryStatusCB callback) final; @@ -72,7 +57,7 @@ class MEDIA_MOJO_EXPORT MojoCdmHelper final : public CdmAuxiliaryHelper, void ConnectToPlatformVerification(); // Provides interfaces when needed. - service_manager::mojom::InterfaceProvider* interface_provider_; + mojom::FrameInterfaceFactory* frame_interfaces_; // Connections to the additional services. For the mojom classes, if a // connection error occurs, we will not be able to reconnect to the @@ -89,10 +74,6 @@ class MEDIA_MOJO_EXPORT MojoCdmHelper final : public CdmAuxiliaryHelper, // TODO(xhwang): Switch to use UniquePtrComparator. std::vector<std::unique_ptr<MojoCdmFileIO>> cdm_file_io_set_; -#if BUILDFLAG(ENABLE_CDM_PROXY) - std::unique_ptr<MojoCdmProxy> cdm_proxy_; -#endif - base::WeakPtrFactory<MojoCdmHelper> weak_factory_{this}; DISALLOW_COPY_AND_ASSIGN(MojoCdmHelper); }; diff --git a/chromium/media/mojo/services/mojo_cdm_helper_unittest.cc b/chromium/media/mojo/services/mojo_cdm_helper_unittest.cc index fcdec73a081..1b52d254b27 100644 --- a/chromium/media/mojo/services/mojo_cdm_helper_unittest.cc +++ b/chromium/media/mojo/services/mojo_cdm_helper_unittest.cc @@ -13,8 +13,6 @@ #include "mojo/public/cpp/bindings/associated_receiver.h" #include "mojo/public/cpp/bindings/pending_receiver.h" #include "mojo/public/cpp/bindings/self_owned_receiver.h" -#include "services/service_manager/public/cpp/binder_registry.h" -#include "services/service_manager/public/mojom/interface_provider.mojom.h" #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" @@ -66,36 +64,27 @@ class MockCdmStorage : public mojom::CdmStorage { mojo::AssociatedReceiver<mojom::CdmFile> client_receiver_{&cdm_file_}; }; -void CreateCdmStorage(mojo::PendingReceiver<mojom::CdmStorage> receiver) { - mojo::MakeSelfOwnedReceiver(std::make_unique<MockCdmStorage>(), - std::move(receiver)); -} - -class TestInterfaceProvider : public service_manager::mojom::InterfaceProvider { +class TestFrameInterfaceFactory : public mojom::FrameInterfaceFactory { public: - TestInterfaceProvider() { - registry_.AddInterface(base::Bind(&CreateCdmStorage)); - } - ~TestInterfaceProvider() override = default; - - void GetInterface(const std::string& interface_name, - mojo::ScopedMessagePipeHandle handle) override { - registry_.BindInterface(interface_name, std::move(handle)); + void CreateProvisionFetcher( + mojo::PendingReceiver<mojom::ProvisionFetcher>) override {} + void CreateCdmStorage( + mojo::PendingReceiver<mojom::CdmStorage> receiver) override { + mojo::MakeSelfOwnedReceiver(std::make_unique<MockCdmStorage>(), + std::move(receiver)); } - - private: - service_manager::BinderRegistry registry_; + void BindEmbedderReceiver(mojo::GenericPendingReceiver) override {} }; } // namespace class MojoCdmHelperTest : public testing::Test { protected: - MojoCdmHelperTest() : helper_(&test_interface_provider_) {} + MojoCdmHelperTest() : helper_(&frame_interfaces_) {} ~MojoCdmHelperTest() override = default; base::test::TaskEnvironment task_environment_; - TestInterfaceProvider test_interface_provider_; + TestFrameInterfaceFactory frame_interfaces_; MockFileIOClient file_io_client_; MojoCdmHelper helper_; }; diff --git a/chromium/media/mojo/services/mojo_cdm_promise.cc b/chromium/media/mojo/services/mojo_cdm_promise.cc index 41b18ec793e..5bd1295a42f 100644 --- a/chromium/media/mojo/services/mojo_cdm_promise.cc +++ b/chromium/media/mojo/services/mojo_cdm_promise.cc @@ -9,7 +9,7 @@ #include <vector> #include "base/bind.h" -#include "base/logging.h" +#include "base/check.h" #include "media/base/content_decryption_module.h" #include "media/base/decryptor.h" diff --git a/chromium/media/mojo/services/mojo_cdm_proxy.cc b/chromium/media/mojo/services/mojo_cdm_proxy.cc deleted file mode 100644 index 4c454fa19c4..00000000000 --- a/chromium/media/mojo/services/mojo_cdm_proxy.cc +++ /dev/null @@ -1,217 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/mojo/services/mojo_cdm_proxy.h" - -#include <vector> - -#include "base/bind.h" -#include "base/callback.h" -#include "base/logging.h" -#include "media/base/cdm_context.h" -#include "mojo/public/cpp/bindings/callback_helpers.h" - -namespace media { - -namespace { - -inline std::ostream& operator<<(std::ostream& out, CdmProxy::Status status) { - switch (status) { - case CdmProxy::Status::kOk: - return out << "kOk"; - case CdmProxy::Status::kFail: - return out << "kFail"; - } - NOTREACHED(); - return out << "Invalid Status!"; -} - -cdm::CdmProxyClient::Status ToCdmStatus(CdmProxy::Status status) { - switch (status) { - case CdmProxy::Status::kOk: - return cdm::CdmProxyClient::Status::kOk; - case CdmProxy::Status::kFail: - return cdm::CdmProxyClient::Status::kFail; - } - - NOTREACHED() << "Unexpected status: " << status; - return cdm::CdmProxyClient::Status::kFail; -} - -cdm::CdmProxyClient::Protocol ToCdmProtocol(CdmProxy::Protocol protocol) { - switch (protocol) { - case CdmProxy::Protocol::kNone: - return cdm::CdmProxyClient::Protocol::kNone; - case CdmProxy::Protocol::kIntel: - return cdm::CdmProxyClient::Protocol::kIntel; - } - - NOTREACHED() << "Unexpected protocol: " << static_cast<int32_t>(protocol); - return cdm::CdmProxyClient::Protocol::kNone; -} - -CdmProxy::Function ToMediaFunction(cdm::CdmProxy::Function function) { - switch (function) { - case cdm::CdmProxy::Function::kIntelNegotiateCryptoSessionKeyExchange: - return CdmProxy::Function::kIntelNegotiateCryptoSessionKeyExchange; - } - - // TODO(xhwang): Return an invalid function? - NOTREACHED() << "Unexpected function: " << static_cast<int32_t>(function); - return CdmProxy::Function::kIntelNegotiateCryptoSessionKeyExchange; -} - -CdmProxy::KeyType ToMediaKeyType(cdm::CdmProxy::KeyType key_type) { - switch (key_type) { - case cdm::CdmProxy::KeyType::kDecryptOnly: - return CdmProxy::KeyType::kDecryptOnly; - case cdm::CdmProxy::KeyType::kDecryptAndDecode: - return CdmProxy::KeyType::kDecryptAndDecode; - } - - NOTREACHED() << "Unexpected key type: " << static_cast<int32_t>(key_type); - return CdmProxy::KeyType::kDecryptOnly; -} - -} // namespace - -MojoCdmProxy::MojoCdmProxy( - mojo::PendingRemote<mojom::CdmProxy> cdm_proxy_remote, - cdm::CdmProxyClient* client) - : cdm_proxy_remote_(std::move(cdm_proxy_remote)), client_(client) { - DVLOG(1) << __func__; - DCHECK(client); -} - -MojoCdmProxy::~MojoCdmProxy() { - DVLOG(1) << __func__; -} - -void MojoCdmProxy::Initialize() { - DVLOG(2) << __func__; - - auto callback = mojo::WrapCallbackWithDefaultInvokeIfNotRun( - base::BindOnce(&MojoCdmProxy::OnInitialized, weak_factory_.GetWeakPtr()), - media::CdmProxy::Status::kFail, media::CdmProxy::Protocol::kNone, 0, - CdmContext::kInvalidCdmId); - cdm_proxy_remote_->Initialize(client_receiver_.BindNewEndpointAndPassRemote(), - std::move(callback)); -} - -void MojoCdmProxy::Process(Function function, - uint32_t crypto_session_id, - const uint8_t* input_data, - uint32_t input_data_size, - uint32_t expected_output_data_size) { - DVLOG(3) << __func__; - CHECK(client_) << "Initialize not called."; - - auto callback = mojo::WrapCallbackWithDefaultInvokeIfNotRun( - base::BindOnce(&MojoCdmProxy::OnProcessed, weak_factory_.GetWeakPtr()), - media::CdmProxy::Status::kFail, std::vector<uint8_t>()); - - cdm_proxy_remote_->Process( - ToMediaFunction(function), crypto_session_id, - std::vector<uint8_t>(input_data, input_data + input_data_size), - expected_output_data_size, std::move(callback)); -} - -void MojoCdmProxy::CreateMediaCryptoSession(const uint8_t* input_data, - uint32_t input_data_size) { - DVLOG(3) << __func__; - CHECK(client_) << "Initialize not called."; - - auto callback = mojo::WrapCallbackWithDefaultInvokeIfNotRun( - base::BindOnce(&MojoCdmProxy::OnMediaCryptoSessionCreated, - weak_factory_.GetWeakPtr()), - media::CdmProxy::Status::kFail, 0, 0); - - cdm_proxy_remote_->CreateMediaCryptoSession( - std::vector<uint8_t>(input_data, input_data + input_data_size), - std::move(callback)); -} - -void MojoCdmProxy::SetKey(uint32_t crypto_session_id, - const uint8_t* key_id, - uint32_t key_id_size, - KeyType key_type, - const uint8_t* key_blob, - uint32_t key_blob_size) { - DVLOG(3) << __func__; - CHECK(client_) << "Initialize not called."; - - auto callback = mojo::WrapCallbackWithDefaultInvokeIfNotRun( - base::BindOnce(&MojoCdmProxy::OnKeySet, weak_factory_.GetWeakPtr()), - media::CdmProxy::Status::kFail); - - cdm_proxy_remote_->SetKey( - crypto_session_id, std::vector<uint8_t>(key_id, key_id + key_id_size), - ToMediaKeyType(key_type), - std::vector<uint8_t>(key_blob, key_blob + key_blob_size), - std::move(callback)); -} - -void MojoCdmProxy::RemoveKey(uint32_t crypto_session_id, - const uint8_t* key_id, - uint32_t key_id_size) { - DVLOG(3) << __func__; - CHECK(client_) << "Initialize not called."; - - auto callback = mojo::WrapCallbackWithDefaultInvokeIfNotRun( - base::BindOnce(&MojoCdmProxy::OnKeyRemoved, weak_factory_.GetWeakPtr()), - media::CdmProxy::Status::kFail); - - cdm_proxy_remote_->RemoveKey( - crypto_session_id, std::vector<uint8_t>(key_id, key_id + key_id_size), - std::move(callback)); -} - -void MojoCdmProxy::NotifyHardwareReset() { - DVLOG(2) << __func__; - client_->NotifyHardwareReset(); -} - -int MojoCdmProxy::GetCdmId() { - DVLOG(2) << __func__ << ": cdm_id = " << cdm_id_; - return cdm_id_; -} - -void MojoCdmProxy::OnInitialized(media::CdmProxy::Status status, - media::CdmProxy::Protocol protocol, - uint32_t crypto_session_id, - int cdm_id) { - DVLOG(3) << __func__ << ": status = " << status - << ", crypto_session_id = " << crypto_session_id; - cdm_id_ = cdm_id; - client_->OnInitialized(ToCdmStatus(status), ToCdmProtocol(protocol), - crypto_session_id); -} - -void MojoCdmProxy::OnProcessed(media::CdmProxy::Status status, - const std::vector<uint8_t>& output_data) { - DVLOG(3) << __func__ << ": status = " << status; - client_->OnProcessed(ToCdmStatus(status), output_data.data(), - output_data.size()); -} - -void MojoCdmProxy::OnMediaCryptoSessionCreated(media::CdmProxy::Status status, - uint32_t crypto_session_id, - uint64_t output_data) { - DVLOG(3) << __func__ << ": status = " << status - << ", crypto_session_id = " << crypto_session_id; - client_->OnMediaCryptoSessionCreated(ToCdmStatus(status), crypto_session_id, - output_data); -} - -void MojoCdmProxy::OnKeySet(media::CdmProxy::Status status) { - DVLOG(3) << __func__ << ": status = " << status; - client_->OnKeySet(ToCdmStatus(status)); -} - -void MojoCdmProxy::OnKeyRemoved(media::CdmProxy::Status status) { - DVLOG(3) << __func__ << ": status = " << status; - client_->OnKeyRemoved(ToCdmStatus(status)); -} - -} // namespace media diff --git a/chromium/media/mojo/services/mojo_cdm_proxy.h b/chromium/media/mojo/services/mojo_cdm_proxy.h deleted file mode 100644 index 5c7264052b2..00000000000 --- a/chromium/media/mojo/services/mojo_cdm_proxy.h +++ /dev/null @@ -1,85 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MEDIA_MOJO_SERVICES_MOJO_CDM_PROXY_H_ -#define MEDIA_MOJO_SERVICES_MOJO_CDM_PROXY_H_ - -#include <stdint.h> - -#include <vector> - -#include "base/macros.h" -#include "base/memory/weak_ptr.h" -#include "media/base/cdm_context.h" -#include "media/cdm/api/content_decryption_module.h" -#include "media/mojo/mojom/cdm_proxy.mojom.h" -#include "media/mojo/services/media_mojo_export.h" -#include "mojo/public/cpp/bindings/associated_receiver.h" -#include "mojo/public/cpp/bindings/pending_remote.h" -#include "mojo/public/cpp/bindings/remote.h" - -namespace media { - -// Implements a cdm::CdmProxy that communicates with mojom::CdmProxy. -class MEDIA_MOJO_EXPORT MojoCdmProxy : public cdm::CdmProxy, - mojom::CdmProxyClient { - public: - MojoCdmProxy(mojo::PendingRemote<mojom::CdmProxy> cdm_proxy_remote, - cdm::CdmProxyClient* client); - ~MojoCdmProxy() override; - - // cdm::CdmProxy implementation. - void Initialize() final; - void Process(Function function, - uint32_t crypto_session_id, - const uint8_t* input_data, - uint32_t input_data_size, - uint32_t expected_output_data_size) final; - void CreateMediaCryptoSession(const uint8_t* input_data, - uint32_t input_data_size) final; - void SetKey(uint32_t crypto_session_id, - const uint8_t* key_id, - uint32_t key_id_size, - KeyType key_type, - const uint8_t* key_blob, - uint32_t key_blob_size) final; - void RemoveKey(uint32_t crypto_session_id, - const uint8_t* key_id, - uint32_t key_id_size) final; - - // mojom::CdmProxyClient implementation. - void NotifyHardwareReset() final; - - // Returns the CDM ID associated with the remote CdmProxy. - int GetCdmId(); - - private: - void OnInitialized(media::CdmProxy::Status status, - media::CdmProxy::Protocol protocol, - uint32_t crypto_session_id, - int cdm_id); - void OnProcessed(media::CdmProxy::Status status, - const std::vector<uint8_t>& output_data); - void OnMediaCryptoSessionCreated(media::CdmProxy::Status status, - uint32_t crypto_session_id, - uint64_t output_data); - void OnKeySet(media::CdmProxy::Status status); - void OnKeyRemoved(media::CdmProxy::Status status); - - mojo::Remote<mojom::CdmProxy> cdm_proxy_remote_; - cdm::CdmProxyClient* client_; - - mojo::AssociatedReceiver<mojom::CdmProxyClient> client_receiver_{this}; - - int cdm_id_ = CdmContext::kInvalidCdmId; - - // NOTE: Weak pointers must be invalidated before all other member variables. - base::WeakPtrFactory<MojoCdmProxy> weak_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(MojoCdmProxy); -}; - -} // namespace media - -#endif // MEDIA_MOJO_SERVICES_MOJO_CDM_PROXY_H_ diff --git a/chromium/media/mojo/services/mojo_cdm_proxy_service.cc b/chromium/media/mojo/services/mojo_cdm_proxy_service.cc deleted file mode 100644 index 1c74993b907..00000000000 --- a/chromium/media/mojo/services/mojo_cdm_proxy_service.cc +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/mojo/services/mojo_cdm_proxy_service.h" - -#include "base/bind.h" -#include "base/macros.h" -#include "media/mojo/services/mojo_cdm_service_context.h" - -namespace media { - -MojoCdmProxyService::MojoCdmProxyService( - std::unique_ptr<::media::CdmProxy> cdm_proxy, - MojoCdmServiceContext* context) - : cdm_proxy_(std::move(cdm_proxy)), context_(context) { - DVLOG(1) << __func__; - DCHECK(cdm_proxy_); - DCHECK(context_); -} - -MojoCdmProxyService::~MojoCdmProxyService() { - DVLOG(1) << __func__; - - if (cdm_id_ != CdmContext::kInvalidCdmId) - context_->UnregisterCdmProxy(cdm_id_); -} - -void MojoCdmProxyService::Initialize( - mojo::PendingAssociatedRemote<mojom::CdmProxyClient> client, - InitializeCallback callback) { - DVLOG(2) << __func__; - - CHECK(!has_initialize_been_called_) << "Initialize should only happen once"; - has_initialize_been_called_ = true; - - client_.Bind(std::move(client)); - - cdm_proxy_->Initialize( - this, base::BindOnce(&MojoCdmProxyService::OnInitialized, - weak_factory_.GetWeakPtr(), std::move(callback))); -} - -void MojoCdmProxyService::Process(media::CdmProxy::Function function, - uint32_t crypto_session_id, - const std::vector<uint8_t>& input_data, - uint32_t expected_output_data_size, - ProcessCallback callback) { - DVLOG(3) << __func__; - cdm_proxy_->Process(function, crypto_session_id, input_data, - expected_output_data_size, std::move(callback)); -} - -void MojoCdmProxyService::CreateMediaCryptoSession( - const std::vector<uint8_t>& input_data, - CreateMediaCryptoSessionCallback callback) { - DVLOG(3) << __func__; - cdm_proxy_->CreateMediaCryptoSession(input_data, std::move(callback)); -} - -void MojoCdmProxyService::SetKey(uint32_t crypto_session_id, - const std::vector<uint8_t>& key_id, - media::CdmProxy::KeyType key_type, - const std::vector<uint8_t>& key_blob, - SetKeyCallback callback) { - DVLOG(3) << __func__; - cdm_proxy_->SetKey(crypto_session_id, key_id, key_type, key_blob, - std::move(callback)); -} - -void MojoCdmProxyService::RemoveKey(uint32_t crypto_session_id, - const std::vector<uint8_t>& key_id, - RemoveKeyCallback callback) { - DVLOG(3) << __func__; - cdm_proxy_->RemoveKey(crypto_session_id, key_id, std::move(callback)); -} - -void MojoCdmProxyService::NotifyHardwareReset() { - DVLOG(2) << __func__; - client_->NotifyHardwareReset(); -} - -base::WeakPtr<CdmContext> MojoCdmProxyService::GetCdmContext() { - DVLOG(2) << __func__; - return cdm_proxy_->GetCdmContext(); -} - -void MojoCdmProxyService::OnInitialized(InitializeCallback callback, - ::media::CdmProxy::Status status, - ::media::CdmProxy::Protocol protocol, - uint32_t crypto_session_id) { - CHECK_EQ(cdm_id_, CdmContext::kInvalidCdmId) - << "CDM proxy should only be created once."; - - if (status == ::media::CdmProxy::Status::kOk) - cdm_id_ = context_->RegisterCdmProxy(this); - - std::move(callback).Run(status, protocol, crypto_session_id, cdm_id_); -} - -} // namespace media diff --git a/chromium/media/mojo/services/mojo_cdm_proxy_service.h b/chromium/media/mojo/services/mojo_cdm_proxy_service.h deleted file mode 100644 index e8eaa950260..00000000000 --- a/chromium/media/mojo/services/mojo_cdm_proxy_service.h +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MEDIA_MOJO_SERVICES_MOJO_CDM_PROXY_SERVICE_H_ -#define MEDIA_MOJO_SERVICES_MOJO_CDM_PROXY_SERVICE_H_ - -#include <stdint.h> - -#include <memory> - -#include "base/compiler_specific.h" -#include "base/macros.h" -#include "base/memory/weak_ptr.h" -#include "media/base/cdm_context.h" -#include "media/cdm/cdm_proxy.h" -#include "media/mojo/mojom/cdm_proxy.mojom.h" -#include "media/mojo/services/media_mojo_export.h" -#include "mojo/public/cpp/bindings/associated_remote.h" -#include "mojo/public/cpp/bindings/pending_associated_remote.h" - -namespace media { - -class MojoCdmServiceContext; - -// A mojom::CdmProxy implementation backed by a media::CdmProxy. -class MEDIA_MOJO_EXPORT MojoCdmProxyService : public mojom::CdmProxy, - public CdmProxy::Client { - public: - MojoCdmProxyService(std::unique_ptr<::media::CdmProxy> cdm_proxy, - MojoCdmServiceContext* context); - - ~MojoCdmProxyService() final; - - // mojom::CdmProxy implementation. - void Initialize(mojo::PendingAssociatedRemote<mojom::CdmProxyClient> client, - InitializeCallback callback) final; - void Process(media::CdmProxy::Function function, - uint32_t crypto_session_id, - const std::vector<uint8_t>& input_data, - uint32_t expected_output_data_size, - ProcessCallback callback) final; - void CreateMediaCryptoSession( - const std::vector<uint8_t>& input_data, - CreateMediaCryptoSessionCallback callback) final; - void SetKey(uint32_t crypto_session_id, - const std::vector<uint8_t>& key_id, - media::CdmProxy::KeyType key_type, - const std::vector<uint8_t>& key_blob, - SetKeyCallback callback) final; - void RemoveKey(uint32_t crypto_session_id, - const std::vector<uint8_t>& key_id, - RemoveKeyCallback callback) final; - - // CdmProxy::Client implementation. - void NotifyHardwareReset() final; - - // Get CdmContext to be used by the media pipeline. - base::WeakPtr<CdmContext> GetCdmContext(); - - int GetCdmIdForTesting() const { return cdm_id_; } - - private: - void OnInitialized(InitializeCallback callback, - ::media::CdmProxy::Status status, - ::media::CdmProxy::Protocol protocol, - uint32_t crypto_session_id); - - bool has_initialize_been_called_ = false; - - std::unique_ptr<::media::CdmProxy> cdm_proxy_; - MojoCdmServiceContext* const context_ = nullptr; - - mojo::AssociatedRemote<mojom::CdmProxyClient> client_; - - // Set to a valid CDM ID if the |cdm_proxy_| is successfully initialized. - int cdm_id_ = CdmContext::kInvalidCdmId; - - // NOTE: Weak pointers must be invalidated before all other member variables. - base::WeakPtrFactory<MojoCdmProxyService> weak_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(MojoCdmProxyService); -}; - -} // namespace media - -#endif // MEDIA_MOJO_SERVICES_MOJO_CDM_PROXY_SERVICE_H_ diff --git a/chromium/media/mojo/services/mojo_cdm_proxy_unittest.cc b/chromium/media/mojo/services/mojo_cdm_proxy_unittest.cc deleted file mode 100644 index 8088e2ffafe..00000000000 --- a/chromium/media/mojo/services/mojo_cdm_proxy_unittest.cc +++ /dev/null @@ -1,390 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include <stdint.h> - -#include <memory> - -#include "base/bind.h" -#include "base/macros.h" -#include "base/memory/weak_ptr.h" -#include "base/run_loop.h" -#include "base/test/gmock_callback_support.h" -#include "base/test/gtest_util.h" -#include "base/test/test_message_loop.h" -#include "media/base/mock_filters.h" -#include "media/cdm/cdm_proxy_context.h" -#include "media/mojo/mojom/cdm_proxy.mojom.h" -#include "media/mojo/services/mojo_cdm_proxy.h" -#include "media/mojo/services/mojo_cdm_proxy_service.h" -#include "media/mojo/services/mojo_cdm_service_context.h" -#include "mojo/public/cpp/bindings/interface_request.h" -#include "mojo/public/cpp/bindings/receiver.h" -#include "testing/gtest/include/gtest/gtest.h" - -using ::testing::_; -using ::testing::NotNull; -using ::testing::SaveArg; -using ::testing::StrictMock; - -namespace media { - -namespace { - -MATCHER_P(StatusEq, status, "") { - return (arg == cdm::CdmProxyClient::Status::kOk && - status == media::CdmProxy::Status::kOk) || - (arg == cdm::CdmProxyClient::Status::kFail && - status == media::CdmProxy::Status::kFail); -} - -constexpr uint32_t kCryptoSessionId = 1010; - -class MockCdmProxyContext : public CdmProxyContext {}; - -class MockCdmProxy : public media::CdmProxy, public media::CdmContext { - public: - MockCdmProxy() {} - ~MockCdmProxy() override = default; - - // media::CdmProxy implementation. - - base::WeakPtr<CdmContext> GetCdmContext() override { - return weak_factory_.GetWeakPtr(); - } - - MOCK_METHOD2(Initialize, void(Client* client, InitializeCB init_cb)); - - MOCK_METHOD5(Process, - void(Function function, - uint32_t crypto_session_id, - const std::vector<uint8_t>& input_data, - uint32_t expected_output_data_size, - ProcessCB process_cb)); - - MOCK_METHOD2(CreateMediaCryptoSession, - void(const std::vector<uint8_t>& input_data, - CreateMediaCryptoSessionCB create_media_crypto_session_cb)); - - MOCK_METHOD5(SetKey, - void(uint32_t crypto_session_id, - const std::vector<uint8_t>& key_id, - KeyType key_type, - const std::vector<uint8_t>& key_blob, - SetKeyCB set_key_cb)); - MOCK_METHOD3(RemoveKey, - void(uint32_t crypto_session_id, - const std::vector<uint8_t>& key_id, - RemoveKeyCB remove_key_cb)); - - // media::CdmContext implementation. - CdmProxyContext* GetCdmProxyContext() override { - return &mock_cdm_proxy_context_; - } - - private: - MockCdmProxyContext mock_cdm_proxy_context_; - base::WeakPtrFactory<MockCdmProxy> weak_factory_{this}; -}; - -class MockCdmProxyClient : public cdm::CdmProxyClient { - public: - MockCdmProxyClient() = default; - ~MockCdmProxyClient() override = default; - - MOCK_METHOD3(OnInitialized, - void(Status status, - Protocol protocol, - uint32_t crypto_session_id)); - MOCK_METHOD3(OnProcessed, - void(Status status, - const uint8_t* output_data, - uint32_t output_data_size)); - MOCK_METHOD3(OnMediaCryptoSessionCreated, - void(Status status, - uint32_t crypto_session_id, - uint64_t output_data)); - MOCK_METHOD1(OnKeySet, void(Status status)); - MOCK_METHOD1(OnKeyRemoved, void(Status status)); - MOCK_METHOD0(NotifyHardwareReset, void()); -}; - -} // namespace - -class MojoCdmProxyTest : public ::testing::Test { - public: - using Status = CdmProxy::Status; - - MojoCdmProxyTest() { - // Client side setup. - mojo::PendingRemote<mojom::CdmProxy> cdm_proxy_remote; - auto receiver = cdm_proxy_remote.InitWithNewPipeAndPassReceiver(); - mojo_cdm_proxy_.reset( - new MojoCdmProxy(std::move(cdm_proxy_remote), &client_)); - cdm_proxy_ = mojo_cdm_proxy_.get(); - - // Service side setup. - std::unique_ptr<MockCdmProxy> mock_cdm_proxy(new MockCdmProxy()); - mock_cdm_proxy_ = mock_cdm_proxy.get(); - mojo_cdm_proxy_service_.reset(new MojoCdmProxyService( - std::move(mock_cdm_proxy), &mojo_cdm_service_context_)); - receiver_.reset(new mojo::Receiver<mojom::CdmProxy>( - mojo_cdm_proxy_service_.get(), std::move(receiver))); - receiver_->set_disconnect_handler(base::BindOnce( - &MojoCdmProxyTest::OnConnectionError, base::Unretained(this))); - - base::RunLoop().RunUntilIdle(); - } - - ~MojoCdmProxyTest() override = default; - - void Initialize(Status expected_status = Status::kOk, - bool has_connection = true) { - if (has_connection) { - EXPECT_CALL(*mock_cdm_proxy_, Initialize(NotNull(), _)) - .WillOnce([&](auto, auto init_cb) { - std::move(init_cb).Run(expected_status, CdmProxy::Protocol::kNone, - kCryptoSessionId); - }); - EXPECT_CALL(client_, - OnInitialized(StatusEq(expected_status), - cdm::CdmProxyClient::kNone, kCryptoSessionId)) - .WillOnce(SaveArg<2>(&crypto_session_id_)); - } else { - // Client should always be called even without connection. But we only - // care about status in this case. - EXPECT_CALL(client_, OnInitialized(StatusEq(expected_status), _, _)); - } - - cdm_proxy_->Initialize(); - base::RunLoop().RunUntilIdle(); - } - - void Process(Status expected_status = Status::kOk, - bool has_connection = true) { - const std::vector<uint8_t> kInputData = {1, 2}; - const uint32_t kExpectedOutputDataSize = 111; - const std::vector<uint8_t> kOutputData = {3, 4, 5}; - - if (has_connection) { - EXPECT_CALL( - *mock_cdm_proxy_, - Process(CdmProxy::Function::kIntelNegotiateCryptoSessionKeyExchange, - crypto_session_id_, kInputData, kExpectedOutputDataSize, _)) - .WillOnce([&](auto, auto, auto, auto, auto process_cb) { - std::move(process_cb).Run(expected_status, kOutputData); - }); - EXPECT_CALL(client_, OnProcessed(StatusEq(expected_status), NotNull(), - kOutputData.size())); - } else { - // Client should always be called even without connection. But we only - // care about status in this case. - EXPECT_CALL(client_, OnProcessed(StatusEq(expected_status), _, _)); - } - - cdm_proxy_->Process( - cdm::CdmProxy::Function::kIntelNegotiateCryptoSessionKeyExchange, - crypto_session_id_, kInputData.data(), kInputData.size(), - kExpectedOutputDataSize); - base::RunLoop().RunUntilIdle(); - } - - void CreateMediaCryptoSession(Status expected_status = Status::kOk, - bool has_connection = true) { - const std::vector<uint8_t> kInputData = {6, 7}; - const uint32_t kMediaCryptoSessionId = 222; - const uint64_t kOutputData = 333; - - if (has_connection) { - EXPECT_CALL(*mock_cdm_proxy_, CreateMediaCryptoSession(kInputData, _)) - .WillOnce([&](auto, auto create_media_crypto_session_cb) { - std::move(create_media_crypto_session_cb) - .Run(expected_status, kMediaCryptoSessionId, kOutputData); - }); - EXPECT_CALL(client_, OnMediaCryptoSessionCreated( - StatusEq(expected_status), kMediaCryptoSessionId, - kOutputData)); - } else { - // Client should always be called even without connection. But we only - // care about status in this case. - EXPECT_CALL(client_, - OnMediaCryptoSessionCreated(StatusEq(expected_status), _, _)); - } - - cdm_proxy_->CreateMediaCryptoSession(kInputData.data(), kInputData.size()); - base::RunLoop().RunUntilIdle(); - } - - void SetKey() { - const std::vector<uint8_t> key_id = {8, 9}; - const std::vector<uint8_t> key_blob = {10, 11, 12}; - EXPECT_CALL(*mock_cdm_proxy_, - SetKey(crypto_session_id_, key_id, _, key_blob, _)) - .WillOnce([&](auto, auto, auto, auto, auto set_key_cb) { - std::move(set_key_cb).Run(Status::kOk); - }); - EXPECT_CALL(client_, OnKeySet(StatusEq(Status::kOk))); - cdm_proxy_->SetKey(crypto_session_id_, key_id.data(), key_id.size(), - cdm::CdmProxy::KeyType::kDecryptOnly, key_blob.data(), - key_blob.size()); - base::RunLoop().RunUntilIdle(); - } - - void RemoveKey() { - const std::vector<uint8_t> key_id = {13, 14}; - EXPECT_CALL(*mock_cdm_proxy_, RemoveKey(crypto_session_id_, key_id, _)) - .WillOnce([&](auto, auto, auto remove_key_cb) { - std::move(remove_key_cb).Run(Status::kOk); - }); - EXPECT_CALL(client_, OnKeyRemoved(StatusEq(Status::kOk))); - cdm_proxy_->RemoveKey(crypto_session_id_, key_id.data(), key_id.size()); - base::RunLoop().RunUntilIdle(); - } - - // Simulate connecting the media component with the CdmContext. Can only be - // called after the CdmProxy is successfully initialized (see Initialize()). - void SetCdm() { - int cdm_id = mojo_cdm_proxy_service_->GetCdmIdForTesting(); - cdm_context_ref_ = mojo_cdm_service_context_.GetCdmContextRef(cdm_id); - cdm_context_ = cdm_context_ref_->GetCdmContext(); - } - - CdmProxyContext* GetCdmProxyContext() { - return cdm_context_->GetCdmProxyContext(); - } - - void Destroy() { - mojo_cdm_proxy_.reset(); - base::RunLoop().RunUntilIdle(); - } - - void OnConnectionError() { mojo_cdm_proxy_service_.reset(); } - - void ForceConnectionError() { - receiver_->ResetWithReason(2, "Test closed connection."); - mojo_cdm_proxy_service_.reset(); - base::RunLoop().RunUntilIdle(); - } - - base::TestMessageLoop message_loop_; - uint32_t crypto_session_id_ = 0; - - // Client side members. - StrictMock<MockCdmProxyClient> client_; - std::unique_ptr<MojoCdmProxy> mojo_cdm_proxy_; - cdm::CdmProxy* cdm_proxy_ = nullptr; - - // Service side members. - MojoCdmServiceContext mojo_cdm_service_context_; - std::unique_ptr<MojoCdmProxyService> mojo_cdm_proxy_service_; - std::unique_ptr<mojo::Receiver<mojom::CdmProxy>> receiver_; - MockCdmProxy* mock_cdm_proxy_ = nullptr; - - // Media component side members. - std::unique_ptr<CdmContextRef> cdm_context_ref_; - CdmContext* cdm_context_ = nullptr; - - private: - DISALLOW_COPY_AND_ASSIGN(MojoCdmProxyTest); -}; - -TEST_F(MojoCdmProxyTest, Initialize) { - Initialize(); -} - -TEST_F(MojoCdmProxyTest, Initialize_Failure) { - Initialize(Status::kFail); -} - -TEST_F(MojoCdmProxyTest, Initialize_Twice) { - Initialize(); - EXPECT_CHECK_DEATH(Initialize()); -} - -TEST_F(MojoCdmProxyTest, Process) { - Initialize(); - Process(); -} - -TEST_F(MojoCdmProxyTest, Process_Failure) { - Initialize(); - Process(Status::kFail); -} - -TEST_F(MojoCdmProxyTest, CreateMediaCryptoSession) { - Initialize(); - Process(); - CreateMediaCryptoSession(); -} - -TEST_F(MojoCdmProxyTest, CreateMediaCryptoSession_Failure) { - Initialize(); - Process(); - CreateMediaCryptoSession(Status::kFail); -} - -TEST_F(MojoCdmProxyTest, SetKey) { - Initialize(); - Process(); - CreateMediaCryptoSession(); - SetKey(); -} - -TEST_F(MojoCdmProxyTest, RemoveKey) { - Initialize(); - Process(); - CreateMediaCryptoSession(); - RemoveKey(); -} - -TEST_F(MojoCdmProxyTest, Destroy) { - Initialize(); - Process(); - EXPECT_TRUE(mojo_cdm_proxy_service_); - Destroy(); - EXPECT_FALSE(mojo_cdm_proxy_service_); -} - -TEST_F(MojoCdmProxyTest, ConnectionError_BeforeInitialize) { - ForceConnectionError(); - Initialize(Status::kFail, false); -} - -TEST_F(MojoCdmProxyTest, ConnectionError_AfterInitialize) { - Initialize(); - Process(); - CreateMediaCryptoSession(); - - ForceConnectionError(); - - // Calling Process() and CreateMediaCryptoSession() without connection. These - // calls should fail but the client should still get notified (about the - // failure). - Process(Status::kFail, false); - CreateMediaCryptoSession(Status::kFail, false); -} - -TEST_F(MojoCdmProxyTest, GetCdmProxyContext) { - Initialize(); - SetCdm(); - EXPECT_TRUE(GetCdmProxyContext()); -} - -TEST_F(MojoCdmProxyTest, GetCdmProxyContext_AfterDestroy) { - Initialize(); - SetCdm(); - EXPECT_TRUE(GetCdmProxyContext()); - Destroy(); - EXPECT_FALSE(GetCdmProxyContext()); -} - -TEST_F(MojoCdmProxyTest, GetCdmProxyContext_AfterConnectionError) { - Initialize(); - SetCdm(); - EXPECT_TRUE(GetCdmProxyContext()); - ForceConnectionError(); - EXPECT_FALSE(GetCdmProxyContext()); -} - -} // namespace media diff --git a/chromium/media/mojo/services/mojo_cdm_service.cc b/chromium/media/mojo/services/mojo_cdm_service.cc index 80910478b16..16e60b82eb9 100644 --- a/chromium/media/mojo/services/mojo_cdm_service.cc +++ b/chromium/media/mojo/services/mojo_cdm_service.cc @@ -188,16 +188,8 @@ void MojoCdmService::OnCdmCreated( &MojoCdmService::OnDecryptorConnectionError, base::Unretained(this))); } - // If the |context_| is not null, we should support connecting the |cdm| with - // the media player in the same process, which also has access to the - // |context_|. Hence pass back the |cdm_id_| obtained from the |context_|. - // Otherwise, if the |cdm| has a valid CDM ID by itself, this CDM can proxy - // all or parts of its functionalities to another remote CDM or CdmProxy. In - // this case, just we pass the remote CDM ID back. - int cdm_id = context_ ? cdm_id_ : cdm_context->GetCdmId(); - cdm_promise_result->success = true; - std::move(callback).Run(std::move(cdm_promise_result), cdm_id, + std::move(callback).Run(std::move(cdm_promise_result), cdm_id_, std::move(decryptor_remote)); } diff --git a/chromium/media/mojo/services/mojo_cdm_service_context.cc b/chromium/media/mojo/services/mojo_cdm_service_context.cc index a49f74ea49a..6279a4c717f 100644 --- a/chromium/media/mojo/services/mojo_cdm_service_context.cc +++ b/chromium/media/mojo/services/mojo_cdm_service_context.cc @@ -11,58 +11,18 @@ #include "media/cdm/cdm_context_ref_impl.h" #include "media/mojo/services/mojo_cdm_service.h" -#if BUILDFLAG(ENABLE_CDM_PROXY) -#include "media/mojo/services/mojo_cdm_proxy_service.h" -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - namespace media { namespace { // Helper function to get the next unique (per-process) CDM ID to be assigned to -// a CDM or CdmProxy. It will be used to locate the CDM by the media players -// living in the same process. +// a CDM. It will be used to locate the CDM by the media players living in the +// same process. int GetNextCdmId() { static int g_next_cdm_id = CdmContext::kInvalidCdmId + 1; return g_next_cdm_id++; } -#if BUILDFLAG(ENABLE_CDM_PROXY) -class CdmProxyContextRef : public CdmContextRef, public CdmContext { - public: - explicit CdmProxyContextRef(base::WeakPtr<CdmContext> cdm_context) - : cdm_context_(cdm_context) {} - ~CdmProxyContextRef() final {} - - // CdmContextRef implementation. - CdmContext* GetCdmContext() final { return this; } - - private: - // CdmContext implementation. - std::unique_ptr<CallbackRegistration> RegisterEventCB( - EventCB event_cb) final { - DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); - return cdm_context_ ? cdm_context_->RegisterEventCB(std::move(event_cb)) - : nullptr; - } - - Decryptor* GetDecryptor() final { - DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); - return cdm_context_ ? cdm_context_->GetDecryptor() : nullptr; - } - - CdmProxyContext* GetCdmProxyContext() final { - DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); - return cdm_context_ ? cdm_context_->GetCdmProxyContext() : nullptr; - } - - base::WeakPtr<CdmContext> cdm_context_; - THREAD_CHECKER(thread_checker_); - - DISALLOW_COPY_AND_ASSIGN(CdmProxyContextRef); -}; -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - } // namespace MojoCdmServiceContext::MojoCdmServiceContext() = default; @@ -83,23 +43,6 @@ void MojoCdmServiceContext::UnregisterCdm(int cdm_id) { cdm_services_.erase(cdm_id); } -#if BUILDFLAG(ENABLE_CDM_PROXY) -int MojoCdmServiceContext::RegisterCdmProxy( - MojoCdmProxyService* cdm_proxy_service) { - DCHECK(cdm_proxy_service); - int cdm_id = GetNextCdmId(); - cdm_proxy_services_[cdm_id] = cdm_proxy_service; - DVLOG(1) << __func__ << ": CdmProxyService registered with CDM ID " << cdm_id; - return cdm_id; -} - -void MojoCdmServiceContext::UnregisterCdmProxy(int cdm_id) { - DVLOG(1) << __func__ << ": cdm_id = " << cdm_id; - DCHECK(cdm_proxy_services_.count(cdm_id)); - cdm_proxy_services_.erase(cdm_id); -} -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - std::unique_ptr<CdmContextRef> MojoCdmServiceContext::GetCdmContextRef( int cdm_id) { DVLOG(1) << __func__ << ": cdm_id = " << cdm_id; @@ -114,15 +57,6 @@ std::unique_ptr<CdmContextRef> MojoCdmServiceContext::GetCdmContextRef( return std::make_unique<CdmContextRefImpl>(cdm_service->second->GetCdm()); } -#if BUILDFLAG(ENABLE_CDM_PROXY) - // Next check all CdmProxies. - auto cdm_proxy_service = cdm_proxy_services_.find(cdm_id); - if (cdm_proxy_service != cdm_proxy_services_.end()) { - return std::make_unique<CdmProxyContextRef>( - cdm_proxy_service->second->GetCdmContext()); - } -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - LOG(ERROR) << "CdmContextRef cannot be obtained for CDM ID: " << cdm_id; return nullptr; } diff --git a/chromium/media/mojo/services/mojo_cdm_service_context.h b/chromium/media/mojo/services/mojo_cdm_service_context.h index f074cf00b19..07290ba2a8f 100644 --- a/chromium/media/mojo/services/mojo_cdm_service_context.h +++ b/chromium/media/mojo/services/mojo_cdm_service_context.h @@ -16,10 +16,8 @@ namespace media { -class CdmProxy; class CdmContextRef; class MojoCdmService; -class MojoCdmProxyService; // A class that creates, owns and manages all MojoCdmService instances. class MEDIA_MOJO_EXPORT MojoCdmServiceContext { @@ -33,15 +31,6 @@ class MEDIA_MOJO_EXPORT MojoCdmServiceContext { // Unregisters the CDM. Must be called before the CDM is destroyed. void UnregisterCdm(int cdm_id); -#if BUILDFLAG(ENABLE_CDM_PROXY) - // Registers the |cdm_proxy_service| and returns a unique (per-process) CDM - // ID. - int RegisterCdmProxy(MojoCdmProxyService* cdm_proxy_service); - - // Unregisters the CdmProxy. Must be called before the CdmProxy is destroyed. - void UnregisterCdmProxy(int cdm_id); -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - // Returns the CdmContextRef associated with |cdm_id|. std::unique_ptr<CdmContextRef> GetCdmContextRef(int cdm_id); @@ -49,11 +38,6 @@ class MEDIA_MOJO_EXPORT MojoCdmServiceContext { // A map between CDM ID and MojoCdmService. std::map<int, MojoCdmService*> cdm_services_; -#if BUILDFLAG(ENABLE_CDM_PROXY) - // A map between CDM ID and MojoCdmProxyService. - std::map<int, MojoCdmProxyService*> cdm_proxy_services_; -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - DISALLOW_COPY_AND_ASSIGN(MojoCdmServiceContext); }; diff --git a/chromium/media/mojo/services/mojo_decryptor_service.cc b/chromium/media/mojo/services/mojo_decryptor_service.cc index 32ad28c74ac..900808c4f37 100644 --- a/chromium/media/mojo/services/mojo_decryptor_service.cc +++ b/chromium/media/mojo/services/mojo_decryptor_service.cc @@ -48,29 +48,6 @@ class FrameResourceReleaserImpl final : public mojom::FrameResourceReleaser { } // namespace -// static -std::unique_ptr<MojoDecryptorService> MojoDecryptorService::Create( - int cdm_id, - MojoCdmServiceContext* mojo_cdm_service_context) { - auto cdm_context_ref = mojo_cdm_service_context->GetCdmContextRef(cdm_id); - if (!cdm_context_ref) { - DVLOG(1) << "CdmContextRef not found for CDM ID: " << cdm_id; - return nullptr; - } - - auto* cdm_context = cdm_context_ref->GetCdmContext(); - DCHECK(cdm_context); - - auto* decryptor = cdm_context->GetDecryptor(); - if (!decryptor) { - DVLOG(1) << "CdmContext does not support Decryptor"; - return nullptr; - } - - return std::make_unique<MojoDecryptorService>(decryptor, - std::move(cdm_context_ref)); -} - MojoDecryptorService::MojoDecryptorService( media::Decryptor* decryptor, std::unique_ptr<CdmContextRef> cdm_context_ref) diff --git a/chromium/media/mojo/services/mojo_decryptor_service.h b/chromium/media/mojo/services/mojo_decryptor_service.h index b4fb2e7a699..4c9d6a02b0e 100644 --- a/chromium/media/mojo/services/mojo_decryptor_service.h +++ b/chromium/media/mojo/services/mojo_decryptor_service.h @@ -21,7 +21,6 @@ namespace media { class DecoderBuffer; -class MojoCdmServiceContext; class MojoDecoderBufferReader; class MojoDecoderBufferWriter; @@ -32,10 +31,6 @@ class MEDIA_MOJO_EXPORT MojoDecryptorService : public mojom::Decryptor { using StreamType = media::Decryptor::StreamType; using Status = media::Decryptor::Status; - static std::unique_ptr<MojoDecryptorService> Create( - int cdm_id, - MojoCdmServiceContext* mojo_cdm_service_context); - // If |cdm_context_ref| is null, caller must ensure that |decryptor| outlives // |this|. Otherwise, |decryptor| is guaranteed to be valid as long as // |cdm_context_ref| is held. diff --git a/chromium/media/mojo/services/mojo_media_client.cc b/chromium/media/mojo/services/mojo_media_client.cc index 4b4fca75d96..b9f53ad90c5 100644 --- a/chromium/media/mojo/services/mojo_media_client.cc +++ b/chromium/media/mojo/services/mojo_media_client.cc @@ -11,10 +11,6 @@ #include "media/base/renderer.h" #include "media/base/video_decoder.h" -#if BUILDFLAG(ENABLE_CDM_PROXY) -#include "media/cdm/cdm_proxy.h" -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - namespace media { MojoMediaClient::MojoMediaClient() = default; @@ -44,7 +40,7 @@ std::unique_ptr<VideoDecoder> MojoMediaClient::CreateVideoDecoder( } std::unique_ptr<Renderer> MojoMediaClient::CreateRenderer( - service_manager::mojom::InterfaceProvider* host_interfaces, + mojom::FrameInterfaceFactory* frame_interfaces, scoped_refptr<base::SingleThreadTaskRunner> task_runner, MediaLog* media_log, const std::string& audio_device_id) { @@ -53,7 +49,7 @@ std::unique_ptr<Renderer> MojoMediaClient::CreateRenderer( #if BUILDFLAG(ENABLE_CAST_RENDERER) std::unique_ptr<Renderer> MojoMediaClient::CreateCastRenderer( - service_manager::mojom::InterfaceProvider* host_interfaces, + mojom::FrameInterfaceFactory* frame_interfaces, scoped_refptr<base::SingleThreadTaskRunner> task_runner, MediaLog* media_log, const base::UnguessableToken& overlay_plane_id) { @@ -62,15 +58,8 @@ std::unique_ptr<Renderer> MojoMediaClient::CreateCastRenderer( #endif // BUILDFLAG(ENABLE_CAST_RENDERER) std::unique_ptr<CdmFactory> MojoMediaClient::CreateCdmFactory( - service_manager::mojom::InterfaceProvider* host_interfaces) { - return nullptr; -} - -#if BUILDFLAG(ENABLE_CDM_PROXY) -std::unique_ptr<CdmProxy> MojoMediaClient::CreateCdmProxy( - const base::Token& cdm_guid) { + mojom::FrameInterfaceFactory* frame_interfaces) { return nullptr; } -#endif // BUILDFLAG(ENABLE_CDM_PROXY) } // namespace media diff --git a/chromium/media/mojo/services/mojo_media_client.h b/chromium/media/mojo/services/mojo_media_client.h index 03e62e3f3f0..03263ee3c63 100644 --- a/chromium/media/mojo/services/mojo_media_client.h +++ b/chromium/media/mojo/services/mojo_media_client.h @@ -15,30 +15,23 @@ #include "media/base/overlay_info.h" #include "media/media_buildflags.h" #include "media/mojo/buildflags.h" +#include "media/mojo/mojom/frame_interface_factory.mojom.h" #include "media/mojo/mojom/video_decoder.mojom.h" #include "media/mojo/services/media_mojo_export.h" #include "media/video/supported_video_decoder_config.h" namespace base { class SingleThreadTaskRunner; -class Token; } // namespace base namespace gfx { class ColorSpace; } // namespace gfx -namespace service_manager { -namespace mojom { -class InterfaceProvider; -} // namespace mojom -} // namespace service_manager - namespace media { class AudioDecoder; class CdmFactory; -class CdmProxy; class MediaLog; class Renderer; class VideoDecoder; @@ -76,7 +69,7 @@ class MEDIA_MOJO_EXPORT MojoMediaClient { // TODO(hubbe): Find out whether we should pass in |target_color_space| here. // TODO(guohuideng): Merge this function into CreateCastRenderer. virtual std::unique_ptr<Renderer> CreateRenderer( - service_manager::mojom::InterfaceProvider* host_interfaces, + mojom::FrameInterfaceFactory* frame_interfaces, scoped_refptr<base::SingleThreadTaskRunner> task_runner, MediaLog* media_log, const std::string& audio_device_id); @@ -88,23 +81,17 @@ class MEDIA_MOJO_EXPORT MojoMediaClient { // associtated with. // Chromecast also uses CreateRenderer to create "audio only" renderers. virtual std::unique_ptr<Renderer> CreateCastRenderer( - service_manager::mojom::InterfaceProvider* host_interfaces, + mojom::FrameInterfaceFactory* frame_interfaces, scoped_refptr<base::SingleThreadTaskRunner> task_runner, MediaLog* media_log, const base::UnguessableToken& overlay_plane_id); #endif // BUILDFLAG(ENABLE_CAST_RENDERER) - // Returns the CdmFactory to be used by MojoCdmService. |host_interfaces| can + // Returns the CdmFactory to be used by MojoCdmService. |frame_interfaces| can // be used to request interfaces provided remotely by the host. It may be a // nullptr if the host chose not to bind the InterfacePtr. virtual std::unique_ptr<CdmFactory> CreateCdmFactory( - service_manager::mojom::InterfaceProvider* host_interfaces); - -#if BUILDFLAG(ENABLE_CDM_PROXY) - // Creates a CdmProxy that proxies part of CDM functionalities to a different - // entity, e.g. hardware CDM modules. - virtual std::unique_ptr<CdmProxy> CreateCdmProxy(const base::Token& cdm_guid); -#endif // BUILDFLAG(ENABLE_CDM_PROXY) + mojom::FrameInterfaceFactory* frame_interfaces); protected: MojoMediaClient(); diff --git a/chromium/media/mojo/services/mojo_video_encode_accelerator_provider.cc b/chromium/media/mojo/services/mojo_video_encode_accelerator_provider.cc index c2644513bdc..0ade6cc8fd8 100644 --- a/chromium/media/mojo/services/mojo_video_encode_accelerator_provider.cc +++ b/chromium/media/mojo/services/mojo_video_encode_accelerator_provider.cc @@ -7,7 +7,6 @@ #include <memory> #include <utility> -#include "base/logging.h" #include "media/base/bind_to_current_loop.h" #include "media/base/limits.h" #include "media/gpu/gpu_video_encode_accelerator_factory.h" diff --git a/chromium/media/mojo/services/test_mojo_media_client.cc b/chromium/media/mojo/services/test_mojo_media_client.cc index aa6240b7912..7d51f3db006 100644 --- a/chromium/media/mojo/services/test_mojo_media_client.cc +++ b/chromium/media/mojo/services/test_mojo_media_client.cc @@ -8,6 +8,7 @@ #include "base/run_loop.h" #include "base/threading/thread_task_runner_handle.h" +#include "build/build_config.h" #include "media/audio/audio_device_description.h" #include "media/audio/audio_manager.h" #include "media/audio/audio_output_stream_sink.h" @@ -21,12 +22,6 @@ #include "media/renderers/default_decoder_factory.h" #include "media/renderers/default_renderer_factory.h" -#if BUILDFLAG(ENABLE_CDM_PROXY) -#include "media/cdm/cdm_paths.h" // nogncheck -#include "media/cdm/cdm_proxy.h" // nogncheck -#include "media/cdm/library_cdm/clear_key_cdm/clear_key_cdm_proxy.h" // nogncheck -#endif - namespace media { TestMojoMediaClient::TestMojoMediaClient() = default; @@ -55,7 +50,7 @@ void TestMojoMediaClient::Initialize() { } std::unique_ptr<Renderer> TestMojoMediaClient::CreateRenderer( - service_manager::mojom::InterfaceProvider* host_interfaces, + mojom::FrameInterfaceFactory* frame_interfaces, scoped_refptr<base::SingleThreadTaskRunner> task_runner, MediaLog* media_log, const std::string& /* audio_device_id */) { @@ -65,9 +60,15 @@ std::unique_ptr<Renderer> TestMojoMediaClient::CreateRenderer( } if (!renderer_factory_) { +#if defined(OS_ANDROID) + renderer_factory_ = std::make_unique<DefaultRendererFactory>( + media_log, decoder_factory_.get(), + DefaultRendererFactory::GetGpuFactoriesCB()); +#else renderer_factory_ = std::make_unique<DefaultRendererFactory>( media_log, decoder_factory_.get(), DefaultRendererFactory::GetGpuFactoriesCB(), nullptr); +#endif } // We cannot share AudioOutputStreamSink or NullVideoSink among different @@ -91,29 +92,19 @@ std::unique_ptr<Renderer> TestMojoMediaClient::CreateRenderer( #if BUILDFLAG(ENABLE_CAST_RENDERER) std::unique_ptr<Renderer> TestMojoMediaClient::CreateCastRenderer( - service_manager::mojom::InterfaceProvider* host_interfaces, + mojom::FrameInterfaceFactory* frame_interfaces, scoped_refptr<base::SingleThreadTaskRunner> task_runner, MediaLog* media_log, const base::UnguessableToken& /* overlay_plane_id */) { - return CreateRenderer(host_interfaces, task_runner, media_log, std::string()); + return CreateRenderer(frame_interfaces, task_runner, media_log, + std::string()); } #endif // BUILDFLAG(ENABLE_CAST_RENDERER) std::unique_ptr<CdmFactory> TestMojoMediaClient::CreateCdmFactory( - service_manager::mojom::InterfaceProvider* /* host_interfaces */) { + mojom::FrameInterfaceFactory* /* frame_interfaces */) { DVLOG(1) << __func__; return std::make_unique<DefaultCdmFactory>(); } -#if BUILDFLAG(ENABLE_CDM_PROXY) -std::unique_ptr<CdmProxy> TestMojoMediaClient::CreateCdmProxy( - const base::Token& cdm_guid) { - DVLOG(1) << __func__ << ": cdm_guid = " << cdm_guid.ToString(); - if (cdm_guid == kClearKeyCdmGuid) - return std::make_unique<ClearKeyCdmProxy>(); - - return nullptr; -} -#endif // BUILDFLAG(ENABLE_CDM_PROXY) - } // namespace media diff --git a/chromium/media/mojo/services/test_mojo_media_client.h b/chromium/media/mojo/services/test_mojo_media_client.h index d72fedd59a2..c8899c05404 100644 --- a/chromium/media/mojo/services/test_mojo_media_client.h +++ b/chromium/media/mojo/services/test_mojo_media_client.h @@ -30,22 +30,19 @@ class TestMojoMediaClient : public MojoMediaClient { // MojoMediaClient implementation. void Initialize() final; std::unique_ptr<Renderer> CreateRenderer( - service_manager::mojom::InterfaceProvider* host_interfaces, + mojom::FrameInterfaceFactory* frame_interfaces, scoped_refptr<base::SingleThreadTaskRunner> task_runner, MediaLog* media_log, const std::string& audio_device_id) final; #if BUILDFLAG(ENABLE_CAST_RENDERER) std::unique_ptr<Renderer> CreateCastRenderer( - service_manager::mojom::InterfaceProvider* host_interfaces, + mojom::FrameInterfaceFactory* frame_interfaces, scoped_refptr<base::SingleThreadTaskRunner> task_runner, MediaLog* media_log, const base::UnguessableToken& overlay_plane_id) final; #endif // BUILDFLAG(ENABLE_CAST_RENDERER) std::unique_ptr<CdmFactory> CreateCdmFactory( - service_manager::mojom::InterfaceProvider* /* host_interfaces */) final; -#if BUILDFLAG(ENABLE_CDM_PROXY) - std::unique_ptr<CdmProxy> CreateCdmProxy(const base::Token& cdm_guid) final; -#endif // BUILDFLAG(ENABLE_CDM_PROXY) + mojom::FrameInterfaceFactory* /* frame_interfaces */) final; private: std::unique_ptr<AudioManager> audio_manager_; diff --git a/chromium/media/mojo/services/video_decode_perf_history_unittest.cc b/chromium/media/mojo/services/video_decode_perf_history_unittest.cc index ac736bf0ae0..3a5a11b8d66 100644 --- a/chromium/media/mojo/services/video_decode_perf_history_unittest.cc +++ b/chromium/media/mojo/services/video_decode_perf_history_unittest.cc @@ -52,6 +52,7 @@ class FakeVideoDecodeStatsDB : public VideoDecodeStatsDB { // Call CompleteInitialize(...) to run |init_cb| callback. void Initialize(base::OnceCallback<void(bool)> init_cb) override { + EXPECT_FALSE(!!pendnding_init_cb_); pendnding_init_cb_ = std::move(init_cb); } @@ -59,7 +60,7 @@ class FakeVideoDecodeStatsDB : public VideoDecodeStatsDB { // for success. void CompleteInitialize(bool success) { DVLOG(2) << __func__ << " running with success = " << success; - EXPECT_FALSE(!pendnding_init_cb_); + EXPECT_TRUE(!!pendnding_init_cb_); std::move(pendnding_init_cb_).Run(success); } @@ -1037,4 +1038,50 @@ INSTANTIATE_TEST_SUITE_P(VaryDBInitTiming, VideoDecodePerfHistoryParamTest, ::testing::ValuesIn(kPerfHistoryTestParams)); -} // namespace media +// +// The following test are not parameterized. They instead always hard code +// deferred initialization. +// + +TEST_F(VideoDecodePerfHistoryTest, ClearHistoryTriggersSuccessfulInitialize) { + // Clear the DB. Completion callback shouldn't fire until initialize + // completes. + EXPECT_CALL(*this, MockOnClearedHistory()).Times(0); + perf_history_->ClearHistory( + base::BindOnce(&VideoDecodePerfHistoryParamTest::MockOnClearedHistory, + base::Unretained(this))); + + // Give completion callback a chance to fire. Confirm it did not fire. + task_environment_.RunUntilIdle(); + testing::Mock::VerifyAndClearExpectations(this); + + // Expect completion callback after we successfully initialize. + EXPECT_CALL(*this, MockOnClearedHistory()); + GetFakeDB()->CompleteInitialize(true); + + // Give deferred callback a chance to fire. + task_environment_.RunUntilIdle(); +} + +TEST_F(VideoDecodePerfHistoryTest, ClearHistoryTriggersFailedInitialize) { + // Clear the DB. Completion callback shouldn't fire until initialize + // completes. + EXPECT_CALL(*this, MockOnClearedHistory()).Times(0); + perf_history_->ClearHistory( + base::BindOnce(&VideoDecodePerfHistoryParamTest::MockOnClearedHistory, + base::Unretained(this))); + + // Give completion callback a chance to fire. Confirm it did not fire. + task_environment_.RunUntilIdle(); + testing::Mock::VerifyAndClearExpectations(this); + + // Expect completion callback after completing initialize. "Failure" is still + // a form of completion. + EXPECT_CALL(*this, MockOnClearedHistory()); + GetFakeDB()->CompleteInitialize(false); + + // Give deferred callback a chance to fire. + task_environment_.RunUntilIdle(); +} + +} // namespace media
\ No newline at end of file diff --git a/chromium/media/mojo/services/watch_time_recorder.cc b/chromium/media/mojo/services/watch_time_recorder.cc index 512e4c950a1..6c1d44c7632 100644 --- a/chromium/media/mojo/services/watch_time_recorder.cc +++ b/chromium/media/mojo/services/watch_time_recorder.cc @@ -421,7 +421,7 @@ void WatchTimeRecorder::RecordUkmPlaybackData() { base::flat_set<AudioCodecProfile> aac_profiles; - base::TimeDelta total_watch_time; + base::TimeDelta total_foreground_audible_watch_time; for (auto& ukm_record : ukm_records_) { ukm::builders::Media_BasicPlayback builder(source_id_); @@ -446,7 +446,11 @@ void WatchTimeRecorder::RecordUkmPlaybackData() { // Only one of these keys should be present. DCHECK(!recorded_all_metric); recorded_all_metric = true; - total_watch_time += kv.second; + + // We should only add to the total watchtime if we were not in the + // background and not muted. + if (!properties_->is_muted && !properties_->is_background) + total_foreground_audible_watch_time += kv.second; builder.SetWatchTime(kv.second.InMilliseconds()); if (ukm_record.total_underflow_count) { @@ -554,10 +558,10 @@ void WatchTimeRecorder::RecordUkmPlaybackData() { base::UmaHistogramEnumeration("Media.AudioCodecProfile.AAC", profile); } - if (total_watch_time > base::TimeDelta()) { + if (total_foreground_audible_watch_time > base::TimeDelta()) { std::move(record_playback_cb_) - .Run(total_watch_time, last_timestamp_, properties_->has_video, - properties_->has_audio); + .Run(total_foreground_audible_watch_time, last_timestamp_, + properties_->has_video, properties_->has_audio); } ukm_records_.clear(); diff --git a/chromium/media/parsers/jpeg_parser.cc b/chromium/media/parsers/jpeg_parser.cc index 344f7c36995..17dcc3c020b 100644 --- a/chromium/media/parsers/jpeg_parser.cc +++ b/chromium/media/parsers/jpeg_parser.cc @@ -4,6 +4,8 @@ #include "media/parsers/jpeg_parser.h" +#include <cstring> + #include "base/big_endian.h" #include "base/logging.h" #include "base/stl_util.h" diff --git a/chromium/media/parsers/vp8_bool_decoder.cc b/chromium/media/parsers/vp8_bool_decoder.cc index 4f156ad8daa..17607732f69 100644 --- a/chromium/media/parsers/vp8_bool_decoder.cc +++ b/chromium/media/parsers/vp8_bool_decoder.cc @@ -46,6 +46,7 @@ #include <algorithm> +#include "base/check_op.h" #include "base/numerics/safe_conversions.h" namespace media { diff --git a/chromium/media/parsers/vp8_parser.cc b/chromium/media/parsers/vp8_parser.cc index b38e3048668..b52c59fd070 100644 --- a/chromium/media/parsers/vp8_parser.cc +++ b/chromium/media/parsers/vp8_parser.cc @@ -7,6 +7,8 @@ #include "media/parsers/vp8_parser.h" +#include <cstring> + #include "base/logging.h" namespace media { diff --git a/chromium/media/parsers/webp_parser.cc b/chromium/media/parsers/webp_parser.cc index b0348037e79..9d2ba7a12a8 100644 --- a/chromium/media/parsers/webp_parser.cc +++ b/chromium/media/parsers/webp_parser.cc @@ -9,7 +9,7 @@ #include <string.h> #include "base/bits.h" -#include "base/logging.h" +#include "base/check_op.h" #include "base/numerics/safe_conversions.h" #include "build/build_config.h" #include "media/parsers/vp8_parser.h" diff --git a/chromium/media/remoting/courier_renderer_factory.cc b/chromium/media/remoting/courier_renderer_factory.cc index 22cbb5ef31a..23598e69503 100644 --- a/chromium/media/remoting/courier_renderer_factory.cc +++ b/chromium/media/remoting/courier_renderer_factory.cc @@ -6,7 +6,7 @@ #include <memory> -#include "base/logging.h" +#include "base/check.h" #include "build/build_config.h" #include "build/buildflag.h" #include "media/base/overlay_info.h" diff --git a/chromium/media/remoting/integration_test.cc b/chromium/media/remoting/integration_test.cc index e56be6ae64d..bc0aa888909 100644 --- a/chromium/media/remoting/integration_test.cc +++ b/chromium/media/remoting/integration_test.cc @@ -12,25 +12,24 @@ namespace media { namespace remoting { -namespace { - constexpr int kAppendTimeSec = 1; -std::unique_ptr<Renderer> CreateEnd2EndTestRenderer( - std::unique_ptr<Renderer> default_renderer) { - return std::make_unique<End2EndTestRenderer>(std::move(default_renderer)); -} - -} // namespace - class MediaRemotingIntegrationTest : public testing::Test, public PipelineIntegrationTestBase { public: MediaRemotingIntegrationTest() { - SetWrapRendererCB(base::BindRepeating(&CreateEnd2EndTestRenderer)); + SetCreateRendererCB(base::BindRepeating( + &MediaRemotingIntegrationTest::CreateEnd2EndTestRenderer, + base::Unretained(this))); } private: + std::unique_ptr<Renderer> CreateEnd2EndTestRenderer( + base::Optional<RendererFactoryType> factory_type) { + return std::make_unique<End2EndTestRenderer>( + this->CreateDefaultRenderer(factory_type)); + } + DISALLOW_COPY_AND_ASSIGN(MediaRemotingIntegrationTest); }; diff --git a/chromium/media/remoting/renderer_controller.cc b/chromium/media/remoting/renderer_controller.cc index a6f8b0ce595..ccbddb1b424 100644 --- a/chromium/media/remoting/renderer_controller.cc +++ b/chromium/media/remoting/renderer_controller.cc @@ -296,17 +296,22 @@ void RendererController::OnDataSourceInitialized( UpdateRemotePlaybackAvailabilityMonitoringState(); } +void RendererController::OnHlsManifestDetected() { +#if defined(OS_ANDROID) + is_hls_ = true; + UpdateRemotePlaybackAvailabilityMonitoringState(); +#else + NOTREACHED(); +#endif +} + void RendererController::UpdateRemotePlaybackAvailabilityMonitoringState() { // Currently RemotePlayback-initated media remoting only supports URL flinging // thus the source is supported when the URL is either http or https, video and // audio codecs are supported by the remote playback device; HLS is playable by // Chrome on Android (which is not detected by the pipeline metadata atm). #if defined(OS_ANDROID) - // TODO(tguilbert): Detect the presence of HLS based on demuxing results, - // rather than the URL string. See crbug.com/663503. - const bool is_media_supported = - MediaCodecUtil::IsHLSURL(url_after_redirects_) || - IsRemotePlaybackSupported(); + const bool is_media_supported = is_hls_ || IsRemotePlaybackSupported(); #else const bool is_media_supported = IsAudioOrVideoSupported(); #endif diff --git a/chromium/media/remoting/renderer_controller.h b/chromium/media/remoting/renderer_controller.h index a38bf5b3ff0..8c22bbf09eb 100644 --- a/chromium/media/remoting/renderer_controller.h +++ b/chromium/media/remoting/renderer_controller.h @@ -61,6 +61,7 @@ class RendererController final : public mojom::RemotingSource, void OnPlaying() override; void OnPaused() override; void OnDataSourceInitialized(const GURL& url_after_redirects) override; + void OnHlsManifestDetected() override; void SetClient(MediaObserverClient* client) override; base::WeakPtr<RendererController> GetWeakPtr() { @@ -213,6 +214,8 @@ class RendererController final : public mojom::RemotingSource, // Current data source information. GURL url_after_redirects_; + bool is_hls_ = false; + // Records session events of interest. SessionMetricsRecorder metrics_recorder_; diff --git a/chromium/media/remoting/renderer_controller_unittest.cc b/chromium/media/remoting/renderer_controller_unittest.cc index 809ae074f82..b42a44457c9 100644 --- a/chromium/media/remoting/renderer_controller_unittest.cc +++ b/chromium/media/remoting/renderer_controller_unittest.cc @@ -89,7 +89,9 @@ class RendererControllerTest : public ::testing::Test, unsigned DecodedFrameCount() const override { return decoded_frames_; } - void UpdateRemotePlaybackCompatibility(bool is_compatibe) override {} + void UpdateRemotePlaybackCompatibility(bool is_compatible) override { + is_remote_playback_compatible_ = is_compatible; + } void CreateCdm(bool is_remoting) { is_remoting_cdm_ = is_remoting; } @@ -163,6 +165,7 @@ class RendererControllerTest : public ::testing::Test, bool is_rendering_remotely_ = false; bool is_remoting_cdm_ = false; bool disable_pipeline_suspend_ = false; + bool is_remote_playback_compatible_ = false; size_t decoded_bytes_ = 0; unsigned decoded_frames_ = 0; base::SimpleTestTickClock clock_; @@ -397,5 +400,24 @@ TEST_F(RendererControllerTest, SetClientNullptr) { ExpectInLocalRendering(); } +#if defined(OS_ANDROID) +TEST_F(RendererControllerTest, RemotePlaybackHlsCompatibility) { + controller_ = FakeRemoterFactory::CreateController(true); + controller_->SetClient(this); + + controller_->OnDataSourceInitialized(GURL("http://example.com/foo.m3u8")); + + PipelineMetadata incompatible_metadata; + incompatible_metadata.has_video = false; + incompatible_metadata.has_audio = false; + controller_->OnMetadataChanged(incompatible_metadata); + EXPECT_FALSE(is_remote_playback_compatible_); + + // HLS is compatible with RemotePlayback regardless of the metadata we have. + controller_->OnHlsManifestDetected(); + EXPECT_TRUE(is_remote_playback_compatible_); +} +#endif + } // namespace remoting } // namespace media diff --git a/chromium/media/renderers/BUILD.gn b/chromium/media/renderers/BUILD.gn index 509124a518c..684f41f71fe 100644 --- a/chromium/media/renderers/BUILD.gn +++ b/chromium/media/renderers/BUILD.gn @@ -32,6 +32,8 @@ source_set("renderers") { "video_renderer_impl.h", "video_resource_updater.cc", "video_resource_updater.h", + "yuv_util.cc", + "yuv_util.h", ] deps = [ @@ -59,7 +61,6 @@ source_set("renderers") { if (is_fuchsia) { deps += [ "//fuchsia/engine:switches" ] } - configs += [ "//media:subcomponent_config", @@ -97,7 +98,14 @@ source_set("unit_tests") { "//ui/gl:test_support", ] if (is_win) { - deps += [ "//media/renderers:media_foundation_renderer" ] + sources += [ + "win/media_foundation_renderer_integration_test.cc", + "win/media_foundation_renderer_unittest.cc", + ] + deps += [ + "//media/renderers:media_foundation_renderer", + "//media/test:pipeline_integration_test_base", + ] } } @@ -112,6 +120,9 @@ if (is_win) { "win/media_foundation_audio_stream.h", "win/media_foundation_protection_manager.cc", "win/media_foundation_protection_manager.h", + "win/media_foundation_renderer.cc", + "win/media_foundation_renderer.h", + "win/media_foundation_renderer_extension.h", "win/media_foundation_source_wrapper.cc", "win/media_foundation_source_wrapper.h", "win/media_foundation_stream_wrapper.cc", diff --git a/chromium/media/renderers/default_decoder_factory.cc b/chromium/media/renderers/default_decoder_factory.cc index c3a41f45c68..f86855c087b 100644 --- a/chromium/media/renderers/default_decoder_factory.cc +++ b/chromium/media/renderers/default_decoder_factory.cc @@ -121,10 +121,22 @@ void DefaultDecoderFactory::CreateVideoDecoders( #if defined(OS_FUCHSIA) if (gpu_factories) { auto* context_provider = gpu_factories->GetMediaContextProvider(); - DCHECK(context_provider); - video_decoders->push_back( - CreateFuchsiaVideoDecoder(gpu_factories->SharedImageInterface(), - context_provider->ContextSupport())); + + // GetMediaContextProvider() may return nullptr when the context was lost + // (e.g. after GPU process crash). To handle this case RenderThreadImpl + // creates a new GpuVideoAcceleratorFactories with a new ContextProvider + // instance, but there is no way to get it here. For now just don't add + // FuchsiaVideoDecoder in that scenario. + // + // TODO(crbug.com/580386): Handle context loss properly. + if (context_provider) { + video_decoders->push_back( + CreateFuchsiaVideoDecoder(gpu_factories->SharedImageInterface(), + context_provider->ContextSupport())); + } else { + DLOG(ERROR) + << "Can't created FuchsiaVideoDecoder due to GPU context loss."; + } } if (base::CommandLine::ForCurrentProcess()->HasSwitch( diff --git a/chromium/media/renderers/default_renderer_factory.cc b/chromium/media/renderers/default_renderer_factory.cc index 074896d761a..9d5c2ac959f 100644 --- a/chromium/media/renderers/default_renderer_factory.cc +++ b/chromium/media/renderers/default_renderer_factory.cc @@ -8,6 +8,7 @@ #include <utility> #include "base/bind.h" +#include "build/build_config.h" #include "media/base/audio_buffer.h" #include "media/base/bind_to_current_loop.h" #include "media/base/decoder_factory.h" @@ -19,6 +20,17 @@ namespace media { +#if defined(OS_ANDROID) +DefaultRendererFactory::DefaultRendererFactory( + MediaLog* media_log, + DecoderFactory* decoder_factory, + const GetGpuFactoriesCB& get_gpu_factories_cb) + : media_log_(media_log), + decoder_factory_(decoder_factory), + get_gpu_factories_cb_(get_gpu_factories_cb) { + DCHECK(decoder_factory_); +} +#else DefaultRendererFactory::DefaultRendererFactory( MediaLog* media_log, DecoderFactory* decoder_factory, @@ -30,6 +42,7 @@ DefaultRendererFactory::DefaultRendererFactory( speech_recognition_client_(std::move(speech_recognition_client)) { DCHECK(decoder_factory_); } +#endif DefaultRendererFactory::~DefaultRendererFactory() = default; @@ -115,10 +128,12 @@ std::unique_ptr<Renderer> DefaultRendererFactory::CreateRenderer( void DefaultRendererFactory::TranscribeAudio( scoped_refptr<media::AudioBuffer> buffer) { +#if !defined(OS_ANDROID) if (speech_recognition_client_ && speech_recognition_client_->IsSpeechRecognitionAvailable()) { speech_recognition_client_->AddAudio(std::move(buffer)); } +#endif } } // namespace media diff --git a/chromium/media/renderers/default_renderer_factory.h b/chromium/media/renderers/default_renderer_factory.h index b47761f561f..09de8928651 100644 --- a/chromium/media/renderers/default_renderer_factory.h +++ b/chromium/media/renderers/default_renderer_factory.h @@ -10,9 +10,13 @@ #include "base/callback.h" #include "base/macros.h" +#include "build/build_config.h" #include "media/base/media_export.h" #include "media/base/renderer_factory.h" + +#if !defined(OS_ANDROID) #include "media/base/speech_recognition_client.h" +#endif namespace media { @@ -36,11 +40,17 @@ class MEDIA_EXPORT DefaultRendererFactory : public RendererFactory { using GetGpuFactoriesCB = base::RepeatingCallback<GpuVideoAcceleratorFactories*()>; +#if defined(OS_ANDROID) + DefaultRendererFactory(MediaLog* media_log, + DecoderFactory* decoder_factory, + const GetGpuFactoriesCB& get_gpu_factories_cb); +#else DefaultRendererFactory( MediaLog* media_log, DecoderFactory* decoder_factory, const GetGpuFactoriesCB& get_gpu_factories_cb, std::unique_ptr<SpeechRecognitionClient> speech_recognition_client); +#endif ~DefaultRendererFactory() final; std::unique_ptr<Renderer> CreateRenderer( @@ -71,7 +81,9 @@ class MEDIA_EXPORT DefaultRendererFactory : public RendererFactory { // Creates factories for supporting video accelerators. May be null. GetGpuFactoriesCB get_gpu_factories_cb_; +#if !defined(OS_ANDROID) std::unique_ptr<SpeechRecognitionClient> speech_recognition_client_; +#endif DISALLOW_COPY_AND_ASSIGN(DefaultRendererFactory); }; diff --git a/chromium/media/renderers/paint_canvas_video_renderer.cc b/chromium/media/renderers/paint_canvas_video_renderer.cc index 0fdd0de5bf8..5eaa9529549 100644 --- a/chromium/media/renderers/paint_canvas_video_renderer.cc +++ b/chromium/media/renderers/paint_canvas_video_renderer.cc @@ -32,9 +32,11 @@ #include "gpu/command_buffer/common/shared_image_usage.h" #include "media/base/data_buffer.h" #include "media/base/video_frame.h" +#include "media/renderers/yuv_util.h" #include "third_party/libyuv/include/libyuv.h" #include "third_party/skia/include/core/SkImage.h" #include "third_party/skia/include/core/SkImageGenerator.h" +#include "third_party/skia/include/core/SkImageInfo.h" #include "third_party/skia/include/gpu/GrBackendSurface.h" #include "third_party/skia/include/gpu/GrContext.h" #include "third_party/skia/include/gpu/gl/GrGLTypes.h" @@ -46,12 +48,14 @@ // shown here to indicate where ideal conversions are currently missing. #if SK_B32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_R32_SHIFT == 16 && \ SK_A32_SHIFT == 24 +#define LIBYUV_I400_TO_ARGB libyuv::I400ToARGB #define LIBYUV_I420_TO_ARGB libyuv::I420ToARGB #define LIBYUV_I422_TO_ARGB libyuv::I422ToARGB #define LIBYUV_I444_TO_ARGB libyuv::I444ToARGB #define LIBYUV_I420ALPHA_TO_ARGB libyuv::I420AlphaToARGB +#define LIBYUV_J400_TO_ARGB libyuv::J400ToARGB #define LIBYUV_J420_TO_ARGB libyuv::J420ToARGB #define LIBYUV_J422_TO_ARGB libyuv::J422ToARGB #define LIBYUV_J444_TO_ARGB libyuv::J444ToARGB @@ -83,12 +87,14 @@ #define LIBYUV_NV12_TO_ARGB libyuv::NV12ToARGB #elif SK_R32_SHIFT == 0 && SK_G32_SHIFT == 8 && SK_B32_SHIFT == 16 && \ SK_A32_SHIFT == 24 +#define LIBYUV_I400_TO_ARGB libyuv::I400ToARGB #define LIBYUV_I420_TO_ARGB libyuv::I420ToABGR #define LIBYUV_I422_TO_ARGB libyuv::I422ToABGR #define LIBYUV_I444_TO_ARGB libyuv::I444ToABGR #define LIBYUV_I420ALPHA_TO_ARGB libyuv::I420AlphaToABGR +#define LIBYUV_J400_TO_ARGB libyuv::J400ToARGB #define LIBYUV_J420_TO_ARGB libyuv::J420ToABGR #define LIBYUV_J422_TO_ARGB libyuv::J422ToABGR #define LIBYUV_J444_TO_ARGB libyuv::J444ToABGR @@ -147,32 +153,6 @@ class SyncTokenClientImpl : public VideoFrame::SyncTokenClient { DISALLOW_IMPLICIT_CONSTRUCTORS(SyncTokenClientImpl); }; -sk_sp<SkImage> YUVGrBackendTexturesToSkImage( - GrContext* gr_context, - gfx::ColorSpace video_color_space, - VideoPixelFormat video_format, - GrBackendTexture* yuv_textures, - const GrBackendTexture& result_texture) { - // TODO(hubbe): This should really default to rec709. - // https://crbug.com/828599 - SkYUVColorSpace color_space = kRec601_SkYUVColorSpace; - video_color_space.ToSkYUVColorSpace(&color_space); - - switch (video_format) { - case PIXEL_FORMAT_NV12: - return SkImage::MakeFromNV12TexturesCopyWithExternalBackend( - gr_context, color_space, yuv_textures, kTopLeft_GrSurfaceOrigin, - result_texture); - case PIXEL_FORMAT_I420: - return SkImage::MakeFromYUVTexturesCopyWithExternalBackend( - gr_context, color_space, yuv_textures, kTopLeft_GrSurfaceOrigin, - result_texture); - default: - NOTREACHED(); - return nullptr; - } -} - // Helper class that begins/ends access to a mailbox within a scope. The mailbox // must have been imported into |texture|. class ScopedSharedImageAccess { @@ -242,108 +222,6 @@ GLuint SynchronizeAndImportMailbox(gpu::raster::RasterInterface* ri, return ri->CreateAndConsumeForGpuRaster(mailbox); } -static constexpr size_t kNumYUVPlanes = 3; -struct YUVPlaneTextureInfo { - GrGLTextureInfo texture = {0, 0}; - bool is_shared_image = false; -}; -using YUVTexturesInfo = std::array<YUVPlaneTextureInfo, kNumYUVPlanes>; - -YUVTexturesInfo GetYUVTexturesInfo( - const VideoFrame* video_frame, - viz::RasterContextProvider* raster_context_provider) { - YUVTexturesInfo yuv_textures_info; - - gpu::raster::RasterInterface* ri = raster_context_provider->RasterInterface(); - DCHECK(ri); - // TODO(bsalomon): Use GL_RGB8 once Skia supports it. - // skbug.com/7533 - GrGLenum skia_texture_format = - video_frame->format() == PIXEL_FORMAT_NV12 ? GL_RGBA8 : GL_R8_EXT; - for (size_t i = 0; i < video_frame->NumTextures(); ++i) { - // Get the texture from the mailbox and wrap it in a GrTexture. - const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(i); - DCHECK(mailbox_holder.texture_target == GL_TEXTURE_2D || - mailbox_holder.texture_target == GL_TEXTURE_EXTERNAL_OES || - mailbox_holder.texture_target == GL_TEXTURE_RECTANGLE_ARB) - << "Unsupported texture target " << std::hex << std::showbase - << mailbox_holder.texture_target; - yuv_textures_info[i].texture.fID = SynchronizeAndImportMailbox( - ri, mailbox_holder.sync_token, mailbox_holder.mailbox); - if (mailbox_holder.mailbox.IsSharedImage()) { - yuv_textures_info[i].is_shared_image = true; - ri->BeginSharedImageAccessDirectCHROMIUM( - yuv_textures_info[i].texture.fID, - GL_SHARED_IMAGE_ACCESS_MODE_READ_CHROMIUM); - } - - yuv_textures_info[i].texture.fTarget = mailbox_holder.texture_target; - yuv_textures_info[i].texture.fFormat = skia_texture_format; - } - - return yuv_textures_info; -} - -void DeleteYUVTextures(const VideoFrame* video_frame, - viz::RasterContextProvider* raster_context_provider, - const YUVTexturesInfo& yuv_textures_info) { - gpu::raster::RasterInterface* ri = raster_context_provider->RasterInterface(); - DCHECK(ri); - - for (size_t i = 0; i < video_frame->NumTextures(); ++i) { - if (yuv_textures_info[i].is_shared_image) - ri->EndSharedImageAccessDirectCHROMIUM(yuv_textures_info[i].texture.fID); - ri->DeleteGpuRasterTexture(yuv_textures_info[i].texture.fID); - } -} - -sk_sp<SkImage> NewSkImageFromVideoFrameYUVTexturesWithExternalBackend( - const VideoFrame* video_frame, - viz::RasterContextProvider* raster_context_provider, - unsigned int texture_target, - unsigned int texture_id) { - DCHECK(video_frame->HasTextures()); - GrContext* gr_context = raster_context_provider->GrContext(); - DCHECK(gr_context); - // TODO: We should compare the DCHECK vs when UpdateLastImage calls this - // function. (https://crbug.com/674185) - DCHECK(video_frame->format() == PIXEL_FORMAT_I420 || - video_frame->format() == PIXEL_FORMAT_NV12); - - gfx::Size ya_tex_size = video_frame->coded_size(); - gfx::Size uv_tex_size((ya_tex_size.width() + 1) / 2, - (ya_tex_size.height() + 1) / 2); - - GrGLTextureInfo backend_texture{}; - - YUVTexturesInfo yuv_textures_info = - GetYUVTexturesInfo(video_frame, raster_context_provider); - - GrBackendTexture yuv_textures[3] = { - GrBackendTexture(ya_tex_size.width(), ya_tex_size.height(), - GrMipMapped::kNo, yuv_textures_info[0].texture), - GrBackendTexture(uv_tex_size.width(), uv_tex_size.height(), - GrMipMapped::kNo, yuv_textures_info[1].texture), - GrBackendTexture(uv_tex_size.width(), uv_tex_size.height(), - GrMipMapped::kNo, yuv_textures_info[2].texture), - }; - backend_texture.fID = texture_id; - backend_texture.fTarget = texture_target; - backend_texture.fFormat = GL_RGBA8; - GrBackendTexture result_texture(video_frame->coded_size().width(), - video_frame->coded_size().height(), - GrMipMapped::kNo, backend_texture); - - sk_sp<SkImage> img = YUVGrBackendTexturesToSkImage( - gr_context, video_frame->ColorSpace(), video_frame->format(), - yuv_textures, result_texture); - gr_context->flush(); - - DeleteYUVTextures(video_frame, raster_context_provider, yuv_textures_info); - - return img; -} - const gpu::MailboxHolder& GetVideoFrameMailboxHolder(VideoFrame* video_frame) { DCHECK(video_frame->HasTextures()); DCHECK_EQ(video_frame->NumTextures(), 1u); @@ -377,18 +255,6 @@ GLuint ImportVideoFrameSingleMailbox(gpu::gles2::GLES2Interface* gl, return SynchronizeAndImportMailbox(gl, mailbox_holder.sync_token, *mailbox); } -// TODO(crbug.com/1023270): Remove this function once we're no longer relying on -// texture ids for Mailbox access as that is only supported on -// RasterImplementationGLES. -GLuint ImportVideoFrameSingleMailbox(gpu::raster::RasterInterface* ri, - VideoFrame* video_frame, - gpu::Mailbox* mailbox) { - const gpu::MailboxHolder& mailbox_holder = - GetVideoFrameMailboxHolder(video_frame); - *mailbox = mailbox_holder.mailbox; - return SynchronizeAndImportMailbox(ri, mailbox_holder.sync_token, *mailbox); -} - gpu::Mailbox SynchronizeVideoFrameSingleMailbox( gpu::raster::RasterInterface* ri, VideoFrame* video_frame) { @@ -551,6 +417,18 @@ void ConvertVideoFrameToRGBPixelsTask(const VideoFrame* video_frame, SkYUVColorSpace color_space = kRec601_SkYUVColorSpace; video_frame->ColorSpace().ToSkYUVColorSpace(&color_space); + if (!video_frame->data(VideoFrame::kUPlane) && + !video_frame->data(VideoFrame::kVPlane)) { + DCHECK_EQ(video_frame->format(), PIXEL_FORMAT_I420); + auto func = (color_space == kJPEG_SkYUVColorSpace) ? LIBYUV_J400_TO_ARGB + : LIBYUV_I400_TO_ARGB; + func(plane_meta[VideoFrame::kYPlane].data, + plane_meta[VideoFrame::kYPlane].stride, pixels, row_bytes, width, + rows); + done->Run(); + return; + } + auto convert_yuv = [&](auto&& func) { func(plane_meta[VideoFrame::kYPlane].data, plane_meta[VideoFrame::kYPlane].stride, @@ -869,6 +747,22 @@ class VideoImageGenerator : public cc::PaintImageGenerator { DISALLOW_IMPLICIT_CONSTRUCTORS(VideoImageGenerator); }; +class VideoTextureBacking : public cc::TextureBacking { + public: + explicit VideoTextureBacking(sk_sp<SkImage> sk_image) + : sk_image_(std::move(sk_image)) {} + + const SkImageInfo& GetSkImageInfo() override { + return sk_image_->imageInfo(); + } + gpu::Mailbox GetMailbox() const override { return mailbox_; } + sk_sp<SkImage> GetAcceleratedSkImage() override { return sk_image_; } + + private: + const sk_sp<SkImage> sk_image_; + const gpu::Mailbox mailbox_; +}; + PaintCanvasVideoRenderer::PaintCanvasVideoRenderer() : cache_deleting_timer_( FROM_HERE, @@ -1030,6 +924,7 @@ scoped_refptr<VideoFrame> DownShiftHighbitVideoFrame( VideoPixelFormat format; switch (video_frame->format()) { case PIXEL_FORMAT_YUV420P12: + case PIXEL_FORMAT_YUV420P10: case PIXEL_FORMAT_YUV420P9: format = PIXEL_FORMAT_I420; break; @@ -1065,6 +960,13 @@ scoped_refptr<VideoFrame> DownShiftHighbitVideoFrame( const uint16_t* src = reinterpret_cast<const uint16_t*>(video_frame->data(plane)); uint8_t* dst = ret->data(plane); + if (!src) { + // An AV1 monochrome (grayscale) frame has no U and V planes. Set all U + // and V samples to the neutral value (128). + DCHECK_NE(plane, VideoFrame::kYPlane); + memset(dst, 128, ret->rows(plane) * ret->stride(plane)); + continue; + } for (int row = 0; row < video_frame->rows(plane); row++) { for (int x = 0; x < width; x++) { dst[x] = src[x] >> shift; @@ -1227,6 +1129,17 @@ void PaintCanvasVideoRenderer::ConvertVideoFrameToRGBPixels( temporary_frame = DownShiftHighbitVideoFrame(video_frame); video_frame = temporary_frame.get(); break; + case PIXEL_FORMAT_YUV420P10: + // In AV1, a monochrome (grayscale) frame is represented as a YUV 4:2:0 + // frame with no U and V planes. Since there are no 10-bit versions of + // libyuv::I400ToARGB() and libyuv::J400ToARGB(), convert the frame to an + // 8-bit YUV 4:2:0 frame with U and V planes. + if (!video_frame->data(VideoFrame::kUPlane) && + !video_frame->data(VideoFrame::kVPlane)) { + temporary_frame = DownShiftHighbitVideoFrame(video_frame); + video_frame = temporary_frame.get(); + } + break; case PIXEL_FORMAT_Y16: // Since it is grayscale conversion, we disregard // SK_PMCOLOR_BYTE_ORDER and always use GL_RGBA. @@ -1415,13 +1328,8 @@ bool PaintCanvasVideoRenderer::PrepareVideoFrameForWebGL( destination_gl->GenUnverifiedSyncTokenCHROMIUM( mailbox_holder.sync_token.GetData()); - source_ri->WaitSyncTokenCHROMIUM(mailbox_holder.sync_token.GetConstData()); - - uint32_t shared_texture = - source_ri->CreateAndConsumeForGpuRaster(mailbox_holder.mailbox); - - if (!PrepareVideoFrame(video_frame, raster_context_provider, target, - shared_texture)) { + if (!PrepareVideoFrame(video_frame, raster_context_provider, + mailbox_holder)) { return false; } @@ -1698,9 +1606,7 @@ bool PaintCanvasVideoRenderer::Cache::Recycle() { // We need a new texture ID because skia will destroy the previous one with // the SkImage. texture_ownership_in_skia = false; - source_texture = - SynchronizeAndImportMailbox(raster_context_provider->RasterInterface(), - gpu::SyncToken(), source_mailbox); + source_texture = 0; return true; } @@ -1728,17 +1634,13 @@ bool PaintCanvasVideoRenderer::UpdateLastImage( auto* ri = raster_context_provider->RasterInterface(); DCHECK(ri); - sk_sp<SkImage> source_image; - if (allow_wrap_texture && video_frame->NumTextures() == 1) { cache_.emplace(video_frame->unique_id()); - cache_->source_texture = ImportVideoFrameSingleMailbox( - ri, video_frame.get(), &cache_->source_mailbox); + const gpu::MailboxHolder& holder = + GetVideoFrameMailboxHolder(video_frame.get()); + cache_->source_mailbox = holder.mailbox; + ri->WaitSyncTokenCHROMIUM(holder.sync_token.GetConstData()); cache_->wraps_video_frame_texture = true; - source_image = - WrapGLTexture(video_frame->mailbox_holder(0).texture_target, - cache_->source_texture, video_frame->coded_size(), - video_frame->ColorSpace(), raster_context_provider); } else { if (cache_ && cache_->raster_context_provider == raster_context_provider && @@ -1749,11 +1651,20 @@ bool PaintCanvasVideoRenderer::UpdateLastImage( } else { cache_.emplace(video_frame->unique_id()); auto* sii = raster_context_provider->SharedImageInterface(); + + // TODO(nazabris): Sort out what to do when GLES2 is needed but the + // cached shared image is created without it. + uint32_t flags = + gpu::SHARED_IMAGE_USAGE_GLES2 | gpu::SHARED_IMAGE_USAGE_RASTER; + if (raster_context_provider->ContextCapabilities() + .supports_oop_raster) { + flags |= gpu::SHARED_IMAGE_USAGE_OOP_RASTERIZATION; + } cache_->source_mailbox = sii->CreateSharedImage( viz::ResourceFormat::RGBA_8888, video_frame->coded_size(), - gfx::ColorSpace(), gpu::SHARED_IMAGE_USAGE_GLES2); - cache_->source_texture = SynchronizeAndImportMailbox( - ri, sii->GenUnverifiedSyncToken(), cache_->source_mailbox); + gfx::ColorSpace(), flags); + ri->WaitSyncTokenCHROMIUM( + sii->GenUnverifiedSyncToken().GetConstData()); } DCHECK(!cache_->texture_ownership_in_skia); @@ -1764,19 +1675,31 @@ bool PaintCanvasVideoRenderer::UpdateLastImage( frame_mailbox, cache_->source_mailbox, GL_TEXTURE_2D, 0, 0, 0, 0, video_frame->coded_size().width(), video_frame->coded_size().height(), GL_FALSE, GL_FALSE); - source_image = WrapGLTexture( - GL_TEXTURE_2D, cache_->source_texture, video_frame->coded_size(), - gfx::ColorSpace(), raster_context_provider); } else { - ScopedSharedImageAccess dest_access( - ri, cache_->source_texture, cache_->source_mailbox, - GL_SHARED_IMAGE_ACCESS_MODE_READWRITE_CHROMIUM); - source_image = NewSkImageFromVideoFrameYUVTexturesWithExternalBackend( - video_frame.get(), raster_context_provider, GL_TEXTURE_2D, - cache_->source_texture); + gpu::MailboxHolder dest_holder{cache_->source_mailbox, + gpu::SyncToken(), GL_TEXTURE_2D}; + ConvertFromVideoFrameYUV(video_frame.get(), raster_context_provider, + dest_holder); } raster_context_provider->GrContext()->flush(); } + + // TODO(jochin): Don't always generate SkImage here. + DCHECK(cache_->source_texture == 0); + cache_->source_texture = + ri->CreateAndConsumeForGpuRaster(cache_->source_mailbox); + + // TODO(nazabris): Handle scoped access correctly. This follows the + // current pattern but is most likely bugged. Access should last for the + // lifetime of the SkImage. + ScopedSharedImageAccess(ri, cache_->source_texture, + cache_->source_mailbox); + auto source_image = + WrapGLTexture(cache_->wraps_video_frame_texture + ? video_frame->mailbox_holder(0).texture_target + : GL_TEXTURE_2D, + cache_->source_texture, video_frame->coded_size(), + video_frame->ColorSpace(), raster_context_provider); if (!source_image) { // Couldn't create the SkImage. cache_.reset(); @@ -1809,8 +1732,10 @@ bool PaintCanvasVideoRenderer::UpdateLastImage( kPremul_SkAlphaType, source_image->imageInfo().refColorSpace()); } } - paint_image_builder.set_image(source_subset, - cc::PaintImage::GetNextContentId()); + paint_image_builder.set_texture_backing( + sk_sp<VideoTextureBacking>( + new VideoTextureBacking(std::move(source_subset))), + cc::PaintImage::GetNextContentId()); } else { cache_.emplace(video_frame->unique_id()); paint_image_builder.set_paint_image_generator( @@ -1832,48 +1757,21 @@ bool PaintCanvasVideoRenderer::UpdateLastImage( bool PaintCanvasVideoRenderer::PrepareVideoFrame( scoped_refptr<VideoFrame> video_frame, viz::RasterContextProvider* raster_context_provider, - unsigned int textureTarget, - unsigned int texture) { - cache_.emplace(video_frame->unique_id()); - auto paint_image_builder = - cc::PaintImageBuilder::WithDefault() - .set_id(renderer_stable_id_) - .set_animation_type(cc::PaintImage::AnimationType::VIDEO) - .set_completion_state(cc::PaintImage::CompletionState::DONE); - + const gpu::MailboxHolder& dest_holder) { // Generate a new image. // Note: Skia will hold onto |video_frame| via |video_generator| only when // |video_frame| is software. // Holding |video_frame| longer than this call when using GPUVideoDecoder // could cause problems since the pool of VideoFrames has a fixed size. if (video_frame->HasTextures()) { - DCHECK(raster_context_provider); - DCHECK(raster_context_provider->GrContext()); - DCHECK(raster_context_provider->RasterInterface()); - sk_sp<SkImage> source_image; if (video_frame->NumTextures() > 1) { - source_image = NewSkImageFromVideoFrameYUVTexturesWithExternalBackend( - video_frame.get(), raster_context_provider, textureTarget, texture); - if (!source_image) { - // Couldn't create the SkImage. - cache_.reset(); - return false; - } + ConvertFromVideoFrameYUV(video_frame.get(), raster_context_provider, + dest_holder); } else { // We don't support Android now. - cache_.reset(); return false; } - cache_->coded_size = video_frame->coded_size(); - cache_->visible_rect = video_frame->visible_rect(); - paint_image_builder.set_image( - source_image->makeSubset(gfx::RectToSkIRect(cache_->visible_rect)), - cc::PaintImage::GetNextContentId()); - } else { - paint_image_builder.set_paint_image_generator( - sk_make_sp<VideoImageGenerator>(video_frame)); } - cache_deleting_timer_.Reset(); return true; } diff --git a/chromium/media/renderers/paint_canvas_video_renderer.h b/chromium/media/renderers/paint_canvas_video_renderer.h index d8ca22e360e..d5b1a141670 100644 --- a/chromium/media/renderers/paint_canvas_video_renderer.h +++ b/chromium/media/renderers/paint_canvas_video_renderer.h @@ -245,8 +245,7 @@ class MEDIA_EXPORT PaintCanvasVideoRenderer { bool PrepareVideoFrame(scoped_refptr<VideoFrame> video_frame, viz::RasterContextProvider* raster_context_provider, - unsigned int textureTarget, - unsigned int texture); + const gpu::MailboxHolder& dest_holder); base::Optional<Cache> cache_; diff --git a/chromium/media/renderers/video_resource_updater.cc b/chromium/media/renderers/video_resource_updater.cc index 8eeaa0a6c4d..27678f2b65f 100644 --- a/chromium/media/renderers/video_resource_updater.cc +++ b/chromium/media/renderers/video_resource_updater.cc @@ -528,13 +528,13 @@ void VideoResourceUpdater::AppendQuads(viz::RenderPass* render_pass, gfx::Rect visible_rect = frame->visible_rect(); gfx::Size coded_size = frame->coded_size(); - const float tex_width_scale = - static_cast<float>(visible_rect.width()) / coded_size.width(); - const float tex_height_scale = - static_cast<float>(visible_rect.height()) / coded_size.height(); + const gfx::PointF uv_top_left( + static_cast<float>(visible_rect.x()) / coded_size.width(), + static_cast<float>(visible_rect.y()) / coded_size.height()); - const gfx::PointF uv_top_left(0.f, 0.f); - const gfx::PointF uv_bottom_right(tex_width_scale, tex_height_scale); + const gfx::PointF uv_bottom_right( + static_cast<float>(visible_rect.right()) / coded_size.width(), + static_cast<float>(visible_rect.bottom()) / coded_size.height()); switch (frame_resource_type_) { case VideoFrameResourceType::VIDEO_HOLE: { @@ -620,17 +620,13 @@ void VideoResourceUpdater::AppendQuads(viz::RenderPass* render_pass, protected_video_type = gfx::ProtectedVideoType::kSoftwareProtected; } - const gfx::Vector2dF offset( - static_cast<float>(visible_rect.x()) / coded_size.width(), - static_cast<float>(visible_rect.y()) / coded_size.height()); - auto* texture_quad = render_pass->CreateAndAppendDrawQuad<viz::TextureDrawQuad>(); - texture_quad->SetNew( - shared_quad_state, quad_rect, visible_quad_rect, needs_blending, - frame_resources_[0].id, premultiplied_alpha, uv_top_left + offset, - uv_bottom_right + offset, SK_ColorTRANSPARENT, opacity, flipped, - nearest_neighbor, false, protected_video_type); + texture_quad->SetNew(shared_quad_state, quad_rect, visible_quad_rect, + needs_blending, frame_resources_[0].id, + premultiplied_alpha, uv_top_left, uv_bottom_right, + SK_ColorTRANSPARENT, opacity, flipped, + nearest_neighbor, false, protected_video_type); texture_quad->set_resource_size_in_pixels(coded_size); for (viz::ResourceId resource_id : texture_quad->resources) { resource_provider_->ValidateResource(resource_id); diff --git a/chromium/media/renderers/win/media_engine_extension.cc b/chromium/media/renderers/win/media_engine_extension.cc index 1fb3147f365..fcb34978e31 100644 --- a/chromium/media/renderers/win/media_engine_extension.cc +++ b/chromium/media/renderers/win/media_engine_extension.cc @@ -16,7 +16,7 @@ MediaEngineExtension::MediaEngineExtension() = default; MediaEngineExtension::~MediaEngineExtension() = default; HRESULT MediaEngineExtension::RuntimeClassInitialize() { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); return S_OK; } @@ -35,7 +35,7 @@ HRESULT MediaEngineExtension::BeginCreateObject(BSTR url_bstr, IUnknown** cancel_cookie, IMFAsyncCallback* callback, IUnknown* state) { - DVLOG(1) << __func__ << ": this=" << this << ",type=" << type; + DVLOG_FUNC(1) << "type=" << type; if (cancel_cookie) { // We don't support a cancel cookie. @@ -51,7 +51,7 @@ HRESULT MediaEngineExtension::BeginCreateObject(BSTR url_bstr, } if (type == MF_OBJECT_MEDIASOURCE) { - DVLOG(2) << "Begin to resolve mf_media_source_: this=" << this; + DVLOG_FUNC(2) << "Begin to resolve |mf_media_source_|"; DCHECK(local_source) << "Media Source should have been set"; ComPtr<IMFAsyncResult> async_result; @@ -71,20 +71,20 @@ HRESULT MediaEngineExtension::BeginCreateObject(BSTR url_bstr, HRESULT MediaEngineExtension::CancelObjectCreation( __in IUnknown* cancel_cookie) { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); return MF_E_UNEXPECTED; } HRESULT MediaEngineExtension::EndCreateObject(__in IMFAsyncResult* result, __deref_out IUnknown** ret_obj) { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); *ret_obj = nullptr; if (!pending_create_object_) return MF_E_UNEXPECTED; - DVLOG(2) << "End to resolve mf_media_source_: this=" << this; + DVLOG_FUNC(2) << "End to resolve |mf_media_source_|"; RETURN_IF_FAILED(result->GetStatus()); RETURN_IF_FAILED(result->GetObject(ret_obj)); pending_create_object_ = false; @@ -92,7 +92,7 @@ HRESULT MediaEngineExtension::EndCreateObject(__in IMFAsyncResult* result, } HRESULT MediaEngineExtension::SetMediaSource(IUnknown* mf_media_source) { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); base::AutoLock lock(lock_); if (has_shutdown_) @@ -103,7 +103,7 @@ HRESULT MediaEngineExtension::SetMediaSource(IUnknown* mf_media_source) { // Break cycles. void MediaEngineExtension::Shutdown() { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); base::AutoLock lock(lock_); if (!has_shutdown_) { diff --git a/chromium/media/renderers/win/media_engine_notify_impl.cc b/chromium/media/renderers/win/media_engine_notify_impl.cc index 4a36907a16f..e68adcdbc81 100644 --- a/chromium/media/renderers/win/media_engine_notify_impl.cc +++ b/chromium/media/renderers/win/media_engine_notify_impl.cc @@ -4,10 +4,64 @@ #include "media/renderers/win/media_engine_notify_impl.h" +#include "media/base/win/mf_helpers.h" + namespace media { namespace { +#define ENUM_TO_STRING(enum) \ + case enum: \ + return #enum + +std::string MediaEngineEventToString(MF_MEDIA_ENGINE_EVENT event) { + switch (event) { + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_LOADSTART); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_PROGRESS); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_SUSPEND); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_ABORT); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_ERROR); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_EMPTIED); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_STALLED); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_PLAY); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_PAUSE); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_LOADEDMETADATA); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_LOADEDDATA); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_WAITING); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_PLAYING); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_CANPLAY); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_CANPLAYTHROUGH); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_SEEKING); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_SEEKED); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_TIMEUPDATE); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_ENDED); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_RATECHANGE); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_DURATIONCHANGE); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_VOLUMECHANGE); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_FORMATCHANGE); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_PURGEQUEUEDEVENTS); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_TIMELINE_MARKER); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_BALANCECHANGE); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_DOWNLOADCOMPLETE); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_BUFFERINGSTARTED); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_BUFFERINGENDED); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_FRAMESTEPCOMPLETED); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_NOTIFYSTABLESTATE); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_FIRSTFRAMEREADY); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_TRACKSCHANGE); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_OPMINFO); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_RESOURCELOST); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_DELAYLOADEVENT_CHANGED); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_STREAMRENDERINGERROR); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_SUPPORTEDRATES_CHANGED); + ENUM_TO_STRING(MF_MEDIA_ENGINE_EVENT_AUDIOENDPOINTCHANGE); + default: + return "Unknown MF_MEDIA_ENGINE_EVENT"; + } +} + +#undef ENUM_TO_STRING + PipelineStatus MediaEngineStatusToPipelineStatus( MF_MEDIA_ENGINE_ERR media_engine_status) { switch (media_engine_status) { @@ -37,16 +91,16 @@ MediaEngineNotifyImpl::~MediaEngineNotifyImpl() = default; HRESULT MediaEngineNotifyImpl::RuntimeClassInitialize( ErrorCB error_cb, EndedCB ended_cb, - DurationChangedCB duration_changed_cb, BufferingStateChangedCB buffering_state_changed_cb, - VideoNaturalSizeChangedCB video_natural_size_changed_cb) { - DVLOG(1) << __func__ << ": this=" << this; + VideoNaturalSizeChangedCB video_natural_size_changed_cb, + TimeUpdateCB time_update_cb) { + DVLOG_FUNC(1); error_cb_ = std::move(error_cb); ended_cb_ = std::move(ended_cb); - duration_changed_cb_ = std::move(duration_changed_cb); buffering_state_changed_cb_ = std::move(buffering_state_changed_cb); video_natural_size_changed_cb_ = std::move(video_natural_size_changed_cb); + time_update_cb_ = std::move(time_update_cb); return S_OK; } @@ -57,29 +111,26 @@ HRESULT MediaEngineNotifyImpl::RuntimeClassInitialize( HRESULT MediaEngineNotifyImpl::EventNotify(DWORD event_code, DWORD_PTR param1, DWORD param2) { - DVLOG(3) << __func__ << ": this=" << this << ",eventCode=" << event_code - << ",param1=" << static_cast<unsigned>(param1) - << ",param2=" << static_cast<unsigned>(param2); + auto event = static_cast<MF_MEDIA_ENGINE_EVENT>(event_code); + DVLOG_FUNC(3) << "event=" << MediaEngineEventToString(event); base::AutoLock lock(lock_); if (has_shutdown_) return S_OK; - switch (static_cast<MF_MEDIA_ENGINE_EVENT>(event_code)) { + switch (event) { case MF_MEDIA_ENGINE_EVENT_ERROR: { // |param1| - A member of the MF_MEDIA_ENGINE_ERR enumeration. // |param2| - An HRESULT error code, or zero. MF_MEDIA_ENGINE_ERR error = static_cast<MF_MEDIA_ENGINE_ERR>(param1); - DLOG(ERROR) << __func__ << ": error=" << error << ",hr=" << param2; + LOG(ERROR) << __func__ << ": error=" << error + << ", hr=" << PrintHr(param2); error_cb_.Run(MediaEngineStatusToPipelineStatus(error)); break; } case MF_MEDIA_ENGINE_EVENT_ENDED: ended_cb_.Run(); break; - case MF_MEDIA_ENGINE_EVENT_DURATIONCHANGE: - duration_changed_cb_.Run(); - break; case MF_MEDIA_ENGINE_EVENT_FORMATCHANGE: video_natural_size_changed_cb_.Run(); break; @@ -96,16 +147,19 @@ HRESULT MediaEngineNotifyImpl::EventNotify(DWORD event_code, BufferingState::BUFFERING_HAVE_NOTHING, BufferingStateChangeReason::BUFFERING_CHANGE_REASON_UNKNOWN); break; + case MF_MEDIA_ENGINE_EVENT_TIMEUPDATE: + time_update_cb_.Run(); + break; + default: - DVLOG(3) << __func__ << ": this=" << this - << ", unhandled event_code=" << event_code; + DVLOG_FUNC(2) << "Unhandled event=" << MediaEngineEventToString(event); break; } return S_OK; } void MediaEngineNotifyImpl::Shutdown() { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); base::AutoLock lock(lock_); has_shutdown_ = true; diff --git a/chromium/media/renderers/win/media_engine_notify_impl.h b/chromium/media/renderers/win/media_engine_notify_impl.h index 1fe51656e60..b0dcf6a4001 100644 --- a/chromium/media/renderers/win/media_engine_notify_impl.h +++ b/chromium/media/renderers/win/media_engine_notify_impl.h @@ -18,10 +18,10 @@ namespace media { using ErrorCB = base::RepeatingCallback<void(PipelineStatus)>; using EndedCB = base::RepeatingClosure; -using DurationChangedCB = base::RepeatingClosure; using BufferingStateChangedCB = base::RepeatingCallback<void(BufferingState, BufferingStateChangeReason)>; using VideoNaturalSizeChangedCB = base::RepeatingClosure; +using TimeUpdateCB = base::RepeatingClosure; // Implements IMFMediaEngineNotify required by IMFMediaEngine // (https://docs.microsoft.com/en-us/windows/win32/api/mfmediaengine/nn-mfmediaengine-imfmediaengine). @@ -38,9 +38,9 @@ class MediaEngineNotifyImpl HRESULT RuntimeClassInitialize( ErrorCB error_cb, EndedCB ended_cb, - DurationChangedCB duration_changed_cb, BufferingStateChangedCB buffering_state_changed_cb, - VideoNaturalSizeChangedCB video_natural_size_changed_cb); + VideoNaturalSizeChangedCB video_natural_size_changed_cb, + TimeUpdateCB time_update_cb); // IMFMediaEngineNotify implementation. IFACEMETHODIMP EventNotify(DWORD event_code, @@ -55,9 +55,9 @@ class MediaEngineNotifyImpl // e.g. using BindToCurrentLoop(). ErrorCB error_cb_; EndedCB ended_cb_; - DurationChangedCB duration_changed_cb_; BufferingStateChangedCB buffering_state_changed_cb_; VideoNaturalSizeChangedCB video_natural_size_changed_cb_; + TimeUpdateCB time_update_cb_; // EventNotify is invoked from MF threadpool thread where the callbacks are // called. diff --git a/chromium/media/renderers/win/media_foundation_audio_stream.cc b/chromium/media/renderers/win/media_foundation_audio_stream.cc index ccbf6a4d3b4..68f4485e6c3 100644 --- a/chromium/media/renderers/win/media_foundation_audio_stream.cc +++ b/chromium/media/renderers/win/media_foundation_audio_stream.cc @@ -245,7 +245,7 @@ HRESULT MediaFoundationAACAudioStream::GetMediaType( // accordingly). HRESULT MediaFoundationAACAudioStream::TransformSample( Microsoft::WRL::ComPtr<IMFSample>& sample) { - DVLOG(3) << __func__ << ": this=" << this; + DVLOG_FUNC(3); if (!enable_adts_header_removal_) return S_OK; diff --git a/chromium/media/renderers/win/media_foundation_protection_manager.cc b/chromium/media/renderers/win/media_foundation_protection_manager.cc index 711e59b5952..d7ef8edf861 100644 --- a/chromium/media/renderers/win/media_foundation_protection_manager.cc +++ b/chromium/media/renderers/win/media_foundation_protection_manager.cc @@ -22,7 +22,7 @@ MediaFoundationProtectionManager::MediaFoundationProtectionManager() = default; MediaFoundationProtectionManager::~MediaFoundationProtectionManager() = default; HRESULT MediaFoundationProtectionManager::RuntimeClassInitialize() { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); if (!base::win::ScopedHString::ResolveCoreWinRTStringDelayload()) return E_FAIL; @@ -37,7 +37,7 @@ HRESULT MediaFoundationProtectionManager::RuntimeClassInitialize() { } HRESULT MediaFoundationProtectionManager::SetCdmProxy(IMFCdmProxy* cdm_proxy) { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); DCHECK(cdm_proxy); cdm_proxy_ = cdm_proxy; @@ -49,7 +49,7 @@ HRESULT MediaFoundationProtectionManager::SetCdmProxy(IMFCdmProxy* cdm_proxy) { HRESULT MediaFoundationProtectionManager::SetPMPServer( ABI::Windows::Media::Protection::IMediaProtectionPMPServer* pmp_server) { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); DCHECK(pmp_server); ComPtr<ABI::Windows::Foundation::Collections::IMap<HSTRING, IInspectable*>> @@ -72,7 +72,7 @@ HRESULT MediaFoundationProtectionManager::BeginEnableContent( IMFTopology* topology, IMFAsyncCallback* callback, IUnknown* state) { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); ComPtr<IUnknown> unknown_object; ComPtr<IMFAsyncResult> async_result; @@ -115,7 +115,7 @@ HRESULT MediaFoundationProtectionManager::BeginEnableContent( HRESULT MediaFoundationProtectionManager::EndEnableContent( IMFAsyncResult* async_result) { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); // Get status from the given |async_result| for the purpose of logging. // Returns S_OK as there is no additional work being done here. @@ -164,7 +164,7 @@ HRESULT MediaFoundationProtectionManager::remove_ComponentLoadFailed( HRESULT MediaFoundationProtectionManager::get_Properties( ABI::Windows::Foundation::Collections::IPropertySet** properties) { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); if (!properties) return E_POINTER; return property_set_.CopyTo(properties); diff --git a/chromium/media/renderers/win/media_foundation_renderer.cc b/chromium/media/renderers/win/media_foundation_renderer.cc new file mode 100644 index 00000000000..b8cfde9f7b7 --- /dev/null +++ b/chromium/media/renderers/win/media_foundation_renderer.cc @@ -0,0 +1,641 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/renderers/win/media_foundation_renderer.h" + +#include <Audioclient.h> +#include <mferror.h> +#include <memory> +#include <string> + +#include "base/bind_helpers.h" +#include "base/callback_helpers.h" +#include "base/guid.h" +#include "base/strings/string16.h" +#include "base/strings/string_number_conversions.h" +#include "base/strings/utf_string_conversions.h" +#include "base/win/scoped_bstr.h" +#include "base/win/scoped_hdc.h" +#include "base/win/scoped_propvariant.h" +#include "base/win/windows_version.h" +#include "base/win/wrapped_window_proc.h" +#include "media/base/bind_to_current_loop.h" +#include "media/base/timestamp_constants.h" +#include "media/base/win/mf_helpers.h" + +namespace media { + +using Microsoft::WRL::ComPtr; +using Microsoft::WRL::MakeAndInitialize; + +namespace { + +ATOM g_video_window_class = 0; + +// The |g_video_window_class| atom obtained is used as the |lpClassName| +// parameter in CreateWindowEx(). +// https://docs.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-createwindowexa +// +// To enable OPM +// (https://docs.microsoft.com/en-us/windows/win32/medfound/output-protection-manager) +// protection for video playback, We call CreateWindowEx() to get a window +// and pass it to MFMediaEngine as an attribute. +bool InitializeVideoWindowClass() { + if (g_video_window_class) + return true; + + WNDCLASSEX intermediate_class; + base::win::InitializeWindowClass( + L"VirtualMediaFoundationCdmVideoWindow", + &base::win::WrappedWindowProc<::DefWindowProc>, CS_OWNDC, 0, 0, nullptr, + reinterpret_cast<HBRUSH>(GetStockObject(BLACK_BRUSH)), nullptr, nullptr, + nullptr, &intermediate_class); + g_video_window_class = RegisterClassEx(&intermediate_class); + if (!g_video_window_class) { + HRESULT register_class_error = HRESULT_FROM_WIN32(GetLastError()); + DLOG(ERROR) << "RegisterClass failed: " << PrintHr(register_class_error); + return false; + } + + return true; +} + +} // namespace + +// static +bool MediaFoundationRenderer::IsSupported() { + return base::win::GetVersion() >= base::win::Version::WIN10; +} + +MediaFoundationRenderer::MediaFoundationRenderer( + bool muted, + scoped_refptr<base::SequencedTaskRunner> task_runner, + bool force_dcomp_mode_for_testing) + : muted_(muted), + task_runner_(task_runner), + force_dcomp_mode_for_testing_(force_dcomp_mode_for_testing) { + DVLOG_FUNC(1); +} + +MediaFoundationRenderer::~MediaFoundationRenderer() { + DVLOG_FUNC(1); + + // Perform shutdown/cleanup in the order (shutdown/detach/destroy) we wanted + // without depending on the order of destructors being invoked. We also need + // to invoke MFShutdown() after shutdown/cleanup of MF related objects. + + StopSendingStatistics(); + + if (mf_media_engine_extension_) + mf_media_engine_extension_->Shutdown(); + if (mf_media_engine_notify_) + mf_media_engine_notify_->Shutdown(); + if (mf_media_engine_) + mf_media_engine_->Shutdown(); + + if (mf_source_) + mf_source_->DetachResource(); + + if (dxgi_device_manager_) { + dxgi_device_manager_.Reset(); + MFUnlockDXGIDeviceManager(); + } + if (virtual_video_window_) + DestroyWindow(virtual_video_window_); +} + +void MediaFoundationRenderer::Initialize(MediaResource* media_resource, + RendererClient* client, + PipelineStatusCallback init_cb) { + DVLOG_FUNC(1); + + renderer_client_ = client; + + HRESULT hr = CreateMediaEngine(media_resource); + if (FAILED(hr)) { + DLOG(ERROR) << "Failed to create media engine: " << PrintHr(hr); + std::move(init_cb).Run(PIPELINE_ERROR_INITIALIZATION_FAILED); + } else { + std::move(init_cb).Run(PIPELINE_OK); + } +} + +HRESULT MediaFoundationRenderer::CreateMediaEngine( + MediaResource* media_resource) { + DVLOG_FUNC(1); + + mf_session_life_time_ = InitializeMediaFoundation(); + if (!mf_session_life_time_) + return E_FAIL; + + // TODO(frankli): Only call the followings when there is a video stream. + RETURN_IF_FAILED(InitializeDXGIDeviceManager()); + RETURN_IF_FAILED(InitializeVirtualVideoWindow()); + + // The OnXxx() callbacks are invoked by MF threadpool thread, we would like + // to bind the callbacks to |task_runner_| MessgaeLoop. + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + auto weak_this = weak_factory_.GetWeakPtr(); + RETURN_IF_FAILED(MakeAndInitialize<MediaEngineNotifyImpl>( + &mf_media_engine_notify_, + BindToCurrentLoop(base::BindRepeating( + &MediaFoundationRenderer::OnPlaybackError, weak_this)), + BindToCurrentLoop(base::BindRepeating( + &MediaFoundationRenderer::OnPlaybackEnded, weak_this)), + BindToCurrentLoop(base::BindRepeating( + &MediaFoundationRenderer::OnBufferingStateChanged, weak_this)), + BindToCurrentLoop(base::BindRepeating( + &MediaFoundationRenderer::OnVideoNaturalSizeChanged, weak_this)), + BindToCurrentLoop(base::BindRepeating( + &MediaFoundationRenderer::OnTimeUpdate, weak_this)))); + + ComPtr<IMFAttributes> creation_attributes; + RETURN_IF_FAILED(MFCreateAttributes(&creation_attributes, 6)); + RETURN_IF_FAILED(creation_attributes->SetUnknown( + MF_MEDIA_ENGINE_CALLBACK, mf_media_engine_notify_.Get())); + RETURN_IF_FAILED( + creation_attributes->SetUINT32(MF_MEDIA_ENGINE_CONTENT_PROTECTION_FLAGS, + MF_MEDIA_ENGINE_ENABLE_PROTECTED_CONTENT)); + RETURN_IF_FAILED(creation_attributes->SetUINT32( + MF_MEDIA_ENGINE_AUDIO_CATEGORY, AudioCategory_Media)); + if (virtual_video_window_) { + RETURN_IF_FAILED(creation_attributes->SetUINT64( + MF_MEDIA_ENGINE_OPM_HWND, + reinterpret_cast<uint64_t>(virtual_video_window_))); + } + + if (dxgi_device_manager_) { + RETURN_IF_FAILED(creation_attributes->SetUnknown( + MF_MEDIA_ENGINE_DXGI_MANAGER, dxgi_device_manager_.Get())); + } + + RETURN_IF_FAILED( + MakeAndInitialize<MediaEngineExtension>(&mf_media_engine_extension_)); + RETURN_IF_FAILED(creation_attributes->SetUnknown( + MF_MEDIA_ENGINE_EXTENSION, mf_media_engine_extension_.Get())); + + ComPtr<IMFMediaEngineClassFactory> class_factory; + RETURN_IF_FAILED(CoCreateInstance(CLSID_MFMediaEngineClassFactory, nullptr, + CLSCTX_INPROC_SERVER, + IID_PPV_ARGS(&class_factory))); + // TODO(frankli): Use MF_MEDIA_ENGINE_REAL_TIME_MODE for low latency hint + // instead of 0. + RETURN_IF_FAILED(class_factory->CreateInstance(0, creation_attributes.Get(), + &mf_media_engine_)); + + auto media_resource_type_ = media_resource->GetType(); + if (media_resource_type_ != MediaResource::Type::STREAM) { + DLOG(ERROR) << "MediaResource is not of STREAM"; + return E_INVALIDARG; + } + + if (!playback_element_id_) { + DLOG(ERROR) << "Invalid playback_element_id_."; + return HRESULT_FROM_WIN32(ERROR_INVALID_STATE); + } + + RETURN_IF_FAILED(MakeAndInitialize<MediaFoundationSourceWrapper>( + &mf_source_, playback_element_id_, media_resource, task_runner_)); + + if (force_dcomp_mode_for_testing_) + SetDCompMode(true, base::DoNothing()); + + if (!mf_source_->HasEncryptedStream()) { + // Supports clear stream for testing. + return SetSourceOnMediaEngine(); + } + + // Has encrypted stream. + RETURN_IF_FAILED(MakeAndInitialize<MediaFoundationProtectionManager>( + &content_protection_manager_)); + ComPtr<IMFMediaEngineProtectedContent> protected_media_engine; + RETURN_IF_FAILED(mf_media_engine_.As(&protected_media_engine)); + RETURN_IF_FAILED(protected_media_engine->SetContentProtectionManager( + content_protection_manager_.Get())); + + waiting_for_mf_cdm_ = true; + if (!cdm_context_) + return S_OK; + + // Has |cdm_context_|. + if (!cdm_context_->GetMediaFoundationCdmProxy( + base::BindOnce(&MediaFoundationRenderer::OnCdmProxyReceived, + weak_factory_.GetWeakPtr()))) { + DLOG(ERROR) << __func__ + << ": CdmContext does not support MF CDM interface."; + return MF_E_UNEXPECTED; + } + + return S_OK; +} + +HRESULT MediaFoundationRenderer::SetSourceOnMediaEngine() { + DVLOG_FUNC(1); + + if (!mf_source_) { + LOG(ERROR) << "mf_source_ is null."; + return HRESULT_FROM_WIN32(ERROR_INVALID_STATE); + } + + ComPtr<IUnknown> source_unknown; + RETURN_IF_FAILED(mf_source_.As(&source_unknown)); + RETURN_IF_FAILED( + mf_media_engine_extension_->SetMediaSource(source_unknown.Get())); + + DVLOG(2) << "Set MFRendererSrc scheme as the source for MFMediaEngine."; + base::win::ScopedBstr mf_renderer_source_scheme( + base::ASCIIToUTF16("MFRendererSrc")); + // We need to set our source scheme first in order for the MFMediaEngine to + // load of our custom MFMediaSource. + RETURN_IF_FAILED( + mf_media_engine_->SetSource(mf_renderer_source_scheme.Get())); + + return S_OK; +} + +HRESULT MediaFoundationRenderer::InitializeDXGIDeviceManager() { + UINT device_reset_token; + RETURN_IF_FAILED( + MFLockDXGIDeviceManager(&device_reset_token, &dxgi_device_manager_)); + + ComPtr<ID3D11Device> d3d11_device; + UINT creation_flags = + (D3D11_CREATE_DEVICE_VIDEO_SUPPORT | D3D11_CREATE_DEVICE_BGRA_SUPPORT | + D3D11_CREATE_DEVICE_PREVENT_INTERNAL_THREADING_OPTIMIZATIONS); + static const D3D_FEATURE_LEVEL feature_levels[] = { + D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_10_1, + D3D_FEATURE_LEVEL_10_0, D3D_FEATURE_LEVEL_9_3, D3D_FEATURE_LEVEL_9_2, + D3D_FEATURE_LEVEL_9_1}; + RETURN_IF_FAILED( + D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, 0, creation_flags, + feature_levels, base::size(feature_levels), + D3D11_SDK_VERSION, &d3d11_device, nullptr, nullptr)); + + ComPtr<ID3D10Multithread> multithreaded_device; + RETURN_IF_FAILED(d3d11_device.As(&multithreaded_device)); + multithreaded_device->SetMultithreadProtected(TRUE); + + return dxgi_device_manager_->ResetDevice(d3d11_device.Get(), + device_reset_token); +} + +HRESULT MediaFoundationRenderer::InitializeVirtualVideoWindow() { + if (!InitializeVideoWindowClass()) + return E_FAIL; + + virtual_video_window_ = + CreateWindowEx(WS_EX_NOPARENTNOTIFY | WS_EX_LAYERED | WS_EX_TRANSPARENT | + WS_EX_NOREDIRECTIONBITMAP, + reinterpret_cast<wchar_t*>(g_video_window_class), L"", + WS_POPUP | WS_DISABLED | WS_CLIPSIBLINGS, 0, 0, 1, 1, + nullptr, nullptr, nullptr, nullptr); + if (!virtual_video_window_) { + HRESULT hr = HRESULT_FROM_WIN32(GetLastError()); + DLOG(ERROR) << "Failed to create virtual window: " << PrintHr(hr); + return hr; + } + + return S_OK; +} + +void MediaFoundationRenderer::SetCdm(CdmContext* cdm_context, + CdmAttachedCB cdm_attached_cb) { + DVLOG_FUNC(1); + + if (cdm_context_ || !cdm_context) { + DLOG(ERROR) << "Failed in checking CdmContext."; + std::move(cdm_attached_cb).Run(false); + return; + } + + cdm_context_ = cdm_context; + + if (waiting_for_mf_cdm_) { + if (!cdm_context_->GetMediaFoundationCdmProxy( + base::BindOnce(&MediaFoundationRenderer::OnCdmProxyReceived, + weak_factory_.GetWeakPtr()))) { + DLOG(ERROR) << "Decryptor does not support MF CDM interface."; + std::move(cdm_attached_cb).Run(false); + return; + } + } + + std::move(cdm_attached_cb).Run(true); +} + +void MediaFoundationRenderer::SetLatencyHint( + base::Optional<base::TimeDelta> /*latency_hint*/) { + // TODO(frankli): Ensure MFMediaEngine rendering pipeine is in real time mode. + NOTIMPLEMENTED() << "We do not use the latency hint today"; +} + +// TODO(frankli): Use ComPtr<> for |cdm|. +void MediaFoundationRenderer::OnCdmProxyReceived(IMFCdmProxy* cdm) { + DVLOG_FUNC(1); + + if (!waiting_for_mf_cdm_ || !content_protection_manager_) { + DLOG(ERROR) << "Failed in checking internal state."; + renderer_client_->OnError(PipelineStatus::PIPELINE_ERROR_INVALID_STATE); + return; + } + + waiting_for_mf_cdm_ = false; + + ComPtr<IMFCdmProxy> cdm_proxy; + cdm_proxy.Attach(cdm); + content_protection_manager_->SetCdmProxy(cdm_proxy.Get()); + mf_source_->SetCdmProxy(cdm_proxy.Get()); + HRESULT hr = SetSourceOnMediaEngine(); + if (FAILED(hr)) { + DLOG(ERROR) << "Failed to set source on media engine: " << PrintHr(hr); + renderer_client_->OnError(PipelineStatus::PIPELINE_ERROR_COULD_NOT_RENDER); + return; + } +} + +void MediaFoundationRenderer::Flush(base::OnceClosure flush_cb) { + DVLOG_FUNC(2); + + HRESULT hr = mf_media_engine_->Pause(); + // Ignore any Pause() error. We can continue to flush |mf_source_| instead of + // stopping the playback with error. + DVLOG_IF(1, FAILED(hr)) << "Failed to pause playback on flush: " + << PrintHr(hr); + + StopSendingStatistics(); + mf_source_->FlushStreams(); + std::move(flush_cb).Run(); +} + +void MediaFoundationRenderer::StartPlayingFrom(base::TimeDelta time) { + double current_time = time.InSecondsF(); + DVLOG_FUNC(2) << "current_time=" << current_time; + + // Note: It is okay for |waiting_for_mf_cdm_| to be true here. The + // MFMediaEngine supports calls to Play/SetCurrentTime before a source is set + // (it will apply the relevant changes to the playback state once a source is + // set on it). + + // SetCurrentTime() completes asynchronously. When the seek operation starts, + // the MFMediaEngine sends an MF_MEDIA_ENGINE_EVENT_SEEKING event. When the + // seek operation completes, the MFMediaEngine sends an + // MF_MEDIA_ENGINE_EVENT_SEEKED event. + HRESULT hr = mf_media_engine_->SetCurrentTime(current_time); + if (FAILED(hr)) { + DLOG(ERROR) << "Failed to SetCurrentTime: " << PrintHr(hr); + renderer_client_->OnError(PipelineStatus::PIPELINE_ERROR_COULD_NOT_RENDER); + return; + } + + hr = mf_media_engine_->Play(); + if (FAILED(hr)) { + DLOG(ERROR) << "Failed to start playback: " << PrintHr(hr); + renderer_client_->OnError(PipelineStatus::PIPELINE_ERROR_COULD_NOT_RENDER); + return; + } + + StartSendingStatistics(); +} + +void MediaFoundationRenderer::SetPlaybackRate(double playback_rate) { + DVLOG_FUNC(2) << "playback_rate=" << playback_rate; + + HRESULT hr = mf_media_engine_->SetPlaybackRate(playback_rate); + // Ignore error so that the media continues to play rather than stopped. + DVLOG_IF(1, FAILED(hr)) << "Failed to set playback rate: " << PrintHr(hr); +} + +void MediaFoundationRenderer::SetDCompMode(bool enabled, + SetDCompModeCB callback) { + DVLOG_FUNC(1); + + HRESULT hr = SetDCompModeInternal(enabled); + if (FAILED(hr)) { + DLOG(ERROR) << "Failed to set DComp mode: " << PrintHr(hr); + std::move(callback).Run(false); + return; + } + + std::move(callback).Run(true); +} + +void MediaFoundationRenderer::GetDCompSurface(GetDCompSurfaceCB callback) { + DVLOG_FUNC(1); + + HANDLE surface_handle = INVALID_HANDLE_VALUE; + HRESULT hr = GetDCompSurfaceInternal(&surface_handle); + DVLOG_IF(1, FAILED(hr)) << "Failed to get DComp surface: " << PrintHr(hr); + std::move(callback).Run(std::move(surface_handle)); +} + +// TODO(crbug.com/1070030): Investigate if we need to add +// OnSelectedVideoTracksChanged() to media renderer.mojom. +void MediaFoundationRenderer::SetVideoStreamEnabled(bool enabled) { + DVLOG_FUNC(1) << "enabled=" << enabled; + if (!mf_source_) + return; + + const bool needs_restart = mf_source_->SetVideoStreamEnabled(enabled); + if (needs_restart) { + // If the media source indicates that we need to restart playback (e.g due + // to a newly enabled stream being EOS), queue a pause and play operation. + mf_media_engine_->Pause(); + mf_media_engine_->Play(); + } +} + +void MediaFoundationRenderer::SetPlaybackElementId( + uint64_t playback_element_id) { + DVLOG_FUNC(1) << "playback_element_id=" << playback_element_id; + + playback_element_id_ = playback_element_id; +} + +void MediaFoundationRenderer::SetOutputParams(const gfx::Rect& output_rect) { + DVLOG_FUNC(2); + + HRESULT hr = SetOutputParamsInternal(output_rect); + DVLOG_IF(1, FAILED(hr)) << "Failed to set output parameters: " << PrintHr(hr); +} + +HRESULT MediaFoundationRenderer::SetOutputParamsInternal( + const gfx::Rect& output_rect) { + DVLOG_FUNC(2); + + if (virtual_video_window_ && + !::SetWindowPos(virtual_video_window_, HWND_BOTTOM, output_rect.x(), + output_rect.y(), output_rect.width(), + output_rect.height(), SWP_NOACTIVATE)) { + return HRESULT_FROM_WIN32(GetLastError()); + } + + // TODO(frankli): Update MFMediaEngineEx with |output_rect| change and update + // renderer client with output size. + + return S_OK; +} + +HRESULT MediaFoundationRenderer::GetDCompSurfaceInternal( + HANDLE* surface_handle) { + DVLOG_FUNC(1); + + ComPtr<IMFMediaEngineEx> media_engine_ex; + RETURN_IF_FAILED(mf_media_engine_.As(&media_engine_ex)); + RETURN_IF_FAILED(media_engine_ex->GetVideoSwapchainHandle(surface_handle)); + return S_OK; +} + +HRESULT MediaFoundationRenderer::SetDCompModeInternal(bool enabled) { + DVLOG_FUNC(1) << "enabled=" << enabled; + + ComPtr<IMFMediaEngineEx> media_engine_ex; + RETURN_IF_FAILED(mf_media_engine_.As(&media_engine_ex)); + RETURN_IF_FAILED(media_engine_ex->EnableWindowlessSwapchainMode(enabled)); + return S_OK; +} + +HRESULT MediaFoundationRenderer::PopulateStatistics( + PipelineStatistics& statistics) { + ComPtr<IMFMediaEngineEx> media_engine_ex; + RETURN_IF_FAILED(mf_media_engine_.As(&media_engine_ex)); + base::win::ScopedPropVariant frames_rendered; + RETURN_IF_FAILED(media_engine_ex->GetStatistics( + MF_MEDIA_ENGINE_STATISTIC_FRAMES_RENDERED, frames_rendered.Receive())); + base::win::ScopedPropVariant frames_dropped; + RETURN_IF_FAILED(media_engine_ex->GetStatistics( + MF_MEDIA_ENGINE_STATISTIC_FRAMES_DROPPED, frames_dropped.Receive())); + statistics.video_frames_decoded = frames_rendered.get().ulVal; + statistics.video_frames_dropped = frames_dropped.get().ulVal; + return S_OK; +} + +void MediaFoundationRenderer::SendStatistics() { + PipelineStatistics new_stats = {}; + HRESULT hr = PopulateStatistics(new_stats); + if (FAILED(hr)) { + DVLOG(3) << "Failed to populate pipeline stats: " << PrintHr(hr); + return; + } + + if (statistics_ != new_stats) { + statistics_ = new_stats; + renderer_client_->OnStatisticsUpdate(statistics_); + } +} + +void MediaFoundationRenderer::StartSendingStatistics() { + const auto kPipelineStatsPollingPeriod = + base::TimeDelta::FromMilliseconds(500); + statistics_timer_.Start(FROM_HERE, kPipelineStatsPollingPeriod, this, + &MediaFoundationRenderer::SendStatistics); +} + +void MediaFoundationRenderer::StopSendingStatistics() { + statistics_timer_.Stop(); +} + +void MediaFoundationRenderer::SetVolume(float volume) { + volume_ = volume; + float set_volume = muted_ ? 0 : volume_; + DVLOG_FUNC(2) << "set_volume=" << set_volume; + + HRESULT hr = mf_media_engine_->SetVolume(set_volume); + DVLOG_IF(1, FAILED(hr)) << "Failed to set volume: " << PrintHr(hr); +} + +base::TimeDelta MediaFoundationRenderer::GetMediaTime() { +// GetCurrentTime is expaned as GetTickCount in base/win/windows_types.h +#undef GetCurrentTime + double current_time = mf_media_engine_->GetCurrentTime(); +// Restore macro definition. +#define GetCurrentTime() GetTickCount() + return base::TimeDelta::FromSecondsD(current_time); +} + +void MediaFoundationRenderer::OnPlaybackError(PipelineStatus status) { + DVLOG_FUNC(1) << "status=" << status; + + renderer_client_->OnError(status); + StopSendingStatistics(); +} + +void MediaFoundationRenderer::OnPlaybackEnded() { + DVLOG_FUNC(2); + + renderer_client_->OnEnded(); + StopSendingStatistics(); +} + +void MediaFoundationRenderer::OnBufferingStateChanged( + BufferingState state, + BufferingStateChangeReason reason) { + DVLOG_FUNC(2); + + if (state == BufferingState::BUFFERING_HAVE_ENOUGH) { + max_buffering_state_ = state; + } + + if (state == BufferingState::BUFFERING_HAVE_NOTHING && + max_buffering_state_ != BufferingState::BUFFERING_HAVE_ENOUGH) { + // Prevent sending BUFFERING_HAVE_NOTHING if we haven't previously sent a + // BUFFERING_HAVE_ENOUGH state. + return; + } + + renderer_client_->OnBufferingStateChange(state, reason); +} + +void MediaFoundationRenderer::OnVideoNaturalSizeChanged() { + DVLOG_FUNC(2); + + const bool has_video = mf_media_engine_->HasVideo(); + DVLOG_FUNC(2) << "has_video=" << has_video; + + // Skip if there are no video streams. This can happen because this is + // originated from MF_MEDIA_ENGINE_EVENT_FORMATCHANGE. + if (!has_video) + return; + + DWORD native_width; + DWORD native_height; + HRESULT hr = + mf_media_engine_->GetNativeVideoSize(&native_width, &native_height); + if (FAILED(hr)) { + // TODO(xhwang): Add UMA to probe if this can happen. + DLOG(ERROR) << "Failed to get native video size from MediaEngine, using " + "default (640x320). hr=" + << hr; + native_video_size_ = {640, 320}; + } else { + native_video_size_ = {native_width, native_height}; + } + + // TODO(frankli): Use actual dest rect provided by client instead of video + // size. Will fix the following in another CL. + ComPtr<IMFMediaEngineEx> mf_media_engine_ex; + hr = mf_media_engine_.As(&mf_media_engine_ex); + if (FAILED(hr)) { + DLOG(ERROR) << PrintHr(hr); + return; + } + + RECT video_dest_rect = {0}; + video_dest_rect.right = native_video_size_.width(); + video_dest_rect.bottom = native_video_size_.height(); + hr = + mf_media_engine_ex->UpdateVideoStream(nullptr, &video_dest_rect, nullptr); + if (FAILED(hr)) { + DLOG(ERROR) << PrintHr(hr); + return; + } + + renderer_client_->OnVideoNaturalSizeChange(native_video_size_); + return; +} + +void MediaFoundationRenderer::OnTimeUpdate() { + DVLOG_FUNC(3) << "media_time=" << GetMediaTime(); +} + +} // namespace media diff --git a/chromium/media/renderers/win/media_foundation_renderer.h b/chromium/media/renderers/win/media_foundation_renderer.h new file mode 100644 index 00000000000..99c5193550f --- /dev/null +++ b/chromium/media/renderers/win/media_foundation_renderer.h @@ -0,0 +1,163 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_RENDERERS_WIN_MEDIA_FOUNDATION_RENDERER_H_ +#define MEDIA_RENDERERS_WIN_MEDIA_FOUNDATION_RENDERER_H_ + +#include <d3d11.h> +#include <mfapi.h> +#include <mfmediaengine.h> +#include <wrl.h> + +#include "base/callback.h" +#include "base/macros.h" +#include "base/memory/weak_ptr.h" +#include "base/sequenced_task_runner.h" +#include "base/timer/timer.h" +#include "base/unguessable_token.h" +#include "base/win/windows_types.h" +#include "media/base/buffering_state.h" +#include "media/base/media_resource.h" +#include "media/base/pipeline_status.h" +#include "media/base/renderer.h" +#include "media/base/renderer_client.h" +#include "media/base/win/mf_initializer.h" +#include "media/renderers/win/media_engine_extension.h" +#include "media/renderers/win/media_engine_notify_impl.h" +#include "media/renderers/win/media_foundation_protection_manager.h" +#include "media/renderers/win/media_foundation_renderer_extension.h" +#include "media/renderers/win/media_foundation_source_wrapper.h" + +namespace media { + +// MediaFoundationRenderer bridges the Renderer and Windows MFMediaEngine +// interfaces. +class MediaFoundationRenderer : public Renderer, + public MediaFoundationRendererExtension { + public: + // Whether MediaFoundationRenderer() is supported on the current device. + static bool IsSupported(); + + MediaFoundationRenderer(bool muted, + scoped_refptr<base::SequencedTaskRunner> task_runner, + bool force_dcomp_mode_for_testing = false); + + ~MediaFoundationRenderer() override; + + // TODO(frankli): naming: Change DComp into DirectComposition for interface + // method names in a separate CL. + + // Renderer implementation. + void Initialize(MediaResource* media_resource, + RendererClient* client, + PipelineStatusCallback init_cb) override; + void SetCdm(CdmContext* cdm_context, CdmAttachedCB cdm_attached_cb) override; + void SetLatencyHint(base::Optional<base::TimeDelta> latency_hint) override; + void Flush(base::OnceClosure flush_cb) override; + void StartPlayingFrom(base::TimeDelta time) override; + void SetPlaybackRate(double playback_rate) override; + void SetVolume(float volume) override; + base::TimeDelta GetMediaTime() override; + + // MediaFoundationRendererExtension implementation. + void SetDCompMode(bool enabled, SetDCompModeCB callback) override; + void GetDCompSurface(GetDCompSurfaceCB callback) override; + void SetVideoStreamEnabled(bool enabled) override; + // SetPlaybackElementId() must be called before Initialize() as + // |playback_element_id| is used in Initialize(). + void SetPlaybackElementId(uint64_t playback_element_id) override; + void SetOutputParams(const gfx::Rect& output_rect) override; + + private: + HRESULT CreateMediaEngine(MediaResource* media_resource); + HRESULT InitializeDXGIDeviceManager(); + HRESULT InitializeVirtualVideoWindow(); + + // Update RendererClient with rendering statistics periodically. + HRESULT PopulateStatistics(PipelineStatistics& statistics); + void SendStatistics(); + void StartSendingStatistics(); + void StopSendingStatistics(); + + // Callbacks for |mf_media_engine_notify_|. + void OnPlaybackError(PipelineStatus status); + void OnPlaybackEnded(); + void OnBufferingStateChanged(BufferingState state, + BufferingStateChangeReason reason); + void OnVideoNaturalSizeChanged(); + void OnTimeUpdate(); + + void OnCdmProxyReceived(IMFCdmProxy* cdm); + + HRESULT SetDCompModeInternal(bool enabled); + HRESULT GetDCompSurfaceInternal(HANDLE* surface_handle); + HRESULT SetSourceOnMediaEngine(); + HRESULT SetOutputParamsInternal(const gfx::Rect& output_rect); + + // TODO(crbug.com/1017943): Support Audio Indicator when using + // media::MojoRenderer. For now, keep |muted_| as const. + const bool muted_; + + // Renderer methods are running in the same sequence. + scoped_refptr<base::SequencedTaskRunner> task_runner_; + + // Once set, will force |mf_media_engine_| to use DirectComposition mode. + // This is used for testing. + const bool force_dcomp_mode_for_testing_; + + // Keep this here so it's destroyed after all Media Foundation members below. + MFSessionLifetime mf_session_life_time_; + + RendererClient* renderer_client_; + + Microsoft::WRL::ComPtr<IMFMediaEngine> mf_media_engine_; + Microsoft::WRL::ComPtr<MediaEngineNotifyImpl> mf_media_engine_notify_; + Microsoft::WRL::ComPtr<MediaEngineExtension> mf_media_engine_extension_; + Microsoft::WRL::ComPtr<MediaFoundationSourceWrapper> mf_source_; + // This enables MFMediaEngine to use hardware acceleration for video decoding + // and vdieo processing. + Microsoft::WRL::ComPtr<IMFDXGIDeviceManager> dxgi_device_manager_; + + // Current duration of the media. + base::TimeDelta duration_; + + // This is the same as "natural_size" in Chromium. + gfx::Size native_video_size_; + + // Keep the last volume value being set. + float volume_ = 1.0; + + // Used for RendererClient::OnBufferingStateChange(). + BufferingState max_buffering_state_ = BufferingState::BUFFERING_HAVE_NOTHING; + + // Used for RendererClient::OnStatisticsUpdate(). + PipelineStatistics statistics_ = {}; + base::RepeatingTimer statistics_timer_; + + // An identifier corresponds to a WebMediaPlayer. It allows MFMediaEngine + // to track the same playback session is running as Renderer can be destroyed + // after a period of inactivity by Chromium media pipeliine. + // Init it to an invalid ID. + uint64_t playback_element_id_ = 0; + + // A fake window handle passed to MF-based rendering pipeline for OPM. + HWND virtual_video_window_ = nullptr; + + base::UnguessableToken surface_request_token_; + base::win::ScopedHandle dcomp_surface_handle_; + + bool waiting_for_mf_cdm_ = false; + CdmContext* cdm_context_ = nullptr; + Microsoft::WRL::ComPtr<MediaFoundationProtectionManager> + content_protection_manager_; + + // NOTE: Weak pointers must be invalidated before all other member variables. + base::WeakPtrFactory<MediaFoundationRenderer> weak_factory_{this}; + + DISALLOW_COPY_AND_ASSIGN(MediaFoundationRenderer); +}; + +} // namespace media + +#endif // MEDIA_RENDERERS_WIN_MEDIA_FOUNDATION_RENDERER_H_ diff --git a/chromium/media/renderers/win/media_foundation_renderer_extension.h b/chromium/media/renderers/win/media_foundation_renderer_extension.h new file mode 100644 index 00000000000..4d9b04d57f7 --- /dev/null +++ b/chromium/media/renderers/win/media_foundation_renderer_extension.h @@ -0,0 +1,45 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_RENDERERS_WIN_MEDIA_FOUNDATION_RENDERER_EXTENSION_H_ +#define MEDIA_RENDERERS_WIN_MEDIA_FOUNDATION_RENDERER_EXTENSION_H_ + +#include "base/callback.h" +#include "base/macros.h" +#include "media/base/media_export.h" +#include "ui/gfx/geometry/rect.h" + +namespace media { + +// C++ interface equivalent to mojom::MediaFoundationRendererExtension. +// This interface allows MediaFoundationRenderer to support video rendering +// using Direct Compositon. +class MEDIA_EXPORT MediaFoundationRendererExtension { + public: + virtual ~MediaFoundationRendererExtension() = default; + + // TODO(frankli): naming: Change DComp into DirectComposition for interface + // method names in a separate CL. + + // Enable Direct Composition video rendering. + using SetDCompModeCB = base::OnceCallback<void(bool)>; + virtual void SetDCompMode(bool enabled, SetDCompModeCB callback) = 0; + + // Get a Direct Composition Surface handle. + using GetDCompSurfaceCB = base::OnceCallback<void(HANDLE)>; + virtual void GetDCompSurface(GetDCompSurfaceCB callback) = 0; + + // Notify renderer whether video is enabled. + virtual void SetVideoStreamEnabled(bool enabled) = 0; + + // Provide a unique identifier which maps to a specific playback element. + virtual void SetPlaybackElementId(uint64_t playback_element_id) = 0; + + // Notify renderer of output composition parameters. + virtual void SetOutputParams(const ::gfx::Rect& rect) = 0; +}; + +} // namespace media + +#endif // MEDIA_RENDERERS_WIN_MEDIA_FOUNDATION_RENDERER_EXTENSION_H_ diff --git a/chromium/media/renderers/win/media_foundation_renderer_integration_test.cc b/chromium/media/renderers/win/media_foundation_renderer_integration_test.cc new file mode 100644 index 00000000000..a76833ef47b --- /dev/null +++ b/chromium/media/renderers/win/media_foundation_renderer_integration_test.cc @@ -0,0 +1,96 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/renderers/win/media_foundation_renderer.h" + +#include <memory> + +#include "media/test/pipeline_integration_test_base.h" +#include "media/test/test_media_source.h" + +namespace media { + +namespace { + +// TODO(xhwang): Generalize this to support more codecs, or use CanPlay() or +// IsTypeSupported() which can take mime types directly. +bool CanDecodeVp9() { + if (!MediaFoundationRenderer::IsSupported()) { + LOG(WARNING) << "MediaFoundationRenderer not supported"; + return false; + } + + MFT_REGISTER_TYPE_INFO input_type = {MFMediaType_Video, MFVideoFormat_VP90}; + IMFActivate** activates = nullptr; + UINT32 count = 0; + + if (FAILED(MFTEnumEx(MFT_CATEGORY_VIDEO_DECODER, + MFT_ENUM_FLAG_SYNCMFT | MFT_ENUM_FLAG_ASYNCMFT | + MFT_ENUM_FLAG_HARDWARE, + &input_type, /*output_type=*/nullptr, &activates, + &count))) { + return false; + } + + for (UINT32 i = 0; i < count; ++i) + activates[i]->Release(); + CoTaskMemFree(activates); + + if (count == 0) { + LOG(WARNING) << "No decoder for VP9"; + return false; + } + + return true; +} + +} // namespace + +class MediaFoundationRendererIntegrationTest + : public testing::Test, + public PipelineIntegrationTestBase { + public: + MediaFoundationRendererIntegrationTest() { + SetCreateRendererCB(base::BindRepeating( + &MediaFoundationRendererIntegrationTest::CreateMediaFoundationRenderer, + base::Unretained(this))); + } + + private: + std::unique_ptr<Renderer> CreateMediaFoundationRenderer( + base::Optional<RendererFactoryType> factory_type) { + auto renderer = std::make_unique<MediaFoundationRenderer>( + /*muted=*/false, task_environment_.GetMainThreadTaskRunner(), + /*force_dcomp_mode_for_testing=*/true); + renderer->SetPlaybackElementId(1); // Must be set before Initialize(). + return renderer; + } + + DISALLOW_COPY_AND_ASSIGN(MediaFoundationRendererIntegrationTest); +}; + +TEST_F(MediaFoundationRendererIntegrationTest, BasicPlayback) { + if (!CanDecodeVp9()) + return; + + ASSERT_EQ(PIPELINE_OK, Start("bear-vp9.webm")); + Play(); + ASSERT_TRUE(WaitUntilOnEnded()); +} + +TEST_F(MediaFoundationRendererIntegrationTest, BasicPlayback_MediaSource) { + if (!CanDecodeVp9()) + return; + + TestMediaSource source("bear-vp9.webm", 67504); + EXPECT_EQ(PIPELINE_OK, StartPipelineWithMediaSource(&source)); + source.EndOfStream(); + + Play(); + ASSERT_TRUE(WaitUntilOnEnded()); + source.Shutdown(); + Stop(); +} + +} // namespace media diff --git a/chromium/media/renderers/win/media_foundation_renderer_unittest.cc b/chromium/media/renderers/win/media_foundation_renderer_unittest.cc new file mode 100644 index 00000000000..a5d47595e73 --- /dev/null +++ b/chromium/media/renderers/win/media_foundation_renderer_unittest.cc @@ -0,0 +1,268 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/renderers/win/media_foundation_renderer.h" + +#include <windows.media.protection.h> + +#include "base/bind.h" +#include "base/bind_helpers.h" +#include "base/single_thread_task_runner.h" +#include "base/test/mock_callback.h" +#include "base/test/task_environment.h" +#include "base/win/scoped_com_initializer.h" +#include "media/base/demuxer_stream.h" +#include "media/base/mock_filters.h" +#include "media/base/test_helpers.h" +#include "testing/gmock/include/gmock/gmock.h" + +using ::testing::_; +using ::testing::Invoke; +using ::testing::NiceMock; +using ::testing::Return; +using ::testing::StrictMock; + +namespace media { + +using ABI::Windows::Media::Protection::IMediaProtectionPMPServer; +using Microsoft::WRL::ComPtr; + +#define MOCK_STDCALL_METHOD1(Name, Types) \ + MOCK_METHOD1_WITH_CALLTYPE(STDMETHODCALLTYPE, Name, Types) + +#define MOCK_STDCALL_METHOD2(Name, Types) \ + MOCK_METHOD2_WITH_CALLTYPE(STDMETHODCALLTYPE, Name, Types) + +#define MOCK_STDCALL_METHOD3(Name, Types) \ + MOCK_METHOD3_WITH_CALLTYPE(STDMETHODCALLTYPE, Name, Types) + +#define MOCK_STDCALL_METHOD7(Name, Types) \ + MOCK_METHOD7_WITH_CALLTYPE(STDMETHODCALLTYPE, Name, Types) + +class MockMFCdmProxy + : public Microsoft::WRL::RuntimeClass< + Microsoft::WRL::RuntimeClassFlags<Microsoft::WRL::ClassicCom>, + IMFCdmProxy> { + public: + MockMFCdmProxy(); + ~MockMFCdmProxy() override; + + // IMFCdmProxy. + MOCK_STDCALL_METHOD2(GetPMPServer, + HRESULT(REFIID riid, void** object_result)); + MOCK_STDCALL_METHOD7(GetInputTrustAuthority, + HRESULT(uint64_t playback_element_id, + uint32_t stream_id, + uint32_t stream_count, + const uint8_t* content_init_data, + uint32_t content_init_data_size, + REFIID riid, + IUnknown** object_result)); + + MOCK_STDCALL_METHOD1(RefreshTrustedInput, + HRESULT(uint64_t playback_element_id)); + MOCK_STDCALL_METHOD3(SetLastKeyIds, + HRESULT(uint64_t playback_element_id, + GUID* key_ids, + uint32_t key_ids_count)); + MOCK_STDCALL_METHOD2(ProcessContentEnabler, + HRESULT(IUnknown* request, IMFAsyncResult* result)); +}; + +MockMFCdmProxy::MockMFCdmProxy() = default; +MockMFCdmProxy::~MockMFCdmProxy() = default; + +class MockMediaProtectionPMPServer + : public Microsoft::WRL::RuntimeClass< + Microsoft::WRL::RuntimeClassFlags< + Microsoft::WRL::WinRt | Microsoft::WRL::InhibitRoOriginateError>, + IMediaProtectionPMPServer> { + public: + MockMediaProtectionPMPServer() = default; + virtual ~MockMediaProtectionPMPServer() = default; + + static HRESULT MakeMockMediaProtectionPMPServer( + IMediaProtectionPMPServer** pmp_server) { + *pmp_server = Microsoft::WRL::Make<MockMediaProtectionPMPServer>().Detach(); + return S_OK; + } + + // Return E_NOINTERFACE to avoid a crash when MFMediaEngine tries to use the + // mocked IPropertySet from get_Properties(). + HRESULT STDMETHODCALLTYPE QueryInterface(REFIID riid, + void** object_result) override { + return E_NOINTERFACE; + } + + // ABI::Windows::Media::Protection::IMediaProtectionPMPServer. + MOCK_STDCALL_METHOD1( + get_Properties, + HRESULT( + ABI::Windows::Foundation::Collections::IPropertySet** properties)); +}; + +class MediaFoundationRendererTest : public testing::Test { + public: + MediaFoundationRendererTest() { + if (!MediaFoundationRenderer::IsSupported()) + return; + + MockMediaProtectionPMPServer::MakeMockMediaProtectionPMPServer( + &pmp_server_); + + mf_renderer_ = std::make_unique<MediaFoundationRenderer>( + /*muted=*/false, task_environment_.GetMainThreadTaskRunner()); + // It is required to invoke SetPlaybackElementId() before Initialize(). + mf_renderer_->SetPlaybackElementId(9876543210); + + // Some default actions. + ON_CALL(cdm_context_, GetMediaFoundationCdmProxy(_)) + .WillByDefault( + Invoke(this, &MediaFoundationRendererTest::MockGetMFCdm)); + ON_CALL(mf_cdm_proxy_, GetPMPServer(_, _)) + .WillByDefault( + Invoke(this, &MediaFoundationRendererTest::MockGetPMPServer)); + + // Some expected calls with return values. + EXPECT_CALL(media_resource_, GetAllStreams()) + .WillRepeatedly( + Invoke(this, &MediaFoundationRendererTest::GetAllStreams)); + EXPECT_CALL(media_resource_, GetType()) + .WillRepeatedly(Return(MediaResource::STREAM)); + } + + ~MediaFoundationRendererTest() override { mf_renderer_.reset(); } + + void AddStream(DemuxerStream::Type type, bool encrypted) { + streams_.push_back(CreateMockDemuxerStream(type, encrypted)); + } + + std::vector<DemuxerStream*> GetAllStreams() { + std::vector<DemuxerStream*> streams; + + for (auto& stream : streams_) { + streams.push_back(stream.get()); + } + + return streams; + } + + void OnSendCdmProxy( + CdmContext::GetMediaFoundationCdmProxyCB get_mf_cdm_proxy_cb) { + std::move(get_mf_cdm_proxy_cb).Run(&mf_cdm_proxy_); + } + + bool MockGetMFCdm( + CdmContext::GetMediaFoundationCdmProxyCB get_mf_cdm_proxy_cb) { + // The callback should be invoked asynchronously per API contract. Post + // to make callback from OnSendCdmProxy(). + task_environment_.GetMainThreadTaskRunner()->PostTask( + FROM_HERE, + base::BindOnce(&MediaFoundationRendererTest::OnSendCdmProxy, + base::Unretained(this), std::move(get_mf_cdm_proxy_cb))); + return true; + } + + HRESULT MockGetPMPServer(REFIID riid, LPVOID* object_result) { + ComPtr<IMediaProtectionPMPServer> pmp_server; + if (riid != __uuidof(**(&pmp_server)) || !object_result) { + return E_INVALIDARG; + } + + return pmp_server_.CopyTo( + reinterpret_cast<IMediaProtectionPMPServer**>(object_result)); + } + + protected: + base::win::ScopedCOMInitializer com_initializer_; + base::test::TaskEnvironment task_environment_; + base::MockOnceCallback<void(bool)> set_cdm_cb_; + base::MockOnceCallback<void(PipelineStatus)> renderer_init_cb_; + NiceMock<MockCdmContext> cdm_context_; + NiceMock<MockMediaResource> media_resource_; + NiceMock<MockRendererClient> renderer_client_; + NiceMock<MockMFCdmProxy> mf_cdm_proxy_; + ComPtr<IMediaProtectionPMPServer> pmp_server_; + std::unique_ptr<MediaFoundationRenderer> mf_renderer_; + std::vector<std::unique_ptr<StrictMock<MockDemuxerStream>>> streams_; +}; + +TEST_F(MediaFoundationRendererTest, VerifyInitWithoutSetCdm) { + if (!MediaFoundationRenderer::IsSupported()) + return; + + AddStream(DemuxerStream::AUDIO, /*encrypted=*/false); + AddStream(DemuxerStream::VIDEO, /*encrypted=*/true); + + EXPECT_CALL(renderer_init_cb_, Run(PIPELINE_OK)); + + mf_renderer_->Initialize(&media_resource_, &renderer_client_, + renderer_init_cb_.Get()); + + task_environment_.RunUntilIdle(); +} + +TEST_F(MediaFoundationRendererTest, SetCdmThenInit) { + if (!MediaFoundationRenderer::IsSupported()) + return; + + AddStream(DemuxerStream::AUDIO, /*encrypted=*/true); + AddStream(DemuxerStream::VIDEO, /*encrypted=*/true); + + EXPECT_CALL(set_cdm_cb_, Run(true)); + EXPECT_CALL(renderer_init_cb_, Run(PIPELINE_OK)); + + mf_renderer_->SetCdm(&cdm_context_, set_cdm_cb_.Get()); + mf_renderer_->Initialize(&media_resource_, &renderer_client_, + renderer_init_cb_.Get()); + + task_environment_.RunUntilIdle(); +} + +TEST_F(MediaFoundationRendererTest, InitThenSetCdm) { + if (!MediaFoundationRenderer::IsSupported()) + return; + + AddStream(DemuxerStream::AUDIO, /*encrypted=*/true); + AddStream(DemuxerStream::VIDEO, /*encrypted=*/true); + + EXPECT_CALL(set_cdm_cb_, Run(true)); + EXPECT_CALL(renderer_init_cb_, Run(PIPELINE_OK)); + + mf_renderer_->Initialize(&media_resource_, &renderer_client_, + renderer_init_cb_.Get()); + mf_renderer_->SetCdm(&cdm_context_, set_cdm_cb_.Get()); + + task_environment_.RunUntilIdle(); +} + +TEST_F(MediaFoundationRendererTest, DirectCompositionHandle) { + if (!MediaFoundationRenderer::IsSupported()) + return; + + base::MockCallback<MediaFoundationRendererExtension::SetDCompModeCB> + set_dcomp_mode_cb; + base::MockCallback<MediaFoundationRendererExtension::GetDCompSurfaceCB> + get_dcomp_cb; + + AddStream(DemuxerStream::AUDIO, /*encrypted=*/true); + AddStream(DemuxerStream::VIDEO, /*encrypted=*/true); + + EXPECT_CALL(set_cdm_cb_, Run(true)); + EXPECT_CALL(renderer_init_cb_, Run(PIPELINE_OK)); + EXPECT_CALL(set_dcomp_mode_cb, Run(true)); + // Ignore the DirectComposition handle value returned as our |pmp_server_| + // has no real implementation. + EXPECT_CALL(get_dcomp_cb, Run(_)); + + mf_renderer_->Initialize(&media_resource_, &renderer_client_, + renderer_init_cb_.Get()); + mf_renderer_->SetCdm(&cdm_context_, set_cdm_cb_.Get()); + mf_renderer_->SetDCompMode(true, set_dcomp_mode_cb.Get()); + mf_renderer_->GetDCompSurface(get_dcomp_cb.Get()); + + task_environment_.RunUntilIdle(); +} + +} // namespace media
\ No newline at end of file diff --git a/chromium/media/renderers/win/media_foundation_source_wrapper.cc b/chromium/media/renderers/win/media_foundation_source_wrapper.cc index a0b1a1e3324..fc16deff47d 100644 --- a/chromium/media/renderers/win/media_foundation_source_wrapper.cc +++ b/chromium/media/renderers/win/media_foundation_source_wrapper.cc @@ -16,6 +16,7 @@ namespace media { using Microsoft::WRL::ComPtr; MediaFoundationSourceWrapper::MediaFoundationSourceWrapper() = default; + MediaFoundationSourceWrapper::~MediaFoundationSourceWrapper() { if (!cdm_proxy_) return; @@ -29,14 +30,14 @@ MediaFoundationSourceWrapper::~MediaFoundationSourceWrapper() { HRESULT hr = cdm_proxy_->SetLastKeyIds(playback_element_id_, key_ids.data(), key_ids.size()); DLOG_IF(ERROR, FAILED(hr)) - << "Failed to notify CDM proxy of last Key IDs. hr=" << hr; + << "Failed to notify CDM proxy of last Key IDs: " << PrintHr(hr); } HRESULT MediaFoundationSourceWrapper::RuntimeClassInitialize( uint64_t playback_element_id, MediaResource* media_resource, scoped_refptr<base::SequencedTaskRunner> task_runner) { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); if (media_resource->GetType() != MediaResource::Type::STREAM) { DLOG(ERROR) << "MediaResource is not of Type STREAM"; @@ -61,7 +62,7 @@ HRESULT MediaFoundationSourceWrapper::RuntimeClassInitialize( } void MediaFoundationSourceWrapper::DetachResource() { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); for (auto stream : media_streams_) { stream->DetachDemuxerStream(); @@ -70,7 +71,7 @@ void MediaFoundationSourceWrapper::DetachResource() { HRESULT MediaFoundationSourceWrapper::GetCharacteristics( DWORD* characteristics) { - DVLOG(3) << __func__ << ": this=" << this; + DVLOG_FUNC(3); if (state_ == State::kShutdown) { DLOG(ERROR) << __func__ << ": MF_E_SHUTDOWN"; @@ -110,7 +111,7 @@ HRESULT MediaFoundationSourceWrapper::SelectDefaultStreams( HRESULT MediaFoundationSourceWrapper::CreatePresentationDescriptor( IMFPresentationDescriptor** presentation_descriptor_out) { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); if (state_ == State::kShutdown) { DLOG(ERROR) << __func__ << ": MF_E_SHUTDOWN"; @@ -141,7 +142,7 @@ HRESULT MediaFoundationSourceWrapper::Start( IMFPresentationDescriptor* presentation_descriptor, const GUID* guid_time_format, const PROPVARIANT* start_position) { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); if (state_ == State::kShutdown) { DLOG(ERROR) << __func__ << ": MF_E_SHUTDOWN"; @@ -232,7 +233,7 @@ HRESULT MediaFoundationSourceWrapper::Start( } HRESULT MediaFoundationSourceWrapper::Stop() { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); if (state_ == State::kShutdown) { DLOG(ERROR) << __func__ << ": MF_E_SHUTDOWN"; @@ -252,7 +253,7 @@ HRESULT MediaFoundationSourceWrapper::Stop() { } HRESULT MediaFoundationSourceWrapper::Pause() { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); if (state_ == State::kShutdown) { DLOG(ERROR) << __func__ << ": MF_E_SHUTDOWN"; @@ -278,7 +279,7 @@ HRESULT MediaFoundationSourceWrapper::Pause() { // After this method is called, methods on the media source and all of its // media streams return MF_E_SHUTDOWN (except for IUnknown methods). HRESULT MediaFoundationSourceWrapper::Shutdown() { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); for (auto stream : media_streams_) { stream->DetachParent(); @@ -295,7 +296,7 @@ HRESULT MediaFoundationSourceWrapper::Shutdown() { // HRESULT MediaFoundationSourceWrapper::GetEvent(DWORD flags, IMFMediaEvent** event_out) { - DVLOG(3) << __func__ << ": this=" << this; + DVLOG_FUNC(3); DCHECK(mf_media_event_queue_); // Not tracing hr to avoid the noise from MF_E_NO_EVENTS_AVAILABLE. @@ -304,7 +305,7 @@ HRESULT MediaFoundationSourceWrapper::GetEvent(DWORD flags, HRESULT MediaFoundationSourceWrapper::BeginGetEvent(IMFAsyncCallback* callback, IUnknown* state) { - DVLOG(3) << __func__ << ": this=" << this; + DVLOG_FUNC(3); DCHECK(mf_media_event_queue_); RETURN_IF_FAILED(mf_media_event_queue_->BeginGetEvent(callback, state)); @@ -313,7 +314,7 @@ HRESULT MediaFoundationSourceWrapper::BeginGetEvent(IMFAsyncCallback* callback, HRESULT MediaFoundationSourceWrapper::EndGetEvent(IMFAsyncResult* result, IMFMediaEvent** event_out) { - DVLOG(3) << __func__ << ": this=" << this; + DVLOG_FUNC(3); DCHECK(mf_media_event_queue_); RETURN_IF_FAILED(mf_media_event_queue_->EndGetEvent(result, event_out)); @@ -324,7 +325,7 @@ HRESULT MediaFoundationSourceWrapper::QueueEvent(MediaEventType type, REFGUID extended_type, HRESULT status, const PROPVARIANT* value) { - DVLOG(3) << __func__ << ": this=" << this; + DVLOG_FUNC(3); DCHECK(mf_media_event_queue_); RETURN_IF_FAILED(mf_media_event_queue_->QueueEventParamVar( @@ -336,19 +337,19 @@ HRESULT MediaFoundationSourceWrapper::GetInputTrustAuthority( DWORD stream_id, REFIID riid, IUnknown** object_out) { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); if (stream_id >= StreamCount()) return E_INVALIDARG; if (!cdm_proxy_) { - DLOG(ERROR) << __func__ << ": MF_E_NOT_PROTECTED"; + DVLOG_FUNC(1) << "MF_E_NOT_PROTECTED"; return MF_E_NOT_PROTECTED; } if (!media_streams_[stream_id]->IsEncrypted()) { - DVLOG(1) << __func__ << ". Unprotected stream. stream_id=" << stream_id - << ",this=" << this; + DVLOG_FUNC(1) << "Unprotected stream; stream_id=" << stream_id; + return MF_E_NOT_PROTECTED; } @@ -362,7 +363,7 @@ HRESULT MediaFoundationSourceWrapper::GetInputTrustAuthority( HRESULT MediaFoundationSourceWrapper::GetService(REFGUID guid_service, REFIID riid, LPVOID* result) { - DVLOG(3) << __func__ << ": this=" << this; + DVLOG_FUNC(3); DCHECK(result); if (!IsEqualGUID(guid_service, MF_RATE_CONTROL_SERVICE)) @@ -373,7 +374,7 @@ HRESULT MediaFoundationSourceWrapper::GetService(REFGUID guid_service, HRESULT MediaFoundationSourceWrapper::GetSlowestRate(MFRATE_DIRECTION direction, BOOL supports_thinning, float* rate) { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(3); DCHECK(rate); if (direction == MFRATE_REVERSE) { @@ -386,7 +387,7 @@ HRESULT MediaFoundationSourceWrapper::GetSlowestRate(MFRATE_DIRECTION direction, HRESULT MediaFoundationSourceWrapper::GetFastestRate(MFRATE_DIRECTION direction, BOOL supports_thinning, float* rate) { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(3); DCHECK(rate); if (direction == MFRATE_REVERSE) { @@ -404,7 +405,7 @@ HRESULT MediaFoundationSourceWrapper::GetFastestRate(MFRATE_DIRECTION direction, HRESULT MediaFoundationSourceWrapper::IsRateSupported(BOOL supports_thinning, float new_rate, float* supported_rate) { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2) << "new_rate=" << new_rate; if (state_ == State::kShutdown) return MF_E_SHUTDOWN; @@ -438,12 +439,13 @@ HRESULT MediaFoundationSourceWrapper::IsRateSupported(BOOL supports_thinning, } } + DVLOG_FUNC(2) << PrintHr(hr); return hr; } HRESULT MediaFoundationSourceWrapper::SetRate(BOOL supports_thinning, float rate) { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); if (state_ == State::kShutdown) return MF_E_SHUTDOWN; @@ -460,7 +462,7 @@ HRESULT MediaFoundationSourceWrapper::SetRate(BOOL supports_thinning, HRESULT MediaFoundationSourceWrapper::GetRate(BOOL* supports_thinning, float* rate) { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); *supports_thinning = FALSE; *rate = 0.0f; @@ -477,7 +479,7 @@ uint32_t MediaFoundationSourceWrapper::StreamCount() const { } void MediaFoundationSourceWrapper::CheckForEndOfPresentation() { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); DCHECK(task_runner_->RunsTasksInCurrentSequence()); if (presentation_ended_) { @@ -495,7 +497,7 @@ void MediaFoundationSourceWrapper::CheckForEndOfPresentation() { if (presentation_ended_) { HRESULT hr = QueueEvent(MEEndOfPresentation, GUID_NULL, S_OK, nullptr); DLOG_IF(ERROR, FAILED(hr)) - << "Failed to notify end of presentation. hr=" << hr; + << "Failed to notify end of presentation: " << PrintHr(hr); } } @@ -511,7 +513,7 @@ bool MediaFoundationSourceWrapper::HasEncryptedStream() const { } void MediaFoundationSourceWrapper::SetCdmProxy(IMFCdmProxy* cdm_proxy) { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); DCHECK(task_runner_->RunsTasksInCurrentSequence()); // cdm_proxy_ should never change. @@ -519,11 +521,12 @@ void MediaFoundationSourceWrapper::SetCdmProxy(IMFCdmProxy* cdm_proxy) { cdm_proxy_ = cdm_proxy; HRESULT hr = cdm_proxy_->RefreshTrustedInput(playback_element_id_); - DLOG_IF(ERROR, FAILED(hr)) << "Failed to refresh trusted input. hr=" << hr; + DLOG_IF(ERROR, FAILED(hr)) + << "Failed to refresh trusted input: " << PrintHr(hr); } bool MediaFoundationSourceWrapper::SetVideoStreamEnabled(bool enabled) { - DVLOG(2) << __func__ << ": this=" << this << ",enabled=" << enabled; + DVLOG_FUNC(2) << "enabled=" << enabled; DCHECK(task_runner_->RunsTasksInCurrentSequence()); if (enabled == video_stream_enabled_) @@ -546,7 +549,7 @@ bool MediaFoundationSourceWrapper::SetVideoStreamEnabled(bool enabled) { } void MediaFoundationSourceWrapper::FlushStreams() { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); DCHECK(task_runner_->RunsTasksInCurrentSequence()); for (auto stream : media_streams_) { diff --git a/chromium/media/renderers/win/media_foundation_stream_wrapper.cc b/chromium/media/renderers/win/media_foundation_stream_wrapper.cc index 3ea5d7c1305..bac34b20056 100644 --- a/chromium/media/renderers/win/media_foundation_stream_wrapper.cc +++ b/chromium/media/renderers/win/media_foundation_stream_wrapper.cc @@ -152,7 +152,7 @@ HRESULT MediaFoundationStreamWrapper::RuntimeClassInitialize( int stream_id, IMFMediaSource* parent_source, DemuxerStream* demuxer_stream) { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); { base::AutoLock auto_lock(lock_); @@ -168,27 +168,27 @@ HRESULT MediaFoundationStreamWrapper::RuntimeClassInitialize( void MediaFoundationStreamWrapper::SetTaskRunner( scoped_refptr<base::SequencedTaskRunner> task_runner) { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); task_runner_ = std::move(task_runner); } void MediaFoundationStreamWrapper::DetachParent() { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); base::AutoLock auto_lock(lock_); parent_source_ = nullptr; } void MediaFoundationStreamWrapper::DetachDemuxerStream() { - DVLOG(1) << __func__ << ": this=" << this; + DVLOG_FUNC(1); DCHECK(task_runner_->RunsTasksInCurrentSequence()); demuxer_stream_ = nullptr; } void MediaFoundationStreamWrapper::SetSelected(bool selected) { - DVLOG(2) << __func__ << ": this=" << this << ",selected=" << selected; + DVLOG_FUNC(2) << "selected=" << selected; base::AutoLock auto_lock(lock_); selected_ = selected; @@ -196,20 +196,20 @@ void MediaFoundationStreamWrapper::SetSelected(bool selected) { bool MediaFoundationStreamWrapper::IsSelected() { base::AutoLock auto_lock(lock_); - DVLOG(2) << __func__ << ": this=" << this << ",selected_=" << selected_; + DVLOG_FUNC(2) << "selected_=" << selected_; return selected_; } bool MediaFoundationStreamWrapper::IsEnabled() { base::AutoLock auto_lock(lock_); - DVLOG(2) << __func__ << ": this=" << this << ",enabled_=" << enabled_; + DVLOG_FUNC(2) << "enabled_=" << enabled_; return enabled_; } void MediaFoundationStreamWrapper::SetEnabled(bool enabled) { - DVLOG(2) << __func__ << ": this=" << this << ",enabled=" << enabled; + DVLOG_FUNC(2) << "enabled=" << enabled; { base::AutoLock auto_lock(lock_); @@ -222,7 +222,7 @@ void MediaFoundationStreamWrapper::SetEnabled(bool enabled) { } void MediaFoundationStreamWrapper::SetFlushed(bool flushed) { - DVLOG(2) << __func__ << ": this=" << this << ",flushed=" << flushed; + DVLOG_FUNC(2) << "flushed=" << flushed; base::AutoLock auto_lock(lock_); flushed_ = flushed; @@ -234,14 +234,14 @@ void MediaFoundationStreamWrapper::SetFlushed(bool flushed) { } bool MediaFoundationStreamWrapper::HasEnded() const { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); return stream_ended_; } HRESULT MediaFoundationStreamWrapper::QueueStartedEvent( const PROPVARIANT* start_position) { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); state_ = State::kStarted; RETURN_IF_FAILED(mf_media_event_queue_->QueueEventParamVar( @@ -251,7 +251,7 @@ HRESULT MediaFoundationStreamWrapper::QueueStartedEvent( HRESULT MediaFoundationStreamWrapper::QueueSeekedEvent( const PROPVARIANT* start_position) { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); state_ = State::kStarted; RETURN_IF_FAILED(mf_media_event_queue_->QueueEventParamVar( @@ -260,7 +260,7 @@ HRESULT MediaFoundationStreamWrapper::QueueSeekedEvent( } HRESULT MediaFoundationStreamWrapper::QueueStoppedEvent() { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); state_ = State::kStopped; RETURN_IF_FAILED(mf_media_event_queue_->QueueEventParamVar( @@ -269,7 +269,7 @@ HRESULT MediaFoundationStreamWrapper::QueueStoppedEvent() { } HRESULT MediaFoundationStreamWrapper::QueuePausedEvent() { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); state_ = State::kPaused; RETURN_IF_FAILED(mf_media_event_queue_->QueueEventParamVar( @@ -282,7 +282,7 @@ DemuxerStream::Type MediaFoundationStreamWrapper::StreamType() const { } void MediaFoundationStreamWrapper::ProcessRequestsIfPossible() { - DVLOG(3) << __func__ << ". this=" << this; + DVLOG_FUNC(3); DCHECK(task_runner_->RunsTasksInCurrentSequence()); { @@ -313,16 +313,16 @@ void MediaFoundationStreamWrapper::ProcessRequestsIfPossible() { HRESULT MediaFoundationStreamWrapper::ServiceSampleRequest( IUnknown* token, DecoderBuffer* buffer) { - DVLOG(3) << __func__ << ". this=" << this; + DVLOG_FUNC(3); DCHECK(task_runner_->RunsTasksInCurrentSequence()); lock_.AssertAcquired(); if (buffer->end_of_stream()) { if (!enabled_) { - DVLOG(2) << "Ignoring EOS for disabled stream: this=" << this; + DVLOG_FUNC(2) << "Ignoring EOS for disabled stream"; return S_OK; } - DVLOG(2) << "End of stream: this=" << this; + DVLOG_FUNC(2) << "End of stream"; RETURN_IF_FAILED(mf_media_event_queue_->QueueEventParamUnk( MEEndOfStream, GUID_NULL, S_OK, nullptr)); stream_ended_ = true; @@ -331,9 +331,8 @@ HRESULT MediaFoundationStreamWrapper::ServiceSampleRequest( ->CheckForEndOfPresentation(); } } else { - DVLOG(3) << __func__ << ". this=" << this - << ",buffer ts(ms)=" << buffer->timestamp().InMilliseconds() - << ",is_key_frame=" << buffer->is_key_frame(); + DVLOG_FUNC(3) << "buffer ts=" << buffer->timestamp() + << ", is_key_frame=" << buffer->is_key_frame(); ComPtr<IMFSample> mf_sample; RETURN_IF_FAILED(GenerateSampleFromDecoderBuffer(buffer, &mf_sample)); if (token) { @@ -347,7 +346,7 @@ HRESULT MediaFoundationStreamWrapper::ServiceSampleRequest( } bool MediaFoundationStreamWrapper::ServicePostFlushSampleRequest() { - DVLOG(3) << __func__ << ". this=" << this; + DVLOG_FUNC(3); DCHECK(task_runner_->RunsTasksInCurrentSequence()); base::AutoLock auto_lock(lock_); @@ -360,7 +359,7 @@ bool MediaFoundationStreamWrapper::ServicePostFlushSampleRequest() { HRESULT hr = ServiceSampleRequest(request_token.Get(), post_flush_buffers_.front().get()); if (FAILED(hr)) { - DLOG(WARNING) << "Failed to service post flush sample. hr=" << hr; + DLOG(WARNING) << "Failed to service post flush sample: " << PrintHr(hr); return false; } @@ -370,7 +369,7 @@ bool MediaFoundationStreamWrapper::ServicePostFlushSampleRequest() { } HRESULT MediaFoundationStreamWrapper::QueueFormatChangedEvent() { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); ComPtr<IMFMediaType> media_type; RETURN_IF_FAILED(GetMediaType(&media_type)); @@ -382,7 +381,7 @@ HRESULT MediaFoundationStreamWrapper::QueueFormatChangedEvent() { void MediaFoundationStreamWrapper::OnDemuxerStreamRead( DemuxerStream::Status status, scoped_refptr<DecoderBuffer> buffer) { - DVLOG(3) << __func__ << ". this=" << this; + DVLOG_FUNC(3); { base::AutoLock auto_lock(lock_); DCHECK(pending_stream_read_); @@ -395,39 +394,40 @@ void MediaFoundationStreamWrapper::OnDemuxerStreamRead( // Push |buffer| to process later if needed. Otherwise, process it // immediately. if (flushed_ || !post_flush_buffers_.empty()) { - DVLOG(3) << __func__ << ". this=" << this << ", push buffer."; + DVLOG_FUNC(3) << "push buffer."; post_flush_buffers_.push(buffer); } else { hr = ServiceSampleRequest(token.Get(), buffer.get()); if (FAILED(hr)) { - DLOG(ERROR) << __func__ << ": ServiceSampleRequest failed. hr=" << hr; + DLOG(ERROR) << __func__ + << ": ServiceSampleRequest failed: " << PrintHr(hr); return; } pending_sample_request_tokens_.pop(); } } else if (status == DemuxerStream::Status::kConfigChanged) { - DVLOG(2) << "Stream config changed. this=" << this - << ",AreFormatChangesEnabled=" << AreFormatChangesEnabled(); + DVLOG_FUNC(2) << "Stream config changed, AreFormatChangesEnabled=" + << AreFormatChangesEnabled(); if (AreFormatChangesEnabled()) { hr = QueueFormatChangedEvent(); if (FAILED(hr)) { DLOG(ERROR) << __func__ - << ": QueueFormatChangedEvent failed. hr=" << hr; + << ": QueueFormatChangedEvent failed: " << PrintHr(hr); return; } } } else if (status == DemuxerStream::Status::kError) { - DVLOG(2) << "Stream read error: this=" << this; + DVLOG_FUNC(2) << "Stream read error"; mf_media_event_queue_->QueueEventParamVar( MEError, GUID_NULL, MF_E_INVALID_STREAM_DATA, nullptr); return; } else if (status == DemuxerStream::Status::kAborted) { - DVLOG(2) << "Stream read aborted: this=" << this; + DVLOG_FUNC(2) << "Stream read aborted"; // Continue to ProcessRequestsIfPossible() to satisfy pending sample // request by issuing DemuxerStream::Read() if necessary. } else { NOTREACHED() << "Unexpected demuxer stream status. status=" << status - << ",this=" << this; + << ", this=" << this; } } @@ -437,7 +437,7 @@ void MediaFoundationStreamWrapper::OnDemuxerStreamRead( HRESULT MediaFoundationStreamWrapper::GenerateSampleFromDecoderBuffer( DecoderBuffer* buffer, IMFSample** sample_out) { - DVLOG(3) << __func__ << ". this=" << this; + DVLOG_FUNC(3); ComPtr<IMFSample> mf_sample; RETURN_IF_FAILED(MFCreateSample(&mf_sample)); @@ -478,14 +478,14 @@ HRESULT MediaFoundationStreamWrapper::GenerateSampleFromDecoderBuffer( HRESULT MediaFoundationStreamWrapper::TransformSample( Microsoft::WRL::ComPtr<IMFSample>& sample) { - DVLOG(3) << __func__ << ". this=" << this; + DVLOG_FUNC(3); return S_OK; } HRESULT MediaFoundationStreamWrapper::GetMediaSource( IMFMediaSource** media_source_out) { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); DCHECK(!task_runner_->RunsTasksInCurrentSequence()); base::AutoLock auto_lock(lock_); @@ -499,7 +499,7 @@ HRESULT MediaFoundationStreamWrapper::GetMediaSource( HRESULT MediaFoundationStreamWrapper::GetStreamDescriptor( IMFStreamDescriptor** stream_descriptor_out) { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); if (!mf_stream_descriptor_) { DLOG(ERROR) << __func__ << ": MF_E_NOT_INITIALIZED"; @@ -510,7 +510,7 @@ HRESULT MediaFoundationStreamWrapper::GetStreamDescriptor( } HRESULT MediaFoundationStreamWrapper::RequestSample(IUnknown* token) { - DVLOG(3) << __func__ << ". this=" << this; + DVLOG_FUNC(3); DCHECK(!task_runner_->RunsTasksInCurrentSequence()); base::AutoLock auto_lock(lock_); @@ -527,7 +527,7 @@ HRESULT MediaFoundationStreamWrapper::RequestSample(IUnknown* token) { HRESULT MediaFoundationStreamWrapper::GetEvent(DWORD flags, IMFMediaEvent** event_out) { - DVLOG(3) << __func__ << ". this=" << this; + DVLOG_FUNC(3); DCHECK(mf_media_event_queue_); // Not tracing hr to avoid the noise from MF_E_NO_EVENTS_AVAILABLE. @@ -536,7 +536,7 @@ HRESULT MediaFoundationStreamWrapper::GetEvent(DWORD flags, HRESULT MediaFoundationStreamWrapper::BeginGetEvent(IMFAsyncCallback* callback, IUnknown* state) { - DVLOG(3) << __func__ << ". this=" << this; + DVLOG_FUNC(3); DCHECK(mf_media_event_queue_); RETURN_IF_FAILED(mf_media_event_queue_->BeginGetEvent(callback, state)); @@ -545,7 +545,7 @@ HRESULT MediaFoundationStreamWrapper::BeginGetEvent(IMFAsyncCallback* callback, HRESULT MediaFoundationStreamWrapper::EndGetEvent(IMFAsyncResult* result, IMFMediaEvent** event_out) { - DVLOG(3) << __func__ << ". this=" << this; + DVLOG_FUNC(3); DCHECK(mf_media_event_queue_); RETURN_IF_FAILED(mf_media_event_queue_->EndGetEvent(result, event_out)); @@ -556,7 +556,7 @@ HRESULT MediaFoundationStreamWrapper::QueueEvent(MediaEventType type, REFGUID extended_type, HRESULT status, const PROPVARIANT* value) { - DVLOG(3) << __func__ << ". this=" << this; + DVLOG_FUNC(3); DCHECK(mf_media_event_queue_); RETURN_IF_FAILED(mf_media_event_queue_->QueueEventParamVar( @@ -565,7 +565,7 @@ HRESULT MediaFoundationStreamWrapper::QueueEvent(MediaEventType type, } HRESULT MediaFoundationStreamWrapper::GenerateStreamDescriptor() { - DVLOG(2) << __func__ << ": this=" << this; + DVLOG_FUNC(2); ComPtr<IMFMediaType> media_type; IMFMediaType** mediaTypes = &media_type; diff --git a/chromium/media/renderers/yuv_util.cc b/chromium/media/renderers/yuv_util.cc new file mode 100644 index 00000000000..3a66d63bd94 --- /dev/null +++ b/chromium/media/renderers/yuv_util.cc @@ -0,0 +1,221 @@ +// Copyright (c) 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/renderers/yuv_util.h" + +#include <GLES3/gl3.h> + +#include "components/viz/common/gpu/raster_context_provider.h" +#include "gpu/GLES2/gl2extchromium.h" +#include "gpu/command_buffer/client/raster_interface.h" +#include "gpu/command_buffer/common/mailbox_holder.h" +#include "media/base/video_frame.h" +#include "third_party/skia/include/core/SkImage.h" +#include "third_party/skia/include/gpu/GrContext.h" + +namespace media { + +namespace { + +static constexpr size_t kNumNV12Planes = 2; +static constexpr size_t kNumYUVPlanes = 3; +using YUVMailboxes = std::array<gpu::MailboxHolder, kNumYUVPlanes>; + +YUVMailboxes GetYUVMailboxes(const VideoFrame* video_frame, + gpu::raster::RasterInterface* ri) { + YUVMailboxes mailboxes; + + for (size_t i = 0; i < video_frame->NumTextures(); ++i) { + mailboxes[i] = video_frame->mailbox_holder(i); + DCHECK(mailboxes[i].texture_target == GL_TEXTURE_2D || + mailboxes[i].texture_target == GL_TEXTURE_EXTERNAL_OES || + mailboxes[i].texture_target == GL_TEXTURE_RECTANGLE_ARB) + << "Unsupported texture target " << std::hex << std::showbase + << mailboxes[i].texture_target; + ri->WaitSyncTokenCHROMIUM(mailboxes[i].sync_token.GetConstData()); + } + + return mailboxes; +} + +struct YUVPlaneTextureInfo { + GrGLTextureInfo texture = {0, 0}; + bool is_shared_image = false; +}; +using YUVTexturesInfo = std::array<YUVPlaneTextureInfo, kNumYUVPlanes>; + +YUVTexturesInfo GetYUVTexturesInfo( + const VideoFrame* video_frame, + viz::RasterContextProvider* raster_context_provider) { + gpu::raster::RasterInterface* ri = raster_context_provider->RasterInterface(); + DCHECK(ri); + YUVMailboxes mailboxes = GetYUVMailboxes(video_frame, ri); + YUVTexturesInfo yuv_textures_info; + + GrGLenum skia_texture_format = + video_frame->format() == PIXEL_FORMAT_NV12 ? GL_RGB8 : GL_R8_EXT; + for (size_t i = 0; i < video_frame->NumTextures(); ++i) { + yuv_textures_info[i].texture.fID = + ri->CreateAndConsumeForGpuRaster(mailboxes[i].mailbox); + if (mailboxes[i].mailbox.IsSharedImage()) { + yuv_textures_info[i].is_shared_image = true; + ri->BeginSharedImageAccessDirectCHROMIUM( + yuv_textures_info[i].texture.fID, + GL_SHARED_IMAGE_ACCESS_MODE_READ_CHROMIUM); + } + + yuv_textures_info[i].texture.fTarget = mailboxes[i].texture_target; + yuv_textures_info[i].texture.fFormat = skia_texture_format; + } + + return yuv_textures_info; +} + +void DeleteYUVTextures(const VideoFrame* video_frame, + viz::RasterContextProvider* raster_context_provider, + const YUVTexturesInfo& yuv_textures_info) { + gpu::raster::RasterInterface* ri = raster_context_provider->RasterInterface(); + DCHECK(ri); + + for (size_t i = 0; i < video_frame->NumTextures(); ++i) { + if (yuv_textures_info[i].is_shared_image) + ri->EndSharedImageAccessDirectCHROMIUM(yuv_textures_info[i].texture.fID); + ri->DeleteGpuRasterTexture(yuv_textures_info[i].texture.fID); + } +} + +void ConvertFromVideoFrameYUVWithGrContext( + const VideoFrame* video_frame, + viz::RasterContextProvider* raster_context_provider, + const gpu::MailboxHolder& dest_mailbox_holder) { + gpu::raster::RasterInterface* ri = raster_context_provider->RasterInterface(); + DCHECK(ri); + ri->WaitSyncTokenCHROMIUM(dest_mailbox_holder.sync_token.GetConstData()); + GLuint dest_tex_id = + ri->CreateAndConsumeForGpuRaster(dest_mailbox_holder.mailbox); + if (dest_mailbox_holder.mailbox.IsSharedImage()) { + ri->BeginSharedImageAccessDirectCHROMIUM( + dest_tex_id, GL_SHARED_IMAGE_ACCESS_MODE_READWRITE_CHROMIUM); + } + // Let the SkImage fall out of scope and track the result using dest_tex_id + NewSkImageFromVideoFrameYUVTexturesWithExternalBackend( + video_frame, raster_context_provider, dest_mailbox_holder.texture_target, + dest_tex_id); + if (dest_mailbox_holder.mailbox.IsSharedImage()) + ri->EndSharedImageAccessDirectCHROMIUM(dest_tex_id); + ri->DeleteGpuRasterTexture(dest_tex_id); +} + +SkYUVColorSpace ColorSpaceToSkYUVColorSpace( + const gfx::ColorSpace& color_space) { + // TODO(hubbe): This should really default to rec709. + // https://crbug.com/828599 + SkYUVColorSpace sk_color_space = kRec601_SkYUVColorSpace; + color_space.ToSkYUVColorSpace(&sk_color_space); + return sk_color_space; +} + +} // namespace + +void ConvertFromVideoFrameYUV( + const VideoFrame* video_frame, + viz::RasterContextProvider* raster_context_provider, + const gpu::MailboxHolder& dest_mailbox_holder) { + DCHECK(raster_context_provider); + if (raster_context_provider->GrContext()) { + ConvertFromVideoFrameYUVWithGrContext(video_frame, raster_context_provider, + dest_mailbox_holder); + return; + } + + auto* ri = raster_context_provider->RasterInterface(); + DCHECK(ri); + ri->WaitSyncTokenCHROMIUM(dest_mailbox_holder.sync_token.GetConstData()); + YUVMailboxes mailboxes = GetYUVMailboxes(video_frame, ri); + SkYUVColorSpace color_space = + ColorSpaceToSkYUVColorSpace(video_frame->ColorSpace()); + if (video_frame->format() == PIXEL_FORMAT_I420) { + DCHECK_EQ(video_frame->NumTextures(), kNumYUVPlanes); + ri->ConvertYUVMailboxesToRGB(dest_mailbox_holder.mailbox, color_space, + mailboxes[0].mailbox, mailboxes[1].mailbox, + mailboxes[2].mailbox); + } else { + DCHECK_EQ(video_frame->format(), PIXEL_FORMAT_NV12); + DCHECK_EQ(video_frame->NumTextures(), kNumNV12Planes); + ri->ConvertNV12MailboxesToRGB(dest_mailbox_holder.mailbox, color_space, + mailboxes[0].mailbox, mailboxes[1].mailbox); + } +} + +sk_sp<SkImage> NewSkImageFromVideoFrameYUVTexturesWithExternalBackend( + const VideoFrame* video_frame, + viz::RasterContextProvider* raster_context_provider, + unsigned int texture_target, + unsigned int texture_id) { + DCHECK(video_frame->HasTextures()); + GrContext* gr_context = raster_context_provider->GrContext(); + DCHECK(gr_context); + // TODO: We should compare the DCHECK vs when UpdateLastImage calls this + // function. (https://crbug.com/674185) + DCHECK(video_frame->format() == PIXEL_FORMAT_I420 || + video_frame->format() == PIXEL_FORMAT_NV12); + + gfx::Size ya_tex_size = video_frame->coded_size(); + gfx::Size uv_tex_size((ya_tex_size.width() + 1) / 2, + (ya_tex_size.height() + 1) / 2); + + GrGLTextureInfo backend_texture{}; + + YUVTexturesInfo yuv_textures_info = + GetYUVTexturesInfo(video_frame, raster_context_provider); + + GrBackendTexture yuv_textures[3] = { + GrBackendTexture(ya_tex_size.width(), ya_tex_size.height(), + GrMipMapped::kNo, yuv_textures_info[0].texture), + GrBackendTexture(uv_tex_size.width(), uv_tex_size.height(), + GrMipMapped::kNo, yuv_textures_info[1].texture), + GrBackendTexture(uv_tex_size.width(), uv_tex_size.height(), + GrMipMapped::kNo, yuv_textures_info[2].texture), + }; + backend_texture.fID = texture_id; + backend_texture.fTarget = texture_target; + backend_texture.fFormat = GL_RGBA8; + GrBackendTexture result_texture(video_frame->coded_size().width(), + video_frame->coded_size().height(), + GrMipMapped::kNo, backend_texture); + + sk_sp<SkImage> img = YUVGrBackendTexturesToSkImage( + gr_context, video_frame->ColorSpace(), video_frame->format(), + yuv_textures, result_texture); + gr_context->flush(); + + DeleteYUVTextures(video_frame, raster_context_provider, yuv_textures_info); + + return img; +} + +sk_sp<SkImage> YUVGrBackendTexturesToSkImage( + GrContext* gr_context, + gfx::ColorSpace video_color_space, + VideoPixelFormat video_format, + GrBackendTexture* yuv_textures, + const GrBackendTexture& result_texture) { + SkYUVColorSpace color_space = ColorSpaceToSkYUVColorSpace(video_color_space); + + switch (video_format) { + case PIXEL_FORMAT_NV12: + return SkImage::MakeFromNV12TexturesCopyWithExternalBackend( + gr_context, color_space, yuv_textures, kTopLeft_GrSurfaceOrigin, + result_texture); + case PIXEL_FORMAT_I420: + return SkImage::MakeFromYUVTexturesCopyWithExternalBackend( + gr_context, color_space, yuv_textures, kTopLeft_GrSurfaceOrigin, + result_texture); + default: + NOTREACHED(); + return nullptr; + } +} + +} // namespace media diff --git a/chromium/media/renderers/yuv_util.h b/chromium/media/renderers/yuv_util.h new file mode 100644 index 00000000000..e8fe451ab07 --- /dev/null +++ b/chromium/media/renderers/yuv_util.h @@ -0,0 +1,54 @@ +// Copyright (c) 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_RENDERERS_YUV_UTIL_H_ +#define MEDIA_RENDERERS_YUV_UTIL_H_ + +#include "media/base/media_export.h" +#include "media/base/video_types.h" +#include "third_party/skia/include/core/SkRefCnt.h" +#include "ui/gfx/color_space.h" + +// Skia forward declarations +class GrBackendTexture; +class GrContext; +class SkImage; + +namespace gpu { +struct MailboxHolder; +} + +namespace viz { +class RasterContextProvider; +} // namespace viz + +namespace media { + +class VideoFrame; + +// Converts a YUV video frame to RGB format and stores the results in the +// provided mailbox. The caller of this function maintains ownership of the +// mailbox. +MEDIA_EXPORT void ConvertFromVideoFrameYUV( + const VideoFrame* video_frame, + viz::RasterContextProvider* raster_context_provider, + const gpu::MailboxHolder& dest_mailbox_holder); + +MEDIA_EXPORT sk_sp<SkImage> +NewSkImageFromVideoFrameYUVTexturesWithExternalBackend( + const VideoFrame* video_frame, + viz::RasterContextProvider* raster_context_provider, + unsigned int texture_target, + unsigned int texture_id); + +MEDIA_EXPORT sk_sp<SkImage> YUVGrBackendTexturesToSkImage( + GrContext* gr_context, + gfx::ColorSpace video_color_space, + VideoPixelFormat video_format, + GrBackendTexture* yuv_textures, + const GrBackendTexture& result_texture); + +} // namespace media + +#endif // MEDIA_RENDERERS_YUV_UTIL_H_
\ No newline at end of file diff --git a/chromium/media/tools/constrained_network_server/cns.py b/chromium/media/tools/constrained_network_server/cns.py index 58e2ba000d8..d039d8e7acc 100755 --- a/chromium/media/tools/constrained_network_server/cns.py +++ b/chromium/media/tools/constrained_network_server/cns.py @@ -2,6 +2,41 @@ # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +# +# [VPYTHON:BEGIN] +# wheel: < +# name: "infra/python/wheels/pytz-py2_py3" +# version: "version:2018.4" +# > +# wheel: < +# name: "infra/python/wheels/tempora-py2_py3" +# version: "version:1.11" +# > +# wheel: < +# name: "infra/python/wheels/more-itertools-py2_py3" +# version: "version:4.1.0" +# > +# wheel: < +# name: "infra/python/wheels/backports_functools_lru_cache-py2_py3" +# version: "version:1.5" +# > +# wheel: < +# name: "infra/python/wheels/six-py2_py3" +# version: "version:1.12.0" +# > +# wheel: < +# name: "infra/python/wheels/portend-py2_py3" +# version: "version:2.2" +# > +# wheel: < +# name: "infra/python/wheels/cheroot-py2_py3" +# version: "version:6.2.4" +# > +# wheel: < +# name: "infra/python/wheels/cherrypy-py2_py3" +# version: "version:14.2.0" +# > +# [VPYTHON:END] """Constrained Network Server. Serves files with supplied network constraints. diff --git a/chromium/media/tools/constrained_network_server/cns_test.py b/chromium/media/tools/constrained_network_server/cns_test.py index 5d794956755..a3ac54e368d 100755 --- a/chromium/media/tools/constrained_network_server/cns_test.py +++ b/chromium/media/tools/constrained_network_server/cns_test.py @@ -1,8 +1,43 @@ -#!/usr/bin/env python +#!/usr/bin/env vpython # Copyright (c) 2012 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +# [VPYTHON:BEGIN] +# wheel: < +# name: "infra/python/wheels/pytz-py2_py3" +# version: "version:2018.4" +# > +# wheel: < +# name: "infra/python/wheels/tempora-py2_py3" +# version: "version:1.11" +# > +# wheel: < +# name: "infra/python/wheels/more-itertools-py2_py3" +# version: "version:4.1.0" +# > +# wheel: < +# name: "infra/python/wheels/backports_functools_lru_cache-py2_py3" +# version: "version:1.5" +# > +# wheel: < +# name: "infra/python/wheels/six-py2_py3" +# version: "version:1.12.0" +# > +# wheel: < +# name: "infra/python/wheels/portend-py2_py3" +# version: "version:2.2" +# > +# wheel: < +# name: "infra/python/wheels/cheroot-py2_py3" +# version: "version:6.2.4" +# > +# wheel: < +# name: "infra/python/wheels/cherrypy-py2_py3" +# version: "version:14.2.0" +# > +# [VPYTHON:END] + """Tests for Constrained Network Server.""" import os import signal diff --git a/chromium/media/video/BUILD.gn b/chromium/media/video/BUILD.gn index f6c7977c7ad..4eacf416cf1 100644 --- a/chromium/media/video/BUILD.gn +++ b/chromium/media/video/BUILD.gn @@ -66,6 +66,14 @@ source_set("video") { "//build/config/compiler:no_size_t_to_int_warning", "//media:subcomponent_config", ] + + if (media_use_libvpx) { + sources += [ + "vpx_video_encoder.cc", + "vpx_video_encoder.h", + ] + deps += [ "//third_party/libvpx" ] + } } # Note: This is a roll-up only target; do not expand the visibility. DEPS should diff --git a/chromium/media/video/fake_video_encode_accelerator.cc b/chromium/media/video/fake_video_encode_accelerator.cc index 30f2251ccdf..783f8f0ba7f 100644 --- a/chromium/media/video/fake_video_encode_accelerator.cc +++ b/chromium/media/video/fake_video_encode_accelerator.cc @@ -5,8 +5,8 @@ #include "media/video/fake_video_encode_accelerator.h" #include "base/bind.h" +#include "base/check.h" #include "base/location.h" -#include "base/logging.h" #include "base/single_thread_task_runner.h" namespace media { diff --git a/chromium/media/video/h264_bit_reader.cc b/chromium/media/video/h264_bit_reader.cc index b3a13719a81..0fe227055c2 100644 --- a/chromium/media/video/h264_bit_reader.cc +++ b/chromium/media/video/h264_bit_reader.cc @@ -3,7 +3,7 @@ // found in the LICENSE file. #include "media/video/h264_bit_reader.h" -#include "base/logging.h" +#include "base/check.h" namespace media { diff --git a/chromium/media/video/picture.cc b/chromium/media/video/picture.cc index 316ce15e5b8..3e169be0b80 100644 --- a/chromium/media/video/picture.cc +++ b/chromium/media/video/picture.cc @@ -2,9 +2,11 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -#include "base/macros.h" #include "media/video/picture.h" +#include "base/logging.h" +#include "base/macros.h" + namespace media { PictureBuffer::PictureBuffer(int32_t id, const gfx::Size& size) diff --git a/chromium/media/video/video_decode_accelerator.h b/chromium/media/video/video_decode_accelerator.h index da79cfc4c83..c574411cea4 100644 --- a/chromium/media/video/video_decode_accelerator.h +++ b/chromium/media/video/video_decode_accelerator.h @@ -39,7 +39,7 @@ namespace media { // implement the backend of PPB_VideoDecoder_Dev. class MEDIA_EXPORT VideoDecodeAccelerator { public: - // Specification of a decoding profile supported by an decoder. + // Specification of a decoding profile supported by a decoder. // |max_resolution| and |min_resolution| are inclusive. struct MEDIA_EXPORT SupportedProfile { SupportedProfile(); diff --git a/chromium/media/video/video_encode_accelerator.cc b/chromium/media/video/video_encode_accelerator.cc index 01ed6203d05..81528abd02a 100644 --- a/chromium/media/video/video_encode_accelerator.cc +++ b/chromium/media/video/video_encode_accelerator.cc @@ -41,7 +41,8 @@ VideoEncodeAccelerator::Config::Config( base::Optional<uint32_t> gop_length, base::Optional<uint8_t> h264_output_level, base::Optional<StorageType> storage_type, - ContentType content_type) + ContentType content_type, + const std::vector<SpatialLayer>& spatial_layers) : input_format(input_format), input_visible_size(input_visible_size), output_profile(output_profile), @@ -51,7 +52,8 @@ VideoEncodeAccelerator::Config::Config( gop_length(gop_length), h264_output_level(h264_output_level), storage_type(storage_type), - content_type(content_type) {} + content_type(content_type), + spatial_layers(spatial_layers) {} VideoEncodeAccelerator::Config::~Config() = default; @@ -74,9 +76,31 @@ std::string VideoEncodeAccelerator::Config::AsHumanReadableString() const { str += base::StringPrintf(", h264_output_level: %u", h264_output_level.value()); } + + for (size_t i = 0; i < spatial_layers.size(); ++i) { + const auto& sl = spatial_layers[i]; + str += base::StringPrintf( + "\nSL#%zu: width=%d, height=%d, bitrate_bps=%u" + ", framerate=%u, max_qp=%u" + ", num_of_temporal_layers=%u", + i, sl.width, sl.height, sl.bitrate_bps, sl.framerate, sl.max_qp, + sl.num_of_temporal_layers); + } return str; } +bool VideoEncodeAccelerator::Config::HasTemporalLayer() const { + for (const auto& sl : spatial_layers) { + if (sl.num_of_temporal_layers > 1u) + return true; + } + return false; +} + +bool VideoEncodeAccelerator::Config::HasSpatialLayer() const { + return spatial_layers.size() > 1u; +} + void VideoEncodeAccelerator::Client::NotifyEncoderInfoChange( const VideoEncoderInfo& info) { // Do nothing if a client doesn't use the info. diff --git a/chromium/media/video/video_encode_accelerator.h b/chromium/media/video/video_encode_accelerator.h index 4ed0582116e..84a31261293 100644 --- a/chromium/media/video/video_encode_accelerator.h +++ b/chromium/media/video/video_encode_accelerator.h @@ -110,6 +110,22 @@ class MEDIA_EXPORT VideoEncodeAccelerator { // kDmabuf if a video frame is referred by dmabuf. enum class StorageType { kShmem, kDmabuf }; + struct MEDIA_EXPORT SpatialLayer { + // The encoder dimension of the spatial layer. + int32_t width = 0; + int32_t height = 0; + // The bitrate of encoded output stream of the spatial layer in bits per + // second. + uint32_t bitrate_bps = 0u; + uint32_t framerate = 0u; + // The recommended maximum qp value of the spatial layer. VEA can ignore + // this value. + uint8_t max_qp = 0u; + // The number of temporal layers of the spatial layer. The detail of + // the temporal layer structure is up to VideoEncodeAccelerator. + uint8_t num_of_temporal_layers = 0u; + }; + Config(); Config(const Config& config); @@ -121,12 +137,16 @@ class MEDIA_EXPORT VideoEncodeAccelerator { base::Optional<uint32_t> gop_length = base::nullopt, base::Optional<uint8_t> h264_output_level = base::nullopt, base::Optional<StorageType> storage_type = base::nullopt, - ContentType content_type = ContentType::kCamera); + ContentType content_type = ContentType::kCamera, + const std::vector<SpatialLayer>& spatial_layers = {}); ~Config(); std::string AsHumanReadableString() const; + bool HasTemporalLayer() const; + bool HasSpatialLayer() const; + // Frame format of input stream (as would be reported by // VideoFrame::format() for frames passed to Encode()). VideoPixelFormat input_format; @@ -168,6 +188,12 @@ class MEDIA_EXPORT VideoEncodeAccelerator { // bright colors. With this content hint the encoder may choose to optimize // for the given use case. ContentType content_type; + + // The configuration for spatial layers. This is not empty if and only if + // either spatial or temporal layer encoding is configured. When this is not + // empty, VideoEncodeAccelerator should refer the width, height, bitrate and + // etc. of |spatial_layers|. + std::vector<SpatialLayer> spatial_layers; }; // Interface for clients that use VideoEncodeAccelerator. These callbacks will diff --git a/chromium/media/video/video_encoder_info.h b/chromium/media/video/video_encoder_info.h index 97ca2cd8859..0258afb2134 100644 --- a/chromium/media/video/video_encoder_info.h +++ b/chromium/media/video/video_encoder_info.h @@ -9,6 +9,7 @@ #include <string> #include <vector> +#include "base/optional.h" #include "media/base/media_export.h" #include "ui/gfx/geometry/size.h" @@ -23,8 +24,10 @@ struct MEDIA_EXPORT ScalingSettings { ScalingSettings(const ScalingSettings&); ~ScalingSettings(); - int min_qp = 4; - int max_qp = 157; + // Quantization Parameter in ScalingSettings are codec specific. + // The range of qp is 0-51 (H264), 0-127 (VP8) and 0-255 (VP9 and AV1). + int min_qp = 0; + int max_qp = 255; }; struct MEDIA_EXPORT ResolutionBitrateLimit { @@ -56,7 +59,7 @@ struct MEDIA_EXPORT VideoEncoderInfo { bool is_hardware_accelerated = true; bool supports_simulcast = false; - ScalingSettings scaling_settings; + base::Optional<ScalingSettings> scaling_settings; std::vector<uint8_t> fps_allocation[kMaxSpatialLayers]; std::vector<ResolutionBitrateLimit> resolution_bitrate_limits; }; diff --git a/chromium/media/video/vpx_video_encoder.cc b/chromium/media/video/vpx_video_encoder.cc new file mode 100644 index 00000000000..808d585b919 --- /dev/null +++ b/chromium/media/video/vpx_video_encoder.cc @@ -0,0 +1,278 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/video/vpx_video_encoder.h" +#include "base/strings/stringprintf.h" +#include "base/time/time.h" +#include "media/base/bind_to_current_loop.h" +#include "media/base/video_frame.h" +#include "third_party/libvpx/source/libvpx/vpx/vp8cx.h" + +namespace media { + +namespace { +Status SetUpVpxConfig(const VideoEncoder::Options& opts, + vpx_codec_enc_cfg_t* config) { + if (opts.width <= 0 || opts.height <= 0) + return Status(StatusCode::kEncoderUnsupportedConfig, + "Negative width or height values"); + + config->g_pass = VPX_RC_ONE_PASS; + config->g_lag_in_frames = 0; + config->rc_resize_allowed = 0; + config->rc_dropframe_thresh = 0; // Don't drop frames + config->g_timebase.num = 1; + config->g_timebase.den = base::Time::kMicrosecondsPerSecond; + + if (opts.threads.has_value()) + config->g_threads = opts.threads.value(); + config->g_w = opts.width; + config->g_h = opts.height; + + // Insert keyframes at will with a given max interval + if (opts.keyframe_interval.has_value()) { + config->kf_mode = VPX_KF_AUTO; + config->kf_min_dist = 0; + config->kf_max_dist = opts.keyframe_interval.value(); + } + + if (opts.bitrate.has_value()) { + config->rc_end_usage = VPX_CBR; + config->rc_target_bitrate = opts.bitrate.value() / 1000; + } else { + config->rc_end_usage = VPX_VBR; + } + return Status(); +} +} // namespace + +VpxVideoEncoder::VpxVideoEncoder() = default; + +void VpxVideoEncoder::Initialize(VideoCodecProfile profile, + const Options& options, + OutputCB output_cb, + StatusCB done_cb) { + done_cb = media::BindToCurrentLoop(std::move(done_cb)); + if (codec_) { + std::move(done_cb).Run(StatusCode::kEncoderInitializeTwice); + return; + } + + vpx_codec_iface_t* iface = nullptr; + if (profile == media::VP8PROFILE_ANY) { + iface = vpx_codec_vp8_cx(); + } else if (profile >= media::VP9PROFILE_PROFILE0 && + profile <= media::VP9PROFILE_PROFILE3) { + iface = vpx_codec_vp9_cx(); + } else { + auto status = Status(StatusCode::kEncoderUnsupportedProfile) + .WithData("profile", profile); + std::move(done_cb).Run(status); + return; + } + + auto vpx_error = vpx_codec_enc_config_default(iface, &codec_config_, 0); + if (vpx_error != VPX_CODEC_OK) { + auto status = Status(StatusCode::kEncoderInitializationError, + "Failed to get default VPX config.") + .WithData("vpx_error", vpx_error); + std::move(done_cb).Run(status); + return; + } + switch (profile) { + case media::VP9PROFILE_PROFILE1: + codec_config_.g_profile = 1; + break; + case media::VP9PROFILE_PROFILE2: + codec_config_.g_profile = 2; + break; + case media::VP9PROFILE_PROFILE3: + codec_config_.g_profile = 3; + break; + default: + codec_config_.g_profile = 0; + break; + } + + auto status = SetUpVpxConfig(options, &codec_config_); + if (!status.is_ok()) { + std::move(done_cb).Run(status); + return; + } + + codec_ = new vpx_codec_ctx_t; + vpx_error = vpx_codec_enc_init(codec_, iface, &codec_config_, 0 /* flags */); + if (vpx_error != VPX_CODEC_OK) { + std::string msg = base::StringPrintf("VPX encoder initialization error: %s", + vpx_codec_err_to_string(vpx_error)); + + status = Status(StatusCode::kEncoderInitializationError, msg); + std::move(done_cb).Run(status); + return; + } + + // Due to https://bugs.chromium.org/p/webm/issues/detail?id=1684 + // values less than 5 crash VP9 encoder. + vpx_error = vpx_codec_control(codec_, VP8E_SET_CPUUSED, 5); + if (vpx_error != VPX_CODEC_OK) { + std::string msg = + base::StringPrintf("VPX encoder VP8E_SET_CPUUSED error: %s", + vpx_codec_err_to_string(vpx_error)); + + status = Status(StatusCode::kEncoderInitializationError, msg); + std::move(done_cb).Run(status); + return; + } + + options_ = options; + output_cb_ = media::BindToCurrentLoop(std::move(output_cb)); + std::move(done_cb).Run(Status()); +} + +void VpxVideoEncoder::Encode(scoped_refptr<const VideoFrame> frame, + bool key_frame, + StatusCB done_cb) { + Status status; + done_cb = media::BindToCurrentLoop(std::move(done_cb)); + if (!codec_) { + std::move(done_cb).Run(StatusCode::kEncoderInitializeNeverCompleted); + return; + } + + if (!frame) { + std::move(done_cb).Run(Status(StatusCode::kEncoderFailedEncode, + "No frame provided for encoding.")); + return; + } + if (!frame->IsMappable() || frame->format() != media::PIXEL_FORMAT_I420) { + status = + Status(StatusCode::kEncoderFailedEncode, "Unexpected frame format.") + .WithData("IsMappable", frame->IsMappable()) + .WithData("format", frame->format()); + std::move(done_cb).Run(std::move(status)); + return; + } + + auto size = frame->visible_rect().size(); + auto* img_data = + const_cast<unsigned char*>(frame->data(media::VideoFrame::kYPlane)); + vpx_image_t vpx_image; + if (&vpx_image != vpx_img_wrap(&vpx_image, VPX_IMG_FMT_I420, size.width(), + size.height(), 1 /* align */, img_data)) { + std::move(done_cb).Run(StatusCode::kEncoderFailedEncode); + return; + } + + vpx_image.planes[VPX_PLANE_Y] = const_cast<unsigned char*>( + frame->visible_data(media::VideoFrame::kYPlane)); + vpx_image.planes[VPX_PLANE_U] = const_cast<unsigned char*>( + frame->visible_data(media::VideoFrame::kUPlane)); + vpx_image.planes[VPX_PLANE_V] = const_cast<unsigned char*>( + frame->visible_data(media::VideoFrame::kVPlane)); + vpx_image.stride[VPX_PLANE_Y] = frame->stride(media::VideoFrame::kYPlane); + vpx_image.stride[VPX_PLANE_U] = frame->stride(media::VideoFrame::kUPlane); + vpx_image.stride[VPX_PLANE_V] = frame->stride(media::VideoFrame::kVPlane); + + auto timestamp = frame->timestamp().InMicroseconds(); + auto duration = GetFrameDuration(*frame); + auto deadline = VPX_DL_REALTIME; + vpx_codec_flags_t flags = key_frame ? VPX_EFLAG_FORCE_KF : 0; + auto vpx_error = vpx_codec_encode(codec_, &vpx_image, timestamp, duration, + flags, deadline); + + if (vpx_error != VPX_CODEC_OK) { + std::string msg = base::StringPrintf("VPX encoding error: %s (%s)", + vpx_codec_err_to_string(vpx_error), + vpx_codec_error_detail(codec_)); + status = Status(StatusCode::kEncoderFailedEncode, msg) + .WithData("vpx_error", vpx_error); + std::move(done_cb).Run(std::move(status)); + return; + } + + DrainOutputs(); + std::move(done_cb).Run(Status()); +} + +void VpxVideoEncoder::ChangeOptions(const Options& options, StatusCB done_cb) { + done_cb = media::BindToCurrentLoop(std::move(done_cb)); + if (!codec_) { + std::move(done_cb).Run(StatusCode::kEncoderInitializeNeverCompleted); + return; + } + + vpx_codec_enc_cfg_t new_config = codec_config_; + auto status = SetUpVpxConfig(options, &new_config); + if (status.is_ok()) { + auto vpx_error = vpx_codec_enc_config_set(codec_, &new_config); + if (vpx_error == VPX_CODEC_OK) { + codec_config_ = new_config; + options_ = options; + } else { + status = Status(StatusCode::kEncoderUnsupportedConfig, + "Failed to set new VPX config") + .WithData("vpx_error", vpx_error); + } + } + + std::move(done_cb).Run(std::move(status)); + return; +} + +uint64_t VpxVideoEncoder::GetFrameDuration(const VideoFrame& frame) { + base::TimeDelta result; + if (!frame.metadata()->GetTimeDelta(media::VideoFrameMetadata::FRAME_DURATION, + &result)) { + result = base::TimeDelta::FromSecondsD(1.0 / options_.framerate); + } + return result.InMicroseconds(); +} + +VpxVideoEncoder::~VpxVideoEncoder() { + if (!codec_) + return; + + auto error = vpx_codec_destroy(codec_); + DCHECK_EQ(error, VPX_CODEC_OK); + delete codec_; +} + +void VpxVideoEncoder::Flush(StatusCB done_cb) { + done_cb = media::BindToCurrentLoop(std::move(done_cb)); + if (!codec_) { + std::move(done_cb).Run(StatusCode::kEncoderInitializeNeverCompleted); + return; + } + + auto vpx_error = vpx_codec_encode(codec_, nullptr, -1, 0, 0, 0); + if (vpx_error != VPX_CODEC_OK) { + std::string msg = base::StringPrintf("VPX flushing error: %s (%s)", + vpx_codec_err_to_string(vpx_error), + vpx_codec_error_detail(codec_)); + Status status = Status(StatusCode::kEncoderFailedEncode, msg) + .WithData("vpx_error", vpx_error); + std::move(done_cb).Run(std::move(status)); + return; + } + DrainOutputs(); + std::move(done_cb).Run(Status()); +} + +void VpxVideoEncoder::DrainOutputs() { + vpx_codec_iter_t iter = nullptr; + const vpx_codec_cx_pkt_t* pkt = nullptr; + while ((pkt = vpx_codec_get_cx_data(codec_, &iter))) { + if (pkt->kind == VPX_CODEC_CX_FRAME_PKT) { + VideoEncoderOutput result; + result.key_frame = (pkt->data.frame.flags & VPX_FRAME_IS_KEY) != 0; + result.timestamp = base::TimeDelta::FromMicroseconds(pkt->data.frame.pts); + result.size = pkt->data.frame.sz; + result.data.reset(new uint8_t[result.size]); + memcpy(result.data.get(), pkt->data.frame.buf, result.size); + output_cb_.Run(std::move(result)); + } + } +} + +} // namespace media diff --git a/chromium/media/video/vpx_video_encoder.h b/chromium/media/video/vpx_video_encoder.h new file mode 100644 index 00000000000..4b6c9ff763e --- /dev/null +++ b/chromium/media/video/vpx_video_encoder.h @@ -0,0 +1,45 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_VIDEO_VPX_VIDEO_ENCODER_H_ +#define MEDIA_VIDEO_VPX_VIDEO_ENCODER_H_ + +#include <memory> + +#include "base/callback_forward.h" +#include "media/base/media_export.h" +#include "media/base/video_encoder.h" +#include "third_party/libvpx/source/libvpx/vpx/vpx_encoder.h" +#include "ui/gfx/geometry/size.h" + +namespace media { + +class MEDIA_EXPORT VpxVideoEncoder : public VideoEncoder { + public: + VpxVideoEncoder(); + ~VpxVideoEncoder() override; + + // VideoDecoder implementation. + void Initialize(VideoCodecProfile profile, + const Options& options, + OutputCB output_cb, + StatusCB done_cb) override; + void Encode(scoped_refptr<const VideoFrame> frame, + bool key_frame, + StatusCB done_cb) override; + void ChangeOptions(const Options& options, StatusCB done_cb) override; + void Flush(StatusCB done_cb) override; + + private: + uint64_t GetFrameDuration(const VideoFrame& frame); + void DrainOutputs(); + + vpx_codec_ctx_t* codec_ = nullptr; + vpx_codec_enc_cfg_t codec_config_ = {}; + Options options_; + OutputCB output_cb_; +}; + +} // namespace media +#endif // MEDIA_VIDEO_VPX_VIDEO_ENCODER_H_
\ No newline at end of file diff --git a/chromium/media/webcodecs/BUILD.gn b/chromium/media/webcodecs/BUILD.gn new file mode 100644 index 00000000000..325aa27053d --- /dev/null +++ b/chromium/media/webcodecs/BUILD.gn @@ -0,0 +1,39 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//testing/test.gni") + +source_set("webcodecs") { + # Do not expand the visibility here without double-checking with OWNERS, this + # is a roll-up target which is part of the //media component. Most other DEPs + # should be using //media and not directly DEP this roll-up target. + visibility = [ "//media" ] + + sources = [ + "wc_decoder_selector.cc", + "wc_decoder_selector.h", + ] + + public_deps = [ + "//base", + "//media/base", + "//media/filters", + ] + + deps = [] + + configs += [ "//media:subcomponent_config" ] +} + +source_set("unit_tests") { + testonly = true + sources = [ "wc_decoder_selector_unittest.cc" ] + + deps = [ + "//base/test:test_support", + "//media:test_support", + "//testing/gmock", + "//testing/gtest", + ] +} diff --git a/chromium/media/webcodecs/wc_decoder_selector.cc b/chromium/media/webcodecs/wc_decoder_selector.cc new file mode 100644 index 00000000000..73b927dc4cc --- /dev/null +++ b/chromium/media/webcodecs/wc_decoder_selector.cc @@ -0,0 +1,142 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/webcodecs/wc_decoder_selector.h" + +#include "base/bind.h" +#include "base/check_op.h" +#include "base/notreached.h" +#include "base/single_thread_task_runner.h" +#include "media/base/channel_layout.h" +#include "media/base/demuxer_stream.h" +#include "media/filters/decrypting_demuxer_stream.h" + +namespace media { + +// Demuxing isn't part of WebCodecs. This shim allows us to reuse decoder +// selection logic from <video>. +// TODO(chcunningham): Maybe refactor DecoderSelector to separate dependency on +// DemuxerStream. DecoderSelection doesn't conceptually require a Demuxer. The +// tough part is re-working DecryptingDemuxerStream. +template <DemuxerStream::Type StreamType> +class ShimDemuxerStream : public DemuxerStream { + public: + using DecoderConfigType = + typename DecoderStreamTraits<StreamType>::DecoderConfigType; + + ~ShimDemuxerStream() override = default; + + void Read(ReadCB read_cb) override { NOTREACHED(); } + bool IsReadPending() const override { + NOTREACHED(); + return false; + } + + void Configure(DecoderConfigType config); + + AudioDecoderConfig audio_decoder_config() override { + DCHECK_EQ(type(), DemuxerStream::AUDIO); + return audio_decoder_config_; + } + + VideoDecoderConfig video_decoder_config() override { + DCHECK_EQ(type(), DemuxerStream::VIDEO); + return video_decoder_config_; + } + + Type type() const override { return stream_type; } + + bool SupportsConfigChanges() override { + NOTREACHED(); + return true; + } + + private: + static const DemuxerStream::Type stream_type = StreamType; + + AudioDecoderConfig audio_decoder_config_; + VideoDecoderConfig video_decoder_config_; +}; + +template <> +void ShimDemuxerStream<DemuxerStream::AUDIO>::Configure( + DecoderConfigType config) { + audio_decoder_config_ = config; +} + +template <> +void ShimDemuxerStream<DemuxerStream::VIDEO>::Configure( + DecoderConfigType config) { + video_decoder_config_ = config; +} + +template <DemuxerStream::Type StreamType> +WebCodecsDecoderSelector<StreamType>::WebCodecsDecoderSelector( + scoped_refptr<base::SingleThreadTaskRunner> task_runner, + CreateDecodersCB create_decoders_cb, + typename Decoder::OutputCB output_cb) + : impl_(std::move(task_runner), + std::move(create_decoders_cb), + &null_media_log_), + demuxer_stream_(new ShimDemuxerStream<StreamType>()), + stream_traits_(CreateStreamTraits()), + output_cb_(output_cb) { + impl_.Initialize(stream_traits_.get(), demuxer_stream_.get(), + nullptr /*CdmContext*/, WaitingCB()); +} + +template <DemuxerStream::Type StreamType> +WebCodecsDecoderSelector<StreamType>::~WebCodecsDecoderSelector() {} + +template <DemuxerStream::Type StreamType> +void WebCodecsDecoderSelector<StreamType>::SelectDecoder( + const DecoderConfig& config, + SelectDecoderCB select_decoder_cb) { + // |impl_| will internally use this the |config| from our ShimDemuxerStream. + demuxer_stream_->Configure(config); + + // |impl_| uses a WeakFactory for its SelectDecoderCB, so we're safe to use + // Unretained here. + impl_.SelectDecoder( + base::BindOnce(&WebCodecsDecoderSelector<StreamType>::OnDecoderSelected, + base::Unretained(this), std::move(select_decoder_cb)), + output_cb_); +} + +template <> +std::unique_ptr<WebCodecsAudioDecoderSelector::StreamTraits> +WebCodecsDecoderSelector<DemuxerStream::AUDIO>::CreateStreamTraits() { + // TODO(chcunningham): Consider plumbing real hw channel layout. + return std::make_unique<WebCodecsDecoderSelector::StreamTraits>( + &null_media_log_, CHANNEL_LAYOUT_NONE); +} + +template <> +std::unique_ptr<WebCodecsVideoDecoderSelector::StreamTraits> +WebCodecsDecoderSelector<DemuxerStream::VIDEO>::CreateStreamTraits() { + return std::make_unique<WebCodecsDecoderSelector::StreamTraits>( + &null_media_log_); +} + +template <DemuxerStream::Type StreamType> +void WebCodecsDecoderSelector<StreamType>::OnDecoderSelected( + SelectDecoderCB select_decoder_cb, + std::unique_ptr<Decoder> decoder, + std::unique_ptr<DecryptingDemuxerStream> decrypting_demuxer_stream) { + DCHECK(!decrypting_demuxer_stream); + + // We immediately finalize decoder selection. From a spec POV we strongly + // prefer to avoid replicating our internal design of having to wait for the + // first frame to arrive before we consider configuration successful. + // TODO(chcunningham): Measure first frame decode failures and find other ways + // to solve (or minimize) the problem. + impl_.FinalizeDecoderSelection(); + + std::move(select_decoder_cb).Run(std::move(decoder)); +} + +template class WebCodecsDecoderSelector<DemuxerStream::VIDEO>; +template class WebCodecsDecoderSelector<DemuxerStream::AUDIO>; + +} // namespace media diff --git a/chromium/media/webcodecs/wc_decoder_selector.h b/chromium/media/webcodecs/wc_decoder_selector.h new file mode 100644 index 00000000000..207350f1a94 --- /dev/null +++ b/chromium/media/webcodecs/wc_decoder_selector.h @@ -0,0 +1,83 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_WEBCODECS_WC_DECODER_SELECTOR_H_ +#define MEDIA_WEBCODECS_WC_DECODER_SELECTOR_H_ + +#include <memory> + +#include "media/base/demuxer_stream.h" +#include "media/base/media_export.h" +#include "media/base/media_util.h" +#include "media/filters/decoder_selector.h" +#include "media/filters/decoder_stream_traits.h" + +namespace media { + +template <DemuxerStream::Type StreamType> +class ShimDemuxerStream; + +template <DemuxerStream::Type StreamType> +class MEDIA_EXPORT WebCodecsDecoderSelector { + public: + typedef DecoderStreamTraits<StreamType> StreamTraits; + typedef typename StreamTraits::DecoderType Decoder; + typedef typename StreamTraits::DecoderConfigType DecoderConfig; + + // Callback to create a list of decoders to select from. + using CreateDecodersCB = + base::RepeatingCallback<std::vector<std::unique_ptr<Decoder>>()>; + + // Emits the result of a single call to SelectDecoder(). Parameter is + // the initialized Decoder. nullptr if selection failed. The caller owns the + // Decoder. + using SelectDecoderCB = base::OnceCallback<void(std::unique_ptr<Decoder>)>; + + WebCodecsDecoderSelector( + scoped_refptr<base::SingleThreadTaskRunner> task_runner, + CreateDecodersCB create_decoders_cb, + typename Decoder::OutputCB output_cb); + + // Aborts any pending decoder selection. + ~WebCodecsDecoderSelector(); + + // Selects and initializes a decoder using |config|. Decoder will + // be returned via |select_decoder_cb| posted to |task_runner_|. Subsequent + // calls will again select from the full list of decoders. + void SelectDecoder(const DecoderConfig& config, + SelectDecoderCB select_decoder_cb); + + private: + // Helper to create |stream_traits_|. + std::unique_ptr<StreamTraits> CreateStreamTraits(); + + // Proxy SelectDecoderCB from impl_ to our |select_decoder_cb|. + void OnDecoderSelected(SelectDecoderCB select_decoder_cb, + std::unique_ptr<Decoder> decoder, + std::unique_ptr<DecryptingDemuxerStream>); + + // Implements heavy lifting for decoder selection. + DecoderSelector<StreamType> impl_; + + // Shim to satisfy dependencies of |impl_|. Provides DecoderConfig to |impl_|. + std::unique_ptr<ShimDemuxerStream<StreamType>> demuxer_stream_; + + // Helper to unify API for configuring audio/video decoders. + std::unique_ptr<StreamTraits> stream_traits_; + + // Repeating callback for decoder outputs. + typename Decoder::OutputCB output_cb_; + + // TODO(chcunningham): Route MEDIA_LOG for WebCodecs. + NullMediaLog null_media_log_; +}; + +typedef WebCodecsDecoderSelector<DemuxerStream::VIDEO> + WebCodecsVideoDecoderSelector; +typedef WebCodecsDecoderSelector<DemuxerStream::AUDIO> + WebCodecsAudioDecoderSelector; + +} // namespace media + +#endif // MEDIA_WEBCODECS_WC_DECODER_SELECTOR_H_ diff --git a/chromium/media/webcodecs/wc_decoder_selector_unittest.cc b/chromium/media/webcodecs/wc_decoder_selector_unittest.cc new file mode 100644 index 00000000000..a14258468ee --- /dev/null +++ b/chromium/media/webcodecs/wc_decoder_selector_unittest.cc @@ -0,0 +1,240 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <vector> + +#include "base/test/task_environment.h" +#include "media/base/demuxer_stream.h" +#include "media/base/media_util.h" +#include "media/base/mock_filters.h" +#include "media/base/status.h" +#include "media/base/test_helpers.h" +#include "media/base/video_decoder.h" +#include "media/filters/decoder_stream.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +#include "media/webcodecs/wc_decoder_selector.h" + +using ::testing::_; +using ::testing::IsNull; +using ::testing::StrictMock; + +namespace media { + +namespace { + +enum DecoderCapability { + kFail, + kSucceed, +}; + +const char kNoDecoder[] = ""; +const char kDecoder1[] = "Decoder1"; +const char kDecoder2[] = "Decoder2"; + +// Specializations for the AUDIO version of the test. +class AudioDecoderSelectorTestParam { + public: + static constexpr DemuxerStream::Type kStreamType = DemuxerStream::AUDIO; + + using DecoderSelector = WebCodecsDecoderSelector<DemuxerStream::AUDIO>; + using MockDecoder = MockAudioDecoder; + using Output = AudioBuffer; + + static AudioDecoderConfig CreateConfig() { return TestAudioConfig::Normal(); } + + // Create a config that won't match the return of CreateConfig(). + static AudioDecoderConfig CreateAlternateConfig() { + return TestAudioConfig::NormalEncrypted(); + } + + // Decoder::Initialize() takes different parameters depending on the type. + static void ExpectInitialize(MockDecoder* decoder, + DecoderCapability capability, + AudioDecoderConfig expected_config) { + EXPECT_CALL(*decoder, Initialize_(_, _, _, _, _)) + .WillRepeatedly([capability, expected_config]( + const AudioDecoderConfig& config, CdmContext*, + AudioDecoder::InitCB& init_cb, + const AudioDecoder::OutputCB&, const WaitingCB&) { + EXPECT_TRUE(config.Matches(expected_config)); + std::move(init_cb).Run(capability == kSucceed + ? OkStatus() + : StatusCode::kCodeOnlyForTesting); + }); + } +}; + +// Specializations for the VIDEO version of the test. +class VideoDecoderSelectorTestParam { + public: + static constexpr DemuxerStream::Type kStreamType = DemuxerStream::VIDEO; + + using DecoderSelector = WebCodecsDecoderSelector<DemuxerStream::VIDEO>; + using MockDecoder = MockVideoDecoder; + using Output = VideoFrame; + + static VideoDecoderConfig CreateConfig() { return TestVideoConfig::Normal(); } + + // Create a config that won't match the return of CreateConfig(). + static VideoDecoderConfig CreateAlternateConfig() { + return TestVideoConfig::LargeEncrypted(); + } + + static void ExpectInitialize(MockDecoder* decoder, + DecoderCapability capability, + VideoDecoderConfig expected_config) { + EXPECT_CALL(*decoder, Initialize_(_, _, _, _, _, _)) + .WillRepeatedly([capability, expected_config]( + const VideoDecoderConfig& config, bool low_delay, + CdmContext*, VideoDecoder::InitCB& init_cb, + const VideoDecoder::OutputCB&, const WaitingCB&) { + EXPECT_TRUE(config.Matches(expected_config)); + std::move(init_cb).Run(capability == kSucceed + ? OkStatus() + : StatusCode::kCodeOnlyForTesting); + }); + } +}; + +// Allocate storage for the member variables. +constexpr DemuxerStream::Type AudioDecoderSelectorTestParam::kStreamType; +constexpr DemuxerStream::Type VideoDecoderSelectorTestParam::kStreamType; + +} // namespace + +// Note: The parameter is called TypeParam in the test cases regardless of what +// we call it here. It's been named the same for convenience. +// Note: The test fixtures inherit from this class. Inside the test cases the +// test fixture class is called TestFixture. +template <typename TypeParam> +class WebCodecsDecoderSelectorTest : public ::testing::Test { + public: + // Convenience aliases. + using Self = WebCodecsDecoderSelectorTest<TypeParam>; + using Decoder = typename TypeParam::DecoderSelector::Decoder; + using DecoderConfig = typename TypeParam::DecoderSelector::DecoderConfig; + using MockDecoder = typename TypeParam::MockDecoder; + using Output = typename TypeParam::Output; + + WebCodecsDecoderSelectorTest() { CreateDecoderSelector(); } + + void OnOutput(scoped_refptr<Output> output) { NOTREACHED(); } + + MOCK_METHOD1_T(OnDecoderSelected, void(std::string)); + + void OnDecoderSelectedThunk(std::unique_ptr<Decoder> decoder) { + // Report only the name of the decoder, since that's what the tests care + // about. The decoder will be destructed immediately. + OnDecoderSelected(decoder ? decoder->GetDisplayName() : kNoDecoder); + } + + void AddMockDecoder(const std::string& decoder_name, + DecoderCapability capability) { + // Actual decoders are created in CreateDecoders(), which may be called + // multiple times by the DecoderSelector. + mock_decoders_to_create_.emplace_back(decoder_name, capability); + } + + std::vector<std::unique_ptr<Decoder>> CreateDecoders() { + std::vector<std::unique_ptr<Decoder>> decoders; + + for (const auto& info : mock_decoders_to_create_) { + std::unique_ptr<StrictMock<MockDecoder>> decoder = + std::make_unique<StrictMock<MockDecoder>>(info.first); + TypeParam::ExpectInitialize(decoder.get(), info.second, + last_set_decoder_config_); + decoders.push_back(std::move(decoder)); + } + + return decoders; + } + + void CreateDecoderSelector() { + decoder_selector_ = + std::make_unique<WebCodecsDecoderSelector<TypeParam::kStreamType>>( + task_environment_.GetMainThreadTaskRunner(), + base::BindRepeating(&Self::CreateDecoders, base::Unretained(this)), + base::BindRepeating(&Self::OnOutput, base::Unretained(this))); + } + + void SelectDecoder(DecoderConfig config = TypeParam::CreateConfig()) { + last_set_decoder_config_ = config; + decoder_selector_->SelectDecoder( + config, + base::BindOnce(&Self::OnDecoderSelectedThunk, base::Unretained(this))); + RunUntilIdle(); + } + + void RunUntilIdle() { task_environment_.RunUntilIdle(); } + + base::test::TaskEnvironment task_environment_; + NullMediaLog media_log_; + + DecoderConfig last_set_decoder_config_; + + std::unique_ptr<WebCodecsDecoderSelector<TypeParam::kStreamType>> + decoder_selector_; + + std::vector<std::pair<std::string, DecoderCapability>> + mock_decoders_to_create_; + + private: + DISALLOW_COPY_AND_ASSIGN(WebCodecsDecoderSelectorTest); +}; + +using WebCodecsDecoderSelectorTestParams = + ::testing::Types<AudioDecoderSelectorTestParam, + VideoDecoderSelectorTestParam>; +TYPED_TEST_SUITE(WebCodecsDecoderSelectorTest, + WebCodecsDecoderSelectorTestParams); + +TYPED_TEST(WebCodecsDecoderSelectorTest, NoDecoders) { + EXPECT_CALL(*this, OnDecoderSelected(kNoDecoder)); + this->SelectDecoder(); +} + +TYPED_TEST(WebCodecsDecoderSelectorTest, OneDecoder) { + this->AddMockDecoder(kDecoder1, kSucceed); + + EXPECT_CALL(*this, OnDecoderSelected(kDecoder1)); + this->SelectDecoder(); +} + +TYPED_TEST(WebCodecsDecoderSelectorTest, TwoDecoders) { + this->AddMockDecoder(kDecoder1, kFail); + this->AddMockDecoder(kDecoder2, kSucceed); + + EXPECT_CALL(*this, OnDecoderSelected(kDecoder2)); + this->SelectDecoder(); +} + +TYPED_TEST(WebCodecsDecoderSelectorTest, TwoDecoders_SelectAgain) { + this->AddMockDecoder(kDecoder1, kSucceed); + this->AddMockDecoder(kDecoder2, kSucceed); + + EXPECT_CALL(*this, OnDecoderSelected(kDecoder1)); + this->SelectDecoder(); + + // Selecting again should give (a new instance of) the same decoder. + EXPECT_CALL(*this, OnDecoderSelected(kDecoder1)); + this->SelectDecoder(); +} + +TYPED_TEST(WebCodecsDecoderSelectorTest, TwoDecoders_NewConfigSelectAgain) { + this->AddMockDecoder(kDecoder1, kSucceed); + this->AddMockDecoder(kDecoder2, kSucceed); + + EXPECT_CALL(*this, OnDecoderSelected(kDecoder1)); + this->SelectDecoder(TypeParam::CreateConfig()); + + // Selecting again should give (a new instance of) the same decoder. + EXPECT_CALL(*this, OnDecoderSelected(kDecoder1)); + // Select again with a different config. Expected config verified during + // CreateDecoders() the SelectDecoder() call. + this->SelectDecoder(TypeParam::CreateAlternateConfig()); +} + +} // namespace media diff --git a/chromium/media/webrtc/BUILD.gn b/chromium/media/webrtc/BUILD.gn index 8ba246a9f12..61bec7b7dd8 100644 --- a/chromium/media/webrtc/BUILD.gn +++ b/chromium/media/webrtc/BUILD.gn @@ -3,15 +3,8 @@ # found in the LICENSE file. import("//media/media_options.gni") -import("//media/webrtc/audio_processing.gni") import("//third_party/webrtc/webrtc.gni") -config("audio_processing_build_flag") { - if (audio_processing_in_audio_service_supported) { - defines = [ "AUDIO_PROCESSING_IN_AUDIO_SERVICE" ] - } -} - component("webrtc") { output_name = "media_webrtc" @@ -31,33 +24,8 @@ component("webrtc") { "//media:shared_memory_support", "//third_party/webrtc_overrides:webrtc_component", ] - - if (audio_processing_in_audio_service_supported) { - # Only build this on platforms where it's supported and used. - sources += [ - "audio_processor.cc", - "audio_processor.h", - "audio_processor_controls.h", - ] - - deps += [ "//media" ] - - public_configs = [ ":audio_processing_build_flag" ] - } } source_set("unit_tests") { testonly = true - if (audio_processing_in_audio_service_supported) { - deps = [ - "//base", - "//base/test:test_support", - "//media:test_support", - "//media/webrtc", - "//testing/gmock", - "//testing/gtest", - "//third_party/webrtc_overrides:webrtc_component", - ] - sources = [ "audio_processor_unittest.cc" ] - } } diff --git a/chromium/media/webrtc/audio_processing.gni b/chromium/media/webrtc/audio_processing.gni deleted file mode 100644 index 8645de00541..00000000000 --- a/chromium/media/webrtc/audio_processing.gni +++ /dev/null @@ -1,4 +0,0 @@ -declare_args() { - # Note: the audio service must be sandboxed for us to use the APM there. - audio_processing_in_audio_service_supported = is_linux || is_win || is_mac -} diff --git a/chromium/media/webrtc/audio_processor.cc b/chromium/media/webrtc/audio_processor.cc deleted file mode 100644 index 0e7b791aa5b..00000000000 --- a/chromium/media/webrtc/audio_processor.cc +++ /dev/null @@ -1,361 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/webrtc/audio_processor.h" - -#include <array> -#include <utility> - -#include "base/bind.h" -#include "base/command_line.h" -#include "base/feature_list.h" -#include "base/files/file_util.h" -#include "base/memory/ptr_util.h" -#include "base/metrics/field_trial.h" -#include "base/metrics/field_trial_params.h" -#include "base/metrics/histogram_macros.h" -#include "base/strings/string_number_conversions.h" -#include "base/task/post_task.h" -#include "base/task/task_traits.h" -#include "base/task/thread_pool.h" -#include "media/base/limits.h" -#include "media/webrtc/helpers.h" -#include "media/webrtc/webrtc_switches.h" -#include "third_party/webrtc/api/audio/echo_canceller3_factory.h" -#include "third_party/webrtc/modules/audio_processing/aec_dump/aec_dump_factory.h" -#include "third_party/webrtc_overrides/task_queue_factory.h" - -namespace media { - -namespace { - -bool UseMultiChannelCaptureProcessing() { - return base::FeatureList::IsEnabled( - features::kWebRtcEnableCaptureMultiChannelApm); -} - -constexpr int kBuffersPerSecond = 100; // 10 ms per buffer. - -} // namespace - -AudioProcessor::ProcessingResult::ProcessingResult( - const AudioBus& audio, - base::Optional<double> new_volume) - : audio(audio), new_volume(new_volume) {} -AudioProcessor::ProcessingResult::ProcessingResult(const ProcessingResult& b) = - default; -AudioProcessor::ProcessingResult::~ProcessingResult() = default; - -AudioProcessor::AudioProcessor(const AudioParameters& audio_parameters, - const AudioProcessingSettings& settings) - : audio_parameters_(audio_parameters), - settings_(settings), - output_bus_(AudioBus::Create(audio_parameters_)), - audio_delay_stats_reporter_(kBuffersPerSecond), - use_capture_multi_channel_processing_( - UseMultiChannelCaptureProcessing()) { - DCHECK(audio_parameters.IsValid()); - DCHECK_EQ(audio_parameters_.GetBufferDuration(), - base::TimeDelta::FromMilliseconds(10)); - InitializeAPM(); - output_ptrs_.reserve(audio_parameters_.channels()); - for (int i = 0; i < audio_parameters_.channels(); ++i) { - output_ptrs_.push_back(output_bus_->channel(i)); - } -} - -AudioProcessor::~AudioProcessor() { - StopEchoCancellationDump(); -} - -// Process the audio from source and return a pointer to the processed data. -AudioProcessor::ProcessingResult AudioProcessor::ProcessCapture( - const AudioBus& source, - base::TimeTicks capture_time, - double volume, - bool key_pressed) { - base::Optional<double> new_volume; - - if (audio_processing_) { - UpdateDelayEstimate(capture_time); - UpdateAnalogLevel(volume); - audio_processing_->set_stream_key_pressed(key_pressed); - - // Writes to |output_bus_|. - FeedDataToAPM(source); - - UpdateTypingDetected(key_pressed); - new_volume = GetNewVolumeFromAGC(volume); - } else { - source.CopyTo(output_bus_.get()); - } - - if (settings_.stereo_mirroring && - audio_parameters_.channel_layout() == CHANNEL_LAYOUT_STEREO) { - output_bus_->SwapChannels(0, 1); - } - - return {*output_bus_, new_volume}; -} - -void AudioProcessor::AnalyzePlayout(const AudioBus& audio, - const AudioParameters& parameters, - base::TimeTicks playout_time) { - if (!audio_processing_) - return; - - render_delay_ = playout_time - base::TimeTicks::Now(); - - DCHECK_GE(parameters.channels(), 1); - DCHECK_LE(parameters.channels(), audio.channels()); - DCHECK_LE(parameters.channels(), media::limits::kMaxChannels); - webrtc::StreamConfig input_stream_config = CreateStreamConfig(parameters); - // If the input audio appears to contain upmixed mono audio, then APM is only - // given the left channel. This reduces computational complexity and improves - // convergence of audio processing algorithms. - // TODO(crbug.com/1023337): Ensure correct channel count in input audio bus. - assume_upmixed_mono_playout_ = - assume_upmixed_mono_playout_ && LeftAndRightChannelsAreSymmetric(audio); - if (assume_upmixed_mono_playout_) { - input_stream_config.set_num_channels(1); - } - - std::array<const float*, media::limits::kMaxChannels> input_ptrs; - for (int i = 0; i < static_cast<int>(input_stream_config.num_channels()); - ++i) { - input_ptrs[i] = audio.channel(i); - } - - const int apm_error = audio_processing_->AnalyzeReverseStream( - input_ptrs.data(), input_stream_config); - - DCHECK_EQ(apm_error, webrtc::AudioProcessing::kNoError); -} - -void AudioProcessor::GetStats(GetStatsCB callback) { - webrtc::AudioProcessorInterface::AudioProcessorStatistics out = {}; - if (audio_processing_) { - out.typing_noise_detected = typing_detected_; - out.apm_statistics = audio_processing_->GetStatistics(has_reverse_stream_); - } - std::move(callback).Run(out); -} - -void AudioProcessor::StartEchoCancellationDump(base::File file) { - if (!audio_processing_) { - // The destructor of File is blocking. Post it to a task runner to avoid - // blocking the main thread. - base::ThreadPool::PostTask( - FROM_HERE, {base::TaskPriority::LOWEST, base::MayBlock()}, - base::BindOnce([](base::File) {}, std::move(file))); - return; - } - - // Here tasks will be posted on the |worker_queue_|. It must be kept alive - // until StopEchoCancellationDump is called or the webrtc::AudioProcessing - // instance is destroyed. - - DCHECK(file.IsValid()); - - if (!worker_queue_) { - worker_queue_ = std::make_unique<rtc::TaskQueue>( - CreateWebRtcTaskQueue(rtc::TaskQueue::Priority::LOW)); - } - auto aec_dump = webrtc::AecDumpFactory::Create( - FileToFILE(std::move(file), "wb"), -1 /* max_log_size_bytes */, - worker_queue_.get()); - if (!aec_dump) { - // AecDumpFactory::Create takes ownership of stream even if it fails, so we - // don't need to close it. - LOG(ERROR) << "Failed to start AEC debug recording"; - return; - } - audio_processing_->AttachAecDump(std::move(aec_dump)); -} - -void AudioProcessor::StopEchoCancellationDump() { - if (audio_processing_) - audio_processing_->DetachAecDump(); - // Note that deleting an rtc::TaskQueue has to be done from the - // thread that created it. - worker_queue_.reset(); -} - -void AudioProcessor::InitializeAPM() { - // Most features must have some configuration applied before constructing the - // APM, and some after; the initialization is divided into "part 1" and "part - // 2" in those cases. - - // If we use nothing but, possibly, audio mirroring, don't initialize the APM. - if (settings_.echo_cancellation != EchoCancellationType::kAec3 && - settings_.noise_suppression == NoiseSuppressionType::kDisabled && - settings_.automatic_gain_control == AutomaticGainControlType::kDisabled && - !settings_.high_pass_filter && !settings_.typing_detection) { - return; - } - - webrtc::Config ap_config; // Note: ap_config.Set(x) transfers ownership of x. - webrtc::AudioProcessingBuilder ap_builder; - - // AEC setup part 1. - - // Echo cancellation is configured both before and after AudioProcessing - // construction, but before Initialize. - if (settings_.echo_cancellation == EchoCancellationType::kAec3) { - ap_builder.SetEchoControlFactory( - std::make_unique<webrtc::EchoCanceller3Factory>()); - } - - // AGC setup part 1. - if (settings_.automatic_gain_control == - AutomaticGainControlType::kExperimental || - settings_.automatic_gain_control == - AutomaticGainControlType::kHybridExperimental) { - std::string min_volume_str( - base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII( - switches::kAgcStartupMinVolume)); - int startup_min_volume; - // |startup_min_volume| set to 0 in case of failure/empty string, which is - // the default. - base::StringToInt(min_volume_str, &startup_min_volume); - auto* experimental_agc = - new webrtc::ExperimentalAgc(true, startup_min_volume); - experimental_agc->digital_adaptive_disabled = - settings_.automatic_gain_control == - AutomaticGainControlType::kHybridExperimental; - ap_config.Set<webrtc::ExperimentalAgc>(experimental_agc); - } else { - ap_config.Set<webrtc::ExperimentalAgc>(new webrtc::ExperimentalAgc(false)); - } - - // Noise suppression setup part 1. - ap_config.Set<webrtc::ExperimentalNs>(new webrtc::ExperimentalNs( - settings_.noise_suppression == NoiseSuppressionType::kExperimental)); - - // Audio processing module construction. - audio_processing_ = base::WrapUnique(ap_builder.Create(ap_config)); - - webrtc::AudioProcessing::Config apm_config = audio_processing_->GetConfig(); - - // APM audio pipeline setup. - apm_config.pipeline.multi_channel_render = true; - apm_config.pipeline.multi_channel_capture = - use_capture_multi_channel_processing_; - - // Typing detection setup. - if (settings_.typing_detection) { - typing_detector_ = std::make_unique<webrtc::TypingDetection>(); - // Configure the update period to 1s (100 * 10ms) in the typing detector. - typing_detector_->SetParameters(0, 0, 0, 0, 0, 100); - - apm_config.voice_detection.enabled = true; - } - - // AEC setup part 2. - apm_config.echo_canceller.enabled = - settings_.echo_cancellation == EchoCancellationType::kAec3; - apm_config.echo_canceller.mobile_mode = false; - - // High-pass filter setup. - apm_config.high_pass_filter.enabled = settings_.high_pass_filter; - - // Noise suppression setup part 2. - apm_config.noise_suppression.enabled = - settings_.noise_suppression != NoiseSuppressionType::kDisabled; - apm_config.noise_suppression.level = - webrtc::AudioProcessing::Config::NoiseSuppression::kHigh; - - // AGC setup part 2. - apm_config.gain_controller1.enabled = - settings_.automatic_gain_control != AutomaticGainControlType::kDisabled; - apm_config.gain_controller1.mode = - webrtc::AudioProcessing::Config::GainController1::kAdaptiveAnalog; - if (settings_.automatic_gain_control == - AutomaticGainControlType::kExperimental || - settings_.automatic_gain_control == - AutomaticGainControlType::kHybridExperimental) { - apm_config.gain_controller2.enabled = - settings_.automatic_gain_control == - AutomaticGainControlType::kHybridExperimental; - apm_config.gain_controller2.fixed_digital.gain_db = 0.f; - apm_config.gain_controller2.adaptive_digital.enabled = true; - const bool use_peaks_not_rms = base::GetFieldTrialParamByFeatureAsBool( - features::kWebRtcHybridAgc, "use_peaks_not_rms", false); - using Shortcut = - webrtc::AudioProcessing::Config::GainController2::LevelEstimator; - apm_config.gain_controller2.adaptive_digital.level_estimator = - use_peaks_not_rms ? Shortcut::kPeak : Shortcut::kRms; - - const int saturation_margin = base::GetFieldTrialParamByFeatureAsInt( - features::kWebRtcHybridAgc, "saturation_margin", -1); - if (saturation_margin != -1) { - apm_config.gain_controller2.adaptive_digital.extra_saturation_margin_db = - saturation_margin; - } - } - audio_processing_->ApplyConfig(apm_config); -} - -void AudioProcessor::UpdateDelayEstimate(base::TimeTicks capture_time) { - const base::TimeDelta capture_delay = base::TimeTicks::Now() - capture_time; - // Note: this delay calculation doesn't make sense, but it's how it's done - // right now. Instead, the APM should probably attach the playout time to each - // reference buffer it gets and store that internally? - const base::TimeDelta render_delay = render_delay_.load(); - - audio_delay_stats_reporter_.ReportDelay(capture_delay, render_delay); - - const base::TimeDelta total_delay = capture_delay + render_delay; - audio_processing_->set_stream_delay_ms(total_delay.InMilliseconds()); - if (total_delay > base::TimeDelta::FromMilliseconds(300)) { - DLOG(WARNING) << "Large audio delay, capture delay: " - << capture_delay.InMilliseconds() << "ms; render delay: " - << render_delay_.load().InMilliseconds() << "ms"; - } -} - -void AudioProcessor::UpdateAnalogLevel(double volume) { - DCHECK_LE(volume, 1.0); - constexpr double kWebRtcMaxVolume = 255; - const int webrtc_volume = volume * kWebRtcMaxVolume; - audio_processing_->set_stream_analog_level(webrtc_volume); -} - -void AudioProcessor::FeedDataToAPM(const AudioBus& source) { - DCHECK_LE(source.channels(), media::limits::kMaxChannels); - std::array<const float*, media::limits::kMaxChannels> input_ptrs; - for (int i = 0; i < source.channels(); ++i) { - input_ptrs[i] = source.channel(i); - } - - const webrtc::StreamConfig config = CreateStreamConfig(audio_parameters_); - int err = audio_processing_->ProcessStream(input_ptrs.data(), config, config, - output_ptrs_.data()); - DCHECK_EQ(err, 0) << "ProcessStream() error: " << err; -} - -void AudioProcessor::UpdateTypingDetected(bool key_pressed) { - if (typing_detector_) { - // Ignore remote tracks to avoid unnecessary stats computation. - auto voice_detected = - audio_processing_->GetStatistics(false /* has_remote_tracks */) - .voice_detected; - DCHECK(voice_detected.has_value()); - typing_detected_ = typing_detector_->Process(key_pressed, *voice_detected); - } -} - -base::Optional<double> AudioProcessor::GetNewVolumeFromAGC(double volume) { - constexpr double kWebRtcMaxVolume = 255; - const int webrtc_volume = volume * kWebRtcMaxVolume; - const int new_webrtc_volume = - audio_processing_->recommended_stream_analog_level(); - - return new_webrtc_volume == webrtc_volume - ? base::nullopt - : base::Optional<double>(static_cast<double>(new_webrtc_volume) / - kWebRtcMaxVolume); -} - -} // namespace media diff --git a/chromium/media/webrtc/audio_processor.h b/chromium/media/webrtc/audio_processor.h deleted file mode 100644 index babf659ae62..00000000000 --- a/chromium/media/webrtc/audio_processor.h +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MEDIA_WEBRTC_AUDIO_PROCESSOR_H_ -#define MEDIA_WEBRTC_AUDIO_PROCESSOR_H_ - -#include <atomic> -#include <memory> -#include <string> -#include <vector> - -#include "base/component_export.h" -#include "base/macros.h" -#include "base/optional.h" -#include "base/time/time.h" -#include "media/audio/audio_io.h" -#include "media/base/audio_parameters.h" -#include "media/base/audio_processing.h" -#include "media/webrtc/audio_delay_stats_reporter.h" -#include "media/webrtc/audio_processor_controls.h" -#include "third_party/webrtc/modules/audio_processing/include/audio_processing.h" -#include "third_party/webrtc/modules/audio_processing/typing_detection.h" -#include "third_party/webrtc/rtc_base/task_queue.h" - -namespace media { - -// The AudioProcessor wraps the WebRTC AudioProcessingModule. It also provides -// stereo channel mirroring. For now, it will only run in the audio service when -// run outside the browser process. As such, it does not support Android -// specific configuration, like enabling mobile mode. -class COMPONENT_EXPORT(MEDIA_WEBRTC) AudioProcessor final - : public AudioProcessorControls { - public: - // |audio_parameters| describe the format to use, and |settings| what - // processing to perform. The WebRTC APM will resample internally if - // necessary. AudioProcessor uses the same format for input and output. Audio - // must be provided in 10ms chunks. - AudioProcessor(const AudioParameters& audio_parameters, - const AudioProcessingSettings& settings); - - ~AudioProcessor() final; - - // The result of a call to ProcessCapture. |audio| is allocated and owned by - // the AudioProcessor. It is valid until the next call to ProcessCapture or - // until destruction of the AudioProcessor. - struct COMPONENT_EXPORT(MEDIA_WEBRTC) ProcessingResult { - ProcessingResult(const AudioBus& audio, base::Optional<double> new_volume); - ProcessingResult(const ProcessingResult& b); - ~ProcessingResult(); - - const AudioBus& audio; - base::Optional<double> new_volume; - }; - - ProcessingResult ProcessCapture(const AudioBus& source, - base::TimeTicks capture_time, - double volume, - bool key_pressed); - - void AnalyzePlayout(const AudioBus& audio, - const AudioParameters& parameters, - base::TimeTicks playout_time); - - void set_has_reverse_stream(bool has_reverse_stream) { - has_reverse_stream_ = has_reverse_stream; - } - - // AudioProcessorControls implementation. - void GetStats(GetStatsCB callback) override; - void StartEchoCancellationDump(base::File file) override; - void StopEchoCancellationDump() override; - - private: - friend class WebRtcAudioProcessorTest; - - void InitializeAPM(); - - // These functions are all part of ProcessCapture and assume that - // |audio_processing_| is set. - void UpdateDelayEstimate(base::TimeTicks capture_time); - void UpdateAnalogLevel(double volume); - void FeedDataToAPM(const AudioBus& source); - void UpdateTypingDetected(bool key_pressed); - base::Optional<double> GetNewVolumeFromAGC(double volume); - - const AudioParameters audio_parameters_; - const AudioProcessingSettings settings_; - - // State related to interaction with APM. - std::unique_ptr<webrtc::AudioProcessing> audio_processing_; - std::unique_ptr<webrtc::TypingDetection> typing_detector_; - std::atomic<bool> typing_detected_ = {false}; - std::atomic<base::TimeDelta> render_delay_ = {base::TimeDelta()}; - bool has_reverse_stream_ = false; - - // Indicates whether the audio processor playout signal has ever had - // asymmetric left and right channel content. - bool assume_upmixed_mono_playout_ = true; - - // The APM writes the processed data here. - std::unique_ptr<AudioBus> output_bus_; - std::vector<float*> output_ptrs_; - - // For reporting audio delay stats. - AudioDelayStatsReporter audio_delay_stats_reporter_; - - // Low-priority task queue for doing AEC dump recordings. It has to - // out-live |audio_processing_| and be created/destroyed from the same - // thread. - std::unique_ptr<rtc::TaskQueue> worker_queue_; - - // Flag indicating whether capture multi channel processing should be active. - const bool use_capture_multi_channel_processing_; - - DISALLOW_COPY_AND_ASSIGN(AudioProcessor); -}; - -} // namespace media - -#endif // MEDIA_WEBRTC_AUDIO_PROCESSOR_H_ diff --git a/chromium/media/webrtc/audio_processor_unittest.cc b/chromium/media/webrtc/audio_processor_unittest.cc deleted file mode 100644 index c8663af8a10..00000000000 --- a/chromium/media/webrtc/audio_processor_unittest.cc +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include <stddef.h> -#include <stdint.h> - -#include <string> -#include <vector> - -#include "base/files/file_path.h" -#include "base/files/file_util.h" -#include "base/files/scoped_temp_dir.h" -#include "base/logging.h" -#include "base/macros.h" -#include "base/memory/aligned_memory.h" -#include "base/path_service.h" -#include "base/stl_util.h" -#include "base/test/task_environment.h" -#include "base/threading/thread_task_runner_handle.h" -#include "base/time/time.h" -#include "build/build_config.h" -#include "media/base/audio_bus.h" -#include "media/base/audio_parameters.h" -#include "media/webrtc/audio_processor.h" -#include "testing/gmock/include/gmock/gmock.h" -#include "testing/gtest/include/gtest/gtest.h" - -using ::testing::_; -using ::testing::AnyNumber; -using ::testing::AtLeast; -using ::testing::Return; - -namespace media { - -namespace { - -const int kAudioProcessingSampleRate = 48000; -const int kAudioProcessingNumberOfChannels = 1; - -// The number of packers used for testing. -const int kNumberOfPacketsForTest = 100; - -void ReadDataFromSpeechFile(char* data, int length) { - base::FilePath file; - CHECK(base::PathService::Get(base::DIR_SOURCE_ROOT, &file)); - file = file.Append(FILE_PATH_LITERAL("media")) - .Append(FILE_PATH_LITERAL("test")) - .Append(FILE_PATH_LITERAL("data")) - .Append(FILE_PATH_LITERAL("speech_16b_stereo_48kHz.raw")); - DCHECK(base::PathExists(file)); - int64_t data_file_size64 = 0; - DCHECK(base::GetFileSize(file, &data_file_size64)); - EXPECT_EQ(length, base::ReadFile(file, data, length)); - DCHECK(data_file_size64 > length); -} - -} // namespace - -class WebRtcAudioProcessorTest : public ::testing::Test { - public: - WebRtcAudioProcessorTest() - : params_(media::AudioParameters::AUDIO_PCM_LOW_LATENCY, - media::CHANNEL_LAYOUT_STEREO, - 48000, - 480) {} - - protected: - // Helper method to save duplicated code. - void ProcessDataAndVerifyFormat(AudioProcessor* audio_processor, - int expected_output_sample_rate, - int expected_output_channels, - int expected_output_buffer_size) { - // Read the audio data from a file. - const media::AudioParameters& params = audio_processor->audio_parameters_; - const int packet_size = params.frames_per_buffer() * 2 * params.channels(); - const size_t length = packet_size * kNumberOfPacketsForTest; - std::unique_ptr<char[]> capture_data(new char[length]); - ReadDataFromSpeechFile(capture_data.get(), length); - const int16_t* data_ptr = - reinterpret_cast<const int16_t*>(capture_data.get()); - std::unique_ptr<media::AudioBus> data_bus = - media::AudioBus::Create(params.channels(), params.frames_per_buffer()); - - // |data_bus_playout| is used if the capture channels include a keyboard - // channel. |data_bus_playout_to_use| points to the AudioBus to use, either - // |data_bus| or |data_bus_playout|. - std::unique_ptr<media::AudioBus> data_bus_playout; - media::AudioBus* data_bus_playout_to_use = data_bus.get(); - media::AudioParameters playout_params = params; - const bool has_keyboard_mic = params.channel_layout() == - media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC; - if (has_keyboard_mic) { - data_bus_playout = media::AudioBus::CreateWrapper(2); - data_bus_playout->set_frames(params.frames_per_buffer()); - data_bus_playout_to_use = data_bus_playout.get(); - playout_params.Reset(params.format(), CHANNEL_LAYOUT_STEREO, - params.sample_rate(), params.frames_per_buffer()); - } - - const base::TimeDelta input_capture_delay = - base::TimeDelta::FromMilliseconds(20); - for (int i = 0; i < kNumberOfPacketsForTest; ++i) { - data_bus->FromInterleaved<SignedInt16SampleTypeTraits>( - data_ptr, data_bus->frames()); - // |audio_processor| does nothing when the audio processing is off in - // the processor. - webrtc::AudioProcessing* ap = audio_processor->audio_processing_.get(); - const bool is_aec_enabled = ap && ap->GetConfig().echo_canceller.enabled; - if (is_aec_enabled) { - if (has_keyboard_mic) { - for (int i = 0; i < data_bus_playout->channels(); ++i) { - data_bus_playout->SetChannelData( - i, const_cast<float*>(data_bus->channel(i))); - } - } - base::TimeTicks in_the_future = - base::TimeTicks::Now() + base::TimeDelta::FromMilliseconds(10); - audio_processor->AnalyzePlayout(*data_bus_playout_to_use, - playout_params, in_the_future); - } - - auto result = audio_processor->ProcessCapture( - *data_bus, base::TimeTicks::Now() - input_capture_delay, 1.0, false); - data_ptr += params.frames_per_buffer() * params.channels(); - } - } - - void VerifyEnabledComponents(AudioProcessor* audio_processor) { - webrtc::AudioProcessing* audio_processing = - audio_processor->audio_processing_.get(); - webrtc::AudioProcessing::Config ap_config = audio_processing->GetConfig(); - EXPECT_TRUE(ap_config.echo_canceller.enabled); - EXPECT_FALSE(ap_config.echo_canceller.mobile_mode); - EXPECT_TRUE(ap_config.high_pass_filter.enabled); - EXPECT_TRUE(ap_config.gain_controller1.enabled); - EXPECT_EQ(ap_config.gain_controller1.mode, - ap_config.gain_controller1.kAdaptiveAnalog); - EXPECT_TRUE(ap_config.noise_suppression.enabled); - EXPECT_EQ(ap_config.noise_suppression.level, - ap_config.noise_suppression.kHigh); - EXPECT_TRUE(ap_config.voice_detection.enabled); - } - - AudioProcessingSettings GetEnabledAudioProcessingSettings() const { - AudioProcessingSettings settings; - settings.echo_cancellation = EchoCancellationType::kAec3; - settings.noise_suppression = NoiseSuppressionType::kExperimental; - settings.automatic_gain_control = AutomaticGainControlType::kExperimental; - settings.high_pass_filter = true; - settings.typing_detection = true; - return settings; - } - - base::test::TaskEnvironment task_environment_; - media::AudioParameters params_; -}; - -TEST_F(WebRtcAudioProcessorTest, WithAudioProcessing) { - AudioProcessor audio_processor(params_, GetEnabledAudioProcessingSettings()); - VerifyEnabledComponents(&audio_processor); - ProcessDataAndVerifyFormat(&audio_processor, kAudioProcessingSampleRate, - kAudioProcessingNumberOfChannels, - kAudioProcessingSampleRate / 100); -} - -TEST_F(WebRtcAudioProcessorTest, WithoutAnyProcessing) { - // All processing settings are disabled by default - AudioProcessingSettings settings; - const media::AudioParameters source_params( - media::AudioParameters::AUDIO_PCM_LOW_LATENCY, - media::CHANNEL_LAYOUT_STEREO, kAudioProcessingSampleRate, - kAudioProcessingSampleRate / 100); - AudioProcessor audio_processor(source_params, settings); - - ProcessDataAndVerifyFormat(&audio_processor, params_.sample_rate(), - params_.channels(), params_.sample_rate() / 100); -} - -TEST_F(WebRtcAudioProcessorTest, TestAllSampleRates) { - for (int sample_rate : {8000, 16000, 32000, 44100, 48000}) { - int buffer_size = sample_rate / 100; - media::AudioParameters params(media::AudioParameters::AUDIO_PCM_LOW_LATENCY, - media::CHANNEL_LAYOUT_STEREO, sample_rate, - buffer_size); - AudioProcessor audio_processor(params, GetEnabledAudioProcessingSettings()); - - VerifyEnabledComponents(&audio_processor); - - ProcessDataAndVerifyFormat(&audio_processor, kAudioProcessingSampleRate, - kAudioProcessingNumberOfChannels, - kAudioProcessingSampleRate / 100); - } -} - -TEST_F(WebRtcAudioProcessorTest, TestStereoAudio) { - // All processing settings are disabled by default - AudioProcessingSettings settings; - settings.stereo_mirroring = true; - const media::AudioParameters source_params( - media::AudioParameters::AUDIO_PCM_LOW_LATENCY, - media::CHANNEL_LAYOUT_STEREO, kAudioProcessingSampleRate, - kAudioProcessingSampleRate / 100); - AudioProcessor audio_processor(source_params, settings); - - // Construct left and right channels, and assign different values to the - // first data of the left channel and right channel. - const int size = media::AudioBus::CalculateMemorySize(source_params); - std::unique_ptr<float, base::AlignedFreeDeleter> left_channel( - static_cast<float*>(base::AlignedAlloc(size, 32))); - std::unique_ptr<float, base::AlignedFreeDeleter> right_channel( - static_cast<float*>(base::AlignedAlloc(size, 32))); - std::unique_ptr<media::AudioBus> wrapper = - media::AudioBus::CreateWrapper(source_params.channels()); - wrapper->set_frames(source_params.frames_per_buffer()); - wrapper->SetChannelData(0, left_channel.get()); - wrapper->SetChannelData(1, right_channel.get()); - wrapper->Zero(); - float* left_channel_ptr = left_channel.get(); - left_channel_ptr[0] = 1.0f; - - // Run the test consecutively to make sure the stereo channels are not - // flipped back and forth. - static const int kNumberOfPacketsForTest = 100; - const base::TimeDelta pushed_capture_delay = - base::TimeDelta::FromMilliseconds(42); - for (int i = 0; i < kNumberOfPacketsForTest; ++i) { - auto result = audio_processor.ProcessCapture( - *wrapper, base::TimeTicks::Now() + pushed_capture_delay, 1.0, false); - - EXPECT_EQ(result.audio.channel(0)[0], 0); - EXPECT_NE(result.audio.channel(1)[0], 0); - } -} - -TEST_F(WebRtcAudioProcessorTest, TestWithKeyboardMicChannel) { - AudioProcessingSettings settings = GetEnabledAudioProcessingSettings(); - media::AudioParameters params(media::AudioParameters::AUDIO_PCM_LOW_LATENCY, - media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC, - kAudioProcessingSampleRate, - kAudioProcessingSampleRate / 100); - AudioProcessor audio_processor(params, settings); - ProcessDataAndVerifyFormat(&audio_processor, kAudioProcessingSampleRate, - kAudioProcessingNumberOfChannels, - kAudioProcessingSampleRate / 100); -} - -TEST_F(WebRtcAudioProcessorTest, StartStopAecDump) { - base::ScopedTempDir temp_directory; - ASSERT_TRUE(temp_directory.CreateUniqueTempDir()); - base::FilePath temp_file_path; - ASSERT_TRUE(base::CreateTemporaryFileInDir(temp_directory.GetPath(), - &temp_file_path)); - { - AudioProcessor audio_processor(params_, - GetEnabledAudioProcessingSettings()); - - // Start and stop recording. - audio_processor.StartEchoCancellationDump(base::File( - temp_file_path, base::File::FLAG_WRITE | base::File::FLAG_OPEN)); - audio_processor.StopEchoCancellationDump(); - - // Start and wait for d-tor. - audio_processor.StartEchoCancellationDump(base::File( - temp_file_path, base::File::FLAG_WRITE | base::File::FLAG_OPEN)); - } - - // Check that dump file is non-empty after audio processor has been - // destroyed. Note that this test fails when compiling WebRTC - // without protobuf support, rtc_enable_protobuf=false. - std::string output; - ASSERT_TRUE(base::ReadFileToString(temp_file_path, &output)); - ASSERT_FALSE(output.empty()); - // The temporary file is deleted when temp_directory exists scope. -} - -} // namespace media diff --git a/chromium/media/webrtc/webrtc_switches.cc b/chromium/media/webrtc/webrtc_switches.cc index 481e16adc76..05bba547469 100644 --- a/chromium/media/webrtc/webrtc_switches.cc +++ b/chromium/media/webrtc/webrtc_switches.cc @@ -20,12 +20,6 @@ const char kAgcStartupMinVolume[] = "agc-startup-min-volume"; namespace features { -// Enables running WebRTC Audio Processing in the audio service, rather than -// in the renderer process. Should be combined with running the audio service -// out of the browser process, except for when testing locally. -const base::Feature kWebRtcApmInAudioService{"WebRtcApmInAudioService", - base::FEATURE_DISABLED_BY_DEFAULT}; - // Enables multi channel capture audio to be processed without // downmixing in the WebRTC audio processing module when running in the renderer // process. @@ -38,25 +32,3 @@ const base::Feature kWebRtcHybridAgc{"WebRtcHybridAgc", base::FEATURE_DISABLED_BY_DEFAULT}; } // namespace features - -namespace switches { - -const char kForceDisableWebRtcApmInAudioService[] = - "disable-webrtc-apm-in-audio-service"; - -} // namespace switches - -namespace media { - -bool IsWebRtcApmInAudioServiceEnabled() { -#if defined(OS_WIN) || defined(OS_MACOSX) || \ - (defined(OS_LINUX) && !defined(OS_CHROMEOS)) - return base::FeatureList::IsEnabled(features::kWebRtcApmInAudioService) && - !base::CommandLine::ForCurrentProcess()->HasSwitch( - switches::kForceDisableWebRtcApmInAudioService); -#else - return false; -#endif -} - -} // namespace media diff --git a/chromium/media/webrtc/webrtc_switches.h b/chromium/media/webrtc/webrtc_switches.h index de0f83cf1af..e3fd32da833 100644 --- a/chromium/media/webrtc/webrtc_switches.h +++ b/chromium/media/webrtc/webrtc_switches.h @@ -19,9 +19,6 @@ COMPONENT_EXPORT(MEDIA_WEBRTC) extern const char kAgcStartupMinVolume[]; namespace features { COMPONENT_EXPORT(MEDIA_WEBRTC) -extern const base::Feature kWebRtcApmInAudioService; - -COMPONENT_EXPORT(MEDIA_WEBRTC) extern const base::Feature kWebRtcEnableCaptureMultiChannelApm; COMPONENT_EXPORT(MEDIA_WEBRTC) @@ -29,15 +26,4 @@ extern const base::Feature kWebRtcHybridAgc; } // namespace features -namespace switches { -COMPONENT_EXPORT(MEDIA_WEBRTC) -extern const char kForceDisableWebRtcApmInAudioService[]; -} // namespace switches - -namespace media { - -COMPONENT_EXPORT(MEDIA_WEBRTC) bool IsWebRtcApmInAudioServiceEnabled(); - -} // namespace media - #endif // MEDIA_WEBRTC_WEBRTC_SWITCHES_H_ |