diff options
Diffstat (limited to 'chromium/media/base')
112 files changed, 3263 insertions, 1497 deletions
diff --git a/chromium/media/base/BUILD.gn b/chromium/media/base/BUILD.gn index f3ec9d91fd9..bcdea7612cb 100644 --- a/chromium/media/base/BUILD.gn +++ b/chromium/media/base/BUILD.gn @@ -156,8 +156,6 @@ source_set("base") { "encryption_scheme.h", "fake_audio_worker.cc", "fake_audio_worker.h", - "fallback_video_decoder.cc", - "fallback_video_decoder.h", "feedback_signal_accumulator.h", "flinging_controller.h", "format_utils.cc", @@ -230,6 +228,8 @@ source_set("base") { "multi_channel_resampler.h", "null_video_sink.cc", "null_video_sink.h", + "offloading_audio_encoder.cc", + "offloading_audio_encoder.h", "offloading_video_encoder.cc", "offloading_video_encoder.h", "output_device_info.cc", @@ -265,6 +265,8 @@ source_set("base") { "seekable_buffer.h", "serial_runner.cc", "serial_runner.h", + "shared_memory_pool.cc", + "shared_memory_pool.h", "silent_sink_suspender.cc", "silent_sink_suspender.h", "simple_sync_token_client.cc", @@ -285,6 +287,9 @@ source_set("base") { "subsample_entry.h", "supported_types.cc", "supported_types.h", + "supported_types.h", + "supported_video_decoder_config.cc", + "supported_video_decoder_config.h", "text_cue.cc", "text_cue.h", "text_ranges.cc", @@ -361,8 +366,10 @@ source_set("base") { "//build:chromeos_buildflags", "//components/system_media_controls/linux/buildflags", "//gpu/command_buffer/client:interface_base", + "//gpu/command_buffer/client:raster_interface", "//gpu/command_buffer/common", "//gpu/ipc/common:common", + "//skia", "//third_party/libyuv", "//third_party/widevine/cdm:headers", "//ui/display:display", @@ -480,6 +487,7 @@ static_library("test_support") { "//media/base/android:test_support", "//media/filters:test_support", "//media/formats:test_support", + "//media/renderers:test_support", "//media/video:test_support", ] testonly = true @@ -553,7 +561,6 @@ source_set("unit_tests") { "audio_sample_types_unittest.cc", "audio_shifter_unittest.cc", "audio_timestamp_helper_unittest.cc", - "bind_to_current_loop_unittest.cc", "bit_reader_unittest.cc", "callback_holder_unittest.cc", "callback_registry_unittest.cc", @@ -567,7 +574,6 @@ source_set("unit_tests") { "djb2_unittest.cc", "fake_audio_worker_unittest.cc", "fake_demuxer_stream_unittest.cc", - "fallback_video_decoder_unittest.cc", "feedback_signal_accumulator_unittest.cc", "frame_rate_estimator_unittest.cc", "key_systems_unittest.cc", @@ -578,6 +584,7 @@ source_set("unit_tests") { "moving_average_unittest.cc", "multi_channel_resampler_unittest.cc", "null_video_sink_unittest.cc", + "offloading_audio_encoder_unittest.cc", "offloading_video_encoder_unittest.cc", "pipeline_impl_unittest.cc", "ranges_unittest.cc", @@ -585,12 +592,14 @@ source_set("unit_tests") { "renderer_factory_selector_unittest.cc", "seekable_buffer_unittest.cc", "serial_runner_unittest.cc", + "shared_memory_pool_unittest.cc", "silent_sink_suspender_unittest.cc", "sinc_resampler_unittest.cc", "status_unittest.cc", "stream_parser_unittest.cc", "subsample_entry_unittest.cc", "supported_types_unittest.cc", + "supported_video_decoder_config_unittest.cc", "text_ranges_unittest.cc", "text_renderer_unittest.cc", "time_delta_interpolator_unittest.cc", @@ -634,7 +643,10 @@ source_set("unit_tests") { if (is_win) { sources += [ "win/dxgi_device_scope_handle_unittest.cc" ] - deps += [ "//media/base/win:media_foundation_util" ] + deps += [ + "//media/base/win:media_foundation_util", + "//ui/events:test_support", + ] libs = [ "d3d11.lib", "mfplat.lib", diff --git a/chromium/media/base/OWNERS b/chromium/media/base/OWNERS index f4ecf46c4ae..7f62fe2fb3e 100644 --- a/chromium/media/base/OWNERS +++ b/chromium/media/base/OWNERS @@ -1,4 +1,3 @@ per-file *audio*=file://media/audio/OWNERS -per-file media_switches.*=beccahughes@chromium.org per-file media_switches.*=mlamouri@chromium.org diff --git a/chromium/media/base/android/BUILD.gn b/chromium/media/base/android/BUILD.gn index 39766b7c387..886dbbf132e 100644 --- a/chromium/media/base/android/BUILD.gn +++ b/chromium/media/base/android/BUILD.gn @@ -163,7 +163,7 @@ if (is_android) { ":media_java_resources", "//base:base_java", "//base:jni_java", - "//third_party/android_deps:androidx_annotation_annotation_java", + "//third_party/androidx:androidx_annotation_annotation_java", ] annotation_processor_deps = [ "//base/android/jni_generator:jni_processor" ] srcjar_deps = [ @@ -198,7 +198,7 @@ if (is_android) { android_library("display_java") { sources = [ "java/src/org/chromium/media/DisplayCompat.java" ] - deps = [ "//third_party/android_deps:androidx_annotation_annotation_java" ] + deps = [ "//third_party/androidx:androidx_annotation_annotation_java" ] } junit_binary("media_base_junit_tests") { @@ -211,7 +211,7 @@ if (is_android) { ":media_java", "//base:base_java", "//base:base_java_test_support", - "//third_party/android_deps:androidx_test_runner_java", + "//third_party/androidx:androidx_test_runner_java", "//third_party/junit", ] } diff --git a/chromium/media/base/android/media_drm_bridge.cc b/chromium/media/base/android/media_drm_bridge.cc index e5d7978a28f..182dc13ad28 100644 --- a/chromium/media/base/android/media_drm_bridge.cc +++ b/chromium/media/base/android/media_drm_bridge.cc @@ -914,7 +914,7 @@ MediaDrmBridge::~MediaDrmBridge() { } // Rejects all pending promises. - cdm_promise_adapter_.Clear(); + cdm_promise_adapter_.Clear(CdmPromiseAdapter::ClearReason::kDestruction); } MediaDrmBridge::SecurityLevel MediaDrmBridge::GetSecurityLevel() { diff --git a/chromium/media/base/android/media_player_bridge.cc b/chromium/media/base/android/media_player_bridge.cc index eeab111c274..41d0ab6356d 100644 --- a/chromium/media/base/android/media_player_bridge.cc +++ b/chromium/media/base/android/media_player_bridge.cc @@ -80,6 +80,8 @@ MediaPlayerBridge::MediaPlayerBridge(const GURL& url, url_(url), site_for_cookies_(site_for_cookies), top_frame_origin_(top_frame_origin), + pending_retrieve_cookies_(false), + should_prepare_on_retrieved_cookies_(false), user_agent_(user_agent), hide_url_log_(hide_url_log), width_(0), @@ -127,6 +129,7 @@ void MediaPlayerBridge::Initialize() { media::MediaResourceGetter* resource_getter = client_->GetMediaResourceGetter(); + pending_retrieve_cookies_ = true; resource_getter->GetCookies( url_, site_for_cookies_, top_frame_origin_, base::BindOnce(&MediaPlayerBridge::OnCookiesRetrieved, @@ -164,6 +167,17 @@ void MediaPlayerBridge::SetVideoSurface(gl::ScopedJavaSurface surface) { surface_.j_surface()); } +void MediaPlayerBridge::SetPlaybackRate(double playback_rate) { + if (j_media_player_bridge_.is_null()) + return; + + JNIEnv* env = base::android::AttachCurrentThread(); + CHECK(env); + + Java_MediaPlayerBridge_setPlaybackRate(env, j_media_player_bridge_, + playback_rate); +} + void MediaPlayerBridge::Prepare() { DCHECK(j_media_player_bridge_.is_null()); @@ -200,31 +214,52 @@ void MediaPlayerBridge::SetDataSource(const std::string& url) { OnMediaError(MEDIA_ERROR_FORMAT); return; } - } else { - // Create a Java String for the URL. - ScopedJavaLocalRef<jstring> j_url_string = - ConvertUTF8ToJavaString(env, url); - - const std::string data_uri_prefix("data:"); - if (base::StartsWith(url, data_uri_prefix, base::CompareCase::SENSITIVE)) { - if (!Java_MediaPlayerBridge_setDataUriDataSource( - env, j_media_player_bridge_, j_url_string)) { - OnMediaError(MEDIA_ERROR_FORMAT); - } - return; - } - ScopedJavaLocalRef<jstring> j_cookies = ConvertUTF8ToJavaString( - env, cookies_); - ScopedJavaLocalRef<jstring> j_user_agent = ConvertUTF8ToJavaString( - env, user_agent_); + if (!Java_MediaPlayerBridge_prepareAsync(env, j_media_player_bridge_)) + OnMediaError(MEDIA_ERROR_FORMAT); + + return; + } - if (!Java_MediaPlayerBridge_setDataSource(env, j_media_player_bridge_, - j_url_string, j_cookies, - j_user_agent, hide_url_log_)) { + // Create a Java String for the URL. + ScopedJavaLocalRef<jstring> j_url_string = ConvertUTF8ToJavaString(env, url); + + const std::string data_uri_prefix("data:"); + if (base::StartsWith(url, data_uri_prefix, base::CompareCase::SENSITIVE)) { + if (!Java_MediaPlayerBridge_setDataUriDataSource( + env, j_media_player_bridge_, j_url_string)) { OnMediaError(MEDIA_ERROR_FORMAT); - return; } + return; + } + + // Cookies may not have been retrieved yet, delay prepare until they are + // retrieved. + if (pending_retrieve_cookies_) { + should_prepare_on_retrieved_cookies_ = true; + return; + } + SetDataSourceInternal(); +} + +void MediaPlayerBridge::SetDataSourceInternal() { + DCHECK(!pending_retrieve_cookies_); + + JNIEnv* env = base::android::AttachCurrentThread(); + CHECK(env); + + ScopedJavaLocalRef<jstring> j_cookies = + ConvertUTF8ToJavaString(env, cookies_); + ScopedJavaLocalRef<jstring> j_user_agent = + ConvertUTF8ToJavaString(env, user_agent_); + ScopedJavaLocalRef<jstring> j_url_string = + ConvertUTF8ToJavaString(env, url_.spec()); + + if (!Java_MediaPlayerBridge_setDataSource(env, j_media_player_bridge_, + j_url_string, j_cookies, + j_user_agent, hide_url_log_)) { + OnMediaError(MEDIA_ERROR_FORMAT); + return; } if (!Java_MediaPlayerBridge_prepareAsync(env, j_media_player_bridge_)) @@ -267,9 +302,15 @@ void MediaPlayerBridge::OnDidSetDataUriDataSource( void MediaPlayerBridge::OnCookiesRetrieved(const std::string& cookies) { cookies_ = cookies; + pending_retrieve_cookies_ = false; client_->GetMediaResourceGetter()->GetAuthCredentials( url_, base::BindOnce(&MediaPlayerBridge::OnAuthCredentialsRetrieved, weak_factory_.GetWeakPtr())); + + if (should_prepare_on_retrieved_cookies_) { + SetDataSourceInternal(); + should_prepare_on_retrieved_cookies_ = false; + } } void MediaPlayerBridge::OnAuthCredentialsRetrieved( diff --git a/chromium/media/base/android/media_player_bridge.h b/chromium/media/base/android/media_player_bridge.h index 8757e09b24a..52648721aa1 100644 --- a/chromium/media/base/android/media_player_bridge.h +++ b/chromium/media/base/android/media_player_bridge.h @@ -92,6 +92,7 @@ class MEDIA_EXPORT MediaPlayerBridge { // Methods to partially expose the underlying MediaPlayer. void SetVideoSurface(gl::ScopedJavaSurface surface); + void SetPlaybackRate(double playback_rate); void Pause(); void SeekTo(base::TimeDelta timestamp); base::TimeDelta GetCurrentTime(); @@ -153,6 +154,7 @@ class MEDIA_EXPORT MediaPlayerBridge { // Set the data source for the media player. void SetDataSource(const std::string& url); + void SetDataSourceInternal(); // Functions that implements media player control. void StartInternal(); @@ -220,6 +222,12 @@ class MEDIA_EXPORT MediaPlayerBridge { // Used to check for cookie content settings. url::Origin top_frame_origin_; + // Waiting to retrieve cookies for |url_|. + bool pending_retrieve_cookies_; + + // Whether to prepare after cookies retrieved. + bool should_prepare_on_retrieved_cookies_; + // User agent string to be used for media player. const std::string user_agent_; diff --git a/chromium/media/base/async_destroy_video_decoder.h b/chromium/media/base/async_destroy_video_decoder.h index 921109b08f1..948fa8c5a7c 100644 --- a/chromium/media/base/async_destroy_video_decoder.h +++ b/chromium/media/base/async_destroy_video_decoder.h @@ -35,6 +35,11 @@ class AsyncDestroyVideoDecoder final : public VideoDecoder { T::DestroyAsync(std::move(wrapped_decoder_)); } + VideoDecoderType GetDecoderType() const override { + DCHECK(wrapped_decoder_); + return wrapped_decoder_->GetDecoderType(); + } + std::string GetDisplayName() const override { DCHECK(wrapped_decoder_); return wrapped_decoder_->GetDisplayName(); diff --git a/chromium/media/base/audio_block_fifo.cc b/chromium/media/base/audio_block_fifo.cc index 85fa6d27b93..ad49cff0637 100644 --- a/chromium/media/base/audio_block_fifo.cc +++ b/chromium/media/base/audio_block_fifo.cc @@ -116,13 +116,33 @@ void AudioBlockFifo::PushInternal(const void* source, std::min(block_frames_ - write_pos_, frames_to_push); if (source) { - // Deinterleave the content to the FIFO and update the |write_pos_|. - current_block->FromInterleavedPartial(source_ptr, write_pos_, push_frames, - bytes_per_sample); + // Deinterleave the content to the FIFO. + switch (bytes_per_sample) { + case 1: + current_block->FromInterleavedPartial<UnsignedInt8SampleTypeTraits>( + source_ptr, write_pos_, push_frames); + break; + case 2: + current_block->FromInterleavedPartial<SignedInt16SampleTypeTraits>( + reinterpret_cast<const int16_t*>(source_ptr), write_pos_, + push_frames); + break; + case 4: + current_block->FromInterleavedPartial<SignedInt32SampleTypeTraits>( + reinterpret_cast<const int32_t*>(source_ptr), write_pos_, + push_frames); + break; + default: + NOTREACHED() << "Unsupported bytes per sample encountered: " + << bytes_per_sample; + current_block->ZeroFramesPartial(write_pos_, push_frames); + } } else { current_block->ZeroFramesPartial(write_pos_, push_frames); } + write_pos_ = (write_pos_ + push_frames) % block_frames_; + if (!write_pos_) { // The current block is completely filled, increment |write_block_| and // |available_blocks_|. diff --git a/chromium/media/base/audio_buffer.cc b/chromium/media/base/audio_buffer.cc index ad6bd58c9c7..99cbfb6af4c 100644 --- a/chromium/media/base/audio_buffer.cc +++ b/chromium/media/base/audio_buffer.cc @@ -14,6 +14,29 @@ namespace media { +namespace { + +// TODO(https://crbug.com/619628): Use vector instructions to speed this up. +template <class SourceSampleTypeTraits> +void CopyConvertFromInterleaved( + const typename SourceSampleTypeTraits::ValueType* source_buffer, + int num_frames_to_write, + const std::vector<float*> dest) { + const int channels = dest.size(); + for (int ch = 0; ch < channels; ++ch) { + float* dest_data = dest[ch]; + for (int target_frame_index = 0, read_pos_in_source = ch; + target_frame_index < num_frames_to_write; + ++target_frame_index, read_pos_in_source += channels) { + auto source_value = source_buffer[read_pos_in_source]; + dest_data[target_frame_index] = + SourceSampleTypeTraits::ToFloat(source_value); + } + } +} + +} // namespace + static base::TimeDelta CalculateDuration(int frames, double sample_rate) { DCHECK_GT(sample_rate, 0); return base::TimeDelta::FromMicroseconds( @@ -332,6 +355,72 @@ void AudioBuffer::ReadFrames(int frames_to_copy, } } +void AudioBuffer::ReadAllFrames(const std::vector<float*>& dest) const { + // Deinterleave each channel (if necessary) and convert to 32bit + // floating-point with nominal range -1.0 -> +1.0 (if necessary). + + // |dest| must have the same number of channels, and the number of frames + // specified must be in range. + DCHECK(!end_of_stream()); + CHECK_EQ(dest.size(), static_cast<size_t>(channel_count_)); + DCHECK(!IsBitstreamFormat()); + + if (!data_) { + // Special case for an empty buffer. + for (int i = 0; i < channel_count_; ++i) + memset(dest[i], 0, adjusted_frame_count_ * sizeof(float)); + return; + } + + // Note: The conversion steps below will clip values to [1.0, -1.0f]. + + if (sample_format_ == kSampleFormatPlanarF32) { + for (int ch = 0; ch < channel_count_; ++ch) { + float* dest_data = dest[ch]; + const float* source_data = + reinterpret_cast<const float*>(channel_data_[ch]); + for (int i = 0; i < adjusted_frame_count_; ++i) + dest_data[i] = Float32SampleTypeTraits::FromFloat(source_data[i]); + } + return; + } + + if (sample_format_ == kSampleFormatPlanarS16) { + // Format is planar signed16. Convert each value into float and insert into + // output channel data. + for (int ch = 0; ch < channel_count_; ++ch) { + const int16_t* source_data = + reinterpret_cast<const int16_t*>(channel_data_[ch]); + float* dest_data = dest[ch]; + for (int i = 0; i < adjusted_frame_count_; ++i) + dest_data[i] = SignedInt16SampleTypeTraits::ToFloat(source_data[i]); + } + return; + } + + const uint8_t* source_data = data_.get(); + + if (sample_format_ == kSampleFormatF32) { + CopyConvertFromInterleaved<Float32SampleTypeTraits>( + reinterpret_cast<const float*>(source_data), adjusted_frame_count_, + dest); + } else if (sample_format_ == kSampleFormatU8) { + CopyConvertFromInterleaved<UnsignedInt8SampleTypeTraits>( + source_data, adjusted_frame_count_, dest); + } else if (sample_format_ == kSampleFormatS16) { + CopyConvertFromInterleaved<SignedInt16SampleTypeTraits>( + reinterpret_cast<const int16_t*>(source_data), adjusted_frame_count_, + dest); + } else if (sample_format_ == kSampleFormatS24 || + sample_format_ == kSampleFormatS32) { + CopyConvertFromInterleaved<SignedInt32SampleTypeTraits>( + reinterpret_cast<const int32_t*>(source_data), adjusted_frame_count_, + dest); + } else { + NOTREACHED() << "Unsupported audio sample type: " << sample_format_; + } +} + void AudioBuffer::TrimStart(int frames_to_trim) { CHECK_GE(frames_to_trim, 0); CHECK_LE(frames_to_trim, adjusted_frame_count_); diff --git a/chromium/media/base/audio_buffer.h b/chromium/media/base/audio_buffer.h index 55adc82837f..9a90ef1d88d 100644 --- a/chromium/media/base/audio_buffer.h +++ b/chromium/media/base/audio_buffer.h @@ -131,6 +131,14 @@ class MEDIA_EXPORT AudioBuffer int dest_frame_offset, AudioBus* dest) const; + // Copy all |adjusted_frame_count_| frames into |dest|. Each of |dest|'s + // elements correspond to a different channel. It's the caller's + // responsibility to make sure enough memory per channel was allocated. + // The frames are converted and clipped from their source format into planar + // float32 data. + // Note: Bitstream formats are not supported. + void ReadAllFrames(const std::vector<float*>& dest) const; + // Trim an AudioBuffer by removing |frames_to_trim| frames from the start. // Timestamp and duration are adjusted to reflect the fewer frames. // Note that repeated calls to TrimStart() may result in timestamp() and diff --git a/chromium/media/base/audio_buffer_unittest.cc b/chromium/media/base/audio_buffer_unittest.cc index c769bd5adc2..3a30db21d2d 100644 --- a/chromium/media/base/audio_buffer_unittest.cc +++ b/chromium/media/base/audio_buffer_unittest.cc @@ -7,6 +7,7 @@ #include <limits> #include <memory> +#include "base/test/gtest_util.h" #include "media/base/audio_buffer.h" #include "media/base/audio_bus.h" #include "media/base/test_helpers.h" @@ -36,6 +37,14 @@ static void VerifyBusWithOffset(AudioBus* bus, } } +static std::vector<float*> WrapChannelsAsVector(AudioBus* bus) { + std::vector<float*> channels(bus->channels()); + for (size_t ch = 0; ch < channels.size(); ++ch) + channels[ch] = bus->channel(ch); + + return channels; +} + static void VerifyBus(AudioBus* bus, int frames, float start, @@ -253,6 +262,15 @@ TEST(AudioBufferTest, ReadBitstream) { EXPECT_EQ(frames, bus->GetBitstreamFrames()); EXPECT_EQ(data_size, bus->GetBitstreamDataSize()); VerifyBitstreamAudioBus(bus.get(), data_size, 1, 1); + +#if GTEST_HAS_DEATH_TEST + auto vector_backing = AudioBus::Create(channels, frames); + std::vector<float*> wrapped_channels = + WrapChannelsAsVector(vector_backing.get()); + + // ReadAllFrames() does not support bitstream formats. + EXPECT_DCHECK_DEATH(buffer->ReadAllFrames(wrapped_channels)); +#endif // GTEST_HAS_DEATH_TEST } TEST(AudioBufferTest, ReadU8) { @@ -272,6 +290,12 @@ TEST(AudioBufferTest, ReadU8) { for (int i = 0; i < frames; ++i) buffer->ReadFrames(1, i, i, bus.get()); VerifyBus(bus.get(), frames, 0, 1.0f / 127.0f); + + // Verify ReadAllFrames() works for U8. + bus->Zero(); + std::vector<float*> wrapped_channels = WrapChannelsAsVector(bus.get()); + buffer->ReadAllFrames(wrapped_channels); + VerifyBus(bus.get(), frames, 0, 1.0f / 127.0f); } TEST(AudioBufferTest, ReadS16) { @@ -293,6 +317,13 @@ TEST(AudioBufferTest, ReadS16) { buffer->ReadFrames(1, i, i, bus.get()); VerifyBus(bus.get(), frames, 1.0f / std::numeric_limits<int16_t>::max(), 1.0f / std::numeric_limits<int16_t>::max()); + + // Verify ReadAllFrames() works for S16. + bus->Zero(); + std::vector<float*> wrapped_channels = WrapChannelsAsVector(bus.get()); + buffer->ReadAllFrames(wrapped_channels); + VerifyBus(bus.get(), frames, 1.0f / std::numeric_limits<int16_t>::max(), + 1.0f / std::numeric_limits<int16_t>::max()); } TEST(AudioBufferTest, ReadS32) { @@ -313,6 +344,13 @@ TEST(AudioBufferTest, ReadS32) { buffer->ReadFrames(10, 10, 0, bus.get()); VerifyBus(bus.get(), 10, 11.0f / std::numeric_limits<int32_t>::max(), 1.0f / std::numeric_limits<int32_t>::max()); + + // Verify ReadAllFrames() works for S32. + bus->Zero(); + std::vector<float*> wrapped_channels = WrapChannelsAsVector(bus.get()); + buffer->ReadAllFrames(wrapped_channels); + VerifyBus(bus.get(), frames, 1.0f / std::numeric_limits<int32_t>::max(), + 1.0f / std::numeric_limits<int32_t>::max()); } TEST(AudioBufferTest, ReadF32) { @@ -336,6 +374,12 @@ TEST(AudioBufferTest, ReadF32) { bus->Zero(); buffer->ReadFrames(10, 10, 0, bus.get()); VerifyBus(bus.get(), 10, 11, 1, ValueType::kFloat); + + // Verify ReadAllFrames() works for F32. + bus->Zero(); + std::vector<float*> wrapped_channels = WrapChannelsAsVector(bus.get()); + buffer->ReadAllFrames(wrapped_channels); + VerifyBus(bus.get(), frames, 1, 1, ValueType::kFloat); } TEST(AudioBufferTest, ReadS16Planar) { @@ -369,6 +413,13 @@ TEST(AudioBufferTest, ReadS16Planar) { buffer->ReadFrames(0, 10, 0, bus.get()); VerifyBus(bus.get(), frames, 1.0f / std::numeric_limits<int16_t>::max(), 1.0f / std::numeric_limits<int16_t>::max()); + + // Verify ReadAllFrames() works for S16Planar. + bus->Zero(); + std::vector<float*> wrapped_channels = WrapChannelsAsVector(bus.get()); + buffer->ReadAllFrames(wrapped_channels); + VerifyBus(bus.get(), frames, 1.0f / std::numeric_limits<int16_t>::max(), + 1.0f / std::numeric_limits<int16_t>::max()); } TEST(AudioBufferTest, ReadF32Planar) { @@ -397,6 +448,12 @@ TEST(AudioBufferTest, ReadF32Planar) { bus->Zero(); buffer->ReadFrames(20, 50, 0, bus.get()); VerifyBus(bus.get(), 20, 51, 1, ValueType::kFloat); + + // Verify ReadAllFrames() works for F32Planar. + bus->Zero(); + std::vector<float*> wrapped_channels = WrapChannelsAsVector(bus.get()); + buffer->ReadAllFrames(wrapped_channels); + VerifyBus(bus.get(), frames, 1, 1, ValueType::kFloat); } TEST(AudioBufferTest, EmptyBuffer) { @@ -411,10 +468,19 @@ TEST(AudioBufferTest, EmptyBuffer) { EXPECT_EQ(base::TimeDelta::FromMilliseconds(10), buffer->duration()); EXPECT_FALSE(buffer->end_of_stream()); - // Read all 100 frames from the buffer. All data should be 0. + // Read all frames from the buffer. All data should be 0. std::unique_ptr<AudioBus> bus = AudioBus::Create(channels, frames); buffer->ReadFrames(frames, 0, 0, bus.get()); VerifyBus(bus.get(), frames, 0, 0); + + // Set some data to confirm the overwrite. + std::vector<float*> wrapped_channels = WrapChannelsAsVector(bus.get()); + for (float* wrapped_channel : wrapped_channels) + memset(wrapped_channel, 123, frames * sizeof(float)); + + // Verify ReadAllFrames() overrites empty buffers. + buffer->ReadAllFrames(wrapped_channels); + VerifyBus(bus.get(), frames, 0, 0); } TEST(AudioBufferTest, TrimEmptyBuffer) { diff --git a/chromium/media/base/audio_bus.cc b/chromium/media/base/audio_bus.cc index 694b53e8c24..76ee5c24218 100644 --- a/chromium/media/base/audio_bus.cc +++ b/chromium/media/base/audio_bus.cc @@ -273,81 +273,6 @@ void AudioBus::BuildChannelData(int channels, int aligned_frames, float* data) { channel_data_.push_back(data + i * aligned_frames); } -// Forwards to non-deprecated version. -void AudioBus::FromInterleaved(const void* source, - int frames, - int bytes_per_sample) { - DCHECK(!is_bitstream_format_); - switch (bytes_per_sample) { - case 1: - FromInterleaved<UnsignedInt8SampleTypeTraits>( - reinterpret_cast<const uint8_t*>(source), frames); - break; - case 2: - FromInterleaved<SignedInt16SampleTypeTraits>( - reinterpret_cast<const int16_t*>(source), frames); - break; - case 4: - FromInterleaved<SignedInt32SampleTypeTraits>( - reinterpret_cast<const int32_t*>(source), frames); - break; - default: - NOTREACHED() << "Unsupported bytes per sample encountered: " - << bytes_per_sample; - ZeroFrames(frames); - } -} - -// Forwards to non-deprecated version. -void AudioBus::FromInterleavedPartial(const void* source, - int start_frame, - int frames, - int bytes_per_sample) { - DCHECK(!is_bitstream_format_); - switch (bytes_per_sample) { - case 1: - FromInterleavedPartial<UnsignedInt8SampleTypeTraits>( - reinterpret_cast<const uint8_t*>(source), start_frame, frames); - break; - case 2: - FromInterleavedPartial<SignedInt16SampleTypeTraits>( - reinterpret_cast<const int16_t*>(source), start_frame, frames); - break; - case 4: - FromInterleavedPartial<SignedInt32SampleTypeTraits>( - reinterpret_cast<const int32_t*>(source), start_frame, frames); - break; - default: - NOTREACHED() << "Unsupported bytes per sample encountered: " - << bytes_per_sample; - ZeroFramesPartial(start_frame, frames); - } -} - -// Forwards to non-deprecated version. -void AudioBus::ToInterleaved(int frames, - int bytes_per_sample, - void* dest) const { - DCHECK(!is_bitstream_format_); - switch (bytes_per_sample) { - case 1: - ToInterleaved<UnsignedInt8SampleTypeTraits>( - frames, reinterpret_cast<uint8_t*>(dest)); - break; - case 2: - ToInterleaved<SignedInt16SampleTypeTraits>( - frames, reinterpret_cast<int16_t*>(dest)); - break; - case 4: - ToInterleaved<SignedInt32SampleTypeTraits>( - frames, reinterpret_cast<int32_t*>(dest)); - break; - default: - NOTREACHED() << "Unsupported bytes per sample encountered: " - << bytes_per_sample; - } -} - void AudioBus::CopyTo(AudioBus* dest) const { dest->set_is_bitstream_format(is_bitstream_format()); if (is_bitstream_format()) { diff --git a/chromium/media/base/audio_bus.h b/chromium/media/base/audio_bus.h index 024fee8968f..1c520279146 100644 --- a/chromium/media/base/audio_bus.h +++ b/chromium/media/base/audio_bus.h @@ -104,11 +104,6 @@ class MEDIA_SHMEM_EXPORT AudioBus { const typename SourceSampleTypeTraits::ValueType* source_buffer, int num_frames_to_write); - // DEPRECATED (https://crbug.com/580391) - // Please use the version templated with SourceSampleTypeTraits instead. - // TODO(chfremer): Remove (https://crbug.com/619623) - void FromInterleaved(const void* source, int frames, int bytes_per_sample); - // Similar to FromInterleaved...(), but overwrites the frames starting at a // given offset |write_offset_in_frames| and does not zero out frames that are // not overwritten. @@ -118,12 +113,6 @@ class MEDIA_SHMEM_EXPORT AudioBus { int write_offset_in_frames, int num_frames_to_write); - // DEPRECATED (https://crbug.com/580391) - // Please use the version templated with SourceSampleTypeTraits instead. - // TODO(chfremer): Remove (https://crbug.com/619623) - void FromInterleavedPartial(const void* source, int start_frame, int frames, - int bytes_per_sample); - // Reads the sample values stored in this AudioBus instance and places them // into the given |dest_buffer| in interleaved format using the sample format // specified by TargetSampleTypeTraits. For a list of ready-to-use @@ -134,11 +123,6 @@ class MEDIA_SHMEM_EXPORT AudioBus { int num_frames_to_read, typename TargetSampleTypeTraits::ValueType* dest_buffer) const; - // DEPRECATED (https://crbug.com/580391) - // Please use the version templated with TargetSampleTypeTraits instead. - // TODO(chfremer): Remove (https://crbug.com/619623) - void ToInterleaved(int frames, int bytes_per_sample, void* dest) const; - // Similar to ToInterleaved(), but reads the frames starting at a given // offset |read_offset_in_frames|. template <class TargetSampleTypeTraits> diff --git a/chromium/media/base/audio_bus_unittest.cc b/chromium/media/base/audio_bus_unittest.cc index 79bb101b2eb..38d5cbd09da 100644 --- a/chromium/media/base/audio_bus_unittest.cc +++ b/chromium/media/base/audio_bus_unittest.cc @@ -335,40 +335,6 @@ TEST_F(AudioBusTest, FromInterleaved) { kTestVectorFrameCount * sizeof(*expected->channel(ch))); } - // Test deprecated version that takes |bytes_per_sample| as an input. - { - SCOPED_TRACE("uint8_t"); - bus->Zero(); - bus->FromInterleaved(kTestVectorUint8, kTestVectorFrameCount, - sizeof(*kTestVectorUint8)); - - // Biased uint8_t calculations have poor precision, so the epsilon here is - // slightly more permissive than int16_t and int32_t calculations. - VerifyAreEqualWithEpsilon(bus.get(), expected.get(), - 1.0f / (std::numeric_limits<uint8_t>::max() - 1)); - } - { - SCOPED_TRACE("int16_t"); - bus->Zero(); - bus->FromInterleaved(kTestVectorInt16, kTestVectorFrameCount, - sizeof(*kTestVectorInt16)); - VerifyAreEqualWithEpsilon( - bus.get(), expected.get(), - 1.0f / (std::numeric_limits<uint16_t>::max() + 1.0f)); - } - { - SCOPED_TRACE("int32_t"); - bus->Zero(); - bus->FromInterleaved(kTestVectorInt32, kTestVectorFrameCount, - sizeof(*kTestVectorInt32)); - - VerifyAreEqualWithEpsilon( - bus.get(), expected.get(), - 1.0f / (std::numeric_limits<uint32_t>::max() + 1.0f)); - } - - // Test non-deprecated version that takes SampleTypeTraits as a template - // parameter. { SCOPED_TRACE("UnsignedInt8SampleTypeTraits"); bus->Zero(); @@ -424,18 +390,6 @@ TEST_F(AudioBusTest, FromInterleavedPartial) { kPartialFrames * sizeof(*expected->channel(ch))); } - // Test deprecated version that takes |bytes_per_sample| as an input. - { - SCOPED_TRACE("int32_t"); - bus->Zero(); - bus->FromInterleavedPartial( - kTestVectorInt32 + kPartialStart * bus->channels(), kPartialStart, - kPartialFrames, sizeof(*kTestVectorInt32)); - VerifyAreEqual(bus.get(), expected.get()); - } - - // Test non-deprecated version that takes SampleTypeTraits as a template - // parameter. { SCOPED_TRACE("SignedInt32SampleTypeTraits"); bus->Zero(); @@ -456,43 +410,6 @@ TEST_F(AudioBusTest, ToInterleaved) { kTestVectorFrameCount * sizeof(*bus->channel(ch))); } - // Test deprecated version that takes |bytes_per_sample| as an input. - { - SCOPED_TRACE("uint8_t"); - uint8_t test_array[base::size(kTestVectorUint8)]; - bus->ToInterleaved(bus->frames(), sizeof(*kTestVectorUint8), test_array); - ASSERT_EQ(0, - memcmp(test_array, kTestVectorUint8, sizeof(kTestVectorUint8))); - } - { - SCOPED_TRACE("int16_t"); - int16_t test_array[base::size(kTestVectorInt16)]; - bus->ToInterleaved(bus->frames(), sizeof(*kTestVectorInt16), test_array); - ASSERT_EQ(0, - memcmp(test_array, kTestVectorInt16, sizeof(kTestVectorInt16))); - } - { - SCOPED_TRACE("int32_t"); - int32_t test_array[base::size(kTestVectorInt32)]; - bus->ToInterleaved(bus->frames(), sizeof(*kTestVectorInt32), test_array); - - // Some compilers get better precision than others on the half-max test, so - // let the test pass with an off by one check on the half-max. - int32_t alternative_acceptable_result[base::size(kTestVectorInt32)]; - memcpy(alternative_acceptable_result, kTestVectorInt32, - sizeof(kTestVectorInt32)); - ASSERT_EQ(alternative_acceptable_result[4], - std::numeric_limits<int32_t>::max() / 2); - alternative_acceptable_result[4]++; - - ASSERT_TRUE( - memcmp(test_array, kTestVectorInt32, sizeof(kTestVectorInt32)) == 0 || - memcmp(test_array, alternative_acceptable_result, - sizeof(alternative_acceptable_result)) == 0); - } - - // Test non-deprecated version that takes SampleTypeTraits as a template - // parameter. { SCOPED_TRACE("UnsignedInt8SampleTypeTraits"); uint8_t test_array[base::size(kTestVectorUint8)]; diff --git a/chromium/media/base/audio_decoder.h b/chromium/media/base/audio_decoder.h index 7361e251036..245ac2979a0 100644 --- a/chromium/media/base/audio_decoder.h +++ b/chromium/media/base/audio_decoder.h @@ -87,6 +87,9 @@ class MEDIA_EXPORT AudioDecoder : public Decoder { // Returns true if the decoder needs bitstream conversion before decoding. virtual bool NeedsBitstreamConversion() const; + // Returns the type of the decoder for statistics recording purposes. + virtual AudioDecoderType GetDecoderType() const = 0; + private: DISALLOW_COPY_AND_ASSIGN(AudioDecoder); }; diff --git a/chromium/media/base/audio_encoder.cc b/chromium/media/base/audio_encoder.cc index 64b5cab9f0a..8e0b81cf3c5 100644 --- a/chromium/media/base/audio_encoder.cc +++ b/chromium/media/base/audio_encoder.cc @@ -5,13 +5,15 @@ #include "media/base/audio_encoder.h" #include "base/logging.h" +#include "base/no_destructor.h" #include "base/time/time.h" #include "media/base/audio_timestamp_helper.h" namespace media { -// ----------------------------------------------------------------------------- -// EncodedAudioBuffer: +AudioEncoder::Options::Options() = default; +AudioEncoder::Options::Options(const Options&) = default; +AudioEncoder::Options::~Options() = default; EncodedAudioBuffer::EncodedAudioBuffer(const AudioParameters& params, std::unique_ptr<uint8_t[]> data, @@ -26,18 +28,7 @@ EncodedAudioBuffer::EncodedAudioBuffer(EncodedAudioBuffer&&) = default; EncodedAudioBuffer::~EncodedAudioBuffer() = default; -// ----------------------------------------------------------------------------- -// AudioEncoder: - -AudioEncoder::AudioEncoder(const AudioParameters& input_params, - EncodeCB encode_callback, - StatusCB status_callback) - : audio_input_params_(input_params), - encode_callback_(std::move(encode_callback)), - status_callback_(std::move(status_callback)) { - DCHECK(audio_input_params_.IsValid()); - DCHECK(!encode_callback_.is_null()); - DCHECK(!status_callback_.is_null()); +AudioEncoder::AudioEncoder() { DETACH_FROM_SEQUENCE(sequence_checker_); } @@ -45,35 +36,4 @@ AudioEncoder::~AudioEncoder() { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); } -void AudioEncoder::EncodeAudio(const AudioBus& audio_bus, - base::TimeTicks capture_time) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - DCHECK_EQ(audio_bus.channels(), audio_input_params_.channels()); - DCHECK(!capture_time.is_null()); - - DLOG_IF(ERROR, - !last_capture_time_.is_null() && - ((capture_time - last_capture_time_).InSecondsF() > - 1.5f * audio_bus.frames() / audio_input_params().sample_rate())) - << "Possibly frames were skipped, which may result in inaccuarate " - "timestamp calculation."; - - last_capture_time_ = capture_time; - - EncodeAudioImpl(audio_bus, capture_time); -} - -void AudioEncoder::Flush() { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - FlushImpl(); -} - -base::TimeTicks AudioEncoder::ComputeTimestamp( - int num_frames, - base::TimeTicks capture_time) const { - return capture_time - AudioTimestampHelper::FramesToTime( - num_frames, audio_input_params_.sample_rate()); -} - } // namespace media diff --git a/chromium/media/base/audio_encoder.h b/chromium/media/base/audio_encoder.h index c25218c9ee8..2d3362bebca 100644 --- a/chromium/media/base/audio_encoder.h +++ b/chromium/media/base/audio_encoder.h @@ -6,6 +6,7 @@ #define MEDIA_BASE_AUDIO_ENCODER_H_ #include <memory> +#include <vector> #include "base/callback.h" #include "base/sequence_checker.h" @@ -46,71 +47,72 @@ struct MEDIA_EXPORT EncodedAudioBuffer { const base::TimeTicks timestamp; }; -// Defines an interface for audio encoders. Concrete encoders must implement the -// EncodeAudioImpl() function. +// Defines an interface for audio encoders. class MEDIA_EXPORT AudioEncoder { public: + struct MEDIA_EXPORT Options { + Options(); + Options(const Options&); + ~Options(); + + base::Optional<int> bitrate; + + int channels; + + int sample_rate; + }; + + // A sequence of codec specific bytes, commonly known as extradata. + using CodecDescription = std::vector<uint8_t>; + // Signature of the callback invoked to provide the encoded audio data. It is - // invoked on the same sequence on which EncodeAudio() is called. The utility - // media::BindToCurrentLoop() can be used to create a callback that will be - // invoked on the same sequence it is constructed on. - using EncodeCB = base::RepeatingCallback<void(EncodedAudioBuffer output)>; + // invoked on the same sequence on which EncodeAudio() is called. + using OutputCB = + base::RepeatingCallback<void(EncodedAudioBuffer output, + base::Optional<CodecDescription>)>; // Signature of the callback to report errors. - using StatusCB = base::RepeatingCallback<void(Status error)>; - - // Constructs the encoder given the audio parameters of the input to this - // encoder, and a callback to trigger to provide the encoded audio data. - // |input_params| must be valid, and |encode_callback| and |status_callback| - // must not be null callbacks. All calls to EncodeAudio() must happen on the - // same sequence (usually an encoder blocking pool sequence), but the encoder - // itself can be constructed on any sequence. - AudioEncoder(const AudioParameters& input_params, - EncodeCB encode_callback, - StatusCB status_callback); + using StatusCB = base::OnceCallback<void(Status error)>; + + AudioEncoder(); AudioEncoder(const AudioEncoder&) = delete; AudioEncoder& operator=(const AudioEncoder&) = delete; virtual ~AudioEncoder(); - const AudioParameters& audio_input_params() const { - return audio_input_params_; - } - - // Performs various checks before calling EncodeAudioImpl() which does the - // actual encoding. - void EncodeAudio(const AudioBus& audio_bus, base::TimeTicks capture_time); + // Initializes an AudioEncoder with the given input option, executing + // the |done_cb| upon completion. |output_cb| is called for each encoded audio + // chunk. + // + // No AudioEncoder calls should be made before |done_cb| is executed. + virtual void Initialize(const Options& options, + OutputCB output_cb, + StatusCB done_cb) = 0; + + // Requests contents of |audio_bus| to be encoded. + // |capture_time| is a media time at the end of the audio piece in the + // |audio_bus|. + // + // |done_cb| is called upon encode completion and can possible convey an + // encoding error. It doesn't depend on future call to encoder's methods. + // |done_cb| will not be called from within this method. + // + // After the input, or several inputs, are encoded the encoder calls + // |output_cb|. + // |output_cb| may be called before or after |done_cb|, + // including before Encode() returns. + virtual void Encode(std::unique_ptr<AudioBus> audio_bus, + base::TimeTicks capture_time, + StatusCB done_cb) = 0; // Some encoders may choose to buffer audio frames before they encode them. - // This function provides a mechanism to drain and encode any buffered frames - // (if any). Must be called on the encoder sequence. - void Flush(); + // Requests all outputs for already encoded frames to be + // produced via |output_cb| and calls |done_cb| after that. + virtual void Flush(StatusCB done_cb) = 0; protected: - const EncodeCB& encode_callback() const { return encode_callback_; } - const StatusCB& status_callback() const { return status_callback_; } - base::TimeTicks last_capture_time() const { return last_capture_time_; } - - virtual void EncodeAudioImpl(const AudioBus& audio_bus, - base::TimeTicks capture_time) = 0; - - virtual void FlushImpl() = 0; - - // Computes the timestamp of an AudioBus which has |num_frames| and was - // captured at |capture_time|. This timestamp is the capture time of the first - // sample in that AudioBus. - base::TimeTicks ComputeTimestamp(int num_frames, - base::TimeTicks capture_time) const; - - private: - const AudioParameters audio_input_params_; - - const EncodeCB encode_callback_; - - const StatusCB status_callback_; + Options options_; - // The capture time of the most recent |audio_bus| delivered to - // EncodeAudio(). - base::TimeTicks last_capture_time_; + OutputCB output_cb_; SEQUENCE_CHECKER(sequence_checker_); }; diff --git a/chromium/media/base/audio_fifo.cc b/chromium/media/base/audio_fifo.cc index 75408b335fc..e1307316b98 100644 --- a/chromium/media/base/audio_fifo.cc +++ b/chromium/media/base/audio_fifo.cc @@ -7,6 +7,7 @@ #include <cstring> #include "base/check_op.h" +#include "base/trace_event/trace_event.h" namespace media { @@ -62,6 +63,8 @@ void AudioFifo::Push(const AudioBus* source) { const int source_size = source->frames(); CHECK_LE(source_size + frames(), max_frames_); + TRACE_EVENT2(TRACE_DISABLED_BY_DEFAULT("audio"), "AudioFifo::Push", "this", + static_cast<void*>(this), "frames", source_size); // Figure out if wrapping is needed and if so what segment sizes we need // when adding the new audio bus content to the FIFO. int append_size = 0; @@ -99,6 +102,9 @@ void AudioFifo::Consume(AudioBus* destination, // allocated memory in |destination| is sufficient. CHECK_LE(frames_to_consume + start_frame, destination->frames()); + TRACE_EVENT2(TRACE_DISABLED_BY_DEFAULT("audio"), "AudioFifo::Consume", "this", + static_cast<void*>(this), "frames", frames_to_consume); + // Figure out if wrapping is needed and if so what segment sizes we need // when removing audio bus content from the FIFO. int consume_size = 0; diff --git a/chromium/media/base/audio_latency.cc b/chromium/media/base/audio_latency.cc index b55a2da37e1..f15e8f95ba5 100644 --- a/chromium/media/base/audio_latency.cc +++ b/chromium/media/base/audio_latency.cc @@ -25,6 +25,7 @@ namespace media { namespace { + #if !defined(OS_WIN) // Taken from "Bit Twiddling Hacks" // http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 @@ -39,11 +40,30 @@ uint32_t RoundUpToPowerOfTwo(uint32_t v) { return v; } #endif + +#if defined(OS_ANDROID) +// WebAudio renderer's quantum size (frames per callback) that is used for +// calculating the "interactive" buffer size. +// TODO(crbug.com/988121): This number needs to be passed down from Blink when +// user-selectable render quantum size is implemented. +const int kWebAudioRenderQuantumSize = 128; + +// From media/renderers/paint_canvas_video_renderer.cc. To calculate the optimum +// buffer size for Pixel 3/4/5 devices, which has a HW buffer size of 96 frames. +int GCD(int a, int b) { + return a == 0 ? b : GCD(b % a, a); +} + +int LCM(int a, int b) { + return a / GCD(a, b) * b; +} +#endif + } // namespace // static bool AudioLatency::IsResamplingPassthroughSupported(LatencyType type) { -#if BUILDFLAG(IS_ASH) +#if BUILDFLAG(IS_CHROMEOS_ASH) return true; #elif defined(OS_ANDROID) // Only N MR1+ has support for OpenSLES performance modes which allow for @@ -139,13 +159,28 @@ int AudioLatency::GetRtcBufferSize(int sample_rate, int hardware_buffer_size) { // static int AudioLatency::GetInteractiveBufferSize(int hardware_buffer_size) { + CHECK_GT(hardware_buffer_size, 0); + #if defined(OS_ANDROID) // Always log this because it's relatively hard to get this // information out. LOG(INFO) << "audioHardwareBufferSize = " << hardware_buffer_size; -#endif + if (hardware_buffer_size >= kWebAudioRenderQuantumSize) + return hardware_buffer_size; + + // HW buffer size is smaller than the Web Audio's render quantum size, so + // compute LCM to avoid glitches and regulate the workload per callback. + // (e.g. 96 vs 128 -> 384) Also cap the buffer size to 4 render quanta + // (512 frames ~= 10ms at 48K) if LCM goes beyond interactive latency range. + int sensible_buffer_size = std::min( + LCM(hardware_buffer_size, kWebAudioRenderQuantumSize), + kWebAudioRenderQuantumSize * 4); + + return sensible_buffer_size; +#else return hardware_buffer_size; +#endif // defined(OS_ANDROID) } int AudioLatency::GetExactBufferSize(base::TimeDelta duration, diff --git a/chromium/media/base/audio_latency_unittest.cc b/chromium/media/base/audio_latency_unittest.cc index 1f627afe191..aa9ac668785 100644 --- a/chromium/media/base/audio_latency_unittest.cc +++ b/chromium/media/base/audio_latency_unittest.cc @@ -145,8 +145,31 @@ TEST(AudioLatency, HighLatencyBufferSizes) { } TEST(AudioLatency, InteractiveBufferSizes) { - for (int i = 6400; i <= 204800; i *= 2) - EXPECT_EQ(i / 100, AudioLatency::GetInteractiveBufferSize(i / 100)); + // The |first| is a requested buffer size and and the |second| is a computed + // "interactive" buffer size from the method. + std::vector<std::pair<int, int>> buffer_size_pairs = { +#if defined(OS_ANDROID) + {64, 128}, + {96, 384}, // Pixel 3, 4, 5. (See crbug.com/1090441) + {240, 240}, // Nexus 7 + {144, 144}, // Galaxy Nexus + // Irregular device buffer size + {100, 512}, + {127, 512}, +#else + {64, 64}, +#endif // defined(OS_ANDROID) + {128, 128}, + {256, 256}, + {512, 512}, + {1024, 1024}, + {2048, 2048} + }; + + for (auto & buffer_size_pair : buffer_size_pairs) { + EXPECT_EQ(buffer_size_pair.second, + AudioLatency::GetInteractiveBufferSize(buffer_size_pair.first)); + } } TEST(AudioLatency, RtcBufferSizes) { diff --git a/chromium/media/base/audio_renderer.h b/chromium/media/base/audio_renderer.h index 533c46150d7..15701b34294 100644 --- a/chromium/media/base/audio_renderer.h +++ b/chromium/media/base/audio_renderer.h @@ -69,6 +69,9 @@ class MEDIA_EXPORT AudioRenderer { // preservation when playing back at speeds other than 1.0. virtual void SetPreservesPitch(bool preserves_pitch) = 0; + // Sets a flag indicating whether the audio stream was initiated by autoplay. + virtual void SetAutoplayInitiated(bool autoplay_initiated) = 0; + private: DISALLOW_COPY_AND_ASSIGN(AudioRenderer); }; diff --git a/chromium/media/base/bind_to_current_loop.h b/chromium/media/base/bind_to_current_loop.h index ec1352a6786..4ddf4fc31c1 100644 --- a/chromium/media/base/bind_to_current_loop.h +++ b/chromium/media/base/bind_to_current_loop.h @@ -5,121 +5,30 @@ #ifndef MEDIA_BASE_BIND_TO_CURRENT_LOOP_H_ #define MEDIA_BASE_BIND_TO_CURRENT_LOOP_H_ -#include <memory> - -#include "base/bind.h" +#include "base/bind_post_task.h" +#include "base/callback.h" #include "base/location.h" -#include "base/sequenced_task_runner.h" -#include "base/single_thread_task_runner.h" #include "base/threading/sequenced_task_runner_handle.h" -// This is a helper utility for binding a OnceCallback or RepeatingCallback to a -// given TaskRunner. The typical use is when |a| (of class |A|) wants to hand a -// callback such as base::BindOnce(&A::AMethod, a) to |b|, but needs to ensure -// that when |b| executes the callback, it does so on |a|'s task_runner's -// MessageLoop. -// -// Typical usage: request to be called back on the current thread: -// other->StartAsyncProcessAndCallMeBack( -// media::BindToLoop(task_runner, base::BindOnce(&MyClass::MyMethod, this))); -// -// media::BindToLoop returns the same type of callback to the given -// callback. I.e. it returns a RepeatingCallback for a given RepeatingCallback, -// and returns OnceCallback for a given OnceCallback. -// -// The function BindToCurrentLoop is shorthand to bind to the calling function's -// current MessageLoop. +// Helpers for using base::BindPostTask() with the TaskRunner for the current +// sequence, ie. base::SequencedTaskRunnerHandle::Get(). namespace media { -namespace internal { - -template <typename Signature, typename... Args> -base::OnceClosure MakeClosure(base::RepeatingCallback<Signature>* callback, - Args&&... args) { - return base::BindOnce(*callback, std::forward<Args>(args)...); -} - -template <typename Signature, typename... Args> -base::OnceClosure MakeClosure(base::OnceCallback<Signature>* callback, - Args&&... args) { - return base::BindOnce(std::move(*callback), std::forward<Args>(args)...); -} - -template <typename CallbackType> -class TrampolineHelper { - public: - TrampolineHelper(const base::Location& posted_from, - scoped_refptr<base::SequencedTaskRunner> task_runner, - CallbackType callback) - : posted_from_(posted_from), - task_runner_(std::move(task_runner)), - callback_(std::move(callback)) { - DCHECK(task_runner_); - DCHECK(callback_); - } - - template <typename... Args> - void Run(Args... args) { - // MakeClosure consumes |callback_| if it's OnceCallback. - task_runner_->PostTask( - posted_from_, MakeClosure(&callback_, std::forward<Args>(args)...)); - } - - ~TrampolineHelper() { - if (callback_) { - task_runner_->PostTask( - posted_from_, - base::BindOnce(&TrampolineHelper::ClearCallbackOnTargetTaskRunner, - std::move(callback_))); - } - } - - private: - static void ClearCallbackOnTargetTaskRunner(CallbackType) {} - - base::Location posted_from_; - scoped_refptr<base::SequencedTaskRunner> task_runner_; - CallbackType callback_; -}; - -} // namespace internal - -template <typename... Args> -inline base::RepeatingCallback<void(Args...)> BindToLoop( - scoped_refptr<base::SequencedTaskRunner> task_runner, - base::RepeatingCallback<void(Args...)> cb) { - using CallbackType = base::RepeatingCallback<void(Args...)>; - using Helper = internal::TrampolineHelper<CallbackType>; - using RunnerType = void (Helper::*)(Args...); - RunnerType run = &Helper::Run; - // TODO(tzik): Propagate FROM_HERE from the caller. - return base::BindRepeating( - run, std::make_unique<Helper>(FROM_HERE, task_runner, std::move(cb))); -} - -template <typename... Args> -inline base::OnceCallback<void(Args...)> BindToLoop( - scoped_refptr<base::SequencedTaskRunner> task_runner, - base::OnceCallback<void(Args...)> cb) { - using CallbackType = base::OnceCallback<void(Args...)>; - using Helper = internal::TrampolineHelper<CallbackType>; - using RunnerType = void (Helper::*)(Args...); - RunnerType run = &Helper::Run; - // TODO(tzik): Propagate FROM_HERE from the caller. - return base::BindOnce( - run, std::make_unique<Helper>(FROM_HERE, task_runner, std::move(cb))); -} template <typename... Args> inline base::RepeatingCallback<void(Args...)> BindToCurrentLoop( - base::RepeatingCallback<void(Args...)> cb) { - return BindToLoop(base::SequencedTaskRunnerHandle::Get(), std::move(cb)); + base::RepeatingCallback<void(Args...)> cb, + const base::Location& location = FROM_HERE) { + return base::BindPostTask(base::SequencedTaskRunnerHandle::Get(), + std::move(cb), location); } template <typename... Args> inline base::OnceCallback<void(Args...)> BindToCurrentLoop( - base::OnceCallback<void(Args...)> cb) { - return BindToLoop(base::SequencedTaskRunnerHandle::Get(), std::move(cb)); + base::OnceCallback<void(Args...)> cb, + const base::Location& location = FROM_HERE) { + return base::BindPostTask(base::SequencedTaskRunnerHandle::Get(), + std::move(cb), location); } } // namespace media diff --git a/chromium/media/base/bind_to_current_loop_unittest.cc b/chromium/media/base/bind_to_current_loop_unittest.cc deleted file mode 100644 index faddb41d379..00000000000 --- a/chromium/media/base/bind_to_current_loop_unittest.cc +++ /dev/null @@ -1,376 +0,0 @@ -// Copyright (c) 2012 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include "media/base/bind_to_current_loop.h" - -#include <memory> -#include <utility> - -#include "base/bind.h" -#include "base/memory/free_deleter.h" -#include "base/run_loop.h" -#include "base/synchronization/waitable_event.h" -#include "base/test/task_environment.h" -#include "base/threading/thread.h" -#include "base/threading/thread_checker_impl.h" -#include "testing/gtest/include/gtest/gtest.h" - -namespace media { - -void BoundBoolSet(bool* var, bool val) { - *var = val; -} - -void BoundBoolSetFromUniquePtr(bool* var, std::unique_ptr<bool> val) { - *var = *val; -} - -void BoundBoolSetFromUniquePtrFreeDeleter( - bool* var, - std::unique_ptr<bool, base::FreeDeleter> val) { - *var = *val; -} - -void BoundBoolSetFromUniquePtrArray(bool* var, std::unique_ptr<bool[]> val) { - *var = val[0]; -} - -void BoundBoolSetFromConstRef(bool* var, const bool& val) { - *var = val; -} - -void BoundIntegersSet(int* a_var, int* b_var, int a_val, int b_val) { - *a_var = a_val; - *b_var = b_val; -} - -struct ThreadRestrictionChecker { - void Run() { EXPECT_TRUE(thread_checker_.CalledOnValidThread()); } - - ~ThreadRestrictionChecker() { - EXPECT_TRUE(thread_checker_.CalledOnValidThread()); - } - - base::ThreadCheckerImpl thread_checker_; -}; - -void ClearReference(base::OnceClosure cb) {} - -// Various tests that check that the bound function is only actually executed -// on the message loop, not during the original Run. -class BindToCurrentLoopTest : public ::testing::Test { - protected: - base::test::SingleThreadTaskEnvironment task_environment_; -}; - -TEST_F(BindToCurrentLoopTest, RepeatingClosure) { - // Test the closure is run inside the loop, not outside it. - base::WaitableEvent waiter(base::WaitableEvent::ResetPolicy::AUTOMATIC, - base::WaitableEvent::InitialState::NOT_SIGNALED); - base::RepeatingClosure cb = BindToCurrentLoop(base::BindRepeating( - &base::WaitableEvent::Signal, base::Unretained(&waiter))); - cb.Run(); - EXPECT_FALSE(waiter.IsSignaled()); - base::RunLoop().RunUntilIdle(); - EXPECT_TRUE(waiter.IsSignaled()); -} - -TEST_F(BindToCurrentLoopTest, OnceClosure) { - // Test the closure is run inside the loop, not outside it. - base::WaitableEvent waiter(base::WaitableEvent::ResetPolicy::AUTOMATIC, - base::WaitableEvent::InitialState::NOT_SIGNALED); - base::OnceClosure cb = BindToCurrentLoop( - base::BindOnce(&base::WaitableEvent::Signal, base::Unretained(&waiter))); - std::move(cb).Run(); - EXPECT_FALSE(waiter.IsSignaled()); - base::RunLoop().RunUntilIdle(); - EXPECT_TRUE(waiter.IsSignaled()); -} - -TEST_F(BindToCurrentLoopTest, BoolRepeating) { - bool bool_var = false; - base::RepeatingCallback<void(bool)> cb = - BindToCurrentLoop(base::BindRepeating(&BoundBoolSet, &bool_var)); - cb.Run(true); - EXPECT_FALSE(bool_var); - base::RunLoop().RunUntilIdle(); - EXPECT_TRUE(bool_var); - - cb.Run(false); - EXPECT_TRUE(bool_var); - base::RunLoop().RunUntilIdle(); - EXPECT_FALSE(bool_var); -} - -TEST_F(BindToCurrentLoopTest, BoolOnce) { - bool bool_var = false; - base::OnceCallback<void(bool)> cb = - BindToCurrentLoop(base::BindOnce(&BoundBoolSet, &bool_var)); - std::move(cb).Run(true); - EXPECT_FALSE(bool_var); - base::RunLoop().RunUntilIdle(); - EXPECT_TRUE(bool_var); -} - -TEST_F(BindToCurrentLoopTest, BoundUniquePtrBoolRepeating) { - bool bool_val = false; - std::unique_ptr<bool> unique_ptr_bool(new bool(true)); - base::RepeatingClosure cb = BindToCurrentLoop(base::BindRepeating( - &BoundBoolSetFromUniquePtr, &bool_val, base::Passed(&unique_ptr_bool))); - cb.Run(); - EXPECT_FALSE(bool_val); - base::RunLoop().RunUntilIdle(); - EXPECT_TRUE(bool_val); -} - -TEST_F(BindToCurrentLoopTest, BoundUniquePtrBoolOnce) { - bool bool_val = false; - std::unique_ptr<bool> unique_ptr_bool(new bool(true)); - base::OnceClosure cb = BindToCurrentLoop(base::BindOnce( - &BoundBoolSetFromUniquePtr, &bool_val, std::move(unique_ptr_bool))); - std::move(cb).Run(); - EXPECT_FALSE(bool_val); - base::RunLoop().RunUntilIdle(); - EXPECT_TRUE(bool_val); -} - -TEST_F(BindToCurrentLoopTest, PassedUniquePtrBoolRepeating) { - bool bool_val = false; - base::RepeatingCallback<void(std::unique_ptr<bool>)> cb = BindToCurrentLoop( - base::BindRepeating(&BoundBoolSetFromUniquePtr, &bool_val)); - cb.Run(std::make_unique<bool>(true)); - EXPECT_FALSE(bool_val); - base::RunLoop().RunUntilIdle(); - EXPECT_TRUE(bool_val); - - cb.Run(std::make_unique<bool>(false)); - EXPECT_TRUE(bool_val); - base::RunLoop().RunUntilIdle(); - EXPECT_FALSE(bool_val); -} - -TEST_F(BindToCurrentLoopTest, PassedUniquePtrBoolOnce) { - bool bool_val = false; - base::OnceCallback<void(std::unique_ptr<bool>)> cb = - BindToCurrentLoop(base::BindOnce(&BoundBoolSetFromUniquePtr, &bool_val)); - std::move(cb).Run(std::make_unique<bool>(true)); - EXPECT_FALSE(bool_val); - base::RunLoop().RunUntilIdle(); - EXPECT_TRUE(bool_val); -} - -TEST_F(BindToCurrentLoopTest, BoundUniquePtrArrayBoolRepeating) { - bool bool_val = false; - std::unique_ptr<bool[]> unique_ptr_array_bool(new bool[1]); - unique_ptr_array_bool[0] = true; - base::RepeatingClosure cb = BindToCurrentLoop( - base::BindRepeating(&BoundBoolSetFromUniquePtrArray, &bool_val, - base::Passed(&unique_ptr_array_bool))); - cb.Run(); - EXPECT_FALSE(bool_val); - base::RunLoop().RunUntilIdle(); - EXPECT_TRUE(bool_val); -} - -TEST_F(BindToCurrentLoopTest, BoundUniquePtrArrayBoolOnce) { - bool bool_val = false; - std::unique_ptr<bool[]> unique_ptr_array_bool(new bool[1]); - unique_ptr_array_bool[0] = true; - base::OnceClosure cb = BindToCurrentLoop( - base::BindOnce(&BoundBoolSetFromUniquePtrArray, &bool_val, - std::move(unique_ptr_array_bool))); - std::move(cb).Run(); - EXPECT_FALSE(bool_val); - base::RunLoop().RunUntilIdle(); - EXPECT_TRUE(bool_val); -} - -TEST_F(BindToCurrentLoopTest, PassedUniquePtrArrayBoolRepeating) { - bool bool_val = false; - base::RepeatingCallback<void(std::unique_ptr<bool[]>)> cb = BindToCurrentLoop( - base::BindRepeating(&BoundBoolSetFromUniquePtrArray, &bool_val)); - - std::unique_ptr<bool[]> unique_ptr_array_bool(new bool[1]); - unique_ptr_array_bool[0] = true; - cb.Run(std::move(unique_ptr_array_bool)); - EXPECT_FALSE(bool_val); - base::RunLoop().RunUntilIdle(); - EXPECT_TRUE(bool_val); - - unique_ptr_array_bool.reset(new bool[1]); - unique_ptr_array_bool[0] = false; - cb.Run(std::move(unique_ptr_array_bool)); - EXPECT_TRUE(bool_val); - base::RunLoop().RunUntilIdle(); - EXPECT_FALSE(bool_val); -} - -TEST_F(BindToCurrentLoopTest, PassedUniquePtrArrayBoolOnce) { - bool bool_val = false; - base::OnceCallback<void(std::unique_ptr<bool[]>)> cb = BindToCurrentLoop( - base::BindOnce(&BoundBoolSetFromUniquePtrArray, &bool_val)); - - std::unique_ptr<bool[]> unique_ptr_array_bool(new bool[1]); - unique_ptr_array_bool[0] = true; - std::move(cb).Run(std::move(unique_ptr_array_bool)); - EXPECT_FALSE(bool_val); - base::RunLoop().RunUntilIdle(); - EXPECT_TRUE(bool_val); -} - -TEST_F(BindToCurrentLoopTest, BoundUniquePtrFreeDeleterBoolRepeating) { - bool bool_val = false; - std::unique_ptr<bool, base::FreeDeleter> unique_ptr_free_deleter_bool( - static_cast<bool*>(malloc(sizeof(bool)))); - *unique_ptr_free_deleter_bool = true; - base::RepeatingClosure cb = BindToCurrentLoop( - base::BindRepeating(&BoundBoolSetFromUniquePtrFreeDeleter, &bool_val, - base::Passed(&unique_ptr_free_deleter_bool))); - cb.Run(); - EXPECT_FALSE(bool_val); - base::RunLoop().RunUntilIdle(); - EXPECT_TRUE(bool_val); -} - -TEST_F(BindToCurrentLoopTest, BoundUniquePtrFreeDeleterBoolOnce) { - bool bool_val = false; - std::unique_ptr<bool, base::FreeDeleter> unique_ptr_free_deleter_bool( - static_cast<bool*>(malloc(sizeof(bool)))); - *unique_ptr_free_deleter_bool = true; - base::OnceClosure cb = BindToCurrentLoop( - base::BindOnce(&BoundBoolSetFromUniquePtrFreeDeleter, &bool_val, - std::move(unique_ptr_free_deleter_bool))); - std::move(cb).Run(); - EXPECT_FALSE(bool_val); - base::RunLoop().RunUntilIdle(); - EXPECT_TRUE(bool_val); -} - -TEST_F(BindToCurrentLoopTest, PassedUniquePtrFreeDeleterBoolRepeating) { - bool bool_val = false; - base::RepeatingCallback<void(std::unique_ptr<bool, base::FreeDeleter>)> cb = - BindToCurrentLoop(base::BindRepeating( - &BoundBoolSetFromUniquePtrFreeDeleter, &bool_val)); - - std::unique_ptr<bool, base::FreeDeleter> unique_ptr_free_deleter_bool( - static_cast<bool*>(malloc(sizeof(bool)))); - *unique_ptr_free_deleter_bool = true; - cb.Run(std::move(unique_ptr_free_deleter_bool)); - EXPECT_FALSE(bool_val); - base::RunLoop().RunUntilIdle(); - EXPECT_TRUE(bool_val); - - unique_ptr_free_deleter_bool.reset(static_cast<bool*>(malloc(sizeof(bool)))); - *unique_ptr_free_deleter_bool = false; - cb.Run(std::move(unique_ptr_free_deleter_bool)); - EXPECT_TRUE(bool_val); - base::RunLoop().RunUntilIdle(); - EXPECT_FALSE(bool_val); -} - -TEST_F(BindToCurrentLoopTest, PassedUniquePtrFreeDeleterBoolOnce) { - bool bool_val = false; - base::OnceCallback<void(std::unique_ptr<bool, base::FreeDeleter>)> cb = - BindToCurrentLoop( - base::BindOnce(&BoundBoolSetFromUniquePtrFreeDeleter, &bool_val)); - - std::unique_ptr<bool, base::FreeDeleter> unique_ptr_free_deleter_bool( - static_cast<bool*>(malloc(sizeof(bool)))); - *unique_ptr_free_deleter_bool = true; - std::move(cb).Run(std::move(unique_ptr_free_deleter_bool)); - EXPECT_FALSE(bool_val); - base::RunLoop().RunUntilIdle(); - EXPECT_TRUE(bool_val); -} - -TEST_F(BindToCurrentLoopTest, IntegersRepeating) { - int a = 0; - int b = 0; - base::RepeatingCallback<void(int, int)> cb = - BindToCurrentLoop(base::BindRepeating(&BoundIntegersSet, &a, &b)); - cb.Run(1, -1); - EXPECT_EQ(a, 0); - EXPECT_EQ(b, 0); - base::RunLoop().RunUntilIdle(); - EXPECT_EQ(a, 1); - EXPECT_EQ(b, -1); - - cb.Run(2, -2); - EXPECT_EQ(a, 1); - EXPECT_EQ(b, -1); - base::RunLoop().RunUntilIdle(); - EXPECT_EQ(a, 2); - EXPECT_EQ(b, -2); -} - -TEST_F(BindToCurrentLoopTest, IntegersOnce) { - int a = 0; - int b = 0; - base::OnceCallback<void(int, int)> cb = - BindToCurrentLoop(base::BindOnce(&BoundIntegersSet, &a, &b)); - std::move(cb).Run(1, -1); - EXPECT_EQ(a, 0); - EXPECT_EQ(b, 0); - base::RunLoop().RunUntilIdle(); - EXPECT_EQ(a, 1); - EXPECT_EQ(b, -1); -} - -TEST_F(BindToCurrentLoopTest, DestroyedOnBoundLoopRepeating) { - base::Thread target_thread("testing"); - ASSERT_TRUE(target_thread.Start()); - - // Ensure that the bound object is also destroyed on the correct thread even - // if the last reference to the callback is dropped on the other thread. - base::RepeatingClosure cb = BindToCurrentLoop( - base::BindRepeating(&ThreadRestrictionChecker::Run, - std::make_unique<ThreadRestrictionChecker>())); - target_thread.task_runner()->PostTask(FROM_HERE, std::move(cb)); - ASSERT_FALSE(cb); - target_thread.FlushForTesting(); - base::RunLoop().RunUntilIdle(); - - // Ensure that the bound object is destroyed on the target thread even if - // the callback is destroyed without invocation. - cb = BindToCurrentLoop( - base::BindRepeating(&ThreadRestrictionChecker::Run, - std::make_unique<ThreadRestrictionChecker>())); - target_thread.task_runner()->PostTask( - FROM_HERE, base::BindOnce(&ClearReference, std::move(cb))); - target_thread.FlushForTesting(); - ASSERT_FALSE(cb); - base::RunLoop().RunUntilIdle(); - - target_thread.Stop(); -} - -TEST_F(BindToCurrentLoopTest, DestroyedOnBoundLoopOnce) { - base::Thread target_thread("testing"); - ASSERT_TRUE(target_thread.Start()); - - // Ensure that the bound object is also destroyed on the correct thread even - // if the last reference to the callback is dropped on the other thread. - base::OnceClosure cb = BindToCurrentLoop( - base::BindOnce(&ThreadRestrictionChecker::Run, - std::make_unique<ThreadRestrictionChecker>())); - target_thread.task_runner()->PostTask(FROM_HERE, std::move(cb)); - ASSERT_FALSE(cb); - target_thread.FlushForTesting(); - base::RunLoop().RunUntilIdle(); - - // Ensure that the bound object is destroyed on the target thread even if - // the callback is destroyed without invocation. - cb = BindToCurrentLoop( - base::BindOnce(&ThreadRestrictionChecker::Run, - std::make_unique<ThreadRestrictionChecker>())); - target_thread.task_runner()->PostTask( - FROM_HERE, base::BindOnce(&ClearReference, std::move(cb))); - target_thread.FlushForTesting(); - ASSERT_FALSE(cb); - base::RunLoop().RunUntilIdle(); - - target_thread.Stop(); -} - -} // namespace media diff --git a/chromium/media/base/cdm_context.cc b/chromium/media/base/cdm_context.cc index 8f8cd843f64..9b30303dd4b 100644 --- a/chromium/media/base/cdm_context.cc +++ b/chromium/media/base/cdm_context.cc @@ -30,11 +30,11 @@ std::string CdmContext::CdmIdToString(const base::UnguessableToken* cdm_id) { return cdm_id ? cdm_id->ToString() : "null"; } +#if defined(OS_WIN) bool CdmContext::RequiresMediaFoundationRenderer() { return false; } -#if defined(OS_WIN) bool CdmContext::GetMediaFoundationCdmProxy( GetMediaFoundationCdmProxyCB get_mf_cdm_proxy_cb) { return false; @@ -53,7 +53,7 @@ FuchsiaCdmContext* CdmContext::GetFuchsiaCdmContext() { } #endif -#if BUILDFLAG(IS_ASH) +#if BUILDFLAG(IS_CHROMEOS_ASH) chromeos::ChromeOsCdmContext* CdmContext::GetChromeOsCdmContext() { return nullptr; } diff --git a/chromium/media/base/cdm_context.h b/chromium/media/base/cdm_context.h index d555e21e360..aacea9ec764 100644 --- a/chromium/media/base/cdm_context.h +++ b/chromium/media/base/cdm_context.h @@ -19,7 +19,7 @@ struct IMFCdmProxy; #endif -#if BUILDFLAG(IS_ASH) +#if BUILDFLAG(IS_CHROMEOS_ASH) namespace chromeos { class ChromeOsCdmContext; } @@ -86,10 +86,6 @@ class MEDIA_EXPORT CdmContext { // occurs implicitly along with decoding). virtual Decryptor* GetDecryptor(); - // Returns whether the CDM requires Media Foundation-based media Renderer. - // Should only return true on Windows. - virtual bool RequiresMediaFoundationRenderer(); - // Returns an ID that can be used to find a remote CDM, in which case this CDM // serves as a proxy to the remote one. Returns base::nullopt when remote CDM // is not supported (e.g. this CDM is a local CDM). @@ -98,6 +94,11 @@ class MEDIA_EXPORT CdmContext { static std::string CdmIdToString(const base::UnguessableToken* cdm_id); #if defined(OS_WIN) + // Returns whether the CDM requires Media Foundation-based media Renderer. + // This is separate from GetMediaFoundationCdmProxy() since it needs to be + // a sync call called in the render process to setup the media pipeline. + virtual bool RequiresMediaFoundationRenderer(); + using GetMediaFoundationCdmProxyCB = base::OnceCallback<void(Microsoft::WRL::ComPtr<IMFCdmProxy>)>; // This allows a CdmContext to expose an IMFTrustedInput instance for use in @@ -122,7 +123,7 @@ class MEDIA_EXPORT CdmContext { virtual FuchsiaCdmContext* GetFuchsiaCdmContext(); #endif -#if BUILDFLAG(IS_ASH) +#if BUILDFLAG(IS_CHROMEOS_ASH) // Returns a ChromeOsCdmContext interface when the context is backed by the // ChromeOS CdmFactoryDaemon. Otherwise return nullptr. virtual chromeos::ChromeOsCdmContext* GetChromeOsCdmContext(); diff --git a/chromium/media/base/cdm_promise_adapter.cc b/chromium/media/base/cdm_promise_adapter.cc index af1b604963d..da0ae36e712 100644 --- a/chromium/media/base/cdm_promise_adapter.cc +++ b/chromium/media/base/cdm_promise_adapter.cc @@ -8,13 +8,26 @@ namespace media { +namespace { + +CdmPromise::SystemCode ToSystemCode(CdmPromiseAdapter::ClearReason reason) { + switch (reason) { + case CdmPromiseAdapter::ClearReason::kDestruction: + return CdmPromise::SystemCode::kAborted; + case CdmPromiseAdapter::ClearReason::kConnectionError: + return CdmPromise::SystemCode::kConnectionError; + } +} + +} // namespace + CdmPromiseAdapter::CdmPromiseAdapter() : next_promise_id_(kInvalidPromiseId + 1) {} CdmPromiseAdapter::~CdmPromiseAdapter() { DCHECK(thread_checker_.CalledOnValidThread()); DLOG_IF(WARNING, !promises_.empty()) << "There are unfulfilled promises"; - Clear(); + Clear(ClearReason::kDestruction); } uint32_t CdmPromiseAdapter::SavePromise(std::unique_ptr<CdmPromise> promise) { @@ -62,13 +75,12 @@ void CdmPromiseAdapter::RejectPromise(uint32_t promise_id, promise->reject(exception_code, system_code, error_message); } -void CdmPromiseAdapter::Clear() { +void CdmPromiseAdapter::Clear(ClearReason reason) { // Reject all outstanding promises. DCHECK(thread_checker_.CalledOnValidThread()); for (auto& promise : promises_) { promise.second->reject(CdmPromise::Exception::INVALID_STATE_ERROR, - CdmPromise::SystemCode::kAborted, - "Operation aborted."); + ToSystemCode(reason), "Operation aborted."); } promises_.clear(); } diff --git a/chromium/media/base/cdm_promise_adapter.h b/chromium/media/base/cdm_promise_adapter.h index d6a088617eb..f678df4b340 100644 --- a/chromium/media/base/cdm_promise_adapter.h +++ b/chromium/media/base/cdm_promise_adapter.h @@ -42,8 +42,13 @@ class MEDIA_EXPORT CdmPromiseAdapter { uint32_t system_code, const std::string& error_message); + enum class ClearReason { + kDestruction, + kConnectionError, + }; + // Rejects and clears all |promises_|. - void Clear(); + void Clear(ClearReason reason); private: // A map between promise IDs and CdmPromises. diff --git a/chromium/media/base/decoder.cc b/chromium/media/base/decoder.cc index 50c2b8d52fa..8e06f1e2514 100644 --- a/chromium/media/base/decoder.cc +++ b/chromium/media/base/decoder.cc @@ -18,4 +18,56 @@ bool Decoder::SupportsDecryption() const { return false; } +std::string GetDecoderName(VideoDecoderType type) { + switch (type) { + case VideoDecoderType::kUnknown: + return "Unknown Video Decoder"; + case VideoDecoderType::kFFmpeg: + return "FFmpegVideoDecoder"; + case VideoDecoderType::kVpx: + return "VpxVideoDecoder"; + case VideoDecoderType::kAom: + return "AomVideoDecoder"; + case VideoDecoderType::kMojo: + return "MojoVideoDecoder"; + case VideoDecoderType::kDecrypting: + return "DecryptingVideoDecoder"; + case VideoDecoderType::kDav1d: + return "Dav1dVideoDecoder"; + case VideoDecoderType::kFuchsia: + return "FuchsiaVideoDecoder"; + case VideoDecoderType::kMediaCodec: + return "MediaCodecVideoDecoder"; + case VideoDecoderType::kGav1: + return "Gav1VideoDecoder"; + case VideoDecoderType::kD3D11: + return "D3D11VideoDecoder"; + case VideoDecoderType::kVaapi: + return "VaapiVideoDecodeAccelerator"; + case VideoDecoderType::kBroker: + return "VideoDecoderBroker"; + case VideoDecoderType::kChromeOs: + return "VideoDecoderPipeline (ChromeOs)"; + case VideoDecoderType::kVda: + return "VideoDecodeAccelerator"; + } +} + +std::string GetDecoderName(AudioDecoderType type) { + switch (type) { + case AudioDecoderType::kUnknown: + return "Unknown Audio Decoder"; + case AudioDecoderType::kFFmpeg: + return "FFmpegAudioDecoder"; + case AudioDecoderType::kMojo: + return "MojoAudioDecoder"; + case AudioDecoderType::kDecrypting: + return "DecryptingAudioDecoder"; + case AudioDecoderType::kMediaCodec: + return "MediaCodecAudioDecoder"; + case AudioDecoderType::kBroker: + return "AudioDecoderBroker"; + } +} + } // namespace media diff --git a/chromium/media/base/decoder.h b/chromium/media/base/decoder.h index 8ebec529466..fb43f9ca369 100644 --- a/chromium/media/base/decoder.h +++ b/chromium/media/base/decoder.h @@ -14,6 +14,47 @@ namespace media { +// List of known AudioDecoder implementations; recorded to UKM, always add new +// values to the end and do not reorder or delete values from this list. +enum class AudioDecoderType : int { + kUnknown = 0, // Decoder name string is not recognized or n/a. + kFFmpeg = 1, // FFmpegAudioDecoder + kMojo = 2, // MojoAudioDecoder + kDecrypting = 3, // DecryptingAudioDecoder + kMediaCodec = 4, // MediaCodecAudioDecoder (Android) + kBroker = 5, // AudioDecoderBroker + + kMaxValue = kBroker // Keep this at the end and equal to the last entry. +}; + +// List of known VideoDecoder implementations; recorded to UKM, always add new +// values to the end and do not reorder or delete values from this list. +enum class VideoDecoderType : int { + kUnknown = 0, // Decoder name string is not recognized or n/a. + // kGpu = 1, // GpuVideoDecoder (DEPRECATED) + kFFmpeg = 2, // FFmpegVideoDecoder + kVpx = 3, // VpxVideoDecoder + kAom = 4, // AomVideoDecoder + kMojo = 5, // MojoVideoDecoder + kDecrypting = 6, // DecryptingVideoDecoder + kDav1d = 7, // Dav1dVideoDecoder + kFuchsia = 8, // FuchsiaVideoDecoder + kMediaCodec = 9, // MediaCodecVideoDecoder (Android) + kGav1 = 10, // Gav1VideoDecoder + kD3D11 = 11, // D3D11VideoDecoder + kVaapi = 12, // VaapiVideoDecodeAccelerator + kBroker = 13, // VideoDecoderBroker (Webcodecs) + kVda = 14, // VDAVideoDecoder + + // Chromeos uses VideoDecoderPipeline. This could potentially become more + // granulated in the future. + kChromeOs = 15, + kMaxValue = kChromeOs // Keep this at the end and equal to the last entry. +}; + +MEDIA_EXPORT std::string GetDecoderName(AudioDecoderType type); +MEDIA_EXPORT std::string GetDecoderName(VideoDecoderType type); + class MEDIA_EXPORT Decoder { public: virtual ~Decoder(); diff --git a/chromium/media/base/decoder_factory.cc b/chromium/media/base/decoder_factory.cc index 52028b6901c..79209054199 100644 --- a/chromium/media/base/decoder_factory.cc +++ b/chromium/media/base/decoder_factory.cc @@ -17,6 +17,11 @@ void DecoderFactory::CreateAudioDecoders( MediaLog* media_log, std::vector<std::unique_ptr<AudioDecoder>>* audio_decoders) {} +SupportedVideoDecoderConfigs +DecoderFactory::GetSupportedVideoDecoderConfigsForWebRTC() { + return {}; +} + void DecoderFactory::CreateVideoDecoders( scoped_refptr<base::SequencedTaskRunner> task_runner, GpuVideoAcceleratorFactories* gpu_factories, diff --git a/chromium/media/base/decoder_factory.h b/chromium/media/base/decoder_factory.h index 2af1af58f9d..fee2f69ff28 100644 --- a/chromium/media/base/decoder_factory.h +++ b/chromium/media/base/decoder_factory.h @@ -12,6 +12,7 @@ #include "base/memory/ref_counted.h" #include "media/base/media_export.h" #include "media/base/overlay_info.h" +#include "media/base/supported_video_decoder_config.h" namespace base { class SequencedTaskRunner; @@ -41,6 +42,13 @@ class MEDIA_EXPORT DecoderFactory { MediaLog* media_log, std::vector<std::unique_ptr<AudioDecoder>>* audio_decoders); + // Returns the union of all decoder configs supported by the decoders created + // when CreateVideoDecoders is called. + // TODO(crbug.com/1173503): Rename to GetSupportedVideoDecoderConfigs after + // being properly implemented for all factories. + virtual SupportedVideoDecoderConfigs + GetSupportedVideoDecoderConfigsForWebRTC(); + // Creates video decoders and append them to the end of |video_decoders|. // Decoders are single-threaded, each decoder should run on |task_runner|. virtual void CreateVideoDecoders( diff --git a/chromium/media/base/decryptor.h b/chromium/media/base/decryptor.h index dbc91c79805..d88aac96da2 100644 --- a/chromium/media/base/decryptor.h +++ b/chromium/media/base/decryptor.h @@ -109,10 +109,9 @@ class MEDIA_EXPORT Decryptor { // - Set to kError if unexpected error has occurred. In this case the // returned frame(s) must be NULL/empty. // Second parameter: The decoded video frame or audio buffers. - typedef base::RepeatingCallback<void(Status, const AudioFrames&)> - AudioDecodeCB; - typedef base::RepeatingCallback<void(Status, scoped_refptr<VideoFrame>)> - VideoDecodeCB; + using AudioDecodeCB = base::OnceCallback<void(Status, const AudioFrames&)>; + using VideoDecodeCB = + base::OnceCallback<void(Status, scoped_refptr<VideoFrame>)>; // Decrypts and decodes the |encrypted| buffer. The status and the decrypted // buffer are returned via the provided callback. @@ -125,9 +124,9 @@ class MEDIA_EXPORT Decryptor { // AudioDecodeCB has completed. Thus, only one AudioDecodeCB may be pending at // any time. Same for DecryptAndDecodeVideo(); virtual void DecryptAndDecodeAudio(scoped_refptr<DecoderBuffer> encrypted, - const AudioDecodeCB& audio_decode_cb) = 0; + AudioDecodeCB audio_decode_cb) = 0; virtual void DecryptAndDecodeVideo(scoped_refptr<DecoderBuffer> encrypted, - const VideoDecodeCB& video_decode_cb) = 0; + VideoDecodeCB video_decode_cb) = 0; // Resets the decoder to an initialized clean state, cancels any scheduled // decrypt-and-decode operations, and fires any pending diff --git a/chromium/media/base/eme_constants.h b/chromium/media/base/eme_constants.h index 2664065f051..11e7c1ed3ff 100644 --- a/chromium/media/base/eme_constants.h +++ b/chromium/media/base/eme_constants.h @@ -31,7 +31,7 @@ enum EmeCodec : uint32_t { EME_CODEC_AAC = 1 << 4, EME_CODEC_AVC1 = 1 << 5, EME_CODEC_VP9_PROFILE2 = 1 << 6, // VP9 profiles 2 - EME_CODEC_HEVC = 1 << 7, + EME_CODEC_HEVC_PROFILE_MAIN = 1 << 7, EME_CODEC_DOLBY_VISION_AVC = 1 << 8, EME_CODEC_DOLBY_VISION_HEVC = 1 << 9, EME_CODEC_AC3 = 1 << 10, @@ -39,6 +39,7 @@ enum EmeCodec : uint32_t { EME_CODEC_MPEG_H_AUDIO = 1 << 12, EME_CODEC_FLAC = 1 << 13, EME_CODEC_AV1 = 1 << 14, + EME_CODEC_HEVC_PROFILE_MAIN10 = 1 << 15, }; // *_ALL values should only be used for masking, do not use them to specify @@ -70,7 +71,8 @@ constexpr SupportedCodecs GetMp4VideoCodecs() { #if BUILDFLAG(USE_PROPRIETARY_CODECS) codecs |= EME_CODEC_AVC1; #if BUILDFLAG(ENABLE_PLATFORM_HEVC) - codecs |= EME_CODEC_HEVC; + codecs |= EME_CODEC_HEVC_PROFILE_MAIN; + codecs |= EME_CODEC_HEVC_PROFILE_MAIN10; #endif // BUILDFLAG(ENABLE_PLATFORM_HEVC) #if BUILDFLAG(ENABLE_PLATFORM_DOLBY_VISION) codecs |= EME_CODEC_DOLBY_VISION_AVC; @@ -186,14 +188,18 @@ enum class EmeConfigRule { // The configuration option prevents use of hardware-secure codecs. // This rule only has meaning on platforms that distinguish hardware-secure - // codecs (i.e. Android and Windows). + // codecs (i.e. Android, Windows and ChromeOS). HW_SECURE_CODECS_NOT_ALLOWED, // The configuration option is supported if hardware-secure codecs are used. // This rule only has meaning on platforms that distinguish hardware-secure - // codecs (i.e. Android and Windows). + // codecs (i.e. Android, Windows and ChromeOS). HW_SECURE_CODECS_REQUIRED, + // The configuration option is supported on platforms where hardware-secure + // codecs are used and an identifier is also required (i.e. ChromeOS). + IDENTIFIER_AND_HW_SECURE_CODECS_REQUIRED, + // The configuration option is supported without conditions. SUPPORTED, }; diff --git a/chromium/media/base/fake_audio_worker.cc b/chromium/media/base/fake_audio_worker.cc index 2d0bbefd537..8d57a0908f1 100644 --- a/chromium/media/base/fake_audio_worker.cc +++ b/chromium/media/base/fake_audio_worker.cc @@ -57,7 +57,7 @@ class FakeAudioWorker::Worker int64_t frames_elapsed_; // Used to cancel any delayed tasks still inside the worker loop's queue. - base::CancelableClosure worker_task_cb_; + base::CancelableRepeatingClosure worker_task_cb_; THREAD_CHECKER(thread_checker_); diff --git a/chromium/media/base/fallback_video_decoder.cc b/chromium/media/base/fallback_video_decoder.cc deleted file mode 100644 index a10ac08bec8..00000000000 --- a/chromium/media/base/fallback_video_decoder.cc +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include <utility> - -#include "base/bind.h" -#include "base/callback_helpers.h" -#include "base/threading/sequenced_task_runner_handle.h" -#include "media/base/decoder_buffer.h" -#include "media/base/fallback_video_decoder.h" -#include "media/base/video_decoder_config.h" - -namespace media { - -FallbackVideoDecoder::FallbackVideoDecoder( - std::unique_ptr<VideoDecoder> preferred, - std::unique_ptr<VideoDecoder> fallback) - : preferred_decoder_(std::move(preferred)), - fallback_decoder_(std::move(fallback)) {} - -void FallbackVideoDecoder::Initialize(const VideoDecoderConfig& config, - bool low_delay, - CdmContext* cdm_context, - InitCB init_cb, - const OutputCB& output_cb, - const WaitingCB& waiting_cb) { - // If we've already fallen back, just reinitialize the selected decoder. - if (selected_decoder_ && did_fallback_) { - selected_decoder_->Initialize(config, low_delay, cdm_context, - std::move(init_cb), output_cb, waiting_cb); - return; - } - - InitCB fallback_initialize_cb = - base::BindOnce(&FallbackVideoDecoder::FallbackInitialize, - weak_factory_.GetWeakPtr(), config, low_delay, cdm_context, - std::move(init_cb), output_cb, waiting_cb); - - preferred_decoder_->Initialize(config, low_delay, cdm_context, - std::move(fallback_initialize_cb), output_cb, - waiting_cb); -} - -void FallbackVideoDecoder::FallbackInitialize(const VideoDecoderConfig& config, - bool low_delay, - CdmContext* cdm_context, - InitCB init_cb, - const OutputCB& output_cb, - const WaitingCB& waiting_cb, - Status status) { - // The preferred decoder was successfully initialized. - if (status.is_ok()) { - selected_decoder_ = preferred_decoder_.get(); - std::move(init_cb).Run(OkStatus()); - return; - } - - did_fallback_ = true; - // Post destruction of |preferred_decoder_| so that we don't destroy the - // object during the callback. DeleteSoon doesn't handle custom deleters, so - // we post a do-nothing task instead. - base::SequencedTaskRunnerHandle::Get()->PostTask( - FROM_HERE, - base::BindOnce(base::DoNothing::Once<std::unique_ptr<VideoDecoder>>(), - std::move(preferred_decoder_))); - selected_decoder_ = fallback_decoder_.get(); - fallback_decoder_->Initialize(config, low_delay, cdm_context, - std::move(init_cb), output_cb, waiting_cb); -} - -void FallbackVideoDecoder::Decode(scoped_refptr<DecoderBuffer> buffer, - DecodeCB decode_cb) { - DCHECK(selected_decoder_); - selected_decoder_->Decode(std::move(buffer), std::move(decode_cb)); -} - -void FallbackVideoDecoder::Reset(base::OnceClosure reset_cb) { - DCHECK(selected_decoder_); - selected_decoder_->Reset(std::move(reset_cb)); -} - -bool FallbackVideoDecoder::NeedsBitstreamConversion() const { - DCHECK(selected_decoder_); - return selected_decoder_->NeedsBitstreamConversion(); -} - -bool FallbackVideoDecoder::CanReadWithoutStalling() const { - DCHECK(selected_decoder_); - return selected_decoder_->CanReadWithoutStalling(); -} - -int FallbackVideoDecoder::GetMaxDecodeRequests() const { - DCHECK(selected_decoder_); - return selected_decoder_->GetMaxDecodeRequests(); -} - -std::string FallbackVideoDecoder::GetDisplayName() const { - // MojoVideoDecoder always identifies itself as such, and never asks for the - // name of the underlying decoder. - NOTREACHED(); - return "FallbackVideoDecoder"; -} - -FallbackVideoDecoder::~FallbackVideoDecoder() = default; - -} // namespace media diff --git a/chromium/media/base/fallback_video_decoder.h b/chromium/media/base/fallback_video_decoder.h deleted file mode 100644 index 98c06c82949..00000000000 --- a/chromium/media/base/fallback_video_decoder.h +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#ifndef MEDIA_BASE_FALLBACK_VIDEO_DECODER_H_ -#define MEDIA_BASE_FALLBACK_VIDEO_DECODER_H_ - -#include <memory> -#include <string> - -#include "base/memory/weak_ptr.h" -#include "media/base/video_decoder.h" - -namespace media { - -// A Wrapper VideoDecoder which supports a fallback and a preferred decoder. -class MEDIA_EXPORT FallbackVideoDecoder : public VideoDecoder { - public: - FallbackVideoDecoder(std::unique_ptr<VideoDecoder> preferred, - std::unique_ptr<VideoDecoder> fallback); - - // media::VideoDecoder implementation. - std::string GetDisplayName() const override; - void Initialize(const VideoDecoderConfig& config, - bool low_delay, - CdmContext* cdm_context, - InitCB init_cb, - const OutputCB& output_cb, - const WaitingCB& waiting_cb) override; - void Decode(scoped_refptr<DecoderBuffer> buffer, DecodeCB decode_cb) override; - void Reset(base::OnceClosure reset_cb) override; - bool NeedsBitstreamConversion() const override; - bool CanReadWithoutStalling() const override; - int GetMaxDecodeRequests() const override; - - protected: - ~FallbackVideoDecoder() override; - - private: - void FallbackInitialize(const VideoDecoderConfig& config, - bool low_delay, - CdmContext* cdm_context, - InitCB init_cb, - const OutputCB& output_cb, - const WaitingCB& waiting_cb, - Status status); - - std::unique_ptr<media::VideoDecoder> preferred_decoder_; - std::unique_ptr<media::VideoDecoder> fallback_decoder_; - media::VideoDecoder* selected_decoder_ = nullptr; - bool did_fallback_ = false; - - base::WeakPtrFactory<FallbackVideoDecoder> weak_factory_{this}; - - DISALLOW_COPY_AND_ASSIGN(FallbackVideoDecoder); -}; - -} // namespace media - -#endif // MEDIA_BASE_FALLBACK_VIDEO_DECODER_H_ diff --git a/chromium/media/base/fallback_video_decoder_unittest.cc b/chromium/media/base/fallback_video_decoder_unittest.cc deleted file mode 100644 index e996eb5b50c..00000000000 --- a/chromium/media/base/fallback_video_decoder_unittest.cc +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright 2018 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -#include <tuple> - -#include "base/bind.h" -#include "base/callback_helpers.h" -#include "base/run_loop.h" -#include "base/test/gmock_callback_support.h" -#include "base/test/task_environment.h" -#include "media/base/decoder_buffer.h" -#include "media/base/fallback_video_decoder.h" -#include "media/base/mock_filters.h" -#include "media/base/test_helpers.h" -#include "media/base/video_decoder.h" -#include "media/base/video_decoder_config.h" -#include "testing/gmock/include/gmock/gmock.h" -#include "testing/gtest/include/gtest/gtest-param-test.h" -#include "testing/gtest/include/gtest/gtest.h" - -using ::base::test::RunOnceCallback; -using ::testing::_; -using ::testing::StrictMock; - -namespace media { - -class FallbackVideoDecoderUnittest : public ::testing::TestWithParam<bool> { - public: - FallbackVideoDecoderUnittest() - : backup_decoder_(nullptr), - preferred_decoder_(nullptr), - fallback_decoder_(nullptr) {} - - ~FallbackVideoDecoderUnittest() override { Destroy(); } - - std::unique_ptr<VideoDecoder> MakeMockDecoderWithExpectations( - bool is_fallback, - bool preferred_should_succeed) { - std::string n = is_fallback ? "Fallback" : "Preferred"; - StrictMock<MockVideoDecoder>* result = new StrictMock<MockVideoDecoder>(n); - - if (is_fallback && !preferred_should_succeed) { - EXPECT_CALL(*result, Initialize_(_, _, _, _, _, _)) - .WillOnce(RunOnceCallback<3>(OkStatus())); - } - - if (!is_fallback) { - preferred_decoder_ = result; - EXPECT_CALL(*result, Initialize_(_, _, _, _, _, _)) - .WillOnce(RunOnceCallback<3>(preferred_should_succeed - ? OkStatus() - : StatusCode::kCodeOnlyForTesting)); - } else { - backup_decoder_ = result; - } - - return std::unique_ptr<VideoDecoder>(result); - } - - void Initialize(bool preferred_should_succeed) { - fallback_decoder_ = new FallbackVideoDecoder( - MakeMockDecoderWithExpectations(false, preferred_should_succeed), - MakeMockDecoderWithExpectations(true, preferred_should_succeed)); - - fallback_decoder_->Initialize( - video_decoder_config_, false, nullptr, - base::BindOnce([](Status status) { EXPECT_TRUE(status.is_ok()); }), - base::DoNothing(), base::DoNothing()); - } - - protected: - void Destroy() { std::default_delete<VideoDecoder>()(fallback_decoder_); } - - bool PreferredShouldSucceed() { return GetParam(); } - - base::test::TaskEnvironment task_environment_; - - StrictMock<MockVideoDecoder>* backup_decoder_; - StrictMock<MockVideoDecoder>* preferred_decoder_; - VideoDecoder* fallback_decoder_; - VideoDecoderConfig video_decoder_config_; - - private: - DISALLOW_COPY_AND_ASSIGN(FallbackVideoDecoderUnittest); -}; - -INSTANTIATE_TEST_SUITE_P(DoesPreferredInitFail, - FallbackVideoDecoderUnittest, - testing::ValuesIn({true, false})); - -#define EXPECT_ON_CORRECT_DECODER(method) \ - if (PreferredShouldSucceed()) \ - EXPECT_CALL(*preferred_decoder_, method); \ - else \ - EXPECT_CALL(*backup_decoder_, method) // Intentionally leave off semicolon. - -// Do not test the name lookup; it is NOTREACHED. -TEST_P(FallbackVideoDecoderUnittest, MethodsRedirectedAsExpected) { - Initialize(PreferredShouldSucceed()); - - EXPECT_ON_CORRECT_DECODER(Decode_(_, _)); - fallback_decoder_->Decode(nullptr, base::DoNothing()); - - EXPECT_ON_CORRECT_DECODER(Reset_(_)); - fallback_decoder_->Reset(base::DoNothing()); - - EXPECT_ON_CORRECT_DECODER(NeedsBitstreamConversion()); - fallback_decoder_->NeedsBitstreamConversion(); - - EXPECT_ON_CORRECT_DECODER(CanReadWithoutStalling()); - fallback_decoder_->CanReadWithoutStalling(); - - EXPECT_ON_CORRECT_DECODER(GetMaxDecodeRequests()); - fallback_decoder_->GetMaxDecodeRequests(); -} - -// │ first initialization │ second initialization │ -// preferred │ preferred │ backup │ preferred │ backup │ -// will succeed │ init called │ init called │ init called │ init called │ -//───────────────┼─────────────┼─────────────┼─────────────┼─────────────┤ -// false │ ✓ │ ✓ │ x │ ✓ │ -// true │ ✓ │ x │ ✓ │ ✓ │ -TEST_P(FallbackVideoDecoderUnittest, ReinitializeWithPreferredFailing) { - Initialize(PreferredShouldSucceed()); - - // If we succeedd the first time, it should still be alive. - if (PreferredShouldSucceed()) { // fail initialization - EXPECT_CALL(*preferred_decoder_, Initialize_(_, _, _, _, _, _)) - .WillOnce(RunOnceCallback<3>(StatusCode::kCodeOnlyForTesting)); - } - EXPECT_CALL(*backup_decoder_, Initialize_(_, _, _, _, _, _)) - .WillOnce(RunOnceCallback<3>(OkStatus())); - - fallback_decoder_->Initialize( - video_decoder_config_, false, nullptr, - base::BindOnce([](Status status) { EXPECT_TRUE(status.is_ok()); }), - base::DoNothing(), base::DoNothing()); -} - -// │ first initialization │ second initialization │ -// preferred │ preferred │ backup │ preferred │ backup │ -// will succeed │ init called │ init called │ init called │ init called │ -//───────────────┼─────────────┼─────────────┼─────────────┼─────────────┤ -// false │ ✓ │ ✓ │ x │ ✓ │ -// true │ ✓ │ x │ ✓ │ x │ -TEST_P(FallbackVideoDecoderUnittest, ReinitializeWithPreferredSuccessful) { - Initialize(PreferredShouldSucceed()); - - // If we succeedd the first time, it should still be alive. - if (PreferredShouldSucceed()) { - EXPECT_CALL(*preferred_decoder_, Initialize_(_, _, _, _, _, _)) - .WillOnce(RunOnceCallback<3>(OkStatus())); // pass initialization - } else { - // Otherwise, preferred was deleted, and we only backup still exists. - EXPECT_CALL(*backup_decoder_, Initialize_(_, _, _, _, _, _)) - .WillOnce(RunOnceCallback<3>(OkStatus())); - } - - fallback_decoder_->Initialize( - video_decoder_config_, false, nullptr, - base::BindOnce([](Status status) { EXPECT_TRUE(status.is_ok()); }), - base::DoNothing(), base::DoNothing()); -} - -} // namespace media diff --git a/chromium/media/base/ipc/DEPS b/chromium/media/base/ipc/DEPS index 5d8ba2b3c2e..27ed9d90740 100644 --- a/chromium/media/base/ipc/DEPS +++ b/chromium/media/base/ipc/DEPS @@ -2,3 +2,9 @@ include_rules = [ "+ipc", "-media/base/media_export.h", ] + +specific_include_rules = { + "media_param_traits_macros.h": [ + "+third_party/blink/public/platform/web_fullscreen_video_status.h", + ] +} diff --git a/chromium/media/base/ipc/media_param_traits_macros.h b/chromium/media/base/ipc/media_param_traits_macros.h index ecebc25c85b..6bedf46d006 100644 --- a/chromium/media/base/ipc/media_param_traits_macros.h +++ b/chromium/media/base/ipc/media_param_traits_macros.h @@ -17,11 +17,13 @@ #include "media/base/container_names.h" #include "media/base/content_decryption_module.h" #include "media/base/decode_status.h" +#include "media/base/decoder.h" #include "media/base/decrypt_config.h" #include "media/base/decryptor.h" #include "media/base/demuxer_stream.h" #include "media/base/eme_constants.h" #include "media/base/encryption_scheme.h" +#include "media/base/media_content_type.h" #include "media/base/media_log_record.h" #include "media/base/media_status.h" #include "media/base/output_device_info.h" @@ -30,6 +32,7 @@ #include "media/base/sample_format.h" #include "media/base/status_codes.h" #include "media/base/subsample_entry.h" +#include "media/base/supported_video_decoder_config.h" #include "media/base/video_codecs.h" #include "media/base/video_color_space.h" #include "media/base/video_transformation.h" @@ -37,7 +40,7 @@ #include "media/base/waiting.h" #include "media/base/watch_time_keys.h" #include "media/media_buildflags.h" -#include "media/video/supported_video_decoder_config.h" +#include "third_party/blink/public/platform/web_fullscreen_video_status.h" #include "ui/gfx/hdr_metadata.h" #include "ui/gfx/ipc/color/gfx_param_traits_macros.h" @@ -47,6 +50,9 @@ // Enum traits. +IPC_ENUM_TRAITS_MAX_VALUE(blink::WebFullscreenVideoStatus, + blink::WebFullscreenVideoStatus::kMaxValue) + IPC_ENUM_TRAITS_MAX_VALUE(media::AudioCodec, media::AudioCodec::kAudioCodecMax) IPC_ENUM_TRAITS_MAX_VALUE(media::AudioCodecProfile, media::AudioCodecProfile::kMaxValue) @@ -95,6 +101,8 @@ IPC_ENUM_TRAITS_MAX_VALUE(media::EncryptionScheme, IPC_ENUM_TRAITS_MAX_VALUE(media::HdcpVersion, media::HdcpVersion::kHdcpVersionMax) +IPC_ENUM_TRAITS_MAX_VALUE(media::MediaContentType, media::MediaContentType::Max) + IPC_ENUM_TRAITS_MAX_VALUE(media::MediaLogRecord::Type, media::MediaLogRecord::Type::kMaxValue) @@ -123,6 +131,12 @@ IPC_ENUM_TRAITS_MIN_MAX_VALUE(media::VideoCodecProfile, IPC_ENUM_TRAITS_MAX_VALUE(media::VideoDecoderImplementation, media::VideoDecoderImplementation::kMaxValue) +IPC_ENUM_TRAITS_MAX_VALUE(media::VideoDecoderType, + media::VideoDecoderType::kMaxValue) + +IPC_ENUM_TRAITS_MAX_VALUE(media::AudioDecoderType, + media::AudioDecoderType::kMaxValue) + IPC_ENUM_TRAITS_MAX_VALUE(media::VideoPixelFormat, media::PIXEL_FORMAT_MAX) IPC_ENUM_TRAITS_MAX_VALUE(media::VideoRotation, media::VIDEO_ROTATION_MAX) diff --git a/chromium/media/base/key_systems.cc b/chromium/media/base/key_systems.cc index d40a7c017e6..d81b872ff33 100644 --- a/chromium/media/base/key_systems.cc +++ b/chromium/media/base/key_systems.cc @@ -103,7 +103,12 @@ EmeCodec ToVideoEmeCodec(VideoCodec codec, VideoCodecProfile profile) { return EME_CODEC_NONE; } case kCodecHEVC: - return EME_CODEC_HEVC; + // Only handle Main and Main10 profiles for HEVC. + if (profile == HEVCPROFILE_MAIN) + return EME_CODEC_HEVC_PROFILE_MAIN; + if (profile == HEVCPROFILE_MAIN10) + return EME_CODEC_HEVC_PROFILE_MAIN10; + return EME_CODEC_NONE; case kCodecDolbyVision: // Only profiles 0, 4, 5, 7, 8, 9 are valid. Profile 0 and 9 are encoded // based on AVC while profile 4, 5, 7 and 8 are based on HEVC. @@ -674,7 +679,8 @@ EmeConfigRule KeySystemsImpl::GetContentTypeConfigRule( // SupportedCodecs | SupportedSecureCodecs | Result // yes | yes | SUPPORTED // yes | no | HW_SECURE_CODECS_NOT_ALLOWED - // no | any | NOT_SUPPORTED + // no | yes | HW_SECURE_CODECS_REQUIRED + // no | no | NOT_SUPPORTED EmeConfigRule support = EmeConfigRule::SUPPORTED; for (size_t i = 0; i < codecs.size(); i++) { EmeCodec codec = @@ -688,22 +694,29 @@ EmeConfigRule KeySystemsImpl::GetContentTypeConfigRule( // codecs with multiple bits set, e.g. to cover multiple profiles, we check // (codec & mask) == codec instead of (codec & mask) != 0 to make sure all // bits are set. Same below. - if ((codec & key_system_codec_mask & mime_type_codec_mask) != codec) { + if ((codec & key_system_codec_mask & mime_type_codec_mask) != codec && + (codec & key_system_hw_secure_codec_mask & mime_type_codec_mask) != + codec) { DVLOG(2) << "Container/codec pair (" << container_mime_type << " / " << codecs[i] << ") not supported by " << key_system; return EmeConfigRule::NOT_SUPPORTED; } - // Check whether the codec supports a hardware-secure mode (any level). The - // goal is to prevent mixing of non-hardware-secure codecs with - // hardware-secure codecs, since the mode is fixed at CDM creation. - // - // Because the check for regular codec support is early-exit, we don't have - // to consider codecs that are only supported in hardware-secure mode. We - // could do so, and make use of HW_SECURE_CODECS_REQUIRED, if it turns out - // that hardware-secure-only codecs actually exist and are useful. - if ((codec & key_system_hw_secure_codec_mask) != codec) + // Check whether the codec supports a hardware-secure mode (any level). + if ((codec & key_system_hw_secure_codec_mask) != codec) { + DCHECK_EQ(codec & key_system_codec_mask, codec); + if (support == EmeConfigRule::HW_SECURE_CODECS_REQUIRED) + return EmeConfigRule::NOT_SUPPORTED; support = EmeConfigRule::HW_SECURE_CODECS_NOT_ALLOWED; + } + + // Check whether the codec requires a hardware-secure mode (any level). + if ((codec & key_system_codec_mask) != codec) { + DCHECK_EQ(codec & key_system_hw_secure_codec_mask, codec); + if (support == EmeConfigRule::HW_SECURE_CODECS_NOT_ALLOWED) + return EmeConfigRule::NOT_SUPPORTED; + support = EmeConfigRule::HW_SECURE_CODECS_REQUIRED; + } } return support; diff --git a/chromium/media/base/key_systems_unittest.cc b/chromium/media/base/key_systems_unittest.cc index 64a7a4b4336..78685a7b8a2 100644 --- a/chromium/media/base/key_systems_unittest.cc +++ b/chromium/media/base/key_systems_unittest.cc @@ -830,10 +830,7 @@ TEST_F(KeySystemsTest, HardwareSecureCodecs) { EmeConfigRule::SUPPORTED, GetVideoContentTypeConfigRule(kVideoFoo, foovideo_codec(), kExternal)); - // Codec that is supported by hardware secure codec but not otherwise is - // treated as NOT_SUPPORTED instead of HW_SECURE_CODECS_REQUIRED. See - // KeySystemsImpl::GetContentTypeConfigRule() for details. - EXPECT_EQ(EmeConfigRule::NOT_SUPPORTED, + EXPECT_EQ(EmeConfigRule::HW_SECURE_CODECS_REQUIRED, GetVideoContentTypeConfigRule(kVideoFoo, securefoovideo_codec(), kExternal)); } diff --git a/chromium/media/base/limits.h b/chromium/media/base/limits.h index 4d1709424ad..edc8fc6a39e 100644 --- a/chromium/media/base/limits.h +++ b/chromium/media/base/limits.h @@ -31,8 +31,8 @@ enum { // - Vorbis used to be limited to 96 kHz, but no longer has that // restriction. // - Most PC audio hardware is limited to 192 kHz, some specialized DAC - // devices will use 384 kHz though. - kMaxSampleRate = 384000, + // devices will use 768 kHz though. + kMaxSampleRate = 768000, kMinSampleRate = 3000, kMaxChannels = 32, kMaxBytesPerSample = 4, diff --git a/chromium/media/base/logging_override_if_enabled.h b/chromium/media/base/logging_override_if_enabled.h index ee274e79f9f..eeae26ab1c6 100644 --- a/chromium/media/base/logging_override_if_enabled.h +++ b/chromium/media/base/logging_override_if_enabled.h @@ -9,6 +9,7 @@ // Warning: Do NOT include this file in .h files to avoid unexpected override. // TODO(xhwang): Provide a way to choose which |verboselevel| to override. +#include "build/build_config.h" #include "media/media_buildflags.h" #if BUILDFLAG(ENABLE_LOGGING_OVERRIDE) @@ -16,9 +17,20 @@ #error This file must be included after base/logging.h. #endif +#if defined(OS_FUCHSIA) + +#define __DVLOG_0 VLOG(0) +#define __DVLOG_1 VLOG(1) +#define __DVLOG_2 VLOG(2) + +#else + #define __DVLOG_0 LOG(INFO) #define __DVLOG_1 LOG(INFO) #define __DVLOG_2 LOG(INFO) + +#endif // defined(OS_FUCHSIA) + #define __DVLOG_3 EAT_STREAM_PARAMETERS #define __DVLOG_4 EAT_STREAM_PARAMETERS #define __DVLOG_5 EAT_STREAM_PARAMETERS diff --git a/chromium/media/base/mac/color_space_util_mac.h b/chromium/media/base/mac/color_space_util_mac.h index 938b7fd46ab..ab2a4459aad 100644 --- a/chromium/media/base/mac/color_space_util_mac.h +++ b/chromium/media/base/mac/color_space_util_mac.h @@ -5,11 +5,13 @@ #ifndef MEDIA_BASE_MAC_COLOR_SPACE_UTIL_MAC_H_ #define MEDIA_BASE_MAC_COLOR_SPACE_UTIL_MAC_H_ +#include <CoreFoundation/CoreFoundation.h> #include <CoreMedia/CoreMedia.h> #include <CoreVideo/CoreVideo.h> #include "media/base/media_export.h" #include "ui/gfx/color_space.h" +#include "ui/gfx/hdr_metadata.h" namespace media { @@ -17,7 +19,13 @@ MEDIA_EXPORT gfx::ColorSpace GetImageBufferColorSpace( CVImageBufferRef image_buffer); MEDIA_EXPORT gfx::ColorSpace GetFormatDescriptionColorSpace( - CMFormatDescriptionRef format_description) API_AVAILABLE(macos(10.11)); + CMFormatDescriptionRef format_description); + +MEDIA_EXPORT CFDataRef +GenerateContentLightLevelInfo(const gfx::HDRMetadata& hdr_metadata); + +MEDIA_EXPORT CFDataRef +GenerateMasteringDisplayColorVolume(const gfx::HDRMetadata& hdr_metadata); } // namespace media diff --git a/chromium/media/base/mac/color_space_util_mac.mm b/chromium/media/base/mac/color_space_util_mac.mm index 8e76a7ef56d..799cfe29ad3 100644 --- a/chromium/media/base/mac/color_space_util_mac.mm +++ b/chromium/media/base/mac/color_space_util_mac.mm @@ -4,6 +4,7 @@ #include "media/base/mac/color_space_util_mac.h" +#include <simd/simd.h> #include <vector> #include "base/mac/foundation_util.h" @@ -56,14 +57,11 @@ gfx::ColorSpace::PrimaryID GetCoreVideoPrimary(CFTypeRef primaries_untyped) { supported_primaries.push_back( {kCVImageBufferColorPrimaries_SMPTE_C, kCMFormatDescriptionColorPrimaries_SMPTE_C, - gfx::ColorSpace::PrimaryID::SMPTE240M}); - if (@available(macos 10.11, *)) { - supported_primaries.push_back( - {kCVImageBufferColorPrimaries_ITU_R_2020, - kCMFormatDescriptionColorPrimaries_ITU_R_2020, - gfx::ColorSpace::PrimaryID::BT2020}); - } + supported_primaries.push_back( + {kCVImageBufferColorPrimaries_ITU_R_2020, + kCMFormatDescriptionColorPrimaries_ITU_R_2020, + gfx::ColorSpace::PrimaryID::BT2020}); return supported_primaries; }()); @@ -87,10 +85,6 @@ gfx::ColorSpace::TransferID GetCoreVideoTransferFn(CFTypeRef transfer_untyped, static const base::NoDestructor<std::vector<CVImageTransferFn>> kSupportedTransferFuncs([] { std::vector<CVImageTransferFn> supported_transfer_funcs; - // The constants kCMFormatDescriptionTransferFunction_ITU_R_709_2, - // SMPTE_240M_1995, and UseGamma will compile against macOS 10.10 - // because they are #defined to their kCVImageBufferTransferFunction - // equivalents. They are technically not present until macOS 10.11. supported_transfer_funcs.push_back( {kCVImageBufferTransferFunction_ITU_R_709_2, kCMFormatDescriptionTransferFunction_ITU_R_709_2, @@ -103,12 +97,10 @@ gfx::ColorSpace::TransferID GetCoreVideoTransferFn(CFTypeRef transfer_untyped, {kCVImageBufferTransferFunction_UseGamma, kCMFormatDescriptionTransferFunction_UseGamma, gfx::ColorSpace::TransferID::CUSTOM}); - if (@available(macos 10.11, *)) { - supported_transfer_funcs.push_back( - {kCVImageBufferTransferFunction_ITU_R_2020, - kCMFormatDescriptionTransferFunction_ITU_R_2020, - gfx::ColorSpace::TransferID::BT2020_10}); - } + supported_transfer_funcs.push_back( + {kCVImageBufferTransferFunction_ITU_R_2020, + kCMFormatDescriptionTransferFunction_ITU_R_2020, + gfx::ColorSpace::TransferID::BT2020_10}); if (@available(macos 10.12, *)) { supported_transfer_funcs.push_back( {kCVImageBufferTransferFunction_SMPTE_ST_428_1, @@ -199,12 +191,10 @@ gfx::ColorSpace::MatrixID GetCoreVideoMatrix(CFTypeRef matrix_untyped) { {kCVImageBufferYCbCrMatrix_SMPTE_240M_1995, kCMFormatDescriptionYCbCrMatrix_SMPTE_240M_1995, gfx::ColorSpace::MatrixID::SMPTE240M}); - if (@available(macos 10.11, *)) { - supported_matrices.push_back( - {kCVImageBufferYCbCrMatrix_ITU_R_2020, - kCMFormatDescriptionYCbCrMatrix_ITU_R_2020, - gfx::ColorSpace::MatrixID::BT2020_NCL}); - } + supported_matrices.push_back( + {kCVImageBufferYCbCrMatrix_ITU_R_2020, + kCMFormatDescriptionYCbCrMatrix_ITU_R_2020, + gfx::ColorSpace::MatrixID::BT2020_NCL}); return supported_matrices; }()); @@ -282,4 +272,65 @@ gfx::ColorSpace GetFormatDescriptionColorSpace( format_description, kCMFormatDescriptionExtension_YCbCrMatrix)); } +CFDataRef GenerateContentLightLevelInfo(const gfx::HDRMetadata& hdr_metadata) { + // This is a SMPTEST2086 Content Light Level Information box. + struct ContentLightLevelInfoSEI { + uint16_t max_content_light_level; + uint16_t max_frame_average_light_level; + } __attribute__((packed, aligned(2))); + static_assert(sizeof(ContentLightLevelInfoSEI) == 4, "Must be 4 bytes"); + + // Values are stored in big-endian... + ContentLightLevelInfoSEI sei; + sei.max_content_light_level = + __builtin_bswap16(hdr_metadata.max_content_light_level); + sei.max_frame_average_light_level = + __builtin_bswap16(hdr_metadata.max_frame_average_light_level); + + NSData* nsdata_sei = [NSData dataWithBytes:&sei length:4]; + return base::mac::NSToCFCast(nsdata_sei); +} + +CFDataRef GenerateMasteringDisplayColorVolume( + const gfx::HDRMetadata& hdr_metadata) { + // This is a SMPTEST2086 Mastering Display Color Volume box. + struct MasteringDisplayColorVolumeSEI { + vector_ushort2 primaries[3]; // GBR + vector_ushort2 white_point; + uint32_t luminance_max; + uint32_t luminance_min; + } __attribute__((packed, aligned(4))); + static_assert(sizeof(MasteringDisplayColorVolumeSEI) == 24, + "Must be 24 bytes"); + + // Make a copy which we can manipulate. + auto md = hdr_metadata.mastering_metadata; + + constexpr float kColorCoordinateUpperBound = 50000.0f; + md.primary_r.Scale(kColorCoordinateUpperBound); + md.primary_g.Scale(kColorCoordinateUpperBound); + md.primary_b.Scale(kColorCoordinateUpperBound); + md.white_point.Scale(kColorCoordinateUpperBound); + + constexpr float kUnitOfMasteringLuminance = 10000.0f; + md.luminance_max *= kUnitOfMasteringLuminance; + md.luminance_min *= kUnitOfMasteringLuminance; + + // Values are stored in big-endian... + MasteringDisplayColorVolumeSEI sei; + sei.primaries[0].x = __builtin_bswap16(md.primary_g.x() + 0.5f); + sei.primaries[0].y = __builtin_bswap16(md.primary_g.y() + 0.5f); + sei.primaries[1].x = __builtin_bswap16(md.primary_b.x() + 0.5f); + sei.primaries[1].y = __builtin_bswap16(md.primary_b.y() + 0.5f); + sei.primaries[2].x = __builtin_bswap16(md.primary_r.x() + 0.5f); + sei.primaries[2].y = __builtin_bswap16(md.primary_r.y() + 0.5f); + sei.white_point.x = __builtin_bswap16(md.white_point.x() + 0.5f); + sei.white_point.y = __builtin_bswap16(md.white_point.y() + 0.5f); + sei.luminance_max = __builtin_bswap32(md.luminance_max + 0.5f); + sei.luminance_min = __builtin_bswap32(md.luminance_min + 0.5f); + + NSData* nsdata_sei = [NSData dataWithBytes:&sei length:24]; + return base::mac::NSToCFCast(nsdata_sei); +} + } // namespace media diff --git a/chromium/media/base/media_log_properties.cc b/chromium/media/base/media_log_properties.cc index cdfbd8a0f56..a21cf880d0a 100644 --- a/chromium/media/base/media_log_properties.cc +++ b/chromium/media/base/media_log_properties.cc @@ -28,6 +28,8 @@ std::string MediaLogPropertyKeyToString(MediaLogProperty property) { STRINGIFY(kIsRangeHeaderSupported); STRINGIFY(kIsVideoDecryptingDemuxerStream); STRINGIFY(kIsAudioDecryptingDemuxerStream); + STRINGIFY(kVideoEncoderName); + STRINGIFY(kIsPlatformVideoEncoder); STRINGIFY(kAudioDecoderName); STRINGIFY(kIsPlatformAudioDecoder); STRINGIFY(kAudioTracks); diff --git a/chromium/media/base/media_log_properties.h b/chromium/media/base/media_log_properties.h index 5f6a1084443..0f8a01817e8 100644 --- a/chromium/media/base/media_log_properties.h +++ b/chromium/media/base/media_log_properties.h @@ -59,8 +59,8 @@ enum class MediaLogProperty { kIsRangeHeaderSupported, // The name of the decoder implementation currently being used to play the - // media stream. All audio/video decoders have names, such as - // FFMpegVideoDecoder or D3D11VideoDecoder. + // media stream. All audio/video decoders have id numbers defined in + // decoder.h. kVideoDecoderName, kAudioDecoderName, @@ -68,6 +68,10 @@ enum class MediaLogProperty { kIsPlatformVideoDecoder, kIsPlatformAudioDecoder, + // Webcodecs supports encoding video streams. + kVideoEncoderName, + kIsPlatformVideoEncoder, + // Whether this media player is using a decrypting demuxer for the given // audio or video stream. kIsVideoDecryptingDemuxerStream, @@ -101,12 +105,14 @@ MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kIsStreaming, bool); MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kFrameUrl, std::string); MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kFrameTitle, std::string); MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kIsSingleOrigin, bool); -MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kVideoDecoderName, std::string); +MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kVideoDecoderName, VideoDecoderType); MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kIsPlatformVideoDecoder, bool); MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kIsRangeHeaderSupported, bool); MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kIsVideoDecryptingDemuxerStream, bool); -MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kAudioDecoderName, std::string); +MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kAudioDecoderName, AudioDecoderType); MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kIsPlatformAudioDecoder, bool); +MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kVideoEncoderName, std::string); +MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kIsPlatformVideoEncoder, bool); MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kIsAudioDecryptingDemuxerStream, bool); MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kAudioTracks, std::vector<AudioDecoderConfig>); MEDIA_LOG_PROPERTY_SUPPORTS_TYPE(kTextTracks, std::vector<TextTrackConfig>); diff --git a/chromium/media/base/media_serializers.h b/chromium/media/base/media_serializers.h index 42f89c07122..5dbff1ba7fb 100644 --- a/chromium/media/base/media_serializers.h +++ b/chromium/media/base/media_serializers.h @@ -12,6 +12,7 @@ #include "base/strings/stringprintf.h" #include "media/base/audio_decoder_config.h" #include "media/base/buffering_state.h" +#include "media/base/decoder.h" #include "media/base/media_serializers_base.h" #include "media/base/status.h" #include "media/base/status_codes.h" @@ -135,6 +136,22 @@ struct MediaSerializer<base::TimeDelta> { // Enum (simple) template <> +struct MediaSerializer<VideoDecoderType> { + static inline base::Value Serialize(VideoDecoderType value) { + return base::Value(GetDecoderName(value)); + } +}; + +// Enum (simple) +template <> +struct MediaSerializer<AudioDecoderType> { + static inline base::Value Serialize(AudioDecoderType value) { + return base::Value(GetDecoderName(value)); + } +}; + +// Enum (simple) +template <> struct MediaSerializer<AudioCodec> { static inline base::Value Serialize(AudioCodec value) { return base::Value(GetCodecName(value)); diff --git a/chromium/media/base/media_switches.cc b/chromium/media/base/media_switches.cc index 07419d183fc..7215555d66c 100644 --- a/chromium/media/base/media_switches.cc +++ b/chromium/media/base/media_switches.cc @@ -192,6 +192,11 @@ const char kOverrideHardwareSecureCodecsForTesting[] = const char kEnableLiveCaptionPrefForTesting[] = "enable-live-caption-pref-for-testing"; +#if BUILDFLAG(ENABLE_PLATFORM_HEVC) +// Enables playback of clear (unencrypted) HEVC content for testing purposes. +const char kEnableClearHevcForTesting[] = "enable-clear-hevc-for-testing"; +#endif + namespace autoplay { // Autoplay policy that requires a document user activation. @@ -218,6 +223,10 @@ const base::Feature kFFmpegDecodeOpaqueVP8{"FFmpegDecodeOpaqueVP8", const base::Feature kOverlayFullscreenVideo{"overlay-fullscreen-video", base::FEATURE_ENABLED_BY_DEFAULT}; +// TODO(crbug.com/1146594): Flip this to disabled in M92. +const base::Feature kEnableMediaInternals{"enable-media-internals", + base::FEATURE_ENABLED_BY_DEFAULT}; + // Enable Picture-in-Picture. const base::Feature kPictureInPicture { "PictureInPicture", @@ -262,6 +271,20 @@ const base::Feature kMediaCastOverlayButton{"MediaCastOverlayButton", const base::Feature kUseAndroidOverlayAggressively{ "UseAndroidOverlayAggressively", base::FEATURE_ENABLED_BY_DEFAULT}; +// If enabled, RTCVideoDecoderAdapter will wrap a DecoderStream as a video +// decoder, rather than using MojoVideoDecoder. This causes the RTC external +// decoder to have all the decoder selection / fallback/forward logic of the +// non-RTC pipeline. +// TODO(liberato): This also causes the external decoder to use software +// decoding sometimes, which changes the interpretation of "ExternalDecoder". +const base::Feature kUseDecoderStreamForWebRTC{ + "UseDecoderStreamForWebRTC", base::FEATURE_DISABLED_BY_DEFAULT}; + +// If enabled, when RTCVideoDecoderAdapter is used then SW decoders will be +// exposed directly to WebRTC. +const base::Feature kExposeSwDecodersToWebRTC{ + "ExposeSwDecodersToWebRTC", base::FEATURE_DISABLED_BY_DEFAULT}; + // Let video without audio be paused when it is playing in the background. const base::Feature kBackgroundVideoPauseOptimization{ "BackgroundVideoPauseOptimization", base::FEATURE_ENABLED_BY_DEFAULT}; @@ -319,6 +342,10 @@ const base::Feature kD3D11VideoDecoderIgnoreWorkarounds{ const base::Feature kD3D11VideoDecoderVP9Profile2{ "D3D11VideoDecoderEnableVP9Profile2", base::FEATURE_DISABLED_BY_DEFAULT}; +// Enable D3D11VideoDecoder to decode AV1 video. +const base::Feature kD3D11VideoDecoderAV1{"D3D11VideoDecoderEnableAV1", + base::FEATURE_DISABLED_BY_DEFAULT}; + // Tell D3D11VideoDecoder not to switch the D3D11 device to multi-threaded mode. // This is to help us track down IGD crashes. const base::Feature kD3D11VideoDecoderSkipMultithreaded{ @@ -349,7 +376,7 @@ const base::Feature kGav1VideoDecoder{"Gav1VideoDecoder", const base::Feature kGlobalMediaControls { "GlobalMediaControls", #if defined(OS_WIN) || defined(OS_MAC) || defined(OS_LINUX) || \ - BUILDFLAG(IS_LACROS) + BUILDFLAG(IS_CHROMEOS_LACROS) base::FEATURE_ENABLED_BY_DEFAULT #else base::FEATURE_DISABLED_BY_DEFAULT @@ -363,11 +390,11 @@ const base::Feature kGlobalMediaControlsAutoDismiss{ // Show Cast sessions in Global Media Controls. It is no-op if // kGlobalMediaControls is not enabled. const base::Feature kGlobalMediaControlsForCast{ - "GlobalMediaControlsForCast", base::FEATURE_DISABLED_BY_DEFAULT}; + "GlobalMediaControlsForCast", base::FEATURE_ENABLED_BY_DEFAULT}; // Allow Global Media Controls in system tray of CrOS. const base::Feature kGlobalMediaControlsForChromeOS{ - "GlobalMediaControlsForChromeOS", base::FEATURE_DISABLED_BY_DEFAULT}; + "GlobalMediaControlsForChromeOS", base::FEATURE_ENABLED_BY_DEFAULT}; constexpr base::FeatureParam<kCrosGlobalMediaControlsPinOptions>::Option kCrosGlobalMediaControlsParamOptions[] = { @@ -391,7 +418,7 @@ const base::Feature kGlobalMediaControlsOverlayControls{ const base::Feature kGlobalMediaControlsPictureInPicture { "GlobalMediaControlsPictureInPicture", #if defined(OS_WIN) || defined(OS_MAC) || defined(OS_LINUX) || \ - BUILDFLAG(IS_LACROS) + BUILDFLAG(IS_CHROMEOS_LACROS) base::FEATURE_ENABLED_BY_DEFAULT #else base::FEATURE_DISABLED_BY_DEFAULT @@ -410,6 +437,10 @@ const base::Feature kGlobalMediaControlsModernUI{ const base::Feature kSpecCompliantCanPlayThrough{ "SpecCompliantCanPlayThrough", base::FEATURE_ENABLED_BY_DEFAULT}; +// Controls usage of SurfaceLayer for MediaStreams. +const base::Feature kSurfaceLayerForMediaStreams{ + "SurfaceLayerForMediaStreams", base::FEATURE_ENABLED_BY_DEFAULT}; + // Disables the real audio output stream after silent audio has been delivered // for too long. Should save quite a bit of power in the muted video case. const base::Feature kSuspendMutedAudio{"SuspendMutedAudio", @@ -428,6 +459,18 @@ const base::Feature kUseR16Texture{"use-r16-texture", const base::Feature kUnifiedAutoplay{"UnifiedAutoplay", base::FEATURE_ENABLED_BY_DEFAULT}; +// TODO(crbug.com/1052397): Revisit once build flag switch of lacros-chrome is +// complete. +#if defined(OS_LINUX) || BUILDFLAG(IS_CHROMEOS_LACROS) +// Enable vaapi video decoding on linux. This is already enabled by default on +// chromeos, but needs an experiment on linux. +const base::Feature kVaapiVideoDecodeLinux{"VaapiVideoDecoder", + base::FEATURE_DISABLED_BY_DEFAULT}; + +const base::Feature kVaapiVideoEncodeLinux{"VaapiVideoEncoder", + base::FEATURE_DISABLED_BY_DEFAULT}; +#endif // defined(OS_LINUX) || BUILDFLAG(IS_CHROMEOS_LACROS) + // Enable VA-API hardware decode acceleration for AV1. const base::Feature kVaapiAV1Decoder{"VaapiAV1Decoder", base::FEATURE_DISABLED_BY_DEFAULT}; @@ -436,6 +479,12 @@ const base::Feature kVaapiAV1Decoder{"VaapiAV1Decoder", const base::Feature kVaapiLowPowerEncoderGen9x{ "VaapiLowPowerEncoderGen9x", base::FEATURE_DISABLED_BY_DEFAULT}; +// Deny specific (likely small) resolutions for VA-API hardware decode and +// encode acceleration. +// TOOD(b/171041334): Enable by default once the ARC++ hw codecs issue is fixed. +const base::Feature kVaapiEnforceVideoMinMaxResolution{ + "VaapiEnforceVideoMinMaxResolution", base::FEATURE_DISABLED_BY_DEFAULT}; + // Enable VA-API hardware encode acceleration for VP8. const base::Feature kVaapiVP8Encoder{"VaapiVP8Encoder", base::FEATURE_ENABLED_BY_DEFAULT}; @@ -444,11 +493,11 @@ const base::Feature kVaapiVP8Encoder{"VaapiVP8Encoder", const base::Feature kVaapiVP9Encoder{"VaapiVP9Encoder", base::FEATURE_ENABLED_BY_DEFAULT}; -#if defined(ARCH_CPU_X86_FAMILY) && BUILDFLAG(IS_ASH) +#if defined(ARCH_CPU_X86_FAMILY) && BUILDFLAG(IS_CHROMEOS_ASH) // Enable VP9 k-SVC decoding with HW decoder for webrtc use case on ChromeOS. const base::Feature kVp9kSVCHWDecoding{"Vp9kSVCHWDecoding", base::FEATURE_ENABLED_BY_DEFAULT}; -#endif // defined(ARCH_CPU_X86_FAMILY) && BUILDFLAG(IS_ASH) +#endif // defined(ARCH_CPU_X86_FAMILY) && BUILDFLAG(IS_CHROMEOS_ASH) // Inform video blitter of video color space. const base::Feature kVideoBlitColorAccuracy{"video-blit-color-accuracy", @@ -469,6 +518,10 @@ const base::Feature kLiveCaption{"LiveCaption", const base::Feature kUseSodaForLiveCaption{"UseSodaForLiveCaption", base::FEATURE_DISABLED_BY_DEFAULT}; +// Live Caption runs system-wide on ChromeOS, as opposed to just in the browser. +const base::Feature kLiveCaptionSystemWideOnChromeOS{ + "LiveCaptionSystemWideOnChromeOS", base::FEATURE_DISABLED_BY_DEFAULT}; + // Prevents UrlProvisionFetcher from making a provisioning request. If // specified, any provisioning request made will not be sent to the provisioning // server, and the response will indicate a failure to communicate with the @@ -502,7 +555,7 @@ const base::Feature kWidevineAv1ForceSupportForTesting{ // Enables handling of hardware media keys for controlling media. const base::Feature kHardwareMediaKeyHandling { "HardwareMediaKeyHandling", -#if BUILDFLAG(IS_ASH) || defined(OS_WIN) || defined(OS_MAC) || \ +#if BUILDFLAG(IS_CHROMEOS_ASH) || defined(OS_WIN) || defined(OS_MAC) || \ BUILDFLAG(USE_MPRIS) base::FEATURE_ENABLED_BY_DEFAULT #else @@ -540,10 +593,6 @@ const base::Feature kAutoplayIgnoreWebAudio{"AutoplayIgnoreWebAudio", const base::Feature kAutoplayDisableSettings{"AutoplayDisableSettings", base::FEATURE_DISABLED_BY_DEFAULT}; -// Whether we should allow autoplay whitelisting via sounds settings. -const base::Feature kAutoplayWhitelistSettings{ - "AutoplayWhitelistSettings", base::FEATURE_ENABLED_BY_DEFAULT}; - #if defined(OS_ANDROID) // Should we allow video playback to use an overlay if it's not needed for // security? Normally, we'd always want to allow this, except as part of the @@ -610,7 +659,7 @@ const base::Feature kUsePooledSharedImageVideoProvider{ "UsePooledSharedImageVideoProvider", base::FEATURE_ENABLED_BY_DEFAULT}; #endif // defined(OS_ANDROID) -#if BUILDFLAG(IS_ASH) && BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION) +#if BUILDFLAG(IS_CHROMEOS_ASH) && BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION) // Enable the hardware-accelerated direct video decoder instead of the one // needing the VdaVideoDecoder adapter. This flag is used mainly as a // chrome:flag for developers debugging issues. TODO(b/159825227): remove when @@ -625,7 +674,8 @@ const base::Feature kUseChromeOSDirectVideoDecoder{ const base::Feature kUseAlternateVideoDecoderImplementation{ "UseAlternateVideoDecoderImplementation", base::FEATURE_DISABLED_BY_DEFAULT}; -#endif // BUILDFLAG(IS_ASH) && BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION) +#endif // BUILDFLAG(IS_CHROMEOS_ASH) && + // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION) #if defined(OS_WIN) // Does NV12->NV12 video copy on the main thread right before the texture's @@ -639,6 +689,11 @@ const base::Feature kDelayCopyNV12Textures{"DelayCopyNV12Textures", const base::Feature kDirectShowGetPhotoState{"DirectShowGetPhotoState", base::FEATURE_ENABLED_BY_DEFAULT}; +// Includes Infrared cameras in the list returned for EnumerateDevices() on +// Windows. +const base::Feature kIncludeIRCamerasInDeviceEnumeration{ + "IncludeIRCamerasInDeviceEnumeration", base::FEATURE_DISABLED_BY_DEFAULT}; + // Enables asynchronous H264 HW encode acceleration using Media Foundation for // Windows. const base::Feature kMediaFoundationAsyncH264Encoding{ @@ -652,6 +707,10 @@ const base::Feature MEDIA_EXPORT kMediaFoundationAV1Decoding{ const base::Feature kMediaFoundationVideoCapture{ "MediaFoundationVideoCapture", base::FEATURE_ENABLED_BY_DEFAULT}; +// Enables MediaFoundation based video capture with D3D11 +const base::Feature kMediaFoundationD3D11VideoCapture{ + "MediaFoundationD3D11VideoCapture", base::FEATURE_DISABLED_BY_DEFAULT}; + // Enables VP8 decode acceleration for Windows. const base::Feature MEDIA_EXPORT kMediaFoundationVP8Decoding{ "MediaFoundationVP8Decoding", base::FEATURE_DISABLED_BY_DEFAULT}; @@ -671,17 +730,19 @@ const base::Feature MEDIA_EXPORT kWasapiRawAudioCapture{ // Controls whether the next version mac capturer, including power improvements, // zero copy operation, and other improvements, is active. const base::Feature MEDIA_EXPORT kAVFoundationCaptureV2{ - "AVFoundationCaptureV2", base::FEATURE_ENABLED_BY_DEFAULT}; + "AVFoundationCaptureV2", base::FEATURE_DISABLED_BY_DEFAULT}; // Controls whether or not the V2 capturer exports IOSurfaces for zero-copy. // This feature only has any effect if kAVFoundationCaptureV2 is also enabled. const base::Feature MEDIA_EXPORT kAVFoundationCaptureV2ZeroCopy{ "AVFoundationCaptureV2ZeroCopy", base::FEATURE_ENABLED_BY_DEFAULT}; - -const base::Feature MEDIA_EXPORT kVideoToolboxVp9Decoding{ - "VideoToolboxVp9Decoding", base::FEATURE_DISABLED_BY_DEFAULT}; #endif // defined(OS_MAC) +#if BUILDFLAG(IS_CHROMEOS_ASH) +const base::Feature MEDIA_EXPORT kDeprecateLowUsageCodecs{ + "DeprecateLowUsageCodecs", base::FEATURE_ENABLED_BY_DEFAULT}; +#endif // BUILDFLAG(IS_CHROMEOS_ASH) + std::string GetEffectiveAutoplayPolicy(const base::CommandLine& command_line) { // Return the autoplay policy set in the command line, if any. if (command_line.HasSwitch(switches::kAutoplayPolicy)) @@ -731,15 +792,16 @@ const base::Feature kMediaEngagementHTTPSOnly{ // Enables Media Feeds to allow sites to provide specific recommendations for // users. -const base::Feature kMediaFeeds{"MediaFeeds", base::FEATURE_ENABLED_BY_DEFAULT}; +const base::Feature kMediaFeeds{"MediaFeeds", + base::FEATURE_DISABLED_BY_DEFAULT}; // Enables fetching Media Feeds periodically in the background. const base::Feature kMediaFeedsBackgroundFetching{ - "MediaFeedsBackgroundFetching", base::FEATURE_ENABLED_BY_DEFAULT}; + "MediaFeedsBackgroundFetching", base::FEATURE_DISABLED_BY_DEFAULT}; // Enables checking Media Feeds against safe search to prevent adult content. const base::Feature kMediaFeedsSafeSearch{"MediaFeedsSafeSearch", - base::FEATURE_ENABLED_BY_DEFAULT}; + base::FEATURE_DISABLED_BY_DEFAULT}; // Enables experimental local learning for media. Used in the context of media // capabilities only. Adds reporting only; does not change media behavior. @@ -769,7 +831,7 @@ const base::Feature kMediaPowerExperiment{"MediaPowerExperiment", // has audio focus enabled. const base::Feature kAudioFocusDuckFlash { "AudioFocusDuckFlash", -#if BUILDFLAG(IS_ASH) +#if BUILDFLAG(IS_CHROMEOS_ASH) base::FEATURE_ENABLED_BY_DEFAULT #else base::FEATURE_DISABLED_BY_DEFAULT @@ -797,6 +859,9 @@ const base::Feature kInternalMediaSession { const base::Feature kKaleidoscope{"Kaleidoscope", base::FEATURE_ENABLED_BY_DEFAULT}; +const base::Feature kKaleidoscopeInMenu{"KaleidoscopeInMenu", + base::FEATURE_DISABLED_BY_DEFAULT}; + const base::Feature kKaleidoscopeForceShowFirstRunExperience{ "KaleidoscopeForceShowFirstRunExperience", base::FEATURE_DISABLED_BY_DEFAULT}; @@ -824,17 +889,10 @@ bool IsVideoCaptureAcceleratedJpegDecodingEnabled() { switches::kUseFakeMjpegDecodeAccelerator)) { return true; } -#if BUILDFLAG(IS_ASH) +#if BUILDFLAG(IS_CHROMEOS_ASH) return true; #endif return false; } -// When enabled, causes the H264Decoder to treat each DecoderBuffer sent to it -// as a complete frame, rather than waiting for a following indicator for frame -// completeness. Temporary flag to allow verifying if this change breaks -// anything. -const base::Feature kH264DecoderBufferIsCompleteFrame{ - "H264DecoderBufferIsCompleteFrame", base::FEATURE_ENABLED_BY_DEFAULT}; - } // namespace media diff --git a/chromium/media/base/media_switches.h b/chromium/media/base/media_switches.h index e08ba0324d0..62396246cf6 100644 --- a/chromium/media/base/media_switches.h +++ b/chromium/media/base/media_switches.h @@ -86,6 +86,10 @@ MEDIA_EXPORT extern const char kOverrideEnabledCdmInterfaceVersion[]; MEDIA_EXPORT extern const char kOverrideHardwareSecureCodecsForTesting[]; MEDIA_EXPORT extern const char kEnableLiveCaptionPrefForTesting[]; +#if BUILDFLAG(ENABLE_PLATFORM_HEVC) +MEDIA_EXPORT extern const char kEnableClearHevcForTesting[]; +#endif + namespace autoplay { MEDIA_EXPORT extern const char kDocumentUserActivationRequiredPolicy[]; @@ -105,7 +109,6 @@ MEDIA_EXPORT extern const base::Feature kAudioFocusDuckFlash; MEDIA_EXPORT extern const base::Feature kAudioFocusLossSuspendMediaSession; MEDIA_EXPORT extern const base::Feature kAutoplayIgnoreWebAudio; MEDIA_EXPORT extern const base::Feature kAutoplayDisableSettings; -MEDIA_EXPORT extern const base::Feature kAutoplayWhitelistSettings; MEDIA_EXPORT extern const base::Feature kBackgroundVideoPauseOptimization; MEDIA_EXPORT extern const base::Feature kBresenhamCadence; MEDIA_EXPORT extern const base::Feature kCdmHostVerification; @@ -114,9 +117,12 @@ MEDIA_EXPORT extern const base::Feature kD3D11PrintCodecOnCrash; MEDIA_EXPORT extern const base::Feature kD3D11VideoDecoder; MEDIA_EXPORT extern const base::Feature kD3D11VideoDecoderIgnoreWorkarounds; MEDIA_EXPORT extern const base::Feature kD3D11VideoDecoderVP9Profile2; +MEDIA_EXPORT extern const base::Feature kD3D11VideoDecoderAV1; MEDIA_EXPORT extern const base::Feature kD3D11VideoDecoderSkipMultithreaded; MEDIA_EXPORT extern const base::Feature kD3D11VideoDecoderAlwaysCopy; MEDIA_EXPORT extern const base::Feature kD3D11VideoDecoderAllowOverlay; +MEDIA_EXPORT extern const base::Feature kEnableMediaInternals; +MEDIA_EXPORT extern const base::Feature kExposeSwDecodersToWebRTC; MEDIA_EXPORT extern const base::Feature kExternalClearKeyForTesting; MEDIA_EXPORT extern const base::Feature kFFmpegDecodeOpaqueVP8; MEDIA_EXPORT extern const base::Feature kFailUrlProvisionFetcherForTesting; @@ -130,16 +136,17 @@ MEDIA_EXPORT extern const base::Feature kGlobalMediaControlsOverlayControls; MEDIA_EXPORT extern const base::Feature kGlobalMediaControlsPictureInPicture; MEDIA_EXPORT extern const base::Feature kGlobalMediaControlsSeamlessTransfer; MEDIA_EXPORT extern const base::Feature kGlobalMediaControlsModernUI; -MEDIA_EXPORT extern const base::Feature kH264DecoderBufferIsCompleteFrame; MEDIA_EXPORT extern const base::Feature kHardwareMediaKeyHandling; MEDIA_EXPORT extern const base::Feature kHardwareSecureDecryption; MEDIA_EXPORT extern const base::Feature kInternalMediaSession; MEDIA_EXPORT extern const base::Feature kKaleidoscope; +MEDIA_EXPORT extern const base::Feature kKaleidoscopeInMenu; MEDIA_EXPORT extern const base::Feature kKaleidoscopeForceShowFirstRunExperience; MEDIA_EXPORT extern const base::Feature kKaleidoscopeModule; MEDIA_EXPORT extern const base::Feature kKaleidoscopeModuleCacheOnly; MEDIA_EXPORT extern const base::Feature kLiveCaption; +MEDIA_EXPORT extern const base::Feature kLiveCaptionSystemWideOnChromeOS; MEDIA_EXPORT extern const base::Feature kLowDelayVideoRenderingOnLiveStream; MEDIA_EXPORT extern const base::Feature kMediaCapabilitiesQueryGpuFactories; MEDIA_EXPORT extern const base::Feature kMediaCapabilitiesWithParameters; @@ -164,16 +171,25 @@ MEDIA_EXPORT extern const base::Feature kRecordMediaEngagementScores; MEDIA_EXPORT extern const base::Feature kRecordWebAudioEngagement; MEDIA_EXPORT extern const base::Feature kResumeBackgroundVideo; MEDIA_EXPORT extern const base::Feature kRevokeMediaSourceObjectURLOnAttach; +MEDIA_EXPORT extern const base::Feature kSurfaceLayerForMediaStreams; MEDIA_EXPORT extern const base::Feature kSuspendMutedAudio; MEDIA_EXPORT extern const base::Feature kSpecCompliantCanPlayThrough; MEDIA_EXPORT extern const base::Feature kUnifiedAutoplay; MEDIA_EXPORT extern const base::Feature kUseAndroidOverlayAggressively; +MEDIA_EXPORT extern const base::Feature kUseDecoderStreamForWebRTC; MEDIA_EXPORT extern const base::Feature kUseFakeDeviceForMediaStream; MEDIA_EXPORT extern const base::Feature kUseMediaHistoryStore; MEDIA_EXPORT extern const base::Feature kUseR16Texture; MEDIA_EXPORT extern const base::Feature kUseSodaForLiveCaption; +// TODO(crbug.com/1052397): Revisit once build flag switch of lacros-chrome is +// complete. +#if (defined(OS_LINUX) || BUILDFLAG(IS_CHROMEOS_LACROS)) +MEDIA_EXPORT extern const base::Feature kVaapiVideoDecodeLinux; +MEDIA_EXPORT extern const base::Feature kVaapiVideoEncodeLinux; +#endif // defined(OS_LINUX) || BUILDFLAG(IS_CHROMEOS_LACROS)) MEDIA_EXPORT extern const base::Feature kVaapiAV1Decoder; MEDIA_EXPORT extern const base::Feature kVaapiLowPowerEncoderGen9x; +MEDIA_EXPORT extern const base::Feature kVaapiEnforceVideoMinMaxResolution; MEDIA_EXPORT extern const base::Feature kVaapiVP8Encoder; MEDIA_EXPORT extern const base::Feature kVaapiVP9Encoder; MEDIA_EXPORT extern const base::Feature kVideoBlitColorAccuracy; @@ -184,9 +200,9 @@ MEDIA_EXPORT extern const base::Feature kResolutionBasedDecoderPriority; MEDIA_EXPORT extern const base::Feature kForceHardwareVideoDecoders; MEDIA_EXPORT extern const base::Feature kForceHardwareAudioDecoders; -#if defined(ARCH_CPU_X86_FAMILY) && BUILDFLAG(IS_ASH) +#if defined(ARCH_CPU_X86_FAMILY) && BUILDFLAG(IS_CHROMEOS_ASH) MEDIA_EXPORT extern const base::Feature kVp9kSVCHWDecoding; -#endif // defined(ARCH_CPU_X86_FAMILY) && BUILDFLAG(IS_ASH) +#endif // defined(ARCH_CPU_X86_FAMILY) && BUILDFLAG(IS_CHROMEOS_ASH) #if defined(OS_ANDROID) MEDIA_EXPORT extern const base::Feature kAllowNonSecureOverlays; @@ -203,26 +219,32 @@ MEDIA_EXPORT extern const base::Feature kUseAudioLatencyFromHAL; MEDIA_EXPORT extern const base::Feature kUsePooledSharedImageVideoProvider; #endif // defined(OS_ANDROID) -#if BUILDFLAG(IS_ASH) && BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION) +#if BUILDFLAG(IS_CHROMEOS_ASH) && BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION) MEDIA_EXPORT extern const base::Feature kUseChromeOSDirectVideoDecoder; MEDIA_EXPORT extern const base::Feature kUseAlternateVideoDecoderImplementation; -#endif // BUILDFLAG(IS_ASH) && BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION) +#endif // BUILDFLAG(IS_CHROMEOS_ASH) && + // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION) #if defined(OS_WIN) MEDIA_EXPORT extern const base::Feature kDelayCopyNV12Textures; MEDIA_EXPORT extern const base::Feature kDirectShowGetPhotoState; +MEDIA_EXPORT extern const base::Feature kIncludeIRCamerasInDeviceEnumeration; MEDIA_EXPORT extern const base::Feature kMediaFoundationAsyncH264Encoding; MEDIA_EXPORT extern const base::Feature kMediaFoundationAV1Decoding; MEDIA_EXPORT extern const base::Feature kMediaFoundationVideoCapture; MEDIA_EXPORT extern const base::Feature kMediaFoundationVP8Decoding; +MEDIA_EXPORT extern const base::Feature kMediaFoundationD3D11VideoCapture; MEDIA_EXPORT extern const base::Feature kWasapiRawAudioCapture; #endif // defined(OS_WIN) #if defined(OS_MAC) MEDIA_EXPORT extern const base::Feature kAVFoundationCaptureV2; MEDIA_EXPORT extern const base::Feature kAVFoundationCaptureV2ZeroCopy; -MEDIA_EXPORT extern const base::Feature kVideoToolboxVp9Decoding; +#endif + +#if BUILDFLAG(IS_CHROMEOS_ASH) +MEDIA_EXPORT extern const base::Feature kDeprecateLowUsageCodecs; #endif // Based on a |command_line| and the current platform, returns the effective diff --git a/chromium/media/base/media_track.h b/chromium/media/base/media_track.h index a4cbb7a0240..c0e020a9caa 100644 --- a/chromium/media/base/media_track.h +++ b/chromium/media/base/media_track.h @@ -7,7 +7,7 @@ #include <string> -#include "base/util/type_safety/strong_alias.h" +#include "base/types/strong_alias.h" #include "media/base/media_export.h" #include "media/base/stream_parser.h" @@ -16,10 +16,10 @@ namespace media { class MEDIA_EXPORT MediaTrack { public: enum Type { Text, Audio, Video }; - using Id = util::StrongAlias<class IdTag, std::string>; - using Kind = util::StrongAlias<class KindTag, std::string>; - using Label = util::StrongAlias<class LabelTag, std::string>; - using Language = util::StrongAlias<class LanguageTag, std::string>; + using Id = base::StrongAlias<class IdTag, std::string>; + using Kind = base::StrongAlias<class KindTag, std::string>; + using Label = base::StrongAlias<class LabelTag, std::string>; + using Language = base::StrongAlias<class LanguageTag, std::string>; MediaTrack(Type type, StreamParser::TrackId bytestream_track_id, const Kind& kind, diff --git a/chromium/media/base/mime_util_internal.cc b/chromium/media/base/mime_util_internal.cc index 39d62476358..6c554eb112e 100644 --- a/chromium/media/base/mime_util_internal.cc +++ b/chromium/media/base/mime_util_internal.cc @@ -565,13 +565,12 @@ bool MimeUtil::IsCodecSupportedOnAndroid( case THEORA: return false; - // AV1 is not supported on Android yet. - case AV1: - return false; - // ---------------------------------------------------------------------- // The remaining codecs may be supported depending on platform abilities. // ---------------------------------------------------------------------- + case AV1: + return BUILDFLAG(ENABLE_AV1_DECODER); + case MPEG2_AAC: // MPEG2_AAC cannot be used in HLS (mpegurl suffix), but this is enforced // in the parsing step by excluding MPEG2_AAC from the list of diff --git a/chromium/media/base/mime_util_unittest.cc b/chromium/media/base/mime_util_unittest.cc index c539b94b976..ae9451dab30 100644 --- a/chromium/media/base/mime_util_unittest.cc +++ b/chromium/media/base/mime_util_unittest.cc @@ -563,7 +563,6 @@ TEST(IsCodecSupportedOnAndroidTest, EncryptedCodecBehavior) { switch (codec) { // These codecs are never supported by the Android platform. case MimeUtil::INVALID_CODEC: - case MimeUtil::AV1: case MimeUtil::MPEG_H_AUDIO: case MimeUtil::THEORA: EXPECT_FALSE(result); @@ -611,6 +610,10 @@ TEST(IsCodecSupportedOnAndroidTest, EncryptedCodecBehavior) { case MimeUtil::EAC3: EXPECT_EQ(HasEac3Support(), result); break; + + case MimeUtil::AV1: + EXPECT_EQ(BUILDFLAG(ENABLE_AV1_DECODER), result); + break; } }); } @@ -630,7 +633,6 @@ TEST(IsCodecSupportedOnAndroidTest, ClearCodecBehavior) { case MimeUtil::INVALID_CODEC: case MimeUtil::MPEG_H_AUDIO: case MimeUtil::THEORA: - case MimeUtil::AV1: EXPECT_FALSE(result); break; @@ -671,6 +673,10 @@ TEST(IsCodecSupportedOnAndroidTest, ClearCodecBehavior) { case MimeUtil::EAC3: EXPECT_EQ(HasEac3Support(), result); break; + + case MimeUtil::AV1: + EXPECT_EQ(BUILDFLAG(ENABLE_AV1_DECODER), result); + break; } }); } diff --git a/chromium/media/base/mock_filters.cc b/chromium/media/base/mock_filters.cc index f7fc5c0cf91..4e8d7c481aa 100644 --- a/chromium/media/base/mock_filters.cc +++ b/chromium/media/base/mock_filters.cc @@ -84,6 +84,7 @@ MockVideoDecoder::MockVideoDecoder(bool is_platform_decoder, supports_decryption_(supports_decryption), decoder_name_(std::move(decoder_name)) { ON_CALL(*this, CanReadWithoutStalling()).WillByDefault(Return(true)); + ON_CALL(*this, IsOptimizedForRTC()).WillByDefault(Return(false)); } MockVideoDecoder::~MockVideoDecoder() = default; @@ -100,6 +101,15 @@ std::string MockVideoDecoder::GetDisplayName() const { return decoder_name_; } +VideoDecoderType MockVideoDecoder::GetDecoderType() const { + return VideoDecoderType::kUnknown; +} + +MockAudioEncoder::MockAudioEncoder() = default; +MockAudioEncoder::~MockAudioEncoder() { + OnDestruct(); +} + MockVideoEncoder::MockVideoEncoder() = default; MockVideoEncoder::~MockVideoEncoder() { Dtor(); @@ -131,6 +141,10 @@ std::string MockAudioDecoder::GetDisplayName() const { return decoder_name_; } +AudioDecoderType MockAudioDecoder::GetDecoderType() const { + return AudioDecoderType::kUnknown; +} + MockRendererClient::MockRendererClient() = default; MockRendererClient::~MockRendererClient() = default; diff --git a/chromium/media/base/mock_filters.h b/chromium/media/base/mock_filters.h index 832c7b4650d..f9fae0efb25 100644 --- a/chromium/media/base/mock_filters.h +++ b/chromium/media/base/mock_filters.h @@ -16,6 +16,7 @@ #include "build/build_config.h" #include "media/base/audio_decoder.h" #include "media/base/audio_decoder_config.h" +#include "media/base/audio_encoder.h" #include "media/base/audio_parameters.h" #include "media/base/audio_renderer.h" #include "media/base/callback_registry.h" @@ -70,8 +71,8 @@ class MockPipelineClient : public Pipeline::Client { MOCK_METHOD1(OnVideoOpacityChange, void(bool)); MOCK_METHOD1(OnVideoFrameRateChange, void(base::Optional<int>)); MOCK_METHOD0(OnVideoAverageKeyframeDistanceUpdate, void()); - MOCK_METHOD1(OnAudioDecoderChange, void(const PipelineDecoderInfo&)); - MOCK_METHOD1(OnVideoDecoderChange, void(const PipelineDecoderInfo&)); + MOCK_METHOD1(OnAudioDecoderChange, void(const AudioDecoderInfo&)); + MOCK_METHOD1(OnVideoDecoderChange, void(const VideoDecoderInfo&)); MOCK_METHOD1(OnRemotePlayStateChange, void(MediaStatus::State state)); }; @@ -117,6 +118,7 @@ class MockPipeline : public Pipeline { MOCK_METHOD1(SetVolume, void(float)); MOCK_METHOD1(SetLatencyHint, void(base::Optional<base::TimeDelta>)); MOCK_METHOD1(SetPreservesPitch, void(bool)); + MOCK_METHOD1(SetAutoplayInitiated, void(bool)); // TODO(sandersd): These should probably have setters too. MOCK_CONST_METHOD0(GetMediaTime, base::TimeDelta()); @@ -228,6 +230,7 @@ class MockVideoDecoder : public VideoDecoder { bool IsPlatformDecoder() const override; bool SupportsDecryption() const override; std::string GetDisplayName() const override; + VideoDecoderType GetDecoderType() const override; // VideoDecoder implementation. void Initialize(const VideoDecoderConfig& config, @@ -254,6 +257,7 @@ class MockVideoDecoder : public VideoDecoder { MOCK_CONST_METHOD0(GetMaxDecodeRequests, int()); MOCK_CONST_METHOD0(CanReadWithoutStalling, bool()); MOCK_CONST_METHOD0(NeedsBitstreamConversion, bool()); + MOCK_CONST_METHOD0(IsOptimizedForRTC, bool()); private: const bool is_platform_decoder_; @@ -262,6 +266,35 @@ class MockVideoDecoder : public VideoDecoder { DISALLOW_COPY_AND_ASSIGN(MockVideoDecoder); }; +class MockAudioEncoder : public AudioEncoder { + public: + MockAudioEncoder(); + ~MockAudioEncoder() override; + + // AudioEncoder implementation. + MOCK_METHOD(void, + Initialize, + (const AudioEncoder::Options& options, + AudioEncoder::OutputCB output_cb, + AudioEncoder::StatusCB done_cb), + (override)); + + MOCK_METHOD(void, + Encode, + (std::unique_ptr<AudioBus> audio_bus, + base::TimeTicks capture_time, + AudioEncoder::StatusCB done_cb), + (override)); + + MOCK_METHOD(void, Flush, (AudioEncoder::StatusCB done_cb), (override)); + + // A function for mocking destructor calls + MOCK_METHOD(void, OnDestruct, ()); + + private: + DISALLOW_COPY_AND_ASSIGN(MockAudioEncoder); +}; + class MockVideoEncoder : public VideoEncoder { public: MockVideoEncoder(); @@ -312,6 +345,7 @@ class MockAudioDecoder : public AudioDecoder { bool IsPlatformDecoder() const override; bool SupportsDecryption() const override; std::string GetDisplayName() const override; + AudioDecoderType GetDecoderType() const override; // AudioDecoder implementation. void Initialize(const AudioDecoderConfig& config, @@ -414,6 +448,7 @@ class MockAudioRenderer : public AudioRenderer { MOCK_METHOD1(SetLatencyHint, void(base::Optional<base::TimeDelta> latency_hint)); MOCK_METHOD1(SetPreservesPitch, void(bool)); + MOCK_METHOD1(SetAutoplayInitiated, void(bool)); private: DISALLOW_COPY_AND_ASSIGN(MockAudioRenderer); @@ -436,6 +471,7 @@ class MockRenderer : public Renderer { PipelineStatusCallback& init_cb)); MOCK_METHOD1(SetLatencyHint, void(base::Optional<base::TimeDelta>)); MOCK_METHOD1(SetPreservesPitch, void(bool)); + MOCK_METHOD1(SetAutoplayInitiated, void(bool)); void Flush(base::OnceClosure flush_cb) override { OnFlush(flush_cb); } MOCK_METHOD1(OnFlush, void(base::OnceClosure& flush_cb)); MOCK_METHOD1(StartPlayingFrom, void(base::TimeDelta timestamp)); @@ -562,10 +598,10 @@ class MockDecryptor : public Decryptor { void(const VideoDecoderConfig& config, DecoderInitCB init_cb)); MOCK_METHOD2(DecryptAndDecodeAudio, void(scoped_refptr<DecoderBuffer> encrypted, - const AudioDecodeCB& audio_decode_cb)); + AudioDecodeCB audio_decode_cb)); MOCK_METHOD2(DecryptAndDecodeVideo, void(scoped_refptr<DecoderBuffer> encrypted, - const VideoDecodeCB& video_decode_cb)); + VideoDecodeCB video_decode_cb)); MOCK_METHOD1(ResetDecoder, void(StreamType stream_type)); MOCK_METHOD1(DeinitializeDecoder, void(StreamType stream_type)); MOCK_METHOD0(CanAlwaysDecrypt, bool()); @@ -582,9 +618,9 @@ class MockCdmContext : public CdmContext { MOCK_METHOD1(RegisterEventCB, std::unique_ptr<CallbackRegistration>(EventCB event_cb)); MOCK_METHOD0(GetDecryptor, Decryptor*()); - MOCK_METHOD0(RequiresMediaFoundationRenderer, bool()); #if defined(OS_WIN) + MOCK_METHOD0(RequiresMediaFoundationRenderer, bool()); MOCK_METHOD1(GetMediaFoundationCdmProxy, bool(GetMediaFoundationCdmProxyCB get_mf_cdm_proxy_cb)); #endif diff --git a/chromium/media/base/null_video_sink.h b/chromium/media/base/null_video_sink.h index 3b972a9a156..988799c4881 100644 --- a/chromium/media/base/null_video_sink.h +++ b/chromium/media/base/null_video_sink.h @@ -66,7 +66,7 @@ class MEDIA_EXPORT NullVideoSink : public VideoRendererSink { RenderCallback* callback_; // Manages cancellation of periodic Render() callback task. - base::CancelableClosure cancelable_worker_; + base::CancelableRepeatingClosure cancelable_worker_; // Used to determine when a new frame is received. scoped_refptr<VideoFrame> last_frame_; diff --git a/chromium/media/base/offloading_audio_encoder.cc b/chromium/media/base/offloading_audio_encoder.cc new file mode 100644 index 00000000000..a44788396d4 --- /dev/null +++ b/chromium/media/base/offloading_audio_encoder.cc @@ -0,0 +1,76 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/base/offloading_audio_encoder.h" + +#include "base/bind_post_task.h" +#include "base/sequenced_task_runner.h" +#include "base/task/task_traits.h" +#include "base/task/thread_pool.h" +#include "base/threading/sequenced_task_runner_handle.h" + +namespace media { + +OffloadingAudioEncoder::OffloadingAudioEncoder( + std::unique_ptr<AudioEncoder> wrapped_encoder, + const scoped_refptr<base::SequencedTaskRunner> work_runner, + const scoped_refptr<base::SequencedTaskRunner> callback_runner) + : wrapped_encoder_(std::move(wrapped_encoder)), + work_runner_(std::move(work_runner)), + callback_runner_(std::move(callback_runner)) { + DCHECK(wrapped_encoder_); + DCHECK(work_runner_); + DCHECK(callback_runner_); + DCHECK_NE(callback_runner_, work_runner_); +} + +OffloadingAudioEncoder::OffloadingAudioEncoder( + std::unique_ptr<AudioEncoder> wrapped_encoder) + : OffloadingAudioEncoder(std::move(wrapped_encoder), + base::ThreadPool::CreateSequencedTaskRunner( + {base::TaskPriority::USER_BLOCKING}), + base::SequencedTaskRunnerHandle::Get()) {} + +void OffloadingAudioEncoder::Initialize(const Options& options, + OutputCB output_cb, + StatusCB done_cb) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + work_runner_->PostTask( + FROM_HERE, base::BindOnce(&AudioEncoder::Initialize, + base::Unretained(wrapped_encoder_.get()), + options, WrapCallback(std::move(output_cb)), + WrapCallback(std::move(done_cb)))); +} + +void OffloadingAudioEncoder::Encode(std::unique_ptr<AudioBus> audio_bus, + base::TimeTicks capture_time, + StatusCB done_cb) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + work_runner_->PostTask( + FROM_HERE, base::BindOnce(&AudioEncoder::Encode, + base::Unretained(wrapped_encoder_.get()), + std::move(audio_bus), capture_time, + WrapCallback(std::move(done_cb)))); +} + +void OffloadingAudioEncoder::Flush(StatusCB done_cb) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + work_runner_->PostTask( + FROM_HERE, base::BindOnce(&AudioEncoder::Flush, + base::Unretained(wrapped_encoder_.get()), + WrapCallback(std::move(done_cb)))); +} + +OffloadingAudioEncoder::~OffloadingAudioEncoder() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + work_runner_->DeleteSoon(FROM_HERE, std::move(wrapped_encoder_)); +} + +template <class T> +T OffloadingAudioEncoder::WrapCallback(T cb) { + DCHECK(callback_runner_); + return base::BindPostTask(callback_runner_, std::move(cb)); +} + +} // namespace media
\ No newline at end of file diff --git a/chromium/media/base/offloading_audio_encoder.h b/chromium/media/base/offloading_audio_encoder.h new file mode 100644 index 00000000000..39f421821a4 --- /dev/null +++ b/chromium/media/base/offloading_audio_encoder.h @@ -0,0 +1,62 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_BASE_OFFLOADING_AUDIO_ENCODER_H_ +#define MEDIA_BASE_OFFLOADING_AUDIO_ENCODER_H_ + +#include <memory> +#include <type_traits> + +#include "base/sequence_checker.h" +#include "media/base/audio_encoder.h" + +namespace base { +class SequencedTaskRunner; +} + +namespace media { + +// A wrapper around audio encoder that offloads all the calls to a dedicated +// task runner. It's used to move synchronous software encoding work off the +// current (main) thread. +class MEDIA_EXPORT OffloadingAudioEncoder final : public AudioEncoder { + public: + // |work_runner| - task runner for encoding work + // |callback_runner| - all encoder's callbacks will be executed on this task + // runner. + OffloadingAudioEncoder( + std::unique_ptr<AudioEncoder> wrapped_encoder, + const scoped_refptr<base::SequencedTaskRunner> work_runner, + const scoped_refptr<base::SequencedTaskRunner> callback_runner); + + // Uses current task runner for callbacks and asks thread pool for a new task + // runner to do actual encoding work. + explicit OffloadingAudioEncoder( + std::unique_ptr<AudioEncoder> wrapped_encoder); + + ~OffloadingAudioEncoder() override; + + void Initialize(const Options& options, + OutputCB output_cb, + StatusCB done_cb) override; + + void Encode(std::unique_ptr<AudioBus> audio_bus, + base::TimeTicks capture_time, + StatusCB done_cb) override; + + void Flush(StatusCB done_cb) override; + + private: + template <class T> + T WrapCallback(T cb); + + std::unique_ptr<AudioEncoder> wrapped_encoder_; + const scoped_refptr<base::SequencedTaskRunner> work_runner_; + const scoped_refptr<base::SequencedTaskRunner> callback_runner_; + SEQUENCE_CHECKER(sequence_checker_); +}; + +} // namespace media + +#endif // MEDIA_BASE_OFFLOADING_AUDIO_ENCODER_H_ diff --git a/chromium/media/base/offloading_audio_encoder_unittest.cc b/chromium/media/base/offloading_audio_encoder_unittest.cc new file mode 100644 index 00000000000..aba0fd86ab7 --- /dev/null +++ b/chromium/media/base/offloading_audio_encoder_unittest.cc @@ -0,0 +1,127 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <memory> +#include <vector> + +#include "base/bind.h" +#include "base/callback_helpers.h" +#include "base/run_loop.h" +#include "base/sequenced_task_runner.h" +#include "base/test/bind.h" +#include "base/test/gmock_callback_support.h" +#include "base/test/task_environment.h" +#include "media/base/media_util.h" +#include "media/base/mock_filters.h" +#include "media/base/offloading_audio_encoder.h" +#include "testing/gtest/include/gtest/gtest.h" + +using ::base::test::RunCallback; +using ::base::test::RunOnceCallback; +using ::testing::_; +using ::testing::DoAll; +using ::testing::Invoke; +using ::testing::Return; + +namespace media { + +class OffloadingAudioEncoderTest : public testing::Test { + protected: + void SetUp() override { + auto mock_audio_encoder = std::make_unique<MockAudioEncoder>(); + mock_audio_encoder_ = mock_audio_encoder.get(); + work_runner_ = base::ThreadPool::CreateSequencedTaskRunner({}); + callback_runner_ = base::SequencedTaskRunnerHandle::Get(); + offloading_encoder_ = std::make_unique<OffloadingAudioEncoder>( + std::move(mock_audio_encoder), work_runner_, callback_runner_); + EXPECT_CALL(*mock_audio_encoder_, OnDestruct()).WillOnce(Invoke([this]() { + EXPECT_TRUE(work_runner_->RunsTasksInCurrentSequence()); + })); + } + + void RunLoop() { task_environment_.RunUntilIdle(); } + + base::test::TaskEnvironment task_environment_; + scoped_refptr<base::SequencedTaskRunner> work_runner_; + scoped_refptr<base::SequencedTaskRunner> callback_runner_; + MockAudioEncoder* mock_audio_encoder_; + std::unique_ptr<OffloadingAudioEncoder> offloading_encoder_; +}; + +TEST_F(OffloadingAudioEncoderTest, Initialize) { + bool called_done = false; + bool called_output = false; + AudioEncoder::Options options; + AudioEncoder::OutputCB output_cb = base::BindLambdaForTesting( + [&](EncodedAudioBuffer, base::Optional<AudioEncoder::CodecDescription>) { + EXPECT_TRUE(callback_runner_->RunsTasksInCurrentSequence()); + called_output = true; + }); + AudioEncoder::StatusCB done_cb = base::BindLambdaForTesting([&](Status s) { + EXPECT_TRUE(callback_runner_->RunsTasksInCurrentSequence()); + called_done = true; + }); + + EXPECT_CALL(*mock_audio_encoder_, Initialize(_, _, _)) + .WillOnce(Invoke([this](const AudioEncoder::Options& options, + AudioEncoder::OutputCB output_cb, + AudioEncoder::StatusCB done_cb) { + EXPECT_TRUE(work_runner_->RunsTasksInCurrentSequence()); + AudioParameters params; + EncodedAudioBuffer buf(params, nullptr, 0, base::TimeTicks()); + std::move(done_cb).Run(Status()); + + // Usually |output_cb| is not called by Initialize() but for this + // test it doesn't matter. We only care about a task runner used + // for running |output_cb|, and not what triggers those callback. + std::move(output_cb).Run(std::move(buf), {}); + })); + + offloading_encoder_->Initialize(options, std::move(output_cb), + std::move(done_cb)); + RunLoop(); + EXPECT_TRUE(called_done); + EXPECT_TRUE(called_output); +} + +TEST_F(OffloadingAudioEncoderTest, Encode) { + bool called_done = false; + AudioEncoder::StatusCB done_cb = base::BindLambdaForTesting([&](Status s) { + EXPECT_TRUE(callback_runner_->RunsTasksInCurrentSequence()); + called_done = true; + }); + + EXPECT_CALL(*mock_audio_encoder_, Encode(_, _, _)) + .WillOnce(Invoke([this](std::unique_ptr<AudioBus> audio_bus, + base::TimeTicks capture_time, + AudioEncoder::StatusCB done_cb) { + EXPECT_TRUE(work_runner_->RunsTasksInCurrentSequence()); + std::move(done_cb).Run(Status()); + })); + + base::TimeTicks ts; + offloading_encoder_->Encode(nullptr, ts, std::move(done_cb)); + RunLoop(); + EXPECT_TRUE(called_done); +} + +TEST_F(OffloadingAudioEncoderTest, Flush) { + bool called_done = false; + AudioEncoder::StatusCB done_cb = base::BindLambdaForTesting([&](Status s) { + EXPECT_TRUE(callback_runner_->RunsTasksInCurrentSequence()); + called_done = true; + }); + + EXPECT_CALL(*mock_audio_encoder_, Flush(_)) + .WillOnce(Invoke([this](AudioEncoder::StatusCB done_cb) { + EXPECT_TRUE(work_runner_->RunsTasksInCurrentSequence()); + std::move(done_cb).Run(Status()); + })); + + offloading_encoder_->Flush(std::move(done_cb)); + RunLoop(); + EXPECT_TRUE(called_done); +} + +} // namespace media diff --git a/chromium/media/base/offloading_video_encoder.cc b/chromium/media/base/offloading_video_encoder.cc index 02acf7135fe..473edb9e727 100644 --- a/chromium/media/base/offloading_video_encoder.cc +++ b/chromium/media/base/offloading_video_encoder.cc @@ -4,10 +4,11 @@ #include "media/base/offloading_video_encoder.h" +#include "base/bind_post_task.h" #include "base/sequenced_task_runner.h" #include "base/task/task_traits.h" #include "base/task/thread_pool.h" -#include "media/base/bind_to_current_loop.h" +#include "base/threading/sequenced_task_runner_handle.h" #include "media/base/video_frame.h" namespace media { @@ -83,7 +84,7 @@ OffloadingVideoEncoder::~OffloadingVideoEncoder() { template <class T> T OffloadingVideoEncoder::WrapCallback(T cb) { DCHECK(callback_runner_); - return media::BindToLoop(callback_runner_.get(), std::move(cb)); + return base::BindPostTask(callback_runner_, std::move(cb)); } -} // namespace media
\ No newline at end of file +} // namespace media diff --git a/chromium/media/base/pipeline.h b/chromium/media/base/pipeline.h index 72a22a031c9..3e1180fb019 100644 --- a/chromium/media/base/pipeline.h +++ b/chromium/media/base/pipeline.h @@ -79,8 +79,8 @@ class MEDIA_EXPORT Pipeline { // Executed whenever the underlying AudioDecoder or VideoDecoder changes // during playback. - virtual void OnAudioDecoderChange(const PipelineDecoderInfo& info) = 0; - virtual void OnVideoDecoderChange(const PipelineDecoderInfo& info) = 0; + virtual void OnAudioDecoderChange(const AudioDecoderInfo& info) = 0; + virtual void OnVideoDecoderChange(const VideoDecoderInfo& info) = 0; // Executed whenever the video frame rate changes. |fps| will be unset if // the frame rate is unstable. The duration used for the frame rate is @@ -229,6 +229,9 @@ class MEDIA_EXPORT Pipeline { // different than 1.0. virtual void SetPreservesPitch(bool preserves_pitch) = 0; + // Sets a flag indicating whether the audio stream was initiated by autoplay. + virtual void SetAutoplayInitiated(bool autoplay_initiated) = 0; + // Returns the current media playback time, which progresses from 0 until // GetMediaDuration(). virtual base::TimeDelta GetMediaTime() const = 0; diff --git a/chromium/media/base/pipeline_impl.cc b/chromium/media/base/pipeline_impl.cc index 67f988b9dde..2d4cdcc0274 100644 --- a/chromium/media/base/pipeline_impl.cc +++ b/chromium/media/base/pipeline_impl.cc @@ -18,8 +18,10 @@ #include "base/synchronization/lock.h" #include "base/synchronization/waitable_event.h" #include "base/threading/thread_task_runner_handle.h" +#include "build/build_config.h" #include "media/base/bind_to_current_loop.h" #include "media/base/cdm_context.h" +#include "media/base/decoder.h" #include "media/base/demuxer.h" #include "media/base/media_log.h" #include "media/base/media_switches.h" @@ -70,6 +72,7 @@ class PipelineImpl::RendererWrapper final : public DemuxerHost, void SetVolume(float volume); void SetLatencyHint(base::Optional<base::TimeDelta> latency_hint); void SetPreservesPitch(bool preserves_pitch); + void SetAutoplayInitiated(bool autoplay_initiated); base::TimeDelta GetMediaTime() const; Ranges<base::TimeDelta> GetBufferedTimeRanges() const; bool DidLoadingProgress(); @@ -196,6 +199,8 @@ class PipelineImpl::RendererWrapper final : public DemuxerHost, // By default, apply pitch adjustments. bool preserves_pitch_ = true; + bool autoplay_initiated_ = false; + // Lock used to serialize |shared_state_|. // TODO(crbug.com/893739): Add GUARDED_BY annotations. mutable base::Lock shared_state_lock_; @@ -496,6 +501,18 @@ void PipelineImpl::RendererWrapper::SetPreservesPitch(bool preserves_pitch) { shared_state_.renderer->SetPreservesPitch(preserves_pitch_); } +void PipelineImpl::RendererWrapper::SetAutoplayInitiated( + bool autoplay_initiated) { + DCHECK(media_task_runner_->BelongsToCurrentThread()); + + if (autoplay_initiated_ == autoplay_initiated) + return; + + autoplay_initiated_ = autoplay_initiated; + if (shared_state_.renderer) + shared_state_.renderer->SetAutoplayInitiated(autoplay_initiated_); +} + base::TimeDelta PipelineImpl::RendererWrapper::GetMediaTime() const { DCHECK(main_task_runner_->BelongsToCurrentThread()); @@ -563,8 +580,11 @@ void PipelineImpl::RendererWrapper::CreateRendererInternal( << "CDM should be available now if has encrypted stream"; base::Optional<RendererFactoryType> factory_type; + +#if defined(OS_WIN) if (cdm_context_ && cdm_context_->RequiresMediaFoundationRenderer()) factory_type = RendererFactoryType::kMediaFoundation; +#endif // defined(OS_WIN) // TODO(xhwang): During Resume(), the |default_renderer_| might already match // the |factory_type|, in which case we shouldn't need to create a new one. @@ -761,7 +781,7 @@ void PipelineImpl::RendererWrapper::OnStatisticsUpdate( shared_state_.statistics.audio_memory_usage += stats.audio_memory_usage; shared_state_.statistics.video_memory_usage += stats.video_memory_usage; - if (!stats.audio_decoder_info.decoder_name.empty() && + if (stats.audio_decoder_info.decoder_type != AudioDecoderType::kUnknown && stats.audio_decoder_info != shared_state_.statistics.audio_decoder_info) { shared_state_.statistics.audio_decoder_info = stats.audio_decoder_info; main_task_runner_->PostTask( @@ -769,7 +789,7 @@ void PipelineImpl::RendererWrapper::OnStatisticsUpdate( weak_pipeline_, stats.audio_decoder_info)); } - if (!stats.video_decoder_info.decoder_name.empty() && + if (stats.video_decoder_info.decoder_type != VideoDecoderType::kUnknown && stats.video_decoder_info != shared_state_.statistics.video_decoder_info) { shared_state_.statistics.video_decoder_info = stats.video_decoder_info; main_task_runner_->PostTask( @@ -1385,6 +1405,15 @@ void PipelineImpl::SetPreservesPitch(bool preserves_pitch) { preserves_pitch)); } +void PipelineImpl::SetAutoplayInitiated(bool autoplay_initiated) { + DCHECK(thread_checker_.CalledOnValidThread()); + + media_task_runner_->PostTask( + FROM_HERE, base::BindOnce(&RendererWrapper::SetAutoplayInitiated, + base::Unretained(renderer_wrapper_.get()), + autoplay_initiated)); +} + base::TimeDelta PipelineImpl::GetMediaTime() const { DCHECK(thread_checker_.CalledOnValidThread()); @@ -1607,7 +1636,7 @@ void PipelineImpl::OnVideoAverageKeyframeDistanceUpdate() { client_->OnVideoAverageKeyframeDistanceUpdate(); } -void PipelineImpl::OnAudioDecoderChange(const PipelineDecoderInfo& info) { +void PipelineImpl::OnAudioDecoderChange(const AudioDecoderInfo& info) { DVLOG(2) << __func__ << ": info=" << info; DCHECK(thread_checker_.CalledOnValidThread()); DCHECK(IsRunning()); @@ -1616,7 +1645,7 @@ void PipelineImpl::OnAudioDecoderChange(const PipelineDecoderInfo& info) { client_->OnAudioDecoderChange(info); } -void PipelineImpl::OnVideoDecoderChange(const PipelineDecoderInfo& info) { +void PipelineImpl::OnVideoDecoderChange(const VideoDecoderInfo& info) { DVLOG(2) << __func__ << ": info=" << info; DCHECK(thread_checker_.CalledOnValidThread()); DCHECK(IsRunning()); diff --git a/chromium/media/base/pipeline_impl.h b/chromium/media/base/pipeline_impl.h index d9444600f6f..beb88adb2b6 100644 --- a/chromium/media/base/pipeline_impl.h +++ b/chromium/media/base/pipeline_impl.h @@ -105,6 +105,7 @@ class MEDIA_EXPORT PipelineImpl : public Pipeline { void SetVolume(float volume) override; void SetLatencyHint(base::Optional<base::TimeDelta> latency_hint) override; void SetPreservesPitch(bool preserves_pitch) override; + void SetAutoplayInitiated(bool autoplay_initiated) override; base::TimeDelta GetMediaTime() const override; Ranges<base::TimeDelta> GetBufferedTimeRanges() const override; base::TimeDelta GetMediaDuration() const override; @@ -161,8 +162,8 @@ class MEDIA_EXPORT PipelineImpl : public Pipeline { void OnVideoNaturalSizeChange(const gfx::Size& size); void OnVideoOpacityChange(bool opaque); void OnVideoAverageKeyframeDistanceUpdate(); - void OnAudioDecoderChange(const PipelineDecoderInfo& info); - void OnVideoDecoderChange(const PipelineDecoderInfo& info); + void OnAudioDecoderChange(const AudioDecoderInfo& info); + void OnVideoDecoderChange(const VideoDecoderInfo& info); void OnRemotePlayStateChange(MediaStatus::State state); void OnVideoFrameRateChange(base::Optional<int> fps); diff --git a/chromium/media/base/pipeline_impl_unittest.cc b/chromium/media/base/pipeline_impl_unittest.cc index cb31db062f6..c3f411582b8 100644 --- a/chromium/media/base/pipeline_impl_unittest.cc +++ b/chromium/media/base/pipeline_impl_unittest.cc @@ -724,14 +724,14 @@ TEST_F(PipelineImplTest, OnStatisticsUpdate) { StartPipelineAndExpect(PIPELINE_OK); PipelineStatistics stats; - stats.audio_decoder_info.decoder_name = "TestAudioDecoderName"; + stats.audio_decoder_info.decoder_type = AudioDecoderType::kMojo; stats.audio_decoder_info.is_platform_decoder = false; EXPECT_CALL(callbacks_, OnAudioDecoderChange(_)); renderer_client_->OnStatisticsUpdate(stats); base::RunLoop().RunUntilIdle(); // VideoDecoderInfo changed and we expect OnVideoDecoderChange() to be called. - stats.video_decoder_info.decoder_name = "TestVideoDecoderName"; + stats.video_decoder_info.decoder_type = VideoDecoderType::kMojo; stats.video_decoder_info.is_platform_decoder = true; EXPECT_CALL(callbacks_, OnVideoDecoderChange(_)); renderer_client_->OnStatisticsUpdate(stats); @@ -749,7 +749,7 @@ TEST_F(PipelineImplTest, OnStatisticsUpdate) { base::RunLoop().RunUntilIdle(); // Both info changed. - stats.audio_decoder_info.decoder_name = "NewTestAudioDecoderName"; + stats.audio_decoder_info.decoder_type = AudioDecoderType::kFFmpeg; stats.video_decoder_info.has_decrypting_demuxer_stream = true; EXPECT_CALL(callbacks_, OnAudioDecoderChange(_)); EXPECT_CALL(callbacks_, OnVideoDecoderChange(_)); diff --git a/chromium/media/base/pipeline_status.cc b/chromium/media/base/pipeline_status.cc index 17e47f1e48e..ab7dfbee853 100644 --- a/chromium/media/base/pipeline_status.cc +++ b/chromium/media/base/pipeline_status.cc @@ -8,6 +8,96 @@ namespace media { +base::Optional<PipelineStatus> StatusCodeToPipelineStatus(StatusCode status) { + switch (status) { + case StatusCode::kOk: + return PIPELINE_OK; + case StatusCode::kPipelineErrorNetwork: + return PIPELINE_ERROR_NETWORK; + case StatusCode::kPipelineErrorDecode: + return PIPELINE_ERROR_DECODE; + case StatusCode::kPipelineErrorAbort: + return PIPELINE_ERROR_ABORT; + case StatusCode::kPipelineErrorInitializationFailed: + return PIPELINE_ERROR_INITIALIZATION_FAILED; + case StatusCode::kPipelineErrorCouldNotRender: + return PIPELINE_ERROR_COULD_NOT_RENDER; + case StatusCode::kPipelineErrorRead: + return PIPELINE_ERROR_READ; + case StatusCode::kPipelineErrorInvalidState: + return PIPELINE_ERROR_INVALID_STATE; + case StatusCode::kPipelineErrorDemuxerErrorCouldNotOpen: + return DEMUXER_ERROR_COULD_NOT_OPEN; + case StatusCode::kPipelineErrorDemuxerErrorCouldNotParse: + return DEMUXER_ERROR_COULD_NOT_PARSE; + case StatusCode::kPipelineErrorDemuxerErrorNoSupportedStreams: + return DEMUXER_ERROR_NO_SUPPORTED_STREAMS; + case StatusCode::kPipelineErrorDecoderErrorNotSupported: + return DECODER_ERROR_NOT_SUPPORTED; + case StatusCode::kPipelineErrorChuckDemuxerErrorAppendFailed: + return CHUNK_DEMUXER_ERROR_APPEND_FAILED; + case StatusCode::kPipelineErrorChunkDemuxerErrorEosStatusDecodeError: + return CHUNK_DEMUXER_ERROR_EOS_STATUS_DECODE_ERROR; + case StatusCode::kPipelineErrorChunkDemuxerErrorEosStatusNetworkError: + return CHUNK_DEMUXER_ERROR_EOS_STATUS_NETWORK_ERROR; + case StatusCode::kPipelineErrorAudioRendererError: + return AUDIO_RENDERER_ERROR; + case StatusCode::kPipelineErrorExternalRendererFailed: + return PIPELINE_ERROR_EXTERNAL_RENDERER_FAILED; + case StatusCode::kPipelineErrorDemuxerErrorDetectedHLS: + return DEMUXER_ERROR_DETECTED_HLS; + default: + NOTREACHED(); + return base::nullopt; + } +} + +StatusCode PipelineStatusToStatusCode(PipelineStatus status) { + switch (status) { + case PIPELINE_OK: + return StatusCode::kOk; + case PIPELINE_ERROR_NETWORK: + return StatusCode::kPipelineErrorNetwork; + case PIPELINE_ERROR_DECODE: + return StatusCode::kPipelineErrorDecode; + case PIPELINE_ERROR_ABORT: + return StatusCode::kPipelineErrorAbort; + case PIPELINE_ERROR_INITIALIZATION_FAILED: + return StatusCode::kPipelineErrorInitializationFailed; + case PIPELINE_ERROR_COULD_NOT_RENDER: + return StatusCode::kPipelineErrorCouldNotRender; + case PIPELINE_ERROR_READ: + return StatusCode::kPipelineErrorRead; + case PIPELINE_ERROR_INVALID_STATE: + return StatusCode::kPipelineErrorInvalidState; + case DEMUXER_ERROR_COULD_NOT_OPEN: + return StatusCode::kPipelineErrorDemuxerErrorCouldNotOpen; + case DEMUXER_ERROR_COULD_NOT_PARSE: + return StatusCode::kPipelineErrorDemuxerErrorCouldNotParse; + case DEMUXER_ERROR_NO_SUPPORTED_STREAMS: + return StatusCode::kPipelineErrorDemuxerErrorNoSupportedStreams; + case DECODER_ERROR_NOT_SUPPORTED: + return StatusCode::kPipelineErrorDecoderErrorNotSupported; + case CHUNK_DEMUXER_ERROR_APPEND_FAILED: + return StatusCode::kPipelineErrorChuckDemuxerErrorAppendFailed; + case CHUNK_DEMUXER_ERROR_EOS_STATUS_DECODE_ERROR: + return StatusCode::kPipelineErrorChunkDemuxerErrorEosStatusDecodeError; + case CHUNK_DEMUXER_ERROR_EOS_STATUS_NETWORK_ERROR: + return StatusCode::kPipelineErrorChunkDemuxerErrorEosStatusNetworkError; + case AUDIO_RENDERER_ERROR: + return StatusCode::kPipelineErrorAudioRendererError; + case PIPELINE_ERROR_EXTERNAL_RENDERER_FAILED: + return StatusCode::kPipelineErrorExternalRendererFailed; + case DEMUXER_ERROR_DETECTED_HLS: + return StatusCode::kPipelineErrorDemuxerErrorDetectedHLS; + } + + NOTREACHED(); + // TODO(crbug.com/1153465): Log pipeline status that failed to convert. + // Return a generic decode error. + return StatusCode::kPipelineErrorDecode; +} + std::string PipelineStatusToString(PipelineStatus status) { #define STRINGIFY_STATUS_CASE(status) \ case status: \ @@ -72,24 +162,4 @@ bool operator!=(const PipelineStatistics& first, return !(first == second); } -bool operator==(const PipelineDecoderInfo& first, - const PipelineDecoderInfo& second) { - return first.decoder_name == second.decoder_name && - first.is_platform_decoder == second.is_platform_decoder && - first.has_decrypting_demuxer_stream == - second.has_decrypting_demuxer_stream; -} - -bool operator!=(const PipelineDecoderInfo& first, - const PipelineDecoderInfo& second) { - return !(first == second); -} - -std::ostream& operator<<(std::ostream& out, const PipelineDecoderInfo& info) { - return out << "{decoder_name:" << info.decoder_name << "," - << "is_platform_decoder:" << info.is_platform_decoder << "," - << "has_decrypting_demuxer_stream:" - << info.has_decrypting_demuxer_stream << "}"; -} - } // namespace media diff --git a/chromium/media/base/pipeline_status.h b/chromium/media/base/pipeline_status.h index 0fc1a6b78ba..a9e448040de 100644 --- a/chromium/media/base/pipeline_status.h +++ b/chromium/media/base/pipeline_status.h @@ -10,8 +10,11 @@ #include <string> #include "base/callback.h" +#include "base/optional.h" #include "base/time/time.h" +#include "media/base/decoder.h" #include "media/base/media_export.h" +#include "media/base/status.h" #include "media/base/timestamp_constants.h" namespace media { @@ -59,6 +62,10 @@ enum PipelineStatus { PIPELINE_STATUS_MAX = DEMUXER_ERROR_DETECTED_HLS, }; +MEDIA_EXPORT base::Optional<PipelineStatus> StatusCodeToPipelineStatus( + StatusCode status); +MEDIA_EXPORT StatusCode PipelineStatusToStatusCode(PipelineStatus status); + // Returns a string version of the status, unique to each PipelineStatus, and // not including any ':'. This makes it suitable for usage in // MediaError.message as the UA-specific-error-code. @@ -71,18 +78,44 @@ MEDIA_EXPORT std::ostream& operator<<(std::ostream& out, PipelineStatus status); using PipelineStatusCB = base::RepeatingCallback<void(PipelineStatus)>; using PipelineStatusCallback = base::OnceCallback<void(PipelineStatus)>; +template <typename DecoderTypeId> struct PipelineDecoderInfo { bool is_platform_decoder = false; bool has_decrypting_demuxer_stream = false; - std::string decoder_name; + DecoderTypeId decoder_type = DecoderTypeId::kUnknown; }; -MEDIA_EXPORT bool operator==(const PipelineDecoderInfo& first, - const PipelineDecoderInfo& second); -MEDIA_EXPORT bool operator!=(const PipelineDecoderInfo& first, - const PipelineDecoderInfo& second); -MEDIA_EXPORT std::ostream& operator<<(std::ostream& out, - const PipelineDecoderInfo& info); +using AudioDecoderInfo = PipelineDecoderInfo<AudioDecoderType>; +using VideoDecoderInfo = PipelineDecoderInfo<VideoDecoderType>; + +template <typename DecoderTypeId> +MEDIA_EXPORT inline bool operator==( + const PipelineDecoderInfo<DecoderTypeId>& first, + const PipelineDecoderInfo<DecoderTypeId>& second) { + return first.decoder_type == second.decoder_type && + first.is_platform_decoder == second.is_platform_decoder && + first.has_decrypting_demuxer_stream == + second.has_decrypting_demuxer_stream; +} + +template <typename DecoderTypeId> +MEDIA_EXPORT inline bool operator!=( + const PipelineDecoderInfo<DecoderTypeId>& first, + const PipelineDecoderInfo<DecoderTypeId>& second) { + return !(first == second); +} + +template <typename DecoderTypeId> +MEDIA_EXPORT inline std::ostream& operator<<( + std::ostream& out, + const PipelineDecoderInfo<DecoderTypeId>& info) { + // TODO(IN THIS CL DON'T FORGET) make a converter to print name. + return out << "{decoder_type:" << static_cast<int64_t>(info.decoder_type) + << "," + << "is_platform_decoder:" << info.is_platform_decoder << "," + << "has_decrypting_demuxer_stream:" + << info.has_decrypting_demuxer_stream << "}"; +} struct MEDIA_EXPORT PipelineStatistics { PipelineStatistics(); @@ -105,8 +138,8 @@ struct MEDIA_EXPORT PipelineStatistics { // Note: Keep these fields at the end of the structure, if you move them you // need to also update the test ProtoUtilsTest::PipelineStatisticsConversion. - PipelineDecoderInfo audio_decoder_info; - PipelineDecoderInfo video_decoder_info; + AudioDecoderInfo audio_decoder_info; + VideoDecoderInfo video_decoder_info; // NOTE: always update operator== implementation in pipeline_status.cc when // adding a field to this struct. Leave this comment at the end. diff --git a/chromium/media/base/renderer.cc b/chromium/media/base/renderer.cc index c1970f95e89..08c4acaa94e 100644 --- a/chromium/media/base/renderer.cc +++ b/chromium/media/base/renderer.cc @@ -34,4 +34,8 @@ void Renderer::SetPreservesPitch(bool preserves_pitch) { // Not supported by most renderers. } +void Renderer::SetAutoplayInitiated(bool autoplay_initiated) { + // Not supported by most renderers. +} + } // namespace media diff --git a/chromium/media/base/renderer.h b/chromium/media/base/renderer.h index b6e1a73373b..9244948c287 100644 --- a/chromium/media/base/renderer.h +++ b/chromium/media/base/renderer.h @@ -55,6 +55,9 @@ class MEDIA_EXPORT Renderer { // different than 1.0. virtual void SetPreservesPitch(bool preserves_pitch); + // Sets a flag indicating whether the audio stream was initiated by autoplay. + virtual void SetAutoplayInitiated(bool autoplay_initiated); + // The following functions must be called after Initialize(). // Discards any buffered data, executing |flush_cb| when completed. diff --git a/chromium/media/base/renderer_factory_selector.cc b/chromium/media/base/renderer_factory_selector.cc index 3695c447d9c..0e912d86a9b 100644 --- a/chromium/media/base/renderer_factory_selector.cc +++ b/chromium/media/base/renderer_factory_selector.cc @@ -40,6 +40,7 @@ void RendererFactorySelector::AddFactory( std::unique_ptr<RendererFactory> factory) { DCHECK(factory); DCHECK(!factories_.count(type)); + DVLOG(2) << __func__ << ": type=" << static_cast<int>(type); factories_[type] = std::move(factory); } diff --git a/chromium/media/base/sample_rates.cc b/chromium/media/base/sample_rates.cc index 6af08622e9f..9d55ba747e8 100644 --- a/chromium/media/base/sample_rates.cc +++ b/chromium/media/base/sample_rates.cc @@ -50,6 +50,9 @@ bool ToAudioSampleRate(int sample_rate, AudioSampleRate* asr) { case 384000: *asr = k384000Hz; return true; + case 768000: + *asr = k768000Hz; + return true; } return false; } diff --git a/chromium/media/base/sample_rates.h b/chromium/media/base/sample_rates.h index 165b3911387..edba90ed2a6 100644 --- a/chromium/media/base/sample_rates.h +++ b/chromium/media/base/sample_rates.h @@ -25,6 +25,7 @@ enum AudioSampleRate { k192000Hz = 10, k24000Hz = 11, k384000Hz = 12, + k768000Hz = 13, // Must always equal the largest value ever reported: kAudioSampleRateMax = k384000Hz, }; diff --git a/chromium/media/base/shared_memory_pool.cc b/chromium/media/base/shared_memory_pool.cc new file mode 100644 index 00000000000..acbca840f9d --- /dev/null +++ b/chromium/media/base/shared_memory_pool.cc @@ -0,0 +1,109 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/base/shared_memory_pool.h" + +#include "base/logging.h" + +namespace { +constexpr size_t kMaxStoredBuffers = 32; +} // namespace + +namespace media { + +SharedMemoryPool::SharedMemoryPool() = default; + +SharedMemoryPool::~SharedMemoryPool() = default; + +SharedMemoryPool::SharedMemoryHandle::SharedMemoryHandle( + base::UnsafeSharedMemoryRegion region, + base::WritableSharedMemoryMapping mapping, + scoped_refptr<SharedMemoryPool> pool) + : region_(std::move(region)), + mapping_(std::move(mapping)), + pool_(std::move(pool)) { + CHECK(pool_); + DCHECK(region_.IsValid()); + DCHECK(mapping_.IsValid()); +} + +SharedMemoryPool::SharedMemoryHandle::~SharedMemoryHandle() { + pool_->ReleaseBuffer(std::move(region_), std::move(mapping_)); +} + +base::UnsafeSharedMemoryRegion* +SharedMemoryPool::SharedMemoryHandle::GetRegion() { + return ®ion_; +} + +base::WritableSharedMemoryMapping* +SharedMemoryPool::SharedMemoryHandle::GetMapping() { + return &mapping_; +} + +std::unique_ptr<SharedMemoryPool::SharedMemoryHandle> +SharedMemoryPool::MaybeAllocateBuffer(size_t region_size) { + base::AutoLock lock(lock_); + + DCHECK_GE(region_size, 0u); + if (is_shutdown_) + return nullptr; + + // Only change the configured size if bigger region is requested to avoid + // unncecessary reallocations. + if (region_size > region_size_) { + mappings_.clear(); + regions_.clear(); + region_size_ = region_size; + } + if (!regions_.empty()) { + DCHECK_EQ(mappings_.size(), regions_.size()); + DCHECK_GE(regions_.back().GetSize(), region_size_); + auto handle = std::make_unique<SharedMemoryHandle>( + std::move(regions_.back()), std::move(mappings_.back()), this); + regions_.pop_back(); + mappings_.pop_back(); + return handle; + } + + auto region = base::UnsafeSharedMemoryRegion::Create(region_size_); + if (!region.IsValid()) + return nullptr; + + base::WritableSharedMemoryMapping mapping = region.Map(); + if (!mapping.IsValid()) + return nullptr; + + return std::make_unique<SharedMemoryHandle>(std::move(region), + std::move(mapping), this); +} + +void SharedMemoryPool::Shutdown() { + base::AutoLock lock(lock_); + DCHECK(!is_shutdown_); + is_shutdown_ = true; + mappings_.clear(); + regions_.clear(); +} + +void SharedMemoryPool::ReleaseBuffer( + base::UnsafeSharedMemoryRegion region, + base::WritableSharedMemoryMapping mapping) { + base::AutoLock lock(lock_); + // Only return regions which are at least as big as the current configuration. + if (is_shutdown_ || regions_.size() >= kMaxStoredBuffers || + !region.IsValid() || region.GetSize() < region_size_) { + DLOG(WARNING) << "Not returning SharedMemoryRegion to the pool:" + << " is_shutdown: " << (is_shutdown_ ? "true" : "false") + << " stored regions: " << regions_.size() + << " configured size: " << region_size_ + << " this region size: " << region.GetSize() + << " valid: " << (region.IsValid() ? "true" : "false"); + return; + } + regions_.emplace_back(std::move(region)); + mappings_.emplace_back(std::move(mapping)); +} + +} // namespace media diff --git a/chromium/media/base/shared_memory_pool.h b/chromium/media/base/shared_memory_pool.h new file mode 100644 index 00000000000..4de5d710230 --- /dev/null +++ b/chromium/media/base/shared_memory_pool.h @@ -0,0 +1,77 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_BASE_SHARED_MEMORY_POOL_H_ +#define MEDIA_BASE_SHARED_MEMORY_POOL_H_ + +#include <vector> + +#include "base/memory/ref_counted.h" +#include "base/memory/unsafe_shared_memory_region.h" +#include "base/synchronization/lock.h" +#include "media/base/media_export.h" + +namespace media { + +// SharedMemoryPool manages allocation and pooling of UnsafeSharedMemoryRegions. +// It is thread-safe. +// May return bigger regions than requested. +// If a requested size is increased, all stored regions are purged. +// Regions are returned to the buffer on destruction of |SharedMemoryHandle| if +// they are of a correct size. +class MEDIA_EXPORT SharedMemoryPool + : public base::RefCountedThreadSafe<SharedMemoryPool> { + public: + // Used to store the allocation result. + // This class returns memory to the pool upon destruction. + class MEDIA_EXPORT SharedMemoryHandle { + public: + SharedMemoryHandle(base::UnsafeSharedMemoryRegion region, + base::WritableSharedMemoryMapping mapping, + scoped_refptr<SharedMemoryPool> pool); + ~SharedMemoryHandle(); + // Disallow copy and assign. + SharedMemoryHandle(const SharedMemoryHandle&) = delete; + SharedMemoryHandle& operator=(const SharedMemoryHandle&) = delete; + + base::UnsafeSharedMemoryRegion* GetRegion(); + + base::WritableSharedMemoryMapping* GetMapping(); + + private: + base::UnsafeSharedMemoryRegion region_; + base::WritableSharedMemoryMapping mapping_; + scoped_refptr<SharedMemoryPool> pool_; + }; + + SharedMemoryPool(); + // Disallow copy and assign. + SharedMemoryPool(const SharedMemoryPool&) = delete; + SharedMemoryPool& operator=(const SharedMemoryPool&) = delete; + + // Allocates a region of the given |size| or reuses a previous allocation if + // possible. + std::unique_ptr<SharedMemoryHandle> MaybeAllocateBuffer(size_t size); + + // Shuts down the pool, freeing all currently unused allocations and freeing + // outstanding ones as they are returned. + void Shutdown(); + + private: + friend class base::RefCountedThreadSafe<SharedMemoryPool>; + ~SharedMemoryPool(); + + void ReleaseBuffer(base::UnsafeSharedMemoryRegion region, + base::WritableSharedMemoryMapping mapping); + + base::Lock lock_; + size_t region_size_ GUARDED_BY(lock_) = 0u; + std::vector<base::UnsafeSharedMemoryRegion> regions_ GUARDED_BY(lock_); + std::vector<base::WritableSharedMemoryMapping> mappings_ GUARDED_BY(lock_); + bool is_shutdown_ GUARDED_BY(lock_) = false; +}; + +} // namespace media + +#endif // MEDIA_BASE_SHARED_MEMORY_POOL_H_ diff --git a/chromium/media/base/shared_memory_pool_unittest.cc b/chromium/media/base/shared_memory_pool_unittest.cc new file mode 100644 index 00000000000..2fb9f951621 --- /dev/null +++ b/chromium/media/base/shared_memory_pool_unittest.cc @@ -0,0 +1,57 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/base/shared_memory_pool.h" + +#include "testing/gtest/include/gtest/gtest.h" + +namespace media { + +TEST(SharedMemoryPoolTest, CreatesRegion) { + scoped_refptr<SharedMemoryPool> pool( + base::MakeRefCounted<SharedMemoryPool>()); + auto handle = pool->MaybeAllocateBuffer(1000); + ASSERT_TRUE(handle); + ASSERT_TRUE(handle->GetRegion()); + EXPECT_TRUE(handle->GetRegion()->IsValid()); + ASSERT_TRUE(handle->GetMapping()); + EXPECT_TRUE(handle->GetMapping()->IsValid()); +} + +TEST(SharedMemoryPoolTest, ReusesRegions) { + scoped_refptr<SharedMemoryPool> pool( + base::MakeRefCounted<SharedMemoryPool>()); + auto handle = pool->MaybeAllocateBuffer(1000u); + ASSERT_TRUE(handle); + base::UnsafeSharedMemoryRegion* region = handle->GetRegion(); + auto id1 = region->GetGUID(); + + // Return memory to the pool. + handle.reset(); + + handle = pool->MaybeAllocateBuffer(1000u); + region = handle->GetRegion(); + // Should reuse the freed region. + EXPECT_EQ(id1, region->GetGUID()); +} + +TEST(SharedMemoryPoolTest, RespectsSize) { + scoped_refptr<SharedMemoryPool> pool( + base::MakeRefCounted<SharedMemoryPool>()); + auto handle = pool->MaybeAllocateBuffer(1000u); + ASSERT_TRUE(handle); + ASSERT_TRUE(handle->GetRegion()); + EXPECT_GE(handle->GetRegion()->GetSize(), 1000u); + + handle = pool->MaybeAllocateBuffer(100u); + ASSERT_TRUE(handle); + ASSERT_TRUE(handle->GetRegion()); + EXPECT_GE(handle->GetRegion()->GetSize(), 100u); + + handle = pool->MaybeAllocateBuffer(1100u); + ASSERT_TRUE(handle); + ASSERT_TRUE(handle->GetRegion()); + EXPECT_GE(handle->GetRegion()->GetSize(), 1100u); +} +} // namespace media diff --git a/chromium/media/base/silent_sink_suspender.h b/chromium/media/base/silent_sink_suspender.h index 52fffadbfe4..40fc611c402 100644 --- a/chromium/media/base/silent_sink_suspender.h +++ b/chromium/media/base/silent_sink_suspender.h @@ -114,7 +114,7 @@ class MEDIA_EXPORT SilentSinkSuspender // A cancelable task that is posted to switch to or from the |fake_sink_| // after a period of silence or first non-silent audio respective. We do this // on Android to save battery consumption. - base::CancelableCallback<void(bool)> sink_transition_callback_; + base::CancelableRepeatingCallback<void(bool)> sink_transition_callback_; // Audio output delay at the moment when transition to |fake_sink_| starts. base::TimeDelta latest_output_delay_; diff --git a/chromium/media/base/sinc_resampler.cc b/chromium/media/base/sinc_resampler.cc index 79b5e2fbe12..e6ff66411ff 100644 --- a/chromium/media/base/sinc_resampler.cc +++ b/chromium/media/base/sinc_resampler.cc @@ -79,6 +79,7 @@ #include "base/check_op.h" #include "base/numerics/math_constants.h" +#include "base/trace_event/trace_event.h" #include "build/build_config.h" #include "cc/base/math_util.h" @@ -228,6 +229,8 @@ void SincResampler::SetRatio(double io_sample_rate_ratio) { } void SincResampler::Resample(int frames, float* destination) { + TRACE_EVENT1(TRACE_DISABLED_BY_DEFAULT("audio"), "SincResampler::Resample", + "io sample rate ratio", io_sample_rate_ratio_); int remaining_frames = frames; // Step (1) -- Prime the input buffer at the start of the input stream. diff --git a/chromium/media/base/status.h b/chromium/media/base/status.h index ef49cc32781..8a7b4e0b9c7 100644 --- a/chromium/media/base/status.h +++ b/chromium/media/base/status.h @@ -166,8 +166,8 @@ MEDIA_EXPORT Status OkStatus(); // } // // auto result = FactoryFn(); -// if (result.has_error()) return std::move(result.error()); -// my_object_ = std::move(result.value()); +// if (result.has_error()) return std::move(result).error(); +// my_object_ = std::move(result).value(); // // Can also be combined into a single switch using `code()`: // @@ -178,7 +178,7 @@ MEDIA_EXPORT Status OkStatus(); // break; // // Maybe switch on specific non-kOk codes for special processing. // default: // Send unknown errors upwards. -// return std::move(result.error()); +// return std::move(result).error(); // } // // Also useful if one would like to get an enum class return value, unless an @@ -189,8 +189,8 @@ MEDIA_EXPORT Status OkStatus(); // StatusOr<ResultType> Foo() { ... } // // auto result = Foo(); -// if (result.has_error()) return std::move(result.error()); -// switch (result.value()) { +// if (result.has_error()) return std::move(result).error(); +// switch (std::move(result).value()) { // case ResultType::kNeedMoreInput: // ... // } @@ -199,16 +199,16 @@ class StatusOr { public: // All of these may be implicit, so that one may just return Status or // the value in question. - StatusOr(Status&& error) : error_(std::move(error)) { - DCHECK(!this->error().is_ok()); + /* not explicit */ StatusOr(Status&& error) : error_(std::move(error)) { + DCHECK_NE(code(), StatusCode::kOk); } - StatusOr(const Status& error) : error_(error) { - DCHECK(!this->error().is_ok()); + /* not explicit */ StatusOr(const Status& error) : error_(error) { + DCHECK_NE(code(), StatusCode::kOk); } StatusOr(StatusCode code, const base::Location& location = base::Location::Current()) : error_(Status(code, "", location)) { - DCHECK(!error().is_ok()); + DCHECK_NE(code, StatusCode::kOk); } StatusOr(T&& value) : value_(std::move(value)) {} @@ -225,26 +225,38 @@ class StatusOr { // Do we have a value? bool has_value() const { return value_.has_value(); } - // Since we often test for errors, provide this too. - bool has_error() const { return !has_value(); } + // Do we have an error? + bool has_error() const { return error_.has_value(); } // Return the error, if we have one. Up to the caller to make sure that we - // have one via |!has_value()|. - Status& error() { return *error_; } - - const Status& error() const { return *error_; } + // have one via |has_error()|. + // NOTE: once this is called, the StatusOr is defunct and should not be used. + Status error() && { + CHECK(error_); + auto error = std::move(*error_); + error_.reset(); + return error; + } - // Return a ref to the value. It's up to the caller to verify that we have a - // value before calling this. - T& value() { return std::get<0>(*value_); } + // Return the value. It's up to the caller to verify that we have a value + // before calling this. Also, this only works once, after which we will have + // an error. Use like this: std::move(status_or).value(); + // NOTE: once this is called, the StatusOr is defunct and should not be used. + T value() && { + CHECK(value_); + auto value = std::move(std::get<0>(*value_)); + value_.reset(); + return value; + } - // Returns the error code we have, if any, or `kOk` if we have a value. If - // this returns `kOk`, then it is equivalent to has_value(). + // Returns the error code we have, if any, or `kOk`. StatusCode code() const { - return has_error() ? error().code() : StatusCode::kOk; + CHECK(error_ || value_); + return error_ ? error_->code() : StatusCode::kOk; } private: + // Optional error. base::Optional<Status> error_; // We wrap |T| in a container so that windows COM wrappers work. They // override operator& and similar, and won't compile in a base::Optional. diff --git a/chromium/media/base/status_codes.h b/chromium/media/base/status_codes.h index 5c2b7feb3f1..45eee6460fa 100644 --- a/chromium/media/base/status_codes.h +++ b/chromium/media/base/status_codes.h @@ -29,6 +29,7 @@ enum class StatusCode : StatusCodeType { // General errors: 0x00 kAborted = 0x00000001, + kInvalidArgument = 0x00000002, // Decoder Errors: 0x01 kDecoderInitializeNeverCompleted = 0x00000101, @@ -77,6 +78,22 @@ enum class StatusCode : StatusCodeType { kCreateVideoProcessorEnumeratorFailed = 0x00000312, kCreateVideoProcessorFailed = 0x00000313, kQueryVideoContextFailed = 0x00000314, + kAcceleratorFlushFailed = 0x00000315, + kTryAgainNotSupported = 0x00000316, + kCryptoConfigFailed = 0x00000317, + kDecoderBeginFrameFailed = 0x00000318, + kReleaseDecoderBufferFailed = 0x00000319, + kGetPicParamBufferFailed = 0x00000320, + kReleasePicParamBufferFailed = 0x00000321, + kGetBitstreamBufferFailed = 0x00000322, + kReleaseBitstreamBufferFailed = 0x00000323, + kGetSliceControlBufferFailed = 0x00000324, + kReleaseSliceControlBufferFailed = 0x00000325, + kDecoderEndFrameFailed = 0x00000326, + kSubmitDecoderBuffersFailed = 0x00000327, + kGetQuantBufferFailed = 0x00000328, + kReleaseQuantBufferFailed = 0x00000329, + kBitstreamBufferSliceTooBig = 0x00000330, // MojoDecoder Errors: 0x04 kMojoDecoderNoWrappedDecoder = 0x00000401, @@ -122,10 +139,50 @@ enum class StatusCode : StatusCodeType { kVaapiBadImageSize = 0x0000070C, kVaapiNoTexture = 0x0000070D, - // Format errors: 0x08 + // Format Errors: 0x08 kH264ParsingError = 0x00000801, kH264BufferTooSmall = 0x00000802, + // Pipeline Errors: 0x09 + // Deprecated: kPipelineErrorUrlNotFound = 0x00000901, + kPipelineErrorNetwork = 0x00000902, + kPipelineErrorDecode = 0x00000903, + // Deprecated: kPipelineErrorDecrypt = 0x00000904, + kPipelineErrorAbort = 0x00000905, + kPipelineErrorInitializationFailed = 0x00000906, + // Unused: 0x00000907 + kPipelineErrorCouldNotRender = 0x00000908, + kPipelineErrorRead = 0x00000909, + // Deprecated: kPipelineErrorOperationPending = 0x0000090a, + kPipelineErrorInvalidState = 0x0000090b, + // Demuxer related errors. + kPipelineErrorDemuxerErrorCouldNotOpen = 0x0000090c, + kPipelineErrorDemuxerErrorCouldNotParse = 0x0000090d, + kPipelineErrorDemuxerErrorNoSupportedStreams = 0x0000090e, + // Decoder related errors. + kPipelineErrorDecoderErrorNotSupported = 0x0000090f, + // ChunkDemuxer related errors. + kPipelineErrorChuckDemuxerErrorAppendFailed = 0x00000910, + kPipelineErrorChunkDemuxerErrorEosStatusDecodeError = 0x00000911, + kPipelineErrorChunkDemuxerErrorEosStatusNetworkError = 0x00000912, + // Audio rendering errors. + kPipelineErrorAudioRendererError = 0x00000913, + // Deprecated: kPipelineErrorAudioRendererErrorSpliceFailed = 0x00000914, + kPipelineErrorExternalRendererFailed = 0x00000915, + // Android only. Used as a signal to fallback MediaPlayerRenderer, and thus + // not exactly an 'error' per say. + kPipelineErrorDemuxerErrorDetectedHLS = 0x00000916, + + // Frame operation errors: 0x0A + kUnsupportedFrameFormatError = 0x00000A01, + + // DecoderStream errors: 0x0B + kDecoderStreamInErrorState = 0x00000B00, + kDecoderStreamReinitFailed = 0x00000B01, + // This is a temporary error for use while the demuxer doesn't return a + // proper status. + kDecoderStreamDemuxerError = 0x00000B02, + // DecodeStatus temporary codes. These names were chosen to match the // DecodeStatus enum, so that un-converted code can DecodeStatus::OK/etc. // Note that OK must result in Status::is_ok(), since converted code will diff --git a/chromium/media/base/status_unittest.cc b/chromium/media/base/status_unittest.cc index 0a2f7cc5e21..2da38b2ca90 100644 --- a/chromium/media/base/status_unittest.cc +++ b/chromium/media/base/status_unittest.cc @@ -201,56 +201,51 @@ TEST_F(StatusTest, StatusOrTypicalUsage) { } TEST_F(StatusTest, StatusOrWithMoveOnlyType) { - StatusOr<std::unique_ptr<int>> error_or(std::make_unique<int>(123)); - EXPECT_TRUE(error_or.has_value()); - EXPECT_FALSE(error_or.has_error()); - std::unique_ptr<int> result = std::move(error_or.value()); - EXPECT_EQ(error_or.value(), nullptr); + StatusOr<std::unique_ptr<int>> status_or(std::make_unique<int>(123)); + EXPECT_TRUE(status_or.has_value()); + EXPECT_FALSE(status_or.has_error()); + std::unique_ptr<int> result = std::move(status_or).value(); EXPECT_NE(result.get(), nullptr); EXPECT_EQ(*result, 123); } TEST_F(StatusTest, StatusOrWithCopyableType) { - StatusOr<int> error_or(123); - EXPECT_TRUE(error_or.has_value()); - EXPECT_FALSE(error_or.has_error()); - int result = std::move(error_or.value()); + StatusOr<int> status_or(123); + EXPECT_TRUE(status_or.has_value()); + EXPECT_FALSE(status_or.has_error()); + int result = std::move(status_or).value(); EXPECT_EQ(result, 123); - // Should be unaffected by the move. - EXPECT_EQ(error_or.value(), 123); } TEST_F(StatusTest, StatusOrMoveConstructionAndAssignment) { // Make sure that we can move-construct and move-assign a move-only value. - StatusOr<std::unique_ptr<int>> error_or_0(std::make_unique<int>(123)); + StatusOr<std::unique_ptr<int>> status_or_0(std::make_unique<int>(123)); - StatusOr<std::unique_ptr<int>> error_or_1(std::move(error_or_0)); - EXPECT_EQ(error_or_0.value(), nullptr); + StatusOr<std::unique_ptr<int>> status_or_1(std::move(status_or_0)); - StatusOr<std::unique_ptr<int>> error_or_2 = std::move(error_or_1); - EXPECT_EQ(error_or_1.value(), nullptr); + StatusOr<std::unique_ptr<int>> status_or_2 = std::move(status_or_1); - // |error_or_2| should have gotten the original. - std::unique_ptr<int> value = std::move(error_or_2.value()); + // |status_or_2| should have gotten the original. + std::unique_ptr<int> value = std::move(status_or_2).value(); EXPECT_EQ(*value, 123); } TEST_F(StatusTest, StatusOrCopyWorks) { // Make sure that we can move-construct and move-assign a move-only value. - StatusOr<int> error_or_0(123); - StatusOr<int> error_or_1(std::move(error_or_0)); - StatusOr<int> error_or_2 = std::move(error_or_1); - EXPECT_EQ(error_or_2.value(), 123); + StatusOr<int> status_or_0(123); + StatusOr<int> status_or_1(std::move(status_or_0)); + StatusOr<int> status_or_2 = std::move(status_or_1); + EXPECT_EQ(std::move(status_or_2).value(), 123); } TEST_F(StatusTest, StatusOrCodeIsOkWithValue) { - StatusOr<int> error_or(123); - EXPECT_EQ(error_or.code(), StatusCode::kOk); + StatusOr<int> status_or(123); + EXPECT_EQ(status_or.code(), StatusCode::kOk); } TEST_F(StatusTest, StatusOrCodeIsNotOkWithoutValue) { - StatusOr<int> error_or(StatusCode::kCodeOnlyForTesting); - EXPECT_EQ(error_or.code(), StatusCode::kCodeOnlyForTesting); + StatusOr<int> status_or(StatusCode::kCodeOnlyForTesting); + EXPECT_EQ(status_or.code(), StatusCode::kCodeOnlyForTesting); } } // namespace media diff --git a/chromium/media/base/stream_parser.cc b/chromium/media/base/stream_parser.cc index ccf06a63557..87041a8e92c 100644 --- a/chromium/media/base/stream_parser.cc +++ b/chromium/media/base/stream_parser.cc @@ -19,6 +19,12 @@ StreamParser::StreamParser() = default; StreamParser::~StreamParser() = default; +// Default implementation of ProcessChunks() is not fully implemented. +bool StreamParser::ProcessChunks(std::unique_ptr<BufferQueue> buffer_queue) { + NOTIMPLEMENTED(); // Likely the wrong type of parser is being used. + return false; +} + static bool MergeBufferQueuesInternal( const std::vector<const StreamParser::BufferQueue*>& buffer_queues, StreamParser::BufferQueue* merged_buffers) { diff --git a/chromium/media/base/stream_parser.h b/chromium/media/base/stream_parser.h index 3eaf9798975..43d788e0182 100644 --- a/chromium/media/base/stream_parser.h +++ b/chromium/media/base/stream_parser.h @@ -140,7 +140,12 @@ class MEDIA_EXPORT StreamParser { // Called when there is new data to parse. // // Returns true if the parse succeeds. + // + // Regular "bytestream-formatted" StreamParsers should fully implement + // Parse(), but WebCodecsEncodedChunkStreamParsers should instead fully + // implement ProcessChunks(). virtual bool Parse(const uint8_t* buf, int size) = 0; + virtual bool ProcessChunks(std::unique_ptr<BufferQueue> buffer_queue); private: DISALLOW_COPY_AND_ASSIGN(StreamParser); diff --git a/chromium/media/base/supported_types.cc b/chromium/media/base/supported_types.cc index 05de5ab4464..a50314fe513 100644 --- a/chromium/media/base/supported_types.cc +++ b/chromium/media/base/supported_types.cc @@ -4,6 +4,7 @@ #include "media/base/supported_types.h" +#include "base/command_line.h" #include "base/feature_list.h" #include "base/logging.h" #include "base/no_destructor.h" @@ -51,6 +52,25 @@ bool IsSupportedHdrMetadata(const gfx::HdrMetadataType& hdr_metadata_type) { return false; } +#if BUILDFLAG(ENABLE_PLATFORM_HEVC) && BUILDFLAG(USE_CHROMEOS_PROTECTED_MEDIA) +bool IsHevcProfileSupported(VideoCodecProfile profile) { + if (!base::CommandLine::ForCurrentProcess()->HasSwitch( + switches::kEnableClearHevcForTesting)) { + return false; + } + switch (profile) { + case HEVCPROFILE_MAIN: // fallthrough + case HEVCPROFILE_MAIN10: + return true; + case HEVCPROFILE_MAIN_STILL_PICTURE: + return false; + default: + NOTREACHED(); + } + return false; +} +#endif // ENABLE_PLATFORM_HEVC && USE_CHROMEOS_PROTECTED_MEDIA + } // namespace bool IsSupportedAudioType(const AudioType& type) { @@ -243,7 +263,7 @@ bool IsDefaultSupportedAudioType(const AudioType& type) { case kCodecAMR_NB: case kCodecAMR_WB: case kCodecGSM_MS: -#if BUILDFLAG(IS_ASH) +#if BUILDFLAG(IS_CHROMEOS_ASH) return true; #else return false; @@ -300,7 +320,8 @@ bool IsDefaultSupportedVideoType(const VideoType& type) { return IsColorSpaceSupported(type.color_space); #else #if defined(OS_ANDROID) - if (base::android::BuildInfo::GetInstance()->is_at_least_q() && + if (base::android::BuildInfo::GetInstance()->sdk_int() >= + base::android::SDK_VERSION_Q && IsColorSpaceSupported(type.color_space)) { return true; } @@ -317,15 +338,21 @@ bool IsDefaultSupportedVideoType(const VideoType& type) { case kCodecTheora: return true; + case kCodecHEVC: +#if BUILDFLAG(ENABLE_PLATFORM_HEVC) && BUILDFLAG(USE_CHROMEOS_PROTECTED_MEDIA) + return IsColorSpaceSupported(type.color_space) && + IsHevcProfileSupported(type.profile); +#else + return false; +#endif case kUnknownVideoCodec: case kCodecVC1: case kCodecMPEG2: - case kCodecHEVC: case kCodecDolbyVision: return false; case kCodecMPEG4: -#if BUILDFLAG(IS_ASH) +#if BUILDFLAG(IS_CHROMEOS_ASH) return true; #else return false; diff --git a/chromium/media/base/supported_types_unittest.cc b/chromium/media/base/supported_types_unittest.cc index f7e04b945c7..eddc5b3ccc1 100644 --- a/chromium/media/base/supported_types_unittest.cc +++ b/chromium/media/base/supported_types_unittest.cc @@ -20,7 +20,7 @@ const bool kPropCodecsEnabled = true; const bool kPropCodecsEnabled = false; #endif -#if BUILDFLAG(IS_ASH) && BUILDFLAG(USE_PROPRIETARY_CODECS) +#if BUILDFLAG(IS_CHROMEOS_ASH) && BUILDFLAG(USE_PROPRIETARY_CODECS) const bool kMpeg4Supported = true; #else const bool kMpeg4Supported = false; @@ -171,8 +171,8 @@ TEST(SupportedTypesTest, IsSupportedVideoType_VP9Profiles) { // VP9 Profile2 are supported on x86, ChromeOS on ARM and Mac/Win on ARM64. // See third_party/libvpx/BUILD.gn. -#if defined(ARCH_CPU_X86_FAMILY) || \ - (defined(ARCH_CPU_ARM_FAMILY) && BUILDFLAG(IS_ASH)) || \ +#if defined(ARCH_CPU_X86_FAMILY) || \ + (defined(ARCH_CPU_ARM_FAMILY) && BUILDFLAG(IS_CHROMEOS_ASH)) || \ (defined(ARCH_CPU_ARM64) && (defined(OS_MAC) || defined(OS_WIN))) EXPECT_TRUE(IsSupportedVideoType( {kCodecVP9, VP9PROFILE_PROFILE2, kUnspecifiedLevel, kColorSpace})); diff --git a/chromium/media/base/supported_video_decoder_config.cc b/chromium/media/base/supported_video_decoder_config.cc new file mode 100644 index 00000000000..b3966073ee6 --- /dev/null +++ b/chromium/media/base/supported_video_decoder_config.cc @@ -0,0 +1,64 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/base/supported_video_decoder_config.h" + +namespace media { + +SupportedVideoDecoderConfig::SupportedVideoDecoderConfig() = default; + +SupportedVideoDecoderConfig::SupportedVideoDecoderConfig( + VideoCodecProfile profile_min, + VideoCodecProfile profile_max, + const gfx::Size& coded_size_min, + const gfx::Size& coded_size_max, + bool allow_encrypted, + bool require_encrypted) + : profile_min(profile_min), + profile_max(profile_max), + coded_size_min(coded_size_min), + coded_size_max(coded_size_max), + allow_encrypted(allow_encrypted), + require_encrypted(require_encrypted) {} + +SupportedVideoDecoderConfig::~SupportedVideoDecoderConfig() = default; + +bool SupportedVideoDecoderConfig::Matches( + const VideoDecoderConfig& config) const { + if (config.profile() < profile_min || config.profile() > profile_max) + return false; + + if (config.is_encrypted()) { + if (!allow_encrypted) + return false; + } else { + if (require_encrypted) + return false; + } + + if (config.coded_size().width() < coded_size_min.width()) + return false; + if (config.coded_size().height() < coded_size_min.height()) + return false; + + if (config.coded_size().width() > coded_size_max.width()) + return false; + if (config.coded_size().height() > coded_size_max.height()) + return false; + + return true; +} + +// static +bool IsVideoDecoderConfigSupported( + const SupportedVideoDecoderConfigs& supported_configs, + const VideoDecoderConfig& config) { + for (const auto& c : supported_configs) { + if (c.Matches(config)) + return true; + } + return false; +} + +} // namespace media diff --git a/chromium/media/base/supported_video_decoder_config.h b/chromium/media/base/supported_video_decoder_config.h new file mode 100644 index 00000000000..b0fcd1ce850 --- /dev/null +++ b/chromium/media/base/supported_video_decoder_config.h @@ -0,0 +1,82 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_BASE_SUPPORTED_VIDEO_DECODER_CONFIG_H_ +#define MEDIA_BASE_SUPPORTED_VIDEO_DECODER_CONFIG_H_ + +#include <vector> + +#include "base/containers/flat_map.h" +#include "base/macros.h" +#include "media/base/media_export.h" +#include "media/base/video_codecs.h" +#include "media/base/video_decoder_config.h" +#include "ui/gfx/geometry/size.h" + +namespace media { + +// The min and max resolution used by SW decoders (dav1d, libgav1, libvpx and +// ffmpeg for example) when queried about decoding capabilities. For now match +// the supported resolutions of HW decoders. +constexpr gfx::Size kDefaultSwDecodeSizeMin(8, 8); +constexpr gfx::Size kDefaultSwDecodeSizeMax(8192, 8192); + +// Specification of a range of configurations that are supported by a video +// decoder. Also provides the ability to check if a VideoDecoderConfig matches +// the supported range. +struct MEDIA_EXPORT SupportedVideoDecoderConfig { + SupportedVideoDecoderConfig(); + SupportedVideoDecoderConfig(VideoCodecProfile profile_min, + VideoCodecProfile profile_max, + const gfx::Size& coded_size_min, + const gfx::Size& coded_size_max, + bool allow_encrypted, + bool require_encrypted); + ~SupportedVideoDecoderConfig(); + + // Returns true if and only if |config| is a supported config. + bool Matches(const VideoDecoderConfig& config) const; + + // Range of VideoCodecProfiles to match, inclusive. + VideoCodecProfile profile_min = VIDEO_CODEC_PROFILE_UNKNOWN; + VideoCodecProfile profile_max = VIDEO_CODEC_PROFILE_UNKNOWN; + + // Coded size range, inclusive. + gfx::Size coded_size_min; + gfx::Size coded_size_max; + + // TODO(liberato): consider switching these to "allow_clear" and + // "allow_encrypted", so that they're orthogonal. + + // If true, then this will match encrypted configs. + bool allow_encrypted = true; + + // If true, then unencrypted configs will not match. + bool require_encrypted = false; + + // Allow copy and assignment. +}; + +// Enumeration of possible implementations for (Mojo)VideoDecoders. +enum class VideoDecoderImplementation { + kDefault = 0, + kAlternate = 1, + kMaxValue = kAlternate +}; + +using SupportedVideoDecoderConfigs = std::vector<SupportedVideoDecoderConfig>; + +// Map of mojo VideoDecoder implementations to the vector of configs that they +// (probably) support. +using SupportedVideoDecoderConfigMap = + base::flat_map<VideoDecoderImplementation, SupportedVideoDecoderConfigs>; + +// Helper method to determine if |config| is supported by |supported_configs|. +MEDIA_EXPORT bool IsVideoDecoderConfigSupported( + const SupportedVideoDecoderConfigs& supported_configs, + const VideoDecoderConfig& config); + +} // namespace media + +#endif // MEDIA_BASE_SUPPORTED_VIDEO_DECODER_CONFIG_H_ diff --git a/chromium/media/base/supported_video_decoder_config_unittest.cc b/chromium/media/base/supported_video_decoder_config_unittest.cc new file mode 100644 index 00000000000..dd3dabdbe3d --- /dev/null +++ b/chromium/media/base/supported_video_decoder_config_unittest.cc @@ -0,0 +1,104 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/base/supported_video_decoder_config.h" +#include "media/base/test_helpers.h" +#include "media/base/video_codecs.h" +#include "testing/gtest/include/gtest/gtest.h" + +namespace media { + +class SupportedVideoDecoderConfigTest : public ::testing::Test { + public: + SupportedVideoDecoderConfigTest() + : decoder_config_( + TestVideoConfig::NormalCodecProfile(kCodecH264, + H264PROFILE_EXTENDED)) { + supported_config_.profile_min = H264PROFILE_MIN; + supported_config_.profile_max = H264PROFILE_MAX; + supported_config_.coded_size_min = gfx::Size(10, 20); + supported_config_.coded_size_max = gfx::Size(10000, 20000); + supported_config_.allow_encrypted = true; + supported_config_.require_encrypted = false; + } + + SupportedVideoDecoderConfig supported_config_; + + // Decoder config that matches |supported_config_|. + VideoDecoderConfig decoder_config_; +}; + +TEST_F(SupportedVideoDecoderConfigTest, ConstructionWithArgs) { + SupportedVideoDecoderConfig config2( + supported_config_.profile_min, supported_config_.profile_max, + supported_config_.coded_size_min, supported_config_.coded_size_max, + supported_config_.allow_encrypted, supported_config_.require_encrypted); + EXPECT_EQ(supported_config_.profile_min, config2.profile_min); + EXPECT_EQ(supported_config_.profile_max, config2.profile_max); + EXPECT_EQ(supported_config_.coded_size_min, config2.coded_size_min); + EXPECT_EQ(supported_config_.coded_size_max, config2.coded_size_max); + EXPECT_EQ(supported_config_.allow_encrypted, config2.allow_encrypted); + EXPECT_EQ(supported_config_.require_encrypted, config2.require_encrypted); +} + +TEST_F(SupportedVideoDecoderConfigTest, MatchingConfigMatches) { + EXPECT_TRUE(supported_config_.Matches(decoder_config_)); + + // Since |supported_config_| allows encrypted, this should also succeed. + decoder_config_.SetIsEncrypted(true); + EXPECT_TRUE(supported_config_.Matches(decoder_config_)); +} + +TEST_F(SupportedVideoDecoderConfigTest, LowerProfileMismatches) { + // Raise |profile_min| above |decoder_config_|. + supported_config_.profile_min = H264PROFILE_HIGH; + EXPECT_FALSE(supported_config_.Matches(decoder_config_)); +} + +TEST_F(SupportedVideoDecoderConfigTest, HigherProfileMismatches) { + // Lower |profile_max| below |decoder_config_|. + supported_config_.profile_max = H264PROFILE_MAIN; + EXPECT_FALSE(supported_config_.Matches(decoder_config_)); +} + +TEST_F(SupportedVideoDecoderConfigTest, SmallerMinWidthMismatches) { + supported_config_.coded_size_min = + gfx::Size(decoder_config_.coded_size().width() + 1, 0); + EXPECT_FALSE(supported_config_.Matches(decoder_config_)); +} + +TEST_F(SupportedVideoDecoderConfigTest, SmallerMinHeightMismatches) { + supported_config_.coded_size_min = + gfx::Size(0, decoder_config_.coded_size().height() + 1); + EXPECT_FALSE(supported_config_.Matches(decoder_config_)); +} + +TEST_F(SupportedVideoDecoderConfigTest, LargerMaxWidthMismatches) { + supported_config_.coded_size_max = + gfx::Size(decoder_config_.coded_size().width() - 1, 10000); + EXPECT_FALSE(supported_config_.Matches(decoder_config_)); +} + +TEST_F(SupportedVideoDecoderConfigTest, LargerMaxHeightMismatches) { + supported_config_.coded_size_max = + gfx::Size(10000, decoder_config_.coded_size().height() - 1); + EXPECT_FALSE(supported_config_.Matches(decoder_config_)); +} + +TEST_F(SupportedVideoDecoderConfigTest, RequiredEncryptionMismatches) { + supported_config_.require_encrypted = true; + EXPECT_FALSE(supported_config_.Matches(decoder_config_)); + + // The encrypted version should succeed. + decoder_config_.SetIsEncrypted(true); + EXPECT_TRUE(supported_config_.Matches(decoder_config_)); +} + +TEST_F(SupportedVideoDecoderConfigTest, AllowedEncryptionMismatches) { + supported_config_.allow_encrypted = false; + decoder_config_.SetIsEncrypted(true); + EXPECT_FALSE(supported_config_.Matches(decoder_config_)); +} + +} // namespace media diff --git a/chromium/media/base/user_input_monitor_unittest.cc b/chromium/media/base/user_input_monitor_unittest.cc index 5e85052bad6..74238a2beb4 100644 --- a/chromium/media/base/user_input_monitor_unittest.cc +++ b/chromium/media/base/user_input_monitor_unittest.cc @@ -19,19 +19,39 @@ #if defined(USE_OZONE) #include "ui/base/ui_base_features.h" // nogncheck +#include "ui/ozone/public/ozone_platform.h" // nogncheck +#endif + +#if defined(OS_WIN) +#include "ui/events/test/keyboard_hook_monitor_utils.h" #endif namespace media { -TEST(UserInputMonitorTest, CreatePlatformSpecific) { +namespace { + +class UserInputMonitorTest : public testing::Test { + protected: + // testing::Test. + void SetUp() override { #if defined(USE_OZONE) - // TODO(crbug.com/1109112): enable those tests for Ozone. - // Here, the only issue why they don't work is that the Ozone platform is not - // initialised. - if (features::IsUsingOzonePlatform()) - return; + if (features::IsUsingOzonePlatform()) { + if (ui::OzonePlatform::GetPlatformNameForTest() == "drm") { + // OzonePlatformDrm::InitializeUI hangs in tests on the DRM platform. + GTEST_SKIP(); + } + // Initialise Ozone in single process mode, as all tests do. + ui::OzonePlatform::InitParams params; + params.single_process = true; + ui::OzonePlatform::InitializeForUI(params); + } #endif + } +}; + +} // namespace +TEST_F(UserInputMonitorTest, CreatePlatformSpecific) { #if defined(OS_LINUX) || defined(OS_CHROMEOS) base::test::TaskEnvironment task_environment( base::test::TaskEnvironment::MainThreadType::IO); @@ -53,15 +73,7 @@ TEST(UserInputMonitorTest, CreatePlatformSpecific) { base::RunLoop().RunUntilIdle(); } -TEST(UserInputMonitorTest, CreatePlatformSpecificWithMapping) { -#if defined(USE_OZONE) - // TODO(crbug.com/1109112): enable those tests for Ozone. - // Here, the only issue why they don't work is that the Ozone platform is not - // initialised. - if (features::IsUsingOzonePlatform()) - return; -#endif - +TEST_F(UserInputMonitorTest, CreatePlatformSpecificWithMapping) { #if defined(OS_LINUX) || defined(OS_CHROMEOS) base::test::TaskEnvironment task_environment( base::test::TaskEnvironment::MainThreadType::IO); @@ -90,7 +102,7 @@ TEST(UserInputMonitorTest, CreatePlatformSpecificWithMapping) { EXPECT_EQ(0u, ReadKeyPressMonitorCount(readonly_mapping)); } -TEST(UserInputMonitorTest, ReadWriteKeyPressMonitorCount) { +TEST_F(UserInputMonitorTest, ReadWriteKeyPressMonitorCount) { std::unique_ptr<base::MappedReadOnlyRegion> shmem = std::make_unique<base::MappedReadOnlyRegion>( base::ReadOnlySharedMemoryRegion::Create(sizeof(uint32_t))); @@ -102,4 +114,96 @@ TEST(UserInputMonitorTest, ReadWriteKeyPressMonitorCount) { EXPECT_EQ(count, ReadKeyPressMonitorCount(readonly_mapping)); } +#if defined(OS_WIN) + +// +// Windows specific scenarios which require simulating keyboard hook events. +// + +TEST_F(UserInputMonitorTest, BlockMonitoringAfterMonitoringEnabled) { + base::test::TaskEnvironment task_environment( + base::test::TaskEnvironment::MainThreadType::UI); + + std::unique_ptr<UserInputMonitor> monitor = UserInputMonitor::Create( + base::ThreadTaskRunnerHandle::Get(), base::ThreadTaskRunnerHandle::Get()); + + if (!monitor) + return; + + monitor->EnableKeyPressMonitoring(); + ui::SimulateKeyboardHookRegistered(); + ui::SimulateKeyboardHookUnregistered(); + monitor->DisableKeyPressMonitoring(); + + monitor.reset(); + base::RunLoop().RunUntilIdle(); +} + +TEST_F(UserInputMonitorTest, BlockMonitoringBeforeMonitoringEnabled) { + base::test::TaskEnvironment task_environment( + base::test::TaskEnvironment::MainThreadType::UI); + + std::unique_ptr<UserInputMonitor> monitor = UserInputMonitor::Create( + base::ThreadTaskRunnerHandle::Get(), base::ThreadTaskRunnerHandle::Get()); + + if (!monitor) + return; + + ui::SimulateKeyboardHookRegistered(); + monitor->EnableKeyPressMonitoring(); + ui::SimulateKeyboardHookUnregistered(); + monitor->DisableKeyPressMonitoring(); + + monitor.reset(); + base::RunLoop().RunUntilIdle(); +} + +TEST_F(UserInputMonitorTest, UnblockMonitoringAfterMonitoringDisabled) { + base::test::TaskEnvironment task_environment( + base::test::TaskEnvironment::MainThreadType::UI); + + std::unique_ptr<UserInputMonitor> monitor = UserInputMonitor::Create( + base::ThreadTaskRunnerHandle::Get(), base::ThreadTaskRunnerHandle::Get()); + + if (!monitor) + return; + + monitor->EnableKeyPressMonitoring(); + ui::SimulateKeyboardHookRegistered(); + monitor->DisableKeyPressMonitoring(); + ui::SimulateKeyboardHookUnregistered(); + + monitor.reset(); + base::RunLoop().RunUntilIdle(); +} + +TEST_F(UserInputMonitorTest, BlockKeypressMonitoringWithSharedMemoryBuffer) { + base::test::TaskEnvironment task_environment( + base::test::TaskEnvironment::MainThreadType::UI); + + std::unique_ptr<UserInputMonitor> monitor = UserInputMonitor::Create( + base::ThreadTaskRunnerHandle::Get(), base::ThreadTaskRunnerHandle::Get()); + + if (!monitor) + return; + + base::ReadOnlySharedMemoryMapping readonly_mapping = + static_cast<UserInputMonitorBase*>(monitor.get()) + ->EnableKeyPressMonitoringWithMapping() + .Map(); + EXPECT_EQ(0u, ReadKeyPressMonitorCount(readonly_mapping)); + ui::SimulateKeyboardHookRegistered(); + EXPECT_EQ(0u, ReadKeyPressMonitorCount(readonly_mapping)); + ui::SimulateKeyboardHookUnregistered(); + EXPECT_EQ(0u, ReadKeyPressMonitorCount(readonly_mapping)); + monitor->DisableKeyPressMonitoring(); + + monitor.reset(); + base::RunLoop().RunUntilIdle(); + + // Check that read only region remains valid after disable. + EXPECT_EQ(0u, ReadKeyPressMonitorCount(readonly_mapping)); +} +#endif // defined(OS_WIN) + } // namespace media diff --git a/chromium/media/base/user_input_monitor_win.cc b/chromium/media/base/user_input_monitor_win.cc index 8187d23f282..6082070ace0 100644 --- a/chromium/media/base/user_input_monitor_win.cc +++ b/chromium/media/base/user_input_monitor_win.cc @@ -20,6 +20,8 @@ #include "third_party/skia/include/core/SkPoint.h" #include "ui/events/keyboard_event_counter.h" #include "ui/events/keycodes/keyboard_code_conversion_win.h" +#include "ui/events/win/keyboard_hook_monitor.h" +#include "ui/events/win/keyboard_hook_observer.h" namespace media { namespace { @@ -41,7 +43,8 @@ std::unique_ptr<RAWINPUTDEVICE> GetRawInputDevices(HWND hwnd, DWORD flags) { // UserInputMonitorWin since it needs to be deleted on the UI thread. class UserInputMonitorWinCore : public base::SupportsWeakPtr<UserInputMonitorWinCore>, - public base::CurrentThread::DestructionObserver { + public base::CurrentThread::DestructionObserver, + public ui::KeyboardHookObserver { public: enum EventBitMask { MOUSE_EVENT_MASK = 1, @@ -55,6 +58,10 @@ class UserInputMonitorWinCore // DestructionObserver overrides. void WillDestroyCurrentMessageLoop() override; + // KeyboardHookObserver implementation. + void OnHookRegistered() override; + void OnHookUnregistered() override; + uint32_t GetKeyPressCount() const; void StartMonitor(); void StartMonitorWithMapping(base::WritableSharedMemoryMapping mapping); @@ -69,6 +76,9 @@ class UserInputMonitorWinCore LPARAM lparam, LRESULT* result); + void CreateRawInputWindow(); + void DestroyRawInputWindow(); + // Task runner on which |window_| is created. scoped_refptr<base::SingleThreadTaskRunner> ui_task_runner_; @@ -79,6 +89,9 @@ class UserInputMonitorWinCore std::unique_ptr<base::win::MessageWindow> window_; ui::KeyboardEventCounter counter_; + bool pause_monitoring_ = false; + bool start_monitoring_after_hook_removed_ = false; + DISALLOW_COPY_AND_ASSIGN(UserInputMonitorWinCore); }; @@ -106,10 +119,17 @@ class UserInputMonitorWin : public UserInputMonitorBase { UserInputMonitorWinCore::UserInputMonitorWinCore( scoped_refptr<base::SingleThreadTaskRunner> ui_task_runner) - : ui_task_runner_(ui_task_runner) {} + : ui_task_runner_(ui_task_runner) { + // Register this instance with the KeyboardHookMonitor to listen for changes + // in the KeyboardHook registration state. Since this instance may have been + // constructed after a hook was registered, check the current state as well. + ui::KeyboardHookMonitor::GetInstance()->AddObserver(this); + pause_monitoring_ = ui::KeyboardHookMonitor::GetInstance()->IsActive(); +} UserInputMonitorWinCore::~UserInputMonitorWinCore() { DCHECK(!window_); + ui::KeyboardHookMonitor::GetInstance()->RemoveObserver(this); } void UserInputMonitorWinCore::WillDestroyCurrentMessageLoop() { @@ -124,6 +144,54 @@ uint32_t UserInputMonitorWinCore::GetKeyPressCount() const { void UserInputMonitorWinCore::StartMonitor() { DCHECK(ui_task_runner_->BelongsToCurrentThread()); + if (pause_monitoring_) { + start_monitoring_after_hook_removed_ = true; + return; + } + + CreateRawInputWindow(); +} + +void UserInputMonitorWinCore::StartMonitorWithMapping( + base::WritableSharedMemoryMapping mapping) { + StartMonitor(); + key_press_count_mapping_ = + std::make_unique<base::WritableSharedMemoryMapping>(std::move(mapping)); +} + +void UserInputMonitorWinCore::StopMonitor() { + DCHECK(ui_task_runner_->BelongsToCurrentThread()); + + DestroyRawInputWindow(); + start_monitoring_after_hook_removed_ = false; + + key_press_count_mapping_.reset(); +} + +void UserInputMonitorWinCore::OnHookRegistered() { + DCHECK(ui_task_runner_->BelongsToCurrentThread()); + DCHECK(!pause_monitoring_); + pause_monitoring_ = true; + + // Don't destroy |key_press_count_mapping_| as this is a temporary block and + // we want to allow monitoring to continue using the same shared memory once + // monitoring is unblocked. + DestroyRawInputWindow(); +} + +void UserInputMonitorWinCore::OnHookUnregistered() { + DCHECK(ui_task_runner_->BelongsToCurrentThread()); + DCHECK(pause_monitoring_); + pause_monitoring_ = false; + + if (start_monitoring_after_hook_removed_) { + start_monitoring_after_hook_removed_ = false; + StartMonitor(); + } +} + +void UserInputMonitorWinCore::CreateRawInputWindow() { + DCHECK(ui_task_runner_->BelongsToCurrentThread()); if (window_) return; @@ -149,16 +217,8 @@ void UserInputMonitorWinCore::StartMonitor() { base::CurrentThread::Get()->AddDestructionObserver(this); } -void UserInputMonitorWinCore::StartMonitorWithMapping( - base::WritableSharedMemoryMapping mapping) { - StartMonitor(); - key_press_count_mapping_ = - std::make_unique<base::WritableSharedMemoryMapping>(std::move(mapping)); -} - -void UserInputMonitorWinCore::StopMonitor() { +void UserInputMonitorWinCore::DestroyRawInputWindow() { DCHECK(ui_task_runner_->BelongsToCurrentThread()); - if (!window_) return; @@ -168,11 +228,8 @@ void UserInputMonitorWinCore::StopMonitor() { if (!RegisterRawInputDevices(device.get(), 1, sizeof(*device))) { PLOG(INFO) << "RegisterRawInputDevices() failed for RIDEV_REMOVE"; } - window_ = nullptr; - key_press_count_mapping_.reset(); - // Stop observing message loop destruction if no event is being monitored. base::CurrentThread::Get()->RemoveDestructionObserver(this); } diff --git a/chromium/media/base/video_decoder.cc b/chromium/media/base/video_decoder.cc index 80ac2f5ab0e..a593bb3e6a1 100644 --- a/chromium/media/base/video_decoder.cc +++ b/chromium/media/base/video_decoder.cc @@ -29,6 +29,10 @@ int VideoDecoder::GetMaxDecodeRequests() const { return 1; } +bool VideoDecoder::IsOptimizedForRTC() const { + return false; +} + // static int VideoDecoder::GetRecommendedThreadCount(int desired_threads) { // If the thread count is specified on the command line, respect it so long as diff --git a/chromium/media/base/video_decoder.h b/chromium/media/base/video_decoder.h index b66186c625e..8c6cb290dc2 100644 --- a/chromium/media/base/video_decoder.h +++ b/chromium/media/base/video_decoder.h @@ -115,6 +115,10 @@ class MEDIA_EXPORT VideoDecoder : public Decoder { // Returns maximum number of parallel decode requests. virtual int GetMaxDecodeRequests() const; + // Returns true if and only if this decoder is optimized for decoding RTC + // streams. The default is false. + virtual bool IsOptimizedForRTC() const; + // Returns the recommended number of threads for software video decoding. If // the --video-threads command line option is specified and is valid, that // value is returned. Otherwise |desired_threads| is clamped to the number of @@ -122,6 +126,12 @@ class MEDIA_EXPORT VideoDecoder : public Decoder { // [|limits::kMinVideoDecodeThreads|, |limits::kMaxVideoDecodeThreads|]. static int GetRecommendedThreadCount(int desired_threads); + // Returns the type of the decoder for statistics recording purposes. + // For meta-decoders (those which wrap other decoders, ie, MojoVideoDecoder) + // this should return the underlying type, if it is known, otherwise return + // its own type. + virtual VideoDecoderType GetDecoderType() const = 0; + private: DISALLOW_COPY_AND_ASSIGN(VideoDecoder); }; diff --git a/chromium/media/base/video_decoder_config.cc b/chromium/media/base/video_decoder_config.cc index 85cc6a64396..79af2bfd372 100644 --- a/chromium/media/base/video_decoder_config.cc +++ b/chromium/media/base/video_decoder_config.cc @@ -117,7 +117,11 @@ std::string VideoDecoderConfig::AsHumanReadableString() const { << hdr_metadata()->mastering_metadata.primary_b.x() << "," << hdr_metadata()->mastering_metadata.primary_b.y() << ") wp(" << hdr_metadata()->mastering_metadata.white_point.x() << "," - << hdr_metadata()->mastering_metadata.white_point.y() << ")"; + << hdr_metadata()->mastering_metadata.white_point.y() + << "), max_content_light_level=" + << hdr_metadata()->max_content_light_level + << ", max_frame_average_light_level=" + << hdr_metadata()->max_frame_average_light_level; } return s.str(); diff --git a/chromium/media/base/video_decoder_config.h b/chromium/media/base/video_decoder_config.h index 5d04d524bad..144f7d61a15 100644 --- a/chromium/media/base/video_decoder_config.h +++ b/chromium/media/base/video_decoder_config.h @@ -167,6 +167,10 @@ class MEDIA_EXPORT VideoDecoderConfig { // useful for decryptors that decrypts an encrypted stream to a clear stream. void SetIsEncrypted(bool is_encrypted); + // Sets whether this config is for WebRTC or not. + void set_is_rtc(bool is_rtc) { is_rtc_ = is_rtc; } + bool is_rtc() const { return is_rtc_; } + private: VideoCodec codec_ = kUnknownVideoCodec; VideoCodecProfile profile_ = VIDEO_CODEC_PROFILE_UNKNOWN; @@ -191,6 +195,7 @@ class MEDIA_EXPORT VideoDecoderConfig { VideoColorSpace color_space_info_; base::Optional<gfx::HDRMetadata> hdr_metadata_; + bool is_rtc_ = false; // Not using DISALLOW_COPY_AND_ASSIGN here intentionally to allow the compiler // generated copy constructor and assignment operator. Since the extra data is diff --git a/chromium/media/base/video_encoder.h b/chromium/media/base/video_encoder.h index 1513367c15c..d1a590a4086 100644 --- a/chromium/media/base/video_encoder.h +++ b/chromium/media/base/video_encoder.h @@ -34,6 +34,11 @@ struct MEDIA_EXPORT VideoEncoderOutput { class MEDIA_EXPORT VideoEncoder { public: + // TODO: Move this to a new file if there are more codec specific options. + struct MEDIA_EXPORT AvcOptions { + bool produce_annexb = false; + }; + struct MEDIA_EXPORT Options { Options(); Options(const Options&); @@ -44,6 +49,9 @@ class MEDIA_EXPORT VideoEncoder { gfx::Size frame_size; base::Optional<int> keyframe_interval = 10000; + + // Only used for H264 encoding. + AvcOptions avc; }; // A sequence of codec specific bytes, commonly known as extradata. diff --git a/chromium/media/base/video_frame.cc b/chromium/media/base/video_frame.cc index f3833229a48..914a4a3a267 100644 --- a/chromium/media/base/video_frame.cc +++ b/chromium/media/base/video_frame.cc @@ -146,6 +146,7 @@ gfx::Size VideoFrame::SampleSize(VideoPixelFormat format, size_t plane) { case PIXEL_FORMAT_XR30: case PIXEL_FORMAT_XB30: case PIXEL_FORMAT_BGRA: + case PIXEL_FORMAT_RGBAF16: break; } } @@ -178,6 +179,7 @@ static bool RequiresEvenSizeAllocation(VideoPixelFormat format) { case PIXEL_FORMAT_XR30: case PIXEL_FORMAT_XB30: case PIXEL_FORMAT_BGRA: + case PIXEL_FORMAT_RGBAF16: return false; case PIXEL_FORMAT_NV12: case PIXEL_FORMAT_NV21: @@ -233,6 +235,9 @@ static base::Optional<VideoFrameLayout> GetDefaultLayout( break; case PIXEL_FORMAT_ARGB: + case PIXEL_FORMAT_XRGB: + case PIXEL_FORMAT_ABGR: + case PIXEL_FORMAT_XBGR: planes = std::vector<ColorPlaneLayout>{ColorPlaneLayout( coded_size.width() * 4, 0, coded_size.GetArea() * 4)}; break; @@ -252,10 +257,8 @@ static base::Optional<VideoFrameLayout> GetDefaultLayout( default: // TODO(miu): This function should support any pixel format. // http://crbug.com/555909 . - DLOG(ERROR) - << "Only PIXEL_FORMAT_I420, PIXEL_FORMAT_Y16, PIXEL_FORMAT_NV12, " - "and PIXEL_FORMAT_ARGB formats are supported: " - << VideoPixelFormatToString(format); + DLOG(ERROR) << "Unsupported pixel format" + << VideoPixelFormatToString(format); return base::nullopt; } @@ -309,7 +312,7 @@ bool VideoFrame::IsValidConfig(VideoPixelFormat format, return true; // Make sure new formats are properly accounted for in the method. - static_assert(PIXEL_FORMAT_MAX == 32, + static_assert(PIXEL_FORMAT_MAX == 33, "Added pixel format, please review IsValidConfig()"); if (format == PIXEL_FORMAT_UNKNOWN) { @@ -341,7 +344,7 @@ scoped_refptr<VideoFrame> VideoFrame::CreateVideoHoleFrame( scoped_refptr<VideoFrame> frame = new VideoFrame(*layout, StorageType::STORAGE_OPAQUE, gfx::Rect(natural_size), natural_size, timestamp); - frame->metadata()->overlay_plane_id = overlay_plane_id; + frame->metadata().overlay_plane_id = overlay_plane_id; return frame; } @@ -368,7 +371,8 @@ scoped_refptr<VideoFrame> VideoFrame::WrapNativeTextures( if (format != PIXEL_FORMAT_ARGB && format != PIXEL_FORMAT_XRGB && format != PIXEL_FORMAT_NV12 && format != PIXEL_FORMAT_I420 && format != PIXEL_FORMAT_ABGR && format != PIXEL_FORMAT_XR30 && - format != PIXEL_FORMAT_XB30 && format != PIXEL_FORMAT_P016LE) { + format != PIXEL_FORMAT_XB30 && format != PIXEL_FORMAT_P016LE && + format != PIXEL_FORMAT_RGBAF16) { DLOG(ERROR) << "Unsupported pixel format: " << VideoPixelFormatToString(format); return nullptr; @@ -434,6 +438,18 @@ scoped_refptr<VideoFrame> VideoFrame::WrapExternalDataWithLayout( return nullptr; } + const auto& last_plane = layout.planes()[layout.planes().size() - 1]; + const size_t required_size = last_plane.offset + last_plane.size; + if (data_size < required_size) { + DLOG(ERROR) << __func__ << " Provided data size is too small. Provided " + << data_size << " bytes, but " << required_size + << " bytes are required." + << ConfigToString(layout.format(), storage_type, + layout.coded_size(), visible_rect, + natural_size); + return nullptr; + } + scoped_refptr<VideoFrame> frame = new VideoFrame( layout, storage_type, visible_rect, natural_size, timestamp); @@ -845,15 +861,37 @@ scoped_refptr<VideoFrame> VideoFrame::WrapVideoFrame( return nullptr; } + size_t new_plane_count = NumPlanes(format); + base::Optional<VideoFrameLayout> new_layout; + if (format == frame->format()) { + new_layout = frame->layout(); + } else { + std::vector<ColorPlaneLayout> new_planes = frame->layout().planes(); + if (new_plane_count > new_planes.size()) { + DLOG(ERROR) << " Wrapping frame has more planes than old one." + << " old plane count: " << new_planes.size() + << " new plane count: " << new_plane_count; + return nullptr; + } + new_planes.resize(new_plane_count); + new_layout = VideoFrameLayout::CreateWithPlanes(format, frame->coded_size(), + new_planes); + } + + if (!new_layout.has_value()) { + DLOG(ERROR) << " Can't create layout for the wrapping frame"; + return nullptr; + } + scoped_refptr<VideoFrame> wrapping_frame( - new VideoFrame(frame->layout(), frame->storage_type(), visible_rect, + new VideoFrame(new_layout.value(), frame->storage_type(), visible_rect, natural_size, frame->timestamp())); // Copy all metadata to the wrapped frame-> - wrapping_frame->metadata()->MergeMetadataFrom(frame->metadata()); + wrapping_frame->metadata().MergeMetadataFrom(frame->metadata()); if (frame->IsMappable()) { - for (size_t i = 0; i < NumPlanes(format); ++i) { + for (size_t i = 0; i < new_plane_count; ++i) { wrapping_frame->data_[i] = frame->data_[i]; } } @@ -882,7 +920,7 @@ scoped_refptr<VideoFrame> VideoFrame::CreateEOSFrame() { } scoped_refptr<VideoFrame> frame = new VideoFrame( *layout, STORAGE_UNKNOWN, gfx::Rect(), gfx::Size(), kNoTimestamp); - frame->metadata()->end_of_stream = true; + frame->metadata().end_of_stream = true; return frame; } @@ -940,6 +978,15 @@ size_t VideoFrame::AllocationSize(VideoPixelFormat format, gfx::Size VideoFrame::PlaneSize(VideoPixelFormat format, size_t plane, const gfx::Size& coded_size) { + gfx::Size size = PlaneSizeInSamples(format, plane, coded_size); + size.set_width(size.width() * BytesPerElement(format, plane)); + return size; +} + +// static +gfx::Size VideoFrame::PlaneSizeInSamples(VideoPixelFormat format, + size_t plane, + const gfx::Size& coded_size) { DCHECK(IsValidPlane(format, plane)); int width = coded_size.width(); @@ -955,8 +1002,7 @@ gfx::Size VideoFrame::PlaneSize(VideoPixelFormat format, const gfx::Size subsample = SampleSize(format, plane); DCHECK(width % subsample.width() == 0); DCHECK(height % subsample.height() == 0); - return gfx::Size(BytesPerElement(format, plane) * width / subsample.width(), - height / subsample.height()); + return gfx::Size(width / subsample.width(), height / subsample.height()); } // static @@ -973,7 +1019,7 @@ int VideoFrame::PlaneHorizontalBitsPerPixel(VideoPixelFormat format, int VideoFrame::PlaneBitsPerPixel(VideoPixelFormat format, size_t plane) { DCHECK(IsValidPlane(format, plane)); return PlaneHorizontalBitsPerPixel(format, plane) / - SampleSize(format, plane).height(); + SampleSize(format, plane).height(); } // static @@ -986,6 +1032,8 @@ size_t VideoFrame::RowBytes(size_t plane, VideoPixelFormat format, int width) { int VideoFrame::BytesPerElement(VideoPixelFormat format, size_t plane) { DCHECK(IsValidPlane(format, plane)); switch (format) { + case PIXEL_FORMAT_RGBAF16: + return 8; case PIXEL_FORMAT_ARGB: case PIXEL_FORMAT_BGRA: case PIXEL_FORMAT_XRGB: @@ -1190,8 +1238,8 @@ uint8_t* VideoFrame::visible_data(size_t plane) { static_cast<const VideoFrame*>(this)->visible_data(plane)); } -const gpu::MailboxHolder& -VideoFrame::mailbox_holder(size_t texture_index) const { +const gpu::MailboxHolder& VideoFrame::mailbox_holder( + size_t texture_index) const { DCHECK(HasTextures()); DCHECK(IsValidPlane(format(), texture_index)); return wrapped_frame_ ? wrapped_frame_->mailbox_holders_[texture_index] @@ -1258,13 +1306,15 @@ gpu::SyncToken VideoFrame::UpdateReleaseSyncToken(SyncTokenClient* client) { } std::string VideoFrame::AsHumanReadableString() const { - if (metadata()->end_of_stream) + if (metadata().end_of_stream) return "end of stream"; std::ostringstream s; s << ConfigToString(format(), storage_type_, coded_size(), visible_rect_, natural_size_) << " timestamp:" << timestamp_.InMicroseconds(); + if (HasTextures()) + s << " textures: " << NumTextures(); return s.str(); } diff --git a/chromium/media/base/video_frame.h b/chromium/media/base/video_frame.h index 4185e3ce506..5bf0cacebf1 100644 --- a/chromium/media/base/video_frame.h +++ b/chromium/media/base/video_frame.h @@ -346,6 +346,12 @@ class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> { size_t plane, const gfx::Size& coded_size); + // Returns the plane gfx::Size (in samples) for a plane of the given coded + // size and format. + static gfx::Size PlaneSizeInSamples(VideoPixelFormat format, + size_t plane, + const gfx::Size& coded_size); + // Returns horizontal bits per pixel for given |plane| and |format|. static int PlaneHorizontalBitsPerPixel(VideoPixelFormat format, size_t plane); @@ -559,10 +565,8 @@ class MEDIA_EXPORT VideoFrame : public base::RefCountedThreadSafe<VideoFrame> { // // TODO(miu): Move some of the "extra" members of VideoFrame (below) into // here as a later clean-up step. - // - // TODO(https://crbug.com/1096727): change the return type to const&. - const VideoFrameMetadata* metadata() const { return &metadata_; } - VideoFrameMetadata* metadata() { return &metadata_; } + const VideoFrameMetadata& metadata() const { return metadata_; } + VideoFrameMetadata& metadata() { return metadata_; } void set_metadata(const VideoFrameMetadata& metadata) { metadata_ = metadata; } diff --git a/chromium/media/base/video_frame_feedback.h b/chromium/media/base/video_frame_feedback.h index 3e64b3fded9..348412b1349 100644 --- a/chromium/media/base/video_frame_feedback.h +++ b/chromium/media/base/video_frame_feedback.h @@ -35,6 +35,10 @@ struct MEDIA_EXPORT VideoFrameFeedback { max_framerate_fps == other.max_framerate_fps; } + bool operator!=(const VideoFrameFeedback& other) const { + return !(*this == other); + } + // Combine constraints of two different sinks resulting in constraints fitting // both of them. void Combine(const VideoFrameFeedback& other); diff --git a/chromium/media/base/video_frame_layout.cc b/chromium/media/base/video_frame_layout.cc index ac6b633e240..dcf0dd23cc5 100644 --- a/chromium/media/base/video_frame_layout.cc +++ b/chromium/media/base/video_frame_layout.cc @@ -55,6 +55,7 @@ size_t VideoFrameLayout::NumPlanes(VideoPixelFormat format) { case PIXEL_FORMAT_XBGR: case PIXEL_FORMAT_XR30: case PIXEL_FORMAT_XB30: + case PIXEL_FORMAT_RGBAF16: return 1; case PIXEL_FORMAT_NV12: case PIXEL_FORMAT_NV21: diff --git a/chromium/media/base/video_frame_metadata.cc b/chromium/media/base/video_frame_metadata.cc index 6cd1c5dcfb0..cfe6cbc6747 100644 --- a/chromium/media/base/video_frame_metadata.cc +++ b/chromium/media/base/video_frame_metadata.cc @@ -21,11 +21,11 @@ VideoFrameMetadata::VideoFrameMetadata(const VideoFrameMetadata& other) = default; #define MERGE_FIELD(a, source) \ - if (source->a) \ - this->a = source->a + if (source.a) \ + this->a = source.a void VideoFrameMetadata::MergeMetadataFrom( - const VideoFrameMetadata* metadata_source) { + const VideoFrameMetadata& metadata_source) { MERGE_FIELD(allow_overlay, metadata_source); MERGE_FIELD(capture_begin_time, metadata_source); MERGE_FIELD(capture_end_time, metadata_source); @@ -38,7 +38,7 @@ void VideoFrameMetadata::MergeMetadataFrom( MERGE_FIELD(interactive_content, metadata_source); MERGE_FIELD(reference_time, metadata_source); MERGE_FIELD(read_lock_fences_enabled, metadata_source); - MERGE_FIELD(rotation, metadata_source); + MERGE_FIELD(transformation, metadata_source); MERGE_FIELD(texture_owner, metadata_source); MERGE_FIELD(wants_promotion_hint, metadata_source); MERGE_FIELD(protected_video, metadata_source); @@ -57,6 +57,7 @@ void VideoFrameMetadata::MergeMetadataFrom( MERGE_FIELD(receive_time, metadata_source); MERGE_FIELD(wallclock_frame_duration, metadata_source); MERGE_FIELD(maximum_composition_delay_in_frames, metadata_source); + MERGE_FIELD(hw_protected_validation_id, metadata_source); } } // namespace media diff --git a/chromium/media/base/video_frame_metadata.h b/chromium/media/base/video_frame_metadata.h index ded4b13adf6..3456e0703bd 100644 --- a/chromium/media/base/video_frame_metadata.h +++ b/chromium/media/base/video_frame_metadata.h @@ -46,7 +46,7 @@ struct MEDIA_EXPORT VideoFrameMetadata { }; // Merges internal values from |metadata_source|. - void MergeMetadataFrom(const VideoFrameMetadata* metadata_source); + void MergeMetadataFrom(const VideoFrameMetadata& metadata_source); // Sources of VideoFrames use this marker to indicate that the associated // VideoFrame can be overlaid, case in which its contents do not need to be @@ -114,8 +114,8 @@ struct MEDIA_EXPORT VideoFrameMetadata { // should use read lock fences. bool read_lock_fences_enabled = false; - // Indicates that the frame is rotated. - base::Optional<VideoRotation> rotation; + // Indicates that the frame has a rotation and/or flip. + base::Optional<VideoTransformation> transformation; // Android only: if set, then this frame is not suitable for overlay, even // if ALLOW_OVERLAY is set. However, it allows us to process the overlay @@ -134,6 +134,10 @@ struct MEDIA_EXPORT VideoFrameMetadata { // PROTECTED_VIDEO is also set to true. bool hw_protected = false; + // Identifier used to query if a HW protected video frame can still be + // properly displayed or not. Non-zero when valid. + uint32_t hw_protected_validation_id = 0; + // An UnguessableToken that identifies VideoOverlayFactory that created // this VideoFrame. It's used by Cast to help with video hole punch. base::Optional<base::UnguessableToken> overlay_plane_id; @@ -165,7 +169,7 @@ struct MEDIA_EXPORT VideoFrameMetadata { // The RTP timestamp associated with this video frame. Stored as a double // since base::DictionaryValue doesn't have a uint32_t type. // - // https://w3c.github.io/webrtc-pc/#dom-rtcrtpcontributingsource + // https://w3c.github.io/webrtc-pc/#dom-rtcrtpcontributingsource-rtptimestamp base::Optional<double> rtp_timestamp; // For video frames coming from a remote source, this is the time the diff --git a/chromium/media/base/video_frame_unittest.cc b/chromium/media/base/video_frame_unittest.cc index 30f2bbc64e0..50836987e0d 100644 --- a/chromium/media/base/video_frame_unittest.cc +++ b/chromium/media/base/video_frame_unittest.cc @@ -59,8 +59,8 @@ media::VideoFrameMetadata GetFullVideoFrameMetadata() { // gfx::Rects metadata.capture_update_rect = gfx::Rect(12, 34, 360, 480); - // media::VideoRotations - metadata.rotation = media::VideoRotation::VIDEO_ROTATION_90; + // media::VideoTransformation + metadata.transformation = media::VIDEO_ROTATION_90; // media::VideoFrameMetadata::CopyMode metadata.copy_mode = media::VideoFrameMetadata::CopyMode::kCopyToNewTexture; @@ -119,7 +119,7 @@ void VerifyVideoFrameMetadataEquality(const media::VideoFrameMetadata& a, EXPECT_EQ(a.interactive_content, b.interactive_content); EXPECT_EQ(a.reference_time, b.reference_time); EXPECT_EQ(a.read_lock_fences_enabled, b.read_lock_fences_enabled); - EXPECT_EQ(a.rotation, b.rotation); + EXPECT_EQ(a.transformation, b.transformation); EXPECT_EQ(a.texture_owner, b.texture_owner); EXPECT_EQ(a.wants_promotion_hint, b.wants_promotion_hint); EXPECT_EQ(a.protected_video, b.protected_video); @@ -290,7 +290,7 @@ TEST(VideoFrame, CreateFrame) { // Test an empty frame. frame = VideoFrame::CreateEOSFrame(); - EXPECT_TRUE(frame->metadata()->end_of_stream); + EXPECT_TRUE(frame->metadata().end_of_stream); } TEST(VideoFrame, CreateZeroInitializedFrame) { @@ -326,7 +326,7 @@ TEST(VideoFrame, CreateBlackFrame) { // Test basic properties. EXPECT_EQ(0, frame->timestamp().InMicroseconds()); - EXPECT_FALSE(frame->metadata()->end_of_stream); + EXPECT_FALSE(frame->metadata().end_of_stream); // Test |frame| properties. EXPECT_EQ(PIXEL_FORMAT_I420, frame->format()); @@ -368,7 +368,7 @@ TEST(VideoFrame, WrapVideoFrame) { gfx::Rect visible_rect(1, 1, 1, 1); gfx::Size natural_size = visible_rect.size(); - wrapped_frame->metadata()->frame_duration = kFrameDuration; + wrapped_frame->metadata().frame_duration = kFrameDuration; frame = media::VideoFrame::WrapVideoFrame( wrapped_frame, wrapped_frame->format(), visible_rect, natural_size); wrapped_frame->AddDestructionObserver( @@ -382,12 +382,12 @@ TEST(VideoFrame, WrapVideoFrame) { EXPECT_EQ(natural_size, frame->natural_size()); // Verify metadata was copied to the wrapped frame. - EXPECT_EQ(*frame->metadata()->frame_duration, kFrameDuration); + EXPECT_EQ(*frame->metadata().frame_duration, kFrameDuration); // Verify the metadata copy was a deep copy. wrapped_frame->clear_metadata(); - EXPECT_NE(wrapped_frame->metadata()->frame_duration.has_value(), - frame->metadata()->frame_duration.has_value()); + EXPECT_NE(wrapped_frame->metadata().frame_duration.has_value(), + frame->metadata().frame_duration.has_value()); } // Verify that |wrapped_frame| outlives |frame|. @@ -725,6 +725,10 @@ TEST(VideoFrame, AllocationSize_OddSize) { EXPECT_EQ(30u, VideoFrame::AllocationSize(format, size)) << VideoPixelFormatToString(format); break; + case PIXEL_FORMAT_RGBAF16: + EXPECT_EQ(120u, VideoFrame::AllocationSize(format, size)) + << VideoPixelFormatToString(format); + break; case PIXEL_FORMAT_MJPEG: case PIXEL_FORMAT_UNKNOWN: continue; @@ -738,11 +742,11 @@ TEST(VideoFrameMetadata, MergeMetadata) { VideoFrameMetadata empty_metadata; // Merging empty metadata into full metadata should be a no-op. - full_metadata.MergeMetadataFrom(&empty_metadata); + full_metadata.MergeMetadataFrom(empty_metadata); VerifyVideoFrameMetadataEquality(full_metadata, reference_metadata); // Merging full metadata into empty metadata should fill it up. - empty_metadata.MergeMetadataFrom(&full_metadata); + empty_metadata.MergeMetadataFrom(full_metadata); VerifyVideoFrameMetadataEquality(empty_metadata, reference_metadata); } @@ -761,7 +765,7 @@ TEST(VideoFrameMetadata, PartialMergeMetadata) { partial_metadata.allow_overlay = false; // Merging partial metadata into full metadata partially override it. - full_metadata.MergeMetadataFrom(&partial_metadata); + full_metadata.MergeMetadataFrom(partial_metadata); EXPECT_EQ(partial_metadata.capture_update_rect, kTempRect); EXPECT_EQ(partial_metadata.reference_time, kTempTicks); diff --git a/chromium/media/base/video_types.cc b/chromium/media/base/video_types.cc index 6814a364db1..56b39a1ed76 100644 --- a/chromium/media/base/video_types.cc +++ b/chromium/media/base/video_types.cc @@ -73,6 +73,8 @@ std::string VideoPixelFormatToString(VideoPixelFormat format) { return "PIXEL_FORMAT_XB30"; case PIXEL_FORMAT_BGRA: return "PIXEL_FORMAT_BGRA"; + case PIXEL_FORMAT_RGBAF16: + return "PIXEL_FORMAT_RGBAF16"; } NOTREACHED() << "Invalid VideoPixelFormat provided: " << format; return ""; @@ -128,6 +130,7 @@ bool IsYuvPlanar(VideoPixelFormat format) { case PIXEL_FORMAT_XR30: case PIXEL_FORMAT_XB30: case PIXEL_FORMAT_BGRA: + case PIXEL_FORMAT_RGBAF16: return false; } return false; @@ -166,6 +169,7 @@ bool IsOpaque(VideoPixelFormat format) { case PIXEL_FORMAT_ARGB: case PIXEL_FORMAT_ABGR: case PIXEL_FORMAT_BGRA: + case PIXEL_FORMAT_RGBAF16: break; } return false; @@ -209,6 +213,7 @@ size_t BitDepth(VideoPixelFormat format) { return 12; case PIXEL_FORMAT_Y16: case PIXEL_FORMAT_P016LE: + case PIXEL_FORMAT_RGBAF16: return 16; } NOTREACHED(); diff --git a/chromium/media/base/video_types.h b/chromium/media/base/video_types.h index 2205036a07c..2187f23272c 100644 --- a/chromium/media/base/video_types.h +++ b/chromium/media/base/video_types.h @@ -77,9 +77,11 @@ enum VideoPixelFormat { PIXEL_FORMAT_BGRA = 32, // 32bpp ARGB (byte-order), 1 plane. + PIXEL_FORMAT_RGBAF16 = 33, // Half float RGBA, 1 plane. + // Please update UMA histogram enumeration when adding new formats here. PIXEL_FORMAT_MAX = - PIXEL_FORMAT_BGRA, // Must always be equal to largest entry logged. + PIXEL_FORMAT_RGBAF16, // Must always be equal to largest entry logged. }; // Returns the name of a Format as a string. diff --git a/chromium/media/base/video_util.cc b/chromium/media/base/video_util.cc index 3daa1cb27d5..5f26611aa07 100644 --- a/chromium/media/base/video_util.cc +++ b/chromium/media/base/video_util.cc @@ -7,12 +7,25 @@ #include <cmath> #include "base/bind.h" +#include "base/callback_helpers.h" #include "base/check_op.h" +#include "base/logging.h" #include "base/notreached.h" #include "base/numerics/safe_conversions.h" #include "base/numerics/safe_math.h" +#include "gpu/GLES2/gl2extchromium.h" +#include "gpu/command_buffer/client/raster_interface.h" +#include "media/base/status_codes.h" #include "media/base/video_frame.h" +#include "media/base/video_frame_pool.h" #include "third_party/libyuv/include/libyuv.h" +#include "third_party/skia/include/core/SkImage.h" +#include "third_party/skia/include/core/SkRefCnt.h" +#include "third_party/skia/include/core/SkSurface.h" +#include "third_party/skia/include/core/SkYUVAPixmaps.h" +#include "third_party/skia/include/gpu/GrDirectContext.h" +#include "third_party/skia/include/gpu/gl/GrGLTypes.h" +#include "ui/gfx/gpu_memory_buffer.h" namespace media { @@ -47,6 +60,202 @@ void FillRegionOutsideVisibleRect(uint8_t* data, } } +std::pair<SkColorType, GrGLenum> GetSkiaAndGlColorTypesForPlane( + VideoPixelFormat format, + size_t plane) { + // TODO(eugene): There is some strange channel switch during RGB readback. + // When frame's pixel format matches GL and Skia color types we get reversed + // channels. But why? + switch (format) { + case PIXEL_FORMAT_NV12: + if (plane == VideoFrame::kUVPlane) + return {kR8G8_unorm_SkColorType, GL_RG8_EXT}; + if (plane == VideoFrame::kYPlane) + return {kAlpha_8_SkColorType, GL_R8_EXT}; + break; + case PIXEL_FORMAT_XBGR: + if (plane == VideoFrame::kARGBPlane) + return {kRGBA_8888_SkColorType, GL_RGBA8_OES}; + break; + case PIXEL_FORMAT_ABGR: + if (plane == VideoFrame::kARGBPlane) + return {kRGBA_8888_SkColorType, GL_RGBA8_OES}; + break; + case PIXEL_FORMAT_XRGB: + if (plane == VideoFrame::kARGBPlane) + return {kBGRA_8888_SkColorType, GL_BGRA8_EXT}; + break; + case PIXEL_FORMAT_ARGB: + if (plane == VideoFrame::kARGBPlane) + return {kBGRA_8888_SkColorType, GL_BGRA8_EXT}; + break; + default: + break; + } + NOTREACHED(); + return {kUnknown_SkColorType, 0}; +} + +scoped_refptr<VideoFrame> ReadbackTextureBackedFrameToMemorySyncGLES( + const VideoFrame& txt_frame, + gpu::raster::RasterInterface* ri, + GrDirectContext* gr_context, + VideoFramePool* pool) { + DCHECK(gr_context); + + if (txt_frame.NumTextures() > 2 || txt_frame.NumTextures() < 1) { + DLOG(ERROR) << "Readback is not possible for this frame: " + << txt_frame.AsHumanReadableString(); + return nullptr; + } + + VideoPixelFormat result_format = txt_frame.format(); + if (txt_frame.NumTextures() == 1 && result_format == PIXEL_FORMAT_NV12) { + // Even though |txt_frame| format is NV12 and it is NV12 in GPU memory, + // the texture is a RGB view that is produced by a shader on the fly. + // So we currently we currently can only read it back as RGB. + result_format = PIXEL_FORMAT_ARGB; + } + + scoped_refptr<VideoFrame> result = + pool + ? pool->CreateFrame(result_format, txt_frame.coded_size(), + txt_frame.visible_rect(), + txt_frame.natural_size(), txt_frame.timestamp()) + : VideoFrame::CreateFrame( + result_format, txt_frame.coded_size(), txt_frame.visible_rect(), + txt_frame.natural_size(), txt_frame.timestamp()); + result->set_color_space(txt_frame.ColorSpace()); + result->metadata().MergeMetadataFrom(txt_frame.metadata()); + + size_t planes = VideoFrame::NumPlanes(result->format()); + for (size_t plane = 0; plane < planes; plane++) { + const gpu::MailboxHolder& holder = txt_frame.mailbox_holder(plane); + if (holder.mailbox.IsZero()) + return nullptr; + ri->WaitSyncTokenCHROMIUM(holder.sync_token.GetConstData()); + + int width = VideoFrame::Columns(plane, result->format(), + result->coded_size().width()); + int height = result->rows(plane); + + auto texture_id = ri->CreateAndConsumeForGpuRaster(holder.mailbox); + if (holder.mailbox.IsSharedImage()) { + ri->BeginSharedImageAccessDirectCHROMIUM( + texture_id, GL_SHARED_IMAGE_ACCESS_MODE_READ_CHROMIUM); + } + + auto cleanup_fn = [](GLuint texture_id, bool shared, + gpu::raster::RasterInterface* ri) { + if (shared) + ri->EndSharedImageAccessDirectCHROMIUM(texture_id); + ri->DeleteGpuRasterTexture(texture_id); + }; + base::ScopedClosureRunner cleanup(base::BindOnce( + cleanup_fn, texture_id, holder.mailbox.IsSharedImage(), ri)); + + GrGLenum texture_format; + SkColorType sk_color_type; + std::tie(sk_color_type, texture_format) = + GetSkiaAndGlColorTypesForPlane(result->format(), plane); + GrGLTextureInfo gl_texture_info; + gl_texture_info.fID = texture_id; + gl_texture_info.fTarget = holder.texture_target; + gl_texture_info.fFormat = texture_format; + + GrBackendTexture texture(width, height, GrMipMapped::kNo, gl_texture_info); + auto image = SkImage::MakeFromTexture( + gr_context, texture, kTopLeft_GrSurfaceOrigin, sk_color_type, + kOpaque_SkAlphaType, nullptr /* colorSpace */); + + if (!image) { + DLOG(ERROR) << "Can't create SkImage from texture!" + << " plane:" << plane; + return nullptr; + } + + auto info = + SkImageInfo::Make(width, height, sk_color_type, kOpaque_SkAlphaType); + SkPixmap pixmap(info, result->data(plane), result->row_bytes(plane)); + if (!image->readPixels(gr_context, pixmap, 0, 0, + SkImage::kDisallow_CachingHint)) { + DLOG(ERROR) << "Plane readback failed." + << " plane:" << plane << " width: " << width + << " height: " << height + << " minRowBytes: " << info.minRowBytes(); + return nullptr; + } + } + + return result; +} + +scoped_refptr<VideoFrame> ReadbackTextureBackedFrameToMemorySyncOOP( + const VideoFrame& txt_frame, + gpu::raster::RasterInterface* ri, + VideoFramePool* pool) { + if (txt_frame.NumTextures() > 2 || txt_frame.NumTextures() < 1) { + DLOG(ERROR) << "Readback is not possible for this frame: " + << txt_frame.AsHumanReadableString(); + return nullptr; + } + + VideoPixelFormat result_format = txt_frame.format(); + if (txt_frame.NumTextures() == 1 && result_format == PIXEL_FORMAT_NV12) { + // Even though |txt_frame| format is NV12 and it is NV12 in GPU memory, + // the texture is a RGB view that is produced by a shader on the fly. + // So we currently we currently can only read it back as RGB. + result_format = PIXEL_FORMAT_ARGB; + } + + scoped_refptr<VideoFrame> result = + pool + ? pool->CreateFrame(result_format, txt_frame.coded_size(), + txt_frame.visible_rect(), + txt_frame.natural_size(), txt_frame.timestamp()) + : VideoFrame::CreateFrame( + result_format, txt_frame.coded_size(), txt_frame.visible_rect(), + txt_frame.natural_size(), txt_frame.timestamp()); + result->set_color_space(txt_frame.ColorSpace()); + result->metadata().MergeMetadataFrom(txt_frame.metadata()); + + size_t planes = VideoFrame::NumPlanes(result->format()); + for (size_t plane = 0; plane < planes; plane++) { + const gpu::MailboxHolder& holder = txt_frame.mailbox_holder(plane); + if (holder.mailbox.IsZero()) { + DLOG(ERROR) << "Can't readback video frame with Zero texture on plane " + << plane; + return nullptr; + } + ri->WaitSyncTokenCHROMIUM(holder.sync_token.GetConstData()); + + int width = VideoFrame::Columns(plane, result->format(), + result->coded_size().width()); + int height = result->rows(plane); + + GrGLenum texture_format; + SkColorType sk_color_type; + std::tie(sk_color_type, texture_format) = + GetSkiaAndGlColorTypesForPlane(result->format(), plane); + + auto info = + SkImageInfo::Make(width, height, sk_color_type, kOpaque_SkAlphaType); + + ri->ReadbackImagePixels(holder.mailbox, info, info.minRowBytes(), 0, 0, + result->data(plane)); + if (ri->GetError() != GL_NO_ERROR) { + DLOG(ERROR) << "Plane readback failed." + << " plane:" << plane << " width: " << width + << " height: " << height + << " minRowBytes: " << info.minRowBytes() + << " error: " << ri->GetError(); + return nullptr; + } + } + + return result; +} + } // namespace double GetPixelAspectRatio(const gfx::Rect& visible_rect, @@ -399,29 +608,42 @@ gfx::Size PadToMatchAspectRatio(const gfx::Size& size, return gfx::Size(size.width(), RoundedDivision(x, target.width())); } -void CopyRGBToVideoFrame(const uint8_t* source, - int stride, - const gfx::Rect& region_in_frame, - VideoFrame* frame) { - const int kY = VideoFrame::kYPlane; - const int kU = VideoFrame::kUPlane; - const int kV = VideoFrame::kVPlane; - CHECK_EQ(frame->stride(kU), frame->stride(kV)); - const int uv_stride = frame->stride(kU); - - if (region_in_frame != gfx::Rect(frame->coded_size())) { - LetterboxVideoFrame(frame, region_in_frame); - } +scoped_refptr<VideoFrame> ConvertToMemoryMappedFrame( + scoped_refptr<VideoFrame> video_frame) { + DCHECK(video_frame); + DCHECK(video_frame->HasGpuMemoryBuffer()); - const int y_offset = - region_in_frame.x() + (region_in_frame.y() * frame->stride(kY)); - const int uv_offset = - region_in_frame.x() / 2 + (region_in_frame.y() / 2 * uv_stride); + auto* gmb = video_frame->GetGpuMemoryBuffer(); + if (!gmb->Map()) + return nullptr; - libyuv::ARGBToI420(source, stride, frame->data(kY) + y_offset, - frame->stride(kY), frame->data(kU) + uv_offset, uv_stride, - frame->data(kV) + uv_offset, uv_stride, - region_in_frame.width(), region_in_frame.height()); + const size_t num_planes = VideoFrame::NumPlanes(video_frame->format()); + uint8_t* plane_addrs[VideoFrame::kMaxPlanes] = {}; + for (size_t i = 0; i < num_planes; i++) + plane_addrs[i] = static_cast<uint8_t*>(gmb->memory(i)); + + auto mapped_frame = VideoFrame::WrapExternalYuvDataWithLayout( + video_frame->layout(), video_frame->visible_rect(), + video_frame->natural_size(), plane_addrs[0], plane_addrs[1], + plane_addrs[2], video_frame->timestamp()); + + if (!mapped_frame) { + gmb->Unmap(); + return nullptr; + } + + mapped_frame->set_color_space(video_frame->ColorSpace()); + mapped_frame->metadata().MergeMetadataFrom(video_frame->metadata()); + + // Pass |video_frame| so that it outlives |mapped_frame| and the mapped buffer + // is unmapped on destruction. + mapped_frame->AddDestructionObserver(base::BindOnce( + [](scoped_refptr<VideoFrame> frame) { + DCHECK(frame->HasGpuMemoryBuffer()); + frame->GetGpuMemoryBuffer()->Unmap(); + }, + std::move(video_frame))); + return mapped_frame; } scoped_refptr<VideoFrame> WrapAsI420VideoFrame( @@ -429,12 +651,8 @@ scoped_refptr<VideoFrame> WrapAsI420VideoFrame( DCHECK_EQ(VideoFrame::STORAGE_OWNED_MEMORY, frame->storage_type()); DCHECK_EQ(PIXEL_FORMAT_I420A, frame->format()); - scoped_refptr<media::VideoFrame> wrapped_frame = - media::VideoFrame::WrapVideoFrame(frame, PIXEL_FORMAT_I420, - frame->visible_rect(), - frame->natural_size()); - if (!wrapped_frame) - return nullptr; + scoped_refptr<VideoFrame> wrapped_frame = VideoFrame::WrapVideoFrame( + frame, PIXEL_FORMAT_I420, frame->visible_rect(), frame->natural_size()); return wrapped_frame; } @@ -487,4 +705,287 @@ bool I420CopyWithPadding(const VideoFrame& src_frame, VideoFrame* dst_frame) { return true; } +scoped_refptr<VideoFrame> ReadbackTextureBackedFrameToMemorySync( + const VideoFrame& txt_frame, + gpu::raster::RasterInterface* ri, + GrDirectContext* gr_context, + VideoFramePool* pool) { + DCHECK(ri); + + if (gr_context) { + return ReadbackTextureBackedFrameToMemorySyncGLES(txt_frame, ri, gr_context, + pool); + } + return ReadbackTextureBackedFrameToMemorySyncOOP(txt_frame, ri, pool); +} + +Status ConvertAndScaleFrame(const VideoFrame& src_frame, + VideoFrame& dst_frame, + std::vector<uint8_t>& tmp_buf) { + constexpr auto kDefaultFiltering = libyuv::kFilterBox; + if (!src_frame.IsMappable() || !dst_frame.IsMappable()) + return Status(StatusCode::kUnsupportedFrameFormatError); + + if ((dst_frame.format() == PIXEL_FORMAT_I420 || + dst_frame.format() == PIXEL_FORMAT_NV12) && + (src_frame.format() == PIXEL_FORMAT_XBGR || + src_frame.format() == PIXEL_FORMAT_XRGB || + src_frame.format() == PIXEL_FORMAT_ABGR || + src_frame.format() == PIXEL_FORMAT_ARGB)) { + // libyuv's RGB to YUV methods always output BT.601. + dst_frame.set_color_space(gfx::ColorSpace::CreateREC601()); + + size_t src_stride = src_frame.stride(VideoFrame::kARGBPlane); + const uint8_t* src_data = src_frame.visible_data(VideoFrame::kARGBPlane); + if (src_frame.visible_rect() != dst_frame.visible_rect()) { + size_t tmp_buffer_size = VideoFrame::AllocationSize( + src_frame.format(), dst_frame.coded_size()); + if (tmp_buf.size() < tmp_buffer_size) + tmp_buf.resize(tmp_buffer_size); + + size_t stride = + VideoFrame::RowBytes(VideoFrame::kARGBPlane, src_frame.format(), + dst_frame.visible_rect().width()); + int error = libyuv::ARGBScale( + src_data, src_stride, src_frame.visible_rect().width(), + src_frame.visible_rect().height(), tmp_buf.data(), stride, + dst_frame.visible_rect().width(), dst_frame.visible_rect().height(), + kDefaultFiltering); + if (error) + return Status(StatusCode::kInvalidArgument); + src_data = tmp_buf.data(); + src_stride = stride; + } + + if (dst_frame.format() == PIXEL_FORMAT_I420) { + auto convert_fn = (src_frame.format() == PIXEL_FORMAT_XBGR || + src_frame.format() == PIXEL_FORMAT_ABGR) + ? libyuv::ABGRToI420 + : libyuv::ARGBToI420; + int error = convert_fn( + src_data, src_stride, dst_frame.visible_data(VideoFrame::kYPlane), + dst_frame.stride(VideoFrame::kYPlane), + dst_frame.visible_data(VideoFrame::kUPlane), + dst_frame.stride(VideoFrame::kUPlane), + dst_frame.visible_data(VideoFrame::kVPlane), + dst_frame.stride(VideoFrame::kVPlane), + dst_frame.visible_rect().width(), dst_frame.visible_rect().height()); + return error ? Status(StatusCode::kInvalidArgument) : Status(); + } + + auto convert_fn = (src_frame.format() == PIXEL_FORMAT_XBGR || + src_frame.format() == PIXEL_FORMAT_ABGR) + ? libyuv::ABGRToNV12 + : libyuv::ARGBToNV12; + int error = convert_fn( + src_data, src_stride, dst_frame.visible_data(VideoFrame::kYPlane), + dst_frame.stride(VideoFrame::kYPlane), + dst_frame.visible_data(VideoFrame::kUVPlane), + dst_frame.stride(VideoFrame::kUVPlane), + dst_frame.visible_rect().width(), dst_frame.visible_rect().height()); + return error ? Status(StatusCode::kInvalidArgument) : Status(); + } + + // Converting between YUV formats doesn't change the color space. + dst_frame.set_color_space(src_frame.ColorSpace()); + + // Both frames are I420, only scaling is required. + if (dst_frame.format() == PIXEL_FORMAT_I420 && + src_frame.format() == PIXEL_FORMAT_I420) { + int error = libyuv::I420Scale( + src_frame.visible_data(VideoFrame::kYPlane), + src_frame.stride(VideoFrame::kYPlane), + src_frame.visible_data(VideoFrame::kUPlane), + src_frame.stride(VideoFrame::kUPlane), + src_frame.visible_data(VideoFrame::kVPlane), + src_frame.stride(VideoFrame::kVPlane), src_frame.visible_rect().width(), + src_frame.visible_rect().height(), + dst_frame.visible_data(VideoFrame::kYPlane), + dst_frame.stride(VideoFrame::kYPlane), + dst_frame.visible_data(VideoFrame::kUPlane), + dst_frame.stride(VideoFrame::kUPlane), + dst_frame.visible_data(VideoFrame::kVPlane), + dst_frame.stride(VideoFrame::kVPlane), dst_frame.visible_rect().width(), + dst_frame.visible_rect().height(), kDefaultFiltering); + return error ? Status(StatusCode::kInvalidArgument) : Status(); + } + + // Both frames are NV12, only scaling is required. + if (dst_frame.format() == PIXEL_FORMAT_NV12 && + src_frame.format() == PIXEL_FORMAT_NV12) { + int error = libyuv::NV12Scale( + src_frame.visible_data(VideoFrame::kYPlane), + src_frame.stride(VideoFrame::kYPlane), + src_frame.visible_data(VideoFrame::kUVPlane), + src_frame.stride(VideoFrame::kUVPlane), + src_frame.visible_rect().width(), src_frame.visible_rect().height(), + dst_frame.visible_data(VideoFrame::kYPlane), + dst_frame.stride(VideoFrame::kYPlane), + dst_frame.visible_data(VideoFrame::kUVPlane), + dst_frame.stride(VideoFrame::kUVPlane), + dst_frame.visible_rect().width(), dst_frame.visible_rect().height(), + kDefaultFiltering); + return error ? Status(StatusCode::kInvalidArgument) : Status(); + } + + if (dst_frame.format() == PIXEL_FORMAT_I420 && + src_frame.format() == PIXEL_FORMAT_NV12) { + if (src_frame.visible_rect() == dst_frame.visible_rect()) { + // Both frames have the same size, only NV12-to-I420 conversion is + // required. + int error = libyuv::NV12ToI420( + src_frame.visible_data(VideoFrame::kYPlane), + src_frame.stride(VideoFrame::kYPlane), + src_frame.visible_data(VideoFrame::kUVPlane), + src_frame.stride(VideoFrame::kUVPlane), + dst_frame.visible_data(VideoFrame::kYPlane), + dst_frame.stride(VideoFrame::kYPlane), + dst_frame.visible_data(VideoFrame::kUPlane), + dst_frame.stride(VideoFrame::kUPlane), + dst_frame.visible_data(VideoFrame::kVPlane), + dst_frame.stride(VideoFrame::kVPlane), + dst_frame.visible_rect().width(), dst_frame.visible_rect().height()); + return error ? Status(StatusCode::kInvalidArgument) : Status(); + } else { + // Both resize and NV12-to-I420 conversion are required. + // First, split UV planes into two, basically producing a I420 frame. + const int tmp_uv_width = (src_frame.visible_rect().width() + 1) / 2; + const int tmp_uv_height = (src_frame.visible_rect().height() + 1) / 2; + size_t tmp_buffer_size = tmp_uv_width * tmp_uv_height * 2; + if (tmp_buf.size() < tmp_buffer_size) + tmp_buf.resize(tmp_buffer_size); + + uint8_t* tmp_u = tmp_buf.data(); + uint8_t* tmp_v = tmp_u + tmp_uv_width * tmp_uv_height; + DCHECK_EQ(tmp_buf.data() + tmp_buffer_size, + tmp_v + (tmp_uv_width * tmp_uv_height)); + libyuv::SplitUVPlane(src_frame.visible_data(VideoFrame::kUVPlane), + src_frame.stride(VideoFrame::kUVPlane), tmp_u, + tmp_uv_width, tmp_v, tmp_uv_width, tmp_uv_width, + tmp_uv_height); + + // Second, scale resulting I420 frame into the destination. + int error = libyuv::I420Scale( + src_frame.visible_data(VideoFrame::kYPlane), + src_frame.stride(VideoFrame::kYPlane), + tmp_u, // Temporary U-plane for src UV-plane. + tmp_uv_width, + tmp_v, // Temporary V-plane for src UV-plane. + tmp_uv_width, src_frame.visible_rect().width(), + src_frame.visible_rect().height(), + dst_frame.visible_data(VideoFrame::kYPlane), + dst_frame.stride(VideoFrame::kYPlane), + dst_frame.visible_data(VideoFrame::kUPlane), + dst_frame.stride(VideoFrame::kUPlane), + dst_frame.visible_data(VideoFrame::kVPlane), + dst_frame.stride(VideoFrame::kVPlane), + dst_frame.visible_rect().width(), dst_frame.visible_rect().height(), + kDefaultFiltering); + return error ? Status(StatusCode::kInvalidArgument) : Status(); + } + } + + if (dst_frame.format() == PIXEL_FORMAT_NV12 && + src_frame.format() == PIXEL_FORMAT_I420) { + if (src_frame.visible_rect() == dst_frame.visible_rect()) { + // Both frames have the same size, only I420-to-NV12 conversion is + // required. + int error = libyuv::I420ToNV12( + src_frame.visible_data(VideoFrame::kYPlane), + src_frame.stride(VideoFrame::kYPlane), + src_frame.visible_data(VideoFrame::kUPlane), + src_frame.stride(VideoFrame::kUPlane), + src_frame.visible_data(VideoFrame::kVPlane), + src_frame.stride(VideoFrame::kVPlane), + dst_frame.visible_data(VideoFrame::kYPlane), + dst_frame.stride(VideoFrame::kYPlane), + dst_frame.visible_data(VideoFrame::kUVPlane), + dst_frame.stride(VideoFrame::kUVPlane), + dst_frame.visible_rect().width(), dst_frame.visible_rect().height()); + return error ? Status(StatusCode::kInvalidArgument) : Status(); + } else { + // Both resize and I420-to-NV12 conversion are required. + // First, merge U and V planes into one, basically producing a NV12 frame. + const int tmp_uv_width = (src_frame.visible_rect().width() + 1) / 2; + const int tmp_uv_height = (src_frame.visible_rect().height() + 1) / 2; + size_t tmp_buffer_size = tmp_uv_width * tmp_uv_height * 2; + if (tmp_buf.size() < tmp_buffer_size) + tmp_buf.resize(tmp_buffer_size); + + uint8_t* tmp_uv = tmp_buf.data(); + size_t stride_uv = tmp_uv_width * 2; + libyuv::MergeUVPlane(src_frame.visible_data(VideoFrame::kUPlane), + src_frame.stride(VideoFrame::kUPlane), + src_frame.visible_data(VideoFrame::kVPlane), + src_frame.stride(VideoFrame::kVPlane), + tmp_uv, // Temporary for merged UV-plane + stride_uv, // Temporary stride + tmp_uv_width, tmp_uv_height); + + // Second, scale resulting NV12 frame into the destination. + int error = libyuv::NV12Scale( + src_frame.visible_data(VideoFrame::kYPlane), + src_frame.stride(VideoFrame::kYPlane), + tmp_uv, // Temporary for merged UV-plane + stride_uv, // Temporary stride + src_frame.visible_rect().width(), src_frame.visible_rect().height(), + dst_frame.visible_data(VideoFrame::kYPlane), + dst_frame.stride(VideoFrame::kYPlane), + dst_frame.visible_data(VideoFrame::kUVPlane), + dst_frame.stride(VideoFrame::kUVPlane), + dst_frame.visible_rect().width(), dst_frame.visible_rect().height(), + kDefaultFiltering); + return error ? Status(StatusCode::kInvalidArgument) : Status(); + } + } + + return Status(StatusCode::kUnsupportedFrameFormatError) + .WithData("src", src_frame.AsHumanReadableString()) + .WithData("dst", dst_frame.AsHumanReadableString()); +} + +scoped_refptr<VideoFrame> CreateFromSkImage(sk_sp<SkImage> sk_image, + const gfx::Rect& visible_rect, + const gfx::Size& natural_size, + base::TimeDelta timestamp) { + DCHECK(!sk_image->isTextureBacked()); + + // TODO(crbug.com/1073995): Add F16 support. + auto sk_color_type = sk_image->colorType(); + if (sk_color_type != kRGBA_8888_SkColorType && + sk_color_type != kBGRA_8888_SkColorType) { + return nullptr; + } + + SkPixmap pm; + const bool peek_result = sk_image->peekPixels(&pm); + DCHECK(peek_result); + + const auto format = + sk_image->isOpaque() + ? (sk_color_type == kRGBA_8888_SkColorType ? PIXEL_FORMAT_XBGR + : PIXEL_FORMAT_XRGB) + : (sk_color_type == kRGBA_8888_SkColorType ? PIXEL_FORMAT_ABGR + : PIXEL_FORMAT_ARGB); + + auto coded_size = gfx::Size(sk_image->width(), sk_image->height()); + auto layout = VideoFrameLayout::CreateWithStrides( + format, coded_size, std::vector<int32_t>(1, pm.rowBytes())); + if (!layout) + return nullptr; + + auto frame = VideoFrame::WrapExternalDataWithLayout( + *layout, visible_rect, natural_size, + // TODO(crbug.com/1161304): We should be able to wrap readonly memory in + // a VideoFrame instead of using writable_addr() here. + reinterpret_cast<uint8_t*>(pm.writable_addr()), pm.computeByteSize(), + timestamp); + if (!frame) + return nullptr; + + frame->AddDestructionObserver(base::BindOnce( + base::DoNothing::Once<sk_sp<SkImage>>(), std::move(sk_image))); + return frame; +} + } // namespace media diff --git a/chromium/media/base/video_util.h b/chromium/media/base/video_util.h index 42e060a25b7..3590bf00e9a 100644 --- a/chromium/media/base/video_util.h +++ b/chromium/media/base/video_util.h @@ -7,13 +7,26 @@ #include <stdint.h> +#include <vector> + #include "base/memory/ref_counted.h" #include "media/base/media_export.h" +#include "media/base/status.h" +#include "third_party/skia/include/core/SkImage.h" #include "ui/gfx/geometry/rect.h" #include "ui/gfx/geometry/size.h" +class GrDirectContext; + +namespace gpu { +namespace raster { +class RasterInterface; +} // namespace raster +} // namespace gpu + namespace media { +class VideoFramePool; class VideoFrame; // Computes the pixel aspect ratio of a given |visible_rect| from its @@ -134,14 +147,26 @@ MEDIA_EXPORT gfx::Size GetRectSizeFromOrigin(const gfx::Rect& rect); MEDIA_EXPORT gfx::Size PadToMatchAspectRatio(const gfx::Size& size, const gfx::Size& target); -// Copy an RGB bitmap into the specified |region_in_frame| of a YUV video frame. -// Fills the regions outside |region_in_frame| with black. -MEDIA_EXPORT void CopyRGBToVideoFrame(const uint8_t* source, - int stride, - const gfx::Rect& region_in_frame, - VideoFrame* frame); +// A helper function to map GpuMemoryBuffer-based VideoFrame. This function +// maps the given GpuMemoryBuffer of |frame| as-is without converting pixel +// format. The returned VideoFrame owns the |frame|. +MEDIA_EXPORT scoped_refptr<VideoFrame> ConvertToMemoryMappedFrame( + scoped_refptr<VideoFrame> frame); -// Converts a frame with YV12A format into I420 by dropping alpha channel. +// This function synchronously reads pixel data from textures associated with +// |txt_frame| and creates a new CPU memory backed frame. It's needed because +// existing video encoders can't handle texture backed frames. +// +// TODO(crbug.com/1162530): Combine this function with +// media::ConvertAndScaleFrame and put it into a new class +// media:FrameSizeAndFormatConverter. +MEDIA_EXPORT scoped_refptr<VideoFrame> ReadbackTextureBackedFrameToMemorySync( + const VideoFrame& txt_frame, + gpu::raster::RasterInterface* ri, + GrDirectContext* gr_context, + VideoFramePool* pool = nullptr); + +// Converts a frame with I420A format into I420 by dropping alpha channel. MEDIA_EXPORT scoped_refptr<VideoFrame> WrapAsI420VideoFrame( scoped_refptr<VideoFrame> frame); @@ -164,6 +189,23 @@ MEDIA_EXPORT scoped_refptr<VideoFrame> WrapAsI420VideoFrame( MEDIA_EXPORT bool I420CopyWithPadding(const VideoFrame& src_frame, VideoFrame* dst_frame) WARN_UNUSED_RESULT; +// Copy pixel data from |src_frame| to |dst_frame| applying scaling and pixel +// format conversion as needed. Both frames need to be mappabale and have either +// I420 or NV12 pixel format. +MEDIA_EXPORT Status ConvertAndScaleFrame(const VideoFrame& src_frame, + VideoFrame& dst_frame, + std::vector<uint8_t>& tmp_buf) + WARN_UNUSED_RESULT; + +// Backs a VideoFrame with a SkImage. The created frame takes a ref on the +// provided SkImage to make this operation zero copy. Only works with CPU +// backed images. +MEDIA_EXPORT scoped_refptr<VideoFrame> CreateFromSkImage( + sk_sp<SkImage> sk_image, + const gfx::Rect& visible_rect, + const gfx::Size& natural_size, + base::TimeDelta timestamp); + } // namespace media #endif // MEDIA_BASE_VIDEO_UTIL_H_ diff --git a/chromium/media/base/video_util_unittest.cc b/chromium/media/base/video_util_unittest.cc index 79af565e5b0..c6ae0616ddc 100644 --- a/chromium/media/base/video_util_unittest.cc +++ b/chromium/media/base/video_util_unittest.cc @@ -604,4 +604,30 @@ TEST_F(VideoUtilTest, I420CopyWithPadding) { EXPECT_TRUE(VerifyCopyWithPadding(*src_frame, *dst_frame)); } +TEST_F(VideoUtilTest, WrapAsI420VideoFrame) { + gfx::Size size(640, 480); + scoped_refptr<VideoFrame> src_frame = + VideoFrame::CreateFrame(PIXEL_FORMAT_I420A, size, gfx::Rect(size), size, + base::TimeDelta::FromDays(1)); + + scoped_refptr<VideoFrame> dst_frame = WrapAsI420VideoFrame(src_frame); + EXPECT_EQ(dst_frame->format(), PIXEL_FORMAT_I420); + EXPECT_EQ(dst_frame->timestamp(), src_frame->timestamp()); + EXPECT_EQ(dst_frame->coded_size(), src_frame->coded_size()); + EXPECT_EQ(dst_frame->visible_rect(), src_frame->visible_rect()); + EXPECT_EQ(dst_frame->natural_size(), src_frame->natural_size()); + + std::vector<size_t> planes = {VideoFrame::kYPlane, VideoFrame::kUPlane, + VideoFrame::kVPlane}; + for (auto plane : planes) + EXPECT_EQ(dst_frame->data(plane), src_frame->data(plane)); + + // Check that memory for planes is not released upon destruction of the + // original frame pointer (new frame holds a reference). This check relies on + // ASAN. + src_frame.reset(); + for (auto plane : planes) + memset(dst_frame->data(plane), 1, dst_frame->stride(plane)); +} + } // namespace media diff --git a/chromium/media/base/win/BUILD.gn b/chromium/media/base/win/BUILD.gn index b4a7cb764a1..b78090ac9c1 100644 --- a/chromium/media/base/win/BUILD.gn +++ b/chromium/media/base/win/BUILD.gn @@ -6,6 +6,7 @@ assert(is_win) config("delay_load_mf") { ldflags = [ + "/DELAYLOAD:d3d11.dll", "/DELAYLOAD:mf.dll", "/DELAYLOAD:mfplat.dll", "/DELAYLOAD:mfreadwrite.dll", @@ -15,6 +16,8 @@ config("delay_load_mf") { component("media_foundation_util") { defines = [ "MF_INITIALIZER_IMPLEMENTATION" ] sources = [ + "dxgi_device_manager.cc", + "dxgi_device_manager.h", "mf_helpers.cc", "mf_helpers.h", "mf_initializer.cc", @@ -31,9 +34,11 @@ component("media_foundation_util") { "//media:shared_memory_support", ] libs = [ + "d3d11.lib", "mf.lib", "mfplat.lib", "mfreadwrite.lib", + "dxguid.lib", ] # MediaFoundation is not available on Windows N, so must be delay loaded. diff --git a/chromium/media/base/win/dxgi_device_manager.cc b/chromium/media/base/win/dxgi_device_manager.cc new file mode 100644 index 00000000000..75d102e4376 --- /dev/null +++ b/chromium/media/base/win/dxgi_device_manager.cc @@ -0,0 +1,143 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/base/win/dxgi_device_manager.h" + +#include <mfcaptureengine.h> +#include <mfreadwrite.h> + +#include "base/win/windows_version.h" +#include "media/base/win/mf_helpers.h" + +namespace media { + +DXGIDeviceScopedHandle::DXGIDeviceScopedHandle( + IMFDXGIDeviceManager* device_manager) + : device_manager_(device_manager) {} + +DXGIDeviceScopedHandle::~DXGIDeviceScopedHandle() { + if (device_handle_ == INVALID_HANDLE_VALUE) { + return; + } + + HRESULT hr = device_manager_->CloseDeviceHandle(device_handle_); + LOG_IF(ERROR, FAILED(hr)) << "Failed to close device handle"; + device_handle_ = INVALID_HANDLE_VALUE; +} + +HRESULT DXGIDeviceScopedHandle::LockDevice(REFIID riid, void** device_out) { + HRESULT hr = S_OK; + if (device_handle_ == INVALID_HANDLE_VALUE) { + hr = device_manager_->OpenDeviceHandle(&device_handle_); + RETURN_ON_HR_FAILURE( + hr, "Failed to open device handle on MF DXGI device manager", hr); + } + // see + // https://docs.microsoft.com/en-us/windows/win32/api/mfobjects/nf-mfobjects-imfdxgidevicemanager-lockdevice + // for details of LockDevice call. + hr = device_manager_->LockDevice(device_handle_, riid, device_out, + /*block=*/FALSE); + return hr; +} + +Microsoft::WRL::ComPtr<ID3D11Device> DXGIDeviceScopedHandle::GetDevice() { + HRESULT hr = S_OK; + if (device_handle_ == INVALID_HANDLE_VALUE) { + hr = device_manager_->OpenDeviceHandle(&device_handle_); + RETURN_ON_HR_FAILURE( + hr, "Failed to open device handle on MF DXGI device manager", nullptr); + } + Microsoft::WRL::ComPtr<ID3D11Device> device; + hr = device_manager_->GetVideoService(device_handle_, IID_PPV_ARGS(&device)); + RETURN_ON_HR_FAILURE(hr, "Failed to get device from MF DXGI device manager", + nullptr); + return device; +} + +scoped_refptr<DXGIDeviceManager> DXGIDeviceManager::Create() { + if (base::win::GetVersion() < base::win::Version::WIN8 || + (!::GetModuleHandle(L"mfplat.dll") && !::LoadLibrary(L"mfplat.dll"))) { + // The MF DXGI Device manager is only supported on Win8 or later + // Additionally, it is not supported when mfplat.dll isn't available + DLOG(ERROR) + << "MF DXGI Device Manager not supported on current version of Windows"; + return nullptr; + } + Microsoft::WRL::ComPtr<IMFDXGIDeviceManager> mf_dxgi_device_manager; + UINT d3d_device_reset_token = 0; + HRESULT hr = MFCreateDXGIDeviceManager(&d3d_device_reset_token, + &mf_dxgi_device_manager); + RETURN_ON_HR_FAILURE(hr, "Failed to create MF DXGI device manager", nullptr); + auto dxgi_device_manager = base::WrapRefCounted(new DXGIDeviceManager( + std::move(mf_dxgi_device_manager), d3d_device_reset_token)); + if (dxgi_device_manager && FAILED(dxgi_device_manager->ResetDevice())) { + // If setting a device failed, ensure that an empty scoped_refptr is + // returned as the dxgi_device_manager is not usable without a device. + return nullptr; + } + return dxgi_device_manager; +} + +DXGIDeviceManager::DXGIDeviceManager( + Microsoft::WRL::ComPtr<IMFDXGIDeviceManager> mf_dxgi_device_manager, + UINT d3d_device_reset_token) + : mf_dxgi_device_manager_(std::move(mf_dxgi_device_manager)), + d3d_device_reset_token_(d3d_device_reset_token) {} + +DXGIDeviceManager::~DXGIDeviceManager() = default; + +HRESULT DXGIDeviceManager::ResetDevice() { + Microsoft::WRL::ComPtr<ID3D11Device> d3d_device; + constexpr uint32_t kDeviceFlags = + D3D11_CREATE_DEVICE_VIDEO_SUPPORT | D3D11_CREATE_DEVICE_BGRA_SUPPORT; + HRESULT hr = D3D11CreateDevice(nullptr, D3D_DRIVER_TYPE_HARDWARE, nullptr, + kDeviceFlags, nullptr, 0, D3D11_SDK_VERSION, + &d3d_device, nullptr, nullptr); + RETURN_ON_HR_FAILURE(hr, "D3D11 device creation failed", hr); + hr = mf_dxgi_device_manager_->ResetDevice(d3d_device.Get(), + d3d_device_reset_token_); + RETURN_ON_HR_FAILURE(hr, "Failed to reset device on MF DXGI device manager", + hr); + return S_OK; +} + +HRESULT DXGIDeviceManager::RegisterInCaptureEngineAttributes( + IMFAttributes* attributes) { + HRESULT hr = attributes->SetUnknown(MF_CAPTURE_ENGINE_D3D_MANAGER, + mf_dxgi_device_manager_.Get()); + RETURN_ON_HR_FAILURE( + hr, "Failed to set MF_CAPTURE_ENGINE_D3D_MANAGER attribute", hr); + return S_OK; +} + +HRESULT DXGIDeviceManager::RegisterInSourceReaderAttributes( + IMFAttributes* attributes) { + HRESULT hr = attributes->SetUnknown(MF_SOURCE_READER_D3D_MANAGER, + mf_dxgi_device_manager_.Get()); + RETURN_ON_HR_FAILURE( + hr, "Failed to set MF_SOURCE_READER_D3D_MANAGER attribute", hr); + return S_OK; +} + +HRESULT DXGIDeviceManager::RegisterWithMediaSource( + Microsoft::WRL::ComPtr<IMFMediaSource> media_source) { + Microsoft::WRL::ComPtr<IMFMediaSourceEx> source_ext; + HRESULT hr = media_source.As(&source_ext); + RETURN_ON_HR_FAILURE(hr, "Failed to query IMFMediaSourceEx", hr); + hr = source_ext->SetD3DManager(mf_dxgi_device_manager_.Get()); + RETURN_ON_HR_FAILURE(hr, "Failed to set D3D manager", hr); + return S_OK; +} + +Microsoft::WRL::ComPtr<ID3D11Device> DXGIDeviceManager::GetDevice() { + DXGIDeviceScopedHandle device_handle(mf_dxgi_device_manager_.Get()); + return device_handle.GetDevice(); +} + +Microsoft::WRL::ComPtr<IMFDXGIDeviceManager> +DXGIDeviceManager::GetMFDXGIDeviceManager() { + return mf_dxgi_device_manager_; +} + +} // namespace media diff --git a/chromium/media/base/win/dxgi_device_manager.h b/chromium/media/base/win/dxgi_device_manager.h new file mode 100644 index 00000000000..d489b3160c2 --- /dev/null +++ b/chromium/media/base/win/dxgi_device_manager.h @@ -0,0 +1,76 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef MEDIA_BASE_WIN_DXGI_DEVICE_MANAGER_H_ +#define MEDIA_BASE_WIN_DXGI_DEVICE_MANAGER_H_ + +#include <d3d11.h> +#include <mfidl.h> +#include <wrl/client.h> + +#include "base/memory/ref_counted.h" +#include "base/memory/scoped_refptr.h" +#include "media/base/win/mf_initializer_export.h" + +namespace media { + +// Wrap around the usage of device handle from |device_manager|. +class MF_INITIALIZER_EXPORT DXGIDeviceScopedHandle { + public: + explicit DXGIDeviceScopedHandle(IMFDXGIDeviceManager* device_manager); + DXGIDeviceScopedHandle(const DXGIDeviceScopedHandle&) = delete; + DXGIDeviceScopedHandle& operator=(const DXGIDeviceScopedHandle&) = delete; + ~DXGIDeviceScopedHandle(); + + HRESULT LockDevice(REFIID riid, void** device_out); + Microsoft::WRL::ComPtr<ID3D11Device> GetDevice(); + + private: + Microsoft::WRL::ComPtr<IMFDXGIDeviceManager> device_manager_; + + HANDLE device_handle_ = INVALID_HANDLE_VALUE; +}; + +class MF_INITIALIZER_EXPORT DXGIDeviceManager + : public base::RefCounted<DXGIDeviceManager> { + public: + DXGIDeviceManager(const DXGIDeviceManager&) = delete; + DXGIDeviceManager& operator=(const DXGIDeviceManager&) = delete; + + // Returns a DXGIDeviceManager with associated D3D device set, or nullptr on + // failure. + static scoped_refptr<DXGIDeviceManager> Create(); + + // Associates a new D3D device with the DXGI Device Manager + virtual HRESULT ResetDevice(); + + // Registers this manager in capture engine attributes. + HRESULT RegisterInCaptureEngineAttributes(IMFAttributes* attributes); + + // Registers this manager in source reader attributes. + HRESULT RegisterInSourceReaderAttributes(IMFAttributes* attributes); + + // Registers this manager with a media source + HRESULT RegisterWithMediaSource( + Microsoft::WRL::ComPtr<IMFMediaSource> media_source); + + // Directly access D3D device stored in DXGI device manager + virtual Microsoft::WRL::ComPtr<ID3D11Device> GetDevice(); + + Microsoft::WRL::ComPtr<IMFDXGIDeviceManager> GetMFDXGIDeviceManager(); + + protected: + friend class base::RefCounted<DXGIDeviceManager>; + DXGIDeviceManager( + Microsoft::WRL::ComPtr<IMFDXGIDeviceManager> mf_dxgi_device_manager, + UINT d3d_device_reset_token); + virtual ~DXGIDeviceManager(); + + Microsoft::WRL::ComPtr<IMFDXGIDeviceManager> mf_dxgi_device_manager_; + UINT d3d_device_reset_token_ = 0; +}; + +} // namespace media + +#endif // MEDIA_BASE_WIN_DXGI_DEVICE_MANAGER_H_ diff --git a/chromium/media/base/win/dxgi_device_scope_handle_unittest.cc b/chromium/media/base/win/dxgi_device_scope_handle_unittest.cc index 5058d1bfd31..be16636583e 100644 --- a/chromium/media/base/win/dxgi_device_scope_handle_unittest.cc +++ b/chromium/media/base/win/dxgi_device_scope_handle_unittest.cc @@ -7,7 +7,7 @@ #include "base/win/windows_version.h" #include "media/base/test_helpers.h" -#include "media/base/win/mf_helpers.h" +#include "media/base/win/dxgi_device_manager.h" #include "media/base/win/mf_initializer.h" namespace media { @@ -66,7 +66,7 @@ class DXGIDeviceScopedHandleTest : public testing::Test { const bool test_supported_; }; -TEST_F(DXGIDeviceScopedHandleTest, UseDXGIDeviceScopedHandle) { +TEST_F(DXGIDeviceScopedHandleTest, LockDevice) { if (!test_supported_) return; @@ -88,4 +88,24 @@ TEST_F(DXGIDeviceScopedHandleTest, UseDXGIDeviceScopedHandle) { ASSERT_HRESULT_SUCCEEDED(device_handle_3.LockDevice(IID_PPV_ARGS(&device3))); } +TEST_F(DXGIDeviceScopedHandleTest, GetDevice) { + if (!test_supported_) + return; + + { + // Create DXGIDeviceScopedHandle in an inner scope. + DXGIDeviceScopedHandle device_handle_1(dxgi_device_man_.Get()); + } + { + // Create DXGIDeviceScopedHandle in an inner scope with GetDevice call. + DXGIDeviceScopedHandle device_handle_2(dxgi_device_man_.Get()); + ComPtr<ID3D11Device> device2 = device_handle_2.GetDevice(); + EXPECT_NE(device2, nullptr); + } + // Use the device in an outer scope. + DXGIDeviceScopedHandle device_handle_3(dxgi_device_man_.Get()); + ComPtr<ID3D11Device> device3 = device_handle_3.GetDevice(); + EXPECT_NE(device3, nullptr); +} + } // namespace media
\ No newline at end of file diff --git a/chromium/media/base/win/hresult_status_helper.cc b/chromium/media/base/win/hresult_status_helper.cc index e3d6a43ebc0..fe141bb5ac9 100644 --- a/chromium/media/base/win/hresult_status_helper.cc +++ b/chromium/media/base/win/hresult_status_helper.cc @@ -5,6 +5,7 @@ #include "media/base/win/hresult_status_helper.h" #include "base/logging.h" +#include "base/strings/string_util.h" namespace media { @@ -15,8 +16,12 @@ Status HresultToStatus(HRESULT hresult, if (SUCCEEDED(hresult)) return OkStatus(); + std::string sys_err = logging::SystemErrorCodeToString(hresult); + if (!base::IsStringUTF8AllowingNoncharacters(sys_err)) + sys_err = "System error string is invalid"; + return Status(code, message == nullptr ? "HRESULT" : message, location) - .WithData("value", logging::SystemErrorCodeToString(hresult)); + .WithData("value", sys_err); } } // namespace media diff --git a/chromium/media/base/win/mf_helpers.cc b/chromium/media/base/win/mf_helpers.cc index 99a1f83970a..37cf1183709 100644 --- a/chromium/media/base/win/mf_helpers.cc +++ b/chromium/media/base/win/mf_helpers.cc @@ -4,6 +4,8 @@ #include "media/base/win/mf_helpers.h" +#include <d3d11.h> + #include "base/check_op.h" namespace media { @@ -51,34 +53,6 @@ MediaBufferScopedPointer::~MediaBufferScopedPointer() { CHECK(SUCCEEDED(hr)); } -DXGIDeviceScopedHandle::DXGIDeviceScopedHandle( - IMFDXGIDeviceManager* device_manager) - : device_manager_(device_manager) {} - -DXGIDeviceScopedHandle::~DXGIDeviceScopedHandle() { - if (device_handle_ != INVALID_HANDLE_VALUE) { - HRESULT hr = device_manager_->CloseDeviceHandle(device_handle_); - CHECK(SUCCEEDED(hr)); - device_handle_ = INVALID_HANDLE_VALUE; - } -} - -HRESULT DXGIDeviceScopedHandle::LockDevice(REFIID riid, void** device_out) { - HRESULT hr; - if (device_handle_ == INVALID_HANDLE_VALUE) { - hr = device_manager_->OpenDeviceHandle(&device_handle_); - if (FAILED(hr)) { - return hr; - } - } - // see - // https://docs.microsoft.com/en-us/windows/win32/api/mfobjects/nf-mfobjects-imfdxgidevicemanager-lockdevice - // for details of LockDevice call. - hr = device_manager_->LockDevice(device_handle_, riid, device_out, - /*block=*/FALSE); - return hr; -} - HRESULT CopyCoTaskMemWideString(LPCWSTR in_string, LPWSTR* out_string) { if (!in_string || !out_string) { return E_INVALIDARG; @@ -94,4 +68,10 @@ HRESULT CopyCoTaskMemWideString(LPCWSTR in_string, LPWSTR* out_string) { return S_OK; } +HRESULT SetDebugName(ID3D11DeviceChild* d3d11_device_child, + const char* debug_string) { + return d3d11_device_child->SetPrivateData(WKPDID_D3DDebugObjectName, + strlen(debug_string), debug_string); +} + } // namespace media diff --git a/chromium/media/base/win/mf_helpers.h b/chromium/media/base/win/mf_helpers.h index aa56e5e437e..6bebf0310f5 100644 --- a/chromium/media/base/win/mf_helpers.h +++ b/chromium/media/base/win/mf_helpers.h @@ -13,6 +13,8 @@ #include "base/macros.h" #include "media/base/win/mf_initializer_export.h" +struct ID3D11DeviceChild; + namespace media { // Helper function to print HRESULT to std::string. @@ -62,6 +64,7 @@ class MF_INITIALIZER_EXPORT MediaBufferScopedPointer { uint8_t* get() { return buffer_; } DWORD current_length() const { return current_length_; } + DWORD max_length() const { return max_length_; } private: Microsoft::WRL::ComPtr<IMFMediaBuffer> media_buffer_; @@ -72,24 +75,14 @@ class MF_INITIALIZER_EXPORT MediaBufferScopedPointer { DISALLOW_COPY_AND_ASSIGN(MediaBufferScopedPointer); }; -// Wrap around the usage of device handle from |device_manager|. -class MF_INITIALIZER_EXPORT DXGIDeviceScopedHandle { - public: - explicit DXGIDeviceScopedHandle(IMFDXGIDeviceManager* device_manager); - ~DXGIDeviceScopedHandle(); - - HRESULT LockDevice(REFIID riid, void** device_out); - - private: - Microsoft::WRL::ComPtr<IMFDXGIDeviceManager> device_manager_; - - HANDLE device_handle_ = INVALID_HANDLE_VALUE; -}; - // Copies |in_string| to |out_string| that is allocated with CoTaskMemAlloc(). MF_INITIALIZER_EXPORT HRESULT CopyCoTaskMemWideString(LPCWSTR in_string, LPWSTR* out_string); +// Set the debug name of a D3D11 resource for use with ETW debugging tools. +// D3D11 retains the string passed to this function. +MF_INITIALIZER_EXPORT HRESULT +SetDebugName(ID3D11DeviceChild* d3d11_device_child, const char* debug_string); } // namespace media #endif // MEDIA_BASE_WIN_MF_HELPERS_H_ |