diff options
Diffstat (limited to 'chromium/content/browser/media')
73 files changed, 6058 insertions, 1091 deletions
diff --git a/chromium/content/browser/media/OWNERS b/chromium/content/browser/media/OWNERS index 7d1710685d7..966a9171d93 100644 --- a/chromium/content/browser/media/OWNERS +++ b/chromium/content/browser/media/OWNERS @@ -1,11 +1,9 @@ file://media/OWNERS olka@chromium.org maxmorin@chromium.org +miu@chromium.org per-file media_devices_*=guidou@chromium.org per-file midi_*=toyoshim@chromium.org -# For changes related to the tab media indicators. -per-file audio_stream_monitor*=miu@chromium.org - # COMPONENT: Internals>Media diff --git a/chromium/content/browser/media/android/media_resource_getter_impl.cc b/chromium/content/browser/media/android/media_resource_getter_impl.cc index 73d5d89a975..2633945e003 100644 --- a/chromium/content/browser/media/android/media_resource_getter_impl.cc +++ b/chromium/content/browser/media/android/media_resource_getter_impl.cc @@ -7,6 +7,7 @@ #include "base/bind.h" #include "base/macros.h" #include "base/path_service.h" +#include "base/single_thread_task_runner.h" #include "base/task_scheduler/post_task.h" #include "content/browser/child_process_security_policy_impl.h" #include "content/browser/fileapi/browser_file_system_helper.h" @@ -97,7 +98,7 @@ static void RequestPlaformPathFromFileSystemURL( url, &platform_path); base::FilePath data_storage_path; - PathService::Get(base::DIR_ANDROID_APP_DATA, &data_storage_path); + base::PathService::Get(base::DIR_ANDROID_APP_DATA, &data_storage_path); if (data_storage_path.IsParent(platform_path)) ReturnResultOnUIThread(std::move(callback), platform_path.value()); else diff --git a/chromium/content/browser/media/audio_input_stream_broker.cc b/chromium/content/browser/media/audio_input_stream_broker.cc new file mode 100644 index 00000000000..e0a2f97fdda --- /dev/null +++ b/chromium/content/browser/media/audio_input_stream_broker.cc @@ -0,0 +1,219 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/audio_input_stream_broker.h" + +#include <utility> + +#include "base/command_line.h" +#include "base/memory/read_only_shared_memory_region.h" +#include "base/metrics/histogram_macros.h" +#include "base/trace_event/trace_event.h" +#include "content/browser/browser_main_loop.h" +#include "content/browser/media/media_internals.h" +#include "content/public/browser/browser_thread.h" +#include "content/public/browser/content_browser_client.h" +#include "content/public/browser/media_observer.h" +#include "content/public/browser/render_process_host.h" +#include "content/public/common/content_client.h" +#include "media/audio/audio_logging.h" +#include "media/base/media_switches.h" +#include "media/base/user_input_monitor.h" +#include "mojo/public/cpp/system/platform_handle.h" + +#if defined(OS_CHROMEOS) +#include "content/browser/media/keyboard_mic_registration.h" +#endif + +namespace content { + +AudioInputStreamBroker::AudioInputStreamBroker( + int render_process_id, + int render_frame_id, + const std::string& device_id, + const media::AudioParameters& params, + uint32_t shared_memory_count, + bool enable_agc, + AudioStreamBroker::DeleterCallback deleter, + mojom::RendererAudioInputStreamFactoryClientPtr renderer_factory_client) + : AudioStreamBroker(render_process_id, render_frame_id), + device_id_(device_id), + params_(params), + shared_memory_count_(shared_memory_count), + enable_agc_(enable_agc), + deleter_(std::move(deleter)), + renderer_factory_client_(std::move(renderer_factory_client)), + observer_binding_(this), + weak_ptr_factory_(this) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + DCHECK(renderer_factory_client_); + DCHECK(deleter_); + TRACE_EVENT_NESTABLE_ASYNC_BEGIN0("audio", "AudioInputStreamBroker", this); + + // Unretained is safe because |this| owns |renderer_factory_client_|. + renderer_factory_client_.set_connection_error_handler( + base::BindOnce(&AudioInputStreamBroker::Cleanup, base::Unretained(this))); + + // Notify RenderProcessHost about input stream so the renderer is not + // background. + auto* process_host = RenderProcessHost::FromID(render_process_id); + if (process_host) + process_host->OnMediaStreamAdded(); + + if (base::CommandLine::ForCurrentProcess()->HasSwitch( + switches::kUseFakeDeviceForMediaStream)) { + params_.set_format(media::AudioParameters::AUDIO_FAKE); + } + + BrowserMainLoop* browser_main_loop = BrowserMainLoop::GetInstance(); + // May be null in unit tests. + if (!browser_main_loop) + return; + +#if defined(OS_CHROMEOS) + if (params_.channel_layout() == + media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC) { + browser_main_loop->keyboard_mic_registration()->Register(); + } +#else + user_input_monitor_ = static_cast<media::UserInputMonitorBase*>( + browser_main_loop->user_input_monitor()); +#endif +} + +AudioInputStreamBroker::~AudioInputStreamBroker() { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + +#if defined(OS_CHROMEOS) + if (params_.channel_layout() == + media::CHANNEL_LAYOUT_STEREO_AND_KEYBOARD_MIC) { + BrowserMainLoop* browser_main_loop = BrowserMainLoop::GetInstance(); + + // May be null in unit tests. + if (browser_main_loop) + browser_main_loop->keyboard_mic_registration()->Deregister(); + } +#else + if (user_input_monitor_) + user_input_monitor_->DisableKeyPressMonitoring(); +#endif + + auto* process_host = RenderProcessHost::FromID(render_process_id()); + if (process_host) + process_host->OnMediaStreamRemoved(); + + // TODO(https://crbug.com/829317) update tab recording indicator. + + if (awaiting_created_) { + TRACE_EVENT_NESTABLE_ASYNC_END1("audio", "CreateStream", this, "success", + "failed or cancelled"); + } + TRACE_EVENT_NESTABLE_ASYNC_END1("audio", "AudioInputStreamBroker", this, + "disconnect reason", + static_cast<uint32_t>(disconnect_reason_)); + + UMA_HISTOGRAM_ENUMERATION("Media.Audio.Capture.StreamBrokerDisconnectReason", + disconnect_reason_); +} + +void AudioInputStreamBroker::CreateStream( + audio::mojom::StreamFactory* factory) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + DCHECK(!observer_binding_.is_bound()); + DCHECK(!client_request_); + TRACE_EVENT_NESTABLE_ASYNC_BEGIN1("audio", "CreateStream", this, "device id", + device_id_); + awaiting_created_ = true; + + base::ReadOnlySharedMemoryRegion key_press_count_buffer; + if (user_input_monitor_) { + key_press_count_buffer = + user_input_monitor_->EnableKeyPressMonitoringWithMapping(); + } + + media::mojom::AudioInputStreamClientPtr client; + client_request_ = mojo::MakeRequest(&client); + + media::mojom::AudioInputStreamPtr stream; + media::mojom::AudioInputStreamRequest stream_request = + mojo::MakeRequest(&stream); + + media::mojom::AudioInputStreamObserverPtr observer_ptr; + observer_binding_.Bind(mojo::MakeRequest(&observer_ptr)); + + // Unretained is safe because |this| owns |observer_binding_|. + observer_binding_.set_connection_error_with_reason_handler(base::BindOnce( + &AudioInputStreamBroker::ObserverBindingLost, base::Unretained(this))); + + // Note that the component id for AudioLog is used to differentiate between + // several users of the same audio log. Since this audio log is for a single + // stream, the component id used doesn't matter. + // TODO(https://crbug.com/836226) pass valid user input monitor handle when + // switching to audio service input streams. + constexpr int log_component_id = 0; + factory->CreateInputStream( + std::move(stream_request), std::move(client), std::move(observer_ptr), + MediaInternals::GetInstance()->CreateMojoAudioLog( + media::AudioLogFactory::AudioComponent::AUDIO_INPUT_CONTROLLER, + log_component_id, render_process_id(), render_frame_id()), + device_id_, params_, shared_memory_count_, enable_agc_, + mojo::WrapReadOnlySharedMemoryRegion(std::move(key_press_count_buffer)), + base::BindOnce(&AudioInputStreamBroker::StreamCreated, + weak_ptr_factory_.GetWeakPtr(), std::move(stream))); +} + +void AudioInputStreamBroker::DidStartRecording() { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + // TODO(https://crbug.com/829317) update tab recording indicator. +} + +void AudioInputStreamBroker::StreamCreated( + media::mojom::AudioInputStreamPtr stream, + media::mojom::AudioDataPipePtr data_pipe, + bool initially_muted, + const base::Optional<base::UnguessableToken>& stream_id) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + awaiting_created_ = false; + TRACE_EVENT_NESTABLE_ASYNC_END1("audio", "CreateStream", this, "success", + !!data_pipe); + + if (!data_pipe) { + disconnect_reason_ = media::mojom::AudioInputStreamObserver:: + DisconnectReason::kStreamCreationFailed; + Cleanup(); + return; + } + + DCHECK(stream_id.has_value()); + DCHECK(renderer_factory_client_); + renderer_factory_client_->StreamCreated( + std::move(stream), std::move(client_request_), std::move(data_pipe), + initially_muted, stream_id); +} +void AudioInputStreamBroker::ObserverBindingLost( + uint32_t reason, + const std::string& description) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + const uint32_t maxValidReason = static_cast<uint32_t>( + media::mojom::AudioInputStreamObserver::DisconnectReason::kMaxValue); + if (reason > maxValidReason) { + DLOG(ERROR) << "Invalid reason: " << reason; + } else if (disconnect_reason_ == media::mojom::AudioInputStreamObserver:: + DisconnectReason::kDocumentDestroyed) { + disconnect_reason_ = + static_cast<media::mojom::AudioInputStreamObserver::DisconnectReason>( + reason); + } + + Cleanup(); +} + +void AudioInputStreamBroker::Cleanup() { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + std::move(deleter_).Run(this); +} + +} // namespace content diff --git a/chromium/content/browser/media/audio_input_stream_broker.h b/chromium/content/browser/media/audio_input_stream_broker.h new file mode 100644 index 00000000000..524fb3408e3 --- /dev/null +++ b/chromium/content/browser/media/audio_input_stream_broker.h @@ -0,0 +1,86 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONTENT_BROWSER_MEDIA_AUDIO_INPUT_STREAM_BROKER_H_ +#define CONTENT_BROWSER_MEDIA_AUDIO_INPUT_STREAM_BROKER_H_ + +#include <string> + +#include "base/memory/weak_ptr.h" +#include "base/sequence_checker.h" +#include "content/browser/media/audio_stream_broker.h" +#include "content/common/content_export.h" +#include "content/common/media/renderer_audio_input_stream_factory.mojom.h" +#include "media/base/audio_parameters.h" +#include "media/mojo/interfaces/audio_input_stream.mojom.h" +#include "mojo/public/cpp/bindings/binding.h" +#include "services/audio/public/mojom/stream_factory.mojom.h" + +namespace media { +class UserInputMonitorBase; +} + +namespace content { + +// AudioInputStreamBroker is used to broker a connection between a client +// (typically renderer) and the audio service. It is operated on the UI thread. +class CONTENT_EXPORT AudioInputStreamBroker final + : public AudioStreamBroker, + public media::mojom::AudioInputStreamObserver { + public: + AudioInputStreamBroker( + int render_process_id, + int render_frame_id, + const std::string& device_id, + const media::AudioParameters& params, + uint32_t shared_memory_count, + bool enable_agc, + AudioStreamBroker::DeleterCallback deleter, + mojom::RendererAudioInputStreamFactoryClientPtr renderer_factory_client); + + ~AudioInputStreamBroker() final; + + // Creates the stream. + void CreateStream(audio::mojom::StreamFactory* factory) final; + + // media::AudioInputStreamObserver implementation. + void DidStartRecording() final; + + private: + void StreamCreated(media::mojom::AudioInputStreamPtr stream, + media::mojom::AudioDataPipePtr data_pipe, + bool initially_muted, + const base::Optional<base::UnguessableToken>& stream_id); + + void ObserverBindingLost(uint32_t reason, const std::string& description); + + void Cleanup(); + + const std::string device_id_; + media::AudioParameters params_; + const uint32_t shared_memory_count_; + const bool enable_agc_; + media::UserInputMonitorBase* user_input_monitor_ = nullptr; + + // Indicates that CreateStream has been called, but not StreamCreated. + bool awaiting_created_ = false; + + DeleterCallback deleter_; + + mojom::RendererAudioInputStreamFactoryClientPtr renderer_factory_client_; + mojo::Binding<AudioInputStreamObserver> observer_binding_; + media::mojom::AudioInputStreamClientRequest client_request_; + + media::mojom::AudioInputStreamObserver::DisconnectReason disconnect_reason_ = + media::mojom::AudioInputStreamObserver::DisconnectReason:: + kDocumentDestroyed; + + base::WeakPtrFactory<AudioInputStreamBroker> weak_ptr_factory_; + + DISALLOW_COPY_AND_ASSIGN(AudioInputStreamBroker); +}; + +} // namespace content + +#endif // CONTENT_BROWSER_MEDIA_AUDIO_INPUT_STREAM_BROKER_H_ diff --git a/chromium/content/browser/media/audio_input_stream_broker_unittest.cc b/chromium/content/browser/media/audio_input_stream_broker_unittest.cc new file mode 100644 index 00000000000..809f63f1504 --- /dev/null +++ b/chromium/content/browser/media/audio_input_stream_broker_unittest.cc @@ -0,0 +1,278 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/audio_input_stream_broker.h" + +#include <memory> +#include <utility> + +#include "base/sync_socket.h" +#include "base/test/mock_callback.h" +#include "content/public/test/test_browser_thread_bundle.h" +#include "media/mojo/interfaces/audio_input_stream.mojom.h" +#include "mojo/public/cpp/system/platform_handle.h" +#include "services/audio/public/cpp/fake_stream_factory.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using ::testing::Test; +using ::testing::Mock; +using ::testing::StrictMock; +using ::testing::InSequence; + +namespace content { + +namespace { + +const int kRenderProcessId = 123; +const int kRenderFrameId = 234; +const uint32_t kShMemCount = 10; +const bool kEnableAgc = false; +const char kDeviceId[] = "testdeviceid"; +const bool kInitiallyMuted = false; + +media::AudioParameters TestParams() { + return media::AudioParameters::UnavailableDeviceParams(); +} + +using MockDeleterCallback = StrictMock< + base::MockCallback<base::OnceCallback<void(AudioStreamBroker*)>>>; + +class MockRendererAudioInputStreamFactoryClient + : public mojom::RendererAudioInputStreamFactoryClient { + public: + MockRendererAudioInputStreamFactoryClient() : binding_(this) {} + ~MockRendererAudioInputStreamFactoryClient() override {} + + mojom::RendererAudioInputStreamFactoryClientPtr MakePtr() { + mojom::RendererAudioInputStreamFactoryClientPtr ret; + binding_.Bind(mojo::MakeRequest(&ret)); + return ret; + } + + MOCK_METHOD0(OnStreamCreated, void()); + + void StreamCreated( + media::mojom::AudioInputStreamPtr input_stream, + media::mojom::AudioInputStreamClientRequest client_request, + media::mojom::AudioDataPipePtr data_pipe, + bool initially_muted, + const base::Optional<base::UnguessableToken>& stream_id) override { + EXPECT_TRUE(stream_id.has_value()); + input_stream_ = std::move(input_stream); + client_request_ = std::move(client_request); + OnStreamCreated(); + } + + void CloseBinding() { binding_.Close(); } + + private: + mojo::Binding<mojom::RendererAudioInputStreamFactoryClient> binding_; + media::mojom::AudioInputStreamPtr input_stream_; + media::mojom::AudioInputStreamClientRequest client_request_; + DISALLOW_COPY_AND_ASSIGN(MockRendererAudioInputStreamFactoryClient); +}; + +class MockStreamFactory : public audio::FakeStreamFactory { + public: + MockStreamFactory() {} + ~MockStreamFactory() final {} + + // State of an expected stream creation. |device_id| and |params| are set + // ahead of time and verified during request. The other fields are filled in + // when the request is received. + struct StreamRequestData { + StreamRequestData(const std::string& device_id, + const media::AudioParameters& params) + : device_id(device_id), params(params) {} + + bool requested = false; + media::mojom::AudioInputStreamRequest stream_request; + media::mojom::AudioInputStreamClientPtr client; + media::mojom::AudioInputStreamObserverPtr observer; + media::mojom::AudioLogPtr log; + const std::string device_id; + const media::AudioParameters params; + uint32_t shared_memory_count; + bool enable_agc; + mojo::ScopedSharedBufferHandle key_press_count_buffer; + CreateInputStreamCallback created_callback; + }; + + void ExpectStreamCreation(StreamRequestData* ex) { + stream_request_data_ = ex; + } + + private: + void CreateInputStream(media::mojom::AudioInputStreamRequest stream_request, + media::mojom::AudioInputStreamClientPtr client, + media::mojom::AudioInputStreamObserverPtr observer, + media::mojom::AudioLogPtr log, + const std::string& device_id, + const media::AudioParameters& params, + uint32_t shared_memory_count, + bool enable_agc, + mojo::ScopedSharedBufferHandle key_press_count_buffer, + CreateInputStreamCallback created_callback) final { + // No way to cleanly exit the test here in case of failure, so use CHECK. + CHECK(stream_request_data_); + EXPECT_EQ(stream_request_data_->device_id, device_id); + EXPECT_TRUE(stream_request_data_->params.Equals(params)); + stream_request_data_->requested = true; + stream_request_data_->stream_request = std::move(stream_request); + stream_request_data_->client = std::move(client); + stream_request_data_->observer = std::move(observer); + stream_request_data_->log = std::move(log); + stream_request_data_->shared_memory_count = shared_memory_count; + stream_request_data_->enable_agc = enable_agc; + stream_request_data_->key_press_count_buffer = + std::move(key_press_count_buffer); + stream_request_data_->created_callback = std::move(created_callback); + } + + StreamRequestData* stream_request_data_; + DISALLOW_COPY_AND_ASSIGN(MockStreamFactory); +}; + +struct TestEnvironment { + TestEnvironment() + : broker(std::make_unique<AudioInputStreamBroker>( + kRenderProcessId, + kRenderFrameId, + kDeviceId, + TestParams(), + kShMemCount, + kEnableAgc, + deleter.Get(), + renderer_factory_client.MakePtr())) {} + + void RunUntilIdle() { thread_bundle.RunUntilIdle(); } + + TestBrowserThreadBundle thread_bundle; + MockDeleterCallback deleter; + StrictMock<MockRendererAudioInputStreamFactoryClient> renderer_factory_client; + std::unique_ptr<AudioInputStreamBroker> broker; + MockStreamFactory stream_factory; + audio::mojom::StreamFactoryPtr factory_ptr = stream_factory.MakePtr(); +}; + +} // namespace + +TEST(AudioInputStreamBrokerTest, StoresProcessAndFrameId) { + TestBrowserThreadBundle thread_bundle; + MockDeleterCallback deleter; + StrictMock<MockRendererAudioInputStreamFactoryClient> renderer_factory_client; + + AudioInputStreamBroker broker( + kRenderProcessId, kRenderFrameId, kDeviceId, TestParams(), kShMemCount, + kEnableAgc, deleter.Get(), renderer_factory_client.MakePtr()); + + EXPECT_EQ(kRenderProcessId, broker.render_process_id()); + EXPECT_EQ(kRenderFrameId, broker.render_frame_id()); +} + +TEST(AudioInputStreamBrokerTest, StreamCreationSuccess_Propagates) { + TestEnvironment env; + MockStreamFactory::StreamRequestData stream_request_data(kDeviceId, + TestParams()); + env.stream_factory.ExpectStreamCreation(&stream_request_data); + + env.broker->CreateStream(env.factory_ptr.get()); + env.RunUntilIdle(); + + EXPECT_TRUE(stream_request_data.requested); + + // Set up test IPC primitives. + const size_t shmem_size = 456; + base::SyncSocket socket1, socket2; + base::SyncSocket::CreatePair(&socket1, &socket2); + std::move(stream_request_data.created_callback) + .Run({base::in_place, mojo::SharedBufferHandle::Create(shmem_size), + mojo::WrapPlatformFile(socket1.Release())}, + kInitiallyMuted, base::UnguessableToken::Create()); + + EXPECT_CALL(env.renderer_factory_client, OnStreamCreated()); + + env.RunUntilIdle(); + + Mock::VerifyAndClear(&env.renderer_factory_client); + + env.broker.reset(); +} + +TEST(AudioInputStreamBrokerTest, StreamCreationFailure_CallsDeleter) { + TestEnvironment env; + MockStreamFactory::StreamRequestData stream_request_data(kDeviceId, + TestParams()); + env.stream_factory.ExpectStreamCreation(&stream_request_data); + + env.broker->CreateStream(env.factory_ptr.get()); + env.RunUntilIdle(); + + EXPECT_TRUE(stream_request_data.requested); + EXPECT_CALL(env.deleter, Run(env.broker.release())) + .WillOnce(testing::DeleteArg<0>()); + + std::move(stream_request_data.created_callback) + .Run(nullptr, kInitiallyMuted, base::nullopt); + + env.RunUntilIdle(); +} + +TEST(AudioInputStreamBrokerTest, RendererFactoryClientDisconnect_CallsDeleter) { + InSequence seq; + TestEnvironment env; + MockStreamFactory::StreamRequestData stream_request_data(kDeviceId, + TestParams()); + env.stream_factory.ExpectStreamCreation(&stream_request_data); + + env.broker->CreateStream(env.factory_ptr.get()); + env.RunUntilIdle(); + EXPECT_TRUE(stream_request_data.requested); + + EXPECT_CALL(env.deleter, Run(env.broker.release())) + .WillOnce(testing::DeleteArg<0>()); + env.renderer_factory_client.CloseBinding(); + env.RunUntilIdle(); + Mock::VerifyAndClear(&env.deleter); + + env.stream_factory.CloseBinding(); + env.RunUntilIdle(); +} + +TEST(AudioInputStreamBrokerTest, ObserverDisconnect_CallsDeleter) { + InSequence seq; + TestEnvironment env; + MockStreamFactory::StreamRequestData stream_request_data(kDeviceId, + TestParams()); + env.stream_factory.ExpectStreamCreation(&stream_request_data); + + env.broker->CreateStream(env.factory_ptr.get()); + env.RunUntilIdle(); + EXPECT_TRUE(stream_request_data.requested); + + EXPECT_CALL(env.deleter, Run(env.broker.release())) + .WillOnce(testing::DeleteArg<0>()); + stream_request_data.observer.reset(); + env.RunUntilIdle(); + Mock::VerifyAndClear(&env.deleter); + + env.stream_factory.CloseBinding(); + env.RunUntilIdle(); +} + +TEST(AudioInputStreamBrokerTest, + FactoryDisconnectDuringConstruction_CallsDeleter) { + TestEnvironment env; + + env.broker->CreateStream(env.factory_ptr.get()); + env.stream_factory.CloseBinding(); + + EXPECT_CALL(env.deleter, Run(env.broker.release())) + .WillOnce(testing::DeleteArg<0>()); + + env.RunUntilIdle(); +} + +} // namespace content diff --git a/chromium/content/browser/media/audio_loopback_stream_broker.cc b/chromium/content/browser/media/audio_loopback_stream_broker.cc new file mode 100644 index 00000000000..d7cba0187ea --- /dev/null +++ b/chromium/content/browser/media/audio_loopback_stream_broker.cc @@ -0,0 +1,164 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/audio_loopback_stream_broker.h" + +#include <utility> + +#include "base/unguessable_token.h" +#include "content/browser/web_contents/web_contents_impl.h" +#include "content/public/browser/browser_thread.h" +#include "content/public/browser/render_process_host.h" + +namespace content { + +AudioStreamBrokerFactory::LoopbackSource::LoopbackSource() = default; + +AudioStreamBrokerFactory::LoopbackSource::LoopbackSource( + WebContents* source_contents) + : WebContentsObserver(source_contents) { + DCHECK(source_contents); +} + +AudioStreamBrokerFactory::LoopbackSource::~LoopbackSource() = default; + +base::UnguessableToken AudioStreamBrokerFactory::LoopbackSource::GetGroupID() { + if (WebContentsImpl* source_contents = + static_cast<WebContentsImpl*>(web_contents())) { + return source_contents->GetAudioStreamFactory()->group_id(); + } + return base::UnguessableToken(); +} + +void AudioStreamBrokerFactory::LoopbackSource::OnStartCapturing() { + if (WebContentsImpl* source_contents = + static_cast<WebContentsImpl*>(web_contents())) { + source_contents->IncrementCapturerCount(gfx::Size()); + } +} + +void AudioStreamBrokerFactory::LoopbackSource::OnStopCapturing() { + if (WebContentsImpl* source_contents = + static_cast<WebContentsImpl*>(web_contents())) { + source_contents->DecrementCapturerCount(); + } +} + +void AudioStreamBrokerFactory::LoopbackSource::WebContentsDestroyed() { + if (on_gone_closure_) + std::move(on_gone_closure_).Run(); +} + +AudioLoopbackStreamBroker::AudioLoopbackStreamBroker( + int render_process_id, + int render_frame_id, + std::unique_ptr<AudioStreamBrokerFactory::LoopbackSource> source, + const media::AudioParameters& params, + uint32_t shared_memory_count, + bool mute_source, + AudioStreamBroker::DeleterCallback deleter, + mojom::RendererAudioInputStreamFactoryClientPtr renderer_factory_client) + : AudioStreamBroker(render_process_id, render_frame_id), + source_(std::move(source)), + params_(params), + shared_memory_count_(shared_memory_count), + deleter_(std::move(deleter)), + renderer_factory_client_(std::move(renderer_factory_client)), + observer_binding_(this), + weak_ptr_factory_(this) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + DCHECK(source_); + DCHECK(source_->GetGroupID()); + DCHECK(renderer_factory_client_); + DCHECK(deleter_); + + // Unretained is safe because |this| owns |source_|. + source_->set_on_gone_closure(base::BindOnce( + &AudioLoopbackStreamBroker::Cleanup, base::Unretained(this))); + + if (mute_source) { + muter_.emplace(source_->GetGroupID()); + } + + // Unretained is safe because |this| owns |renderer_factory_client_|. + renderer_factory_client_.set_connection_error_handler(base::BindOnce( + &AudioLoopbackStreamBroker::Cleanup, base::Unretained(this))); + + // Notify the source that we are capturing from it, to prevent its + // backgrounding. + source_->OnStartCapturing(); + + // Notify RenderProcessHost about the input stream, so that the destination + // renderer does not get background. + if (auto* process_host = RenderProcessHost::FromID(render_process_id)) + process_host->OnMediaStreamAdded(); +} + +AudioLoopbackStreamBroker::~AudioLoopbackStreamBroker() { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + source_->OnStopCapturing(); + + if (auto* process_host = RenderProcessHost::FromID(render_process_id())) + process_host->OnMediaStreamRemoved(); +} + +void AudioLoopbackStreamBroker::CreateStream( + audio::mojom::StreamFactory* factory) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + DCHECK(!observer_binding_.is_bound()); + DCHECK(!client_request_); + DCHECK(source_->GetGroupID()); + + if (muter_) // Mute the source. + muter_->Connect(factory); + + media::mojom::AudioInputStreamClientPtr client; + client_request_ = mojo::MakeRequest(&client); + + media::mojom::AudioInputStreamPtr stream; + media::mojom::AudioInputStreamRequest stream_request = + mojo::MakeRequest(&stream); + + media::mojom::AudioInputStreamObserverPtr observer_ptr; + observer_binding_.Bind(mojo::MakeRequest(&observer_ptr)); + + // Unretained is safe because |this| owns |observer_binding_|. + observer_binding_.set_connection_error_handler(base::BindOnce( + &AudioLoopbackStreamBroker::Cleanup, base::Unretained(this))); + + factory->CreateLoopbackStream( + std::move(stream_request), std::move(client), std::move(observer_ptr), + params_, shared_memory_count_, source_->GetGroupID(), + base::BindOnce(&AudioLoopbackStreamBroker::StreamCreated, + weak_ptr_factory_.GetWeakPtr(), std::move(stream))); +} + +void AudioLoopbackStreamBroker::DidStartRecording() { + DCHECK_CURRENTLY_ON(BrowserThread::UI); +} + +void AudioLoopbackStreamBroker::StreamCreated( + media::mojom::AudioInputStreamPtr stream, + media::mojom::AudioDataPipePtr data_pipe) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + if (!data_pipe) { + Cleanup(); + return; + } + + DCHECK(renderer_factory_client_); + renderer_factory_client_->StreamCreated( + std::move(stream), std::move(client_request_), std::move(data_pipe), + false /* |initially_muted|: Loopback streams are never muted. */, + base::nullopt /* |stream_id|: Loopback streams don't have ids */); +} + +void AudioLoopbackStreamBroker::Cleanup() { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + std::move(deleter_).Run(this); +} + +} // namespace content diff --git a/chromium/content/browser/media/audio_loopback_stream_broker.h b/chromium/content/browser/media/audio_loopback_stream_broker.h new file mode 100644 index 00000000000..71778564a65 --- /dev/null +++ b/chromium/content/browser/media/audio_loopback_stream_broker.h @@ -0,0 +1,74 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONTENT_BROWSER_MEDIA_AUDIO_LOOPBACK_STREAM_BROKER_H_ +#define CONTENT_BROWSER_MEDIA_AUDIO_LOOPBACK_STREAM_BROKER_H_ + +#include <string> + +#include "base/memory/weak_ptr.h" +#include "base/optional.h" +#include "base/sequence_checker.h" +#include "content/browser/media/audio_muting_session.h" +#include "content/browser/media/audio_stream_broker.h" +#include "content/common/content_export.h" +#include "content/common/media/renderer_audio_input_stream_factory.mojom.h" +#include "media/base/audio_parameters.h" +#include "media/mojo/interfaces/audio_input_stream.mojom.h" +#include "mojo/public/cpp/bindings/binding.h" +#include "services/audio/public/mojom/stream_factory.mojom.h" + +namespace content { + +// AudioLoopbackStreamBroker is used to broker a connection between a client +// (typically renderer) and the audio service. It is operated on the UI thread. +class CONTENT_EXPORT AudioLoopbackStreamBroker final + : public AudioStreamBroker, + public media::mojom::AudioInputStreamObserver { + public: + AudioLoopbackStreamBroker( + int render_process_id, + int render_frame_id, + std::unique_ptr<AudioStreamBrokerFactory::LoopbackSource> source, + const media::AudioParameters& params, + uint32_t shared_memory_count, + bool mute_source, + AudioStreamBroker::DeleterCallback deleter, + mojom::RendererAudioInputStreamFactoryClientPtr renderer_factory_client); + + ~AudioLoopbackStreamBroker() final; + + // Creates the stream. + void CreateStream(audio::mojom::StreamFactory* factory) final; + + // media::AudioInputStreamObserver implementation. + void DidStartRecording() final; + + private: + void StreamCreated(media::mojom::AudioInputStreamPtr stream, + media::mojom::AudioDataPipePtr data_pipe); + void Cleanup(); + + const std::unique_ptr<AudioStreamBrokerFactory::LoopbackSource> source_; + const media::AudioParameters params_; + const uint32_t shared_memory_count_; + + DeleterCallback deleter_; + + // Constructed only if the loopback source playback should be muted while the + // loopback stream is running. + base::Optional<AudioMutingSession> muter_; + + mojom::RendererAudioInputStreamFactoryClientPtr renderer_factory_client_; + mojo::Binding<AudioInputStreamObserver> observer_binding_; + media::mojom::AudioInputStreamClientRequest client_request_; + + base::WeakPtrFactory<AudioLoopbackStreamBroker> weak_ptr_factory_; + + DISALLOW_COPY_AND_ASSIGN(AudioLoopbackStreamBroker); +}; + +} // namespace content + +#endif // CONTENT_BROWSER_MEDIA_AUDIO_LOOPBACK_STREAM_BROKER_H_ diff --git a/chromium/content/browser/media/audio_loopback_stream_broker_unittest.cc b/chromium/content/browser/media/audio_loopback_stream_broker_unittest.cc new file mode 100644 index 00000000000..cfe6cc7ccb3 --- /dev/null +++ b/chromium/content/browser/media/audio_loopback_stream_broker_unittest.cc @@ -0,0 +1,374 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/audio_loopback_stream_broker.h" + +#include <memory> +#include <utility> + +#include "base/sync_socket.h" +#include "base/test/mock_callback.h" +#include "content/public/test/test_browser_thread_bundle.h" +#include "media/mojo/interfaces/audio_input_stream.mojom.h" +#include "mojo/public/cpp/system/platform_handle.h" +#include "services/audio/public/cpp/fake_stream_factory.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using ::testing::_; +using ::testing::Test; +using ::testing::Mock; +using ::testing::NiceMock; +using ::testing::StrictMock; +using ::testing::InSequence; + +namespace content { + +namespace { + +const int kRenderProcessId = 123; +const int kRenderFrameId = 234; +const uint32_t kShMemCount = 10; + +media::AudioParameters TestParams() { + return media::AudioParameters::UnavailableDeviceParams(); +} + +class MockSource : public AudioStreamBrokerFactory::LoopbackSource { + public: + explicit MockSource(const base::UnguessableToken& group_id) + : group_id_(group_id) {} + ~MockSource() override {} + + // AudioStreamBrokerFactory::LoopbackSource mocking. + base::UnguessableToken GetGroupID() override { return group_id_; } + MOCK_METHOD0(OnStartCapturing, void(void)); + MOCK_METHOD0(OnStopCapturing, void(void)); + + private: + base::UnguessableToken group_id_; + DISALLOW_COPY_AND_ASSIGN(MockSource); +}; + +using MockDeleterCallback = StrictMock< + base::MockCallback<base::OnceCallback<void(AudioStreamBroker*)>>>; + +class MockRendererAudioInputStreamFactoryClient + : public mojom::RendererAudioInputStreamFactoryClient { + public: + MockRendererAudioInputStreamFactoryClient() : binding_(this) {} + ~MockRendererAudioInputStreamFactoryClient() override {} + + mojom::RendererAudioInputStreamFactoryClientPtr MakePtr() { + mojom::RendererAudioInputStreamFactoryClientPtr ret; + binding_.Bind(mojo::MakeRequest(&ret)); + return ret; + } + + MOCK_METHOD0(OnStreamCreated, void()); + + void StreamCreated( + media::mojom::AudioInputStreamPtr input_stream, + media::mojom::AudioInputStreamClientRequest client_request, + media::mojom::AudioDataPipePtr data_pipe, + bool initially_muted, + const base::Optional<base::UnguessableToken>& stream_id) override { + // Loopback streams have no stream ids. + EXPECT_FALSE(stream_id.has_value()); + input_stream_ = std::move(input_stream); + client_request_ = std::move(client_request); + OnStreamCreated(); + } + + void CloseBinding() { binding_.Close(); } + + private: + mojo::Binding<mojom::RendererAudioInputStreamFactoryClient> binding_; + media::mojom::AudioInputStreamPtr input_stream_; + media::mojom::AudioInputStreamClientRequest client_request_; +}; + +class MockStreamFactory : public audio::FakeStreamFactory { + public: + MockStreamFactory() {} + ~MockStreamFactory() final {} + + // State of an expected stream creation. |device_id| and |params| are set + // ahead of time and verified during request. The other fields are filled in + // when the request is received. + struct StreamRequestData { + StreamRequestData(const base::UnguessableToken& group_id, + const media::AudioParameters& params) + : params(params), group_id(group_id) {} + + bool requested = false; + media::mojom::AudioInputStreamRequest stream_request; + media::mojom::AudioInputStreamClientPtr client; + media::mojom::AudioInputStreamObserverPtr observer; + const media::AudioParameters params; + uint32_t shared_memory_count; + base::UnguessableToken group_id; + mojo::ScopedSharedBufferHandle key_press_count_buffer; + CreateLoopbackStreamCallback created_callback; + audio::mojom::LocalMuterAssociatedRequest muter_request; + }; + + void ExpectStreamCreation(StreamRequestData* ex) { + stream_request_data_ = ex; + } + + MOCK_METHOD1(IsMuting, void(const base::UnguessableToken&)); + + private: + void CreateLoopbackStream( + media::mojom::AudioInputStreamRequest stream_request, + media::mojom::AudioInputStreamClientPtr client, + media::mojom::AudioInputStreamObserverPtr observer, + const media::AudioParameters& params, + uint32_t shared_memory_count, + const base::UnguessableToken& group_id, + CreateLoopbackStreamCallback created_callback) final { + // No way to cleanly exit the test here in case of failure, so use CHECK. + CHECK(stream_request_data_); + EXPECT_EQ(stream_request_data_->group_id, group_id); + EXPECT_TRUE(stream_request_data_->params.Equals(params)); + stream_request_data_->requested = true; + stream_request_data_->stream_request = std::move(stream_request); + stream_request_data_->client = std::move(client); + stream_request_data_->observer = std::move(observer); + stream_request_data_->shared_memory_count = shared_memory_count; + stream_request_data_->created_callback = std::move(created_callback); + } + + void BindMuter(audio::mojom::LocalMuterAssociatedRequest request, + const base::UnguessableToken& group_id) final { + stream_request_data_->muter_request = std::move(request); + IsMuting(group_id); + } + + StreamRequestData* stream_request_data_; + + DISALLOW_COPY_AND_ASSIGN(MockStreamFactory); +}; + +const bool kMuteSource = true; + +struct TestEnvironment { + TestEnvironment(const base::UnguessableToken& group_id, bool mute_source) { + // Muting should not start until CreateStream() is called. + EXPECT_CALL(stream_factory, IsMuting(_)).Times(0); + auto mock_source = std::make_unique<NiceMock<MockSource>>(group_id); + source = mock_source.get(); + broker = std::make_unique<AudioLoopbackStreamBroker>( + kRenderProcessId, kRenderFrameId, std::move(mock_source), TestParams(), + kShMemCount, mute_source, deleter.Get(), + renderer_factory_client.MakePtr()); + } + + void RunUntilIdle() { thread_bundle.RunUntilIdle(); } + + TestBrowserThreadBundle thread_bundle; + MockDeleterCallback deleter; + MockSource* source; + StrictMock<MockRendererAudioInputStreamFactoryClient> renderer_factory_client; + std::unique_ptr<AudioLoopbackStreamBroker> broker; + MockStreamFactory stream_factory; + audio::mojom::StreamFactoryPtr factory_ptr = stream_factory.MakePtr(); +}; + +} // namespace + +TEST(AudioLoopbackStreamBrokerTest, StoresProcessAndFrameId) { + InSequence seq; + TestBrowserThreadBundle thread_bundle; + MockDeleterCallback deleter; + StrictMock<MockRendererAudioInputStreamFactoryClient> renderer_factory_client; + auto source = std::make_unique<StrictMock<MockSource>>( + base::UnguessableToken::Create()); + MockSource* mock_source = source.get(); + + EXPECT_CALL(*mock_source, OnStartCapturing()); + + AudioLoopbackStreamBroker broker(kRenderProcessId, kRenderFrameId, + std::move(source), TestParams(), kShMemCount, + !kMuteSource, deleter.Get(), + renderer_factory_client.MakePtr()); + + EXPECT_EQ(kRenderProcessId, broker.render_process_id()); + EXPECT_EQ(kRenderFrameId, broker.render_frame_id()); + + EXPECT_CALL(*mock_source, OnStopCapturing()); +} + +TEST(AudioLoopbackStreamBrokerTest, StreamCreationSuccess_Propagates) { + TestEnvironment env(base::UnguessableToken::Create(), !kMuteSource); + MockStreamFactory::StreamRequestData stream_request_data( + env.source->GetGroupID(), TestParams()); + env.stream_factory.ExpectStreamCreation(&stream_request_data); + + EXPECT_CALL(env.stream_factory, IsMuting(_)).Times(0); + + env.broker->CreateStream(env.factory_ptr.get()); + env.RunUntilIdle(); + + EXPECT_TRUE(stream_request_data.requested); + + // Set up test IPC primitives. + const size_t shmem_size = 456; + base::SyncSocket socket1, socket2; + base::SyncSocket::CreatePair(&socket1, &socket2); + std::move(stream_request_data.created_callback) + .Run({base::in_place, mojo::SharedBufferHandle::Create(shmem_size), + mojo::WrapPlatformFile(socket1.Release())}); + + EXPECT_CALL(env.renderer_factory_client, OnStreamCreated()); + + env.RunUntilIdle(); + + Mock::VerifyAndClear(&env.renderer_factory_client); + env.broker.reset(); +} + +TEST(AudioLoopbackStreamBrokerTest, MutedStreamCreation_Mutes) { + TestEnvironment env(base::UnguessableToken::Create(), kMuteSource); + MockStreamFactory::StreamRequestData stream_request_data( + env.source->GetGroupID(), TestParams()); + env.stream_factory.ExpectStreamCreation(&stream_request_data); + + EXPECT_CALL(env.stream_factory, IsMuting(env.source->GetGroupID())); + + env.broker->CreateStream(env.factory_ptr.get()); + env.RunUntilIdle(); + + EXPECT_TRUE(stream_request_data.requested); + + // Set up test IPC primitives. + const size_t shmem_size = 456; + base::SyncSocket socket1, socket2; + base::SyncSocket::CreatePair(&socket1, &socket2); + std::move(stream_request_data.created_callback) + .Run({base::in_place, mojo::SharedBufferHandle::Create(shmem_size), + mojo::WrapPlatformFile(socket1.Release())}); + + EXPECT_CALL(env.renderer_factory_client, OnStreamCreated()); + + env.RunUntilIdle(); + + Mock::VerifyAndClear(&env.renderer_factory_client); + env.broker.reset(); +} + +TEST(AudioLoopbackStreamBrokerTest, SourceGone_CallsDeleter) { + TestEnvironment env(base::UnguessableToken::Create(), kMuteSource); + MockStreamFactory::StreamRequestData stream_request_data( + env.source->GetGroupID(), TestParams()); + env.stream_factory.ExpectStreamCreation(&stream_request_data); + + EXPECT_CALL(env.stream_factory, IsMuting(env.source->GetGroupID())); + + env.broker->CreateStream(env.factory_ptr.get()); + env.RunUntilIdle(); + + EXPECT_TRUE(stream_request_data.requested); + + // Set up test IPC primitives. + const size_t shmem_size = 456; + base::SyncSocket socket1, socket2; + base::SyncSocket::CreatePair(&socket1, &socket2); + std::move(stream_request_data.created_callback) + .Run({base::in_place, mojo::SharedBufferHandle::Create(shmem_size), + mojo::WrapPlatformFile(socket1.Release())}); + + EXPECT_CALL(env.renderer_factory_client, OnStreamCreated()); + + env.RunUntilIdle(); + + Mock::VerifyAndClear(&env.renderer_factory_client); + + EXPECT_CALL(env.deleter, Run(env.broker.release())) + .WillOnce(testing::DeleteArg<0>()); + + env.source->WebContentsDestroyed(); + + env.RunUntilIdle(); +} + +TEST(AudioLoopbackStreamBrokerTest, StreamCreationFailure_CallsDeleter) { + TestEnvironment env(base::UnguessableToken::Create(), !kMuteSource); + MockStreamFactory::StreamRequestData stream_request_data( + env.source->GetGroupID(), TestParams()); + env.stream_factory.ExpectStreamCreation(&stream_request_data); + + EXPECT_CALL(env.stream_factory, IsMuting(_)).Times(0); + + env.broker->CreateStream(env.factory_ptr.get()); + env.RunUntilIdle(); + + EXPECT_TRUE(stream_request_data.requested); + EXPECT_CALL(env.deleter, Run(env.broker.release())) + .WillOnce(testing::DeleteArg<0>()); + + std::move(stream_request_data.created_callback).Run(nullptr); + + env.RunUntilIdle(); +} + +TEST(AudioLoopbackStreamBrokerTest, + RendererFactoryClientDisconnect_CallsDeleter) { + TestEnvironment env(base::UnguessableToken::Create(), !kMuteSource); + MockStreamFactory::StreamRequestData stream_request_data( + env.source->GetGroupID(), TestParams()); + env.stream_factory.ExpectStreamCreation(&stream_request_data); + + EXPECT_CALL(env.stream_factory, IsMuting(_)).Times(0); + + env.broker->CreateStream(env.factory_ptr.get()); + env.RunUntilIdle(); + EXPECT_TRUE(stream_request_data.requested); + + EXPECT_CALL(env.deleter, Run(env.broker.release())) + .WillOnce(testing::DeleteArg<0>()); + env.renderer_factory_client.CloseBinding(); + env.RunUntilIdle(); + Mock::VerifyAndClear(&env.deleter); + + env.stream_factory.CloseBinding(); + env.RunUntilIdle(); +} + +TEST(AudioLoopbackStreamBrokerTest, ObserverDisconnect_CallsDeleter) { + TestEnvironment env(base::UnguessableToken::Create(), !kMuteSource); + MockStreamFactory::StreamRequestData stream_request_data( + env.source->GetGroupID(), TestParams()); + env.stream_factory.ExpectStreamCreation(&stream_request_data); + + EXPECT_CALL(env.stream_factory, IsMuting(_)).Times(0); + + env.broker->CreateStream(env.factory_ptr.get()); + env.RunUntilIdle(); + EXPECT_TRUE(stream_request_data.requested); + + EXPECT_CALL(env.deleter, Run(env.broker.release())) + .WillOnce(testing::DeleteArg<0>()); + stream_request_data.observer.reset(); + env.RunUntilIdle(); + Mock::VerifyAndClear(&env.deleter); + + env.stream_factory.CloseBinding(); + env.RunUntilIdle(); +} + +TEST(AudioLoopbackStreamBrokerTest, + FactoryDisconnectDuringConstruction_CallsDeleter) { + TestEnvironment env(base::UnguessableToken::Create(), !kMuteSource); + env.broker->CreateStream(env.factory_ptr.get()); + env.stream_factory.CloseBinding(); + + EXPECT_CALL(env.deleter, Run(env.broker.release())) + .WillOnce(testing::DeleteArg<0>()); + + env.RunUntilIdle(); +} + +} // namespace content diff --git a/chromium/content/browser/media/audio_muting_session.cc b/chromium/content/browser/media/audio_muting_session.cc new file mode 100644 index 00000000000..6e8c13d3dc7 --- /dev/null +++ b/chromium/content/browser/media/audio_muting_session.cc @@ -0,0 +1,22 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/audio_muting_session.h" + +namespace content { + +AudioMutingSession::AudioMutingSession(const base::UnguessableToken& group_id) + : group_id_(group_id) {} + +AudioMutingSession::~AudioMutingSession(){}; + +void AudioMutingSession::Connect(audio::mojom::StreamFactory* factory) { + if (muter_) + muter_.reset(); + + DCHECK(factory); + factory->BindMuter(mojo::MakeRequest(&muter_), group_id_); +} + +} // namespace content diff --git a/chromium/content/browser/media/audio_muting_session.h b/chromium/content/browser/media/audio_muting_session.h new file mode 100644 index 00000000000..a13c13c17aa --- /dev/null +++ b/chromium/content/browser/media/audio_muting_session.h @@ -0,0 +1,32 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONTENT_BROWSER_MEDIA_AUDIO_MUTING_SESSION_H_ +#define CONTENT_BROWSER_MEDIA_AUDIO_MUTING_SESSION_H_ + +#include <utility> + +#include "base/unguessable_token.h" +#include "content/common/content_export.h" +#include "services/audio/public/mojom/stream_factory.mojom.h" + +namespace content { + +class CONTENT_EXPORT AudioMutingSession { + public: + explicit AudioMutingSession(const base::UnguessableToken& group_id); + ~AudioMutingSession(); + + void Connect(audio::mojom::StreamFactory* factory); + + private: + const base::UnguessableToken group_id_; + audio::mojom::LocalMuterAssociatedPtr muter_; + + DISALLOW_COPY_AND_ASSIGN(AudioMutingSession); +}; + +} // namespace content + +#endif // CONTENT_BROWSER_MEDIA_AUDIO_MUTING_SESSION_H_ diff --git a/chromium/content/browser/media/audio_output_stream_broker.cc b/chromium/content/browser/media/audio_output_stream_broker.cc new file mode 100644 index 00000000000..21cf65ea694 --- /dev/null +++ b/chromium/content/browser/media/audio_output_stream_broker.cc @@ -0,0 +1,157 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/audio_output_stream_broker.h" + +#include <utility> + +#include "base/metrics/histogram_macros.h" +#include "base/trace_event/trace_event.h" +#include "content/browser/media/media_internals.h" +#include "content/public/browser/content_browser_client.h" +#include "content/public/browser/media_observer.h" +#include "content/public/common/content_client.h" +#include "media/audio/audio_logging.h" + +namespace content { + +AudioOutputStreamBroker::AudioOutputStreamBroker( + int render_process_id, + int render_frame_id, + int stream_id, + const std::string& output_device_id, + const media::AudioParameters& params, + const base::UnguessableToken& group_id, + DeleterCallback deleter, + media::mojom::AudioOutputStreamProviderClientPtr client) + : AudioStreamBroker(render_process_id, render_frame_id), + output_device_id_(output_device_id), + params_(params), + group_id_(group_id), + deleter_(std::move(deleter)), + client_(std::move(client)), + observer_(render_process_id, render_frame_id, stream_id), + observer_binding_(&observer_), + weak_ptr_factory_(this) { + DCHECK(client_); + DCHECK(deleter_); + DCHECK(group_id_); + TRACE_EVENT_NESTABLE_ASYNC_BEGIN0("audio", "AudioOutputStreamBroker", this); + + MediaObserver* media_observer = + GetContentClient()->browser()->GetMediaObserver(); + + // May be null in unit tests. + if (media_observer) + media_observer->OnCreatingAudioStream(render_process_id, render_frame_id); + + // Unretained is safe because |this| owns |client_| + client_.set_connection_error_handler(base::BindOnce( + &AudioOutputStreamBroker::Cleanup, base::Unretained(this))); +} + +AudioOutputStreamBroker::~AudioOutputStreamBroker() { + DCHECK_CALLED_ON_VALID_SEQUENCE(owning_sequence_); + + if (awaiting_created_) { + TRACE_EVENT_NESTABLE_ASYNC_END1("audio", "CreateStream", this, "success", + "failed or cancelled"); + } + TRACE_EVENT_NESTABLE_ASYNC_END1("audio", "AudioOutputStreamBroker", this, + "disconnect reason", + static_cast<uint32_t>(disconnect_reason_)); + + UMA_HISTOGRAM_ENUMERATION("Media.Audio.Render.StreamBrokerDisconnectReason", + disconnect_reason_); +} + +void AudioOutputStreamBroker::CreateStream( + audio::mojom::StreamFactory* factory) { + DCHECK_CALLED_ON_VALID_SEQUENCE(owning_sequence_); + DCHECK(!observer_binding_.is_bound()); + TRACE_EVENT_NESTABLE_ASYNC_BEGIN1("audio", "CreateStream", this, "device id", + output_device_id_); + awaiting_created_ = true; + + // Set up observer ptr. Unretained is safe because |this| owns + // |observer_binding_|. + media::mojom::AudioOutputStreamObserverAssociatedPtrInfo ptr_info; + observer_binding_.Bind(mojo::MakeRequest(&ptr_info)); + observer_binding_.set_connection_error_with_reason_handler(base::BindOnce( + &AudioOutputStreamBroker::ObserverBindingLost, base::Unretained(this))); + + media::mojom::AudioOutputStreamPtr stream; + media::mojom::AudioOutputStreamRequest stream_request = + mojo::MakeRequest(&stream); + + // Note that the component id for AudioLog is used to differentiate between + // several users of the same audio log. Since this audio log is for a single + // stream, the component id used doesn't matter. + constexpr int log_component_id = 0; + factory->CreateOutputStream( + std::move(stream_request), std::move(ptr_info), + MediaInternals::GetInstance()->CreateMojoAudioLog( + media::AudioLogFactory::AudioComponent::AUDIO_OUTPUT_CONTROLLER, + log_component_id, render_process_id(), render_frame_id()), + output_device_id_, params_, group_id_, + base::BindOnce(&AudioOutputStreamBroker::StreamCreated, + weak_ptr_factory_.GetWeakPtr(), std::move(stream))); +} + +void AudioOutputStreamBroker::StreamCreated( + media::mojom::AudioOutputStreamPtr stream, + media::mojom::AudioDataPipePtr data_pipe) { + DCHECK_CALLED_ON_VALID_SEQUENCE(owning_sequence_); + TRACE_EVENT_NESTABLE_ASYNC_END1("audio", "CreateStream", this, "success", + !!data_pipe); + awaiting_created_ = false; + if (!data_pipe) { + // Stream creation failed. Signal error. + client_.ResetWithReason( + static_cast<uint32_t>(media::mojom::AudioOutputStreamObserver:: + DisconnectReason::kPlatformError), + std::string()); + disconnect_reason_ = media::mojom::AudioOutputStreamObserver:: + DisconnectReason::kStreamCreationFailed; + Cleanup(); + return; + } + + client_->Created(std::move(stream), std::move(data_pipe)); +} + +void AudioOutputStreamBroker::ObserverBindingLost( + uint32_t reason, + const std::string& description) { + DCHECK_CALLED_ON_VALID_SEQUENCE(owning_sequence_); + + TRACE_EVENT_NESTABLE_ASYNC_INSTANT1("audio", "ObserverBindingLost", this, + "reset reason", reason); + const uint32_t maxValidReason = static_cast<uint32_t>( + media::mojom::AudioOutputStreamObserver::DisconnectReason::kMaxValue); + if (reason > maxValidReason) { + NOTREACHED() << "Invalid reason: " << reason; + } else if (disconnect_reason_ == media::mojom::AudioOutputStreamObserver:: + DisconnectReason::kDocumentDestroyed) { + disconnect_reason_ = + static_cast<media::mojom::AudioOutputStreamObserver::DisconnectReason>( + reason); + } + + // TODO(https://crbug.com/787806): Don't propagate errors if we can retry + // instead. + client_.ResetWithReason( + static_cast<uint32_t>(media::mojom::AudioOutputStreamObserver:: + DisconnectReason::kPlatformError), + std::string()); + + Cleanup(); +} + +void AudioOutputStreamBroker::Cleanup() { + DCHECK_CALLED_ON_VALID_SEQUENCE(owning_sequence_); + std::move(deleter_).Run(this); +} + +} // namespace content diff --git a/chromium/content/browser/media/audio_output_stream_broker.h b/chromium/content/browser/media/audio_output_stream_broker.h new file mode 100644 index 00000000000..d997649ff6d --- /dev/null +++ b/chromium/content/browser/media/audio_output_stream_broker.h @@ -0,0 +1,79 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONTENT_BROWSER_MEDIA_AUDIO_OUTPUT_STREAM_BROKER_H_ +#define CONTENT_BROWSER_MEDIA_AUDIO_OUTPUT_STREAM_BROKER_H_ + +#include <string> + +#include "base/callback.h" +#include "base/macros.h" +#include "base/memory/weak_ptr.h" +#include "base/sequence_checker.h" +#include "base/unguessable_token.h" +#include "content/browser/media/audio_stream_broker.h" +#include "content/browser/renderer_host/media/audio_output_stream_observer_impl.h" +#include "content/common/content_export.h" +#include "media/base/audio_parameters.h" +#include "media/mojo/interfaces/audio_output_stream.mojom.h" +#include "mojo/public/cpp/bindings/associated_binding.h" +#include "services/audio/public/mojom/stream_factory.mojom.h" + +namespace content { + +// AudioOutputStreamBroker is used to broker a connection between a client +// (typically renderer) and the audio service. It also sets up all objects +// used for monitoring the stream. +class CONTENT_EXPORT AudioOutputStreamBroker final : public AudioStreamBroker { + public: + AudioOutputStreamBroker( + int render_process_id, + int render_frame_id, + int stream_id, + const std::string& output_device_id, + const media::AudioParameters& params, + const base::UnguessableToken& group_id, + DeleterCallback deleter, + media::mojom::AudioOutputStreamProviderClientPtr client); + + ~AudioOutputStreamBroker() final; + + // Creates the stream. + void CreateStream(audio::mojom::StreamFactory* factory) final; + + private: + void StreamCreated(media::mojom::AudioOutputStreamPtr stream, + media::mojom::AudioDataPipePtr data_pipe); + void ObserverBindingLost(uint32_t reason, const std::string& description); + void Cleanup(); + + SEQUENCE_CHECKER(owning_sequence_); + + const std::string output_device_id_; + const media::AudioParameters params_; + const base::UnguessableToken group_id_; + + // Indicates that CreateStream has been called, but not StreamCreated. + bool awaiting_created_ = false; + + DeleterCallback deleter_; + + media::mojom::AudioOutputStreamProviderClientPtr client_; + + AudioOutputStreamObserverImpl observer_; + mojo::AssociatedBinding<media::mojom::AudioOutputStreamObserver> + observer_binding_; + + media::mojom::AudioOutputStreamObserver::DisconnectReason disconnect_reason_ = + media::mojom::AudioOutputStreamObserver::DisconnectReason:: + kDocumentDestroyed; + + base::WeakPtrFactory<AudioOutputStreamBroker> weak_ptr_factory_; + + DISALLOW_COPY_AND_ASSIGN(AudioOutputStreamBroker); +}; + +} // namespace content + +#endif // CONTENT_BROWSER_MEDIA_AUDIO_OUTPUT_STREAM_BROKER_H_ diff --git a/chromium/content/browser/media/audio_output_stream_broker_unittest.cc b/chromium/content/browser/media/audio_output_stream_broker_unittest.cc new file mode 100644 index 00000000000..5427db631be --- /dev/null +++ b/chromium/content/browser/media/audio_output_stream_broker_unittest.cc @@ -0,0 +1,281 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/audio_output_stream_broker.h" + +#include <memory> +#include <utility> + +#include "base/bind.h" +#include "base/bind_helpers.h" +#include "base/macros.h" +#include "base/sync_socket.h" +#include "base/test/mock_callback.h" +#include "base/test/scoped_task_environment.h" +#include "base/unguessable_token.h" +#include "media/base/audio_parameters.h" +#include "media/mojo/interfaces/audio_output_stream.mojom.h" +#include "mojo/public/cpp/bindings/binding.h" +#include "mojo/public/cpp/bindings/interface_request.h" +#include "mojo/public/cpp/system/buffer.h" +#include "mojo/public/cpp/system/platform_handle.h" +#include "services/audio/public/cpp/fake_stream_factory.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using ::testing::Test; +using ::testing::Mock; +using ::testing::StrictMock; +using ::testing::InSequence; + +namespace content { + +namespace { + +const int kRenderProcessId = 123; +const int kRenderFrameId = 234; +const int kStreamId = 345; +const size_t kShMemSize = 456; +const char kDeviceId[] = "testdeviceid"; + +media::AudioParameters TestParams() { + return media::AudioParameters::UnavailableDeviceParams(); +} + +using MockDeleterCallback = StrictMock< + base::MockCallback<base::OnceCallback<void(AudioStreamBroker*)>>>; + +class MockAudioOutputStreamProviderClient + : public media::mojom::AudioOutputStreamProviderClient { + public: + MockAudioOutputStreamProviderClient() : binding_(this) {} + ~MockAudioOutputStreamProviderClient() override {} + + void Created(media::mojom::AudioOutputStreamPtr, + media::mojom::AudioDataPipePtr) override { + OnCreated(); + } + + MOCK_METHOD0(OnCreated, void()); + + MOCK_METHOD2(ConnectionError, void(uint32_t, const std::string&)); + + media::mojom::AudioOutputStreamProviderClientPtr MakePtr() { + media::mojom::AudioOutputStreamProviderClientPtr ptr; + binding_.Bind(mojo::MakeRequest(&ptr)); + binding_.set_connection_error_with_reason_handler( + base::BindOnce(&MockAudioOutputStreamProviderClient::ConnectionError, + base::Unretained(this))); + return ptr; + } + + void CloseBinding() { binding_.Close(); } + + private: + mojo::Binding<media::mojom::AudioOutputStreamProviderClient> binding_; + DISALLOW_COPY_AND_ASSIGN(MockAudioOutputStreamProviderClient); +}; + +class MockStreamFactory : public audio::FakeStreamFactory { + public: + MockStreamFactory() {} + ~MockStreamFactory() final {} + + // State of an expected stream creation. |output_device_id|, |params|, + // and |groups_id| are set ahead of time and verified during request. + // The other fields are filled in when the request is received. + struct StreamRequestData { + StreamRequestData(const std::string& output_device_id, + const media::AudioParameters& params, + const base::UnguessableToken& group_id) + : output_device_id(output_device_id), + params(params), + group_id(group_id) {} + + bool requested = false; + media::mojom::AudioOutputStreamRequest stream_request; + media::mojom::AudioOutputStreamObserverAssociatedPtrInfo observer_info; + media::mojom::AudioLogPtr log; + const std::string output_device_id; + const media::AudioParameters params; + const base::UnguessableToken group_id; + CreateOutputStreamCallback created_callback; + }; + + void ExpectStreamCreation(StreamRequestData* ex) { + stream_request_data_ = ex; + } + + private: + void CreateOutputStream( + media::mojom::AudioOutputStreamRequest stream_request, + media::mojom::AudioOutputStreamObserverAssociatedPtrInfo observer_info, + media::mojom::AudioLogPtr log, + const std::string& output_device_id, + const media::AudioParameters& params, + const base::UnguessableToken& group_id, + CreateOutputStreamCallback created_callback) final { + // No way to cleanly exit the test here in case of failure, so use CHECK. + CHECK(stream_request_data_); + EXPECT_EQ(stream_request_data_->output_device_id, output_device_id); + EXPECT_TRUE(stream_request_data_->params.Equals(params)); + EXPECT_EQ(stream_request_data_->group_id, group_id); + stream_request_data_->requested = true; + stream_request_data_->stream_request = std::move(stream_request); + stream_request_data_->observer_info = std::move(observer_info); + stream_request_data_->log = std::move(log); + stream_request_data_->created_callback = std::move(created_callback); + } + + StreamRequestData* stream_request_data_; + DISALLOW_COPY_AND_ASSIGN(MockStreamFactory); +}; + +// This struct collects test state we need without doing anything fancy. +struct TestEnvironment { + TestEnvironment() + : group(base::UnguessableToken::Create()), + broker(std::make_unique<AudioOutputStreamBroker>( + kRenderProcessId, + kRenderFrameId, + kStreamId, + kDeviceId, + TestParams(), + group, + deleter.Get(), + provider_client.MakePtr())) {} + + void RunUntilIdle() { env.RunUntilIdle(); } + + base::test::ScopedTaskEnvironment env; + base::UnguessableToken group; + MockDeleterCallback deleter; + StrictMock<MockAudioOutputStreamProviderClient> provider_client; + std::unique_ptr<AudioOutputStreamBroker> broker; + MockStreamFactory stream_factory; + audio::mojom::StreamFactoryPtr factory_ptr = stream_factory.MakePtr(); +}; + +} // namespace + +TEST(AudioOutputStreamBrokerTest, StoresProcessAndFrameId) { + base::test::ScopedTaskEnvironment env; + MockDeleterCallback deleter; + StrictMock<MockAudioOutputStreamProviderClient> provider_client; + + AudioOutputStreamBroker broker(kRenderProcessId, kRenderFrameId, kStreamId, + kDeviceId, TestParams(), + base::UnguessableToken::Create(), + deleter.Get(), provider_client.MakePtr()); + + EXPECT_EQ(kRenderProcessId, broker.render_process_id()); + EXPECT_EQ(kRenderFrameId, broker.render_frame_id()); +} + +TEST(AudioOutputStreamBrokerTest, ClientDisconnect_CallsDeleter) { + TestEnvironment env; + + EXPECT_CALL(env.deleter, Run(env.broker.release())) + .WillOnce(testing::DeleteArg<0>()); + env.provider_client.CloseBinding(); + env.RunUntilIdle(); +} + +TEST(AudioOutputStreamBrokerTest, StreamCreationSuccess_Propagates) { + TestEnvironment env; + MockStreamFactory::StreamRequestData stream_request_data( + kDeviceId, TestParams(), env.group); + env.stream_factory.ExpectStreamCreation(&stream_request_data); + + env.broker->CreateStream(env.factory_ptr.get()); + env.RunUntilIdle(); + + EXPECT_TRUE(stream_request_data.requested); + + // Set up test IPC primitives. + base::SyncSocket socket1, socket2; + base::SyncSocket::CreatePair(&socket1, &socket2); + std::move(stream_request_data.created_callback) + .Run({base::in_place, mojo::SharedBufferHandle::Create(kShMemSize), + mojo::WrapPlatformFile(socket1.Release())}); + + EXPECT_CALL(env.provider_client, OnCreated()); + + env.RunUntilIdle(); + + Mock::VerifyAndClear(&env.provider_client); + + env.broker.reset(); +} + +TEST(AudioOutputStreamBrokerTest, + StreamCreationFailure_PropagatesErrorAndCallsDeleter) { + TestEnvironment env; + MockStreamFactory::StreamRequestData stream_request_data( + kDeviceId, TestParams(), env.group); + env.stream_factory.ExpectStreamCreation(&stream_request_data); + + env.broker->CreateStream(env.factory_ptr.get()); + env.RunUntilIdle(); + + EXPECT_TRUE(stream_request_data.requested); + EXPECT_CALL(env.provider_client, + ConnectionError(static_cast<uint32_t>( + media::mojom::AudioOutputStreamObserver:: + DisconnectReason::kPlatformError), + std::string())); + EXPECT_CALL(env.deleter, Run(env.broker.release())) + .WillOnce(testing::DeleteArg<0>()); + + std::move(stream_request_data.created_callback).Run(nullptr); + + env.RunUntilIdle(); +} + +TEST(AudioOutputStreamBrokerTest, + ObserverDisconnect_PropagatesErrorAndCallsDeleter) { + TestEnvironment env; + MockStreamFactory::StreamRequestData stream_request_data( + kDeviceId, TestParams(), env.group); + env.stream_factory.ExpectStreamCreation(&stream_request_data); + + env.broker->CreateStream(env.factory_ptr.get()); + env.RunUntilIdle(); + + EXPECT_TRUE(stream_request_data.requested); + EXPECT_CALL(env.provider_client, + ConnectionError(static_cast<uint32_t>( + media::mojom::AudioOutputStreamObserver:: + DisconnectReason::kPlatformError), + std::string())); + EXPECT_CALL(env.deleter, Run(env.broker.release())) + .WillOnce(testing::DeleteArg<0>()); + + // This results in a connection error. + stream_request_data.observer_info.PassHandle(); + + env.RunUntilIdle(); + env.stream_factory.CloseBinding(); + env.RunUntilIdle(); +} + +TEST(AudioOutputStreamBrokerTest, + FactoryDisconnectDuringConstruction_PropagatesErrorAndCallsDeleter) { + TestEnvironment env; + + env.broker->CreateStream(env.factory_ptr.get()); + env.stream_factory.CloseBinding(); + + EXPECT_CALL(env.deleter, Run(env.broker.release())) + .WillOnce(testing::DeleteArg<0>()); + EXPECT_CALL(env.provider_client, + ConnectionError(static_cast<uint32_t>( + media::mojom::AudioOutputStreamObserver:: + DisconnectReason::kPlatformError), + std::string())); + + env.RunUntilIdle(); +} + +} // namespace content diff --git a/chromium/content/browser/media/audio_stream_broker.cc b/chromium/content/browser/media/audio_stream_broker.cc new file mode 100644 index 00000000000..6ff668c3b9e --- /dev/null +++ b/chromium/content/browser/media/audio_stream_broker.cc @@ -0,0 +1,85 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/audio_stream_broker.h" + +#include <utility> + +#include "content/browser/media/audio_input_stream_broker.h" +#include "content/browser/media/audio_loopback_stream_broker.h" +#include "content/browser/media/audio_output_stream_broker.h" + +namespace content { + +namespace { + +class AudioStreamBrokerFactoryImpl final : public AudioStreamBrokerFactory { + public: + AudioStreamBrokerFactoryImpl() = default; + ~AudioStreamBrokerFactoryImpl() final = default; + + std::unique_ptr<AudioStreamBroker> CreateAudioInputStreamBroker( + int render_process_id, + int render_frame_id, + const std::string& device_id, + const media::AudioParameters& params, + uint32_t shared_memory_count, + bool enable_agc, + AudioStreamBroker::DeleterCallback deleter, + mojom::RendererAudioInputStreamFactoryClientPtr renderer_factory_client) + final { + return std::make_unique<AudioInputStreamBroker>( + render_process_id, render_frame_id, device_id, params, + shared_memory_count, enable_agc, std::move(deleter), + std::move(renderer_factory_client)); + } + + std::unique_ptr<AudioStreamBroker> CreateAudioLoopbackStreamBroker( + int render_process_id, + int render_frame_id, + std::unique_ptr<LoopbackSource> source, + const media::AudioParameters& params, + uint32_t shared_memory_count, + bool mute_source, + AudioStreamBroker::DeleterCallback deleter, + mojom::RendererAudioInputStreamFactoryClientPtr renderer_factory_client) + final { + return std::make_unique<AudioLoopbackStreamBroker>( + render_process_id, render_frame_id, std::move(source), params, + shared_memory_count, mute_source, std::move(deleter), + std::move(renderer_factory_client)); + } + + std::unique_ptr<AudioStreamBroker> CreateAudioOutputStreamBroker( + int render_process_id, + int render_frame_id, + int stream_id, + const std::string& output_device_id, + const media::AudioParameters& params, + const base::UnguessableToken& group_id, + AudioStreamBroker::DeleterCallback deleter, + media::mojom::AudioOutputStreamProviderClientPtr client) final { + return std::make_unique<AudioOutputStreamBroker>( + render_process_id, render_frame_id, stream_id, output_device_id, params, + group_id, std::move(deleter), std::move(client)); + } +}; + +} // namespace + +AudioStreamBroker::AudioStreamBroker(int render_process_id, int render_frame_id) + : render_process_id_(render_process_id), + render_frame_id_(render_frame_id) {} +AudioStreamBroker::~AudioStreamBroker() {} + +AudioStreamBrokerFactory::AudioStreamBrokerFactory() {} +AudioStreamBrokerFactory::~AudioStreamBrokerFactory() {} + +// static +std::unique_ptr<AudioStreamBrokerFactory> +AudioStreamBrokerFactory::CreateImpl() { + return std::make_unique<AudioStreamBrokerFactoryImpl>(); +} + +} // namespace content diff --git a/chromium/content/browser/media/audio_stream_broker.h b/chromium/content/browser/media/audio_stream_broker.h new file mode 100644 index 00000000000..40af0eac01b --- /dev/null +++ b/chromium/content/browser/media/audio_stream_broker.h @@ -0,0 +1,139 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONTENT_BROWSER_MEDIA_AUDIO_STREAM_BROKER_H_ +#define CONTENT_BROWSER_MEDIA_AUDIO_STREAM_BROKER_H_ + +#include <memory> +#include <string> + +#include "base/callback.h" +#include "base/macros.h" +#include "content/common/content_export.h" +#include "content/common/media/renderer_audio_input_stream_factory.mojom.h" +#include "content/public/browser/web_contents_observer.h" +#include "media/mojo/interfaces/audio_input_stream.mojom.h" +#include "media/mojo/interfaces/audio_output_stream.mojom.h" + +namespace audio { +namespace mojom { +class StreamFactory; +} +} // namespace audio + +namespace base { +class UnguessableToken; +} + +namespace media { +class AudioParameters; +} + +namespace content { +class WebContents; + +// An AudioStreamBroker is used to broker a connection between a client +// (typically renderer) and the audio service. It also sets up all objects +// used for monitoring the stream. +class CONTENT_EXPORT AudioStreamBroker { + public: + using DeleterCallback = base::OnceCallback<void(AudioStreamBroker*)>; + + AudioStreamBroker(int render_process_id, int render_frame_id); + virtual ~AudioStreamBroker(); + + virtual void CreateStream(audio::mojom::StreamFactory* factory) = 0; + + int render_process_id() const { return render_process_id_; } + int render_frame_id() const { return render_frame_id_; } + + protected: + const int render_process_id_; + const int render_frame_id_; + + private: + DISALLOW_COPY_AND_ASSIGN(AudioStreamBroker); +}; + +// Used for dependency injection into ForwardingAudioStreamFactory. +class CONTENT_EXPORT AudioStreamBrokerFactory { + public: + class CONTENT_EXPORT LoopbackSource : public WebContentsObserver { + public: + explicit LoopbackSource(WebContents* source_contents); + ~LoopbackSource() override; + + // Virtual for mocking in tests. + // Will return an empty token if the source is not present. + + virtual base::UnguessableToken GetGroupID(); + + // Signals the source WebContents that capturing started. + virtual void OnStartCapturing(); + + // Signals the source WebContents that capturing stopped. + virtual void OnStopCapturing(); + + // Sets the closure to run when the source WebContents is gone. + void set_on_gone_closure(base::OnceClosure on_gone_closure) { + on_gone_closure_ = std::move(on_gone_closure); + } + + // WebContentsObserver implementation. + void WebContentsDestroyed() override; + + protected: + LoopbackSource(); + + private: + base::OnceClosure on_gone_closure_; + DISALLOW_COPY_AND_ASSIGN(LoopbackSource); + }; + + static std::unique_ptr<AudioStreamBrokerFactory> CreateImpl(); + + AudioStreamBrokerFactory(); + virtual ~AudioStreamBrokerFactory(); + + virtual std::unique_ptr<AudioStreamBroker> CreateAudioInputStreamBroker( + int render_process_id, + int render_frame_id, + const std::string& device_id, + const media::AudioParameters& params, + uint32_t shared_memory_count, + bool enable_agc, + AudioStreamBroker::DeleterCallback deleter, + mojom::RendererAudioInputStreamFactoryClientPtr + renderer_factory_client) = 0; + + virtual std::unique_ptr<AudioStreamBroker> CreateAudioLoopbackStreamBroker( + int render_process_id, + int render_frame_id, + std::unique_ptr<LoopbackSource> source, + const media::AudioParameters& params, + uint32_t shared_memory_count, + bool mute_source, + AudioStreamBroker::DeleterCallback deleter, + mojom::RendererAudioInputStreamFactoryClientPtr + renderer_factory_client) = 0; + + virtual std::unique_ptr<AudioStreamBroker> CreateAudioOutputStreamBroker( + int render_process_id, + int render_frame_id, + int stream_id, + const std::string& output_device_id, + const media::AudioParameters& params, + const base::UnguessableToken& group_id, + AudioStreamBroker::DeleterCallback deleter, + media::mojom::AudioOutputStreamProviderClientPtr client) = 0; + + // TODO(https://crbug.com/830493): Other kinds of streams. + + private: + DISALLOW_COPY_AND_ASSIGN(AudioStreamBrokerFactory); +}; + +} // namespace content + +#endif // CONTENT_BROWSER_MEDIA_AUDIO_STREAM_BROKER_H_ diff --git a/chromium/content/browser/media/capture/OWNERS b/chromium/content/browser/media/capture/OWNERS index c9e162eb22d..31e7db3290b 100644 --- a/chromium/content/browser/media/capture/OWNERS +++ b/chromium/content/browser/media/capture/OWNERS @@ -8,5 +8,4 @@ per-file desktop_capture_device_unittest.cc=zijiehe@chromium.org per-file image_capture*=mcasas@chromium.org -# TEAM: media-capture-and-streams@grotations.appspotmail.com -# COMPONENT: Blink>GetUserMedia +# COMPONENT: Internals>Media>ScreenCapture diff --git a/chromium/content/browser/media/capture/audio_mirroring_manager_unittest.cc b/chromium/content/browser/media/capture/audio_mirroring_manager_unittest.cc index 2d6efe047d8..a485e46bf68 100644 --- a/chromium/content/browser/media/capture/audio_mirroring_manager_unittest.cc +++ b/chromium/content/browser/media/capture/audio_mirroring_manager_unittest.cc @@ -114,7 +114,6 @@ class AudioMirroringManagerTest : public testing::Test { : params_(AudioParameters::AUDIO_FAKE, media::CHANNEL_LAYOUT_STEREO, AudioParameters::kAudioCDSampleRate, - 16, AudioParameters::kAudioCDSampleRate / 10) {} MockDiverter* CreateStream(int render_process_id, diff --git a/chromium/content/browser/media/capture/aura_window_video_capture_device.cc b/chromium/content/browser/media/capture/aura_window_video_capture_device.cc new file mode 100644 index 00000000000..93160c08dcf --- /dev/null +++ b/chromium/content/browser/media/capture/aura_window_video_capture_device.cc @@ -0,0 +1,171 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/capture/aura_window_video_capture_device.h" + +#include "base/bind.h" +#include "base/location.h" +#include "base/macros.h" +#include "base/memory/ref_counted.h" +#include "base/memory/weak_ptr.h" +#include "base/threading/thread_task_runner_handle.h" +#include "content/public/browser/browser_thread.h" +#include "content/public/browser/desktop_media_id.h" +#include "media/base/bind_to_current_loop.h" +#include "mojo/public/cpp/bindings/strong_binding.h" +#include "ui/aura/window.h" +#include "ui/aura/window_observer.h" + +#if defined(OS_CHROMEOS) +#include "content/browser/media/capture/lame_window_capturer_chromeos.h" +#endif + +namespace content { + +// Threading note: This is constructed on the device thread, while the +// destructor and the rest of the class will run exclusively on the UI thread. +class AuraWindowVideoCaptureDevice::WindowTracker + : public aura::WindowObserver, + public base::SupportsWeakPtr< + AuraWindowVideoCaptureDevice::WindowTracker> { + public: + WindowTracker(base::WeakPtr<AuraWindowVideoCaptureDevice> device, + CursorRenderer* cursor_renderer, + const DesktopMediaID& source_id) + : device_(std::move(device)), + device_task_runner_(base::ThreadTaskRunnerHandle::Get()), + cursor_renderer_(cursor_renderer), + target_type_(source_id.type) { + DCHECK(device_task_runner_); + DCHECK(cursor_renderer_); + + BrowserThread::PostTask( + BrowserThread::UI, FROM_HERE, + base::BindOnce(&WindowTracker::ResolveTarget, AsWeakPtr(), source_id)); + } + + ~WindowTracker() final { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + if (target_window_) { + target_window_->RemoveObserver(this); + } + } + + DesktopMediaID::Type target_type() const { return target_type_; } + + aura::Window* target_window() const { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + return target_window_; + } + + private: + // Determines which frame sink and aura::Window should be targeted for capture + // and notifies the device. + void ResolveTarget(const DesktopMediaID& source_id) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + // Since ResolveTarget() should only ever be called once, expect + // |target_window_| to be null at this point. + DCHECK(!target_window_); + + target_window_ = DesktopMediaID::GetAuraWindowById(source_id); + if (target_window_ && +#if defined(OS_CHROMEOS) + // See class comments for LameWindowCapturerChromeOS. + (source_id.type == DesktopMediaID::TYPE_WINDOW || + target_window_->GetFrameSinkId().is_valid()) && +#else + target_window_->GetFrameSinkId().is_valid() && +#endif + true) { + target_window_->AddObserver(this); + device_task_runner_->PostTask( + FROM_HERE, + base::BindOnce(&FrameSinkVideoCaptureDevice::OnTargetChanged, device_, + target_window_->GetFrameSinkId())); + // Note: CursorRenderer runs on the UI thread. It's also important that + // SetTargetView() be called in the current stack while |target_window_| + // is known to be a valid pointer. http://crbug.com/818679 + cursor_renderer_->SetTargetView(target_window_); + } else { + device_task_runner_->PostTask( + FROM_HERE, + base::BindOnce(&FrameSinkVideoCaptureDevice::OnTargetPermanentlyLost, + device_)); + } + } + + // aura::WindowObserver override. + void OnWindowDestroying(aura::Window* window) final { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + DCHECK_EQ(window, target_window_); + + target_window_->RemoveObserver(this); + target_window_ = nullptr; + + device_task_runner_->PostTask( + FROM_HERE, + base::BindOnce(&FrameSinkVideoCaptureDevice::OnTargetPermanentlyLost, + device_)); + cursor_renderer_->SetTargetView(nullptr); + } + + private: + // |device_| may be dereferenced only by tasks run by |device_task_runner_|. + const base::WeakPtr<FrameSinkVideoCaptureDevice> device_; + const scoped_refptr<base::SingleThreadTaskRunner> device_task_runner_; + + // Owned by FrameSinkVideoCaptureDevice. This will be valid for the life of + // WindowTracker because the WindowTracker deleter task will be posted to the + // UI thread before the CursorRenderer deleter task. + CursorRenderer* const cursor_renderer_; + + const DesktopMediaID::Type target_type_; + + aura::Window* target_window_ = nullptr; + + DISALLOW_COPY_AND_ASSIGN(WindowTracker); +}; + +AuraWindowVideoCaptureDevice::AuraWindowVideoCaptureDevice( + const DesktopMediaID& source_id) + : tracker_(new WindowTracker(AsWeakPtr(), cursor_renderer(), source_id)) {} + +AuraWindowVideoCaptureDevice::~AuraWindowVideoCaptureDevice() = default; + +#if defined(OS_CHROMEOS) +void AuraWindowVideoCaptureDevice::CreateCapturer( + viz::mojom::FrameSinkVideoCapturerRequest request) { + BrowserThread::PostTask( + BrowserThread::UI, FROM_HERE, + base::BindOnce( + [](base::WeakPtr<WindowTracker> tracker_ptr, + viz::mojom::FrameSinkVideoCapturerRequest request) { + WindowTracker* const tracker = tracker_ptr.get(); + if (!tracker) { + // WindowTracker was destroyed in the meantime, due to early + // shutdown. + return; + } + + if (tracker->target_type() == DesktopMediaID::TYPE_WINDOW) { + VLOG(1) << "AuraWindowVideoCaptureDevice is using the LAME " + "capturer. :("; + mojo::StrongBinding<viz::mojom::FrameSinkVideoCapturer>::Create( + std::make_unique<LameWindowCapturerChromeOS>( + tracker->target_window()), + std::move(request)); + } else { + VLOG(1) << "AuraWindowVideoCaptureDevice is using the frame " + "sink capturer. :)"; + CreateCapturerViaGlobalManager(std::move(request)); + } + }, + tracker_->AsWeakPtr(), std::move(request))); +} +#endif + +} // namespace content diff --git a/chromium/content/browser/media/capture/aura_window_video_capture_device.h b/chromium/content/browser/media/capture/aura_window_video_capture_device.h new file mode 100644 index 00000000000..0295b3d1951 --- /dev/null +++ b/chromium/content/browser/media/capture/aura_window_video_capture_device.h @@ -0,0 +1,57 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONTENT_BROWSER_MEDIA_CAPTURE_AURA_WINDOW_VIDEO_CAPTURE_DEVICE_H_ +#define CONTENT_BROWSER_MEDIA_CAPTURE_AURA_WINDOW_VIDEO_CAPTURE_DEVICE_H_ + +#include <memory> + +#include "base/macros.h" +#include "base/memory/weak_ptr.h" +#include "build/build_config.h" +#include "content/browser/media/capture/frame_sink_video_capture_device.h" +#include "content/common/content_export.h" +#include "content/public/browser/browser_thread.h" + +namespace aura { +class Window; +} // namespace aura + +namespace content { + +struct DesktopMediaID; + +// Captures the displayed contents of an aura::Window, producing a stream of +// video frames. +class CONTENT_EXPORT AuraWindowVideoCaptureDevice + : public FrameSinkVideoCaptureDevice, + public base::SupportsWeakPtr<AuraWindowVideoCaptureDevice> { + public: + explicit AuraWindowVideoCaptureDevice(const DesktopMediaID& source_id); + ~AuraWindowVideoCaptureDevice() final; + +#if defined(OS_CHROMEOS) + protected: + // Overrides FrameSinkVideoCaptureDevice::CreateCapturer() to create a + // LameWindowCapturerChromeOS for window capture where compositor frame sinks + // are not present. See class comments for LameWindowCapturerChromeOS for + // further details. + void CreateCapturer(viz::mojom::FrameSinkVideoCapturerRequest request) final; +#endif + + private: + // Monitors the target Window and notifies the base class if it is destroyed. + class WindowTracker; + + // A helper that runs on the UI thread to monitor the target aura::Window, and + // post a notification if it is destroyed. + const std::unique_ptr<WindowTracker, BrowserThread::DeleteOnUIThread> + tracker_; + + DISALLOW_COPY_AND_ASSIGN(AuraWindowVideoCaptureDevice); +}; + +} // namespace content + +#endif // CONTENT_BROWSER_MEDIA_CAPTURE_AURA_WINDOW_VIDEO_CAPTURE_DEVICE_H_ diff --git a/chromium/content/browser/media/capture/aura_window_video_capture_device_browsertest.cc b/chromium/content/browser/media/capture/aura_window_video_capture_device_browsertest.cc new file mode 100644 index 00000000000..7ba9955087b --- /dev/null +++ b/chromium/content/browser/media/capture/aura_window_video_capture_device_browsertest.cc @@ -0,0 +1,358 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/capture/aura_window_video_capture_device.h" + +#include <tuple> + +#include "base/macros.h" +#include "base/run_loop.h" +#include "build/build_config.h" +#include "cc/test/pixel_test_utils.h" +#include "content/browser/media/capture/content_capture_device_browsertest_base.h" +#include "content/browser/media/capture/fake_video_capture_stack.h" +#include "content/browser/media/capture/frame_test_util.h" +#include "content/public/browser/browser_thread.h" +#include "content/public/browser/desktop_media_id.h" +#include "content/public/browser/web_contents.h" +#include "content/shell/browser/shell.h" +#include "media/base/video_util.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/skia/include/core/SkBitmap.h" +#include "ui/aura/window.h" +#include "ui/gfx/geometry/rect.h" +#include "ui/gfx/geometry/rect_conversions.h" +#include "ui/gfx/geometry/rect_f.h" + +namespace content { +namespace { + +class AuraWindowVideoCaptureDeviceBrowserTest + : public ContentCaptureDeviceBrowserTestBase { + public: + AuraWindowVideoCaptureDeviceBrowserTest() = default; + ~AuraWindowVideoCaptureDeviceBrowserTest() override = default; + + aura::Window* GetCapturedWindow() const { +#if defined(OS_CHROMEOS) + // Since the LameWindowCapturerChromeOS will be used, just return the normal + // shell window. + return shell()->window(); +#else + // Note: The Window with an associated compositor frame sink (required for + // capture) is the root window, which is an immediate ancestor of the + // aura::Window provided by shell()->window(). + return shell()->window()->GetRootWindow(); +#endif + } + + // Returns the location of the content within the window. + gfx::Rect GetWebContentsRect() const { + aura::Window* const contents_window = + shell()->web_contents()->GetNativeView(); + gfx::Rect rect = gfx::Rect(contents_window->bounds().size()); + aura::Window::ConvertRectToTarget(contents_window, GetCapturedWindow(), + &rect); + return rect; + } + + // Runs the browser until a frame whose content matches the given |color| is + // found in the captured frames queue, or until a testing failure has + // occurred. + void WaitForFrameWithColor(SkColor color) { + VLOG(1) << "Waiting for frame content area filled with color: red=" + << SkColorGetR(color) << ", green=" << SkColorGetG(color) + << ", blue=" << SkColorGetB(color); + + while (!testing::Test::HasFailure()) { + EXPECT_TRUE(capture_stack()->started()); + EXPECT_FALSE(capture_stack()->error_occurred()); + capture_stack()->ExpectNoLogMessages(); + + while (capture_stack()->has_captured_frames() && + !testing::Test::HasFailure()) { + // Pop the next frame from the front of the queue and convert to a RGB + // bitmap for analysis. + const SkBitmap rgb_frame = capture_stack()->NextCapturedFrame(); + EXPECT_FALSE(rgb_frame.empty()); + + // Three regions of the frame will be analyzed: 1) the WebContents + // region containing a solid color, 2) the remaining part of the + // captured window containing the content shell UI, and 3) the + // solid-black letterboxed region surrounding them. + const gfx::Size frame_size(rgb_frame.width(), rgb_frame.height()); + const gfx::Size window_size = GetExpectedSourceSize(); + const gfx::Rect webcontents_rect = GetWebContentsRect(); + + // Compute the Rects representing where the three regions would be in + // the frame. + const gfx::RectF window_in_frame_rect_f( + IsFixedAspectRatioTest() ? media::ComputeLetterboxRegion( + gfx::Rect(frame_size), window_size) + : gfx::Rect(frame_size)); + const gfx::RectF webcontents_in_frame_rect_f = + FrameTestUtil::TransformSimilarly(gfx::Rect(window_size), + window_in_frame_rect_f, + webcontents_rect); + const gfx::Rect window_in_frame_rect = + gfx::ToEnclosingRect(window_in_frame_rect_f); + const gfx::Rect webcontents_in_frame_rect = + gfx::ToEnclosingRect(webcontents_in_frame_rect_f); + + // Determine the average RGB color in the three regions of the frame. + const auto average_webcontents_rgb = FrameTestUtil::ComputeAverageColor( + rgb_frame, webcontents_in_frame_rect, gfx::Rect()); + const auto average_window_rgb = FrameTestUtil::ComputeAverageColor( + rgb_frame, window_in_frame_rect, webcontents_in_frame_rect); + const auto average_letterbox_rgb = FrameTestUtil::ComputeAverageColor( + rgb_frame, gfx::Rect(frame_size), window_in_frame_rect); + + // TODO(crbug/810131): Once color space issues are fixed, remove this. + // At first, it seemed only to affect ChromeOS; but then on Linux, in + // software compositing mode, it seems that sometimes compositing is + // switching color spaces during the test (e.g., expected a 255 red, but + // we see two frames of 238 followed by a "close-enough" frame of 251). + constexpr int max_color_diff = + FrameTestUtil::kMaxInaccurateColorDifference; + + VLOG(1) << "Video frame analysis: size=" << frame_size.ToString() + << ", captured webcontents should be at " + << webcontents_in_frame_rect.ToString() + << " and has average color " << average_webcontents_rgb + << ", captured window should be at " + << window_in_frame_rect.ToString() << " and has average color " + << average_window_rgb << ", letterbox region has average color " + << average_letterbox_rgb + << ", maximum color error=" << max_color_diff; + + // The letterboxed region should always be black. + if (IsFixedAspectRatioTest()) { + EXPECT_TRUE(FrameTestUtil::IsApproximatelySameColor( + SK_ColorBLACK, average_letterbox_rgb, max_color_diff)); + } + + if (testing::Test::HasFailure()) { + ADD_FAILURE() << "Test failure occurred at this frame; PNG dump: " + << cc::GetPNGDataUrl(rgb_frame); + return; + } + + // Return if the WebContents region now has the new |color|. + if (FrameTestUtil::IsApproximatelySameColor( + color, average_webcontents_rgb, max_color_diff)) { + VLOG(1) << "Observed desired frame."; + return; + } else { + VLOG(3) << "PNG dump of undesired frame: " + << cc::GetPNGDataUrl(rgb_frame); + } + } + + // Wait for at least the minimum capture period before checking for more + // captured frames. + base::RunLoop run_loop; + BrowserThread::PostDelayedTask(BrowserThread::UI, FROM_HERE, + run_loop.QuitClosure(), + GetMinCapturePeriod()); + run_loop.Run(); + } + } + + protected: + // Note: Test code should call <BaseClass>::GetExpectedSourceSize() instead of + // this method since it has extra code to sanity-check that the source size is + // not changing during the test. + gfx::Size GetCapturedSourceSize() const final { + return GetCapturedWindow()->bounds().size(); + } + + std::unique_ptr<FrameSinkVideoCaptureDevice> CreateDevice() final { + const DesktopMediaID source_id = DesktopMediaID::RegisterAuraWindow( + DesktopMediaID::TYPE_WINDOW, GetCapturedWindow()); + EXPECT_TRUE(DesktopMediaID::GetAuraWindowById(source_id)); + return std::make_unique<AuraWindowVideoCaptureDevice>(source_id); + } + + void WaitForFirstFrame() final { WaitForFrameWithColor(SK_ColorBLACK); } + + private: + DISALLOW_COPY_AND_ASSIGN(AuraWindowVideoCaptureDeviceBrowserTest); +}; + +// Tests that the device refuses to start if the target window was destroyed +// before the device could start. +IN_PROC_BROWSER_TEST_F(AuraWindowVideoCaptureDeviceBrowserTest, + ErrorsOutIfWindowHasGoneBeforeDeviceStart) { + NavigateToInitialDocument(); + + const DesktopMediaID source_id = DesktopMediaID::RegisterAuraWindow( + DesktopMediaID::TYPE_WINDOW, GetCapturedWindow()); + EXPECT_TRUE(DesktopMediaID::GetAuraWindowById(source_id)); + const auto capture_params = SnapshotCaptureParams(); + + // Close the Shell. This should close the window it owned, making the capture + // target invalid. + shell()->Close(); + + // Create the device. + auto device = std::make_unique<AuraWindowVideoCaptureDevice>(source_id); + // Running the pending UI tasks should cause the device to realize the window + // is gone. + RunUntilIdle(); + + // Attempt to start the device, and expect the video capture stack to have + // been notified of the error. + device->AllocateAndStartWithReceiver(capture_params, + capture_stack()->CreateFrameReceiver()); + EXPECT_FALSE(capture_stack()->started()); + EXPECT_TRUE(capture_stack()->error_occurred()); + capture_stack()->ExpectHasLogMessages(); + + device->StopAndDeAllocate(); + RunUntilIdle(); +} + +// Tests that the device starts, captures a frame, and then gracefully +// errors-out because the target window is destroyed before the device is +// stopped. +IN_PROC_BROWSER_TEST_F(AuraWindowVideoCaptureDeviceBrowserTest, + ErrorsOutWhenWindowIsDestroyed) { + NavigateToInitialDocument(); + AllocateAndStartAndWaitForFirstFrame(); + + // Initially, the device captures any content changes normally. + ChangePageContentColor(SK_ColorRED); + WaitForFrameWithColor(SK_ColorRED); + + // Close the Shell. This should close the window it owned, causing a "target + // permanently lost" error to propagate to the video capture stack. + shell()->Close(); + RunUntilIdle(); + EXPECT_TRUE(capture_stack()->error_occurred()); + capture_stack()->ExpectHasLogMessages(); + + StopAndDeAllocate(); +} + +// Tests that the device stops delivering frames while suspended. When resumed, +// any content changes that occurred during the suspend should cause a new frame +// to be delivered, to ensure the client is up-to-date. +IN_PROC_BROWSER_TEST_F(AuraWindowVideoCaptureDeviceBrowserTest, + SuspendsAndResumes) { + NavigateToInitialDocument(); + AllocateAndStartAndWaitForFirstFrame(); + + // Initially, the device captures any content changes normally. + ChangePageContentColor(SK_ColorRED); + WaitForFrameWithColor(SK_ColorRED); + + // Suspend the device. + device()->MaybeSuspend(); + RunUntilIdle(); + ClearCapturedFramesQueue(); + + // Change the page content and run the browser for five seconds. Expect no + // frames were queued because the device should be suspended. + ChangePageContentColor(SK_ColorGREEN); + base::RunLoop run_loop; + BrowserThread::PostDelayedTask(BrowserThread::UI, FROM_HERE, + run_loop.QuitClosure(), + base::TimeDelta::FromSeconds(5)); + run_loop.Run(); + EXPECT_FALSE(HasCapturedFramesInQueue()); + + // Resume the device and wait for an automatic refresh frame containing the + // content that was updated while the device was suspended. + device()->Resume(); + WaitForFrameWithColor(SK_ColorGREEN); + + StopAndDeAllocate(); +} + +// Tests that the device delivers refresh frames when asked, while the source +// content is not changing. +IN_PROC_BROWSER_TEST_F(AuraWindowVideoCaptureDeviceBrowserTest, + DeliversRefreshFramesUponRequest) { + NavigateToInitialDocument(); + AllocateAndStartAndWaitForFirstFrame(); + + // Set the page content to a known color. + ChangePageContentColor(SK_ColorRED); + WaitForFrameWithColor(SK_ColorRED); + + // Without making any further changes to the source (which would trigger + // frames to be captured), request and wait for ten refresh frames. + for (int i = 0; i < 10; ++i) { + ClearCapturedFramesQueue(); + device()->RequestRefreshFrame(); + WaitForFrameWithColor(SK_ColorRED); + } + + StopAndDeAllocate(); +} + +class AuraWindowVideoCaptureDeviceBrowserTestP + : public AuraWindowVideoCaptureDeviceBrowserTest, + public testing::WithParamInterface<std::tuple<bool, bool>> { + public: + bool IsSoftwareCompositingTest() const override { + return std::get<0>(GetParam()); + } + bool IsFixedAspectRatioTest() const override { + return std::get<1>(GetParam()); + } +}; + +#if defined(OS_CHROMEOS) +INSTANTIATE_TEST_CASE_P( + , + AuraWindowVideoCaptureDeviceBrowserTestP, + testing::Combine( + // Note: On ChromeOS, software compositing is not an option. + testing::Values(false /* GPU-accelerated compositing */), + testing::Values(false /* variable aspect ratio */, + true /* fixed aspect ratio */))); +#else +INSTANTIATE_TEST_CASE_P( + , + AuraWindowVideoCaptureDeviceBrowserTestP, + testing::Combine(testing::Values(false /* GPU-accelerated compositing */, + true /* software compositing */), + testing::Values(false /* variable aspect ratio */, + true /* fixed aspect ratio */))); +#endif // defined(OS_CHROMEOS) + +// Tests that the device successfully captures a series of content changes, +// whether the browser is running with software compositing or GPU-accelerated +// compositing. +IN_PROC_BROWSER_TEST_P(AuraWindowVideoCaptureDeviceBrowserTestP, + CapturesContentChanges) { + SCOPED_TRACE(testing::Message() + << "Test parameters: " + << (IsSoftwareCompositingTest() ? "Software Compositing" + : "GPU Compositing") + << " with " + << (IsFixedAspectRatioTest() ? "Fixed Video Aspect Ratio" + : "Variable Video Aspect Ratio")); + + NavigateToInitialDocument(); + AllocateAndStartAndWaitForFirstFrame(); + + ASSERT_EQ(shell()->web_contents()->GetVisibility(), + content::Visibility::VISIBLE); + + static constexpr SkColor kColorsToCycleThrough[] = { + SK_ColorRED, SK_ColorGREEN, SK_ColorBLUE, SK_ColorYELLOW, + SK_ColorCYAN, SK_ColorMAGENTA, SK_ColorWHITE, + }; + for (SkColor color : kColorsToCycleThrough) { + ChangePageContentColor(color); + WaitForFrameWithColor(color); + } + + StopAndDeAllocate(); +} + +} // namespace +} // namespace content diff --git a/chromium/content/browser/media/capture/content_capture_device_browsertest_base.cc b/chromium/content/browser/media/capture/content_capture_device_browsertest_base.cc new file mode 100644 index 00000000000..73ba0a4823f --- /dev/null +++ b/chromium/content/browser/media/capture/content_capture_device_browsertest_base.cc @@ -0,0 +1,267 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/capture/content_capture_device_browsertest_base.h" + +#include <cmath> +#include <utility> + +#include "base/run_loop.h" +#include "base/strings/stringprintf.h" +#include "content/browser/media/capture/frame_sink_video_capture_device.h" +#include "content/public/browser/browser_thread.h" +#include "content/public/browser/render_frame_host.h" +#include "content/public/browser/web_contents.h" +#include "content/public/test/browser_test_utils.h" +#include "content/public/test/content_browser_test.h" +#include "content/public/test/content_browser_test_utils.h" +#include "content/public/test/test_utils.h" +#include "content/shell/browser/shell.h" +#include "net/dns/mock_host_resolver.h" +#include "net/test/embedded_test_server/embedded_test_server.h" +#include "net/test/embedded_test_server/http_request.h" +#include "net/test/embedded_test_server/http_response.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "url/gurl.h" + +using net::test_server::BasicHttpResponse; +using net::test_server::HttpRequest; +using net::test_server::HttpResponse; + +namespace content { + +ContentCaptureDeviceBrowserTestBase::ContentCaptureDeviceBrowserTestBase() = + default; + +ContentCaptureDeviceBrowserTestBase::~ContentCaptureDeviceBrowserTestBase() = + default; + +void ContentCaptureDeviceBrowserTestBase::ChangePageContentColor( + SkColor color) { + // See the HandleRequest() method for the original documents being modified + // here. + std::string script; + if (IsCrossSiteCaptureTest()) { + const GURL& inner_frame_url = + embedded_test_server()->GetURL(kInnerFrameHostname, kInnerFramePath); + script = base::StringPrintf( + "document.getElementsByTagName('iframe')[0].src = '%s?color=123456';", + inner_frame_url.spec().c_str()); + } else { + script = "document.body.style.backgroundColor = '#123456';"; + } + script.replace(script.find("123456"), 6, + base::StringPrintf("%02x%02x%02x", SkColorGetR(color), + SkColorGetG(color), SkColorGetB(color))); + CHECK(ExecuteScript(shell()->web_contents(), script)); +} + +gfx::Size ContentCaptureDeviceBrowserTestBase::GetExpectedSourceSize() { + const gfx::Size source_size = GetCapturedSourceSize(); + if (expected_source_size_) { + EXPECT_EQ(*expected_source_size_, source_size) + << "Sanity-check failed: Source size changed during this test."; + } else { + expected_source_size_.emplace(source_size); + VLOG(1) << "Captured source size is " << expected_source_size_->ToString(); + } + return source_size; +} + +media::VideoCaptureParams +ContentCaptureDeviceBrowserTestBase::SnapshotCaptureParams() { + constexpr gfx::Size kMaxCaptureSize = gfx::Size(320, 320); + constexpr int kMaxFramesPerSecond = 60; + + gfx::Size capture_size = kMaxCaptureSize; + if (IsFixedAspectRatioTest()) { + // Half either the width or height, depending on the source size. The goal + // is to force obvious letterboxing (or pillarboxing), regardless of how the + // source is currently sized/oriented. + const gfx::Size source_size = GetExpectedSourceSize(); + if (source_size.width() < source_size.height()) { + capture_size.set_height(capture_size.height() / 2); + } else { + capture_size.set_width(capture_size.width() / 2); + } + } + + media::VideoCaptureParams params; + params.requested_format = media::VideoCaptureFormat( + capture_size, kMaxFramesPerSecond, media::PIXEL_FORMAT_I420); + params.resolution_change_policy = + IsFixedAspectRatioTest() + ? media::ResolutionChangePolicy::FIXED_ASPECT_RATIO + : media::ResolutionChangePolicy::ANY_WITHIN_LIMIT; + return params; +} + +base::TimeDelta ContentCaptureDeviceBrowserTestBase::GetMinCapturePeriod() { + return base::TimeDelta::FromMicroseconds( + base::Time::kMicrosecondsPerSecond / + device_->capture_params().requested_format.frame_rate); +} + +void ContentCaptureDeviceBrowserTestBase::NavigateToInitialDocument() { + // If doing a cross-site capture test, navigate to the more-complex document + // that also contains an iframe (rendered in a separate process). Otherwise, + // navigate to the simpler document. + if (IsCrossSiteCaptureTest()) { + ASSERT_TRUE(NavigateToURL( + shell(), + embedded_test_server()->GetURL(kOuterFrameHostname, kOuterFramePath))); + ASSERT_TRUE(WaitForLoadStop(shell()->web_contents())); + + // Confirm the iframe is a cross-process child render frame. + auto* const child_frame = + ChildFrameAt(shell()->web_contents()->GetMainFrame(), 0); + ASSERT_TRUE(child_frame); + ASSERT_TRUE(child_frame->IsCrossProcessSubframe()); + } else { + ASSERT_TRUE( + NavigateToURL(shell(), embedded_test_server()->GetURL( + kSingleFrameHostname, kSingleFramePath))); + ASSERT_TRUE(WaitForLoadStop(shell()->web_contents())); + } +} + +void ContentCaptureDeviceBrowserTestBase:: + AllocateAndStartAndWaitForFirstFrame() { + capture_stack()->Reset(); + + device_ = CreateDevice(); + device_->AllocateAndStartWithReceiver(SnapshotCaptureParams(), + capture_stack()->CreateFrameReceiver()); + RunUntilIdle(); + EXPECT_TRUE(capture_stack()->started()); + EXPECT_FALSE(capture_stack()->error_occurred()); + capture_stack()->ExpectNoLogMessages(); + + WaitForFirstFrame(); +} + +void ContentCaptureDeviceBrowserTestBase::StopAndDeAllocate() { + device_->StopAndDeAllocate(); + RunUntilIdle(); + device_.reset(); +} + +void ContentCaptureDeviceBrowserTestBase::RunUntilIdle() { + base::RunLoop().RunUntilIdle(); +} + +bool ContentCaptureDeviceBrowserTestBase::IsSoftwareCompositingTest() const { + return false; +} + +bool ContentCaptureDeviceBrowserTestBase::IsFixedAspectRatioTest() const { + return false; +} + +bool ContentCaptureDeviceBrowserTestBase::IsCrossSiteCaptureTest() const { + return false; +} + +void ContentCaptureDeviceBrowserTestBase::SetUp() { + // IMPORTANT: Do not add the switches::kUseGpuInTests command line flag: It + // causes the tests to take 12+ seconds just to spin up a render process on + // debug builds. It can also cause test failures in MSAN builds, or exacerbate + // OOM situations on highly-loaded machines. + + // Screen capture requires readback from compositor output. + EnablePixelOutput(); + + // Conditionally force software compositing instead of GPU-accelerated + // compositing. + if (IsSoftwareCompositingTest()) { + UseSoftwareCompositing(); + } + + ContentBrowserTest::SetUp(); +} + +void ContentCaptureDeviceBrowserTestBase::SetUpCommandLine( + base::CommandLine* command_line) { + IsolateAllSitesForTesting(command_line); +} + +void ContentCaptureDeviceBrowserTestBase::SetUpOnMainThread() { + ContentBrowserTest::SetUpOnMainThread(); + + // Set-up and start the embedded test HTTP server. + host_resolver()->AddRule("*", "127.0.0.1"); + embedded_test_server()->RegisterRequestHandler( + base::BindRepeating(&ContentCaptureDeviceBrowserTestBase::HandleRequest, + base::Unretained(this))); + ASSERT_TRUE(embedded_test_server()->Start()); +} + +void ContentCaptureDeviceBrowserTestBase::TearDownOnMainThread() { + ASSERT_TRUE(embedded_test_server()->ShutdownAndWaitUntilComplete()); + + ClearCapturedFramesQueue(); + + // Run any left-over tasks (usually these are delete-soon's and orphaned + // tasks). + RunUntilIdle(); + + ContentBrowserTest::TearDownOnMainThread(); +} + +std::unique_ptr<HttpResponse> +ContentCaptureDeviceBrowserTestBase::HandleRequest(const HttpRequest& request) { + auto response = std::make_unique<BasicHttpResponse>(); + response->set_content_type("text/html"); + const GURL& url = request.GetURL(); + if (url.path() == kOuterFramePath) { + // A page with a solid white fill color, but containing an iframe in its + // upper-left quadrant. + const GURL& inner_frame_url = + embedded_test_server()->GetURL(kInnerFrameHostname, kInnerFramePath); + response->set_content(base::StringPrintf( + "<!doctype html>" + "<body style='background-color: #ffffff;'>" + "<iframe src='%s' style='position:absolute; " + "top:0px; left:0px; margin:none; padding:none; border:none;'>" + "</iframe>" + "<script>" + "window.addEventListener('load', () => {" + " const iframe = document.getElementsByTagName('iframe')[0];" + " iframe.width = document.documentElement.clientWidth / 2;" + " iframe.height = document.documentElement.clientHeight / 2;" + "});" + "</script>" + "</body>", + inner_frame_url.spec().c_str())); + } else { + // A page whose solid fill color is based on a query parameter, or + // defaults to black. + const std::string& query = url.query(); + std::string color = "#000000"; + const auto pos = query.find("color="); + if (pos != std::string::npos) { + color = "#" + query.substr(pos + 6, 6); + } + response->set_content( + base::StringPrintf("<!doctype html>" + "<body style='background-color: %s;'></body>", + color.c_str())); + } + return std::move(response); +} + +// static +constexpr char ContentCaptureDeviceBrowserTestBase::kInnerFrameHostname[]; +// static +constexpr char ContentCaptureDeviceBrowserTestBase::kInnerFramePath[]; +// static +constexpr char ContentCaptureDeviceBrowserTestBase::kOuterFrameHostname[]; +// static +constexpr char ContentCaptureDeviceBrowserTestBase::kOuterFramePath[]; +// static +constexpr char ContentCaptureDeviceBrowserTestBase::kSingleFrameHostname[]; +// static +constexpr char ContentCaptureDeviceBrowserTestBase::kSingleFramePath[]; + +} // namespace content diff --git a/chromium/content/browser/media/capture/content_capture_device_browsertest_base.h b/chromium/content/browser/media/capture/content_capture_device_browsertest_base.h new file mode 100644 index 00000000000..a6f4135f6c1 --- /dev/null +++ b/chromium/content/browser/media/capture/content_capture_device_browsertest_base.h @@ -0,0 +1,129 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONTENT_BROWSER_MEDIA_CAPTURE_CONTENT_CAPTURE_DEVICE_BROWSERTEST_BASE_H_ +#define CONTENT_BROWSER_MEDIA_CAPTURE_CONTENT_CAPTURE_DEVICE_BROWSERTEST_BASE_H_ + +#include <memory> +#include <string> + +#include "base/macros.h" +#include "base/optional.h" +#include "base/time/time.h" +#include "build/build_config.h" +#include "content/browser/media/capture/fake_video_capture_stack.h" +#include "content/public/test/content_browser_test.h" +#include "media/capture/video_capture_types.h" +#include "third_party/skia/include/core/SkColor.h" +#include "ui/gfx/geometry/size.h" + +namespace net { +namespace test_server { +struct HttpRequest; +class HttpResponse; +} // namespace test_server +} // namespace net + +namespace content { + +class FrameSinkVideoCaptureDevice; + +// Common base class for screen capture browser tests. Since this is a +// ContentBrowserTest, it assumes the test environment consists of a content +// shell and a single WebContents. +class ContentCaptureDeviceBrowserTestBase : public ContentBrowserTest { + public: + ContentCaptureDeviceBrowserTestBase(); + ~ContentCaptureDeviceBrowserTestBase() override; + + FakeVideoCaptureStack* capture_stack() { return &capture_stack_; } + FrameSinkVideoCaptureDevice* device() const { return device_.get(); } + + // Alters the solid fill color making up the page content. This will trigger a + // compositor update, which will trigger a frame capture. + void ChangePageContentColor(SkColor color); + + // Returns the captured source size, but also sanity-checks that it is not + // changing during the test. Prefer to use this method instead of + // GetCapturedSourceSize() to improve test stability. + gfx::Size GetExpectedSourceSize(); + + // Returns capture parameters based on the captured source size. + media::VideoCaptureParams SnapshotCaptureParams(); + + // Returns the actual minimum capture period the device is using. This should + // not be called until after AllocateAndStartAndWaitForFirstFrame(). + base::TimeDelta GetMinCapturePeriod(); + + // Navigates to the initial document, according to the current test + // parameters, and waits for page load completion. All test fixtures should + // call this before any of the other methods. + void NavigateToInitialDocument(); + + // Creates and starts the device for frame capture, and checks that the + // initial refresh frame is delivered. + void AllocateAndStartAndWaitForFirstFrame(); + + // Stops and destroys the device. + void StopAndDeAllocate(); + + // Runs the message loop until idle. + void RunUntilIdle(); + + void ClearCapturedFramesQueue() { capture_stack_.ClearCapturedFramesQueue(); } + + bool HasCapturedFramesInQueue() const { + return capture_stack_.has_captured_frames(); + } + + protected: + // These all return false, but can be overridden for parameterized tests to + // change the behavior of this base class. + virtual bool IsSoftwareCompositingTest() const; + virtual bool IsFixedAspectRatioTest() const; + virtual bool IsCrossSiteCaptureTest() const; + + // Returns the size of the original content (i.e., not including any + // stretching/scaling being done to fit it within a video frame). + virtual gfx::Size GetCapturedSourceSize() const = 0; + + // Returns a new FrameSinkVideoCaptureDevice instance. + virtual std::unique_ptr<FrameSinkVideoCaptureDevice> CreateDevice() = 0; + + // Called to wait for the first frame with expected content. + virtual void WaitForFirstFrame() = 0; + + // ContentBrowserTest overrides to enable pixel output and set-up/tear-down + // the embedded HTTP server that provides test content. + void SetUp() override; + void SetUpCommandLine(base::CommandLine* command_line) override; + void SetUpOnMainThread() override; + void TearDownOnMainThread() override; + + private: + // Called by the embedded test HTTP server to provide the document resources. + std::unique_ptr<net::test_server::HttpResponse> HandleRequest( + const net::test_server::HttpRequest& request); + + FakeVideoCaptureStack capture_stack_; + base::Optional<gfx::Size> expected_source_size_; + std::unique_ptr<FrameSinkVideoCaptureDevice> device_; + + // Arbitrary string constants used to refer to each document by + // host+path. Note that the "inner frame" and "outer frame" must have + // different hostnames to engage the cross-site process isolation logic in the + // browser. + static constexpr char kInnerFrameHostname[] = "innerframe.com"; + static constexpr char kInnerFramePath[] = "/inner.html"; + static constexpr char kOuterFrameHostname[] = "outerframe.com"; + static constexpr char kOuterFramePath[] = "/outer.html"; + static constexpr char kSingleFrameHostname[] = "singleframe.com"; + static constexpr char kSingleFramePath[] = "/single.html"; + + DISALLOW_COPY_AND_ASSIGN(ContentCaptureDeviceBrowserTestBase); +}; + +} // namespace content + +#endif // CONTENT_BROWSER_MEDIA_CAPTURE_CONTENT_CAPTURE_DEVICE_BROWSERTEST_BASE_H_ diff --git a/chromium/content/browser/media/capture/cursor_renderer_aura.cc b/chromium/content/browser/media/capture/cursor_renderer_aura.cc index d390783b242..ec13d0da468 100644 --- a/chromium/content/browser/media/capture/cursor_renderer_aura.cc +++ b/chromium/content/browser/media/capture/cursor_renderer_aura.cc @@ -112,9 +112,8 @@ SkBitmap CursorRendererAura::GetLastKnownCursorImage(gfx::Point* hot_point) { } gfx::NativeCursor cursor = window_->GetHost()->last_cursor(); - SkBitmap cursor_bitmap; - ui::GetCursorBitmap(cursor, &cursor_bitmap, hot_point); - return cursor_bitmap; + *hot_point = cursor.GetHotspot(); + return cursor.GetBitmap(); } void CursorRendererAura::OnMouseEvent(ui::MouseEvent* event) { diff --git a/chromium/content/browser/media/capture/cursor_renderer_aura_unittest.cc b/chromium/content/browser/media/capture/cursor_renderer_aura_unittest.cc index b9017fa7d5c..4a5209e9985 100644 --- a/chromium/content/browser/media/capture/cursor_renderer_aura_unittest.cc +++ b/chromium/content/browser/media/capture/cursor_renderer_aura_unittest.cc @@ -41,7 +41,7 @@ class CursorRendererAuraTest : public AuraTestBase { // Initialize the shared global resource bundle that has bitmap // resources needed by CursorRenderer base::FilePath pak_file; - bool r = PathService::Get(base::DIR_MODULE, &pak_file); + bool r = base::PathService::Get(base::DIR_MODULE, &pak_file); DCHECK(r); pak_file = pak_file.Append(FILE_PATH_LITERAL("content_shell.pak")); ui::ResourceBundle::InitSharedInstanceWithPakPath(pak_file); @@ -92,18 +92,18 @@ class CursorRendererAuraTest : public AuraTestBase { void MoveMouseCursorWithinWindow() { gfx::Point point1(20, 20); ui::MouseEvent event1(ui::ET_MOUSE_MOVED, point1, point1, Now(), 0, 0); - aura::Env::GetInstance()->set_last_mouse_location(point1); + aura::Env::GetInstance()->SetLastMouseLocation(point1); cursor_renderer_->OnMouseEvent(&event1); gfx::Point point2(60, 60); ui::MouseEvent event2(ui::ET_MOUSE_MOVED, point2, point2, Now(), 0, 0); - aura::Env::GetInstance()->set_last_mouse_location(point2); + aura::Env::GetInstance()->SetLastMouseLocation(point2); cursor_renderer_->OnMouseEvent(&event2); } void MoveMouseCursorOutsideWindow() { gfx::Point point(1000, 1000); ui::MouseEvent event1(ui::ET_MOUSE_MOVED, point, point, Now(), 0, 0); - aura::Env::GetInstance()->set_last_mouse_location(point); + aura::Env::GetInstance()->SetLastMouseLocation(point); cursor_renderer_->OnMouseEvent(&event1); } diff --git a/chromium/content/browser/media/capture/desktop_capture_device.cc b/chromium/content/browser/media/capture/desktop_capture_device.cc index df6078bc5f7..01954292117 100644 --- a/chromium/content/browser/media/capture/desktop_capture_device.cc +++ b/chromium/content/browser/media/capture/desktop_capture_device.cc @@ -181,6 +181,10 @@ class DesktopCaptureDevice::Core : public webrtc::DesktopCapturer::Callback { // result. bool first_capture_returned_; + // True if the first capture permanent error has been logged. Used to log the + // first capture permanent error. + bool first_permanent_error_logged; + // The type of the capturer. DesktopMediaID::Type capturer_type_; @@ -208,6 +212,7 @@ DesktopCaptureDevice::Core::Core( max_cpu_consumption_percentage_(GetMaximumCpuConsumptionPercentage()), capture_in_progress_(false), first_capture_returned_(false), + first_permanent_error_logged(false), capturer_type_(type), weak_factory_(this) {} @@ -299,8 +304,17 @@ void DesktopCaptureDevice::Core::OnCaptureResult( } if (!success) { - if (result == webrtc::DesktopCapturer::Result::ERROR_PERMANENT) + if (result == webrtc::DesktopCapturer::Result::ERROR_PERMANENT) { + if (!first_permanent_error_logged) { + first_permanent_error_logged = true; + if (capturer_type_ == DesktopMediaID::TYPE_SCREEN) { + IncrementDesktopCaptureCounter(SCREEN_CAPTURER_PERMANENT_ERROR); + } else { + IncrementDesktopCaptureCounter(WINDOW_CAPTURER_PERMANENT_ERROR); + } + } client_->OnError(FROM_HERE, "The desktop capturer has failed."); + } return; } DCHECK(frame); diff --git a/chromium/content/browser/media/capture/desktop_capture_device_aura_unittest.cc b/chromium/content/browser/media/capture/desktop_capture_device_aura_unittest.cc index 8714ce6f501..7de25f82bfa 100644 --- a/chromium/content/browser/media/capture/desktop_capture_device_aura_unittest.cc +++ b/chromium/content/browser/media/capture/desktop_capture_device_aura_unittest.cc @@ -51,6 +51,13 @@ class MockDeviceClient : public media::VideoCaptureDevice::Client { base::TimeTicks reference_time, base::TimeDelta tiemstamp, int frame_feedback_id)); + MOCK_METHOD6(OnIncomingCapturedGfxBuffer, + void(gfx::GpuMemoryBuffer* buffer, + const media::VideoCaptureFormat& frame_format, + int clockwise_rotation, + base::TimeTicks reference_time, + base::TimeDelta timestamp, + int frame_feedback_id)); MOCK_METHOD0(DoReserveOutputBuffer, void(void)); MOCK_METHOD0(DoOnIncomingCapturedBuffer, void(void)); MOCK_METHOD0(DoOnIncomingCapturedVideoFrame, void(void)); @@ -63,10 +70,8 @@ class MockDeviceClient : public media::VideoCaptureDevice::Client { // Trampoline methods to workaround GMOCK problems with std::unique_ptr<>. Buffer ReserveOutputBuffer(const gfx::Size& dimensions, media::VideoPixelFormat format, - media::VideoPixelStorage storage, int frame_feedback_id) override { EXPECT_EQ(media::PIXEL_FORMAT_I420, format); - EXPECT_EQ(media::VideoPixelStorage::CPU, storage); DoReserveOutputBuffer(); return Buffer(); } @@ -87,10 +92,8 @@ class MockDeviceClient : public media::VideoCaptureDevice::Client { } Buffer ResurrectLastOutputBuffer(const gfx::Size& dimensions, media::VideoPixelFormat format, - media::VideoPixelStorage storage, int frame_feedback_id) override { EXPECT_EQ(media::PIXEL_FORMAT_I420, format); - EXPECT_EQ(media::VideoPixelStorage::CPU, storage); DoResurrectLastOutputBuffer(); return Buffer(); } diff --git a/chromium/content/browser/media/capture/desktop_capture_device_uma_types.h b/chromium/content/browser/media/capture/desktop_capture_device_uma_types.h index 07df96a1864..c2ed81b5e19 100644 --- a/chromium/content/browser/media/capture/desktop_capture_device_uma_types.h +++ b/chromium/content/browser/media/capture/desktop_capture_device_uma_types.h @@ -24,6 +24,8 @@ enum DesktopCaptureCounters { SCREEN_CAPTURER_CREATED_WITHOUT_AUDIO, TAB_VIDEO_CAPTURER_CREATED_WITH_AUDIO, TAB_VIDEO_CAPTURER_CREATED_WITHOUT_AUDIO, + SCREEN_CAPTURER_PERMANENT_ERROR, + WINDOW_CAPTURER_PERMANENT_ERROR, DESKTOP_CAPTURE_COUNTER_BOUNDARY }; diff --git a/chromium/content/browser/media/capture/desktop_capture_device_unittest.cc b/chromium/content/browser/media/capture/desktop_capture_device_unittest.cc index 08f262b5262..96e5f8c1e81 100644 --- a/chromium/content/browser/media/capture/desktop_capture_device_unittest.cc +++ b/chromium/content/browser/media/capture/desktop_capture_device_unittest.cc @@ -13,7 +13,7 @@ #include "base/command_line.h" #include "base/macros.h" -#include "base/message_loop/message_loop.h" +#include "base/message_loop/message_loop_current.h" #include "base/strings/string_number_conversions.h" #include "base/synchronization/waitable_event.h" #include "base/test/test_mock_time_task_runner.h" @@ -73,6 +73,13 @@ class MockDeviceClient : public media::VideoCaptureDevice::Client { base::TimeTicks reference_time, base::TimeDelta timestamp, int frame_feedback_id)); + MOCK_METHOD6(OnIncomingCapturedGfxBuffer, + void(gfx::GpuMemoryBuffer* buffer, + const media::VideoCaptureFormat& frame_format, + int clockwise_rotation, + base::TimeTicks reference_time, + base::TimeDelta timestamp, + int frame_feedback_id)); MOCK_METHOD0(DoReserveOutputBuffer, void(void)); MOCK_METHOD0(DoOnIncomingCapturedBuffer, void(void)); MOCK_METHOD0(DoOnIncomingCapturedVideoFrame, void(void)); @@ -85,10 +92,8 @@ class MockDeviceClient : public media::VideoCaptureDevice::Client { // Trampoline methods to workaround GMOCK problems with std::unique_ptr<>. Buffer ReserveOutputBuffer(const gfx::Size& dimensions, media::VideoPixelFormat format, - media::VideoPixelStorage storage, int frame_feedback_id) override { - EXPECT_TRUE(format == media::PIXEL_FORMAT_I420 && - storage == media::VideoPixelStorage::CPU); + EXPECT_TRUE(format == media::PIXEL_FORMAT_I420); DoReserveOutputBuffer(); return Buffer(); } @@ -109,10 +114,8 @@ class MockDeviceClient : public media::VideoCaptureDevice::Client { } Buffer ResurrectLastOutputBuffer(const gfx::Size& dimensions, media::VideoPixelFormat format, - media::VideoPixelStorage storage, int frame_feedback_id) override { - EXPECT_TRUE(format == media::PIXEL_FORMAT_I420 && - storage == media::VideoPixelStorage::CPU); + EXPECT_TRUE(format == media::PIXEL_FORMAT_I420); DoResurrectLastOutputBuffer(); return Buffer(); } @@ -630,7 +633,7 @@ class DesktopCaptureDeviceThrottledTest : public DesktopCaptureDeviceTest { // 'PostNonNestable' is required to make sure the next one // shot capture timer is already pushed when forwaring the // virtual time by the next pending task delay. - base::MessageLoop::current() + base::MessageLoopCurrent::Get() ->task_runner() ->PostNonNestableTask( FROM_HERE, diff --git a/chromium/content/browser/media/capture/fake_video_capture_stack.cc b/chromium/content/browser/media/capture/fake_video_capture_stack.cc new file mode 100644 index 00000000000..8bd1c9c1540 --- /dev/null +++ b/chromium/content/browser/media/capture/fake_video_capture_stack.cc @@ -0,0 +1,159 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/capture/fake_video_capture_stack.h" + +#include <stdint.h> + +#include <utility> + +#include "base/bind_helpers.h" +#include "media/base/video_frame.h" +#include "media/capture/video/video_frame_receiver.h" +#include "media/capture/video_capture_types.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/libyuv/include/libyuv.h" +#include "ui/gfx/geometry/rect.h" + +namespace content { + +FakeVideoCaptureStack::FakeVideoCaptureStack() = default; + +FakeVideoCaptureStack::~FakeVideoCaptureStack() = default; + +void FakeVideoCaptureStack::Reset() { + frames_.clear(); + last_frame_timestamp_ = base::TimeDelta::Min(); +} + +class FakeVideoCaptureStack::Receiver : public media::VideoFrameReceiver { + public: + explicit Receiver(FakeVideoCaptureStack* capture_stack) + : capture_stack_(capture_stack) {} + ~Receiver() final = default; + + private: + using Buffer = media::VideoCaptureDevice::Client::Buffer; + + void OnNewBuffer(int buffer_id, + media::mojom::VideoBufferHandlePtr buffer_handle) final { + buffers_[buffer_id] = std::move(buffer_handle); + } + + void OnFrameReadyInBuffer( + int buffer_id, + int frame_feedback_id, + std::unique_ptr<Buffer::ScopedAccessPermission> access, + media::mojom::VideoFrameInfoPtr frame_info) final { + const auto it = buffers_.find(buffer_id); + CHECK(it != buffers_.end()); + CHECK(it->second->is_shared_buffer_handle()); + mojo::ScopedSharedBufferHandle& buffer = + it->second->get_shared_buffer_handle(); + + const size_t mapped_size = + media::VideoCaptureFormat(frame_info->coded_size, 0.0f, + frame_info->pixel_format) + .ImageAllocationSize(); + mojo::ScopedSharedBufferMapping mapping = buffer->Map(mapped_size); + CHECK(mapping.get()); + + auto frame = media::VideoFrame::WrapExternalData( + frame_info->pixel_format, frame_info->coded_size, + frame_info->visible_rect, frame_info->visible_rect.size(), + reinterpret_cast<uint8_t*>(mapping.get()), mapped_size, + frame_info->timestamp); + CHECK(frame); + frame->metadata()->MergeInternalValuesFrom(frame_info->metadata); + // This destruction observer will unmap the shared memory when the + // VideoFrame goes out-of-scope. + frame->AddDestructionObserver( + base::BindOnce(base::DoNothing::Once<mojo::ScopedSharedBufferMapping>(), + std::move(mapping))); + // This destruction observer will notify the video capture device once all + // downstream code is done using the VideoFrame. + frame->AddDestructionObserver(base::BindOnce( + [](std::unique_ptr<Buffer::ScopedAccessPermission> access) {}, + std::move(access))); + + capture_stack_->OnReceivedFrame(std::move(frame)); + } + + void OnBufferRetired(int buffer_id) final { + const auto it = buffers_.find(buffer_id); + CHECK(it != buffers_.end()); + buffers_.erase(it); + } + + void OnError() final { capture_stack_->error_occurred_ = true; } + + void OnLog(const std::string& message) final { + capture_stack_->log_messages_.push_back(message); + } + + void OnStarted() final { capture_stack_->started_ = true; } + + void OnStartedUsingGpuDecode() final { NOTREACHED(); } + + FakeVideoCaptureStack* const capture_stack_; + base::flat_map<int, media::mojom::VideoBufferHandlePtr> buffers_; + + DISALLOW_COPY_AND_ASSIGN(Receiver); +}; + +std::unique_ptr<media::VideoFrameReceiver> +FakeVideoCaptureStack::CreateFrameReceiver() { + return std::make_unique<Receiver>(this); +} + +SkBitmap FakeVideoCaptureStack::NextCapturedFrame() { + CHECK(!frames_.empty()); + media::VideoFrame& frame = *(frames_.front()); + SkBitmap bitmap; + bitmap.allocN32Pixels(frame.visible_rect().width(), + frame.visible_rect().height()); + // TODO(crbug/810131): This is not Rec.709 colorspace conversion, and so will + // introduce inaccuracies. + libyuv::I420ToARGB(frame.visible_data(media::VideoFrame::kYPlane), + frame.stride(media::VideoFrame::kYPlane), + frame.visible_data(media::VideoFrame::kUPlane), + frame.stride(media::VideoFrame::kUPlane), + frame.visible_data(media::VideoFrame::kVPlane), + frame.stride(media::VideoFrame::kVPlane), + reinterpret_cast<uint8_t*>(bitmap.getPixels()), + static_cast<int>(bitmap.rowBytes()), bitmap.width(), + bitmap.height()); + frames_.pop_front(); + return bitmap; +} + +void FakeVideoCaptureStack::ClearCapturedFramesQueue() { + frames_.clear(); +} + +void FakeVideoCaptureStack::ExpectHasLogMessages() { + EXPECT_FALSE(log_messages_.empty()); + while (!log_messages_.empty()) { + VLOG(1) << "Next log message: " << log_messages_.front(); + log_messages_.pop_front(); + } +} + +void FakeVideoCaptureStack::ExpectNoLogMessages() { + while (!log_messages_.empty()) { + ADD_FAILURE() << "Unexpected log message: " << log_messages_.front(); + log_messages_.pop_front(); + } +} + +void FakeVideoCaptureStack::OnReceivedFrame( + scoped_refptr<media::VideoFrame> frame) { + // Frame timestamps should be monotionically increasing. + EXPECT_LT(last_frame_timestamp_, frame->timestamp()); + last_frame_timestamp_ = frame->timestamp(); + + frames_.emplace_back(std::move(frame)); +} + +} // namespace content diff --git a/chromium/content/browser/media/capture/fake_video_capture_stack.h b/chromium/content/browser/media/capture/fake_video_capture_stack.h new file mode 100644 index 00000000000..8092af19359 --- /dev/null +++ b/chromium/content/browser/media/capture/fake_video_capture_stack.h @@ -0,0 +1,80 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONTENT_BROWSER_MEDIA_CAPTURE_FAKE_VIDEO_CAPTURE_STACK_H_ +#define CONTENT_BROWSER_MEDIA_CAPTURE_FAKE_VIDEO_CAPTURE_STACK_H_ + +#include <memory> +#include <string> + +#include "base/containers/circular_deque.h" +#include "base/memory/scoped_refptr.h" +#include "base/time/time.h" +#include "content/public/browser/browser_thread.h" +#include "third_party/skia/include/core/SkBitmap.h" + +namespace media { +class VideoFrame; +class VideoFrameReceiver; +}; // namespace media + +namespace content { + +// Provides a fake representation of the entire video capture stack. It creates +// a VideoFrameReceiver that a device can deliver video frames to, and adapts +// that to a simple collector of video frames, represented as SkBitmaps, for +// further examination by the browser tests. +class FakeVideoCaptureStack { + public: + FakeVideoCaptureStack(); + ~FakeVideoCaptureStack(); + + // Reset the capture stack to a state where it contains no frames and is + // expecting a first frame. + void Reset(); + + // Returns a VideoFrameReceiver that the implementation under test delivers + // frames to. + std::unique_ptr<media::VideoFrameReceiver> CreateFrameReceiver(); + + // Returns true if the device called VideoFrameReceiver::OnStarted(). + bool started() const { return started_; } + + // Returns true if the device called VideoFrameReceiver::OnError(). + bool error_occurred() const { return error_occurred_; } + + // Accessors to capture frame queue. + bool has_captured_frames() const { return !frames_.empty(); } + SkBitmap NextCapturedFrame(); + void ClearCapturedFramesQueue(); + + // Called when tests expect there to be one or more log messages sent to the + // video capture stack. Turn on verbose logging for a dump of the actual log + // messages. This method clears the queue of log messages. + void ExpectHasLogMessages(); + + // Called when tests expect there to be no log messages sent to the video + // capture stack. + void ExpectNoLogMessages(); + + private: + // A minimal implementation of VideoFrameReceiver that wraps buffers into + // VideoFrame instances and forwards all relevant callbacks and data to the + // parent FakeVideoCaptureStack. + class Receiver; + + // Checks that the frame timestamp is monotonically increasing and then + // stashes it in the |frames_| queue for later examination by the tests. + void OnReceivedFrame(scoped_refptr<media::VideoFrame> frame); + + bool started_ = false; + bool error_occurred_ = false; + base::circular_deque<std::string> log_messages_; + base::circular_deque<scoped_refptr<media::VideoFrame>> frames_; + base::TimeDelta last_frame_timestamp_ = base::TimeDelta::Min(); +}; + +} // namespace content + +#endif // CONTENT_BROWSER_MEDIA_CAPTURE_FAKE_VIDEO_CAPTURE_STACK_H_ diff --git a/chromium/content/browser/media/capture/frame_sink_video_capture_device.cc b/chromium/content/browser/media/capture/frame_sink_video_capture_device.cc index 8db3dc88281..80ad28d503b 100644 --- a/chromium/content/browser/media/capture/frame_sink_video_capture_device.cc +++ b/chromium/content/browser/media/capture/frame_sink_video_capture_device.cc @@ -33,47 +33,6 @@ std::unique_ptr<T, BrowserThread::DeleteOnUIThread> RescopeToUIThread( return std::unique_ptr<T, BrowserThread::DeleteOnUIThread>(ptr.release()); } -// Sets up a mojo message pipe and requests the HostFrameSinkManager create a -// new capturer instance bound to it. Returns the client-side interface. -viz::mojom::FrameSinkVideoCapturerPtrInfo CreateCapturer() { - DCHECK_CURRENTLY_ON(BrowserThread::UI); - - viz::HostFrameSinkManager* const manager = GetHostFrameSinkManager(); - DCHECK(manager); - viz::mojom::FrameSinkVideoCapturerPtr capturer; - manager->CreateVideoCapturer(mojo::MakeRequest(&capturer)); - return capturer.PassInterface(); -} - -// Adapter for a VideoFrameReceiver to get access to the mojo SharedBufferHandle -// for a frame. -class HandleMover - : public media::VideoCaptureDevice::Client::Buffer::HandleProvider { - public: - explicit HandleMover(mojo::ScopedSharedBufferHandle handle) - : handle_(std::move(handle)) {} - ~HandleMover() final {} - - mojo::ScopedSharedBufferHandle GetHandleForInterProcessTransit( - bool read_only) final { - return std::move(handle_); - } - - base::SharedMemoryHandle GetNonOwnedSharedMemoryHandleForLegacyIPC() final { - NOTREACHED(); - return base::SharedMemoryHandle(); - } - - std::unique_ptr<media::VideoCaptureBufferHandle> GetHandleForInProcessAccess() - final { - NOTREACHED(); - return nullptr; - } - - private: - mojo::ScopedSharedBufferHandle handle_; -}; - // Adapter for a VideoFrameReceiver to notify once frame consumption is // complete. VideoFrameReceiver requires owning an object that it will destroy // once consumption is complete. This class adapts between that scheme and @@ -90,9 +49,7 @@ class ScopedFrameDoneHelper } // namespace FrameSinkVideoCaptureDevice::FrameSinkVideoCaptureDevice() - : capturer_creator_(base::BindRepeating(&CreateCapturer)), - binding_(this), - cursor_renderer_(RescopeToUIThread(CursorRenderer::Create( + : cursor_renderer_(RescopeToUIThread(CursorRenderer::Create( CursorRenderer::CURSOR_DISPLAYED_ON_MOUSE_MOVEMENT))), weak_factory_(this) { DCHECK(cursor_renderer_); @@ -133,12 +90,33 @@ void FrameSinkVideoCaptureDevice::AllocateAndStartWithReceiver( &FrameSinkVideoCaptureDevice::RequestRefreshFrame, weak_factory_.GetWeakPtr())))); - // Hop to the UI thread to request a Mojo connection to a new capturer - // instance, and then hop back to the device thread to start using it. - BrowserThread::PostTaskAndReplyWithResult( - BrowserThread::UI, FROM_HERE, base::BindOnce(capturer_creator_), - base::BindOnce(&FrameSinkVideoCaptureDevice::OnCapturerCreated, - weak_factory_.GetWeakPtr())); + // Shutdown the prior capturer, if any. + MaybeStopConsuming(); + + capturer_ = std::make_unique<viz::ClientFrameSinkVideoCapturer>( + base::BindRepeating(&FrameSinkVideoCaptureDevice::CreateCapturer, + base::Unretained(this))); + + capturer_->SetFormat(capture_params_.requested_format.pixel_format, + media::COLOR_SPACE_UNSPECIFIED); + capturer_->SetMinCapturePeriod( + base::TimeDelta::FromMicroseconds(base::saturated_cast<int64_t>( + base::Time::kMicrosecondsPerSecond / + capture_params_.requested_format.frame_rate))); + const auto& constraints = capture_params_.SuggestConstraints(); + capturer_->SetResolutionConstraints(constraints.min_frame_size, + constraints.max_frame_size, + constraints.fixed_aspect_ratio); + + if (target_.is_valid()) { + capturer_->ChangeTarget(target_); + } + + receiver_->OnStarted(); + + if (!suspend_requested_) { + MaybeStartConsuming(); + } } void FrameSinkVideoCaptureDevice::AllocateAndStart( @@ -153,7 +131,7 @@ void FrameSinkVideoCaptureDevice::AllocateAndStart( void FrameSinkVideoCaptureDevice::RequestRefreshFrame() { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - if (capturer_ && binding_.is_bound() && !suspend_requested_) { + if (capturer_ && !suspend_requested_) { capturer_->RequestRefreshFrame(); } } @@ -266,9 +244,11 @@ void FrameSinkVideoCaptureDevice::OnFrameCaptured( // Pass the video frame to the VideoFrameReceiver. This is done by first // passing the shared memory buffer handle and then notifying it that a new // frame is ready to be read from the buffer. - receiver_->OnNewBufferHandle( - static_cast<BufferId>(slot_index), - std::make_unique<HandleMover>(std::move(buffer))); + media::mojom::VideoBufferHandlePtr buffer_handle = + media::mojom::VideoBufferHandle::New(); + buffer_handle->set_shared_buffer_handle(std::move(buffer)); + receiver_->OnNewBuffer(static_cast<BufferId>(slot_index), + std::move(buffer_handle)); receiver_->OnFrameReadyInBuffer( static_cast<BufferId>(slot_index), slot_index, std::make_unique<ScopedFrameDoneHelper>( @@ -288,9 +268,9 @@ void FrameSinkVideoCaptureDevice::OnStopped() { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); // This method would never be called if FrameSinkVideoCaptureDevice explicitly - // called capturer_->Stop(), because the binding is closed at that time. - // Therefore, a call to this method means that the capturer cannot continue; - // and that's a permanent failure. + // called capturer_->StopAndResetConsumer(), because the binding is closed at + // that time. Therefore, a call to this method means that the capturer cannot + // continue; and that's a permanent failure. OnFatalError("Capturer service cannot continue."); } @@ -315,80 +295,47 @@ void FrameSinkVideoCaptureDevice::OnTargetPermanentlyLost() { OnFatalError("Capture target has been permanently lost."); } -void FrameSinkVideoCaptureDevice::SetCapturerCreatorForTesting( - CapturerCreatorCallback creator) { - capturer_creator_ = std::move(creator); -} - void FrameSinkVideoCaptureDevice::WillStart() {} void FrameSinkVideoCaptureDevice::DidStop() {} -void FrameSinkVideoCaptureDevice::OnCapturerCreated( - viz::mojom::FrameSinkVideoCapturerPtrInfo info) { - DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - - if (!receiver_) { - return; // StopAndDeAllocate() occurred in the meantime. - } - - // Shutdown the prior capturer, if any. - MaybeStopConsuming(); - capturer_.reset(); - - // Bind and configure the new capturer. - capturer_.Bind(std::move(info)); - // TODO(miu): Remove this once HostFrameSinkManager will notify this - // consumer when to take recovery steps after VIZ process crashes. - capturer_.set_connection_error_handler(base::BindOnce( - &FrameSinkVideoCaptureDevice::OnFatalError, base::Unretained(this), - "Capturer service connection lost.")); - capturer_->SetFormat(capture_params_.requested_format.pixel_format, - media::COLOR_SPACE_UNSPECIFIED); - capturer_->SetMinCapturePeriod( - base::TimeDelta::FromMicroseconds(base::saturated_cast<int64_t>( - base::Time::kMicrosecondsPerSecond / - capture_params_.requested_format.frame_rate))); - const auto& constraints = capture_params_.SuggestConstraints(); - capturer_->SetResolutionConstraints(constraints.min_frame_size, - constraints.max_frame_size, - constraints.fixed_aspect_ratio); - - if (target_.is_valid()) { - capturer_->ChangeTarget(target_); - } - - receiver_->OnStarted(); +void FrameSinkVideoCaptureDevice::CreateCapturer( + viz::mojom::FrameSinkVideoCapturerRequest request) { + CreateCapturerViaGlobalManager(std::move(request)); +} - if (!suspend_requested_) { - MaybeStartConsuming(); - } +// static +void FrameSinkVideoCaptureDevice::CreateCapturerViaGlobalManager( + viz::mojom::FrameSinkVideoCapturerRequest request) { + // Send the request to UI thread because that's where HostFrameSinkManager + // lives. + BrowserThread::PostTask( + BrowserThread::UI, FROM_HERE, + base::BindOnce( + [](viz::mojom::FrameSinkVideoCapturerRequest request) { + viz::HostFrameSinkManager* const manager = + GetHostFrameSinkManager(); + DCHECK(manager); + manager->CreateVideoCapturer(std::move(request)); + }, + std::move(request))); } void FrameSinkVideoCaptureDevice::MaybeStartConsuming() { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - if (!receiver_ || !capturer_ || binding_.is_bound()) { + if (!receiver_ || !capturer_) { return; } - viz::mojom::FrameSinkVideoConsumerPtr consumer; - binding_.Bind(mojo::MakeRequest(&consumer)); - // TODO(miu): Remove this once HostFrameSinkManager will notify this consumer - // when to take recovery steps after VIZ process crashes. - binding_.set_connection_error_handler(base::BindOnce( - &FrameSinkVideoCaptureDevice::OnFatalError, base::Unretained(this), - "Consumer connection to Capturer service lost.")); - capturer_->Start(std::move(consumer)); + capturer_->Start(this); } void FrameSinkVideoCaptureDevice::MaybeStopConsuming() { DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); - if (capturer_) { - capturer_->Stop(); - } - binding_.Close(); + if (capturer_) + capturer_->StopAndResetConsumer(); } void FrameSinkVideoCaptureDevice::OnFramePropagationComplete( diff --git a/chromium/content/browser/media/capture/frame_sink_video_capture_device.h b/chromium/content/browser/media/capture/frame_sink_video_capture_device.h index bca2bf237a8..fefe7b9d10c 100644 --- a/chromium/content/browser/media/capture/frame_sink_video_capture_device.h +++ b/chromium/content/browser/media/capture/frame_sink_video_capture_device.h @@ -13,6 +13,7 @@ #include "base/optional.h" #include "base/sequence_checker.h" #include "components/viz/common/surfaces/frame_sink_id.h" +#include "components/viz/host/client_frame_sink_video_capturer.h" #include "content/browser/media/capture/cursor_renderer.h" #include "content/common/content_export.h" #include "content/public/browser/browser_thread.h" @@ -21,7 +22,6 @@ #include "media/capture/video/video_frame_receiver.h" #include "media/capture/video_capture_types.h" #include "mojo/public/cpp/bindings/binding.h" -#include "services/viz/privileged/interfaces/compositing/frame_sink_video_capture.mojom.h" namespace content { @@ -43,9 +43,6 @@ class CONTENT_EXPORT FrameSinkVideoCaptureDevice : public media::VideoCaptureDevice, public viz::mojom::FrameSinkVideoConsumer { public: - using CapturerCreatorCallback = - base::RepeatingCallback<viz::mojom::FrameSinkVideoCapturerPtrInfo()>; - FrameSinkVideoCaptureDevice(); ~FrameSinkVideoCaptureDevice() override; @@ -87,9 +84,6 @@ class CONTENT_EXPORT FrameSinkVideoCaptureDevice void OnTargetChanged(const viz::FrameSinkId& frame_sink_id); void OnTargetPermanentlyLost(); - // Overrides the callback that is run to create the capturer. - void SetCapturerCreatorForTesting(CapturerCreatorCallback creator); - protected: CursorRenderer* cursor_renderer() const { return cursor_renderer_.get(); } @@ -97,12 +91,20 @@ class CONTENT_EXPORT FrameSinkVideoCaptureDevice virtual void WillStart(); virtual void DidStop(); + // Establishes connection to FrameSinkVideoCapturer. The default + // implementation calls CreateCapturerViaGlobalManager(), but subclasses + // and/or tests may provide alternatives. + virtual void CreateCapturer( + viz::mojom::FrameSinkVideoCapturerRequest request); + + // Establishes connection to FrameSinkVideoCapturer using the global + // viz::HostFrameSinkManager. + static void CreateCapturerViaGlobalManager( + viz::mojom::FrameSinkVideoCapturerRequest request); + private: using BufferId = decltype(media::VideoCaptureDevice::Client::Buffer::id); - // Bind a newly-created capturer, configure it, and resuming consuming. - void OnCapturerCreated(viz::mojom::FrameSinkVideoCapturerPtrInfo info); - // If not consuming and all preconditions are met, set up and start consuming. void MaybeStartConsuming(); @@ -135,16 +137,7 @@ class CONTENT_EXPORT FrameSinkVideoCaptureDevice // cleared by StopAndDeAllocate(). std::unique_ptr<media::VideoFrameReceiver> receiver_; - // Callback that is run to request a capturer be created and returns the - // client-side interface. This callback will be run on the UI BrowserThread. - // The constructor provides a default, but unit tests can override this. - CapturerCreatorCallback capturer_creator_; - - // Mojo pointer to the capturer instance in VIZ. - viz::mojom::FrameSinkVideoCapturerPtr capturer_; - - // Mojo binding to this instance as a consumer of frames from the capturer. - mojo::Binding<viz::mojom::FrameSinkVideoConsumer> binding_; + std::unique_ptr<viz::ClientFrameSinkVideoCapturer> capturer_; // A pool of structs that hold state relevant to frames currently being // processed by VideoFrameReceiver. Each "slot" is re-used by later frames. diff --git a/chromium/content/browser/media/capture/frame_sink_video_capture_device_unittest.cc b/chromium/content/browser/media/capture/frame_sink_video_capture_device_unittest.cc index f52055ef111..b9cc7f8bad8 100644 --- a/chromium/content/browser/media/capture/frame_sink_video_capture_device_unittest.cc +++ b/chromium/content/browser/media/capture/frame_sink_video_capture_device_unittest.cc @@ -60,13 +60,12 @@ constexpr int kMaxFrameRate = 25; // It evenly divides 1 million usec. constexpr base::TimeDelta kMinCapturePeriod = base::TimeDelta::FromMicroseconds( base::Time::kMicrosecondsPerSecond / kMaxFrameRate); constexpr media::VideoPixelFormat kFormat = media::PIXEL_FORMAT_I420; -constexpr media::VideoPixelStorage kStorage = media::VideoPixelStorage::CPU; // Helper to return the capture parameters packaged in a VideoCaptureParams. media::VideoCaptureParams GetCaptureParams() { media::VideoCaptureParams params; params.requested_format = - media::VideoCaptureFormat(kResolution, kMaxFrameRate, kFormat, kStorage); + media::VideoCaptureFormat(kResolution, kMaxFrameRate, kFormat); return params; } @@ -141,22 +140,22 @@ class MockVideoFrameReceiver : public media::VideoFrameReceiver { ~MockVideoFrameReceiver() override { DCHECK_ON_DEVICE_THREAD(); - EXPECT_TRUE(handle_providers_.empty()); + EXPECT_TRUE(buffer_handles_.empty()); EXPECT_TRUE(feedback_ids_.empty()); EXPECT_TRUE(access_permissions_.empty()); EXPECT_TRUE(frame_infos_.empty()); } - void OnNewBufferHandle( - int buffer_id, - std::unique_ptr<Buffer::HandleProvider> handle_provider) final { + void OnNewBuffer(int buffer_id, + media::mojom::VideoBufferHandlePtr buffer_handle) final { DCHECK_ON_DEVICE_THREAD(); - auto* const raw_pointer = handle_provider.get(); - handle_providers_[buffer_id] = std::move(handle_provider); - MockOnNewBufferHandle(buffer_id, raw_pointer); + auto* const raw_pointer = buffer_handle.get(); + buffer_handles_[buffer_id] = std::move(buffer_handle); + MockOnNewBuffer(buffer_id, raw_pointer); } - MOCK_METHOD2(MockOnNewBufferHandle, - void(int buffer_id, Buffer::HandleProvider* handle_provider)); + MOCK_METHOD2(MockOnNewBuffer, + void(int buffer_id, + media::mojom::VideoBufferHandle* buffer_handle)); void OnFrameReadyInBuffer( int buffer_id, int frame_feedback_id, @@ -184,13 +183,14 @@ class MockVideoFrameReceiver : public media::VideoFrameReceiver { mojo::ScopedSharedBufferHandle TakeBufferHandle(int buffer_id) { DCHECK_NOT_ON_DEVICE_THREAD(); - const auto it = handle_providers_.find(buffer_id); - if (it == handle_providers_.end()) { + const auto it = buffer_handles_.find(buffer_id); + if (it == buffer_handles_.end()) { ADD_FAILURE() << "Missing entry for buffer_id=" << buffer_id; return mojo::ScopedSharedBufferHandle(); } - auto buffer = it->second->GetHandleForInterProcessTransit(true); - handle_providers_.erase(it); + CHECK(it->second->is_shared_buffer_handle()); + auto buffer = std::move(it->second->get_shared_buffer_handle()); + buffer_handles_.erase(it); return buffer; } @@ -229,14 +229,36 @@ class MockVideoFrameReceiver : public media::VideoFrameReceiver { } private: - base::flat_map<int, std::unique_ptr<Buffer::HandleProvider>> - handle_providers_; + base::flat_map<int, media::mojom::VideoBufferHandlePtr> buffer_handles_; base::flat_map<int, int> feedback_ids_; base::flat_map<int, std::unique_ptr<Buffer::ScopedAccessPermission>> access_permissions_; base::flat_map<int, media::mojom::VideoFrameInfoPtr> frame_infos_; }; +// A FrameSinkVideoCaptureDevice, but with CreateCapturer() overridden to bind +// to a MockFrameSinkVideoCapturer instead of the real thing. +class FrameSinkVideoCaptureDeviceForTest : public FrameSinkVideoCaptureDevice { + public: + explicit FrameSinkVideoCaptureDeviceForTest( + MockFrameSinkVideoCapturer* capturer) + : capturer_(capturer) {} + + protected: + void CreateCapturer(viz::mojom::FrameSinkVideoCapturerRequest request) final { + BrowserThread::PostTask( + BrowserThread::UI, FROM_HERE, + base::BindOnce( + [](MockFrameSinkVideoCapturer* capturer, + viz::mojom::FrameSinkVideoCapturerRequest request) { + capturer->Bind(std::move(request)); + }, + capturer_, std::move(request))); + } + + MockFrameSinkVideoCapturer* const capturer_; +}; + // Convenience macros to make a non-blocking FrameSinkVideoCaptureDevice method // call on the device thread. #define POST_DEVICE_METHOD_CALL0(method) \ @@ -259,21 +281,11 @@ class FrameSinkVideoCaptureDeviceTest : public testing::Test { // until complete. POST_DEVICE_TASK(base::BindOnce( [](FrameSinkVideoCaptureDeviceTest* test) { - test->device_ = std::make_unique<FrameSinkVideoCaptureDevice>(); + test->device_ = std::make_unique<FrameSinkVideoCaptureDeviceForTest>( + &test->capturer_); }, this)); WAIT_FOR_DEVICE_TASKS(); - - // Set an override to "create" the mock capturer instance instead of the - // real thing. - device_->SetCapturerCreatorForTesting(base::BindRepeating( - [](MockFrameSinkVideoCapturer* capturer) { - DCHECK_CURRENTLY_ON(BrowserThread::UI); - viz::mojom::FrameSinkVideoCapturerPtr capturer_ptr; - capturer->Bind(mojo::MakeRequest(&capturer_ptr)); - return capturer_ptr.PassInterface(); - }, - &capturer_)); } void TearDown() override { @@ -348,7 +360,7 @@ class FrameSinkVideoCaptureDeviceTest : public testing::Test { std::move(buffer), buffer_size, media::mojom::VideoFrameInfo::New( kMinCapturePeriod * frame_number, - base::Value(base::Value::Type::DICTIONARY), kFormat, kStorage, + base::Value(base::Value::Type::DICTIONARY), kFormat, kResolution, gfx::Rect(kResolution)), gfx::Rect(kResolution), gfx::Rect(kResolution), viz::mojom::FrameSinkVideoConsumerFrameCallbacksPtr( @@ -390,39 +402,6 @@ class FrameSinkVideoCaptureDeviceTest : public testing::Test { std::unique_ptr<FrameSinkVideoCaptureDevice> device_; }; -// Tests a racy start condition: Ensure that nothing bad happens if -// StopAndDeAllocate() is called before the capturer creation completes. -TEST_F(FrameSinkVideoCaptureDeviceTest, - AllocatesAndDeallocatesBeforeCapturerCreated) { - auto receiver = std::make_unique<MockVideoFrameReceiver>(); - EXPECT_CALL(*receiver, OnStarted()).Times(0); - EXPECT_CALL(*receiver, OnError()).Times(0); - - EXPECT_CALL(capturer_, SetFormat(_, _)).Times(0); - EXPECT_CALL(capturer_, SetMinCapturePeriod(_)).Times(0); - EXPECT_CALL(capturer_, SetResolutionConstraints(_, _, _)).Times(0); - EXPECT_CALL(capturer_, ChangeTarget(_)).Times(0); - EXPECT_CALL(capturer_, MockStart(_)).Times(0); - - EXPECT_FALSE(capturer_.is_bound()); - POST_DEVICE_METHOD_CALL(AllocateAndStartWithReceiver, GetCaptureParams(), - std::move(receiver)); - // A task is pending on the UI thread to create the capturer. Call - // StopAndDeAllocate() before that task runs. - POST_DEVICE_METHOD_CALL0(StopAndDeAllocate); - WAIT_FOR_DEVICE_TASKS(); - - // Now, run the task on the UI thread, which will post the reply back to the - // device thread. - RUN_UI_TASKS(); - EXPECT_TRUE(capturer_.is_bound()); - - // Now, when the reply task on the device thread is run, the - // FrameSinkVideoCaptureDevice should realize that StopAndDeAllocate() was - // called in the meantime and abort. - WAIT_FOR_DEVICE_TASKS(); -} - // Tests a normal session, progressing through the start, frame capture, and // stop phases. TEST_F(FrameSinkVideoCaptureDeviceTest, CapturesAndDeliversFrames) { @@ -452,7 +431,7 @@ TEST_F(FrameSinkVideoCaptureDeviceTest, CapturesAndDeliversFrames) { const int first_frame_number = next_frame_number; for (int i = 0; i < in_flight_count; ++i) { Expectation new_buffer_called = - EXPECT_CALL(*receiver, MockOnNewBufferHandle(Ge(0), NotNull())) + EXPECT_CALL(*receiver, MockOnNewBuffer(Ge(0), NotNull())) .WillOnce(SaveArg<0>(&buffer_ids[i])); EXPECT_CALL(*receiver, MockOnFrameReadyInBuffer(Eq(ByRef(buffer_ids[i])), Ge(0), @@ -478,7 +457,6 @@ TEST_F(FrameSinkVideoCaptureDeviceTest, CapturesAndDeliversFrames) { ASSERT_TRUE(info); EXPECT_EQ(kMinCapturePeriod * frame_number, info->timestamp); EXPECT_EQ(kFormat, info->pixel_format); - EXPECT_EQ(kStorage, info->storage_type); EXPECT_EQ(kResolution, info->coded_size); EXPECT_EQ(gfx::Rect(kResolution), info->visible_rect); } diff --git a/chromium/content/browser/media/capture/frame_test_util.cc b/chromium/content/browser/media/capture/frame_test_util.cc new file mode 100644 index 00000000000..7f4e218fa24 --- /dev/null +++ b/chromium/content/browser/media/capture/frame_test_util.cc @@ -0,0 +1,93 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/capture/frame_test_util.h" + +#include <stdint.h> + +#include <cmath> + +#include "ui/gfx/geometry/rect.h" +#include "ui/gfx/geometry/rect_conversions.h" +#include "ui/gfx/geometry/rect_f.h" +#include "ui/gfx/geometry/size.h" +#include "ui/gfx/transform.h" + +namespace content { + +// static +FrameTestUtil::RGB FrameTestUtil::ComputeAverageColor( + SkBitmap frame, + const gfx::Rect& raw_include_rect, + const gfx::Rect& raw_exclude_rect) { + // Clip the rects to the valid region within |frame|. Also, only the subregion + // of |exclude_rect| within |include_rect| is relevant. + gfx::Rect include_rect = raw_include_rect; + include_rect.Intersect(gfx::Rect(0, 0, frame.width(), frame.height())); + gfx::Rect exclude_rect = raw_exclude_rect; + exclude_rect.Intersect(include_rect); + + // Sum up the color values in each color channel for all pixels in + // |include_rect| not contained by |exclude_rect|. + int64_t include_sums[3] = {0}; + for (int y = include_rect.y(), bottom = include_rect.bottom(); y < bottom; + ++y) { + for (int x = include_rect.x(), right = include_rect.right(); x < right; + ++x) { + const SkColor color = frame.getColor(x, y); + if (exclude_rect.Contains(x, y)) { + continue; + } + include_sums[0] += SkColorGetR(color); + include_sums[1] += SkColorGetG(color); + include_sums[2] += SkColorGetB(color); + } + } + + // Divide the sums by the area to compute the average color. + const int include_area = + include_rect.size().GetArea() - exclude_rect.size().GetArea(); + if (include_area <= 0) { + return RGB{NAN, NAN, NAN}; + } else { + const auto include_area_f = static_cast<double>(include_area); + return RGB{include_sums[0] / include_area_f, + include_sums[1] / include_area_f, + include_sums[2] / include_area_f}; + } +} + +// static +bool FrameTestUtil::IsApproximatelySameColor(SkColor color, + const RGB& rgb, + int max_diff) { + const double r_diff = std::abs(SkColorGetR(color) - rgb.r); + const double g_diff = std::abs(SkColorGetG(color) - rgb.g); + const double b_diff = std::abs(SkColorGetB(color) - rgb.b); + return r_diff < max_diff && g_diff < max_diff && b_diff < max_diff; +} + +// static +gfx::RectF FrameTestUtil::TransformSimilarly(const gfx::Rect& original, + const gfx::RectF& transformed, + const gfx::Rect& rect) { + if (original.IsEmpty()) { + return gfx::RectF(transformed.x() - original.x(), + transformed.y() - original.y(), 0.0f, 0.0f); + } + // The following is the scale-then-translate 2D matrix. + const gfx::Transform transform(transformed.width() / original.width(), 0.0f, + 0.0f, transformed.height() / original.height(), + transformed.x() - original.x(), + transformed.y() - original.y()); + gfx::RectF result(rect); + transform.TransformRect(&result); + return result; +} + +std::ostream& operator<<(std::ostream& out, const FrameTestUtil::RGB& rgb) { + return (out << "{r=" << rgb.r << ",g=" << rgb.g << ",b=" << rgb.b << '}'); +} + +} // namespace content diff --git a/chromium/content/browser/media/capture/frame_test_util.h b/chromium/content/browser/media/capture/frame_test_util.h new file mode 100644 index 00000000000..c3667e59b48 --- /dev/null +++ b/chromium/content/browser/media/capture/frame_test_util.h @@ -0,0 +1,64 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONTENT_BROWSER_MEDIA_CAPTURE_FRAME_TEST_UTIL_H_ +#define CONTENT_BROWSER_MEDIA_CAPTURE_FRAME_TEST_UTIL_H_ + +#include <ostream> + +#include "third_party/skia/include/core/SkBitmap.h" +#include "third_party/skia/include/core/SkColor.h" + +namespace gfx { +class Rect; +class RectF; +} // namespace gfx + +namespace content { + +class FrameTestUtil { + public: + struct RGB { + double r; + double g; + double b; + }; + + // Returns the average RGB color in |include_rect| except for pixels also in + // |exclude_rect|. + static RGB ComputeAverageColor(SkBitmap frame, + const gfx::Rect& include_rect, + const gfx::Rect& exclude_rect); + + // Returns true if the red, green, and blue components are all within + // |max_diff| of each other. + static bool IsApproximatelySameColor(SkColor color, + const RGB& rgb, + int max_diff = kMaxColorDifference); + + // Determines how |original| has been scaled and translated to become + // |transformed|, and then applies the same transform on |rect| and returns + // the result. + static gfx::RectF TransformSimilarly(const gfx::Rect& original, + const gfx::RectF& transformed, + const gfx::Rect& rect); + + // The default maximum color value difference, assuming there will be a little + // error due to pixel boundaries being rounded after coordinate system + // transforms. + static constexpr int kMaxColorDifference = 16; + + // A much more-relaxed maximum color value difference, assuming errors caused + // by indifference towards color space concerns (and also "studio" versus + // "jpeg" YUV ranges). + // TODO(crbug/810131): Once color space issues are fixed, remove this. + static constexpr int kMaxInaccurateColorDifference = 48; +}; + +// A convenience for logging and gtest expectations output. +std::ostream& operator<<(std::ostream& out, const FrameTestUtil::RGB& rgb); + +} // namespace content + +#endif // CONTENT_BROWSER_MEDIA_CAPTURE_FRAME_TEST_UTIL_H_ diff --git a/chromium/content/browser/media/capture/lame_window_capturer_chromeos.cc b/chromium/content/browser/media/capture/lame_window_capturer_chromeos.cc new file mode 100644 index 00000000000..95e5e769f1a --- /dev/null +++ b/chromium/content/browser/media/capture/lame_window_capturer_chromeos.cc @@ -0,0 +1,373 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/capture/lame_window_capturer_chromeos.h" + +#include "base/bind.h" +#include "base/bind_helpers.h" +#include "base/location.h" +#include "components/viz/common/frame_sinks/copy_output_request.h" +#include "components/viz/common/frame_sinks/copy_output_result.h" +#include "media/base/limits.h" +#include "media/base/video_util.h" +#include "mojo/public/cpp/bindings/strong_binding.h" +#include "ui/gfx/geometry/rect.h" + +using media::VideoFrame; +using media::VideoFrameMetadata; + +namespace content { + +namespace { +// Returns |raw_size| with width and height truncated to even-numbered values. +gfx::Size AdjustSizeForI420Format(const gfx::Size& raw_size) { + return gfx::Size(raw_size.width() & ~1, raw_size.height() & ~1); +} +} // namespace + +// static +constexpr base::TimeDelta LameWindowCapturerChromeOS::kAbsoluteMinCapturePeriod; + +LameWindowCapturerChromeOS::LameWindowCapturerChromeOS(aura::Window* target) + : target_(target), + copy_request_source_(base::UnguessableToken::Create()), + weak_factory_(this) { + if (target_) { + target_->AddObserver(this); + } +} + +LameWindowCapturerChromeOS::~LameWindowCapturerChromeOS() { + if (target_) { + target_->RemoveObserver(this); + } +} + +void LameWindowCapturerChromeOS::SetFormat(media::VideoPixelFormat format, + media::ColorSpace color_space) { + if (format != media::PIXEL_FORMAT_I420) { + LOG(DFATAL) << "Invalid pixel format: Only I420 is supported."; + } + + if (color_space != media::COLOR_SPACE_UNSPECIFIED && + color_space != media::COLOR_SPACE_HD_REC709) { + LOG(DFATAL) << "Unsupported color space: Only BT.709 is supported."; + } +} + +void LameWindowCapturerChromeOS::SetMinCapturePeriod( + base::TimeDelta min_capture_period) { + capture_period_ = std::max(min_capture_period, kAbsoluteMinCapturePeriod); + + // If the capture period is being changed while the timer is already running, + // re-start with the new capture period. + if (timer_.IsRunning()) { + timer_.Start(FROM_HERE, capture_period_, this, + &LameWindowCapturerChromeOS::CaptureNextFrame); + } +} + +void LameWindowCapturerChromeOS::SetMinSizeChangePeriod( + base::TimeDelta min_period) {} + +void LameWindowCapturerChromeOS::SetResolutionConstraints( + const gfx::Size& min_size, + const gfx::Size& max_size, + bool use_fixed_aspect_ratio) { + if (max_size.width() <= 1 || max_size.height() <= 1 || + max_size.width() > media::limits::kMaxDimension || + max_size.height() > media::limits::kMaxDimension) { + LOG(DFATAL) << "Invalid max_size (" << max_size.ToString() + << "): It must be within media::limits."; + return; + } + + // Set the capture size to the max size, adjusted for the I420 format. + capture_size_ = AdjustSizeForI420Format(max_size); + DCHECK(!capture_size_.IsEmpty()); + + // Cancel any in-flight captures that would be using the old size and clear + // the buffer pool. + weak_factory_.InvalidateWeakPtrs(); + buffer_pool_.clear(); + in_flight_count_ = 0; +} + +void LameWindowCapturerChromeOS::SetAutoThrottlingEnabled(bool enabled) { + NOTIMPLEMENTED(); +} + +void LameWindowCapturerChromeOS::ChangeTarget( + const viz::FrameSinkId& frame_sink_id) { + // The LameWindowCapturerChromeOS does not capture from compositor frame + // sinks. +} + +void LameWindowCapturerChromeOS::Start( + viz::mojom::FrameSinkVideoConsumerPtr consumer) { + DCHECK(consumer); + + Stop(); + + consumer_ = std::move(consumer); + // In the future, if the connection to the consumer is lost before a call to + // Stop(), make that call on its behalf. + consumer_.set_connection_error_handler(base::BindOnce( + &LameWindowCapturerChromeOS::Stop, base::Unretained(this))); + + timer_.Start(FROM_HERE, capture_period_, this, + &LameWindowCapturerChromeOS::CaptureNextFrame); +} + +void LameWindowCapturerChromeOS::Stop() { + // Stop the timer, cancel any in-flight frames, and clear the buffer pool. + timer_.Stop(); + weak_factory_.InvalidateWeakPtrs(); + buffer_pool_.clear(); + in_flight_count_ = 0; + + if (consumer_) { + consumer_->OnStopped(); + consumer_.reset(); + } +} + +void LameWindowCapturerChromeOS::RequestRefreshFrame() { + // This is ignored because the LameWindowCapturerChromeOS captures frames + // continuously. +} + +class LameWindowCapturerChromeOS::InFlightFrame + : public viz::mojom::FrameSinkVideoConsumerFrameCallbacks { + public: + InFlightFrame(base::WeakPtr<LameWindowCapturerChromeOS> capturer, + BufferAndSize buffer) + : capturer_(std::move(capturer)), buffer_(std::move(buffer)) {} + + ~InFlightFrame() final { Done(); } + + mojo::ScopedSharedBufferHandle CloneBufferHandle() { + return buffer_.first->Clone( + mojo::SharedBufferHandle::AccessMode::READ_WRITE); + } + + size_t buffer_size() const { return buffer_.second; } + + VideoFrame* video_frame() const { return video_frame_.get(); } + void set_video_frame(scoped_refptr<VideoFrame> frame) { + video_frame_ = std::move(frame); + } + + const gfx::Rect& content_rect() const { return content_rect_; } + void set_content_rect(const gfx::Rect& rect) { content_rect_ = rect; } + + void Done() final { + if (auto* capturer = capturer_.get()) { + DCHECK_GT(capturer->in_flight_count_, 0); + --capturer->in_flight_count_; + // If the capture size hasn't changed, return the buffer to the pool. + if (buffer_.second == + VideoFrame::AllocationSize(media::PIXEL_FORMAT_I420, + capturer->capture_size_)) { + capturer->buffer_pool_.emplace_back(std::move(buffer_)); + } + capturer_ = nullptr; + } + + buffer_.first.reset(); + buffer_.second = 0; + } + + void ProvideFeedback(double utilization) final {} + + private: + base::WeakPtr<LameWindowCapturerChromeOS> capturer_; + BufferAndSize buffer_; + scoped_refptr<VideoFrame> video_frame_; + gfx::Rect content_rect_; + + DISALLOW_COPY_AND_ASSIGN(InFlightFrame); +}; + +void LameWindowCapturerChromeOS::CaptureNextFrame() { + // If the maximum frame in-flight count has been reached, skip this frame. + if (in_flight_count_ >= kMaxFramesInFlight) { + return; + } + + // Attempt to re-use a buffer from the pool. Otherwise, create a new one. + const size_t allocation_size = + VideoFrame::AllocationSize(media::PIXEL_FORMAT_I420, capture_size_); + BufferAndSize buffer; + if (buffer_pool_.empty()) { + buffer.first = mojo::SharedBufferHandle::Create(allocation_size); + if (!buffer.first.is_valid()) { + // If creating the shared memory failed, abort the frame, and hope this + // is a transient problem. + return; + } + buffer.second = allocation_size; + } else { + buffer = std::move(buffer_pool_.back()); + buffer_pool_.pop_back(); + DCHECK(buffer.first.is_valid()); + DCHECK_EQ(buffer.second, allocation_size); + } + + // Map the shared memory buffer, to populate its data. + mojo::ScopedSharedBufferMapping mapping = buffer.first->Map(buffer.second); + if (!mapping) { + // If the shared memory mapping failed, just abort this frame, hoping the + // issue is a transient one (e.g., lack of an available region in address + // space). + return; + } + + // At this point, frame capture will proceed. Create an InFlightFrame to track + // population and consumption of the frame, and to eventually return the + // buffer to the pool and decrement |in_flight_count_|. + ++in_flight_count_; + auto in_flight_frame = std::make_unique<InFlightFrame>( + weak_factory_.GetWeakPtr(), std::move(buffer)); + + // Create a VideoFrame that wraps the mapped buffer. + const base::TimeTicks begin_time = base::TimeTicks::Now(); + if (first_frame_reference_time_.is_null()) { + first_frame_reference_time_ = begin_time; + } + in_flight_frame->set_video_frame(VideoFrame::WrapExternalData( + media::PIXEL_FORMAT_I420, capture_size_, gfx::Rect(capture_size_), + capture_size_, static_cast<uint8_t*>(mapping.get()), allocation_size, + begin_time - first_frame_reference_time_)); + auto* const frame = in_flight_frame->video_frame(); + DCHECK(frame); + frame->AddDestructionObserver( + base::BindOnce(base::DoNothing::Once<mojo::ScopedSharedBufferMapping>(), + std::move(mapping))); + VideoFrameMetadata* const metadata = frame->metadata(); + metadata->SetTimeTicks(VideoFrameMetadata::CAPTURE_BEGIN_TIME, begin_time); + metadata->SetInteger(VideoFrameMetadata::COLOR_SPACE, + media::COLOR_SPACE_HD_REC709); + metadata->SetTimeDelta(VideoFrameMetadata::FRAME_DURATION, capture_period_); + metadata->SetDouble(VideoFrameMetadata::FRAME_RATE, + 1.0 / capture_period_.InSecondsF()); + metadata->SetTimeTicks(VideoFrameMetadata::REFERENCE_TIME, begin_time); + + // Compute the region of the VideoFrame that will contain the content. If + // there is nothing to copy from/to (e.g., the target is gone, or is sized too + // small), send a blank black frame immediately. + const gfx::Size source_size = + target_ ? target_->bounds().size() : gfx::Size(); + const gfx::Rect content_rect = source_size.IsEmpty() + ? gfx::Rect() + : media::ComputeLetterboxRegionForI420( + frame->visible_rect(), source_size); + in_flight_frame->set_content_rect(content_rect); + if (content_rect.IsEmpty()) { + media::LetterboxVideoFrame(frame, gfx::Rect()); + DeliverFrame(std::move(in_flight_frame)); + return; + } + DCHECK(target_); + + // Request a copy of the Layer associated with the |target_| aura::Window. + auto request = std::make_unique<viz::CopyOutputRequest>( + // Note: As of this writing, I420_PLANES is not supported external to VIZ. + viz::CopyOutputRequest::ResultFormat::RGBA_BITMAP, + base::BindOnce(&LameWindowCapturerChromeOS::DidCopyFrame, + weak_factory_.GetWeakPtr(), std::move(in_flight_frame))); + request->set_source(copy_request_source_); + request->set_area(gfx::Rect(source_size)); + request->SetScaleRatio( + gfx::Vector2d(source_size.width(), source_size.height()), + gfx::Vector2d(content_rect.width(), content_rect.height())); + request->set_result_selection(gfx::Rect(content_rect.size())); + target_->layer()->RequestCopyOfOutput(std::move(request)); +} + +void LameWindowCapturerChromeOS::DidCopyFrame( + std::unique_ptr<InFlightFrame> in_flight_frame, + std::unique_ptr<viz::CopyOutputResult> result) { + // Populate the VideoFrame from the CopyOutputResult. + auto* const frame = in_flight_frame->video_frame(); + DCHECK(frame); + const auto& content_rect = in_flight_frame->content_rect(); + const int y_stride = frame->stride(VideoFrame::kYPlane); + uint8_t* const y = frame->visible_data(VideoFrame::kYPlane) + + content_rect.y() * y_stride + content_rect.x(); + const int u_stride = frame->stride(VideoFrame::kUPlane); + uint8_t* const u = frame->visible_data(VideoFrame::kUPlane) + + (content_rect.y() / 2) * u_stride + (content_rect.x() / 2); + const int v_stride = frame->stride(VideoFrame::kVPlane); + uint8_t* const v = frame->visible_data(VideoFrame::kVPlane) + + (content_rect.y() / 2) * v_stride + (content_rect.x() / 2); + if (!result->ReadI420Planes(y, y_stride, u, u_stride, v, v_stride)) { + return; // Copy request failed, punt. + } + + // The result may be smaller than what was requested, if unforeseen clamping + // to the source boundaries occurred by the executor of the copy request. + // However, the result should never contain more than what was requested. + DCHECK_LE(result->size().width(), content_rect.width()); + DCHECK_LE(result->size().height(), content_rect.height()); + media::LetterboxVideoFrame( + frame, gfx::Rect(content_rect.origin(), + AdjustSizeForI420Format(result->size()))); + + DeliverFrame(std::move(in_flight_frame)); +} + +void LameWindowCapturerChromeOS::DeliverFrame( + std::unique_ptr<InFlightFrame> in_flight_frame) { + auto* const frame = in_flight_frame->video_frame(); + DCHECK(frame); + frame->metadata()->SetTimeTicks(VideoFrameMetadata::CAPTURE_END_TIME, + base::TimeTicks::Now()); + + // Clone the buffer handle for the consumer. + mojo::ScopedSharedBufferHandle buffer_for_consumer = + in_flight_frame->CloneBufferHandle(); + if (!buffer_for_consumer.is_valid()) { + return; // This should only fail if the OS is exhausted of handles. + } + const size_t buffer_allocation_size = in_flight_frame->buffer_size(); + + // Assemble frame layout, format, and metadata into a mojo struct to send to + // the consumer. + media::mojom::VideoFrameInfoPtr info = media::mojom::VideoFrameInfo::New(); + info->timestamp = frame->timestamp(); + info->metadata = frame->metadata()->GetInternalValues().Clone(); + info->pixel_format = frame->format(); + info->coded_size = frame->coded_size(); + info->visible_rect = frame->visible_rect(); + const gfx::Rect update_rect = frame->visible_rect(); + const gfx::Rect content_rect = in_flight_frame->content_rect(); + + // Drop the VideoFrame wrapper, which will unmap the shared memory from this + // process. + in_flight_frame->set_video_frame(nullptr); + + // Create a mojo message pipe and bind to the InFlightFrame to wait for the + // Done() signal from the consumer. The mojo::StrongBinding takes ownership of + // the InFlightFrame. + viz::mojom::FrameSinkVideoConsumerFrameCallbacksPtr callbacks; + mojo::MakeStrongBinding(std::move(in_flight_frame), + mojo::MakeRequest(&callbacks)); + + // Send the frame to the consumer. + consumer_->OnFrameCaptured(std::move(buffer_for_consumer), + buffer_allocation_size, std::move(info), + update_rect, content_rect, std::move(callbacks)); +} + +void LameWindowCapturerChromeOS::OnWindowDestroying(aura::Window* window) { + if (window == target_) { + target_->RemoveObserver(this); + target_ = nullptr; + // The capturer may continue running, but it will notice the target is gone + // and produce blank black frames hereafter. + } +} + +} // namespace content diff --git a/chromium/content/browser/media/capture/lame_window_capturer_chromeos.h b/chromium/content/browser/media/capture/lame_window_capturer_chromeos.h new file mode 100644 index 00000000000..6978f755f07 --- /dev/null +++ b/chromium/content/browser/media/capture/lame_window_capturer_chromeos.h @@ -0,0 +1,139 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONTENT_BROWSER_MEDIA_CAPTURE_LAME_WINDOW_CAPTURER_CHROMEOS_H_ +#define CONTENT_BROWSER_MEDIA_CAPTURE_LAME_WINDOW_CAPTURER_CHROMEOS_H_ + +#include <utility> +#include <vector> + +#include "base/macros.h" +#include "base/memory/weak_ptr.h" +#include "base/time/time.h" +#include "base/timer/timer.h" +#include "base/unguessable_token.h" +#include "media/base/video_frame.h" +#include "mojo/public/cpp/system/buffer.h" +#include "services/viz/privileged/interfaces/compositing/frame_sink_video_capture.mojom.h" +#include "ui/aura/window.h" +#include "ui/aura/window_observer.h" +#include "ui/gfx/geometry/size.h" + +namespace viz { +class CopyOutputResult; +} + +namespace content { + +// A minimal FrameSinkVideoCapturer implementation for aura::Window video +// capture on ChromeOS (i.e., not desktop capture, and not WebContents capture), +// in cases where a Window does not host a compositor frame sink. This is far +// less efficient than, and far under-performs, the normal +// FrameSinkVideoCapturer provided by the VIZ service, as it lacks multiple +// design features that would be required for low CPU use and high +// pixels-per-sec throughput. It is a placeholder, until the necessary +// infrastructure exists to provide VIZ FrameSinkVideoCapturer the compositor +// frame sink it needs for aura::Windows in the middle of the window tree +// hierarchy. +// +// As this is not meant to be a full-fledged, long-term implementation, it only +// supports the production of I420-format video (Rec. 709 color space) at a +// maximum rate of 5 FPS, and only a maximum of 3 frames can be in-flight at any +// one time. In addition, since source content changes cannot be detected, this +// capturer indefinitely produces frames at a constant framerate while it is +// running. +// +// TODO(crbug/806366): The goal is to remove this code by 2019. +class LameWindowCapturerChromeOS : public viz::mojom::FrameSinkVideoCapturer, + public aura::WindowObserver { + public: + explicit LameWindowCapturerChromeOS(aura::Window* target); + ~LameWindowCapturerChromeOS() final; + + // viz::mojom::FrameSinkVideoCapturer implementation. + void SetFormat(media::VideoPixelFormat format, + media::ColorSpace color_space) final; + void SetMinCapturePeriod(base::TimeDelta min_capture_period) final; + void SetMinSizeChangePeriod(base::TimeDelta min_period) final; + void SetResolutionConstraints(const gfx::Size& min_size, + const gfx::Size& max_size, + bool use_fixed_aspect_ratio) final; + void SetAutoThrottlingEnabled(bool enabled) final; + void ChangeTarget(const viz::FrameSinkId& frame_sink_id) final; + void Start(viz::mojom::FrameSinkVideoConsumerPtr consumer) final; + void Stop() final; + void RequestRefreshFrame() final; + + private: + // Represents an in-flight frame, being populated by this capturer and then + // delivered to the consumer. When the consumer is done with the frame, this + // returns the buffer back to the pool. + class InFlightFrame; + + // Initiates capture of the next frame. This is called periodically by the + // |timer_|. + void CaptureNextFrame(); + + // Populates the frame from the CopyOutputResult and then calls DeliverFrame. + void DidCopyFrame(std::unique_ptr<InFlightFrame> in_flight_frame, + std::unique_ptr<viz::CopyOutputResult> result); + + // Delivers the frame to the consumer, and sets up the notification path for + // when the consumer is done with the frame. + void DeliverFrame(std::unique_ptr<InFlightFrame> in_flight_frame); + + // aura::WindowObserver override. + void OnWindowDestroying(aura::Window* window) final; + + // The window being captured. If the window is destroyed, this is set to null + // and only blank black frames will be produced thereafter. + aura::Window* target_; + + // Capture parameters. The defaults are according to the mojo interface + // definition comments for viz::mojom::FrameSinkVideoCapturer. + base::TimeDelta capture_period_ = kAbsoluteMinCapturePeriod; + gfx::Size capture_size_ = gfx::Size(640, 360); + + // The current consumer. This is set by Start() and cleared by Stop(). + viz::mojom::FrameSinkVideoConsumerPtr consumer_; + + // A timer that calls CaptureNextFrame() periodically, according to the + // currently-set |capture_period_|. This timer is only running while a + // consumer is present. + base::RepeatingTimer timer_; + + // A pool of shared memory buffers for re-use. + using BufferAndSize = std::pair<mojo::ScopedSharedBufferHandle, size_t>; + std::vector<BufferAndSize> buffer_pool_; + + // The current number of frames in-flight. If incrementing this would be + // exceed kMaxInFlightFrames, frame capture is not attempted. + int in_flight_count_ = 0; + + // Tick clock time of the first frame since Start() was called. This is used + // for generating "media offset" VideoFrame timestamps. + base::TimeTicks first_frame_reference_time_; + + // A value provided in the copy requests to enable VIZ to optimize around + // video capture. + const base::UnguessableToken copy_request_source_; + + // Used for cancelling any outstanding activities' results, once Stop() is + // called and there is no longer a consumer to receive another frame. + base::WeakPtrFactory<LameWindowCapturerChromeOS> weak_factory_; + + // Enforce a very low maximum frame rate (5 FPS), due to the lack of + // design optimizations. See top-level class comments. + static constexpr base::TimeDelta kAbsoluteMinCapturePeriod = + base::TimeDelta::FromMilliseconds(200); + + // The maximum number of frames in-flight at any one time. + static constexpr int kMaxFramesInFlight = 3; + + DISALLOW_COPY_AND_ASSIGN(LameWindowCapturerChromeOS); +}; + +} // namespace content + +#endif // CONTENT_BROWSER_MEDIA_CAPTURE_LAME_WINDOW_CAPTURER_CHROMEOS_H_ diff --git a/chromium/content/browser/media/capture/screen_capture_device_android_unittest.cc b/chromium/content/browser/media/capture/screen_capture_device_android_unittest.cc index 994bfae8014..dda49dcb222 100644 --- a/chromium/content/browser/media/capture/screen_capture_device_android_unittest.cc +++ b/chromium/content/browser/media/capture/screen_capture_device_android_unittest.cc @@ -25,6 +25,13 @@ class MockDeviceClient : public media::VideoCaptureDevice::Client { base::TimeTicks reference_time, base::TimeDelta tiemstamp, int frame_feedback_id)); + MOCK_METHOD6(OnIncomingCapturedGfxBuffer, + void(gfx::GpuMemoryBuffer* buffer, + const media::VideoCaptureFormat& frame_format, + int clockwise_rotation, + base::TimeTicks reference_time, + base::TimeDelta timestamp, + int frame_feedback_id)); MOCK_METHOD0(DoReserveOutputBuffer, void(void)); MOCK_METHOD0(DoOnIncomingCapturedBuffer, void(void)); MOCK_METHOD0(DoOnIncomingCapturedVideoFrame, void(void)); @@ -38,10 +45,8 @@ class MockDeviceClient : public media::VideoCaptureDevice::Client { // Trampoline methods to workaround GMOCK problems with std::unique_ptr<>. Buffer ReserveOutputBuffer(const gfx::Size& dimensions, media::VideoPixelFormat format, - media::VideoPixelStorage storage, int frame_feedback_id) override { EXPECT_EQ(media::PIXEL_FORMAT_I420, format); - EXPECT_EQ(media::VideoPixelStorage::CPU, storage); DoReserveOutputBuffer(); return Buffer(); } @@ -62,10 +67,8 @@ class MockDeviceClient : public media::VideoCaptureDevice::Client { } Buffer ResurrectLastOutputBuffer(const gfx::Size& dimensions, media::VideoPixelFormat format, - media::VideoPixelStorage storage, int frame_feedback_id) override { EXPECT_EQ(media::PIXEL_FORMAT_I420, format); - EXPECT_EQ(media::VideoPixelStorage::CPU, storage); DoResurrectLastOutputBuffer(); return Buffer(); } diff --git a/chromium/content/browser/media/capture/web_contents_audio_input_stream.cc b/chromium/content/browser/media/capture/web_contents_audio_input_stream.cc index 15957b6d3ec..15115a22b8b 100644 --- a/chromium/content/browser/media/capture/web_contents_audio_input_stream.cc +++ b/chromium/content/browser/media/capture/web_contents_audio_input_stream.cc @@ -441,4 +441,9 @@ bool WebContentsAudioInputStream::IsMuted() { return false; } +void WebContentsAudioInputStream::SetOutputDeviceForAec( + const std::string& output_device_id) { + // Not supported. Do nothing. +} + } // namespace content diff --git a/chromium/content/browser/media/capture/web_contents_audio_input_stream.h b/chromium/content/browser/media/capture/web_contents_audio_input_stream.h index a7047aff624..427615b9826 100644 --- a/chromium/content/browser/media/capture/web_contents_audio_input_stream.h +++ b/chromium/content/browser/media/capture/web_contents_audio_input_stream.h @@ -48,6 +48,7 @@ class CONTENT_EXPORT WebContentsAudioInputStream bool SetAutomaticGainControl(bool enabled) override; bool GetAutomaticGainControl() override; bool IsMuted() override; + void SetOutputDeviceForAec(const std::string& output_device_id) override; // Create a new audio mirroring session, or return NULL on error. |device_id| // should be in the format accepted by diff --git a/chromium/content/browser/media/capture/web_contents_audio_input_stream_unittest.cc b/chromium/content/browser/media/capture/web_contents_audio_input_stream_unittest.cc index 7c1911b90a2..e0d3842142b 100644 --- a/chromium/content/browser/media/capture/web_contents_audio_input_stream_unittest.cc +++ b/chromium/content/browser/media/capture/web_contents_audio_input_stream_unittest.cc @@ -55,9 +55,8 @@ const int kAnotherRenderFrameId = 1; const AudioParameters& TestAudioParameters() { static const AudioParameters params( - AudioParameters::AUDIO_FAKE, - media::CHANNEL_LAYOUT_STEREO, - AudioParameters::kAudioCDSampleRate, 16, + AudioParameters::AUDIO_FAKE, media::CHANNEL_LAYOUT_STEREO, + AudioParameters::kAudioCDSampleRate, AudioParameters::kAudioCDSampleRate / 100); return params; } @@ -65,7 +64,7 @@ const AudioParameters& TestAudioParameters() { class MockAudioMirroringManager : public AudioMirroringManager { public: MockAudioMirroringManager() : AudioMirroringManager() {} - virtual ~MockAudioMirroringManager() {} + ~MockAudioMirroringManager() override {} MOCK_METHOD1(StartMirroring, void(MirroringDestination* destination)); MOCK_METHOD1(StopMirroring, void(MirroringDestination* destination)); @@ -84,7 +83,7 @@ class MockWebContentsTracker : public WebContentsTracker { MOCK_METHOD0(Stop, void()); private: - virtual ~MockWebContentsTracker() {} + ~MockWebContentsTracker() override {} DISALLOW_COPY_AND_ASSIGN(MockWebContentsTracker); }; @@ -131,9 +130,7 @@ class MockVirtualAudioInputStream : public VirtualAudioInputStream { Invoke(&real_, &VirtualAudioInputStream::RemoveInputProvider)); } - ~MockVirtualAudioInputStream() { - DCHECK(real_stream_is_closed_); - } + ~MockVirtualAudioInputStream() override { DCHECK(real_stream_is_closed_); } MOCK_METHOD0(Open, bool()); MOCK_METHOD1(Start, void(AudioInputStream::AudioInputCallback*)); diff --git a/chromium/content/browser/media/capture/web_contents_video_capture_device_browsertest.cc b/chromium/content/browser/media/capture/web_contents_video_capture_device_browsertest.cc index 5a6951ca012..643c570b0a0 100644 --- a/chromium/content/browser/media/capture/web_contents_video_capture_device_browsertest.cc +++ b/chromium/content/browser/media/capture/web_contents_video_capture_device_browsertest.cc @@ -4,300 +4,43 @@ #include "content/browser/media/capture/web_contents_video_capture_device.h" -#include <stdint.h> - -#include <array> -#include <cmath> -#include <string> #include <tuple> -#include "base/containers/circular_deque.h" -#include "base/containers/flat_map.h" +#include "base/macros.h" #include "base/run_loop.h" -#include "base/strings/stringprintf.h" #include "build/build_config.h" #include "cc/test/pixel_test_utils.h" +#include "content/browser/media/capture/content_capture_device_browsertest_base.h" +#include "content/browser/media/capture/fake_video_capture_stack.h" +#include "content/browser/media/capture/frame_test_util.h" #include "content/public/browser/browser_thread.h" #include "content/public/browser/render_frame_host.h" #include "content/public/browser/render_process_host.h" #include "content/public/browser/render_widget_host_view.h" #include "content/public/browser/web_contents.h" -#include "content/public/test/browser_test_utils.h" -#include "content/public/test/content_browser_test.h" -#include "content/public/test/content_browser_test_utils.h" -#include "content/public/test/test_utils.h" #include "content/shell/browser/shell.h" -#include "media/base/video_frame.h" #include "media/base/video_util.h" -#include "media/capture/video/video_frame_receiver.h" -#include "media/capture/video_capture_types.h" -#include "net/dns/mock_host_resolver.h" -#include "net/test/embedded_test_server/embedded_test_server.h" -#include "net/test/embedded_test_server/http_request.h" -#include "net/test/embedded_test_server/http_response.h" #include "testing/gtest/include/gtest/gtest.h" -#include "third_party/libyuv/include/libyuv.h" #include "third_party/skia/include/core/SkBitmap.h" -#include "third_party/skia/include/core/SkColor.h" #include "ui/gfx/geometry/rect.h" -#include "ui/gfx/geometry/size.h" -#include "url/gurl.h" - -using net::test_server::BasicHttpResponse; -using net::test_server::HttpRequest; -using net::test_server::HttpResponse; +#include "ui/gfx/geometry/rect_conversions.h" +#include "ui/gfx/geometry/rect_f.h" namespace content { namespace { -// Provides a fake representation of the entire video capture stack. It creates -// a VideoFrameReceiver that the device can deliver VideoFrames to, and adapts -// that to a simple callback structure that allows the browser tests to examine -// each video frame that was captured. -class FakeVideoCaptureStack { - public: - using FrameCallback = - base::RepeatingCallback<void(scoped_refptr<media::VideoFrame> frame)>; - void SetFrameCallback(FrameCallback callback) { - frame_callback_ = std::move(callback); - } - - std::unique_ptr<media::VideoFrameReceiver> CreateFrameReceiver() { - return std::make_unique<FakeVideoFrameReceiver>(this); - } - - bool started() const { return started_; } - bool error_occurred() const { return error_occurred_; } - - void ExpectHasLogMessages() { - EXPECT_FALSE(log_messages_.empty()); - while (!log_messages_.empty()) { - VLOG(1) << "Next log message: " << log_messages_.front(); - log_messages_.pop_front(); - } - } - - void ExpectNoLogMessages() { - while (!log_messages_.empty()) { - ADD_FAILURE() << "Unexpected log message: " << log_messages_.front(); - log_messages_.pop_front(); - } - } - - private: - // A minimal implementation of VideoFrameReceiver that wraps buffers into - // VideoFrame instances and forwards all relevant callbacks and data to the - // parent FakeVideoCaptureStack. - class FakeVideoFrameReceiver : public media::VideoFrameReceiver { - public: - explicit FakeVideoFrameReceiver(FakeVideoCaptureStack* capture_stack) - : capture_stack_(capture_stack) {} - - private: - using Buffer = media::VideoCaptureDevice::Client::Buffer; - - void OnNewBufferHandle( - int buffer_id, - std::unique_ptr<Buffer::HandleProvider> handle_provider) final { - buffers_[buffer_id] = - handle_provider->GetHandleForInterProcessTransit(true); - } - - void OnFrameReadyInBuffer( - int buffer_id, - int frame_feedback_id, - std::unique_ptr<Buffer::ScopedAccessPermission> access, - media::mojom::VideoFrameInfoPtr frame_info) final { - const auto it = buffers_.find(buffer_id); - CHECK(it != buffers_.end()); - mojo::ScopedSharedBufferHandle& buffer = it->second; - - const size_t mapped_size = - media::VideoCaptureFormat(frame_info->coded_size, 0.0f, - frame_info->pixel_format, - frame_info->storage_type) - .ImageAllocationSize(); - mojo::ScopedSharedBufferMapping mapping = buffer->Map(mapped_size); - CHECK(mapping.get()); - - auto frame = media::VideoFrame::WrapExternalData( - frame_info->pixel_format, frame_info->coded_size, - frame_info->visible_rect, frame_info->visible_rect.size(), - reinterpret_cast<uint8_t*>(mapping.get()), mapped_size, - frame_info->timestamp); - CHECK(frame); - frame->metadata()->MergeInternalValuesFrom(frame_info->metadata); - // This destruction observer will unmap the shared memory when the - // VideoFrame goes out-of-scope. - frame->AddDestructionObserver(base::BindOnce( - [](mojo::ScopedSharedBufferMapping mapping) {}, std::move(mapping))); - // This destruction observer will notify the WebContentsVideoCaptureDevice - // once all downstream code is done using the VideoFrame. - frame->AddDestructionObserver(base::BindOnce( - [](std::unique_ptr<Buffer::ScopedAccessPermission> access) {}, - std::move(access))); - - capture_stack_->frame_callback_.Run(std::move(frame)); - } - - void OnBufferRetired(int buffer_id) final { - const auto it = buffers_.find(buffer_id); - CHECK(it != buffers_.end()); - buffers_.erase(it); - } - - void OnError() final { capture_stack_->error_occurred_ = true; } - - void OnLog(const std::string& message) final { - capture_stack_->log_messages_.push_back(message); - } - - void OnStarted() final { capture_stack_->started_ = true; } - - void OnStartedUsingGpuDecode() final { NOTREACHED(); } - - FakeVideoCaptureStack* const capture_stack_; - base::flat_map<int, mojo::ScopedSharedBufferHandle> buffers_; - }; - - FrameCallback frame_callback_; - bool started_ = false; - bool error_occurred_ = false; - base::circular_deque<std::string> log_messages_; -}; - -class WebContentsVideoCaptureDeviceBrowserTest : public ContentBrowserTest { +class WebContentsVideoCaptureDeviceBrowserTest + : public ContentCaptureDeviceBrowserTestBase { public: - FakeVideoCaptureStack* capture_stack() { return &capture_stack_; } - WebContentsVideoCaptureDevice* device() const { return device_.get(); } - - // Alters the solid fill color making up the page content. This will trigger a - // compositor update, which will trigger a frame capture. - void ChangePageContentColor(std::string css_color_hex) { - // See the HandleRequest() method for the original documents being modified - // here. - std::string script; - if (is_cross_site_capture_test()) { - const GURL& inner_frame_url = - embedded_test_server()->GetURL(kInnerFrameHostname, kInnerFramePath); - script = base::StringPrintf( - "document.getElementsByTagName('iframe')[0].src = '%s?color=123456';", - inner_frame_url.spec().c_str()); - } else { - script = "document.body.style.backgroundColor = '#123456';"; - } - script.replace(script.find("123456"), 6, css_color_hex); - CHECK(ExecuteScript(shell()->web_contents(), script)); - } - - // Returns the size of the WebContents top-level frame view. - gfx::Size GetViewSize() const { - return shell() - ->web_contents() - ->GetMainFrame() - ->GetView() - ->GetViewBounds() - .size(); - } - - // Returns capture parameters based on the current size of the source view, - // which is based on the size of the Shell window. - media::VideoCaptureParams SnapshotCaptureParams() const { - constexpr gfx::Size kMaxCaptureSize = gfx::Size(320, 320); - constexpr int kMaxFramesPerSecond = 60; - - gfx::Size capture_size = kMaxCaptureSize; - if (use_fixed_aspect_ratio()) { - // Half either the width or height, depending on the source view size. The - // goal is to force obvious letterboxing (or pillarboxing), regardless of - // how the source view is currently sized. - const gfx::Size view_size = GetViewSize(); - if (view_size.width() < view_size.height()) { - capture_size.set_height(capture_size.height() / 2); - } else { - capture_size.set_width(capture_size.width() / 2); - } - } - - media::VideoCaptureParams params; - params.requested_format = media::VideoCaptureFormat( - capture_size, kMaxFramesPerSecond, media::PIXEL_FORMAT_I420, - media::VideoPixelStorage::CPU); - params.resolution_change_policy = - use_fixed_aspect_ratio() - ? media::ResolutionChangePolicy::FIXED_ASPECT_RATIO - : media::ResolutionChangePolicy::ANY_WITHIN_LIMIT; - return params; - } - - // Navigates to the initial document, according to the current test - // parameters, and waits for page load completion. - void NavigateToInitialDocument() { - // Navigate to the single-frame test's document and record the view size. - ASSERT_TRUE( - NavigateToURL(shell(), embedded_test_server()->GetURL( - kSingleFrameHostname, kSingleFramePath))); - ASSERT_TRUE(WaitForLoadStop(shell()->web_contents())); - expected_view_size_ = GetViewSize(); - VLOG(1) << "View size is " << expected_view_size_.ToString(); - - // If doing a cross-site capture test, navigate to the more-complex document - // that also contains an iframe (rendered in a separate process). - if (is_cross_site_capture_test()) { - ASSERT_TRUE( - NavigateToURL(shell(), embedded_test_server()->GetURL( - kOuterFrameHostname, kOuterFramePath))); - ASSERT_TRUE(WaitForLoadStop(shell()->web_contents())); - - // Confirm the iframe is a cross-process child render frame. - auto* const child_frame = - ChildFrameAt(shell()->web_contents()->GetMainFrame(), 0); - ASSERT_TRUE(child_frame); - ASSERT_TRUE(child_frame->IsCrossProcessSubframe()); - } - } - - // Creates and starts the device for frame capture, and checks that the - // initial refresh frame is delivered. - void AllocateAndStartAndWaitForFirstFrame() { - frames_.clear(); - last_frame_timestamp_ = base::TimeDelta::Min(); - capture_stack()->SetFrameCallback( - base::BindRepeating(&WebContentsVideoCaptureDeviceBrowserTest::OnFrame, - base::Unretained(this))); - - auto* const main_frame = shell()->web_contents()->GetMainFrame(); - device_ = std::make_unique<WebContentsVideoCaptureDevice>( - main_frame->GetProcess()->GetID(), main_frame->GetRoutingID()); - device_->AllocateAndStartWithReceiver( - SnapshotCaptureParams(), capture_stack()->CreateFrameReceiver()); - RunAllPendingInMessageLoop(BrowserThread::UI); - EXPECT_TRUE(capture_stack()->started()); - EXPECT_FALSE(capture_stack()->error_occurred()); - capture_stack()->ExpectNoLogMessages(); - - min_capture_period_ = base::TimeDelta::FromMicroseconds( - base::Time::kMicrosecondsPerSecond / - device_->capture_params().requested_format.frame_rate); - WaitForFrameWithColor(SK_ColorBLACK); - } + WebContentsVideoCaptureDeviceBrowserTest() = default; + ~WebContentsVideoCaptureDeviceBrowserTest() override = default; - // Stops and destroys the device. - void StopAndDeAllocate() { - device_->StopAndDeAllocate(); - RunAllPendingInMessageLoop(BrowserThread::UI); - device_.reset(); - } - - void ClearCapturedFramesQueue() { frames_.clear(); } - - bool HasCapturedFramesInQueue() const { return !frames_.empty(); } - - // Runs the browser until a frame with the given |color| is found in the - // captured frames queue, or until a testing failure has occurred. + // Runs the browser until a frame whose content matches the given |color| is + // found in the captured frames queue, or until a testing failure has + // occurred. void WaitForFrameWithColor(SkColor color) { - VLOG(1) << "Waiting for frame filled with color: red=" << SkColorGetR(color) - << ", green=" << SkColorGetG(color) + VLOG(1) << "Waiting for frame content area filled with color: red=" + << SkColorGetR(color) << ", green=" << SkColorGetG(color) << ", blue=" << SkColorGetB(color); while (!testing::Test::HasFailure()) { @@ -305,52 +48,59 @@ class WebContentsVideoCaptureDeviceBrowserTest : public ContentBrowserTest { EXPECT_FALSE(capture_stack()->error_occurred()); capture_stack()->ExpectNoLogMessages(); - while (!frames_.empty() && !testing::Test::HasFailure()) { + while (capture_stack()->has_captured_frames() && + !testing::Test::HasFailure()) { // Pop the next frame from the front of the queue and convert to a RGB // bitmap for analysis. - SkBitmap rgb_frame = ConvertToSkBitmap(*(frames_.front())); - frames_.pop_front(); + const SkBitmap rgb_frame = capture_stack()->NextCapturedFrame(); EXPECT_FALSE(rgb_frame.empty()); - // Analyze the frame and compute the average color value for each of: 1) - // the upper-left quadrant of the content region; 2) the remaining three - // quadrants of the content region; and 3) the non-content (i.e., - // letterboxed) region. + // Three regions of the frame will be analyzed: 1) the upper-left + // quadrant of the content region where the iframe draws; 2) the + // remaining three quadrants of the content region where the main frame + // draws; and 3) the non-content (i.e., letterboxed) region. const gfx::Size frame_size(rgb_frame.width(), rgb_frame.height()); - const gfx::Size current_view_size = GetViewSize(); - EXPECT_EQ(expected_view_size_, current_view_size) - << "Sanity-check failed: View size changed sized during this test."; - const gfx::Rect content_rect = - use_fixed_aspect_ratio() - ? media::ComputeLetterboxRegion(gfx::Rect(frame_size), - current_view_size) - : gfx::Rect(frame_size); - std::array<double, 3> average_ul_content_rgb; - std::array<double, 3> average_rem_content_rgb; - std::array<double, 3> average_letterbox_rgb; - AnalyzeFrame(rgb_frame, content_rect, &average_ul_content_rgb, - &average_rem_content_rgb, &average_letterbox_rgb); - - const auto ToTriplet = [](const std::array<double, 3>& rgb) { - return base::StringPrintf("(%f,%f,%f)", rgb[0], rgb[1], rgb[2]); - }; - VLOG(1) << "Video frame analysis: size=" << frame_size.ToString() - << ", expected content_rect=" << content_rect.ToString() - << ", average upper-left content quadrant rgb=" - << ToTriplet(average_ul_content_rgb) - << ", average remaining content rgb=" - << ToTriplet(average_rem_content_rgb) - << ", average letterbox rgb=" - << ToTriplet(average_letterbox_rgb); - - // The letterboxed region should be black. - if (use_fixed_aspect_ratio()) { - EXPECT_NEAR(SkColorGetR(SK_ColorBLACK), average_letterbox_rgb[0], - kMaxColorDifference); - EXPECT_NEAR(SkColorGetG(SK_ColorBLACK), average_letterbox_rgb[1], - kMaxColorDifference); - EXPECT_NEAR(SkColorGetB(SK_ColorBLACK), average_letterbox_rgb[2], - kMaxColorDifference); + const gfx::Size source_size = GetExpectedSourceSize(); + const gfx::Rect iframe_rect(0, 0, source_size.width() / 2, + source_size.height() / 2); + + // Compute the Rects representing where the three regions would be in + // the |rgb_frame|. + const gfx::RectF content_in_frame_rect_f( + IsFixedAspectRatioTest() ? media::ComputeLetterboxRegion( + gfx::Rect(frame_size), source_size) + : gfx::Rect(frame_size)); + const gfx::RectF iframe_in_frame_rect_f = + FrameTestUtil::TransformSimilarly( + gfx::Rect(source_size), content_in_frame_rect_f, iframe_rect); + const gfx::Rect content_in_frame_rect = + gfx::ToEnclosingRect(content_in_frame_rect_f); + const gfx::Rect iframe_in_frame_rect = + gfx::ToEnclosingRect(iframe_in_frame_rect_f); + + // Determine the average RGB color in the three regions-of-interest in + // the frame. + const auto average_iframe_rgb = FrameTestUtil::ComputeAverageColor( + rgb_frame, iframe_in_frame_rect, gfx::Rect()); + const auto average_mainframe_rgb = FrameTestUtil::ComputeAverageColor( + rgb_frame, content_in_frame_rect, iframe_in_frame_rect); + const auto average_letterbox_rgb = FrameTestUtil::ComputeAverageColor( + rgb_frame, gfx::Rect(frame_size), content_in_frame_rect); + + VLOG(1) + << "Video frame analysis: size=" << frame_size.ToString() + << ", captured upper-left quadrant of content should be at " + << iframe_in_frame_rect.ToString() << " and has average color " + << average_iframe_rgb + << ", captured remaining quadrants of content should be bound by " + << content_in_frame_rect.ToString() << " and has average color " + << average_mainframe_rgb << ", letterbox region has average color " + << average_letterbox_rgb; + + // The letterboxed region should always be black. + if (IsFixedAspectRatioTest()) { + EXPECT_TRUE(FrameTestUtil::IsApproximatelySameColor( + SK_ColorBLACK, average_letterbox_rgb)); } if (testing::Test::HasFailure()) { @@ -359,14 +109,19 @@ class WebContentsVideoCaptureDeviceBrowserTest : public ContentBrowserTest { return; } - if (is_cross_site_capture_test() && - IsApproximatelySameColor(color, average_ul_content_rgb) && - IsApproximatelySameColor(SK_ColorWHITE, average_rem_content_rgb)) { + // Return if the content region(s) now has/have the expected color(s). + if (IsCrossSiteCaptureTest() && + FrameTestUtil::IsApproximatelySameColor(color, + average_iframe_rgb) && + FrameTestUtil::IsApproximatelySameColor(SK_ColorWHITE, + average_mainframe_rgb)) { VLOG(1) << "Observed desired frame."; return; - } else if (!is_cross_site_capture_test() && - IsApproximatelySameColor(color, average_ul_content_rgb) && - IsApproximatelySameColor(color, average_rem_content_rgb)) { + } else if (!IsCrossSiteCaptureTest() && + FrameTestUtil::IsApproximatelySameColor( + color, average_iframe_rgb) && + FrameTestUtil::IsApproximatelySameColor( + color, average_mainframe_rgb)) { VLOG(1) << "Observed desired frame."; return; } else { @@ -380,226 +135,34 @@ class WebContentsVideoCaptureDeviceBrowserTest : public ContentBrowserTest { base::RunLoop run_loop; BrowserThread::PostDelayedTask(BrowserThread::UI, FROM_HERE, run_loop.QuitClosure(), - min_capture_period_); + GetMinCapturePeriod()); run_loop.Run(); } } protected: - // These are overridden for the parameterized tests. - virtual bool use_software_compositing() const { return false; } - virtual bool use_fixed_aspect_ratio() const { return false; } - virtual bool is_cross_site_capture_test() const { return false; } - - void SetUp() override { - // IMPORTANT: Do not add the switches::kUseGpuInTests command line flag: It - // causes the tests to take 12+ seconds just to spin up a render process on - // debug builds. It can also cause test failures in MSAN builds, or - // exacerbate OOM situations on highly-loaded machines. - - // Screen capture requires readback from compositor output. - EnablePixelOutput(); - - // Conditionally force software compositing instead of GPU-accelerated - // compositing. - if (use_software_compositing()) { - UseSoftwareCompositing(); - } - - ContentBrowserTest::SetUp(); - } - - void SetUpCommandLine(base::CommandLine* command_line) override { - IsolateAllSitesForTesting(command_line); + // Don't call this. Call <BaseClass>::GetExpectedSourceSize() instead. + gfx::Size GetCapturedSourceSize() const final { + return shell() + ->web_contents() + ->GetMainFrame() + ->GetView() + ->GetViewBounds() + .size(); } - void SetUpOnMainThread() override { - ContentBrowserTest::SetUpOnMainThread(); - - // Set-up and start the embedded test HTTP server. - host_resolver()->AddRule("*", "127.0.0.1"); - embedded_test_server()->RegisterRequestHandler(base::BindRepeating( - &WebContentsVideoCaptureDeviceBrowserTest::HandleRequest, - base::Unretained(this))); - ASSERT_TRUE(embedded_test_server()->Start()); + std::unique_ptr<FrameSinkVideoCaptureDevice> CreateDevice() final { + auto* const main_frame = shell()->web_contents()->GetMainFrame(); + return std::make_unique<WebContentsVideoCaptureDevice>( + main_frame->GetProcess()->GetID(), main_frame->GetRoutingID()); } - void TearDownOnMainThread() override { - ASSERT_TRUE(embedded_test_server()->ShutdownAndWaitUntilComplete()); - - frames_.clear(); - - // Run any left-over tasks (usually these are delete-soon's and orphaned - // tasks). - base::RunLoop().RunUntilIdle(); - - ContentBrowserTest::TearDownOnMainThread(); - } + void WaitForFirstFrame() final { WaitForFrameWithColor(SK_ColorBLACK); } private: - void OnFrame(scoped_refptr<media::VideoFrame> frame) { - // Frame timestamps should be monotionically increasing. - EXPECT_LT(last_frame_timestamp_, frame->timestamp()); - last_frame_timestamp_ = frame->timestamp(); - - frames_.emplace_back(std::move(frame)); - } - - // Called by the embedded test HTTP server to provide the document resources. - std::unique_ptr<HttpResponse> HandleRequest(const HttpRequest& request) { - auto response = std::make_unique<BasicHttpResponse>(); - response->set_content_type("text/html"); - const GURL& url = request.GetURL(); - if (url.path() == kOuterFramePath) { - // A page with a solid white fill color, but containing an iframe in its - // upper-left quadrant. - const GURL& inner_frame_url = - embedded_test_server()->GetURL(kInnerFrameHostname, kInnerFramePath); - response->set_content(base::StringPrintf( - "<!doctype html>" - "<body style='background-color: #ffffff;'>" - "<iframe src='%s' width=%d height=%d style='position:absolute; " - "top:0px; left:0px; margin:none; padding:none; border:none;'>" - "</iframe>" - "</body>", - inner_frame_url.spec().c_str(), expected_view_size_.width() / 2, - expected_view_size_.height() / 2)); - } else { - // A page whose solid fill color is based on a query parameter, or - // defaults to black. - const std::string& query = url.query(); - std::string color = "#000000"; - const auto pos = query.find("color="); - if (pos != std::string::npos) { - color = "#" + query.substr(pos + 6, 6); - } - response->set_content( - base::StringPrintf("<!doctype html>" - "<body style='background-color: %s;'></body>", - color.c_str())); - } - return std::move(response); - } - - static SkBitmap ConvertToSkBitmap(const media::VideoFrame& frame) { - SkBitmap bitmap; - bitmap.allocN32Pixels(frame.visible_rect().width(), - frame.visible_rect().height()); - // TODO(miu): This is not Rec.709 colorspace conversion, and so will - // introduce inaccuracies. - libyuv::I420ToARGB(frame.visible_data(media::VideoFrame::kYPlane), - frame.stride(media::VideoFrame::kYPlane), - frame.visible_data(media::VideoFrame::kUPlane), - frame.stride(media::VideoFrame::kUPlane), - frame.visible_data(media::VideoFrame::kVPlane), - frame.stride(media::VideoFrame::kVPlane), - reinterpret_cast<uint8_t*>(bitmap.getPixels()), - static_cast<int>(bitmap.rowBytes()), bitmap.width(), - bitmap.height()); - return bitmap; - } - - // Computes the average color in the frame for each of these regions: 1) the - // upper-left quadrant of the content; 2) the remaining three quadrants of the - // content; 3) the letterboxed regions (if any). - static void AnalyzeFrame(SkBitmap frame, - const gfx::Rect& content_rect, - std::array<double, 3>* average_ul_content_rgb, - std::array<double, 3>* average_rem_content_rgb, - std::array<double, 3>* average_letterbox_rgb) { - const gfx::Rect ul_content_rect(content_rect.x(), content_rect.y(), - content_rect.width() / 2, - content_rect.height() / 2); - int64_t sum_of_ul_content_values[3] = {0}; - int64_t sum_of_rem_content_values[3] = {0}; - int64_t sum_of_letterbox_values[3] = {0}; - for (int y = 0; y < frame.height(); ++y) { - for (int x = 0; x < frame.width(); ++x) { - const SkColor color = frame.getColor(x, y); - int64_t* const sums = - ul_content_rect.Contains(x, y) - ? sum_of_ul_content_values - : (content_rect.Contains(x, y) ? sum_of_rem_content_values - : sum_of_letterbox_values); - sums[0] += SkColorGetR(color); - sums[1] += SkColorGetG(color); - sums[2] += SkColorGetB(color); - } - } - - const double ul_content_area = - static_cast<double>(ul_content_rect.size().GetArea()); - for (int i = 0; i < 3; ++i) { - (*average_ul_content_rgb)[i] = - (ul_content_area <= 0.0) - ? NAN - : (sum_of_ul_content_values[i] / ul_content_area); - } - - const double rem_content_area = static_cast<double>( - content_rect.size().GetArea() - ul_content_rect.size().GetArea()); - for (int i = 0; i < 3; ++i) { - (*average_rem_content_rgb)[i] = - (rem_content_area <= 0.0) - ? NAN - : (sum_of_rem_content_values[i] / rem_content_area); - } - - const double letterbox_area = static_cast<double>( - (frame.width() * frame.height()) - content_rect.size().GetArea()); - for (int i = 0; i < 3; ++i) { - (*average_letterbox_rgb)[i] = - (letterbox_area <= 0.0) - ? NAN - : (sum_of_letterbox_values[i] / letterbox_area); - } - } - - static bool IsApproximatelySameColor(SkColor color, - const std::array<double, 3> rgb) { - const double r_diff = std::abs(SkColorGetR(color) - rgb[0]); - const double g_diff = std::abs(SkColorGetG(color) - rgb[1]); - const double b_diff = std::abs(SkColorGetB(color) - rgb[2]); - return r_diff < kMaxColorDifference && g_diff < kMaxColorDifference && - b_diff < kMaxColorDifference; - } - - FakeVideoCaptureStack capture_stack_; - - gfx::Size expected_view_size_; - - std::unique_ptr<WebContentsVideoCaptureDevice> device_; - base::TimeDelta min_capture_period_; - base::circular_deque<scoped_refptr<media::VideoFrame>> frames_; - base::TimeDelta last_frame_timestamp_; - - static constexpr int kMaxColorDifference = 8; - - // Arbitrary string constants used to refer to each document by - // host+path. Note that the "inner frame" and "outer frame" must have - // different hostnames to engage the cross-site process isolation logic in the - // browser. - static constexpr char kInnerFrameHostname[] = "innerframe.com"; - static constexpr char kInnerFramePath[] = "/inner.html"; - static constexpr char kOuterFrameHostname[] = "outerframe.com"; - static constexpr char kOuterFramePath[] = "/outer.html"; - static constexpr char kSingleFrameHostname[] = "singleframe.com"; - static constexpr char kSingleFramePath[] = "/single.html"; + DISALLOW_COPY_AND_ASSIGN(WebContentsVideoCaptureDeviceBrowserTest); }; -// static -constexpr char WebContentsVideoCaptureDeviceBrowserTest::kInnerFrameHostname[]; -// static -constexpr char WebContentsVideoCaptureDeviceBrowserTest::kInnerFramePath[]; -// static -constexpr char WebContentsVideoCaptureDeviceBrowserTest::kOuterFrameHostname[]; -// static -constexpr char WebContentsVideoCaptureDeviceBrowserTest::kOuterFramePath[]; -// static -constexpr char WebContentsVideoCaptureDeviceBrowserTest::kSingleFrameHostname[]; -// static -constexpr char WebContentsVideoCaptureDeviceBrowserTest::kSingleFramePath[]; - // Tests that the device refuses to start if the WebContents target was // destroyed before the device could start. IN_PROC_BROWSER_TEST_F(WebContentsVideoCaptureDeviceBrowserTest, @@ -621,7 +184,7 @@ IN_PROC_BROWSER_TEST_F(WebContentsVideoCaptureDeviceBrowserTest, render_process_id, render_frame_id); // Running the pending UI tasks should cause the device to realize the // WebContents is gone. - RunAllPendingInMessageLoop(BrowserThread::UI); + RunUntilIdle(); // Attempt to start the device, and expect the video capture stack to have // been notified of the error. @@ -632,7 +195,7 @@ IN_PROC_BROWSER_TEST_F(WebContentsVideoCaptureDeviceBrowserTest, capture_stack()->ExpectHasLogMessages(); device->StopAndDeAllocate(); - RunAllPendingInMessageLoop(BrowserThread::UI); + RunUntilIdle(); } // Tests that the device starts, captures a frame, and then gracefully @@ -643,13 +206,13 @@ IN_PROC_BROWSER_TEST_F(WebContentsVideoCaptureDeviceBrowserTest, AllocateAndStartAndWaitForFirstFrame(); // Initially, the device captures any content changes normally. - ChangePageContentColor("ff0000"); + ChangePageContentColor(SK_ColorRED); WaitForFrameWithColor(SK_ColorRED); // Delete the WebContents instance and the Shell, and allow the the "target // permanently lost" error to propagate to the video capture stack. shell()->web_contents()->Close(); - RunAllPendingInMessageLoop(BrowserThread::UI); + RunUntilIdle(); EXPECT_TRUE(capture_stack()->error_occurred()); capture_stack()->ExpectHasLogMessages(); @@ -665,17 +228,17 @@ IN_PROC_BROWSER_TEST_F(WebContentsVideoCaptureDeviceBrowserTest, AllocateAndStartAndWaitForFirstFrame(); // Initially, the device captures any content changes normally. - ChangePageContentColor("ff0000"); + ChangePageContentColor(SK_ColorRED); WaitForFrameWithColor(SK_ColorRED); // Suspend the device. device()->MaybeSuspend(); - RunAllPendingInMessageLoop(BrowserThread::UI); + RunUntilIdle(); ClearCapturedFramesQueue(); // Change the page content and run the browser for five seconds. Expect no // frames were queued because the device should be suspended. - ChangePageContentColor("00ff00"); + ChangePageContentColor(SK_ColorGREEN); base::RunLoop run_loop; BrowserThread::PostDelayedTask(BrowserThread::UI, FROM_HERE, run_loop.QuitClosure(), @@ -699,7 +262,7 @@ IN_PROC_BROWSER_TEST_F(WebContentsVideoCaptureDeviceBrowserTest, AllocateAndStartAndWaitForFirstFrame(); // Set the page content to a known color. - ChangePageContentColor("ff0000"); + ChangePageContentColor(SK_ColorRED); WaitForFrameWithColor(SK_ColorRED); // Without making any further changes to the source (which would trigger @@ -717,13 +280,13 @@ class WebContentsVideoCaptureDeviceBrowserTestP : public WebContentsVideoCaptureDeviceBrowserTest, public testing::WithParamInterface<std::tuple<bool, bool, bool>> { public: - bool use_software_compositing() const override { + bool IsSoftwareCompositingTest() const override { return std::get<0>(GetParam()); } - bool use_fixed_aspect_ratio() const override { + bool IsFixedAspectRatioTest() const override { return std::get<1>(GetParam()); } - bool is_cross_site_capture_test() const override { + bool IsCrossSiteCaptureTest() const override { return std::get<2>(GetParam()); } }; @@ -733,23 +296,23 @@ INSTANTIATE_TEST_CASE_P( , WebContentsVideoCaptureDeviceBrowserTestP, testing::Combine( - // On ChromeOS, software compositing is not an option. - testing::Values(false), - // Force video frame resolutions to have a fixed aspect ratio? - testing::Values(false, true), - // Test with a document that contains a cross-site iframe? - testing::Values(false, true))); + // Note: On ChromeOS, software compositing is not an option. + testing::Values(false /* GPU-accelerated compositing */), + testing::Values(false /* variable aspect ratio */, + true /* fixed aspect ratio */), + testing::Values(false /* page has only a main frame */, + true /* page contains a cross-site iframe */))); #else INSTANTIATE_TEST_CASE_P( , WebContentsVideoCaptureDeviceBrowserTestP, testing::Combine( - // Use software compositing instead of GPU-accelerated compositing? - testing::Values(false, true), - // Force video frame resolutions to have a fixed aspect ratio? - testing::Values(false, true), - // Test with a document that contains a cross-site iframe? - testing::Values(false, true))); + testing::Values(false /* GPU-accelerated compositing */, + true /* software compositing */), + testing::Values(false /* variable aspect ratio */, + true /* fixed aspect ratio */), + testing::Values(false /* page has only a main frame */, + true /* page contains a cross-site iframe */))); #endif // defined(OS_CHROMEOS) // Tests that the device successfully captures a series of content changes, @@ -760,10 +323,10 @@ IN_PROC_BROWSER_TEST_P(WebContentsVideoCaptureDeviceBrowserTestP, CapturesContentChanges) { SCOPED_TRACE(testing::Message() << "Test parameters: " - << (use_software_compositing() ? "Software Compositing" - : "GPU Compositing") + << (IsSoftwareCompositingTest() ? "Software Compositing" + : "GPU Compositing") << " with " - << (use_fixed_aspect_ratio() ? "Fixed Video Aspect Ratio" + << (IsFixedAspectRatioTest() ? "Fixed Video Aspect Ratio" : "Variable Video Aspect Ratio")); NavigateToInitialDocument(); @@ -771,42 +334,46 @@ IN_PROC_BROWSER_TEST_P(WebContentsVideoCaptureDeviceBrowserTestP, for (int visilibilty_case = 0; visilibilty_case < 3; ++visilibilty_case) { switch (visilibilty_case) { - case 0: - VLOG(1) << "Visibility case: WebContents is showing."; + case 0: { + SCOPED_TRACE(testing::Message() + << "Visibility case: WebContents is showing."); shell()->web_contents()->WasShown(); base::RunLoop().RunUntilIdle(); ASSERT_EQ(shell()->web_contents()->GetVisibility(), content::Visibility::VISIBLE); break; - case 1: - VLOG(1) << "Visibility case: WebContents is hidden."; + } + + case 1: { + SCOPED_TRACE(testing::Message() + << "Visibility case: WebContents is hidden."); shell()->web_contents()->WasHidden(); base::RunLoop().RunUntilIdle(); ASSERT_EQ(shell()->web_contents()->GetVisibility(), content::Visibility::HIDDEN); break; - case 2: - VLOG(1) << "Visibility case: WebContents is showing, but occluded."; + } + + case 2: { + SCOPED_TRACE( + testing::Message() + << "Visibility case: WebContents is showing, but occluded."); shell()->web_contents()->WasShown(); shell()->web_contents()->WasOccluded(); base::RunLoop().RunUntilIdle(); ASSERT_EQ(shell()->web_contents()->GetVisibility(), content::Visibility::OCCLUDED); break; + } } - static const struct { - const char* const css_hex; - SkColor skia; - } kColorsToCycleThrough[] = { - {"ff0000", SK_ColorRED}, {"00ff00", SK_ColorGREEN}, - {"0000ff", SK_ColorBLUE}, {"ffff00", SK_ColorYELLOW}, - {"00ffff", SK_ColorCYAN}, {"ff00ff", SK_ColorMAGENTA}, - {"ffffff", SK_ColorWHITE}, + static constexpr SkColor kColorsToCycleThrough[] = { + SK_ColorRED, SK_ColorGREEN, SK_ColorBLUE, SK_ColorYELLOW, + SK_ColorCYAN, SK_ColorMAGENTA, SK_ColorWHITE, }; - for (const auto color : kColorsToCycleThrough) { - ChangePageContentColor(color.css_hex); - WaitForFrameWithColor(color.skia); + for (SkColor color : kColorsToCycleThrough) { + ChangePageContentColor(color); + WaitForFrameWithColor(color); } } diff --git a/chromium/content/browser/media/cdm_registry_impl_unittest.cc b/chromium/content/browser/media/cdm_registry_impl_unittest.cc index 90040e2d820..96547c4fc7f 100644 --- a/chromium/content/browser/media/cdm_registry_impl_unittest.cc +++ b/chromium/content/browser/media/cdm_registry_impl_unittest.cc @@ -9,6 +9,7 @@ #include <vector> #include "base/base_paths.h" +#include "base/containers/flat_set.h" #include "base/files/file_path.h" #include "base/strings/string_split.h" #include "base/strings/string_util.h" @@ -21,6 +22,7 @@ namespace content { const char kTestCdmName[] = "Test CDM"; +const char kAlternateCdmName[] = "Alternate CDM"; const char kTestCdmGuid[] = "62FE9C4B-384E-48FD-B28A-9F6F248BC8CC"; const char kTestPath[] = "/aa/bb"; const char kVersion1[] = "1.1.1.1"; @@ -41,13 +43,14 @@ class CdmRegistryImplTest : public testing::Test { const std::string& path, const std::vector<media::VideoCodec>& supported_video_codecs, bool supports_persistent_license, + const base::flat_set<media::EncryptionMode>& supported_modes, std::string supported_key_system, bool supports_sub_key_systems = false) { - cdm_registry_.RegisterCdm( - CdmInfo(name, kTestCdmGuid, base::Version(version), - base::FilePath::FromUTF8Unsafe(path), kTestFileSystemId, - supported_video_codecs, supports_persistent_license, - supported_key_system, supports_sub_key_systems)); + cdm_registry_.RegisterCdm(CdmInfo( + name, kTestCdmGuid, base::Version(version), + base::FilePath::FromUTF8Unsafe(path), kTestFileSystemId, + supported_video_codecs, supports_persistent_license, supported_modes, + supported_key_system, supports_sub_key_systems)); } bool IsRegistered(const std::string& name, const std::string& version) { @@ -73,7 +76,8 @@ class CdmRegistryImplTest : public testing::Test { TEST_F(CdmRegistryImplTest, Register) { Register(kTestCdmName, kVersion1, kTestPath, - {media::kCodecVP8, media::kCodecVP9}, true, kTestKeySystem, true); + {media::kCodecVP8, media::kCodecVP9}, true, + {media::EncryptionMode::kCenc}, kTestKeySystem, true); std::vector<CdmInfo> cdms = cdm_registry_.GetAllRegisteredCdms(); ASSERT_EQ(1u, cdms.size()); CdmInfo cdm = cdms[0]; @@ -85,29 +89,38 @@ TEST_F(CdmRegistryImplTest, Register) { EXPECT_EQ(media::kCodecVP8, cdm.supported_video_codecs[0]); EXPECT_EQ(media::kCodecVP9, cdm.supported_video_codecs[1]); EXPECT_TRUE(cdm.supports_persistent_license); + EXPECT_EQ(1u, cdm.supported_encryption_schemes.size()); + EXPECT_EQ( + 1u, cdm.supported_encryption_schemes.count(media::EncryptionMode::kCenc)); EXPECT_EQ(kTestKeySystem, cdm.supported_key_system); EXPECT_TRUE(cdm.supports_sub_key_systems); } TEST_F(CdmRegistryImplTest, ReRegister) { - Register(kTestCdmName, kVersion1, "/bb/cc", {}, false, kTestKeySystem); + Register(kTestCdmName, kVersion1, "/bb/cc", {}, false, + {media::EncryptionMode::kCenc}, kTestKeySystem); EXPECT_TRUE(IsRegistered(kTestCdmName, kVersion1)); // Now register same key system with different values. - Register(kTestCdmName, kVersion1, kTestPath, {}, false, kTestKeySystem); + Register(kTestCdmName, kVersion1, kTestPath, {}, false, + {media::EncryptionMode::kCenc}, kTestKeySystem); EXPECT_TRUE(IsRegistered(kTestCdmName, kVersion1)); } TEST_F(CdmRegistryImplTest, MultipleVersions) { - Register(kTestCdmName, kVersion1, kTestPath, {}, false, kTestKeySystem); - Register(kTestCdmName, kVersion2, "/bb/cc", {}, false, kTestKeySystem); + Register(kTestCdmName, kVersion1, kTestPath, {}, false, + {media::EncryptionMode::kCenc}, kTestKeySystem); + Register(kTestCdmName, kVersion2, "/bb/cc", {}, false, + {media::EncryptionMode::kCenc}, kTestKeySystem); EXPECT_TRUE(IsRegistered(kTestCdmName, kVersion1)); EXPECT_TRUE(IsRegistered(kTestCdmName, kVersion2)); } TEST_F(CdmRegistryImplTest, NewVersionInsertedLast) { - Register(kTestCdmName, kVersion1, kTestPath, {}, false, kTestKeySystem); - Register(kTestCdmName, kVersion2, "/bb/cc", {}, false, kTestKeySystem); + Register(kTestCdmName, kVersion1, kTestPath, {}, false, + {media::EncryptionMode::kCenc}, kTestKeySystem); + Register(kTestCdmName, kVersion2, "/bb/cc", {}, false, + {media::EncryptionMode::kCenc}, kTestKeySystem); const std::vector<std::string> versions = GetVersions(kTestCdmGuid); EXPECT_EQ(2u, versions.size()); @@ -115,4 +128,28 @@ TEST_F(CdmRegistryImplTest, NewVersionInsertedLast) { EXPECT_EQ(kVersion2, versions[1]); } +TEST_F(CdmRegistryImplTest, DifferentNames) { + Register(kTestCdmName, kVersion1, kTestPath, {}, false, + {media::EncryptionMode::kCenc}, kTestKeySystem); + Register(kAlternateCdmName, kVersion1, kTestPath, {}, false, + {media::EncryptionMode::kCbcs}, kTestKeySystem); + EXPECT_TRUE(IsRegistered(kTestCdmName, kVersion1)); + EXPECT_TRUE(IsRegistered(kAlternateCdmName, kVersion1)); +} + +TEST_F(CdmRegistryImplTest, SupportedEncryptionSchemes) { + Register(kTestCdmName, kVersion1, kTestPath, {}, false, + {media::EncryptionMode::kCenc, media::EncryptionMode::kCbcs}, + kTestKeySystem); + + std::vector<CdmInfo> cdms = cdm_registry_.GetAllRegisteredCdms(); + ASSERT_EQ(1u, cdms.size()); + const CdmInfo& cdm = cdms[0]; + EXPECT_EQ(2u, cdm.supported_encryption_schemes.size()); + EXPECT_EQ( + 1u, cdm.supported_encryption_schemes.count(media::EncryptionMode::kCenc)); + EXPECT_EQ( + 1u, cdm.supported_encryption_schemes.count(media::EncryptionMode::kCbcs)); +} + } // namespace content diff --git a/chromium/content/browser/media/encrypted_media_browsertest.cc b/chromium/content/browser/media/encrypted_media_browsertest.cc index 9a01d1a44e9..7096e1e1fe8 100644 --- a/chromium/content/browser/media/encrypted_media_browsertest.cc +++ b/chromium/content/browser/media/encrypted_media_browsertest.cc @@ -47,12 +47,15 @@ const char kExternalClearKeyKeySystem[] = "org.chromium.externalclearkey"; // Supported media types. const char kWebMVorbisAudioOnly[] = "audio/webm; codecs=\"vorbis\""; const char kWebMOpusAudioOnly[] = "audio/webm; codecs=\"opus\""; -const char kWebMVP8VideoOnly[] = "video/webm; codecs=\"vp8\""; -const char kWebMVP9VideoOnly[] = "video/webm; codecs=\"vp9\""; -const char kWebMOpusAudioVP9Video[] = "video/webm; codecs=\"opus, vp9\""; -const char kWebMVorbisAudioVP8Video[] = "video/webm; codecs=\"vorbis, vp8\""; +const char kWebMVp8VideoOnly[] = "video/webm; codecs=\"vp8\""; +const char kWebMVp9VideoOnly[] = "video/webm; codecs=\"vp9\""; +const char kWebMOpusAudioVp9Video[] = "video/webm; codecs=\"opus, vp9\""; +const char kWebMVorbisAudioVp8Video[] = "video/webm; codecs=\"vorbis, vp8\""; +const char kMp4FlacAudioOnly[] = "audio/mp4; codecs=\"flac\""; +const char kMp4Vp9VideoOnly[] = + "video/mp4; codecs=\"vp09.00.10.08.01.02.02.02.00\""; #if BUILDFLAG(USE_PROPRIETARY_CODECS) -const char kMP4VideoOnly[] = "video/mp4; codecs=\"avc1.64001E\""; +const char kMp4Avc1VideoOnly[] = "video/mp4; codecs=\"avc1.64001E\""; #endif // BUILDFLAG(USE_PROPRIETARY_CODECS) // EME-specific test results and errors. @@ -89,14 +92,14 @@ class EncryptedMediaTest void TestSimplePlayback(const std::string& encrypted_media, const std::string& media_type) { - RunSimpleEncryptedMediaTest( - encrypted_media, media_type, CurrentKeySystem(), CurrentSourceType()); + RunSimpleEncryptedMediaTest(encrypted_media, media_type, CurrentKeySystem(), + CurrentSourceType()); } void TestFrameSizeChange() { RunEncryptedMediaTest("encrypted_frame_size_change.html", "frame_size_change-av_enc-v.webm", - kWebMVorbisAudioVP8Video, CurrentKeySystem(), + kWebMVorbisAudioVp8Video, CurrentKeySystem(), CurrentSourceType(), media::kEnded); } @@ -141,18 +144,17 @@ class EncryptedMediaTest src_type, media::kEnded); } -#if BUILDFLAG(USE_PROPRIETARY_CODECS) - void TestMP4EncryptionPlayback(const std::string& media_file, + void TestMp4EncryptionPlayback(const std::string& media_file, + const std::string& media_type, const std::string& expected_title) { if (CurrentSourceType() != SrcType::MSE) { DVLOG(0) << "Skipping test; Can only play MP4 encrypted streams by MSE."; return; } - RunEncryptedMediaTest(kDefaultEmePlayer, media_file, kMP4VideoOnly, + RunEncryptedMediaTest(kDefaultEmePlayer, media_file, media_type, CurrentKeySystem(), SrcType::MSE, expected_title); } -#endif // BUILDFLAG(USE_PROPRIETARY_CODECS) protected: // We want to fail quickly when a test fails because an error is encountered. @@ -205,29 +207,29 @@ IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, Playback_AudioOnly_WebM) { } IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, Playback_AudioClearVideo_WebM) { - TestSimplePlayback("bear-320x240-av_enc-a.webm", kWebMVorbisAudioVP8Video); + TestSimplePlayback("bear-320x240-av_enc-a.webm", kWebMVorbisAudioVp8Video); } IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, Playback_VideoAudio_WebM) { - TestSimplePlayback("bear-320x240-av_enc-av.webm", kWebMVorbisAudioVP8Video); + TestSimplePlayback("bear-320x240-av_enc-av.webm", kWebMVorbisAudioVp8Video); } IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, Playback_VideoOnly_WebM) { - TestSimplePlayback("bear-320x240-v_enc-v.webm", kWebMVP8VideoOnly); + TestSimplePlayback("bear-320x240-v_enc-v.webm", kWebMVp8VideoOnly); } IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, Playback_VideoOnly_WebM_Fullsample) { TestSimplePlayback("bear-320x240-v-vp9_fullsample_enc-v.webm", - kWebMVP9VideoOnly); + kWebMVp9VideoOnly); } IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, Playback_VideoOnly_WebM_Subsample) { TestSimplePlayback("bear-320x240-v-vp9_subsample_enc-v.webm", - kWebMVP9VideoOnly); + kWebMVp9VideoOnly); } IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, Playback_VideoClearAudio_WebM) { - TestSimplePlayback("bear-320x240-av_enc-v.webm", kWebMVorbisAudioVP8Video); + TestSimplePlayback("bear-320x240-av_enc-v.webm", kWebMVorbisAudioVp8Video); } IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, Playback_AudioOnly_WebM_Opus) { @@ -244,7 +246,7 @@ IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, Playback_VideoAudio_WebM_Opus) { return; #endif TestSimplePlayback("bear-320x240-opus-av_enc-av.webm", - kWebMOpusAudioVP9Video); + kWebMOpusAudioVp9Video); } IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, Playback_VideoClearAudio_WebM_Opus) { @@ -252,7 +254,21 @@ IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, Playback_VideoClearAudio_WebM_Opus) { if (!media::PlatformHasOpusSupport()) return; #endif - TestSimplePlayback("bear-320x240-opus-av_enc-v.webm", kWebMOpusAudioVP9Video); + TestSimplePlayback("bear-320x240-opus-av_enc-v.webm", kWebMOpusAudioVp9Video); +} + +IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, Playback_AudioOnly_MP4_FLAC) { + TestMp4EncryptionPlayback("bear-flac-cenc.mp4", kMp4FlacAudioOnly, + media::kEnded); +} + +IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, Playback_VideoOnly_MP4_VP9) { + // MP4 without MSE is not support yet, http://crbug.com/170793. + if (CurrentSourceType() != SrcType::MSE) { + DVLOG(0) << "Skipping test; Can only play MP4 encrypted streams by MSE."; + return; + } + TestSimplePlayback("bear-320x240-v_frag-vp9-cenc.mp4", kMp4Vp9VideoOnly); } // Strictly speaking this is not an "encrypted" media test. Keep it here for @@ -286,23 +302,25 @@ IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, MAYBE_FrameSizeChangeVideo) { #if BUILDFLAG(USE_PROPRIETARY_CODECS) IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, Playback_Encryption_CENC) { - TestMP4EncryptionPlayback("bear-640x360-v_frag-cenc.mp4", media::kEnded); + TestMp4EncryptionPlayback("bear-640x360-v_frag-cenc.mp4", kMp4Avc1VideoOnly, + media::kEnded); } IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, Playback_Encryption_CBC1) { - TestMP4EncryptionPlayback("bear-640x360-v_frag-cbc1.mp4", media::kError); + TestMp4EncryptionPlayback("bear-640x360-v_frag-cbc1.mp4", kMp4Avc1VideoOnly, + media::kError); } IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, Playback_Encryption_CENS) { - TestMP4EncryptionPlayback("bear-640x360-v_frag-cens.mp4", media::kError); + TestMp4EncryptionPlayback("bear-640x360-v_frag-cens.mp4", kMp4Avc1VideoOnly, + media::kError); } IN_PROC_BROWSER_TEST_P(EncryptedMediaTest, Playback_Encryption_CBCS) { -#if BUILDFLAG(ENABLE_CBCS_ENCRYPTION_SCHEME) - TestMP4EncryptionPlayback("bear-640x360-v_frag-cbcs.mp4", media::kEnded); -#else - TestMP4EncryptionPlayback("bear-640x360-v_frag-cbcs.mp4", media::kError); -#endif + std::string expected_result = + BUILDFLAG(ENABLE_CBCS_ENCRYPTION_SCHEME) ? media::kEnded : media::kError; + TestMp4EncryptionPlayback("bear-640x360-v_frag-cbcs.mp4", kMp4Avc1VideoOnly, + expected_result); } #endif // BUILDFLAG(USE_PROPRIETARY_CODECS) diff --git a/chromium/content/browser/media/flinging_renderer.cc b/chromium/content/browser/media/flinging_renderer.cc new file mode 100644 index 00000000000..8828939c4e5 --- /dev/null +++ b/chromium/content/browser/media/flinging_renderer.cc @@ -0,0 +1,100 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/flinging_renderer.h" + +#include "base/memory/ptr_util.h" +#include "content/browser/frame_host/render_frame_host_delegate.h" +#include "content/browser/frame_host/render_frame_host_impl.h" +#include "content/public/browser/content_browser_client.h" +#include "content/public/browser/presentation_service_delegate.h" +#include "content/public/browser/render_frame_host.h" +#include "content/public/common/content_client.h" + +namespace content { + +FlingingRenderer::FlingingRenderer(std::unique_ptr<MediaController> controller) + : controller_(std::move(controller)) {} + +FlingingRenderer::~FlingingRenderer() = default; + +// static +std::unique_ptr<FlingingRenderer> FlingingRenderer::Create( + RenderFrameHost* render_frame_host, + const std::string& presentation_id) { + DVLOG(1) << __func__; + + ContentClient* content_client = GetContentClient(); + if (!content_client) + return nullptr; + + ContentBrowserClient* browser_client = content_client->browser(); + if (!browser_client) + return nullptr; + + ControllerPresentationServiceDelegate* presentation_delegate = + browser_client->GetControllerPresentationServiceDelegate( + static_cast<RenderFrameHostImpl*>(render_frame_host) + ->delegate() + ->GetAsWebContents()); + + if (!presentation_delegate) + return nullptr; + + auto media_controller = presentation_delegate->GetMediaController( + render_frame_host->GetProcess()->GetID(), + render_frame_host->GetRoutingID(), presentation_id); + + if (!media_controller) + return nullptr; + + return base::WrapUnique<FlingingRenderer>( + new FlingingRenderer(std::move(media_controller))); +} + +// media::Renderer implementation +void FlingingRenderer::Initialize(media::MediaResource* media_resource, + media::RendererClient* client, + const media::PipelineStatusCB& init_cb) { + DVLOG(2) << __func__; + init_cb.Run(media::PIPELINE_OK); +} + +void FlingingRenderer::SetCdm(media::CdmContext* cdm_context, + const media::CdmAttachedCB& cdm_attached_cb) { + // The flinging renderer does not support playing encrypted content. + NOTREACHED(); +} + +void FlingingRenderer::Flush(const base::Closure& flush_cb) { + DVLOG(2) << __func__; + // There is nothing to reset, we can no-op the call. + flush_cb.Run(); +} + +void FlingingRenderer::StartPlayingFrom(base::TimeDelta time) { + DVLOG(2) << __func__; + controller_->Seek(time); + controller_->Play(); +} + +void FlingingRenderer::SetPlaybackRate(double playback_rate) { + DVLOG(2) << __func__; + if (playback_rate == 0) + controller_->Pause(); + else + controller_->Play(); +} + +void FlingingRenderer::SetVolume(float volume) { + DVLOG(2) << __func__; + controller_->SetVolume(volume); +} + +base::TimeDelta FlingingRenderer::GetMediaTime() { + // TODO(https://crbug.com/830871): return correct media time. + return base::TimeDelta(); +} + +} // namespace content diff --git a/chromium/content/browser/media/flinging_renderer.h b/chromium/content/browser/media/flinging_renderer.h new file mode 100644 index 00000000000..59a12b8bf07 --- /dev/null +++ b/chromium/content/browser/media/flinging_renderer.h @@ -0,0 +1,62 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONTENT_BROWSER_MEDIA_FLINGING_RENDERER_H_ +#define CONTENT_BROWSER_MEDIA_FLINGING_RENDERER_H_ + +#include "base/callback.h" +#include "content/common/content_export.h" +#include "content/public/browser/media_controller.h" +#include "media/base/media_resource.h" +#include "media/base/renderer.h" +#include "media/base/renderer_client.h" +#include "url/gurl.h" + +namespace content { + +class FlingingRendererTest; +class RenderFrameHost; + +// FlingingRenderer adapts from the media::Renderer interface to the +// MediaController interface. The MediaController is used to issue simple media +// playback commands. In this case, the media we are controlling should be an +// already existing RemotingCastSession, which should have been initiated by a +// blink::RemotePlayback object, using the PresentationService. +class CONTENT_EXPORT FlingingRenderer : public media::Renderer { + public: + // Helper method to create a FlingingRenderer from an already existing + // presentation ID. + // Returns nullptr if there was an error getting the MediaControllor for the + // given presentation ID. + static std::unique_ptr<FlingingRenderer> Create( + RenderFrameHost* render_frame_host, + const std::string& presentation_id); + + ~FlingingRenderer() override; + + // media::Renderer implementation + void Initialize(media::MediaResource* media_resource, + media::RendererClient* client, + const media::PipelineStatusCB& init_cb) override; + void SetCdm(media::CdmContext* cdm_context, + const media::CdmAttachedCB& cdm_attached_cb) override; + void Flush(const base::Closure& flush_cb) override; + void StartPlayingFrom(base::TimeDelta time) override; + void SetPlaybackRate(double playback_rate) override; + void SetVolume(float volume) override; + base::TimeDelta GetMediaTime() override; + + private: + friend class FlingingRendererTest; + + explicit FlingingRenderer(std::unique_ptr<MediaController> controller); + + std::unique_ptr<MediaController> controller_; + + DISALLOW_COPY_AND_ASSIGN(FlingingRenderer); +}; + +} // namespace content + +#endif // CONTENT_BROWSER_MEDIA_FLINGING_RENDERER_H_ diff --git a/chromium/content/browser/media/flinging_renderer_unittest.cc b/chromium/content/browser/media/flinging_renderer_unittest.cc new file mode 100644 index 00000000000..923415d94b8 --- /dev/null +++ b/chromium/content/browser/media/flinging_renderer_unittest.cc @@ -0,0 +1,84 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/flinging_renderer.h" + +#include "base/version.h" +#include "content/public/browser/media_controller.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using ::testing::_; +using ::testing::StrictMock; + +namespace content { + +class MockMediaController : public MediaController { + public: + MOCK_METHOD0(Play, void()); + MOCK_METHOD0(Pause, void()); + MOCK_METHOD1(SetMute, void(bool)); + MOCK_METHOD1(SetVolume, void(float)); + MOCK_METHOD1(Seek, void(base::TimeDelta)); +}; + +class FlingingRendererTest : public testing::Test { + public: + FlingingRendererTest() + : media_controller_(new StrictMock<MockMediaController>()), + renderer_(std::unique_ptr<MediaController>(media_controller_)) {} + + protected: + StrictMock<MockMediaController>* media_controller_; + FlingingRenderer renderer_; +}; + +TEST_F(FlingingRendererTest, StartPlayingFromTime) { + base::TimeDelta seek_time = base::TimeDelta::FromSeconds(10); + EXPECT_CALL(*media_controller_, Play()); + EXPECT_CALL(*media_controller_, Seek(seek_time)); + + renderer_.StartPlayingFrom(seek_time); +} + +TEST_F(FlingingRendererTest, StartPlayingFromBeginning) { + EXPECT_CALL(*media_controller_, Play()); + EXPECT_CALL(*media_controller_, Seek(base::TimeDelta())); + + renderer_.StartPlayingFrom(base::TimeDelta()); +} + +TEST_F(FlingingRendererTest, SetPlaybackRate) { + double playback_rate = 1.0; + EXPECT_CALL(*media_controller_, Play()); + + renderer_.SetPlaybackRate(playback_rate); +} + +TEST_F(FlingingRendererTest, SetPlaybackRateToZero) { + double playback_rate = 0.0; + EXPECT_CALL(*media_controller_, Pause()); + + renderer_.SetPlaybackRate(playback_rate); +} + +// Setting the volume to a positive value should not change the mute state. +TEST_F(FlingingRendererTest, SetVolume) { + float volume = 0.5; + EXPECT_CALL(*media_controller_, SetVolume(volume)); + EXPECT_CALL(*media_controller_, SetMute(_)).Times(0); + + renderer_.SetVolume(volume); +} + +// Setting the volume to 0 should not set the mute state. +TEST_F(FlingingRendererTest, SetVolumeToZero) { + float volume = 0; + EXPECT_CALL(*media_controller_, SetVolume(volume)); + EXPECT_CALL(*media_controller_, SetMute(_)).Times(0); + + renderer_.SetVolume(volume); +} + +} // namespace content diff --git a/chromium/content/browser/media/forwarding_audio_stream_factory.cc b/chromium/content/browser/media/forwarding_audio_stream_factory.cc new file mode 100644 index 00000000000..694f8bfe4d2 --- /dev/null +++ b/chromium/content/browser/media/forwarding_audio_stream_factory.cc @@ -0,0 +1,256 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/forwarding_audio_stream_factory.h" + +#include <utility> + +#include "base/trace_event/trace_event.h" +#include "content/browser/web_contents/web_contents_impl.h" +#include "content/public/browser/browser_thread.h" +#include "content/public/browser/navigation_handle.h" +#include "content/public/browser/render_frame_host.h" +#include "content/public/browser/render_process_host.h" +#include "content/public/browser/web_contents.h" +#include "services/audio/public/mojom/constants.mojom.h" +#include "services/service_manager/public/cpp/connector.h" + +namespace content { + +ForwardingAudioStreamFactory::ForwardingAudioStreamFactory( + WebContents* web_contents, + std::unique_ptr<service_manager::Connector> connector, + std::unique_ptr<AudioStreamBrokerFactory> broker_factory) + : WebContentsObserver(web_contents), + connector_(std::move(connector)), + broker_factory_(std::move(broker_factory)), + group_id_(base::UnguessableToken::Create()) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + DCHECK(web_contents); + DCHECK(broker_factory_); +} + +ForwardingAudioStreamFactory::~ForwardingAudioStreamFactory() { + DCHECK_CURRENTLY_ON(BrowserThread::UI); +} + +// static +ForwardingAudioStreamFactory* ForwardingAudioStreamFactory::ForFrame( + RenderFrameHost* frame) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + auto* contents = + static_cast<WebContentsImpl*>(WebContents::FromRenderFrameHost(frame)); + if (!contents) + return nullptr; + + return contents->GetAudioStreamFactory(); +} + +void ForwardingAudioStreamFactory::CreateInputStream( + RenderFrameHost* frame, + const std::string& device_id, + const media::AudioParameters& params, + uint32_t shared_memory_count, + bool enable_agc, + mojom::RendererAudioInputStreamFactoryClientPtr renderer_factory_client) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + const int process_id = frame->GetProcess()->GetID(); + const int frame_id = frame->GetRoutingID(); + inputs_ + .insert(broker_factory_->CreateAudioInputStreamBroker( + process_id, frame_id, device_id, params, shared_memory_count, + enable_agc, + base::BindOnce(&ForwardingAudioStreamFactory::RemoveInput, + base::Unretained(this)), + std::move(renderer_factory_client))) + .first->get() + ->CreateStream(GetFactory()); +} + +void ForwardingAudioStreamFactory::AssociateInputAndOutputForAec( + const base::UnguessableToken& input_stream_id, + const std::string& raw_output_device_id) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + // Avoid spawning a factory if this for some reason gets called with an + // invalid input_stream_id before any streams are created. + if (!inputs_.empty()) { + GetFactory()->AssociateInputAndOutputForAec(input_stream_id, + raw_output_device_id); + } +} + +void ForwardingAudioStreamFactory::CreateOutputStream( + RenderFrameHost* frame, + const std::string& device_id, + const media::AudioParameters& params, + media::mojom::AudioOutputStreamProviderClientPtr client) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + const int process_id = frame->GetProcess()->GetID(); + const int frame_id = frame->GetRoutingID(); + + outputs_ + .insert(broker_factory_->CreateAudioOutputStreamBroker( + process_id, frame_id, ++stream_id_counter_, device_id, params, + group_id_, + base::BindOnce(&ForwardingAudioStreamFactory::RemoveOutput, + base::Unretained(this)), + std::move(client))) + .first->get() + ->CreateStream(GetFactory()); +} + +void ForwardingAudioStreamFactory::CreateLoopbackStream( + RenderFrameHost* frame, + RenderFrameHost* frame_of_source_web_contents, + const media::AudioParameters& params, + uint32_t shared_memory_count, + bool mute_source, + mojom::RendererAudioInputStreamFactoryClientPtr renderer_factory_client) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + DCHECK(frame); + DCHECK(frame_of_source_web_contents); + + TRACE_EVENT_BEGIN1("audio", "CreateLoopbackStream", "group", + group_id_.GetLowForSerialization()); + + WebContents* source_contents = + WebContents::FromRenderFrameHost(frame_of_source_web_contents); + if (!source_contents) { + TRACE_EVENT_END1("audio", "CreateLoopbackStream", "source", + "failed to find source"); + return; + } + + const int process_id = frame->GetProcess()->GetID(); + const int frame_id = frame->GetRoutingID(); + inputs_ + .insert(broker_factory_->CreateAudioLoopbackStreamBroker( + process_id, frame_id, + std::make_unique<AudioStreamBrokerFactory::LoopbackSource>( + source_contents), + params, shared_memory_count, mute_source, + base::BindOnce(&ForwardingAudioStreamFactory::RemoveInput, + base::Unretained(this)), + std::move(renderer_factory_client))) + .first->get() + ->CreateStream(GetFactory()); + TRACE_EVENT_END1("audio", "CreateLoopbackStream", "source", + static_cast<WebContentsImpl*>(source_contents) + ->GetAudioStreamFactory() + ->group_id() + .GetLowForSerialization()); +} + +void ForwardingAudioStreamFactory::SetMuted(bool muted) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + DCHECK_NE(muted, IsMuted()); + TRACE_EVENT_INSTANT2("audio", "SetMuted", TRACE_EVENT_SCOPE_THREAD, "group", + group_id_.GetLowForSerialization(), "muted", muted); + + if (!muted) { + muter_.reset(); + return; + } + + muter_.emplace(group_id_); + if (remote_factory_) + muter_->Connect(remote_factory_.get()); +} + +bool ForwardingAudioStreamFactory::IsMuted() const { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + return !!muter_; +} + +void ForwardingAudioStreamFactory::FrameDeleted( + RenderFrameHost* render_frame_host) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + CleanupStreamsBelongingTo(render_frame_host); +} + +void ForwardingAudioStreamFactory::CleanupStreamsBelongingTo( + RenderFrameHost* render_frame_host) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + + const int process_id = render_frame_host->GetProcess()->GetID(); + const int frame_id = render_frame_host->GetRoutingID(); + + TRACE_EVENT_BEGIN2("audio", "CleanupStreamsBelongingTo", "group", + group_id_.GetLowForSerialization(), "process id", + process_id); + + auto match_rfh = + [process_id, + frame_id](const std::unique_ptr<AudioStreamBroker>& broker) -> bool { + return broker->render_process_id() == process_id && + broker->render_frame_id() == frame_id; + }; + + base::EraseIf(outputs_, match_rfh); + base::EraseIf(inputs_, match_rfh); + + ResetRemoteFactoryPtrIfIdle(); + + TRACE_EVENT_END1("audio", "CleanupStreamsBelongingTo", "frame_id", frame_id); +} + +void ForwardingAudioStreamFactory::RemoveInput(AudioStreamBroker* broker) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + size_t removed = inputs_.erase(broker); + DCHECK_EQ(1u, removed); + + ResetRemoteFactoryPtrIfIdle(); +} + +void ForwardingAudioStreamFactory::RemoveOutput(AudioStreamBroker* broker) { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + size_t removed = outputs_.erase(broker); + DCHECK_EQ(1u, removed); + + ResetRemoteFactoryPtrIfIdle(); +} + +audio::mojom::StreamFactory* ForwardingAudioStreamFactory::GetFactory() { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + if (!remote_factory_) { + TRACE_EVENT_INSTANT1( + "audio", "ForwardingAudioStreamFactory: Binding new factory", + TRACE_EVENT_SCOPE_THREAD, "group", group_id_.GetLowForSerialization()); + connector_->BindInterface(audio::mojom::kServiceName, + mojo::MakeRequest(&remote_factory_)); + // Unretained is safe because |this| owns |remote_factory_|. + remote_factory_.set_connection_error_handler( + base::BindOnce(&ForwardingAudioStreamFactory::ResetRemoteFactoryPtr, + base::Unretained(this))); + + // Restore the muting session on reconnect. + if (muter_) + muter_->Connect(remote_factory_.get()); + } + + return remote_factory_.get(); +} + +void ForwardingAudioStreamFactory::ResetRemoteFactoryPtrIfIdle() { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + if (inputs_.empty() && outputs_.empty()) + ResetRemoteFactoryPtr(); +} + +void ForwardingAudioStreamFactory::ResetRemoteFactoryPtr() { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + if (remote_factory_) { + TRACE_EVENT_INSTANT1( + "audio", "ForwardingAudioStreamFactory: Resetting factory", + TRACE_EVENT_SCOPE_THREAD, "group", group_id_.GetLowForSerialization()); + } + remote_factory_.reset(); + // The stream brokers will call a callback to be deleted soon, give them a + // chance to signal an error to the client first. +} + +} // namespace content diff --git a/chromium/content/browser/media/forwarding_audio_stream_factory.h b/chromium/content/browser/media/forwarding_audio_stream_factory.h new file mode 100644 index 00000000000..26ffbb89bbb --- /dev/null +++ b/chromium/content/browser/media/forwarding_audio_stream_factory.h @@ -0,0 +1,144 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONTENT_BROWSER_MEDIA_FORWARDING_AUDIO_STREAM_FACTORY_H_ +#define CONTENT_BROWSER_MEDIA_FORWARDING_AUDIO_STREAM_FACTORY_H_ + +#include <memory> +#include <string> + +#include "base/containers/flat_set.h" +#include "base/containers/unique_ptr_adapters.h" +#include "base/macros.h" +#include "base/optional.h" +#include "base/unguessable_token.h" +#include "content/browser/media/audio_muting_session.h" +#include "content/browser/media/audio_stream_broker.h" +#include "content/common/content_export.h" +#include "content/common/media/renderer_audio_input_stream_factory.mojom.h" +#include "content/public/browser/web_contents_observer.h" +#include "services/audio/public/mojom/stream_factory.mojom.h" + +namespace service_manager { +class Connector; +} + +namespace media { +class AudioParameters; +} + +namespace content { + +class AudioStreamBroker; +class RenderFrameHost; +class WebContents; + +// This class handles stream creation operations for a WebContents. +// This class is operated on the UI thread. +class CONTENT_EXPORT ForwardingAudioStreamFactory final + : public WebContentsObserver { + public: + ForwardingAudioStreamFactory( + WebContents* web_contents, + std::unique_ptr<service_manager::Connector> connector, + std::unique_ptr<AudioStreamBrokerFactory> factory); + + ~ForwardingAudioStreamFactory() final; + + // Returns the ForwardingAudioStreamFactory which takes care of stream + // creation for |frame|. Returns null if |frame| is null or if the frame + // doesn't belong to a WebContents. + static ForwardingAudioStreamFactory* ForFrame(RenderFrameHost* frame); + + const base::UnguessableToken& group_id() { return group_id_; } + + // TODO(https://crbug.com/787806): Automatically restore streams on audio + // service restart. + void CreateInputStream( + RenderFrameHost* frame, + const std::string& device_id, + const media::AudioParameters& params, + uint32_t shared_memory_count, + bool enable_agc, + mojom::RendererAudioInputStreamFactoryClientPtr renderer_factory_client); + + void AssociateInputAndOutputForAec( + const base::UnguessableToken& input_stream_id, + const std::string& raw_output_device_id); + + void CreateOutputStream( + RenderFrameHost* frame, + const std::string& device_id, + const media::AudioParameters& params, + media::mojom::AudioOutputStreamProviderClientPtr client); + + void CreateLoopbackStream( + RenderFrameHost* frame, + RenderFrameHost* frame_of_source_web_contents, + const media::AudioParameters& params, + uint32_t shared_memory_count, + bool mute_source, + mojom::RendererAudioInputStreamFactoryClientPtr renderer_factory_client); + + // Sets the muting state for all output streams created through this factory. + void SetMuted(bool muted); + + // Returns the current muting state. + bool IsMuted() const; + + // WebContentsObserver implementation. We observe these events so that we can + // clean up streams belonging to a frame when that frame is destroyed. + void FrameDeleted(RenderFrameHost* render_frame_host) final; + + // E.g. to override binder. + service_manager::Connector* get_connector_for_testing() { + return connector_.get(); + } + + private: + using StreamBrokerSet = base::flat_set<std::unique_ptr<AudioStreamBroker>, + base::UniquePtrComparator>; + + void CleanupStreamsBelongingTo(RenderFrameHost* render_frame_host); + + void RemoveInput(AudioStreamBroker* handle); + void RemoveOutput(AudioStreamBroker* handle); + + audio::mojom::StreamFactory* GetFactory(); + void ResetRemoteFactoryPtrIfIdle(); + void ResetRemoteFactoryPtr(); + + const std::unique_ptr<service_manager::Connector> connector_; + const std::unique_ptr<AudioStreamBrokerFactory> broker_factory_; + + // Unique id indentifying all streams belonging to the WebContents owning + // |this|. + // TODO(https://crbug.com/824019): Use this for loopback. + const base::UnguessableToken group_id_; + + // Lazily acquired. Reset on connection error and when we no longer have any + // streams. Note: we don't want muting to force the connection to be open, + // since we want to clean up the service when not in use. If we have active + // muting but nothing else, we should stop it and start it again when we need + // to reacquire the factory for some other reason. + audio::mojom::StreamFactoryPtr remote_factory_; + + // Running id used for tracking audible streams. We keep count here to avoid + // collisions. + // TODO(https://crbug.com/830494): Refactor to make this unnecessary and + // remove it. + int stream_id_counter_ = 0; + + // Instantiated when |outputs_| should be muted, empty otherwise. + base::Optional<AudioMutingSession> muter_; + + StreamBrokerSet inputs_; + StreamBrokerSet outputs_; + + DISALLOW_COPY_AND_ASSIGN(ForwardingAudioStreamFactory); +}; + +} // namespace content + +#endif // CONTENT_BROWSER_MEDIA_FORWARDING_AUDIO_STREAM_FACTORY_H_ diff --git a/chromium/content/browser/media/forwarding_audio_stream_factory_unittest.cc b/chromium/content/browser/media/forwarding_audio_stream_factory_unittest.cc new file mode 100644 index 00000000000..e0764947aa1 --- /dev/null +++ b/chromium/content/browser/media/forwarding_audio_stream_factory_unittest.cc @@ -0,0 +1,711 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/forwarding_audio_stream_factory.h" + +#include <memory> +#include <utility> + +#include "base/bind.h" +#include "base/bind_helpers.h" +#include "base/macros.h" +#include "base/test/mock_callback.h" +#include "base/unguessable_token.h" +#include "content/common/media/renderer_audio_input_stream_factory.mojom.h" +#include "content/public/browser/render_process_host.h" +#include "content/public/browser/web_contents.h" +#include "content/public/test/test_renderer_host.h" +#include "media/base/audio_parameters.h" +#include "media/mojo/interfaces/audio_output_stream.mojom.h" +#include "mojo/public/cpp/bindings/associated_binding.h" +#include "mojo/public/cpp/bindings/binding.h" +#include "mojo/public/cpp/bindings/interface_request.h" +#include "services/audio/public/cpp/fake_stream_factory.h" +#include "services/audio/public/mojom/constants.mojom.h" +#include "services/audio/public/mojom/stream_factory.mojom.h" +#include "services/service_manager/public/cpp/connector.h" +#include "testing/gmock/include/gmock/gmock.h" +#include "testing/gtest/include/gtest/gtest.h" + +using ::testing::Test; +using ::testing::Mock; +using ::testing::NotNull; +using ::testing::StrictMock; +using ::testing::InSequence; + +namespace content { + +namespace { + +class MockStreamFactory : public audio::FakeStreamFactory, + public audio::mojom::LocalMuter { + public: + MockStreamFactory() : muter_binding_(this) {} + ~MockStreamFactory() final {} + + bool IsConnected() { + return binding_ && !binding_.handle().QuerySignalsState().peer_closed(); + } + bool IsMuterConnected() { return muter_binding_.is_bound(); } + + private: + void BindMuter(audio::mojom::LocalMuterAssociatedRequest request, + const base::UnguessableToken& group_id) final { + muter_binding_.Bind(std::move(request)); + muter_binding_.set_connection_error_handler(base::BindOnce( + &MockStreamFactory::MuterUnbound, base::Unretained(this))); + } + void MuterUnbound() { muter_binding_.Close(); } + + mojo::AssociatedBinding<audio::mojom::LocalMuter> muter_binding_; + DISALLOW_COPY_AND_ASSIGN(MockStreamFactory); +}; + +class MockBroker : public AudioStreamBroker { + public: + explicit MockBroker(RenderFrameHost* rfh) + : AudioStreamBroker(rfh->GetProcess()->GetID(), rfh->GetRoutingID()), + weak_factory_(this) {} + + ~MockBroker() override {} + + MOCK_METHOD1(CreateStream, void(audio::mojom::StreamFactory* factory)); + + // Can be used to verify that |this| has been destructed. + base::WeakPtr<MockBroker> GetWeakPtr() { return weak_factory_.GetWeakPtr(); } + + DeleterCallback deleter; + + private: + base::WeakPtrFactory<MockBroker> weak_factory_; + DISALLOW_COPY_AND_ASSIGN(MockBroker); +}; + +class MockBrokerFactory : public AudioStreamBrokerFactory { + public: + MockBrokerFactory() {} + ~MockBrokerFactory() final { + EXPECT_TRUE(prepared_input_stream_brokers_.empty()) + << "Input broker creation was expected but didn't happen"; + EXPECT_TRUE(prepared_output_stream_brokers_.empty()) + << "Output broker creation was expected but didn't happen"; + } + + void ExpectInputStreamBrokerCreation(std::unique_ptr<MockBroker> broker) { + prepared_input_stream_brokers_.push(std::move(broker)); + } + + void ExpectOutputStreamBrokerCreation(std::unique_ptr<MockBroker> broker) { + prepared_output_stream_brokers_.push(std::move(broker)); + } + + void ExpectLoopbackStreamBrokerCreation(std::unique_ptr<MockBroker> broker) { + prepared_loopback_stream_brokers_.push(std::move(broker)); + } + + std::unique_ptr<AudioStreamBroker> CreateAudioInputStreamBroker( + int render_process_id, + int render_frame_id, + const std::string& device_id, + const media::AudioParameters& params, + uint32_t shared_memory_count, + bool enable_agc, + AudioStreamBroker::DeleterCallback deleter, + mojom::RendererAudioInputStreamFactoryClientPtr renderer_factory_client) + final { + std::unique_ptr<MockBroker> prepared_broker = + std::move(prepared_input_stream_brokers_.front()); + prepared_input_stream_brokers_.pop(); + CHECK_NE(nullptr, prepared_broker.get()); + EXPECT_EQ(render_process_id, prepared_broker->render_process_id()); + EXPECT_EQ(render_frame_id, prepared_broker->render_frame_id()); + prepared_broker->deleter = std::move(deleter); + return std::move(prepared_broker); + } + + std::unique_ptr<AudioStreamBroker> CreateAudioOutputStreamBroker( + int render_process_id, + int render_frame_id, + int stream_id, + const std::string& output_device_id, + const media::AudioParameters& params, + const base::UnguessableToken& group_id, + AudioStreamBroker::DeleterCallback deleter, + media::mojom::AudioOutputStreamProviderClientPtr client) final { + std::unique_ptr<MockBroker> prepared_broker = + std::move(prepared_output_stream_brokers_.front()); + prepared_output_stream_brokers_.pop(); + CHECK_NE(nullptr, prepared_broker.get()); + EXPECT_EQ(render_process_id, prepared_broker->render_process_id()); + EXPECT_EQ(render_frame_id, prepared_broker->render_frame_id()); + prepared_broker->deleter = std::move(deleter); + return std::move(prepared_broker); + } + + std::unique_ptr<AudioStreamBroker> CreateAudioLoopbackStreamBroker( + int render_process_id, + int render_frame_id, + std::unique_ptr<LoopbackSource> source, + const media::AudioParameters& params, + uint32_t shared_memory_count, + bool mute_source, + AudioStreamBroker::DeleterCallback deleter, + mojom::RendererAudioInputStreamFactoryClientPtr renderer_factory_client) + final { + std::unique_ptr<MockBroker> prepared_broker = + std::move(prepared_loopback_stream_brokers_.front()); + prepared_loopback_stream_brokers_.pop(); + CHECK_NE(nullptr, prepared_broker.get()); + EXPECT_EQ(render_process_id, prepared_broker->render_process_id()); + EXPECT_EQ(render_frame_id, prepared_broker->render_frame_id()); + prepared_broker->deleter = std::move(deleter); + return std::move(prepared_broker); + } + + private: + base::queue<std::unique_ptr<MockBroker>> prepared_loopback_stream_brokers_; + base::queue<std::unique_ptr<MockBroker>> prepared_input_stream_brokers_; + base::queue<std::unique_ptr<MockBroker>> prepared_output_stream_brokers_; + DISALLOW_COPY_AND_ASSIGN(MockBrokerFactory); +}; + +class ForwardingAudioStreamFactoryTest : public RenderViewHostTestHarness { + public: + ForwardingAudioStreamFactoryTest() + : connector_(service_manager::Connector::Create(&connector_request_)), + broker_factory_(std::make_unique<MockBrokerFactory>()) { + service_manager::Connector::TestApi connector_test_api(connector_.get()); + connector_test_api.OverrideBinderForTesting( + service_manager::Identity(audio::mojom::kServiceName), + audio::mojom::StreamFactory::Name_, + base::BindRepeating(&ForwardingAudioStreamFactoryTest::BindFactory, + base::Unretained(this))); + } + + ~ForwardingAudioStreamFactoryTest() override {} + + void SetUp() override { + RenderViewHostTestHarness::SetUp(); + RenderFrameHostTester::For(main_rfh())->InitializeRenderFrameIfNeeded(); + other_rfh_ = + RenderFrameHostTester::For(main_rfh())->AppendChild("other_rfh"); + } + + void BindFactory(mojo::ScopedMessagePipeHandle factory_request) { + stream_factory_.binding_.Bind( + audio::mojom::StreamFactoryRequest(std::move(factory_request))); + } + + base::WeakPtr<MockBroker> ExpectLoopbackBrokerConstruction( + RenderFrameHost* rfh) { + auto broker = std::make_unique<StrictMock<MockBroker>>(rfh); + auto weak_broker = broker->GetWeakPtr(); + broker_factory_->ExpectLoopbackStreamBrokerCreation(std::move(broker)); + return weak_broker; + } + + base::WeakPtr<MockBroker> ExpectInputBrokerConstruction( + RenderFrameHost* rfh) { + auto broker = std::make_unique<StrictMock<MockBroker>>(rfh); + auto weak_broker = broker->GetWeakPtr(); + broker_factory_->ExpectInputStreamBrokerCreation(std::move(broker)); + return weak_broker; + } + + base::WeakPtr<MockBroker> ExpectOutputBrokerConstruction( + RenderFrameHost* rfh) { + auto broker = std::make_unique<StrictMock<MockBroker>>(rfh); + auto weak_broker = broker->GetWeakPtr(); + broker_factory_->ExpectOutputStreamBrokerCreation(std::move(broker)); + return weak_broker; + } + + RenderFrameHost* other_rfh() { return other_rfh_; } + + static const char kInputDeviceId[]; + static const char kOutputDeviceId[]; + static const media::AudioParameters kParams; + static const uint32_t kSharedMemoryCount; + static const bool kEnableAgc; + MockStreamFactory stream_factory_; + service_manager::mojom::ConnectorRequest connector_request_; + std::unique_ptr<service_manager::Connector> connector_; + RenderFrameHost* other_rfh_; + std::unique_ptr<MockBrokerFactory> broker_factory_; +}; + +const char ForwardingAudioStreamFactoryTest::kInputDeviceId[] = + "test input device id"; +const char ForwardingAudioStreamFactoryTest::kOutputDeviceId[] = + "test output device id"; +const media::AudioParameters ForwardingAudioStreamFactoryTest::kParams = + media::AudioParameters::UnavailableDeviceParams(); +const uint32_t ForwardingAudioStreamFactoryTest::kSharedMemoryCount = 10; +const bool ForwardingAudioStreamFactoryTest::kEnableAgc = false; +const bool kMuteSource = true; + +} // namespace + +TEST_F(ForwardingAudioStreamFactoryTest, CreateInputStream_CreatesInputStream) { + mojom::RendererAudioInputStreamFactoryClientPtr client; + base::WeakPtr<MockBroker> broker = ExpectInputBrokerConstruction(main_rfh()); + + ForwardingAudioStreamFactory factory(web_contents(), std::move(connector_), + std::move(broker_factory_)); + + EXPECT_CALL(*broker, CreateStream(NotNull())); + mojo::MakeRequest(&client); + factory.CreateInputStream(main_rfh(), kInputDeviceId, kParams, + kSharedMemoryCount, kEnableAgc, std::move(client)); +} + +TEST_F(ForwardingAudioStreamFactoryTest, + CreateLoopbackStream_CreatesLoopbackStream) { + std::unique_ptr<WebContents> source_contents = CreateTestWebContents(); + mojom::RendererAudioInputStreamFactoryClientPtr client; + base::WeakPtr<MockBroker> broker = + ExpectLoopbackBrokerConstruction(main_rfh()); + + ForwardingAudioStreamFactory factory(web_contents(), std::move(connector_), + std::move(broker_factory_)); + + EXPECT_CALL(*broker, CreateStream(NotNull())); + mojo::MakeRequest(&client); + factory.CreateLoopbackStream(main_rfh(), source_contents->GetMainFrame(), + kParams, kSharedMemoryCount, kMuteSource, + std::move(client)); +} + +TEST_F(ForwardingAudioStreamFactoryTest, + CreateOutputStream_CreatesOutputStream) { + media::mojom::AudioOutputStreamProviderClientPtr client; + base::WeakPtr<MockBroker> broker = ExpectOutputBrokerConstruction(main_rfh()); + + ForwardingAudioStreamFactory factory(web_contents(), std::move(connector_), + std::move(broker_factory_)); + + EXPECT_CALL(*broker, CreateStream(NotNull())); + mojo::MakeRequest(&client); + factory.CreateOutputStream(main_rfh(), kOutputDeviceId, kParams, + std::move(client)); +} + +TEST_F(ForwardingAudioStreamFactoryTest, + InputBrokerDeleterCalled_DestroysInputStream) { + mojom::RendererAudioInputStreamFactoryClientPtr client; + base::WeakPtr<MockBroker> main_rfh_broker = + ExpectInputBrokerConstruction(main_rfh()); + base::WeakPtr<MockBroker> other_rfh_broker = + ExpectInputBrokerConstruction(other_rfh()); + + ForwardingAudioStreamFactory factory(web_contents(), std::move(connector_), + std::move(broker_factory_)); + + { + EXPECT_CALL(*main_rfh_broker, CreateStream(NotNull())); + mojo::MakeRequest(&client); + factory.CreateInputStream(main_rfh(), kInputDeviceId, kParams, + kSharedMemoryCount, kEnableAgc, + std::move(client)); + testing::Mock::VerifyAndClear(&*main_rfh_broker); + } + { + EXPECT_CALL(*other_rfh_broker, CreateStream(NotNull())); + mojo::MakeRequest(&client); + factory.CreateInputStream(other_rfh(), kInputDeviceId, kParams, + kSharedMemoryCount, kEnableAgc, + std::move(client)); + testing::Mock::VerifyAndClear(&*other_rfh_broker); + } + + std::move(other_rfh_broker->deleter).Run(&*other_rfh_broker); + EXPECT_FALSE(other_rfh_broker) + << "Input broker should be destructed when deleter is called."; + EXPECT_TRUE(main_rfh_broker); +} + +TEST_F(ForwardingAudioStreamFactoryTest, + LoopbackBrokerDeleterCalled_DestroysInputStream) { + std::unique_ptr<WebContents> source_contents = CreateTestWebContents(); + mojom::RendererAudioInputStreamFactoryClientPtr client; + base::WeakPtr<MockBroker> main_rfh_broker = + ExpectLoopbackBrokerConstruction(main_rfh()); + base::WeakPtr<MockBroker> other_rfh_broker = + ExpectLoopbackBrokerConstruction(other_rfh()); + + ForwardingAudioStreamFactory factory(web_contents(), std::move(connector_), + std::move(broker_factory_)); + + { + EXPECT_CALL(*main_rfh_broker, CreateStream(NotNull())); + mojo::MakeRequest(&client); + factory.CreateLoopbackStream(main_rfh(), source_contents->GetMainFrame(), + kParams, kSharedMemoryCount, kMuteSource, + std::move(client)); + testing::Mock::VerifyAndClear(&*main_rfh_broker); + } + { + EXPECT_CALL(*other_rfh_broker, CreateStream(NotNull())); + mojo::MakeRequest(&client); + factory.CreateLoopbackStream(other_rfh(), source_contents->GetMainFrame(), + kParams, kSharedMemoryCount, kMuteSource, + std::move(client)); + testing::Mock::VerifyAndClear(&*other_rfh_broker); + } + + std::move(other_rfh_broker->deleter).Run(&*other_rfh_broker); + EXPECT_FALSE(other_rfh_broker) + << "Loopback broker should be destructed when deleter is called."; + EXPECT_TRUE(main_rfh_broker); +} + +TEST_F(ForwardingAudioStreamFactoryTest, + OutputBrokerDeleterCalled_DestroysOutputStream) { + media::mojom::AudioOutputStreamProviderClientPtr client; + base::WeakPtr<MockBroker> main_rfh_broker = + ExpectOutputBrokerConstruction(main_rfh()); + base::WeakPtr<MockBroker> other_rfh_broker = + ExpectOutputBrokerConstruction(other_rfh()); + + ForwardingAudioStreamFactory factory(web_contents(), std::move(connector_), + std::move(broker_factory_)); + + { + EXPECT_CALL(*main_rfh_broker, CreateStream(NotNull())); + mojo::MakeRequest(&client); + factory.CreateOutputStream(main_rfh(), kOutputDeviceId, kParams, + std::move(client)); + testing::Mock::VerifyAndClear(&*main_rfh_broker); + } + { + EXPECT_CALL(*other_rfh_broker, CreateStream(NotNull())); + mojo::MakeRequest(&client); + factory.CreateOutputStream(other_rfh(), kOutputDeviceId, kParams, + std::move(client)); + testing::Mock::VerifyAndClear(&*other_rfh_broker); + } + + std::move(other_rfh_broker->deleter).Run(&*other_rfh_broker); + EXPECT_FALSE(other_rfh_broker) + << "Output broker should be destructed when deleter is called."; + EXPECT_TRUE(main_rfh_broker); +} + +TEST_F(ForwardingAudioStreamFactoryTest, DestroyFrame_DestroysRelatedStreams) { + std::unique_ptr<WebContents> source_contents = CreateTestWebContents(); + + mojom::RendererAudioInputStreamFactoryClientPtr input_client; + base::WeakPtr<MockBroker> main_rfh_input_broker = + ExpectInputBrokerConstruction(main_rfh()); + base::WeakPtr<MockBroker> other_rfh_input_broker = + ExpectInputBrokerConstruction(other_rfh()); + + base::WeakPtr<MockBroker> main_rfh_loopback_broker = + ExpectLoopbackBrokerConstruction(main_rfh()); + base::WeakPtr<MockBroker> other_rfh_loopback_broker = + ExpectLoopbackBrokerConstruction(other_rfh()); + + media::mojom::AudioOutputStreamProviderClientPtr output_client; + base::WeakPtr<MockBroker> main_rfh_output_broker = + ExpectOutputBrokerConstruction(main_rfh()); + base::WeakPtr<MockBroker> other_rfh_output_broker = + ExpectOutputBrokerConstruction(other_rfh()); + + ForwardingAudioStreamFactory factory(web_contents(), std::move(connector_), + std::move(broker_factory_)); + + { + EXPECT_CALL(*main_rfh_input_broker, CreateStream(NotNull())); + mojo::MakeRequest(&input_client); + factory.CreateInputStream(main_rfh(), kInputDeviceId, kParams, + kSharedMemoryCount, kEnableAgc, + std::move(input_client)); + testing::Mock::VerifyAndClear(&*main_rfh_input_broker); + } + { + EXPECT_CALL(*other_rfh_input_broker, CreateStream(NotNull())); + mojo::MakeRequest(&input_client); + factory.CreateInputStream(other_rfh(), kInputDeviceId, kParams, + kSharedMemoryCount, kEnableAgc, + std::move(input_client)); + testing::Mock::VerifyAndClear(&*other_rfh_input_broker); + } + + { + EXPECT_CALL(*main_rfh_loopback_broker, CreateStream(NotNull())); + mojo::MakeRequest(&input_client); + factory.CreateLoopbackStream(main_rfh(), source_contents->GetMainFrame(), + kParams, kSharedMemoryCount, kMuteSource, + std::move(input_client)); + testing::Mock::VerifyAndClear(&*main_rfh_loopback_broker); + } + { + EXPECT_CALL(*other_rfh_loopback_broker, CreateStream(NotNull())); + mojo::MakeRequest(&input_client); + factory.CreateLoopbackStream(other_rfh(), source_contents->GetMainFrame(), + kParams, kSharedMemoryCount, kMuteSource, + std::move(input_client)); + testing::Mock::VerifyAndClear(&*other_rfh_loopback_broker); + } + + { + EXPECT_CALL(*main_rfh_output_broker, CreateStream(NotNull())); + mojo::MakeRequest(&output_client); + factory.CreateOutputStream(main_rfh(), kOutputDeviceId, kParams, + std::move(output_client)); + testing::Mock::VerifyAndClear(&*main_rfh_output_broker); + } + { + EXPECT_CALL(*other_rfh_output_broker, CreateStream(NotNull())); + mojo::MakeRequest(&output_client); + factory.CreateOutputStream(other_rfh(), kOutputDeviceId, kParams, + std::move(output_client)); + testing::Mock::VerifyAndClear(&*other_rfh_output_broker); + } + + factory.FrameDeleted(other_rfh()); + base::RunLoop().RunUntilIdle(); + EXPECT_FALSE(other_rfh_input_broker) + << "Input broker should be destructed when owning frame is destructed."; + EXPECT_TRUE(main_rfh_input_broker); + EXPECT_FALSE(other_rfh_loopback_broker) << "Loopback broker should be " + "destructed when owning frame is " + "destructed."; + EXPECT_TRUE(main_rfh_loopback_broker); + EXPECT_FALSE(other_rfh_output_broker) + << "Output broker should be destructed when owning frame is destructed."; + EXPECT_TRUE(main_rfh_output_broker); +} + +TEST_F(ForwardingAudioStreamFactoryTest, DestroyWebContents_DestroysStreams) { + mojom::RendererAudioInputStreamFactoryClientPtr input_client; + base::WeakPtr<MockBroker> input_broker = + ExpectInputBrokerConstruction(main_rfh()); + + media::mojom::AudioOutputStreamProviderClientPtr output_client; + base::WeakPtr<MockBroker> output_broker = + ExpectOutputBrokerConstruction(main_rfh()); + + ForwardingAudioStreamFactory factory(web_contents(), std::move(connector_), + std::move(broker_factory_)); + + EXPECT_CALL(*input_broker, CreateStream(NotNull())); + mojo::MakeRequest(&input_client); + factory.CreateInputStream(main_rfh(), kInputDeviceId, kParams, + kSharedMemoryCount, kEnableAgc, + std::move(input_client)); + + EXPECT_CALL(*output_broker, CreateStream(NotNull())); + mojo::MakeRequest(&output_client); + factory.CreateOutputStream(main_rfh(), kOutputDeviceId, kParams, + std::move(output_client)); + + DeleteContents(); + base::RunLoop().RunUntilIdle(); + EXPECT_FALSE(input_broker) << "Input broker should be destructed when owning " + "WebContents is destructed."; + EXPECT_FALSE(output_broker) + << "Output broker should be destructed when owning " + "WebContents is destructed."; +} + +TEST_F(ForwardingAudioStreamFactoryTest, LastStreamDeleted_ClearsFactoryPtr) { + mojom::RendererAudioInputStreamFactoryClientPtr input_client; + base::WeakPtr<MockBroker> main_rfh_input_broker = + ExpectInputBrokerConstruction(main_rfh()); + base::WeakPtr<MockBroker> other_rfh_input_broker = + ExpectInputBrokerConstruction(other_rfh()); + + media::mojom::AudioOutputStreamProviderClientPtr output_client; + base::WeakPtr<MockBroker> main_rfh_output_broker = + ExpectOutputBrokerConstruction(main_rfh()); + base::WeakPtr<MockBroker> other_rfh_output_broker = + ExpectOutputBrokerConstruction(other_rfh()); + + ForwardingAudioStreamFactory factory(web_contents(), std::move(connector_), + std::move(broker_factory_)); + + { + EXPECT_CALL(*main_rfh_input_broker, CreateStream(NotNull())); + mojo::MakeRequest(&input_client); + factory.CreateInputStream(main_rfh(), kInputDeviceId, kParams, + kSharedMemoryCount, kEnableAgc, + std::move(input_client)); + testing::Mock::VerifyAndClear(&*main_rfh_input_broker); + } + { + EXPECT_CALL(*other_rfh_input_broker, CreateStream(NotNull())); + mojo::MakeRequest(&input_client); + factory.CreateInputStream(other_rfh(), kInputDeviceId, kParams, + kSharedMemoryCount, kEnableAgc, + std::move(input_client)); + testing::Mock::VerifyAndClear(&*other_rfh_input_broker); + } + + { + EXPECT_CALL(*main_rfh_output_broker, CreateStream(NotNull())); + mojo::MakeRequest(&output_client); + factory.CreateOutputStream(main_rfh(), kOutputDeviceId, kParams, + std::move(output_client)); + testing::Mock::VerifyAndClear(&*main_rfh_output_broker); + } + { + EXPECT_CALL(*other_rfh_output_broker, CreateStream(NotNull())); + mojo::MakeRequest(&output_client); + factory.CreateOutputStream(other_rfh(), kOutputDeviceId, kParams, + std::move(output_client)); + testing::Mock::VerifyAndClear(&*other_rfh_output_broker); + } + + base::RunLoop().RunUntilIdle(); + EXPECT_TRUE(stream_factory_.IsConnected()); + std::move(other_rfh_input_broker->deleter).Run(&*other_rfh_input_broker); + base::RunLoop().RunUntilIdle(); + // Connection should still be open, since there are still streams left. + EXPECT_TRUE(stream_factory_.IsConnected()); + std::move(main_rfh_input_broker->deleter).Run(&*main_rfh_input_broker); + base::RunLoop().RunUntilIdle(); + // Connection should still be open, since there are still streams left. + EXPECT_TRUE(stream_factory_.IsConnected()); + std::move(other_rfh_output_broker->deleter).Run(&*other_rfh_output_broker); + base::RunLoop().RunUntilIdle(); + // Connection should still be open, since there's still a stream left. + EXPECT_TRUE(stream_factory_.IsConnected()); + std::move(main_rfh_output_broker->deleter).Run(&*main_rfh_output_broker); + base::RunLoop().RunUntilIdle(); + // Now there are no streams left, connection should be broken. + EXPECT_FALSE(stream_factory_.IsConnected()); +} + +TEST_F(ForwardingAudioStreamFactoryTest, + MuteNoOutputStreams_DoesNotConnectMuter) { + ForwardingAudioStreamFactory factory(web_contents(), std::move(connector_), + std::move(broker_factory_)); + EXPECT_FALSE(factory.IsMuted()); + + factory.SetMuted(true); + base::RunLoop().RunUntilIdle(); + EXPECT_TRUE(factory.IsMuted()); + EXPECT_FALSE(stream_factory_.IsConnected()); + EXPECT_FALSE(stream_factory_.IsMuterConnected()); + + factory.SetMuted(false); + base::RunLoop().RunUntilIdle(); + EXPECT_FALSE(factory.IsMuted()); + EXPECT_FALSE(stream_factory_.IsConnected()); + EXPECT_FALSE(stream_factory_.IsMuterConnected()); +} + +TEST_F(ForwardingAudioStreamFactoryTest, MuteWithOutputStream_ConnectsMuter) { + media::mojom::AudioOutputStreamProviderClientPtr client; + base::WeakPtr<MockBroker> broker = ExpectOutputBrokerConstruction(main_rfh()); + ForwardingAudioStreamFactory factory(web_contents(), std::move(connector_), + std::move(broker_factory_)); + + EXPECT_CALL(*broker, CreateStream(NotNull())); + mojo::MakeRequest(&client); + factory.CreateOutputStream(main_rfh(), kOutputDeviceId, kParams, + std::move(client)); + base::RunLoop().RunUntilIdle(); + testing::Mock::VerifyAndClear(&*broker); + + EXPECT_TRUE(stream_factory_.IsConnected()); + EXPECT_FALSE(factory.IsMuted()); + + factory.SetMuted(true); + base::RunLoop().RunUntilIdle(); + EXPECT_TRUE(factory.IsMuted()); + EXPECT_TRUE(stream_factory_.IsConnected()); + EXPECT_TRUE(stream_factory_.IsMuterConnected()); + + factory.SetMuted(false); + base::RunLoop().RunUntilIdle(); + EXPECT_FALSE(factory.IsMuted()); + EXPECT_TRUE(stream_factory_.IsConnected()); + EXPECT_FALSE(stream_factory_.IsMuterConnected()); +} + +TEST_F(ForwardingAudioStreamFactoryTest, + WhenMuting_ConnectedWhenOutputStreamExists) { + media::mojom::AudioOutputStreamProviderClientPtr client; + base::WeakPtr<MockBroker> broker = ExpectOutputBrokerConstruction(main_rfh()); + ForwardingAudioStreamFactory factory(web_contents(), std::move(connector_), + std::move(broker_factory_)); + + EXPECT_FALSE(stream_factory_.IsConnected()); + EXPECT_FALSE(factory.IsMuted()); + + factory.SetMuted(true); + base::RunLoop().RunUntilIdle(); + EXPECT_TRUE(factory.IsMuted()); + EXPECT_FALSE(stream_factory_.IsConnected()); + EXPECT_FALSE(stream_factory_.IsMuterConnected()); + + EXPECT_CALL(*broker, CreateStream(NotNull())); + mojo::MakeRequest(&client); + factory.CreateOutputStream(main_rfh(), kOutputDeviceId, kParams, + std::move(client)); + base::RunLoop().RunUntilIdle(); + EXPECT_TRUE(factory.IsMuted()); + EXPECT_TRUE(stream_factory_.IsConnected()); + EXPECT_TRUE(stream_factory_.IsMuterConnected()); + testing::Mock::VerifyAndClear(&*broker); + + std::move(broker->deleter).Run(&*broker); + EXPECT_FALSE(broker); + base::RunLoop().RunUntilIdle(); + EXPECT_TRUE(factory.IsMuted()); + EXPECT_FALSE(stream_factory_.IsConnected()); + EXPECT_FALSE(stream_factory_.IsMuterConnected()); +} + +TEST_F(ForwardingAudioStreamFactoryTest, + WhenMuting_AddRemoveSecondStream_DoesNotChangeMuting) { + media::mojom::AudioOutputStreamProviderClientPtr client; + base::WeakPtr<MockBroker> broker = ExpectOutputBrokerConstruction(main_rfh()); + base::WeakPtr<MockBroker> another_broker = + ExpectOutputBrokerConstruction(main_rfh()); + ForwardingAudioStreamFactory factory(web_contents(), std::move(connector_), + std::move(broker_factory_)); + + { + EXPECT_CALL(*broker, CreateStream(NotNull())); + mojo::MakeRequest(&client); + factory.CreateOutputStream(main_rfh(), kOutputDeviceId, kParams, + std::move(client)); + base::RunLoop().RunUntilIdle(); + testing::Mock::VerifyAndClear(&*broker); + } + EXPECT_TRUE(stream_factory_.IsConnected()); + EXPECT_FALSE(factory.IsMuted()); + + factory.SetMuted(true); + base::RunLoop().RunUntilIdle(); + EXPECT_TRUE(factory.IsMuted()); + EXPECT_TRUE(stream_factory_.IsConnected()); + EXPECT_TRUE(stream_factory_.IsMuterConnected()); + + { + EXPECT_CALL(*another_broker, CreateStream(NotNull())); + mojo::MakeRequest(&client); + factory.CreateOutputStream(main_rfh(), kOutputDeviceId, kParams, + std::move(client)); + base::RunLoop().RunUntilIdle(); + testing::Mock::VerifyAndClear(&*another_broker); + } + + EXPECT_TRUE(factory.IsMuted()); + EXPECT_TRUE(stream_factory_.IsConnected()); + EXPECT_TRUE(stream_factory_.IsMuterConnected()); + + std::move(another_broker->deleter).Run(&*another_broker); + EXPECT_FALSE(another_broker); + base::RunLoop().RunUntilIdle(); + + EXPECT_TRUE(factory.IsMuted()); + EXPECT_TRUE(stream_factory_.IsConnected()); + EXPECT_TRUE(stream_factory_.IsMuterConnected()); +} + +} // namespace content diff --git a/chromium/content/browser/media/key_system_support_impl.cc b/chromium/content/browser/media/key_system_support_impl.cc index 0092ef764f1..414da641e76 100644 --- a/chromium/content/browser/media/key_system_support_impl.cc +++ b/chromium/content/browser/media/key_system_support_impl.cc @@ -6,6 +6,7 @@ #include <vector> +#include "base/containers/flat_set.h" #include "base/logging.h" #include "base/metrics/histogram_functions.h" #include "content/public/browser/cdm_registry.h" @@ -62,13 +63,16 @@ void KeySystemSupportImpl::IsKeySystemSupported( std::unique_ptr<CdmInfo> cdm = GetCdmInfoForKeySystem(key_system); if (!cdm) { SendCdmAvailableUMA(key_system, false); - std::move(callback).Run(false, {}, false); + std::move(callback).Run(false, {}, false, {}); return; } + const base::flat_set<media::EncryptionMode>& schemes = + cdm->supported_encryption_schemes; SendCdmAvailableUMA(key_system, true); - std::move(callback).Run(true, cdm->supported_video_codecs, - cdm->supports_persistent_license); + std::move(callback).Run( + true, cdm->supported_video_codecs, cdm->supports_persistent_license, + std::vector<media::EncryptionMode>(schemes.begin(), schemes.end())); } } // namespace content diff --git a/chromium/content/browser/media/key_system_support_impl_unittest.cc b/chromium/content/browser/media/key_system_support_impl_unittest.cc index 719dcf29eec..aa3134ae60e 100644 --- a/chromium/content/browser/media/key_system_support_impl_unittest.cc +++ b/chromium/content/browser/media/key_system_support_impl_unittest.cc @@ -14,6 +14,7 @@ #include "content/public/common/cdm_info.h" #include "content/public/common/webplugininfo.h" #include "content/public/test/test_browser_thread_bundle.h" +#include "media/base/decrypt_config.h" #include "media/base/video_codecs.h" #include "testing/gtest/include/gtest/gtest.h" @@ -39,24 +40,16 @@ class KeySystemSupportTest : public testing::Test { // |supports_persistent_license|. All other values for CdmInfo have some // default value as they're not returned by IsKeySystemSupported(). void Register(const std::string& key_system, - const std::vector<media::VideoCodec> supported_video_codecs, - bool supports_persistent_license) { + const std::vector<media::VideoCodec>& supported_video_codecs, + bool supports_persistent_license, + const base::flat_set<media::EncryptionMode>& supported_modes) { DVLOG(1) << __func__; CdmRegistry::GetInstance()->RegisterCdm( CdmInfo(key_system, kTestCdmGuid, base::Version(kVersion), base::FilePath::FromUTF8Unsafe(kTestPath), kTestFileSystemId, - supported_video_codecs, supports_persistent_license, key_system, - false)); - - // As IsKeySystemSupported() checks for the matching pepper plugin, - // register a dummy one. - // TODO(crbug.com/772160) Remove this when pepper CDM support removed. - PluginService::GetInstance()->RegisterInternalPlugin( - WebPluginInfo(base::ASCIIToUTF16(key_system), - base::FilePath::FromUTF8Unsafe(kTestPath), - base::ASCIIToUTF16(kVersion), base::string16()), - true); + supported_video_codecs, supports_persistent_license, + supported_modes, key_system, false)); } // Determines if |key_system| is registered. If it is, updates |codecs_| @@ -65,7 +58,8 @@ class KeySystemSupportTest : public testing::Test { DVLOG(1) << __func__; bool is_available = false; key_system_support_->IsKeySystemSupported(key_system, &is_available, - &codecs_, &persistent_); + &codecs_, &persistent_, + &encryption_schemes_); return is_available; } @@ -75,6 +69,7 @@ class KeySystemSupportTest : public testing::Test { // Updated by IsSupported(). std::vector<media::VideoCodec> codecs_; bool persistent_; + std::vector<media::EncryptionMode> encryption_schemes_; }; // Note that as CdmRegistry::GetInstance() is a static, it is shared between @@ -86,30 +81,41 @@ TEST_F(KeySystemSupportTest, NoKeySystems) { } TEST_F(KeySystemSupportTest, OneKeySystem) { - Register("KeySystem2", {media::VideoCodec::kCodecVP8}, true); + Register("KeySystem2", {media::VideoCodec::kCodecVP8}, true, + {media::EncryptionMode::kCenc, media::EncryptionMode::kCbcs}); EXPECT_TRUE(IsSupported("KeySystem2")); EXPECT_EQ(1u, codecs_.size()); EXPECT_EQ(media::VideoCodec::kCodecVP8, codecs_[0]); EXPECT_TRUE(persistent_); + EXPECT_EQ(2u, encryption_schemes_.size()); + EXPECT_EQ(media::EncryptionMode::kCenc, encryption_schemes_[0]); + EXPECT_EQ(media::EncryptionMode::kCbcs, encryption_schemes_[1]); } TEST_F(KeySystemSupportTest, MultipleKeySystems) { Register("KeySystem3", - {media::VideoCodec::kCodecVP8, media::VideoCodec::kCodecVP9}, true); - Register("KeySystem4", {media::VideoCodec::kCodecVP9}, false); + {media::VideoCodec::kCodecVP8, media::VideoCodec::kCodecVP9}, true, + {media::EncryptionMode::kCenc}); + Register("KeySystem4", {media::VideoCodec::kCodecVP9}, false, + {media::EncryptionMode::kCbcs}); EXPECT_TRUE(IsSupported("KeySystem3")); EXPECT_EQ(2u, codecs_.size()); EXPECT_EQ(media::VideoCodec::kCodecVP8, codecs_[0]); EXPECT_EQ(media::VideoCodec::kCodecVP9, codecs_[1]); EXPECT_TRUE(persistent_); + EXPECT_EQ(1u, encryption_schemes_.size()); + EXPECT_EQ(media::EncryptionMode::kCenc, encryption_schemes_[0]); EXPECT_TRUE(IsSupported("KeySystem4")); EXPECT_EQ(1u, codecs_.size()); EXPECT_EQ(media::VideoCodec::kCodecVP9, codecs_[0]); EXPECT_FALSE(persistent_); + EXPECT_EQ(1u, encryption_schemes_.size()); + EXPECT_EQ(media::EncryptionMode::kCbcs, encryption_schemes_[0]); } TEST_F(KeySystemSupportTest, MissingKeySystem) { - Register("KeySystem5", {media::VideoCodec::kCodecVP8}, true); + Register("KeySystem5", {media::VideoCodec::kCodecVP8}, true, + {media::EncryptionMode::kCenc}); EXPECT_FALSE(IsSupported("KeySystem6")); } diff --git a/chromium/content/browser/media/keyboard_mic_registration.cc b/chromium/content/browser/media/keyboard_mic_registration.cc new file mode 100644 index 00000000000..785f6d3fa7f --- /dev/null +++ b/chromium/content/browser/media/keyboard_mic_registration.cc @@ -0,0 +1,31 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "content/browser/media/keyboard_mic_registration.h" + +#include "chromeos/audio/cras_audio_handler.h" +#include "content/public/browser/browser_thread.h" + +namespace content { + +KeyboardMicRegistration::KeyboardMicRegistration() = default; + +KeyboardMicRegistration::~KeyboardMicRegistration() { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + DCHECK_EQ(0, register_count_); +} + +void KeyboardMicRegistration::Register() { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + if (++register_count_ == 1) + chromeos::CrasAudioHandler::Get()->SetKeyboardMicActive(true); +} + +void KeyboardMicRegistration::Deregister() { + DCHECK_CURRENTLY_ON(BrowserThread::UI); + if (--register_count_ == 0) + chromeos::CrasAudioHandler::Get()->SetKeyboardMicActive(false); +} + +} // namespace content diff --git a/chromium/content/browser/media/keyboard_mic_registration.h b/chromium/content/browser/media/keyboard_mic_registration.h new file mode 100644 index 00000000000..267d0dee1ce --- /dev/null +++ b/chromium/content/browser/media/keyboard_mic_registration.h @@ -0,0 +1,31 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef CONTENT_BROWSER_MEDIA_KEYBOARD_MIC_REGISTRATION_H_ +#define CONTENT_BROWSER_MEDIA_KEYBOARD_MIC_REGISTRATION_H_ + +#include "base/macros.h" + +namespace content { + +// Chrome OS keyboard mic stream registration. Used on UI thread only and owned +// by BrowserMainLoop; instance must be obtained through +// BrowserMainLoop::keyboard_mic_registration(). +class KeyboardMicRegistration { + public: + KeyboardMicRegistration(); + ~KeyboardMicRegistration(); + + void Register(); + void Deregister(); + + private: + int register_count_ = 0; + + DISALLOW_COPY_AND_ASSIGN(KeyboardMicRegistration); +}; + +} // namespace content + +#endif // CONTENT_BROWSER_MEDIA_KEYBOARD_MIC_REGISTRATION_H_ diff --git a/chromium/content/browser/media/media_canplaytype_browsertest.cc b/chromium/content/browser/media/media_canplaytype_browsertest.cc index 11bfbc8548e..f6706151c8f 100644 --- a/chromium/content/browser/media/media_canplaytype_browsertest.cc +++ b/chromium/content/browser/media/media_canplaytype_browsertest.cc @@ -15,7 +15,7 @@ #include "content/shell/browser/shell.h" #include "media/base/media_switches.h" #include "media/media_buildflags.h" -#include "third_party/libaom/av1_features.h" +#include "third_party/libaom/av1_buildflags.h" #include "ui/display/display_switches.h" #if defined(OS_ANDROID) diff --git a/chromium/content/browser/media/media_devices_util.cc b/chromium/content/browser/media/media_devices_util.cc index 9440bc7f4b7..58badef2512 100644 --- a/chromium/content/browser/media/media_devices_util.cc +++ b/chromium/content/browser/media/media_devices_util.cc @@ -13,10 +13,12 @@ #include "base/strings/string_tokenizer.h" #include "content/browser/frame_host/render_frame_host_delegate.h" #include "content/browser/frame_host/render_frame_host_impl.h" +#include "content/browser/web_contents/web_contents_impl.h" #include "content/public/browser/browser_context.h" #include "content/public/browser/browser_thread.h" #include "content/public/browser/media_device_id.h" #include "content/public/browser/render_frame_host.h" +#include "content/public/browser/web_contents.h" #include "content/public/common/media_stream_request.h" #include "media/base/media_switches.h" @@ -91,6 +93,15 @@ std::string GetDefaultMediaDeviceIDFromCommandLine( } // namespace +MediaDeviceSaltAndOrigin::MediaDeviceSaltAndOrigin() = default; + +MediaDeviceSaltAndOrigin::MediaDeviceSaltAndOrigin(std::string device_id_salt, + std::string group_id_salt, + url::Origin origin) + : device_id_salt(std::move(device_id_salt)), + group_id_salt(std::move(group_id_salt)), + origin(std::move(origin)) {} + void GetDefaultMediaDeviceID( MediaDeviceType device_type, int render_process_id, @@ -113,46 +124,55 @@ void GetDefaultMediaDeviceID( callback); } -std::pair<std::string, url::Origin> GetMediaDeviceSaltAndOrigin( - int render_process_id, - int render_frame_id) { +MediaDeviceSaltAndOrigin GetMediaDeviceSaltAndOrigin(int render_process_id, + int render_frame_id) { DCHECK_CURRENTLY_ON(BrowserThread::UI); RenderFrameHost* frame_host = RenderFrameHost::FromID(render_process_id, render_frame_id); RenderProcessHost* process_host = RenderProcessHost::FromID(render_process_id); - return std::make_pair( + WebContentsImpl* web_contents = static_cast<WebContentsImpl*>( + WebContents::FromRenderFrameHost(frame_host)); + + std::string device_id_salt = process_host ? process_host->GetBrowserContext()->GetMediaDeviceIDSalt() - : std::string(), - frame_host ? frame_host->GetLastCommittedOrigin() : url::Origin()); + : std::string(); + std::string group_id_salt = + device_id_salt + (web_contents + ? web_contents->GetMediaDeviceGroupIDSaltBase() + : std::string()); + url::Origin origin = + frame_host ? frame_host->GetLastCommittedOrigin() : url::Origin(); + + return {std::move(device_id_salt), std::move(group_id_salt), + std::move(origin)}; } -MediaDeviceInfo TranslateMediaDeviceInfo(bool has_permission, - const std::string& device_id_salt, - const std::string& group_id_salt, - const url::Origin& security_origin, - const MediaDeviceInfo& device_info) { +MediaDeviceInfo TranslateMediaDeviceInfo( + bool has_permission, + const MediaDeviceSaltAndOrigin& salt_and_origin, + const MediaDeviceInfo& device_info) { return MediaDeviceInfo( - GetHMACForMediaDeviceID(device_id_salt, security_origin, - device_info.device_id), + GetHMACForMediaDeviceID(salt_and_origin.device_id_salt, + salt_and_origin.origin, device_info.device_id), has_permission ? device_info.label : std::string(), device_info.group_id.empty() ? std::string() - : GetHMACForMediaDeviceID(group_id_salt, security_origin, - device_info.group_id)); + : GetHMACForMediaDeviceID(salt_and_origin.group_id_salt, + salt_and_origin.origin, + device_info.group_id), + has_permission ? device_info.video_facing + : media::MEDIA_VIDEO_FACING_NONE); } MediaDeviceInfoArray TranslateMediaDeviceInfoArray( bool has_permission, - const std::string& device_id_salt, - const std::string& group_id_salt, - const url::Origin& security_origin, + const MediaDeviceSaltAndOrigin& salt_and_origin, const MediaDeviceInfoArray& device_infos) { MediaDeviceInfoArray result; for (const auto& device_info : device_infos) { - result.push_back(TranslateMediaDeviceInfo(has_permission, device_id_salt, - group_id_salt, security_origin, - device_info)); + result.push_back( + TranslateMediaDeviceInfo(has_permission, salt_and_origin, device_info)); } return result; } diff --git a/chromium/content/browser/media/media_devices_util.h b/chromium/content/browser/media/media_devices_util.h index c43e6909617..d2a02ef498e 100644 --- a/chromium/content/browser/media/media_devices_util.h +++ b/chromium/content/browser/media/media_devices_util.h @@ -17,45 +17,52 @@ namespace content { // Returns the ID of the user-default device ID via |callback|. // If no such device ID can be found, |callback| receives an empty string. -void CONTENT_EXPORT GetDefaultMediaDeviceID( +CONTENT_EXPORT void GetDefaultMediaDeviceID( MediaDeviceType device_type, int render_process_id, int render_frame_id, const base::Callback<void(const std::string&)>& callback); +struct CONTENT_EXPORT MediaDeviceSaltAndOrigin { + MediaDeviceSaltAndOrigin(); + MediaDeviceSaltAndOrigin(std::string device_id_salt, + std::string group_id_salt, + url::Origin origin); + + std::string device_id_salt; + std::string group_id_salt; + url::Origin origin; +}; + // Returns the current media device ID salt and security origin for the given // |render_process_id| and |render_frame_id|. These values are used to produce // unique media-device IDs for each origin and renderer process. These values // should not be cached since the user can explicitly change them at any time. // This function must run on the UI thread. -std::pair<std::string, url::Origin> GetMediaDeviceSaltAndOrigin( - int render_process_id, - int render_frame_id); +MediaDeviceSaltAndOrigin GetMediaDeviceSaltAndOrigin(int render_process_id, + int render_frame_id); // Returns a translated version of |device_info| suitable for use in a renderer // process. // The |device_id| field is hashed using |device_id_salt| and |security_origin|. // The |group_id| field is hashed using |group_id_salt| and |security_origin|. // The |label| field is removed if |has_permission| is false. -MediaDeviceInfo TranslateMediaDeviceInfo(bool has_permission, - const std::string& device_id_salt, - const std::string& group_id_salt, - const url::Origin& security_origin, - const MediaDeviceInfo& device_info); +MediaDeviceInfo TranslateMediaDeviceInfo( + bool has_permission, + const MediaDeviceSaltAndOrigin& salt_and_origin, + const MediaDeviceInfo& device_info); // Returns a translated version of |device_infos|, with each element translated // using TranslateMediaDeviceInfo(). MediaDeviceInfoArray TranslateMediaDeviceInfoArray( bool has_permission, - const std::string& device_id_salt, - const std::string& group_id_salt, - const url::Origin& security_origin, + const MediaDeviceSaltAndOrigin& salt_and_origin, const MediaDeviceInfoArray& device_infos); // Type definition to make it easier to use mock alternatives to // GetMediaDeviceSaltAndOrigin. using MediaDeviceSaltAndOriginCallback = - base::RepeatingCallback<std::pair<std::string, url::Origin>(int, int)>; + base::RepeatingCallback<MediaDeviceSaltAndOrigin(int, int)>; } // namespace content diff --git a/chromium/content/browser/media/media_interface_proxy.cc b/chromium/content/browser/media/media_interface_proxy.cc index 31bf2c0f4d8..82b0c8c61ac 100644 --- a/chromium/content/browser/media/media_interface_proxy.cc +++ b/chromium/content/browser/media/media_interface_proxy.cc @@ -27,7 +27,7 @@ #include "content/public/browser/provision_fetcher_impl.h" #include "content/public/browser/render_process_host.h" #include "content/public/browser/storage_partition.h" -#include "net/url_request/url_request_context_getter.h" +#include "services/network/public/cpp/shared_url_loader_factory.h" #endif #if BUILDFLAG(ENABLE_LIBRARY_CDMS) @@ -46,6 +46,7 @@ #if defined(OS_ANDROID) #include "content/browser/media/android/media_player_renderer.h" +#include "content/browser/media/flinging_renderer.h" #include "media/mojo/services/mojo_renderer_service.h" // nogncheck #endif @@ -161,6 +162,17 @@ void MediaInterfaceProxy::CreateRenderer( CreateMediaPlayerRenderer(std::move(request)); return; } + + if (type == media::mojom::HostedRendererType::kFlinging) { + std::unique_ptr<FlingingRenderer> renderer = + FlingingRenderer::Create(render_frame_host_, type_specific_id); + + media::MojoRendererService::Create( + nullptr, std::move(renderer), + media::MojoRendererService::InitiateSurfaceRequestCB(), + std::move(request)); + return; + } #endif InterfaceFactory* factory = GetMediaInterfaceFactory(); @@ -183,6 +195,14 @@ void MediaInterfaceProxy::CreateCdm( #endif } +void MediaInterfaceProxy::CreateDecryptor( + int cdm_id, + media::mojom::DecryptorRequest request) { + InterfaceFactory* factory = GetMediaInterfaceFactory(); + if (factory) + factory->CreateDecryptor(cdm_id, std::move(request)); +} + void MediaInterfaceProxy::CreateCdmProxy( const std::string& cdm_guid, media::mojom::CdmProxyRequest request) { @@ -202,12 +222,12 @@ MediaInterfaceProxy::GetFrameServices(const std::string& cdm_guid, #if BUILDFLAG(ENABLE_MOJO_CDM) // TODO(slan): Wrap these into a RenderFrame specific ProvisionFetcher impl. - net::URLRequestContextGetter* context_getter = - BrowserContext::GetDefaultStoragePartition( - render_frame_host_->GetProcess()->GetBrowserContext()) - ->GetURLRequestContext(); provider->registry()->AddInterface(base::BindRepeating( - &ProvisionFetcherImpl::Create, base::RetainedRef(context_getter))); + &ProvisionFetcherImpl::Create, + base::RetainedRef( + BrowserContext::GetDefaultStoragePartition( + render_frame_host_->GetProcess()->GetBrowserContext()) + ->GetURLLoaderFactoryForBrowserProcess()))); #if BUILDFLAG(ENABLE_LIBRARY_CDMS) // Only provide CdmStorageImpl when we have a valid |cdm_file_system_id|, @@ -386,8 +406,8 @@ void MediaInterfaceProxy::CreateMediaPlayerRenderer( base::BindRepeating(&MediaPlayerRenderer::InitiateScopedSurfaceRequest, base::Unretained(renderer.get())); - media::MojoRendererService::Create(std::move(renderer), surface_request_cb, - std::move(request)); + media::MojoRendererService::Create(nullptr, std::move(renderer), + surface_request_cb, std::move(request)); } #endif // defined(OS_ANDROID) diff --git a/chromium/content/browser/media/media_interface_proxy.h b/chromium/content/browser/media/media_interface_proxy.h index 08b7661a4a2..ce7e4dfb739 100644 --- a/chromium/content/browser/media/media_interface_proxy.h +++ b/chromium/content/browser/media/media_interface_proxy.h @@ -51,6 +51,8 @@ class MediaInterfaceProxy : public media::mojom::InterfaceFactory { media::mojom::RendererRequest request) final; void CreateCdm(const std::string& key_system, media::mojom::ContentDecryptionModuleRequest request) final; + void CreateDecryptor(int cdm_id, + media::mojom::DecryptorRequest request) final; void CreateCdmProxy(const std::string& cdm_guid, media::mojom::CdmProxyRequest request) final; diff --git a/chromium/content/browser/media/media_internals.cc b/chromium/content/browser/media/media_internals.cc index 6e58bdce30f..e245a5feada 100644 --- a/chromium/content/browser/media/media_internals.cc +++ b/chromium/content/browser/media/media_internals.cc @@ -469,7 +469,8 @@ std::string MediaInternals::MediaInternalsUMAHandler::GetUMANameForAVStream( uma_name += "DDS."; } - if (player_info.video_decoder == media::GpuVideoDecoder::kDecoderName) { + if (player_info.video_decoder == media::GpuVideoDecoder::kDecoderName || + player_info.video_decoder == "MojoVideoDecoder") { uma_name += "HW"; } else { uma_name += "SW"; diff --git a/chromium/content/browser/media/media_internals_unittest.cc b/chromium/content/browser/media/media_internals_unittest.cc index 506227d219e..a02bcadfbf0 100644 --- a/chromium/content/browser/media/media_internals_unittest.cc +++ b/chromium/content/browser/media/media_internals_unittest.cc @@ -115,38 +115,6 @@ class MediaInternalsVideoCaptureDeviceTest : public testing::Test, MediaInternals::UpdateCallback update_cb_; }; -// TODO(chfremer): Consider removing this. This test seem be -// a duplicate implementation of the functionality under test. -// https://crbug.com/630694 -#if defined(OS_WIN) || defined(OS_MACOSX) || defined(OS_LINUX) || \ - defined(OS_ANDROID) -TEST_F(MediaInternalsVideoCaptureDeviceTest, - AllCaptureApiTypesHaveProperStringRepresentation) { - using VideoCaptureApi = media::VideoCaptureApi; - std::map<VideoCaptureApi, std::string> api_to_string_map; - api_to_string_map[VideoCaptureApi::LINUX_V4L2_SINGLE_PLANE] = "V4L2 SPLANE"; - api_to_string_map[VideoCaptureApi::WIN_MEDIA_FOUNDATION] = "Media Foundation"; - api_to_string_map[VideoCaptureApi::WIN_MEDIA_FOUNDATION_SENSOR] = - "Media Foundation Sensor Camera"; - api_to_string_map[VideoCaptureApi::WIN_DIRECT_SHOW] = "Direct Show"; - api_to_string_map[VideoCaptureApi::MACOSX_AVFOUNDATION] = "AV Foundation"; - api_to_string_map[VideoCaptureApi::MACOSX_DECKLINK] = "DeckLink"; - api_to_string_map[VideoCaptureApi::ANDROID_API1] = "Camera API1"; - api_to_string_map[VideoCaptureApi::ANDROID_API2_LEGACY] = - "Camera API2 Legacy"; - api_to_string_map[VideoCaptureApi::ANDROID_API2_FULL] = "Camera API2 Full"; - api_to_string_map[VideoCaptureApi::ANDROID_API2_LIMITED] = - "Camera API2 Limited"; - EXPECT_EQ(static_cast<size_t>(VideoCaptureApi::UNKNOWN), - api_to_string_map.size()); - for (const auto& map_entry : api_to_string_map) { - media::VideoCaptureDeviceDescriptor descriptor; - descriptor.capture_api = map_entry.first; - EXPECT_EQ(map_entry.second, descriptor.GetCaptureApiTypeString()); - } -} -#endif - TEST_F(MediaInternalsVideoCaptureDeviceTest, VideoCaptureFormatStringIsInExpectedFormat) { // Since media internals will send video capture capabilities to JavaScript in @@ -157,14 +125,11 @@ TEST_F(MediaInternalsVideoCaptureDeviceTest, const float kFrameRate = 30.0f; const gfx::Size kFrameSize(1280, 720); const media::VideoPixelFormat kPixelFormat = media::PIXEL_FORMAT_I420; - const media::VideoPixelStorage kPixelStorage = media::VideoPixelStorage::CPU; const media::VideoCaptureFormat capture_format(kFrameSize, kFrameRate, - kPixelFormat, kPixelStorage); + kPixelFormat); const std::string expected_string = base::StringPrintf( - "(%s)@%.3ffps, pixel format: %s, storage: %s", - kFrameSize.ToString().c_str(), kFrameRate, - media::VideoPixelFormatToString(kPixelFormat).c_str(), - media::VideoCaptureFormat::PixelStorageToString(kPixelStorage).c_str()); + "(%s)@%.3ffps, pixel format: %s", kFrameSize.ToString().c_str(), + kFrameRate, media::VideoPixelFormatToString(kPixelFormat).c_str()); EXPECT_EQ(expected_string, media::VideoCaptureFormat::ToString(capture_format)); } @@ -251,7 +216,7 @@ class MediaInternalsAudioLogTest private: static media::AudioParameters MakeAudioParams() { media::AudioParameters params(media::AudioParameters::AUDIO_PCM_LINEAR, - media::CHANNEL_LAYOUT_MONO, 48000, 16, 128); + media::CHANNEL_LAYOUT_MONO, 48000, 128); params.set_effects(media::AudioParameters::ECHO_CANCELLER | media::AudioParameters::DUCKING); return params; diff --git a/chromium/content/browser/media/media_source_browsertest.cc b/chromium/content/browser/media/media_source_browsertest.cc index e5571a3b16c..5e3cb0006e5 100644 --- a/chromium/content/browser/media/media_source_browsertest.cc +++ b/chromium/content/browser/media/media_source_browsertest.cc @@ -24,14 +24,12 @@ const char kWebMOpusAudioOnly[] = "audio/webm; codecs=\"opus\""; #endif const char kWebMVideoOnly[] = "video/webm; codecs=\"vp8\""; const char kWebMAudioVideo[] = "video/webm; codecs=\"vorbis, vp8\""; - -#if BUILDFLAG(USE_PROPRIETARY_CODECS) const char kMp4FlacAudioOnly[] = "audio/mp4; codecs=\"flac\""; -#if BUILDFLAG(ENABLE_MSE_MPEG2TS_STREAM_PARSER) +#if BUILDFLAG(USE_PROPRIETARY_CODECS) && \ + BUILDFLAG(ENABLE_MSE_MPEG2TS_STREAM_PARSER) const char kMp2tAudioVideo[] = "video/mp2t; codecs=\"mp4a.40.2, avc1.42E01E\""; #endif -#endif namespace content { @@ -51,7 +49,6 @@ class MediaSourceTest : public content::MediaBrowserTest { command_line->AppendSwitchASCII( switches::kAutoplayPolicy, switches::autoplay::kNoUserGestureRequiredPolicy); - scoped_feature_list_.InitAndDisableFeature(media::kMseFlacInIsobmff); } protected: @@ -121,12 +118,11 @@ IN_PROC_BROWSER_TEST_F(MediaSourceTest, Playback_Video_WEBM_Audio_MP4) { true); } -IN_PROC_BROWSER_TEST_F(MediaSourceTest, - Playback_AudioOnly_FLAC_MP4_Unsupported) { - // The feature is disabled by test setup, so verify playback failure. - TestSimplePlayback("bear-flac_frag.mp4", kMp4FlacAudioOnly, media::kFailed); +#endif // BUILDFLAG(USE_PROPRIETARY_CODECS) + +IN_PROC_BROWSER_TEST_F(MediaSourceTest, Playback_AudioOnly_FLAC_MP4) { + TestSimplePlayback("bear-flac_frag.mp4", kMp4FlacAudioOnly, media::kEnded); } -#endif #if BUILDFLAG(USE_PROPRIETARY_CODECS) #if BUILDFLAG(ENABLE_MSE_MPEG2TS_STREAM_PARSER) @@ -136,24 +132,4 @@ IN_PROC_BROWSER_TEST_F(MediaSourceTest, Playback_AudioVideo_Mp2t) { #endif #endif -#if BUILDFLAG(USE_PROPRIETARY_CODECS) -class MediaSourceFlacInIsobmffTest : public content::MediaSourceTest { - public: - void SetUpCommandLine(base::CommandLine* command_line) override { - command_line->AppendSwitchASCII( - switches::kAutoplayPolicy, - switches::autoplay::kNoUserGestureRequiredPolicy); - - // Enable MSE FLAC-in-MP4 feature. - scoped_feature_list_.InitAndEnableFeature(media::kMseFlacInIsobmff); - } -}; - -IN_PROC_BROWSER_TEST_F(MediaSourceFlacInIsobmffTest, - Playback_AudioOnly_FLAC_MP4_Supported) { - // The feature is enabled by test setup, so verify playback success. - TestSimplePlayback("bear-flac_frag.mp4", kMp4FlacAudioOnly, media::kEnded); -} -#endif - } // namespace content diff --git a/chromium/content/browser/media/media_web_contents_observer.cc b/chromium/content/browser/media/media_web_contents_observer.cc index 065f2f24773..b3abe26c982 100644 --- a/chromium/content/browser/media/media_web_contents_observer.cc +++ b/chromium/content/browser/media/media_web_contents_observer.cc @@ -9,6 +9,7 @@ #include "build/build_config.h" #include "content/browser/media/audible_metrics.h" #include "content/browser/media/audio_stream_monitor.h" +#include "content/browser/picture_in_picture/picture_in_picture_window_controller_impl.h" #include "content/browser/web_contents/web_contents_impl.h" #include "content/common/media/media_player_delegate_messages.h" #include "content/public/browser/render_frame_host.h" @@ -134,10 +135,13 @@ bool MediaWebContentsObserver::OnMessageReceived( IPC_MESSAGE_HANDLER(MediaPlayerDelegateHostMsg_OnMediaSizeChanged, OnMediaSizeChanged) IPC_MESSAGE_HANDLER( - MediaPlayerDelegateHostMsg_OnPictureInPictureSourceChanged, - OnPictureInPictureSourceChanged) + MediaPlayerDelegateHostMsg_OnPictureInPictureModeStarted, + OnPictureInPictureModeStarted) IPC_MESSAGE_HANDLER(MediaPlayerDelegateHostMsg_OnPictureInPictureModeEnded, OnPictureInPictureModeEnded) + IPC_MESSAGE_HANDLER( + MediaPlayerDelegateHostMsg_OnPictureInPictureSurfaceChanged, + OnPictureInPictureSurfaceChanged) IPC_MESSAGE_UNHANDLED(handled = false) IPC_END_MESSAGE_MAP() return handled; @@ -145,23 +149,7 @@ bool MediaWebContentsObserver::OnMessageReceived( void MediaWebContentsObserver::OnVisibilityChanged( content::Visibility visibility) { - if (visibility == content::Visibility::HIDDEN) { - // If there are entities capturing screenshots or video (e.g., mirroring), - // don't release the wake lock. - if (!web_contents()->IsBeingCaptured()) { - GetVideoWakeLock()->CancelWakeLock(); - has_video_wake_lock_for_testing_ = false; - } - } else { - // TODO(ke.he@intel.com): Determine whether a tab should be allowed to - // request the wake lock when it's occluded. - DCHECK(visibility == content::Visibility::VISIBLE || - visibility == content::Visibility::OCCLUDED); - - // Restore wake lock if there are active video players running. - if (!active_video_players_.empty()) - LockVideo(); - } + UpdateVideoLock(); } void MediaWebContentsObserver::RequestPersistentVideo(bool value) { @@ -184,15 +172,20 @@ bool MediaWebContentsObserver::IsPlayerActive( return MediaPlayerEntryExists(player_id, active_audio_players_); } +void MediaWebContentsObserver::OnPictureInPictureWindowResize( + const gfx::Size& window_size) { + DCHECK(pip_player_.has_value()); + + RenderFrameHost* frame = pip_player_->first; + int delegate_id = pip_player_->second; + frame->Send(new MediaPlayerDelegateMsg_OnPictureInPictureWindowResize( + frame->GetRoutingID(), delegate_id, window_size)); +} + void MediaWebContentsObserver::OnMediaDestroyed( RenderFrameHost* render_frame_host, int delegate_id) { OnMediaPaused(render_frame_host, delegate_id, true); - - if (pip_player_ && - pip_player_ == MediaPlayerId(render_frame_host, delegate_id)) { - pip_player_.reset(); - } } void MediaWebContentsObserver::OnMediaPaused(RenderFrameHost* render_frame_host, @@ -203,7 +196,8 @@ void MediaWebContentsObserver::OnMediaPaused(RenderFrameHost* render_frame_host, RemoveMediaPlayerEntry(player_id, &active_audio_players_); const bool removed_video = RemoveMediaPlayerEntry(player_id, &active_video_players_); - MaybeCancelVideoLock(); + + UpdateVideoLock(); if (removed_audio || removed_video) { // Notify observers the player has been "paused". @@ -241,9 +235,7 @@ void MediaWebContentsObserver::OnMediaPlaying( if (has_video) { AddMediaPlayerEntry(id, &active_video_players_); - // If we're not hidden and have just created a player, create a wakelock. - if (!web_contents_impl()->IsHidden()) - LockVideo(); + UpdateVideoLock(); } if (!session_controllers_manager_.RequestPlay( @@ -296,16 +288,61 @@ void MediaWebContentsObserver::OnMediaSizeChanged( web_contents_impl()->MediaResized(size, id); } -void MediaWebContentsObserver::OnPictureInPictureSourceChanged( +void MediaWebContentsObserver::OnPictureInPictureModeStarted( RenderFrameHost* render_frame_host, - int delegate_id) { + int delegate_id, + const viz::SurfaceId& surface_id, + const gfx::Size& natural_size, + int request_id) { + DCHECK(surface_id.is_valid()); pip_player_ = MediaPlayerId(render_frame_host, delegate_id); + + UpdateVideoLock(); + + gfx::Size window_size = + web_contents_impl()->EnterPictureInPicture(surface_id, natural_size); + + render_frame_host->Send( + new MediaPlayerDelegateMsg_OnPictureInPictureModeStarted_ACK( + render_frame_host->GetRoutingID(), delegate_id, request_id, + window_size)); } void MediaWebContentsObserver::OnPictureInPictureModeEnded( RenderFrameHost* render_frame_host, - int delegate_id) { - pip_player_.reset(); + int delegate_id, + int request_id) { + // TODO(mlamouri): must be a DCHECK but can't at the moment because we do not + // correctly notify players when switching PIP video in the same tab. + if (pip_player_) { + web_contents_impl()->ExitPictureInPicture(); + + // Reset must happen after notifying the WebContents because it may interact + // with it. + pip_player_.reset(); + + UpdateVideoLock(); + } + + render_frame_host->Send( + new MediaPlayerDelegateMsg_OnPictureInPictureModeEnded_ACK( + render_frame_host->GetRoutingID(), delegate_id, request_id)); +} + +void MediaWebContentsObserver::OnPictureInPictureSurfaceChanged( + RenderFrameHost* render_frame_host, + int delegate_id, + const viz::SurfaceId& surface_id, + const gfx::Size& natural_size) { + DCHECK(surface_id.is_valid()); + DCHECK(pip_player_); + + PictureInPictureWindowControllerImpl* pip_controller = + PictureInPictureWindowControllerImpl::FromWebContents( + web_contents_impl()); + DCHECK(pip_controller); + + pip_controller->EmbedSurface(surface_id, natural_size); } void MediaWebContentsObserver::ClearWakeLocks( @@ -322,7 +359,7 @@ void MediaWebContentsObserver::ClearWakeLocks( audio_players.begin(), audio_players.end(), std::inserter(removed_players, removed_players.end())); - MaybeCancelVideoLock(); + UpdateVideoLock(); // Notify all observers the player has been "paused". for (const auto& id : removed_players) { @@ -379,21 +416,25 @@ void MediaWebContentsObserver::CancelAudioLock() { has_audio_wake_lock_for_testing_ = false; } -void MediaWebContentsObserver::LockVideo() { - DCHECK(!active_video_players_.empty()); - GetVideoWakeLock()->RequestWakeLock(); - has_video_wake_lock_for_testing_ = true; -} +void MediaWebContentsObserver::UpdateVideoLock() { + if (active_video_players_.empty() || + (web_contents()->GetVisibility() == Visibility::HIDDEN && + !web_contents()->IsBeingCaptured() && !pip_player_.has_value())) { + // Need to release a wake lock if one is held. + if (!has_video_wake_lock_) + return; -void MediaWebContentsObserver::CancelVideoLock() { - GetVideoWakeLock()->CancelWakeLock(); - has_video_wake_lock_for_testing_ = false; -} + GetVideoWakeLock()->CancelWakeLock(); + has_video_wake_lock_ = false; + return; + } + + // Need to take a wake lock if not already done. + if (has_video_wake_lock_) + return; -void MediaWebContentsObserver::MaybeCancelVideoLock() { - // If there are no more video players, cancel the video wake lock. - if (active_video_players_.empty()) - CancelVideoLock(); + GetVideoWakeLock()->RequestWakeLock(); + has_video_wake_lock_ = true; } void MediaWebContentsObserver::OnMediaMutedStatusChanged( diff --git a/chromium/content/browser/media/media_web_contents_observer.h b/chromium/content/browser/media/media_web_contents_observer.h index 7b9f1f1d399..2655629a1da 100644 --- a/chromium/content/browser/media/media_web_contents_observer.h +++ b/chromium/content/browser/media/media_web_contents_observer.h @@ -33,6 +33,10 @@ namespace gfx { class Size; } // namespace size +namespace viz { +class SurfaceId; +} // namespace viz + namespace content { // This class manages all RenderFrame based media related managers at the @@ -85,13 +89,16 @@ class CONTENT_EXPORT MediaWebContentsObserver : public WebContentsObserver { // Returns whether or not the given player id is active. bool IsPlayerActive(const MediaPlayerId& player_id) const; + // Called by the Picture-in-Picture controller when the associated window is + // resized. |window_size| represents the new size of the window. It MUST be + // called when there is a player in Picture-in-Picture. + void OnPictureInPictureWindowResize(const gfx::Size& window_size); + bool has_audio_wake_lock_for_testing() const { return has_audio_wake_lock_for_testing_; } - bool has_video_wake_lock_for_testing() const { - return has_video_wake_lock_for_testing_; - } + bool has_video_wake_lock_for_testing() const { return has_video_wake_lock_; } protected: MediaSessionControllersManager* session_controllers_manager() { @@ -119,10 +126,18 @@ class CONTENT_EXPORT MediaWebContentsObserver : public WebContentsObserver { void OnMediaMutedStatusChanged(RenderFrameHost* render_frame_host, int delegate_id, bool muted); - void OnPictureInPictureSourceChanged(RenderFrameHost* render_frame_host, - int delegate_id); + void OnPictureInPictureModeStarted(RenderFrameHost* render_frame_host, + int delegate_id, + const viz::SurfaceId&, + const gfx::Size& natural_size, + int request_id); void OnPictureInPictureModeEnded(RenderFrameHost* render_frame_host, - int delegate_id); + int delegate_id, + int request_id); + void OnPictureInPictureSurfaceChanged(RenderFrameHost*, + int delegate_id, + const viz::SurfaceId&, + const gfx::Size&); // Clear |render_frame_host|'s tracking entry for its WakeLocks. void ClearWakeLocks(RenderFrameHost* render_frame_host); @@ -130,12 +145,10 @@ class CONTENT_EXPORT MediaWebContentsObserver : public WebContentsObserver { device::mojom::WakeLock* GetAudioWakeLock(); device::mojom::WakeLock* GetVideoWakeLock(); + // WakeLock related methods for audio and video. void LockAudio(); - void LockVideo(); - void CancelAudioLock(); - void CancelVideoLock(); - void MaybeCancelVideoLock(); + void UpdateVideoLock(); // Helper methods for adding or removing player entries in |player_map|. void AddMediaPlayerEntry(const MediaPlayerId& id, @@ -161,7 +174,7 @@ class CONTENT_EXPORT MediaWebContentsObserver : public WebContentsObserver { base::Optional<MediaPlayerId> pip_player_; base::Optional<bool> picture_in_picture_allowed_in_fullscreen_; bool has_audio_wake_lock_for_testing_ = false; - bool has_video_wake_lock_for_testing_ = false; + bool has_video_wake_lock_ = false; MediaSessionControllersManager session_controllers_manager_; diff --git a/chromium/content/browser/media/session/audio_focus_manager_unittest.cc b/chromium/content/browser/media/session/audio_focus_manager_unittest.cc index 46d2c1ce196..459ce5089f4 100644 --- a/chromium/content/browser/media/session/audio_focus_manager_unittest.cc +++ b/chromium/content/browser/media/session/audio_focus_manager_unittest.cc @@ -4,6 +4,8 @@ #include "content/browser/media/session/audio_focus_manager.h" +#include <memory> + #include "base/command_line.h" #include "base/run_loop.h" #include "content/browser/media/session/media_session_impl.h" @@ -43,7 +45,8 @@ class AudioFocusManagerTest : public testing::Test { base::CommandLine::ForCurrentProcess()->AppendSwitch( switches::kEnableAudioFocus); rph_factory_.reset(new MockRenderProcessHostFactory()); - RenderProcessHostImpl::set_render_process_host_factory(rph_factory_.get()); + RenderProcessHostImpl::set_render_process_host_factory_for_testing( + rph_factory_.get()); browser_context_.reset(new TestBrowserContext()); pepper_observer_.reset(new MockMediaSessionPlayerObserver()); } @@ -53,7 +56,7 @@ class AudioFocusManagerTest : public testing::Test { base::RunLoop().RunUntilIdle(); browser_context_.reset(); - RenderProcessHostImpl::set_render_process_host_factory(nullptr); + RenderProcessHostImpl::set_render_process_host_factory_for_testing(nullptr); rph_factory_.reset(); } @@ -99,7 +102,7 @@ class AudioFocusManagerTest : public testing::Test { session->AbandonSystemAudioFocusIfNeeded(); } - WebContents* CreateWebContents() { + std::unique_ptr<WebContents> CreateWebContents() { return TestWebContents::Create(browser_context_.get(), SiteInstance::SiteInstance::Create(browser_context_.get())); } diff --git a/chromium/content/browser/media/session/media_session_android.cc b/chromium/content/browser/media/session/media_session_android.cc index b9733ddb545..cd30af13b23 100644 --- a/chromium/content/browser/media/session/media_session_android.cc +++ b/chromium/content/browser/media/session/media_session_android.cc @@ -162,6 +162,14 @@ void MediaSessionAndroid::DidReceiveAction(JNIEnv* env, static_cast<blink::mojom::MediaSessionAction>(action)); } +void MediaSessionAndroid::RequestSystemAudioFocus( + JNIEnv* env, + const base::android::JavaParamRef<jobject>& j_obj) { + DCHECK(media_session()); + static_cast<MediaSessionImpl*>(media_session()) + ->RequestSystemAudioFocus(AudioFocusManager::AudioFocusType::Gain); +} + WebContentsAndroid* MediaSessionAndroid::GetWebContentsAndroid() { MediaSessionImpl* session = static_cast<MediaSessionImpl*>(media_session()); if (!session) diff --git a/chromium/content/browser/media/session/media_session_android.h b/chromium/content/browser/media/session/media_session_android.h index 3d3ea8ea58d..3a95d72f0cd 100644 --- a/chromium/content/browser/media/session/media_session_android.h +++ b/chromium/content/browser/media/session/media_session_android.h @@ -53,6 +53,9 @@ class MediaSessionAndroid final : public MediaSessionObserver { void DidReceiveAction(JNIEnv* env, const base::android::JavaParamRef<jobject>& j_obj, jint action); + void RequestSystemAudioFocus( + JNIEnv* env, + const base::android::JavaParamRef<jobject>& j_obj); private: WebContentsAndroid* GetWebContentsAndroid(); diff --git a/chromium/content/browser/media/session/media_session_impl.cc b/chromium/content/browser/media/session/media_session_impl.cc index 311371a2770..a66c5d02e07 100644 --- a/chromium/content/browser/media/session/media_session_impl.cc +++ b/chromium/content/browser/media/session/media_session_impl.cc @@ -579,6 +579,10 @@ bool MediaSessionImpl::RequestSystemAudioFocus( bool result = delegate_->RequestAudioFocus(audio_focus_type); uma_helper_.RecordRequestAudioFocusResult(result); + // Make sure we are unducked. + if (result) + StopDucking(); + // MediaSessionImpl must change its state & audio focus type AFTER requesting // audio focus. SetAudioFocusState(result ? State::ACTIVE : State::INACTIVE); diff --git a/chromium/content/browser/media/session/media_session_impl.h b/chromium/content/browser/media/session/media_session_impl.h index 3a681b8b3bc..637acb6dfcf 100644 --- a/chromium/content/browser/media/session/media_session_impl.h +++ b/chromium/content/browser/media/session/media_session_impl.h @@ -199,6 +199,11 @@ class MediaSessionImpl : public MediaSession, // to blink::MediaSession corresponding to the current routed service. void DidReceiveAction(blink::mojom::MediaSessionAction action) override; + // Requests audio focus to the AudioFocusDelegate. + // Returns whether the request was granted. + CONTENT_EXPORT bool RequestSystemAudioFocus( + AudioFocusManager::AudioFocusType audio_focus_type); + private: friend class content::WebContentsUserData<MediaSessionImpl>; friend class ::MediaSessionImplBrowserTest; @@ -240,11 +245,6 @@ class MediaSessionImpl : public MediaSession, State new_state); CONTENT_EXPORT void OnResumeInternal(MediaSession::SuspendType suspend_type); - // Requests audio focus to the AudioFocusDelegate. - // Returns whether the request was granted. - CONTENT_EXPORT bool RequestSystemAudioFocus( - AudioFocusManager::AudioFocusType audio_focus_type); - // To be called after a call to AbandonAudioFocus() in order request the // delegate to abandon the audio focus. CONTENT_EXPORT void AbandonSystemAudioFocusIfNeeded(); diff --git a/chromium/content/browser/media/session/media_session_impl_browsertest.cc b/chromium/content/browser/media/session/media_session_impl_browsertest.cc index df48df6df53..28fdcebf58d 100644 --- a/chromium/content/browser/media/session/media_session_impl_browsertest.cc +++ b/chromium/content/browser/media/session/media_session_impl_browsertest.cc @@ -336,6 +336,34 @@ IN_PROC_BROWSER_TEST_F(MediaSessionImplBrowserTest, AudioFocusInitialState) { EXPECT_FALSE(IsActive()); } +IN_PROC_BROWSER_TEST_F(MediaSessionImplBrowserTest, + AddPlayerOnSuspendedFocusUnducks) { + auto player_observer = std::make_unique<MockMediaSessionPlayerObserver>(); + StartNewPlayer(player_observer.get(), media::MediaContentType::Persistent); + + UISuspend(); + EXPECT_FALSE(IsActive()); + + SystemStartDucking(); + EXPECT_EQ(kDuckingVolumeMultiplier, player_observer->GetVolumeMultiplier(0)); + + EXPECT_TRUE( + AddPlayer(player_observer.get(), 0, media::MediaContentType::Persistent)); + EXPECT_EQ(kDefaultVolumeMultiplier, player_observer->GetVolumeMultiplier(0)); +} + +IN_PROC_BROWSER_TEST_F(MediaSessionImplBrowserTest, + CanRequestFocusBeforePlayerCreation) { + auto player_observer = std::make_unique<MockMediaSessionPlayerObserver>(); + + media_session_->RequestSystemAudioFocus( + content::AudioFocusManager::AudioFocusType::Gain); + EXPECT_TRUE(IsActive()); + + StartNewPlayer(player_observer.get(), media::MediaContentType::Persistent); + EXPECT_TRUE(IsActive()); +} + IN_PROC_BROWSER_TEST_F(MediaSessionImplBrowserTest, StartPlayerGivesFocus) { auto player_observer = std::make_unique<MockMediaSessionPlayerObserver>(); diff --git a/chromium/content/browser/media/session/mock_media_session_observer.h b/chromium/content/browser/media/session/mock_media_session_observer.h index 6421fa9b4e6..069fdc9a296 100644 --- a/chromium/content/browser/media/session/mock_media_session_observer.h +++ b/chromium/content/browser/media/session/mock_media_session_observer.h @@ -13,7 +13,7 @@ namespace content { class MockMediaSessionObserver : public MediaSessionObserver { public: MockMediaSessionObserver(MediaSession* media_session); - virtual ~MockMediaSessionObserver() override; + ~MockMediaSessionObserver() override; MOCK_METHOD0(MediaSessionDestroyed, void()); MOCK_METHOD2(MediaSessionStateChanged, diff --git a/chromium/content/browser/media/url_provision_fetcher.cc b/chromium/content/browser/media/url_provision_fetcher.cc index e6f57a7964b..4b77be7c34d 100644 --- a/chromium/content/browser/media/url_provision_fetcher.cc +++ b/chromium/content/browser/media/url_provision_fetcher.cc @@ -8,17 +8,19 @@ #include "media/base/bind_to_current_loop.h" #include "net/base/load_flags.h" #include "net/traffic_annotation/network_traffic_annotation.h" -#include "net/url_request/url_fetcher.h" - -using net::URLFetcher; +#include "services/network/public/cpp/resource_request.h" +#include "services/network/public/cpp/shared_url_loader_factory.h" +#include "services/network/public/cpp/simple_url_loader.h" namespace content { // Implementation of URLProvisionFetcher. URLProvisionFetcher::URLProvisionFetcher( - net::URLRequestContextGetter* context_getter) - : context_getter_(context_getter) {} + scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory) + : url_loader_factory_(std::move(url_loader_factory)) { + DCHECK(url_loader_factory_); +} URLProvisionFetcher::~URLProvisionFetcher() {} @@ -32,7 +34,7 @@ void URLProvisionFetcher::Retrieve( default_url + "&signedRequest=" + request_data; DVLOG(1) << __func__ << ": request:" << request_string; - DCHECK(!request_); + DCHECK(!simple_url_loader_); net::NetworkTrafficAnnotationTag traffic_annotation = net::DefineNetworkTrafficAnnotation("url_prevision_fetcher", R"( semantics { @@ -62,49 +64,48 @@ void URLProvisionFetcher::Retrieve( "Media Identifier permissions." policy_exception_justification: "Not implemented." })"); - request_ = URLFetcher::Create(GURL(request_string), URLFetcher::POST, this, - traffic_annotation); - - // SetUploadData is mandatory even if we are not uploading anything. - request_->SetUploadData("", ""); - request_->AddExtraRequestHeader("User-Agent: Widevine CDM v1.0"); - request_->AddExtraRequestHeader("Content-Type: application/json"); - request_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES | - net::LOAD_DO_NOT_SEND_COOKIES); - - DCHECK(context_getter_); - request_->SetRequestContext(context_getter_); - - request_->Start(); + auto resource_request = std::make_unique<network::ResourceRequest>(); + resource_request->url = GURL(request_string); + resource_request->load_flags = + net::LOAD_DO_NOT_SAVE_COOKIES | net::LOAD_DO_NOT_SEND_COOKIES; + resource_request->method = "POST"; + resource_request->headers.SetHeader("User-Agent", "Widevine CDM v1.0"); + simple_url_loader_ = network::SimpleURLLoader::Create( + std::move(resource_request), traffic_annotation); + simple_url_loader_->AttachStringForUpload("", "application/json"); + simple_url_loader_->DownloadToStringOfUnboundedSizeUntilCrashAndDie( + url_loader_factory_.get(), + base::BindOnce(&URLProvisionFetcher::OnSimpleLoaderComplete, + base::Unretained(this))); } -void URLProvisionFetcher::OnURLFetchComplete(const net::URLFetcher* source) { - DCHECK(source); - - int response_code = source->GetResponseCode(); - DVLOG(1) << __func__ << ": response code:" << source->GetResponseCode(); - - std::string response; +void URLProvisionFetcher::OnSimpleLoaderComplete( + std::unique_ptr<std::string> response_body) { bool success = false; - if (response_code == 200) { - success = source->GetResponseAsString(&response); - DVLOG_IF(1, !success) << __func__ << ": GetResponseAsString() failed"; + std::string response; + if (response_body) { + success = true; + response = std::move(*response_body); } else { + int response_code = -1; + if (simple_url_loader_->ResponseInfo() && + simple_url_loader_->ResponseInfo()->headers) { + response_code = + simple_url_loader_->ResponseInfo()->headers->response_code(); + } DVLOG(1) << "CDM provision: server returned error code " << response_code; } - request_.reset(); - // URLFetcher implementation calls OnURLFetchComplete() on the same thread - // that called Start() and it does this asynchronously. + simple_url_loader_.reset(); response_cb_.Run(success, response); } // Implementation of content public method CreateProvisionFetcher(). std::unique_ptr<media::ProvisionFetcher> CreateProvisionFetcher( - net::URLRequestContextGetter* context_getter) { - DCHECK(context_getter); - return std::make_unique<URLProvisionFetcher>(context_getter); + scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory) { + DCHECK(url_loader_factory); + return std::make_unique<URLProvisionFetcher>(std::move(url_loader_factory)); } } // namespace content diff --git a/chromium/content/browser/media/url_provision_fetcher.h b/chromium/content/browser/media/url_provision_fetcher.h index 02a594b7347..053e2c09a65 100644 --- a/chromium/content/browser/media/url_provision_fetcher.h +++ b/chromium/content/browser/media/url_provision_fetcher.h @@ -7,17 +7,20 @@ #include "base/macros.h" #include "media/base/provision_fetcher.h" -#include "net/url_request/url_fetcher.h" -#include "net/url_request/url_fetcher_delegate.h" + +namespace network { +class SharedURLLoaderFactory; +class SimpleURLLoader; +} // namespace network namespace content { // The ProvisionFetcher that retrieves the data by HTTP POST request. -class URLProvisionFetcher : public media::ProvisionFetcher, - public net::URLFetcherDelegate { +class URLProvisionFetcher : public media::ProvisionFetcher { public: - explicit URLProvisionFetcher(net::URLRequestContextGetter* context_getter); + explicit URLProvisionFetcher( + scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory); ~URLProvisionFetcher() override; // media::ProvisionFetcher implementation. @@ -26,11 +29,10 @@ class URLProvisionFetcher : public media::ProvisionFetcher, const ProvisionFetcher::ResponseCB& response_cb) override; private: - // net::URLFetcherDelegate implementation. - void OnURLFetchComplete(const net::URLFetcher* source) override; + void OnSimpleLoaderComplete(std::unique_ptr<std::string> response_body); - net::URLRequestContextGetter* context_getter_; - std::unique_ptr<net::URLFetcher> request_; + scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory_; + std::unique_ptr<network::SimpleURLLoader> simple_url_loader_; media::ProvisionFetcher::ResponseCB response_cb_; DISALLOW_COPY_AND_ASSIGN(URLProvisionFetcher); |