diff options
Diffstat (limited to 'chromium/third_party/blink/renderer/modules/imagecapture')
7 files changed, 433 insertions, 77 deletions
diff --git a/chromium/third_party/blink/renderer/modules/imagecapture/BUILD.gn b/chromium/third_party/blink/renderer/modules/imagecapture/BUILD.gn index 58b71a57fa8..76602d87c9f 100644 --- a/chromium/third_party/blink/renderer/modules/imagecapture/BUILD.gn +++ b/chromium/third_party/blink/renderer/modules/imagecapture/BUILD.gn @@ -8,12 +8,17 @@ blink_modules_sources("imagecapture") { sources = [ "image_capture.cc", "image_capture.h", + "image_capture_frame_grabber.cc", + "image_capture_frame_grabber.h", "media_settings_range.h", "photo_capabilities.cc", "photo_capabilities.h", ] deps = [ + "//media", "//media/capture/mojom:image_capture_blink", + "//skia", + "//third_party/libyuv", ] } diff --git a/chromium/third_party/blink/renderer/modules/imagecapture/DEPS b/chromium/third_party/blink/renderer/modules/imagecapture/DEPS index 1307c6ddf7f..8328eae6ced 100644 --- a/chromium/third_party/blink/renderer/modules/imagecapture/DEPS +++ b/chromium/third_party/blink/renderer/modules/imagecapture/DEPS @@ -1,3 +1,8 @@ include_rules = [ "+media/capture/mojom/image_capture.mojom-blink.h", + + "+media/base", + "+skia/ext/platform_canvas.h", + "+third_party/libyuv", + "+third_party/skia/include/core", ] diff --git a/chromium/third_party/blink/renderer/modules/imagecapture/OWNERS b/chromium/third_party/blink/renderer/modules/imagecapture/OWNERS index 1ac08b05517..ab8e7bac0ca 100644 --- a/chromium/third_party/blink/renderer/modules/imagecapture/OWNERS +++ b/chromium/third_party/blink/renderer/modules/imagecapture/OWNERS @@ -1,5 +1,7 @@ -mcasas@chromium.org reillyg@chromium.org +# Original (legacy) owner. +mcasas@chromium.org + # COMPONENT: Blink>ImageCapture # TEAM: webrtc-dev@chromium.org diff --git a/chromium/third_party/blink/renderer/modules/imagecapture/image_capture.cc b/chromium/third_party/blink/renderer/modules/imagecapture/image_capture.cc index 122e66ff4ff..60562ee7bac 100644 --- a/chromium/third_party/blink/renderer/modules/imagecapture/image_capture.cc +++ b/chromium/third_party/blink/renderer/modules/imagecapture/image_capture.cc @@ -4,12 +4,12 @@ #include "third_party/blink/renderer/modules/imagecapture/image_capture.h" +#include <memory> #include <utility> #include "services/service_manager/public/cpp/interface_provider.h" #include "third_party/blink/public/platform/interface_provider.h" #include "third_party/blink/public/platform/platform.h" -#include "third_party/blink/public/platform/web_image_capture_frame_grabber.h" #include "third_party/blink/public/platform/web_media_stream_track.h" #include "third_party/blink/renderer/bindings/core/v8/callback_promise_adapter.h" #include "third_party/blink/renderer/bindings/core/v8/script_promise_resolver.h" @@ -18,11 +18,13 @@ #include "third_party/blink/renderer/core/frame/local_frame.h" #include "third_party/blink/renderer/core/imagebitmap/image_bitmap.h" #include "third_party/blink/renderer/modules/event_target_modules.h" +#include "third_party/blink/renderer/modules/imagecapture/image_capture_frame_grabber.h" #include "third_party/blink/renderer/modules/imagecapture/media_settings_range.h" #include "third_party/blink/renderer/modules/imagecapture/photo_capabilities.h" #include "third_party/blink/renderer/modules/mediastream/media_stream_track.h" #include "third_party/blink/renderer/modules/mediastream/media_track_capabilities.h" #include "third_party/blink/renderer/modules/mediastream/media_track_constraints.h" +#include "third_party/blink/renderer/platform/heap/heap.h" #include "third_party/blink/renderer/platform/mojo/mojo_helper.h" #include "third_party/blink/renderer/platform/wtf/functional.h" @@ -126,8 +128,8 @@ ScriptPromise ImageCapture::getPhotoCapabilities(ScriptState* script_state) { ScriptPromise promise = resolver->Promise(); if (!service_) { - resolver->Reject(DOMException::Create(DOMExceptionCode::kNotFoundError, - kNoServiceError)); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotFoundError, kNoServiceError)); return promise; } service_requests_.insert(resolver); @@ -152,8 +154,8 @@ ScriptPromise ImageCapture::getPhotoSettings(ScriptState* script_state) { ScriptPromise promise = resolver->Promise(); if (!service_) { - resolver->Reject(DOMException::Create(DOMExceptionCode::kNotFoundError, - kNoServiceError)); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotFoundError, kNoServiceError)); return promise; } service_requests_.insert(resolver); @@ -182,15 +184,15 @@ ScriptPromise ImageCapture::setOptions(ScriptState* script_state, ScriptPromise promise = resolver->Promise(); if (TrackIsInactive(*stream_track_)) { - resolver->Reject( - DOMException::Create(DOMExceptionCode::kInvalidStateError, - "The associated Track is in an invalid state.")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kInvalidStateError, + "The associated Track is in an invalid state.")); return promise; } if (!service_) { - resolver->Reject(DOMException::Create(DOMExceptionCode::kNotFoundError, - kNoServiceError)); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotFoundError, kNoServiceError)); return promise; } service_requests_.insert(resolver); @@ -204,9 +206,9 @@ ScriptPromise ImageCapture::setOptions(ScriptState* script_state, if (photo_capabilities_ && (height < photo_capabilities_->imageHeight()->min() || height > photo_capabilities_->imageHeight()->max())) { - resolver->Reject( - DOMException::Create(DOMExceptionCode::kNotSupportedError, - "imageHeight setting out of range")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotSupportedError, + "imageHeight setting out of range")); return promise; } settings->height = height; @@ -217,9 +219,9 @@ ScriptPromise ImageCapture::setOptions(ScriptState* script_state, if (photo_capabilities_ && (width < photo_capabilities_->imageWidth()->min() || width > photo_capabilities_->imageWidth()->max())) { - resolver->Reject( - DOMException::Create(DOMExceptionCode::kNotSupportedError, - "imageWidth setting out of range")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotSupportedError, + "imageWidth setting out of range")); return promise; } settings->width = width; @@ -229,9 +231,9 @@ ScriptPromise ImageCapture::setOptions(ScriptState* script_state, if (settings->has_red_eye_reduction) { if (photo_capabilities_ && !photo_capabilities_->IsRedEyeReductionControllable()) { - resolver->Reject( - DOMException::Create(DOMExceptionCode::kNotSupportedError, - "redEyeReduction is not controllable.")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotSupportedError, + "redEyeReduction is not controllable.")); return promise; } settings->red_eye_reduction = photo_settings->redEyeReduction(); @@ -242,7 +244,7 @@ ScriptPromise ImageCapture::setOptions(ScriptState* script_state, const String fill_light_mode = photo_settings->fillLightMode(); if (photo_capabilities_ && photo_capabilities_->fillLightMode().Find( fill_light_mode) == kNotFound) { - resolver->Reject(DOMException::Create( + resolver->Reject(MakeGarbageCollected<DOMException>( DOMExceptionCode::kNotSupportedError, "Unsupported fillLightMode")); return promise; } @@ -263,14 +265,14 @@ ScriptPromise ImageCapture::takePhoto(ScriptState* script_state) { ScriptPromise promise = resolver->Promise(); if (TrackIsInactive(*stream_track_)) { - resolver->Reject( - DOMException::Create(DOMExceptionCode::kInvalidStateError, - "The associated Track is in an invalid state.")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kInvalidStateError, + "The associated Track is in an invalid state.")); return promise; } if (!service_) { - resolver->Reject(DOMException::Create(DOMExceptionCode::kNotFoundError, - kNoServiceError)); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotFoundError, kNoServiceError)); return promise; } @@ -304,19 +306,19 @@ ScriptPromise ImageCapture::grabFrame(ScriptState* script_state) { ScriptPromise promise = resolver->Promise(); if (TrackIsInactive(*stream_track_)) { - resolver->Reject( - DOMException::Create(DOMExceptionCode::kInvalidStateError, - "The associated Track is in an invalid state.")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kInvalidStateError, + "The associated Track is in an invalid state.")); return promise; } // Create |m_frameGrabber| the first time. if (!frame_grabber_) { - frame_grabber_ = Platform::Current()->CreateImageCaptureFrameGrabber(); + frame_grabber_ = std::make_unique<ImageCaptureFrameGrabber>(); } if (!frame_grabber_) { - resolver->Reject(DOMException::Create( + resolver->Reject(MakeGarbageCollected<DOMException>( DOMExceptionCode::kUnknownError, "Couldn't create platform resources")); return promise; } @@ -343,8 +345,8 @@ void ImageCapture::SetMediaTrackConstraints( const HeapVector<Member<MediaTrackConstraintSet>>& constraints_vector) { DCHECK_GT(constraints_vector.size(), 0u); if (!service_) { - resolver->Reject(DOMException::Create(DOMExceptionCode::kNotFoundError, - kNoServiceError)); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotFoundError, kNoServiceError)); return; } // TODO(mcasas): add support more than one single advanced constraint. @@ -367,8 +369,8 @@ void ImageCapture::SetMediaTrackConstraints( (constraints->hasFocusDistance() && !capabilities_->hasFocusDistance()) || (constraints->hasZoom() && !capabilities_->hasZoom()) || (constraints->hasTorch() && !capabilities_->hasTorch())) { - resolver->Reject(DOMException::Create(DOMExceptionCode::kNotSupportedError, - "Unsupported constraint(s)")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotSupportedError, "Unsupported constraint(s)")); return; } @@ -384,9 +386,9 @@ void ImageCapture::SetMediaTrackConstraints( constraints->whiteBalanceMode().GetAsString(); if (capabilities_->whiteBalanceMode().Find(white_balance_mode) == kNotFound) { - resolver->Reject( - DOMException::Create(DOMExceptionCode::kNotSupportedError, - "Unsupported whiteBalanceMode.")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotSupportedError, + "Unsupported whiteBalanceMode.")); return; } temp_constraints->setWhiteBalanceMode(constraints->whiteBalanceMode()); @@ -397,7 +399,7 @@ void ImageCapture::SetMediaTrackConstraints( if (settings->has_exposure_mode) { const auto exposure_mode = constraints->exposureMode().GetAsString(); if (capabilities_->exposureMode().Find(exposure_mode) == kNotFound) { - resolver->Reject(DOMException::Create( + resolver->Reject(MakeGarbageCollected<DOMException>( DOMExceptionCode::kNotSupportedError, "Unsupported exposureMode.")); return; } @@ -410,7 +412,7 @@ void ImageCapture::SetMediaTrackConstraints( if (settings->has_focus_mode) { const auto focus_mode = constraints->focusMode().GetAsString(); if (capabilities_->focusMode().Find(focus_mode) == kNotFound) { - resolver->Reject(DOMException::Create( + resolver->Reject(MakeGarbageCollected<DOMException>( DOMExceptionCode::kNotSupportedError, "Unsupported focusMode.")); return; } @@ -440,9 +442,9 @@ void ImageCapture::SetMediaTrackConstraints( constraints->exposureCompensation().GetAsDouble(); if (exposure_compensation < capabilities_->exposureCompensation()->min() || exposure_compensation > capabilities_->exposureCompensation()->max()) { - resolver->Reject( - DOMException::Create(DOMExceptionCode::kNotSupportedError, - "exposureCompensation setting out of range")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotSupportedError, + "exposureCompensation setting out of range")); return; } temp_constraints->setExposureCompensation( @@ -456,9 +458,9 @@ void ImageCapture::SetMediaTrackConstraints( const auto exposure_time = constraints->exposureTime().GetAsDouble(); if (exposure_time < capabilities_->exposureTime()->min() || exposure_time > capabilities_->exposureTime()->max()) { - resolver->Reject( - DOMException::Create(DOMExceptionCode::kNotSupportedError, - "exposureTime setting out of range")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotSupportedError, + "exposureTime setting out of range")); return; } temp_constraints->setExposureTime(constraints->exposureTime()); @@ -471,9 +473,9 @@ void ImageCapture::SetMediaTrackConstraints( constraints->colorTemperature().GetAsDouble(); if (color_temperature < capabilities_->colorTemperature()->min() || color_temperature > capabilities_->colorTemperature()->max()) { - resolver->Reject( - DOMException::Create(DOMExceptionCode::kNotSupportedError, - "colorTemperature setting out of range")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotSupportedError, + "colorTemperature setting out of range")); return; } temp_constraints->setColorTemperature(constraints->colorTemperature()); @@ -484,7 +486,7 @@ void ImageCapture::SetMediaTrackConstraints( const auto iso = constraints->iso().GetAsDouble(); if (iso < capabilities_->iso()->min() || iso > capabilities_->iso()->max()) { - resolver->Reject(DOMException::Create( + resolver->Reject(MakeGarbageCollected<DOMException>( DOMExceptionCode::kNotSupportedError, "iso setting out of range")); return; } @@ -498,9 +500,9 @@ void ImageCapture::SetMediaTrackConstraints( const auto brightness = constraints->brightness().GetAsDouble(); if (brightness < capabilities_->brightness()->min() || brightness > capabilities_->brightness()->max()) { - resolver->Reject( - DOMException::Create(DOMExceptionCode::kNotSupportedError, - "brightness setting out of range")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotSupportedError, + "brightness setting out of range")); return; } temp_constraints->setBrightness(constraints->brightness()); @@ -512,9 +514,9 @@ void ImageCapture::SetMediaTrackConstraints( const auto contrast = constraints->contrast().GetAsDouble(); if (contrast < capabilities_->contrast()->min() || contrast > capabilities_->contrast()->max()) { - resolver->Reject( - DOMException::Create(DOMExceptionCode::kNotSupportedError, - "contrast setting out of range")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotSupportedError, + "contrast setting out of range")); return; } temp_constraints->setContrast(constraints->contrast()); @@ -526,9 +528,9 @@ void ImageCapture::SetMediaTrackConstraints( const auto saturation = constraints->saturation().GetAsDouble(); if (saturation < capabilities_->saturation()->min() || saturation > capabilities_->saturation()->max()) { - resolver->Reject( - DOMException::Create(DOMExceptionCode::kNotSupportedError, - "saturation setting out of range")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotSupportedError, + "saturation setting out of range")); return; } temp_constraints->setSaturation(constraints->saturation()); @@ -540,9 +542,9 @@ void ImageCapture::SetMediaTrackConstraints( const auto sharpness = constraints->sharpness().GetAsDouble(); if (sharpness < capabilities_->sharpness()->min() || sharpness > capabilities_->sharpness()->max()) { - resolver->Reject( - DOMException::Create(DOMExceptionCode::kNotSupportedError, - "sharpness setting out of range")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotSupportedError, + "sharpness setting out of range")); return; } temp_constraints->setSharpness(constraints->sharpness()); @@ -555,9 +557,9 @@ void ImageCapture::SetMediaTrackConstraints( const auto focus_distance = constraints->focusDistance().GetAsDouble(); if (focus_distance < capabilities_->focusDistance()->min() || focus_distance > capabilities_->focusDistance()->max()) { - resolver->Reject( - DOMException::Create(DOMExceptionCode::kNotSupportedError, - "focusDistance setting out of range")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotSupportedError, + "focusDistance setting out of range")); return; } temp_constraints->setFocusDistance(constraints->focusDistance()); @@ -569,7 +571,7 @@ void ImageCapture::SetMediaTrackConstraints( const auto zoom = constraints->zoom().GetAsDouble(); if (zoom < capabilities_->zoom()->min() || zoom > capabilities_->zoom()->max()) { - resolver->Reject(DOMException::Create( + resolver->Reject(MakeGarbageCollected<DOMException>( DOMExceptionCode::kNotSupportedError, "zoom setting out of range")); return; } @@ -583,7 +585,7 @@ void ImageCapture::SetMediaTrackConstraints( if (settings->has_torch) { const auto torch = constraints->torch().GetAsBoolean(); if (torch && !capabilities_->torch()) { - resolver->Reject(DOMException::Create( + resolver->Reject(MakeGarbageCollected<DOMException>( DOMExceptionCode::kNotSupportedError, "torch not supported")); return; } @@ -689,8 +691,8 @@ void ImageCapture::OnMojoGetPhotoState( DCHECK(service_requests_.Contains(resolver)); if (photo_state.is_null()) { - resolver->Reject(DOMException::Create(DOMExceptionCode::kUnknownError, - "platform error")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kUnknownError, "platform error")); service_requests_.erase(resolver); return; } @@ -740,8 +742,8 @@ void ImageCapture::OnMojoSetOptions(ScriptPromiseResolver* resolver, TRACE_EVENT_SCOPE_PROCESS); if (!result) { - resolver->Reject(DOMException::Create(DOMExceptionCode::kUnknownError, - "setOptions failed")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kUnknownError, "setOptions failed")); service_requests_.erase(resolver); return; } @@ -766,8 +768,8 @@ void ImageCapture::OnMojoTakePhoto(ScriptPromiseResolver* resolver, // TODO(mcasas): Should be using a mojo::StructTraits. if (blob->data.IsEmpty()) { - resolver->Reject(DOMException::Create(DOMExceptionCode::kUnknownError, - "platform error")); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kUnknownError, "platform error")); } else { resolver->Resolve( Blob::Create(blob->data.data(), blob->data.size(), blob->mime_type)); @@ -889,8 +891,8 @@ void ImageCapture::UpdateMediaTrackCapabilities( void ImageCapture::OnServiceConnectionError() { service_.reset(); for (ScriptPromiseResolver* resolver : service_requests_) { - resolver->Reject(DOMException::Create(DOMExceptionCode::kNotFoundError, - kNoServiceError)); + resolver->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotFoundError, kNoServiceError)); } service_requests_.clear(); } diff --git a/chromium/third_party/blink/renderer/modules/imagecapture/image_capture.h b/chromium/third_party/blink/renderer/modules/imagecapture/image_capture.h index ea0caa46869..fc3963d8a36 100644 --- a/chromium/third_party/blink/renderer/modules/imagecapture/image_capture.h +++ b/chromium/third_party/blink/renderer/modules/imagecapture/image_capture.h @@ -21,10 +21,10 @@ namespace blink { class ExceptionState; +class ImageCaptureFrameGrabber; class MediaStreamTrack; class PhotoCapabilities; class ScriptPromiseResolver; -class WebImageCaptureFrameGrabber; // TODO(mcasas): Consider adding a web test checking that this class is not // garbage collected while it has event listeners. @@ -98,7 +98,7 @@ class MODULES_EXPORT ImageCapture final void ResolveWithPhotoCapabilities(ScriptPromiseResolver*); Member<MediaStreamTrack> stream_track_; - std::unique_ptr<WebImageCaptureFrameGrabber> frame_grabber_; + std::unique_ptr<ImageCaptureFrameGrabber> frame_grabber_; media::mojom::blink::ImageCapturePtr service_; Member<MediaTrackCapabilities> capabilities_; diff --git a/chromium/third_party/blink/renderer/modules/imagecapture/image_capture_frame_grabber.cc b/chromium/third_party/blink/renderer/modules/imagecapture/image_capture_frame_grabber.cc new file mode 100644 index 00000000000..2653e58c3cb --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/imagecapture/image_capture_frame_grabber.cc @@ -0,0 +1,187 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "third_party/blink/renderer/modules/imagecapture/image_capture_frame_grabber.h" + +#include "media/base/bind_to_current_loop.h" +#include "media/base/video_frame.h" +#include "media/base/video_types.h" +#include "media/base/video_util.h" +#include "skia/ext/platform_canvas.h" +#include "third_party/blink/public/platform/web_media_stream_source.h" +#include "third_party/blink/public/platform/web_media_stream_track.h" +#include "third_party/blink/renderer/core/imagebitmap/image_bitmap.h" +#include "third_party/blink/renderer/platform/cross_thread_functional.h" +#include "third_party/blink/renderer/platform/wtf/functional.h" +#include "third_party/blink/renderer/platform/wtf/thread_safe_ref_counted.h" +#include "third_party/libyuv/include/libyuv.h" +#include "third_party/skia/include/core/SkImage.h" +#include "third_party/skia/include/core/SkSurface.h" + +namespace blink { + +namespace { + +void OnError(std::unique_ptr<ImageCaptureGrabFrameCallbacks> callbacks) { + callbacks->OnError(); +} + +} // anonymous namespace + +// Template specialization of [1], needed to be able to pass callbacks +// that have ScopedWebCallbacks paramaters across threads. +// +// [1] third_party/blink/renderer/platform/cross_thread_copier.h. +template <typename T> +struct CrossThreadCopier<ScopedWebCallbacks<T>> + : public CrossThreadCopierPassThrough<ScopedWebCallbacks<T>> { + STATIC_ONLY(CrossThreadCopier); + using Type = ScopedWebCallbacks<T>; + static ScopedWebCallbacks<T> Copy(ScopedWebCallbacks<T> pointer) { + return pointer; + } +}; + +// Ref-counted class to receive a single VideoFrame on IO thread, convert it and +// send it to |main_task_runner_|, where this class is created and destroyed. +class ImageCaptureFrameGrabber::SingleShotFrameHandler + : public WTF::ThreadSafeRefCounted<SingleShotFrameHandler> { + public: + SingleShotFrameHandler() : first_frame_received_(false) {} + + // Receives a |frame| and converts its pixels into a SkImage via an internal + // PaintSurface and SkPixmap. Alpha channel, if any, is copied. + using SkImageDeliverCB = WTF::CrossThreadFunction<void(sk_sp<SkImage>)>; + void OnVideoFrameOnIOThread( + SkImageDeliverCB callback, + scoped_refptr<base::SingleThreadTaskRunner> task_runner, + scoped_refptr<media::VideoFrame> frame, + base::TimeTicks current_time); + + private: + friend class WTF::ThreadSafeRefCounted<SingleShotFrameHandler>; + + // Flag to indicate that the first frames has been processed, and subsequent + // ones can be safely discarded. + bool first_frame_received_; + + DISALLOW_COPY_AND_ASSIGN(SingleShotFrameHandler); +}; + +void ImageCaptureFrameGrabber::SingleShotFrameHandler::OnVideoFrameOnIOThread( + SkImageDeliverCB callback, + scoped_refptr<base::SingleThreadTaskRunner> task_runner, + scoped_refptr<media::VideoFrame> frame, + base::TimeTicks /* current_time */) { + DCHECK(frame->format() == media::PIXEL_FORMAT_I420 || + frame->format() == media::PIXEL_FORMAT_I420A); + + if (first_frame_received_) + return; + first_frame_received_ = true; + + const SkAlphaType alpha = media::IsOpaque(frame->format()) + ? kOpaque_SkAlphaType + : kPremul_SkAlphaType; + const SkImageInfo info = SkImageInfo::MakeN32( + frame->visible_rect().width(), frame->visible_rect().height(), alpha); + + sk_sp<SkSurface> surface = SkSurface::MakeRaster(info); + DCHECK(surface); + + auto wrapper_callback = media::BindToLoop( + std::move(task_runner), ConvertToBaseCallback(std::move(callback))); + + SkPixmap pixmap; + if (!skia::GetWritablePixels(surface->getCanvas(), &pixmap)) { + DLOG(ERROR) << "Error trying to map SkSurface's pixels"; + std::move(wrapper_callback).Run(sk_sp<SkImage>()); + return; + } + + const uint32_t destination_pixel_format = + (kN32_SkColorType == kRGBA_8888_SkColorType) ? libyuv::FOURCC_ABGR + : libyuv::FOURCC_ARGB; + + libyuv::ConvertFromI420(frame->visible_data(media::VideoFrame::kYPlane), + frame->stride(media::VideoFrame::kYPlane), + frame->visible_data(media::VideoFrame::kUPlane), + frame->stride(media::VideoFrame::kUPlane), + frame->visible_data(media::VideoFrame::kVPlane), + frame->stride(media::VideoFrame::kVPlane), + static_cast<uint8_t*>(pixmap.writable_addr()), + pixmap.width() * 4, pixmap.width(), pixmap.height(), + destination_pixel_format); + + if (frame->format() == media::PIXEL_FORMAT_I420A) { + DCHECK(!info.isOpaque()); + // This function copies any plane into the alpha channel of an ARGB image. + libyuv::ARGBCopyYToAlpha(frame->visible_data(media::VideoFrame::kAPlane), + frame->stride(media::VideoFrame::kAPlane), + static_cast<uint8_t*>(pixmap.writable_addr()), + pixmap.width() * 4, pixmap.width(), + pixmap.height()); + } + + std::move(wrapper_callback).Run(surface->makeImageSnapshot()); +} + +ImageCaptureFrameGrabber::ImageCaptureFrameGrabber() + : frame_grab_in_progress_(false), weak_factory_(this) {} + +ImageCaptureFrameGrabber::~ImageCaptureFrameGrabber() { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); +} + +void ImageCaptureFrameGrabber::GrabFrame( + WebMediaStreamTrack* track, + std::unique_ptr<ImageCaptureGrabFrameCallbacks> callbacks, + scoped_refptr<base::SingleThreadTaskRunner> task_runner) { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + DCHECK(!!callbacks); + + DCHECK(track && !track->IsNull() && track->GetPlatformTrack()); + DCHECK_EQ(WebMediaStreamSource::kTypeVideo, track->Source().GetType()); + + if (frame_grab_in_progress_) { + // Reject grabFrame()s too close back to back. + callbacks->OnError(); + return; + } + + auto scoped_callbacks = + MakeScopedWebCallbacks(std::move(callbacks), WTF::Bind(&OnError)); + + // A SingleShotFrameHandler is bound and given to the Track to guarantee that + // only one VideoFrame is converted and delivered to OnSkImage(), otherwise + // SKImages might be sent to resolved |callbacks| while DisconnectFromTrack() + // is being processed, which might be further held up if UI is busy, see + // https://crbug.com/623042. + frame_grab_in_progress_ = true; + MediaStreamVideoSink::ConnectToTrack( + *track, + ConvertToBaseCallback(CrossThreadBind( + &SingleShotFrameHandler::OnVideoFrameOnIOThread, + base::MakeRefCounted<SingleShotFrameHandler>(), + WTF::Passed(CrossThreadBind( + &ImageCaptureFrameGrabber::OnSkImage, weak_factory_.GetWeakPtr(), + WTF::Passed(std::move(scoped_callbacks)))), + WTF::Passed(std::move(task_runner)))), + false); +} + +void ImageCaptureFrameGrabber::OnSkImage( + ScopedWebCallbacks<ImageCaptureGrabFrameCallbacks> callbacks, + sk_sp<SkImage> image) { + DCHECK_CALLED_ON_VALID_THREAD(thread_checker_); + + MediaStreamVideoSink::DisconnectFromTrack(); + frame_grab_in_progress_ = false; + if (image) + callbacks.PassCallbacks()->OnSuccess(image); + else + callbacks.PassCallbacks()->OnError(); +} + +} // namespace blink diff --git a/chromium/third_party/blink/renderer/modules/imagecapture/image_capture_frame_grabber.h b/chromium/third_party/blink/renderer/modules/imagecapture/image_capture_frame_grabber.h new file mode 100644 index 00000000000..5e0755f9d85 --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/imagecapture/image_capture_frame_grabber.h @@ -0,0 +1,155 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_IMAGECAPTURE_IMAGE_CAPTURE_FRAME_GRABBER_H_ +#define THIRD_PARTY_BLINK_RENDERER_MODULES_IMAGECAPTURE_IMAGE_CAPTURE_FRAME_GRABBER_H_ + +#include <memory> + +#include "base/callback.h" +#include "base/macros.h" +#include "base/memory/weak_ptr.h" +#include "base/single_thread_task_runner.h" +#include "base/threading/thread_checker.h" +#include "third_party/blink/public/platform/web_callbacks.h" +#include "third_party/blink/public/web/modules/mediastream/media_stream_video_sink.h" +#include "third_party/blink/renderer/bindings/core/v8/callback_promise_adapter.h" +#include "third_party/skia/include/core/SkRefCnt.h" + +class SkImage; + +namespace blink { + +class ImageBitmap; +class WebMediaStreamTrack; + +// A ScopedWebCallbacks is a move-only scoper which helps manage the lifetime of +// a blink::WebCallbacks object. This is particularly useful when you're +// simultaneously dealing with the following two conditions: +// +// 1. Your WebCallbacks implementation requires either onSuccess or onError to +// be called before it's destroyed. This is the case with +// CallbackPromiseAdapter for example, because its underlying +// ScriptPromiseResolver must be resolved or rejected before destruction. +// +// 2. You are passing ownership of the WebCallbacks to code which may +// silenty drop it. A common way for this to happen is to bind the +// WebCallbacks as an argument to a base::Callback which gets destroyed +// before it can run. +// +// While it's possible to individually track the lifetime of pending +// WebCallbacks, this becomes cumbersome when dealing with many different +// callbacks types. ScopedWebCallbacks provides a generic and relatively +// lightweight solution to this problem. +// +// Example usage: +// +// using FooCallbacks = blink::WebCallbacks<const Foo&, const FooError&>; +// +// void RespondWithSuccess(ScopedWebCallbacks<FooCallbacks> callbacks) { +// callbacks.PassCallbacks()->onSuccess(Foo("everything is great")); +// } +// +// void OnCallbacksDropped(std::unique_ptr<FooCallbacks> callbacks) { +// // Ownership of the FooCallbacks is passed to this function if +// // ScopedWebCallbacks::PassCallbacks isn't called before the +// // ScopedWebCallbacks is destroyed. +// callbacks->onError(FooError("everything is terrible")); +// } +// +// // Blink client implementation +// void FooClientImpl::doMagic(std::unique_ptr<FooCallbacks> callbacks) { +// auto scoped_callbacks = make_scoped_web_callbacks( +// std::move(callbacks), base::BindOnce(&OnCallbacksDropped)); +// +// // Call to some lower-level service which may never run the callback we +// // give it. +// foo_service_->DoMagic(base::BindOnce(&RespondWithSuccess, +// std::move(scoped_callbacks))); +// } +// +// If the bound RespondWithSuccess callback actually runs, PassCallbacks() will +// reliquish ownership of the WebCallbacks object to a temporary scoped_ptr +// which will be destroyed immediately after onSuccess is called. +// +// If the bound RespondWithSuccess callback is instead destroyed first, +// the ScopedWebCallbacks destructor will invoke OnCallbacksDropped, executing +// our desired default behavior before deleting the WebCallbacks. +template <typename CallbacksType> +class ScopedWebCallbacks { + public: + using DestructionCallback = + base::OnceCallback<void(std::unique_ptr<CallbacksType> callbacks)>; + + ScopedWebCallbacks(std::unique_ptr<CallbacksType> callbacks, + DestructionCallback destruction_callback) + : callbacks_(std::move(callbacks)), + destruction_callback_(std::move(destruction_callback)) {} + + ~ScopedWebCallbacks() { + if (destruction_callback_) + std::move(destruction_callback_).Run(std::move(callbacks_)); + } + + ScopedWebCallbacks(ScopedWebCallbacks&& other) = default; + ScopedWebCallbacks(const ScopedWebCallbacks& other) = delete; + + ScopedWebCallbacks& operator=(ScopedWebCallbacks&& other) = default; + ScopedWebCallbacks& operator=(const ScopedWebCallbacks& other) = delete; + + std::unique_ptr<CallbacksType> PassCallbacks() { + destruction_callback_ = DestructionCallback(); + return std::move(callbacks_); + } + + private: + std::unique_ptr<CallbacksType> callbacks_; + DestructionCallback destruction_callback_; +}; + +template <typename CallbacksType> +ScopedWebCallbacks<CallbacksType> MakeScopedWebCallbacks( + std::unique_ptr<CallbacksType> callbacks, + typename ScopedWebCallbacks<CallbacksType>::DestructionCallback + destruction_callback) { + return ScopedWebCallbacks<CallbacksType>(std::move(callbacks), + std::move(destruction_callback)); +} + +using ImageCaptureGrabFrameCallbacks = + CallbackPromiseAdapter<ImageBitmap, void>; + +// This class grabs Video Frames from a given Media Stream Video Track, binding +// a method of an ephemeral SingleShotFrameHandler every time grabFrame() is +// called. This method receives an incoming media::VideoFrame on a background +// thread and converts it into the appropriate SkBitmap which is sent back to +// OnSkBitmap(). This class is single threaded throughout. +class ImageCaptureFrameGrabber final : public MediaStreamVideoSink { + public: + ImageCaptureFrameGrabber(); + ~ImageCaptureFrameGrabber() override; + + void GrabFrame(WebMediaStreamTrack* track, + std::unique_ptr<ImageCaptureGrabFrameCallbacks> callbacks, + scoped_refptr<base::SingleThreadTaskRunner> task_runner); + + private: + // Internal class to receive, convert and forward one frame. + class SingleShotFrameHandler; + + void OnSkImage(ScopedWebCallbacks<ImageCaptureGrabFrameCallbacks> callbacks, + sk_sp<SkImage> image); + + // Flag to indicate that there is a frame grabbing in progress. + bool frame_grab_in_progress_; + + THREAD_CHECKER(thread_checker_); + base::WeakPtrFactory<ImageCaptureFrameGrabber> weak_factory_; + + DISALLOW_COPY_AND_ASSIGN(ImageCaptureFrameGrabber); +}; + +} // namespace blink + +#endif // THIRD_PARTY_BLINK_RENDERER_MODULES_IMAGECAPTURE_IMAGE_CAPTURE_FRAME_GRABBER_H_ |