diff options
Diffstat (limited to 'chromium/third_party/blink/renderer/modules/webcodecs')
29 files changed, 1434 insertions, 0 deletions
diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/BUILD.gn b/chromium/third_party/blink/renderer/modules/webcodecs/BUILD.gn new file mode 100644 index 00000000000..677518de66b --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/BUILD.gn @@ -0,0 +1,44 @@ +# Copyright 2019 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import("//third_party/blink/renderer/modules/modules.gni") + +blink_modules_sources("webcodecs") { + sources = [ + "encoded_video_chunk.cc", + "encoded_video_chunk.h", + "encoded_video_metadata.h", + "video_decoder.cc", + "video_decoder.h", + "video_frame.cc", + "video_frame.h", + "video_track_reader.cc", + "video_track_reader.h", + "video_track_writer.cc", + "video_track_writer.h", + ] +} + +source_set("unit_tests") { + testonly = true + sources = [ + "encoded_video_chunk_test.cc", + "video_frame_test.cc", + "video_track_reader_writer_test.cc", + ] + + configs += [ + "//third_party/blink/renderer:config", + "//third_party/blink/renderer:inside_blink", + "//third_party/blink/renderer/core:blink_core_pch", + ] + + deps = [ + "//testing/gmock", + "//testing/gtest", + "//third_party/blink/renderer/modules", + "//third_party/blink/renderer/platform", + "//third_party/blink/renderer/platform/wtf", + ] +} diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/DEPS b/chromium/third_party/blink/renderer/modules/webcodecs/DEPS new file mode 100644 index 00000000000..21f825ca9a3 --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/DEPS @@ -0,0 +1,9 @@ +include_rules = [ + "+base/run_loop.h", + "+media/base/media_util.h", + "+media/base/video_decoder.h", + "+media/base/video_frame.h", + "+media/base/status.h", + "+ui/gfx/geometry/rect.h", + "+ui/gfx/geometry/size.h", +] diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/OWNERS b/chromium/third_party/blink/renderer/modules/webcodecs/OWNERS new file mode 100644 index 00000000000..cc72639d7c4 --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/OWNERS @@ -0,0 +1,3 @@ +mlamouri@chromium.org +sandersd@chromium.org +dalecurtis@chromium.org diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/README.md b/chromium/third_party/blink/renderer/modules/webcodecs/README.md new file mode 100644 index 00000000000..35d9e504ee7 --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/README.md @@ -0,0 +1,9 @@ +# WebCodecs API + +This directory will contain the implementation of +https://github.com/WICG/web-codecs/, which is a low-level API for encode and +decode of audio and video. + +It will use the existing codec implementations in src/media used by the video +stack, WebRTC, and MediaRecorder, such as media::DecoderFactory, +media::VideoEncodeAccelerator, and media::VideoFrame.
\ No newline at end of file diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.cc b/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.cc new file mode 100644 index 00000000000..ad165d8dd39 --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.cc @@ -0,0 +1,64 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.h" + +#include <utility> + +#include "third_party/blink/renderer/core/typed_arrays/dom_array_buffer.h" +#include "third_party/blink/renderer/platform/wtf/text/wtf_string.h" + +namespace blink { + +EncodedVideoChunk* EncodedVideoChunk::Create(String type, + uint64_t timestamp, + const DOMArrayPiece& data) { + return EncodedVideoChunk::Create(type, timestamp, 0 /* duration */, data); +} + +EncodedVideoChunk* EncodedVideoChunk::Create(String type, + uint64_t timestamp, + uint64_t duration, + const DOMArrayPiece& data) { + EncodedVideoMetadata metadata; + metadata.timestamp = base::TimeDelta::FromMicroseconds(timestamp); + metadata.key_frame = (type == "key"); + if (duration) + metadata.duration = base::TimeDelta::FromMicroseconds(duration); + return MakeGarbageCollected<EncodedVideoChunk>( + metadata, DOMArrayBuffer::Create(data.Bytes(), data.ByteLengthAsSizeT())); +} + +EncodedVideoChunk::EncodedVideoChunk(EncodedVideoMetadata metadata, + DOMArrayBuffer* buffer) + : metadata_(metadata), buffer_(buffer) {} + +String EncodedVideoChunk::type() const { + return metadata_.key_frame ? "key" : "delta"; +} + +uint64_t EncodedVideoChunk::timestamp() const { + return metadata_.timestamp.InMicroseconds(); +} + +base::Optional<uint64_t> EncodedVideoChunk::duration() const { + if (!metadata_.duration) + return base::nullopt; + return metadata_.duration->InMicroseconds(); +} + +uint64_t EncodedVideoChunk::duration(bool* is_null) const { + if (!metadata_.duration) { + *is_null = true; + return 0; + } + *is_null = false; + return metadata_.duration->InMicroseconds(); +} + +DOMArrayBuffer* EncodedVideoChunk::data() const { + return buffer_; +} + +} // namespace blink diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.h b/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.h new file mode 100644 index 00000000000..4d063fdfcb2 --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.h @@ -0,0 +1,54 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_ENCODED_VIDEO_CHUNK_H_ +#define THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_ENCODED_VIDEO_CHUNK_H_ + +#include "third_party/blink/renderer/core/typed_arrays/dom_array_piece.h" +#include "third_party/blink/renderer/modules/modules_export.h" +#include "third_party/blink/renderer/modules/webcodecs/encoded_video_metadata.h" +#include "third_party/blink/renderer/platform/bindings/script_wrappable.h" + +namespace blink { + +class DOMArrayBuffer; + +class MODULES_EXPORT EncodedVideoChunk final : public ScriptWrappable { + DEFINE_WRAPPERTYPEINFO(); + + public: + EncodedVideoChunk(EncodedVideoMetadata metadata, DOMArrayBuffer* buffer); + + static EncodedVideoChunk* Create(String type, + uint64_t timestamp, + const DOMArrayPiece& data); + static EncodedVideoChunk* Create(String type, + uint64_t timestamp, + uint64_t duration, + const DOMArrayPiece& data); + + // encoded_video_chunk.idl implementation. + String type() const; + uint64_t timestamp() const; + base::Optional<uint64_t> duration() const; + // TODO(crbug.com/1060971): Remove |is_null| version. + uint64_t duration(bool* is_null) const; // DEPRECATED + uint64_t duration(bool& is_null) const { // DEPRECATED + return duration(&is_null); + } + DOMArrayBuffer* data() const; + + void Trace(Visitor* visitor) override { + visitor->Trace(buffer_); + ScriptWrappable::Trace(visitor); + } + + private: + EncodedVideoMetadata metadata_; + Member<DOMArrayBuffer> buffer_; +}; + +} // namespace blink + +#endif // THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_ENCODED_VIDEO_CHUNK_H_ diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.idl b/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.idl new file mode 100644 index 00000000000..b2eff05301b --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.idl @@ -0,0 +1,20 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +enum EncodedVideoChunkType { + "key", + "delta", +}; + +[ + RuntimeEnabled=WebCodecs +] interface EncodedVideoChunk { + constructor(EncodedVideoChunkType type, unsigned long long timestamp, BufferSource data); + constructor(EncodedVideoChunkType type, unsigned long long timestamp, unsigned long long duration, BufferSource data); + readonly attribute EncodedVideoChunkType type; + // TODO: Add frame dependency information + readonly attribute unsigned long long timestamp; // microseconds + readonly attribute unsigned long long? duration; // microseconds + readonly attribute ArrayBuffer data; +};
\ No newline at end of file diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_chunk_test.cc b/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_chunk_test.cc new file mode 100644 index 00000000000..9844820876d --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_chunk_test.cc @@ -0,0 +1,58 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/blink/renderer/platform/wtf/text/wtf_string.h" + +namespace blink { + +namespace { + +class EncodedVideoChunkTest : public testing::Test { + public: + DOMArrayBuffer* StringToBuffer(std::string data) { + return DOMArrayBuffer::Create(data.data(), data.size()); + } + + std::string BufferToString(DOMArrayBuffer* buffer) { + return std::string(static_cast<char*>(buffer->Data()), + buffer->ByteLengthAsSizeT()); + } +}; + +TEST_F(EncodedVideoChunkTest, ConstructorAndAttributes) { + String type = "key"; + uint64_t timestamp = 1000000; + std::string data = "test"; + auto* encoded = + EncodedVideoChunk::Create(type, timestamp, StringToBuffer(data)); + + EXPECT_EQ(type, encoded->type()); + EXPECT_EQ(timestamp, encoded->timestamp()); + EXPECT_EQ(data, BufferToString(encoded->data())); + bool is_null = false; + encoded->duration(&is_null); + EXPECT_TRUE(is_null); +} + +TEST_F(EncodedVideoChunkTest, ConstructorWithDuration) { + String type = "key"; + uint64_t timestamp = 1000000; + uint64_t duration = 16667; + std::string data = "test"; + auto* encoded = EncodedVideoChunk::Create(type, timestamp, duration, + StringToBuffer(data)); + + EXPECT_EQ(type, encoded->type()); + EXPECT_EQ(timestamp, encoded->timestamp()); + EXPECT_EQ(data, BufferToString(encoded->data())); + bool is_null = false; + EXPECT_EQ(duration, encoded->duration(&is_null)); + EXPECT_FALSE(is_null); +} + +} // namespace + +} // namespace blink diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_config.idl b/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_config.idl new file mode 100644 index 00000000000..7af04a69e78 --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_config.idl @@ -0,0 +1,21 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// https://github.com/WICG/web-codecs + +dictionary EncodedVideoConfig { + // Codec string, eg. "avc1.42001e" or "vp09.00.10.08". + // TODO(sandersd): Accept "avc1" if |description| is provided? + required DOMString codec; + + // avcC, vpcC, or etc. + BufferSource description; + + // If provided, these override in-band configuration. + double sampleAspect; + // TODO(sandersd): color space. + + // TODO(sandersd): Constraints (sequential access) and requirements + // (imagebitmap, colorspace conversion). +}; diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_metadata.h b/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_metadata.h new file mode 100644 index 00000000000..124b3bf6c72 --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/encoded_video_metadata.h @@ -0,0 +1,17 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_ENCODED_VIDEO_METADATA_H_ +#define THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_ENCODED_VIDEO_METADATA_H_ + +#include "base/optional.h" +#include "base/time/time.h" + +struct EncodedVideoMetadata { + bool key_frame = false; + base::TimeDelta timestamp; + base::Optional<base::TimeDelta> duration; +}; + +#endif // THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_ENCODED_VIDEO_METADATA_H_ diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/idls.gni b/chromium/third_party/blink/renderer/modules/webcodecs/idls.gni new file mode 100644 index 00000000000..973ad53968d --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/idls.gni @@ -0,0 +1,30 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +modules_idl_files = [ + "encoded_video_chunk.idl", + "video_decoder.idl", + "video_frame.idl", + "video_track_reader.idl", + "video_track_writer.idl", +] + +modules_callback_function_idl_files = [ + "video_decoder_output_callback.idl", + "webcodecs_error_callback.idl", +] + +modules_dictionary_idl_files = [ + "encoded_video_config.idl", + "video_decoder_init.idl", + "video_track_writer_parameters.idl", +] + +# IDL files that either define partial interfaces or target (right side of) +# `includes`. +modules_dependency_idl_files = [] + +# Similar to |modules_dependency_idl_files| but limited to things that are +# exposed only for testing. +modules_testing_dependency_idl_files = [] diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/video_decoder.cc b/chromium/third_party/blink/renderer/modules/webcodecs/video_decoder.cc new file mode 100644 index 00000000000..cc422bb0ecf --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/video_decoder.cc @@ -0,0 +1,158 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "third_party/blink/renderer/modules/webcodecs/video_decoder.h" + +#include "base/logging.h" +#include "base/time/time.h" +#include "media/base/media_util.h" +#include "media/base/video_decoder.h" +#include "third_party/blink/renderer/bindings/modules/v8/v8_encoded_video_chunk.h" +#include "third_party/blink/renderer/bindings/modules/v8/v8_encoded_video_config.h" +#include "third_party/blink/renderer/core/dom/dom_exception.h" +#include "third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.h" +#include "third_party/blink/renderer/modules/webcodecs/video_frame.h" +#include "third_party/blink/renderer/platform/bindings/exception_code.h" +#include "third_party/blink/renderer/platform/bindings/exception_state.h" +#include "third_party/blink/renderer/platform/bindings/script_state.h" +#include "third_party/blink/renderer/platform/wtf/functional.h" +#include "ui/gfx/geometry/rect.h" +#include "ui/gfx/geometry/size.h" + +namespace blink { + +// static +VideoDecoder* VideoDecoder::Create(ScriptState* script_state, + const VideoDecoderInit* init, + ExceptionState& exception_state) { + return MakeGarbageCollected<VideoDecoder>(script_state, init, + exception_state); +} + +VideoDecoder::VideoDecoder(ScriptState* script_state, + const VideoDecoderInit* init, + ExceptionState& exception_state) + : script_state_(script_state) { + DVLOG(1) << __func__; + // TODO(sandersd): Extract callbacks from |init|. +} + +VideoDecoder::~VideoDecoder() { + DVLOG(1) << __func__; +} + +int32_t VideoDecoder::decodeQueueSize() { + return requested_decodes_; +} + +int32_t VideoDecoder::decodeProcessingCount() { + return pending_decodes_.size(); +} + +ScriptPromise VideoDecoder::configure(const EncodedVideoConfig* config, + ExceptionState&) { + DVLOG(1) << __func__; + Request* request = MakeGarbageCollected<Request>(); + request->type = Request::Type::kConfigure; + request->config = config; + return EnqueueRequest(request); +} + +ScriptPromise VideoDecoder::decode(const EncodedVideoChunk* chunk, + ExceptionState&) { + DVLOG(3) << __func__; + requested_decodes_++; + Request* request = MakeGarbageCollected<Request>(); + request->type = Request::Type::kDecode; + request->chunk = chunk; + return EnqueueRequest(request); +} + +ScriptPromise VideoDecoder::flush(ExceptionState&) { + DVLOG(3) << __func__; + Request* request = MakeGarbageCollected<Request>(); + request->type = Request::Type::kFlush; + return EnqueueRequest(request); +} + +ScriptPromise VideoDecoder::reset(ExceptionState&) { + DVLOG(3) << __func__; + requested_resets_++; + Request* request = MakeGarbageCollected<Request>(); + request->type = Request::Type::kReset; + return EnqueueRequest(request); +} + +ScriptPromise VideoDecoder::EnqueueRequest(Request* request) { + ScriptPromiseResolver* resolver = + MakeGarbageCollected<ScriptPromiseResolver>(script_state_); + request->resolver = resolver; + requests_.push_back(request); + + // If there were no requests before, trigger request processing. + if (requests_.size() == 1) + ProcessRequests(); + + return resolver->Promise(); +} + +void VideoDecoder::ProcessRequests() {} + +void VideoDecoder::HandleError() {} + +void VideoDecoder::OnInitializeDone(media::Status status) { + DVLOG(3) << __func__; + + Request* request = requests_.TakeFirst(); + DCHECK_EQ(request->type, Request::Type::kConfigure); + + if (!status.is_ok()) { + // TODO(tmathmeyer) this drops the media error - should we consider logging + // it or converting it to the DOMException type somehow? + request->resolver.Release()->Reject(MakeGarbageCollected<DOMException>( + DOMExceptionCode::kNotSupportedError, "Codec initialization failed.")); + HandleError(); + return; + } + + request->resolver.Release()->Resolve(); + ProcessRequests(); +} + +void VideoDecoder::OnDecodeDone(uint32_t id, media::DecodeStatus status) { + DVLOG(3) << __func__; + DCHECK(pending_decodes_.Contains(id)); + + if (status != media::DecodeStatus::OK) { + // TODO(sandersd): Handle ABORTED during Reset. + HandleError(); + return; + } + + auto it = pending_decodes_.find(id); + it->value->resolver.Release()->Resolve(); + pending_decodes_.erase(it); + ProcessRequests(); +} + +void VideoDecoder::OnOutput(scoped_refptr<media::VideoFrame> frame) { + DVLOG(3) << __func__; + // TODO(sandersd): Call output callback. + // MakeGarbageCollected<VideoFrame>(frame) +} + +void VideoDecoder::Trace(Visitor* visitor) { + visitor->Trace(script_state_); + visitor->Trace(requests_); + visitor->Trace(pending_decodes_); + ScriptWrappable::Trace(visitor); +} + +void VideoDecoder::Request::Trace(Visitor* visitor) { + visitor->Trace(config); + visitor->Trace(chunk); + visitor->Trace(resolver); +} + +} // namespace blink diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/video_decoder.h b/chromium/third_party/blink/renderer/modules/webcodecs/video_decoder.h new file mode 100644 index 00000000000..03c62537f19 --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/video_decoder.h @@ -0,0 +1,90 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_DECODER_H_ +#define THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_DECODER_H_ + +#include <stdint.h> +#include <memory> + +#include "media/base/status.h" +#include "media/base/video_decoder.h" +#include "third_party/blink/renderer/bindings/core/v8/script_promise.h" +#include "third_party/blink/renderer/bindings/core/v8/script_promise_resolver.h" +#include "third_party/blink/renderer/modules/modules_export.h" +#include "third_party/blink/renderer/platform/bindings/script_wrappable.h" +#include "third_party/blink/renderer/platform/heap/garbage_collected.h" +#include "third_party/blink/renderer/platform/heap/heap.h" +#include "third_party/blink/renderer/platform/heap/heap_allocator.h" +#include "third_party/blink/renderer/platform/heap/member.h" + +namespace blink { + +class EncodedVideoChunk; +class EncodedVideoConfig; +class ExceptionState; +class ScriptState; +class VideoDecoderInit; + +class MODULES_EXPORT VideoDecoder final : public ScriptWrappable { + DEFINE_WRAPPERTYPEINFO(); + + public: + static VideoDecoder* Create(ScriptState*, + const VideoDecoderInit*, + ExceptionState&); + + VideoDecoder(ScriptState*, const VideoDecoderInit*, ExceptionState&); + ~VideoDecoder() override; + + // video_decoder.idl implementation. + int32_t decodeQueueSize(); + int32_t decodeProcessingCount(); + ScriptPromise configure(const EncodedVideoConfig*, ExceptionState&); + ScriptPromise decode(const EncodedVideoChunk*, ExceptionState&); + ScriptPromise flush(ExceptionState&); + ScriptPromise reset(ExceptionState&); + + // GarbageCollected override. + void Trace(Visitor*) override; + + private: + struct Request : public GarbageCollected<Request> { + enum class Type { + kConfigure, + kDecode, + kFlush, + kReset, + }; + + void Trace(Visitor*); + + Type type; + Member<const EncodedVideoConfig> config; + Member<const EncodedVideoChunk> chunk; + Member<ScriptPromiseResolver> resolver; + }; + + ScriptPromise EnqueueRequest(Request* request); + void ProcessRequests(); + void HandleError(); + + // Called by |decoder_|. + void OnInitializeDone(media::Status status); + void OnDecodeDone(uint32_t id, media::DecodeStatus); + void OnOutput(scoped_refptr<media::VideoFrame>); + + Member<ScriptState> script_state_; + + HeapDeque<Member<Request>> requests_; + int32_t requested_decodes_ = 0; + int32_t requested_resets_ = 0; + + std::unique_ptr<media::VideoDecoder> decoder_; + HeapHashMap<uint32_t, Member<Request>> pending_decodes_; +}; + +} // namespace blink + +#endif // THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_DECODER_H_ diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/video_decoder.idl b/chromium/third_party/blink/renderer/modules/webcodecs/video_decoder.idl new file mode 100644 index 00000000000..8d930506f5e --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/video_decoder.idl @@ -0,0 +1,87 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// https://github.com/WICG/web-codecs + +// A VideoDecoder processes a queue of configure, decode, and flush requests. +// Requests are taken from the queue sequentially but may be processed +// concurrently. +// +// TODO(sandersd): Specify a tune() implementation for changing decoder +// parameters (separate from stream parameters). This is more important for +// encoders. +[RuntimeEnabled=WebCodecs] +interface VideoDecoder { + // |init| includes an |output| callback for emitting VideoFrames and an + // |error| callback for emitting decode errors. + // + // When in an error state, methods other than reset() will fail. + // + // TODO(sandersd): Consider adding a state or last error attribute. + // TODO(sandersd): Consider aborting pending decodes on error, rather than + // waiting for reset(). + [CallWith=ScriptState, RaisesException] constructor(VideoDecoderInit init); + + // The number of queued decode requests. This does not include requests that + // have been taken for processing. + // + // Applications can minimize underflow by enqueueing decode requests until + // |decodeQueueSize| is greater than a constant. + readonly attribute long decodeQueueSize; + + // The number of decode requests currently being processed. + // + // Applications can minimize resource consumption and decode latency by + // enqueueing decode requests only when |decodeQueueSize| and + // |decodeProcessingCount| are small. + // + // TODO(sandersd): Consider counting queued decode requests as well. This + // could be simpler for apps. + readonly attribute long decodeProcessingCount; + + // Enqueue a request to set or change the stream configuration. + // + // The next enqueued decode request must be for a keyframe. + // + // Resolved after emitting output for all earlier decode requests. + // + // TODO(sandersd): Test that resolution (a microtask) interleaves between + // outputs callback calls in all cases. + // TODO(sandersd): Move the keyframe rule into the bytestream registry. + [RaisesException] Promise<void> configure(EncodedVideoConfig config); + + // Enqueue a request to decode an input chunk. + // + // You must call configure() before calling enqueue() for the first time. + // + // Resolved after decoding of the input chunk has started (that is, after + // decreasing |decodeQueueSize|). + // + // TODO(sandersd): Change to a dictionary type. + // TODO(sandersd): Should we guarantee that resolution occurs in order? + // TODO(sandersd): Add status to result. + // TODO(sandersd): Buffer return. + [RaisesException] Promise<void> decode(EncodedVideoChunk chunk); + + // Enqueue a request to finish decoding queued input chunks. + // + // The next enqueued input chunk must be a keyframe. + // + // Resolved after emitting output for all earlier decode requests. + // + // TODO(sandersd): Consider relaxing the keyframe requirement. + [RaisesException] Promise<void> flush(); + + // Discard all pending work. + // + // Output for earlier decode requests will not be emitted, even if processing + // has already started. + // + // The next enqueued input chunk must be a keyframe. + // + // Resolved after all earlier enqueue() promises have been resolved. + // + // TODO(sandersd): Require configure() after reset()? + [RaisesException] Promise<void> reset(); +}; diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/video_decoder_init.idl b/chromium/third_party/blink/renderer/modules/webcodecs/video_decoder_init.idl new file mode 100644 index 00000000000..5335bbaebd5 --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/video_decoder_init.idl @@ -0,0 +1,10 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// https://github.com/WICG/web-codecs + +dictionary VideoDecoderInit { + VideoDecoderOutputCallback output; + WebCodecsErrorCallback error; +}; diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/video_decoder_output_callback.idl b/chromium/third_party/blink/renderer/modules/webcodecs/video_decoder_output_callback.idl new file mode 100644 index 00000000000..3daff3e415d --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/video_decoder_output_callback.idl @@ -0,0 +1,8 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// https://github.com/WICG/web-codecs + +[RuntimeEnabled=WebCodecs] +callback VideoDecoderOutputCallback = void(VideoFrame output); diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/video_frame.cc b/chromium/third_party/blink/renderer/modules/webcodecs/video_frame.cc new file mode 100644 index 00000000000..fca64b49cbf --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/video_frame.cc @@ -0,0 +1,60 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "third_party/blink/renderer/modules/webcodecs/video_frame.h" + +#include <utility> + +#include "media/base/video_frame.h" + +namespace blink { + +VideoFrame::VideoFrame(scoped_refptr<media::VideoFrame> frame) + : frame_(std::move(frame)) { + DCHECK(frame_); +} + +scoped_refptr<media::VideoFrame> VideoFrame::frame() { + return frame_; +} + +scoped_refptr<const media::VideoFrame> VideoFrame::frame() const { + return frame_; +} + +uint64_t VideoFrame::timestamp() const { + if (!frame_) + return 0; + return frame_->timestamp().InMicroseconds(); +} + +uint32_t VideoFrame::coded_width() const { + if (!frame_) + return 0; + return frame_->coded_size().width(); +} + +uint32_t VideoFrame::coded_height() const { + if (!frame_) + return 0; + return frame_->coded_size().height(); +} + +uint32_t VideoFrame::visible_width() const { + if (!frame_) + return 0; + return frame_->visible_rect().width(); +} + +uint32_t VideoFrame::visible_height() const { + if (!frame_) + return 0; + return frame_->visible_rect().height(); +} + +void VideoFrame::release() { + frame_.reset(); +} + +} // namespace blink diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/video_frame.h b/chromium/third_party/blink/renderer/modules/webcodecs/video_frame.h new file mode 100644 index 00000000000..269543f12de --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/video_frame.h @@ -0,0 +1,41 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_FRAME_H_ +#define THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_FRAME_H_ + +#include "media/base/video_frame.h" +#include "third_party/blink/renderer/modules/modules_export.h" +#include "third_party/blink/renderer/platform/bindings/script_wrappable.h" +#include "third_party/blink/renderer/platform/wtf/text/wtf_string.h" + +namespace blink { + +class MODULES_EXPORT VideoFrame final : public ScriptWrappable { + DEFINE_WRAPPERTYPEINFO(); + + public: + explicit VideoFrame(scoped_refptr<media::VideoFrame> frame); + + // video_track_frame.idl implementation. + uint64_t timestamp() const; + + uint32_t coded_width() const; + uint32_t coded_height() const; + uint32_t visible_width() const; + uint32_t visible_height() const; + + void release(); + + // Convenience functions + scoped_refptr<media::VideoFrame> frame(); + scoped_refptr<const media::VideoFrame> frame() const; + + private: + scoped_refptr<media::VideoFrame> frame_; +}; + +} // namespace blink + +#endif // THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_FRAME_H_ diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/video_frame.idl b/chromium/third_party/blink/renderer/modules/webcodecs/video_frame.idl new file mode 100644 index 00000000000..5b4f67254f9 --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/video_frame.idl @@ -0,0 +1,17 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +[ + RuntimeEnabled=WebCodecs +] interface VideoFrame { + void release(); + + readonly attribute unsigned long long timestamp; // microseconds + + readonly attribute unsigned long coded_width; + readonly attribute unsigned long coded_height; + + readonly attribute unsigned long visible_width; + readonly attribute unsigned long visible_height; +};
\ No newline at end of file diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/video_frame_test.cc b/chromium/third_party/blink/renderer/modules/webcodecs/video_frame_test.cc new file mode 100644 index 00000000000..d3e6d42cd91 --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/video_frame_test.cc @@ -0,0 +1,62 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "media/base/video_frame.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/blink/renderer/modules/webcodecs/video_frame.h" +#include "ui/gfx/geometry/rect.h" +#include "ui/gfx/geometry/size.h" + +namespace blink { + +namespace { + +class VideoFrameTest : public testing::Test { + public: + VideoFrame* CreateBlinkVideoFrame( + scoped_refptr<media::VideoFrame> media_frame) { + return MakeGarbageCollected<VideoFrame>(std::move(media_frame)); + } + scoped_refptr<media::VideoFrame> CreateBlackMediaVideoFrame( + base::TimeDelta timestamp, + media::VideoPixelFormat format, + const gfx::Size& coded_size, + const gfx::Size& visible_size) { + scoped_refptr<media::VideoFrame> media_frame = + media::VideoFrame::WrapVideoFrame( + media::VideoFrame::CreateBlackFrame(coded_size), format, + gfx::Rect(visible_size) /* visible_rect */, + visible_size /* natural_size */); + media_frame->set_timestamp(timestamp); + return media_frame; + } +}; + +TEST_F(VideoFrameTest, ConstructorAndAttributes) { + scoped_refptr<media::VideoFrame> media_frame = CreateBlackMediaVideoFrame( + base::TimeDelta::FromMicroseconds(1000), media::PIXEL_FORMAT_I420, + gfx::Size(112, 208) /* coded_size */, + gfx::Size(100, 200) /* visible_size */); + VideoFrame* blink_frame = CreateBlinkVideoFrame(media_frame); + + EXPECT_EQ(1000u, blink_frame->timestamp()); + EXPECT_EQ(112u, blink_frame->coded_width()); + EXPECT_EQ(208u, blink_frame->coded_height()); + EXPECT_EQ(100u, blink_frame->visible_width()); + EXPECT_EQ(200u, blink_frame->visible_height()); + EXPECT_EQ(media_frame, blink_frame->frame()); + + blink_frame->release(); + + EXPECT_EQ(0u, blink_frame->timestamp()); + EXPECT_EQ(0u, blink_frame->coded_width()); + EXPECT_EQ(0u, blink_frame->coded_height()); + EXPECT_EQ(0u, blink_frame->visible_width()); + EXPECT_EQ(0u, blink_frame->visible_height()); + EXPECT_EQ(nullptr, blink_frame->frame()); +} + +} // namespace + +} // namespace blink diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/video_track_reader.cc b/chromium/third_party/blink/renderer/modules/webcodecs/video_track_reader.cc new file mode 100644 index 00000000000..d1b603f0dfb --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/video_track_reader.cc @@ -0,0 +1,121 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "third_party/blink/renderer/modules/webcodecs/video_track_reader.h" + +#include "media/base/video_frame.h" +#include "third_party/blink/public/web/modules/mediastream/media_stream_video_sink.h" +#include "third_party/blink/public/web/modules/mediastream/media_stream_video_track.h" +#include "third_party/blink/renderer/core/streams/readable_stream.h" +#include "third_party/blink/renderer/core/streams/readable_stream_default_controller_with_script_scope.h" +#include "third_party/blink/renderer/core/streams/underlying_source_base.h" +#include "third_party/blink/renderer/modules/webcodecs/video_frame.h" +#include "third_party/blink/renderer/platform/bindings/exception_state.h" +#include "third_party/blink/renderer/platform/bindings/script_state.h" +#include "third_party/blink/renderer/platform/scheduler/public/post_cross_thread_task.h" +#include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h" + +namespace blink { + +class VideoTrackReadableStreamSource final : public UnderlyingSourceBase, + public MediaStreamVideoSink { + public: + VideoTrackReadableStreamSource(ScriptState* script_state, + MediaStreamTrack* track) + : UnderlyingSourceBase(script_state), + task_runner_(ExecutionContext::From(script_state) + ->GetTaskRunner(TaskType::kInternalMedia)) { + ConnectToTrack(track->Component(), + ConvertToBaseRepeatingCallback(CrossThreadBindRepeating( + &VideoTrackReadableStreamSource::OnFrameFromVideoTrack, + WrapCrossThreadPersistent(this))), + false /* is_sink_secure */); + } + + // Callback of MediaStreamVideoSink::ConnectToTrack + void OnFrameFromVideoTrack(scoped_refptr<media::VideoFrame> media_frame, + base::TimeTicks estimated_capture_time) { + // The value of estimated_capture_time here seems to almost always be the + // system clock and most implementations of this callback ignore it. + // So, we will also ignore it. + DCHECK(media_frame); + PostCrossThreadTask( + *task_runner_.get(), FROM_HERE, + CrossThreadBindOnce( + &VideoTrackReadableStreamSource::OnFrameFromVideoTrackOnTaskRunner, + WrapCrossThreadPersistent(this), std::move(media_frame))); + } + + void OnFrameFromVideoTrackOnTaskRunner( + scoped_refptr<media::VideoFrame> media_frame) { + Controller()->Enqueue( + MakeGarbageCollected<VideoFrame>(std::move(media_frame))); + } + + // MediaStreamVideoSink override + void OnReadyStateChanged(WebMediaStreamSource::ReadyState state) override { + if (state == WebMediaStreamSource::kReadyStateEnded) + Close(); + } + + // UnderlyingSourceBase overrides. + ScriptPromise pull(ScriptState* script_state) override { + // Since video tracks are all push-based with no back pressure, there's + // nothing to do here. + return ScriptPromise::CastUndefined(script_state); + } + + ScriptPromise Cancel(ScriptState* script_state, ScriptValue reason) override { + Close(); + return ScriptPromise::CastUndefined(script_state); + } + + void ContextDestroyed() override { DisconnectFromTrack(); } + + void Close() { + if (Controller()) + Controller()->Close(); + DisconnectFromTrack(); + } + + // VideoFrames will be queue on this task runner. + const scoped_refptr<base::SingleThreadTaskRunner> task_runner_; +}; + +VideoTrackReader* VideoTrackReader::Create(ScriptState* script_state, + MediaStreamTrack* track, + ExceptionState& exception_state) { + if (track->kind() != "video") { + exception_state.ThrowTypeError( + "Can only read video frames from video tracks."); + return nullptr; + } + + if (!script_state->ContextIsValid()) { // when the context is detached + exception_state.ThrowDOMException(DOMExceptionCode::kNotSupportedError, + "The context has been destroyed"); + + return nullptr; + } + + auto* source = + MakeGarbageCollected<VideoTrackReadableStreamSource>(script_state, track); + auto* readable = + ReadableStream::CreateWithCountQueueingStrategy(script_state, source, 0); + return MakeGarbageCollected<VideoTrackReader>(readable); +} + +VideoTrackReader::VideoTrackReader(ReadableStream* readable) + : readable_(readable) {} + +ReadableStream* VideoTrackReader::readable() const { + return readable_; +} + +void VideoTrackReader::Trace(Visitor* visitor) { + visitor->Trace(readable_); + ScriptWrappable::Trace(visitor); +} + +} // namespace blink diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/video_track_reader.h b/chromium/third_party/blink/renderer/modules/webcodecs/video_track_reader.h new file mode 100644 index 00000000000..1928a08ba6f --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/video_track_reader.h @@ -0,0 +1,38 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_TRACK_READER_H_ +#define THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_TRACK_READER_H_ + +#include "third_party/blink/renderer/modules/mediastream/media_stream_track.h" +#include "third_party/blink/renderer/modules/modules_export.h" +#include "third_party/blink/renderer/platform/bindings/script_wrappable.h" + +namespace blink { + +class ReadableStream; +class ScriptState; + +class MODULES_EXPORT VideoTrackReader final : public ScriptWrappable { + DEFINE_WRAPPERTYPEINFO(); + + public: + static VideoTrackReader* Create(ScriptState* script_state, + MediaStreamTrack* track, + ExceptionState& exception_state); + explicit VideoTrackReader(ReadableStream* readable); + ReadableStream* readable() const; + + void Trace(Visitor* visitor) override; + + private: + VideoTrackReader(const VideoTrackReader&) = delete; + VideoTrackReader& operator=(const VideoTrackReader&) = delete; + + Member<ReadableStream> readable_; +}; + +} // namespace blink + +#endif // THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_TRACK_READER_H_ diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/video_track_reader.idl b/chromium/third_party/blink/renderer/modules/webcodecs/video_track_reader.idl new file mode 100644 index 00000000000..bb5aded4d5f --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/video_track_reader.idl @@ -0,0 +1,13 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// See spec in developement at https://github.com/WICG/web-codecs/blob/master/index.bs +// and https://github.com/WICG/web-codecs/blob/master/explainer.md. +[ + RuntimeEnabled=WebCodecs +] interface VideoTrackReader { + [CallWith=ScriptState, RaisesException] + constructor(MediaStreamTrack track); + readonly attribute ReadableStream readable; // of VideoFrame +};
\ No newline at end of file diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/video_track_reader_writer_test.cc b/chromium/third_party/blink/renderer/modules/webcodecs/video_track_reader_writer_test.cc new file mode 100644 index 00000000000..f576da4559f --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/video_track_reader_writer_test.cc @@ -0,0 +1,133 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "base/run_loop.h" +#include "testing/gtest/include/gtest/gtest.h" +#include "third_party/blink/renderer/bindings/core/v8/v8_binding_for_testing.h" +#include "third_party/blink/renderer/bindings/modules/v8/v8_video_frame.h" +#include "third_party/blink/renderer/bindings/modules/v8/v8_video_track_writer_parameters.h" +#include "third_party/blink/renderer/core/streams/readable_stream.h" +#include "third_party/blink/renderer/core/streams/writable_stream.h" +#include "third_party/blink/renderer/core/streams/writable_stream_default_writer.h" +#include "third_party/blink/renderer/modules/webcodecs/video_frame.h" +#include "third_party/blink/renderer/modules/webcodecs/video_track_reader.h" +#include "third_party/blink/renderer/modules/webcodecs/video_track_writer.h" +#include "third_party/blink/renderer/platform/bindings/exception_state.h" +#include "third_party/blink/renderer/platform/bindings/script_state.h" +#include "third_party/blink/renderer/platform/testing/io_task_runner_testing_platform_support.h" + +namespace blink { + +class VideoTrackReaderWriterTest : public testing::Test { + public: + void TearDown() override { + RunIOUntilIdle(); + ThreadState::Current()->CollectAllGarbageForTesting(); + } + + protected: + VideoFrame* CreateBlackVideoFrame() { + return MakeGarbageCollected<VideoFrame>( + media::VideoFrame::CreateBlackFrame(gfx::Size(100, 100))); + } + + void RunIOUntilIdle() const { + // Tracks use the IO thread to send frames to sinks. Make sure that + // tasks on IO thread are completed before moving on. + base::RunLoop run_loop; + platform_->GetIOTaskRunner()->PostTaskAndReply( + FROM_HERE, base::BindOnce([] {}), run_loop.QuitClosure()); + run_loop.Run(); + base::RunLoop().RunUntilIdle(); + } + + private: + ScopedTestingPlatformSupport<IOTaskRunnerTestingPlatformSupport> platform_; +}; + +TEST_F(VideoTrackReaderWriterTest, WriteAndRead) { + V8TestingScope scope; + auto* script_state = scope.GetScriptState(); + + VideoTrackWriterParameters params; + params.setReleaseFrames(false); + auto* writer = + VideoTrackWriter::Create(script_state, ¶ms, ASSERT_NO_EXCEPTION); + auto* reader = VideoTrackReader::Create(script_state, writer->track(), + ASSERT_NO_EXCEPTION); + + auto* frame = CreateBlackVideoFrame(); + writer->writable() + ->getWriter(script_state, ASSERT_NO_EXCEPTION) + ->write(script_state, + ScriptValue(scope.GetIsolate(), ToV8(frame, script_state)), + ASSERT_NO_EXCEPTION); + + auto read_promise = reader->readable() + ->getReader(script_state, ASSERT_NO_EXCEPTION) + ->read(script_state, ASSERT_NO_EXCEPTION); + auto v8_read_promise = read_promise.V8Value().As<v8::Promise>(); + + EXPECT_EQ(v8::Promise::kPending, v8_read_promise->State()); + + RunIOUntilIdle(); + + EXPECT_EQ(v8::Promise::kFulfilled, v8_read_promise->State()); + + auto* read_frame = V8VideoFrame::ToImplWithTypeCheck( + scope.GetIsolate(), + v8_read_promise->Result() + ->ToObject(scope.GetContext()) + .ToLocalChecked() + ->Get(scope.GetContext(), V8String(scope.GetIsolate(), "value")) + .ToLocalChecked()); + + ASSERT_TRUE(frame); + EXPECT_EQ(frame->frame(), read_frame->frame()); + + // Auto-release turned off + EXPECT_NE(nullptr, frame->frame()); +} + +TEST_F(VideoTrackReaderWriterTest, AutoRelease) { + V8TestingScope scope; + auto* script_state = scope.GetScriptState(); + + VideoTrackWriterParameters params; + params.setReleaseFrames(true); + auto* writer = + VideoTrackWriter::Create(script_state, ¶ms, ASSERT_NO_EXCEPTION); + + auto* frame = CreateBlackVideoFrame(); + writer->writable() + ->getWriter(script_state, ASSERT_NO_EXCEPTION) + ->write(script_state, + ScriptValue(scope.GetIsolate(), ToV8(frame, script_state)), + ASSERT_NO_EXCEPTION); + + RunIOUntilIdle(); + + // Auto-release worked + EXPECT_EQ(nullptr, frame->frame()); +} + +TEST_F(VideoTrackReaderWriterTest, Abort) { + V8TestingScope scope; + auto* script_state = scope.GetScriptState(); + + VideoTrackWriterParameters params; + params.setReleaseFrames(false); + auto* writer = + VideoTrackWriter::Create(script_state, ¶ms, ASSERT_NO_EXCEPTION); + + EXPECT_EQ(writer->track()->readyState(), "live"); + + writer->writable()->abort(script_state, ASSERT_NO_EXCEPTION); + + RunIOUntilIdle(); + + EXPECT_TRUE(writer->track()->Ended()); +} + +} // namespace blink diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/video_track_writer.cc b/chromium/third_party/blink/renderer/modules/webcodecs/video_track_writer.cc new file mode 100644 index 00000000000..1d3089a2c52 --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/video_track_writer.cc @@ -0,0 +1,194 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this Sink code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include "third_party/blink/renderer/modules/webcodecs/video_track_writer.h" + +#include "media/base/video_frame.h" +#include "third_party/blink/public/mojom/mediastream/media_stream.mojom-blink.h" +#include "third_party/blink/public/web/modules/mediastream/media_stream_video_source.h" +#include "third_party/blink/public/web/modules/mediastream/media_stream_video_track.h" +#include "third_party/blink/renderer/bindings/modules/v8/v8_video_frame.h" +#include "third_party/blink/renderer/bindings/modules/v8/v8_video_track_writer_parameters.h" +#include "third_party/blink/renderer/core/execution_context/execution_context.h" +#include "third_party/blink/renderer/core/streams/underlying_sink_base.h" +#include "third_party/blink/renderer/core/streams/writable_stream.h" +#include "third_party/blink/renderer/core/streams/writable_stream_default_controller.h" +#include "third_party/blink/renderer/modules/mediastream/media_stream_track.h" +#include "third_party/blink/renderer/modules/webcodecs/video_frame.h" +#include "third_party/blink/renderer/platform/bindings/exception_state.h" +#include "third_party/blink/renderer/platform/bindings/script_state.h" +#include "third_party/blink/renderer/platform/scheduler/public/post_cross_thread_task.h" +#include "third_party/blink/renderer/platform/wtf/cross_thread_functional.h" +#include "third_party/blink/renderer/platform/wtf/uuid.h" + +namespace blink { + +// Simplifies the creation of video tracks. Just do this: +// auto source = std::make_unique<PushableVideoTrackSource>(); +// auto* track = CreateVideoTrackFromSource(script_state, source); +// for each frame: +// source->PushFrame(video_frame, capture_time); +// source->Stop(); +class PushableVideoTrackSource : public MediaStreamVideoSource { + public: + void PushFrame(scoped_refptr<media::VideoFrame> video_frame, + base::TimeTicks estimated_capture_time) { + if (!running_) + return; + + // Note that although use of the IO thread is rare in blink, it's required + // by any implementation of MediaStreamVideoSource, which is made clear by + // the documentation of MediaStreamVideoSource::StartSourceImpl which reads + // "An implementation must call |frame_callback| on the IO thread." + // Also see the DCHECK at VideoTrackAdapter::DeliverFrameOnIO + // and the other of implementations of MediaStreamVideoSource at + // MediaStreamRemoteVideoSource::StartSourceImpl, + // CastReceiverSession::StartVideo, + // CanvasCaptureHandler::SendFrame, + // and HtmlVideoElementCapturerSource::sendNewFrame. + PostCrossThreadTask( + *io_task_runner(), FROM_HERE, + CrossThreadBindOnce(deliver_frame_cb_, std::move(video_frame), + estimated_capture_time)); + } + + void Stop() { + DoStopSource(); + running_ = false; + } + + void StartSourceImpl(VideoCaptureDeliverFrameCB frame_callback, + EncodedVideoFrameCB encoded_frame_callback) override { + DCHECK(frame_callback); + running_ = true; + deliver_frame_cb_ = frame_callback; + OnStartDone(mojom::blink::MediaStreamRequestResult::OK); + } + + void StopSourceImpl() override { running_ = false; } + + private: + bool running_ = false; + VideoCaptureDeliverFrameCB deliver_frame_cb_; +}; + +// Implements a WritableStream's UnderlyingSinkBase by pushing frames into a +// PushableVideoTrackSource. Also optionally releases the frames. +class VideoTrackWritableStreamSink final : public UnderlyingSinkBase { + public: + // The source must out live the sink. + VideoTrackWritableStreamSink(PushableVideoTrackSource* source, + bool release_frames) + : source_(source), release_frames_(release_frames) {} + + // UnderlyingSinkBase overrides. + ScriptPromise start(ScriptState* script_state, + WritableStreamDefaultController* controller, + ExceptionState& exception_state) override { + // We're ready write away + return ScriptPromise::CastUndefined(script_state); + } + + ScriptPromise write(ScriptState* script_state, + ScriptValue chunk, + WritableStreamDefaultController* controller, + ExceptionState& exception_state) override { + VideoFrame* video_frame = V8VideoFrame::ToImplWithTypeCheck( + script_state->GetIsolate(), chunk.V8Value()); + if (!video_frame) { + exception_state.ThrowTypeError("Provided chunk is not a VideoFrame."); + return ScriptPromise(); + } + + base::TimeTicks estimated_capture_time = base::TimeTicks::Now(); + source_->PushFrame(video_frame->frame(), estimated_capture_time); + + if (release_frames_) + video_frame->release(); + + return ScriptPromise::CastUndefined(script_state); + } + + ScriptPromise abort(ScriptState* script_state, + ScriptValue reason, + ExceptionState& exception_state) override { + source_->Stop(); + return ScriptPromise::CastUndefined(script_state); + } + + ScriptPromise close(ScriptState* script_state, + ExceptionState& exception_state) override { + source_->Stop(); + return ScriptPromise::CastUndefined(script_state); + } + + PushableVideoTrackSource* source_; + bool release_frames_; +}; + +MediaStreamTrack* CreateVideoTrackFromSource( + ScriptState* script_state, + std::unique_ptr<MediaStreamVideoSource> video_source) { + // Get "video_source.get()" before std::move(source) into owner. + auto* video_source_ptr = video_source.get(); + + String track_id = WTF::CreateCanonicalUUIDString(); + MediaStreamSource* video_source_owner = + MakeGarbageCollected<MediaStreamSource>( + track_id, MediaStreamSource::kTypeVideo, track_id /* name */, + false /* remote */); + video_source->SetOwner(video_source_owner); + video_source_owner->SetPlatformSource(std::move(video_source)); + + return MakeGarbageCollected<MediaStreamTrack>( + ExecutionContext::From(script_state), + MediaStreamVideoTrack::CreateVideoTrack( + video_source_ptr, MediaStreamVideoSource::ConstraintsOnceCallback(), + true /* enabled */)); +} + +VideoTrackWriter* VideoTrackWriter::Create( + ScriptState* script_state, + const VideoTrackWriterParameters* params, + ExceptionState& exception_state) { + if (!script_state->ContextIsValid()) { // when the context is detached + exception_state.ThrowDOMException(DOMExceptionCode::kNotSupportedError, + "The context has been destroyed"); + + return nullptr; + } + + std::unique_ptr<PushableVideoTrackSource> track_source = + std::make_unique<PushableVideoTrackSource>(); + VideoTrackWritableStreamSink* writable_sink = + MakeGarbageCollected<VideoTrackWritableStreamSink>( + track_source.get(), params->releaseFrames()); + + auto* track = + CreateVideoTrackFromSource(script_state, std::move(track_source)); + auto* writable = WritableStream::Create( + script_state, ScriptValue::From(script_state, writable_sink), + exception_state); + return MakeGarbageCollected<VideoTrackWriter>(track, writable); +} + +VideoTrackWriter::VideoTrackWriter(MediaStreamTrack* track, + WritableStream* writable) + : track_(track), writable_(writable) {} + +WritableStream* VideoTrackWriter::writable() { + return writable_; +} + +MediaStreamTrack* VideoTrackWriter::track() { + return track_; +} + +void VideoTrackWriter::Trace(Visitor* visitor) { + visitor->Trace(track_); + visitor->Trace(writable_); + ScriptWrappable::Trace(visitor); +} + +} // namespace blink diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/video_track_writer.h b/chromium/third_party/blink/renderer/modules/webcodecs/video_track_writer.h new file mode 100644 index 00000000000..1b925f6b80c --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/video_track_writer.h @@ -0,0 +1,42 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_TRACK_WRITER_H_ +#define THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_TRACK_WRITER_H_ + +#include "third_party/blink/renderer/modules/mediastream/media_stream_track.h" +#include "third_party/blink/renderer/modules/modules_export.h" +#include "third_party/blink/renderer/platform/bindings/script_wrappable.h" + +namespace blink { + +class ScriptState; +class WritableStream; +class VideoTrackWriterParameters; + +class MODULES_EXPORT VideoTrackWriter final : public ScriptWrappable { + DEFINE_WRAPPERTYPEINFO(); + + public: + // video_track_writer.idl implementation + static VideoTrackWriter* Create(ScriptState* script_state, + const VideoTrackWriterParameters* params, + ExceptionState& exception_state); + VideoTrackWriter(MediaStreamTrack* track, WritableStream* writable); + MediaStreamTrack* track(); + WritableStream* writable(); + + // GarbageCollected override + void Trace(Visitor* visitor) override; + + private: + Member<MediaStreamTrack> track_; + Member<WritableStream> writable_; + + DISALLOW_COPY_AND_ASSIGN(VideoTrackWriter); +}; + +} // namespace blink + +#endif // THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_TRACK_WRITER_H_ diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/video_track_writer.idl b/chromium/third_party/blink/renderer/modules/webcodecs/video_track_writer.idl new file mode 100644 index 00000000000..aa5d3fd1fe2 --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/video_track_writer.idl @@ -0,0 +1,14 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// See spec in developement at https://github.com/WICG/web-codecs/blob/master/index.bs +// and https://github.com/WICG/web-codecs/blob/master/explainer.md. +[ + RuntimeEnabled=WebCodecs +] interface VideoTrackWriter { + [CallWith=ScriptState, RaisesException] + constructor(VideoTrackWriterParameters params); + readonly attribute WritableStream writable; // of VideoFrame + readonly attribute MediaStreamTrack track; +};
\ No newline at end of file diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/video_track_writer_parameters.idl b/chromium/third_party/blink/renderer/modules/webcodecs/video_track_writer_parameters.idl new file mode 100644 index 00000000000..7687d4949b3 --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/video_track_writer_parameters.idl @@ -0,0 +1,9 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// See spec in developement at https://github.com/WICG/web-codecs/blob/master/index.bs +// and https://github.com/WICG/web-codecs/blob/master/explainer.md. +dictionary VideoTrackWriterParameters { + boolean releaseFrames = true; +};
\ No newline at end of file diff --git a/chromium/third_party/blink/renderer/modules/webcodecs/webcodecs_error_callback.idl b/chromium/third_party/blink/renderer/modules/webcodecs/webcodecs_error_callback.idl new file mode 100644 index 00000000000..4dcaf89f186 --- /dev/null +++ b/chromium/third_party/blink/renderer/modules/webcodecs/webcodecs_error_callback.idl @@ -0,0 +1,8 @@ +// Copyright 2020 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// https://github.com/WICG/web-codecs + +[RuntimeEnabled=WebCodecs] +callback WebCodecsErrorCallback = void(DOMException error); |