diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2018-08-28 15:28:34 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2018-08-28 13:54:51 +0000 |
commit | 2a19c63448c84c1805fb1a585c3651318bb86ca7 (patch) | |
tree | eb17888e8531aa6ee5e85721bd553b832a7e5156 /chromium/third_party/webrtc/video/video_quality_test.cc | |
parent | b014812705fc80bff0a5c120dfcef88f349816dc (diff) | |
download | qtwebengine-chromium-2a19c63448c84c1805fb1a585c3651318bb86ca7.tar.gz |
BASELINE: Update Chromium to 69.0.3497.70
Change-Id: I2b7b56e4e7a8b26656930def0d4575dc32b900a0
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>
Diffstat (limited to 'chromium/third_party/webrtc/video/video_quality_test.cc')
-rw-r--r-- | chromium/third_party/webrtc/video/video_quality_test.cc | 1423 |
1 files changed, 122 insertions, 1301 deletions
diff --git a/chromium/third_party/webrtc/video/video_quality_test.cc b/chromium/third_party/webrtc/video/video_quality_test.cc index b87e6040555..abc8376704f 100644 --- a/chromium/third_party/webrtc/video/video_quality_test.cc +++ b/chromium/third_party/webrtc/video/video_quality_test.cc @@ -22,39 +22,16 @@ #include "media/engine/vp8_encoder_simulcast_proxy.h" #include "media/engine/webrtcvideoengine.h" #include "modules/audio_mixer/audio_mixer_impl.h" -#include "modules/rtp_rtcp/source/rtp_format.h" -#include "modules/rtp_rtcp/source/rtp_utility.h" #include "modules/video_coding/codecs/h264/include/h264.h" #include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h" #include "modules/video_coding/codecs/vp8/include/vp8.h" -#include "modules/video_coding/codecs/vp8/include/vp8_common_types.h" #include "modules/video_coding/codecs/vp9/include/vp9.h" -#include "rtc_base/cpu_time.h" -#include "rtc_base/flags.h" -#include "rtc_base/format_macros.h" -#include "rtc_base/memory_usage.h" -#include "rtc_base/pathutils.h" -#include "system_wrappers/include/cpu_info.h" -#include "test/rtp_file_writer.h" #include "test/run_loop.h" -#include "test/statistics.h" #include "test/testsupport/fileutils.h" -#include "test/testsupport/frame_writer.h" -#include "test/testsupport/perf_test.h" -#include "test/testsupport/test_artifacts.h" -#include "test/vcm_capturer.h" #include "test/video_renderer.h" - -DEFINE_bool(save_worst_frame, - false, - "Enable saving a frame with the lowest PSNR to a jpeg file in the " - "test_artifacts_dir"); +#include "video/video_analyzer.h" namespace { - -constexpr int kSendStatsPollingIntervalMs = 1000; - -constexpr size_t kMaxComparisons = 10; constexpr char kSyncGroup[] = "av_sync"; constexpr int kOpusMinBitrateBps = 6000; constexpr int kOpusBitrateFbBps = 32000; @@ -86,1027 +63,30 @@ class VideoStreamFactory std::vector<webrtc::VideoStream> streams_; }; - -bool IsFlexfec(int payload_type) { - return payload_type == webrtc::VideoQualityTest::kFlexfecPayloadType; -} - } // namespace namespace webrtc { -class VideoAnalyzer : public PacketReceiver, - public Transport, - public rtc::VideoSinkInterface<VideoFrame> { - public: - VideoAnalyzer(test::LayerFilteringTransport* transport, - const std::string& test_label, - double avg_psnr_threshold, - double avg_ssim_threshold, - int duration_frames, - FILE* graph_data_output_file, - const std::string& graph_title, - uint32_t ssrc_to_analyze, - uint32_t rtx_ssrc_to_analyze, - size_t selected_stream, - int selected_sl, - int selected_tl, - bool is_quick_test_enabled, - Clock* clock, - std::string rtp_dump_name) - : transport_(transport), - receiver_(nullptr), - call_(nullptr), - send_stream_(nullptr), - receive_stream_(nullptr), - captured_frame_forwarder_(this, clock), - test_label_(test_label), - graph_data_output_file_(graph_data_output_file), - graph_title_(graph_title), - ssrc_to_analyze_(ssrc_to_analyze), - rtx_ssrc_to_analyze_(rtx_ssrc_to_analyze), - selected_stream_(selected_stream), - selected_sl_(selected_sl), - selected_tl_(selected_tl), - pre_encode_proxy_(this), - encode_timing_proxy_(this), - last_fec_bytes_(0), - frames_to_process_(duration_frames), - frames_recorded_(0), - frames_processed_(0), - dropped_frames_(0), - dropped_frames_before_first_encode_(0), - dropped_frames_before_rendering_(0), - last_render_time_(0), - last_render_delta_ms_(0), - last_unfreeze_time_ms_(0), - rtp_timestamp_delta_(0), - total_media_bytes_(0), - first_sending_time_(0), - last_sending_time_(0), - cpu_time_(0), - wallclock_time_(0), - avg_psnr_threshold_(avg_psnr_threshold), - avg_ssim_threshold_(avg_ssim_threshold), - is_quick_test_enabled_(is_quick_test_enabled), - stats_polling_thread_(&PollStatsThread, this, "StatsPoller"), - comparison_available_event_(false, false), - done_(true, false), - clock_(clock), - start_ms_(clock->TimeInMilliseconds()) { - // Create thread pool for CPU-expensive PSNR/SSIM calculations. - - // Try to use about as many threads as cores, but leave kMinCoresLeft alone, - // so that we don't accidentally starve "real" worker threads (codec etc). - // Also, don't allocate more than kMaxComparisonThreads, even if there are - // spare cores. - - uint32_t num_cores = CpuInfo::DetectNumberOfCores(); - RTC_DCHECK_GE(num_cores, 1); - static const uint32_t kMinCoresLeft = 4; - static const uint32_t kMaxComparisonThreads = 8; - - if (num_cores <= kMinCoresLeft) { - num_cores = 1; - } else { - num_cores -= kMinCoresLeft; - num_cores = std::min(num_cores, kMaxComparisonThreads); - } - - for (uint32_t i = 0; i < num_cores; ++i) { - rtc::PlatformThread* thread = - new rtc::PlatformThread(&FrameComparisonThread, this, "Analyzer"); - thread->Start(); - comparison_thread_pool_.push_back(thread); - } - - if (!rtp_dump_name.empty()) { - fprintf(stdout, "Writing rtp dump to %s\n", rtp_dump_name.c_str()); - rtp_file_writer_.reset(test::RtpFileWriter::Create( - test::RtpFileWriter::kRtpDump, rtp_dump_name)); - } - } - - ~VideoAnalyzer() { - for (rtc::PlatformThread* thread : comparison_thread_pool_) { - thread->Stop(); - delete thread; - } - } - - virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; } - - void SetSource(test::VideoCapturer* video_capturer, bool respect_sink_wants) { - if (respect_sink_wants) - captured_frame_forwarder_.SetSource(video_capturer); - rtc::VideoSinkWants wants; - video_capturer->AddOrUpdateSink(InputInterface(), wants); - } - - void SetCall(Call* call) { - rtc::CritScope lock(&crit_); - RTC_DCHECK(!call_); - call_ = call; - } - - void SetSendStream(VideoSendStream* stream) { - rtc::CritScope lock(&crit_); - RTC_DCHECK(!send_stream_); - send_stream_ = stream; - } - - void SetReceiveStream(VideoReceiveStream* stream) { - rtc::CritScope lock(&crit_); - RTC_DCHECK(!receive_stream_); - receive_stream_ = stream; - } - - rtc::VideoSinkInterface<VideoFrame>* InputInterface() { - return &captured_frame_forwarder_; - } - rtc::VideoSourceInterface<VideoFrame>* OutputInterface() { - return &captured_frame_forwarder_; - } - - DeliveryStatus DeliverPacket(MediaType media_type, - rtc::CopyOnWriteBuffer packet, - const PacketTime& packet_time) override { - // Ignore timestamps of RTCP packets. They're not synchronized with - // RTP packet timestamps and so they would confuse wrap_handler_. - if (RtpHeaderParser::IsRtcp(packet.cdata(), packet.size())) { - return receiver_->DeliverPacket(media_type, std::move(packet), - packet_time); - } - - if (rtp_file_writer_) { - test::RtpPacket p; - memcpy(p.data, packet.cdata(), packet.size()); - p.length = packet.size(); - p.original_length = packet.size(); - p.time_ms = clock_->TimeInMilliseconds() - start_ms_; - rtp_file_writer_->WritePacket(&p); - } - - RtpUtility::RtpHeaderParser parser(packet.cdata(), packet.size()); - RTPHeader header; - parser.Parse(&header); - if (!IsFlexfec(header.payloadType) && - (header.ssrc == ssrc_to_analyze_ || - header.ssrc == rtx_ssrc_to_analyze_)) { - // Ignore FlexFEC timestamps, to avoid collisions with media timestamps. - // (FlexFEC and media are sent on different SSRCs, which have different - // timestamps spaces.) - // Also ignore packets from wrong SSRC, but include retransmits. - rtc::CritScope lock(&crit_); - int64_t timestamp = - wrap_handler_.Unwrap(header.timestamp - rtp_timestamp_delta_); - recv_times_[timestamp] = - Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); - } - - return receiver_->DeliverPacket(media_type, std::move(packet), packet_time); - } - - void PreEncodeOnFrame(const VideoFrame& video_frame) { - rtc::CritScope lock(&crit_); - if (!first_encoded_timestamp_) { - while (frames_.front().timestamp() != video_frame.timestamp()) { - ++dropped_frames_before_first_encode_; - frames_.pop_front(); - RTC_CHECK(!frames_.empty()); - } - first_encoded_timestamp_ = video_frame.timestamp(); - } - } - - void PostEncodeFrameCallback(const EncodedFrame& encoded_frame) { - rtc::CritScope lock(&crit_); - if (!first_sent_timestamp_ && - encoded_frame.stream_id_ == selected_stream_) { - first_sent_timestamp_ = encoded_frame.timestamp_; - } - } - - bool SendRtp(const uint8_t* packet, - size_t length, - const PacketOptions& options) override { - RtpUtility::RtpHeaderParser parser(packet, length); - RTPHeader header; - parser.Parse(&header); - - int64_t current_time = - Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); - - bool result = transport_->SendRtp(packet, length, options); - { - rtc::CritScope lock(&crit_); - if (rtp_timestamp_delta_ == 0 && header.ssrc == ssrc_to_analyze_) { - RTC_CHECK(static_cast<bool>(first_sent_timestamp_)); - rtp_timestamp_delta_ = header.timestamp - *first_sent_timestamp_; - } - - if (!IsFlexfec(header.payloadType) && header.ssrc == ssrc_to_analyze_) { - // Ignore FlexFEC timestamps, to avoid collisions with media timestamps. - // (FlexFEC and media are sent on different SSRCs, which have different - // timestamps spaces.) - // Also ignore packets from wrong SSRC and retransmits. - int64_t timestamp = - wrap_handler_.Unwrap(header.timestamp - rtp_timestamp_delta_); - send_times_[timestamp] = current_time; - - if (IsInSelectedSpatialAndTemporalLayer(packet, length, header)) { - encoded_frame_sizes_[timestamp] += - length - (header.headerLength + header.paddingLength); - total_media_bytes_ += - length - (header.headerLength + header.paddingLength); - } - if (first_sending_time_ == 0) - first_sending_time_ = current_time; - last_sending_time_ = current_time; - } - } - return result; - } - - bool SendRtcp(const uint8_t* packet, size_t length) override { - return transport_->SendRtcp(packet, length); - } - - void OnFrame(const VideoFrame& video_frame) override { - int64_t render_time_ms = - Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); - - rtc::CritScope lock(&crit_); - - StartExcludingCpuThreadTime(); - - int64_t send_timestamp = - wrap_handler_.Unwrap(video_frame.timestamp() - rtp_timestamp_delta_); - - while (wrap_handler_.Unwrap(frames_.front().timestamp()) < send_timestamp) { - if (!last_rendered_frame_) { - // No previous frame rendered, this one was dropped after sending but - // before rendering. - ++dropped_frames_before_rendering_; - } else { - AddFrameComparison(frames_.front(), *last_rendered_frame_, true, - render_time_ms); - } - frames_.pop_front(); - RTC_DCHECK(!frames_.empty()); - } - - VideoFrame reference_frame = frames_.front(); - frames_.pop_front(); - int64_t reference_timestamp = - wrap_handler_.Unwrap(reference_frame.timestamp()); - if (send_timestamp == reference_timestamp - 1) { - // TODO(ivica): Make this work for > 2 streams. - // Look at RTPSender::BuildRTPHeader. - ++send_timestamp; - } - ASSERT_EQ(reference_timestamp, send_timestamp); - - AddFrameComparison(reference_frame, video_frame, false, render_time_ms); - - last_rendered_frame_ = video_frame; - - StopExcludingCpuThreadTime(); - } - - void Wait() { - // Frame comparisons can be very expensive. Wait for test to be done, but - // at time-out check if frames_processed is going up. If so, give it more - // time, otherwise fail. Hopefully this will reduce test flakiness. - - stats_polling_thread_.Start(); - - int last_frames_processed = -1; - int iteration = 0; - while (!done_.Wait(VideoQualityTest::kDefaultTimeoutMs)) { - int frames_processed; - { - rtc::CritScope crit(&comparison_lock_); - frames_processed = frames_processed_; - } - - // Print some output so test infrastructure won't think we've crashed. - const char* kKeepAliveMessages[3] = { - "Uh, I'm-I'm not quite dead, sir.", - "Uh, I-I think uh, I could pull through, sir.", - "Actually, I think I'm all right to come with you--"}; - printf("- %s\n", kKeepAliveMessages[iteration++ % 3]); - - if (last_frames_processed == -1) { - last_frames_processed = frames_processed; - continue; - } - if (frames_processed == last_frames_processed) { - EXPECT_GT(frames_processed, last_frames_processed) - << "Analyzer stalled while waiting for test to finish."; - done_.Set(); - break; - } - last_frames_processed = frames_processed; - } - - if (iteration > 0) - printf("- Farewell, sweet Concorde!\n"); - - stats_polling_thread_.Stop(); - } - - rtc::VideoSinkInterface<VideoFrame>* pre_encode_proxy() { - return &pre_encode_proxy_; - } - EncodedFrameObserver* encode_timing_proxy() { return &encode_timing_proxy_; } - - void StartMeasuringCpuProcessTime() { - rtc::CritScope lock(&cpu_measurement_lock_); - cpu_time_ -= rtc::GetProcessCpuTimeNanos(); - wallclock_time_ -= rtc::SystemTimeNanos(); - } - - void StopMeasuringCpuProcessTime() { - rtc::CritScope lock(&cpu_measurement_lock_); - cpu_time_ += rtc::GetProcessCpuTimeNanos(); - wallclock_time_ += rtc::SystemTimeNanos(); - } - - void StartExcludingCpuThreadTime() { - rtc::CritScope lock(&cpu_measurement_lock_); - cpu_time_ += rtc::GetThreadCpuTimeNanos(); - } - - void StopExcludingCpuThreadTime() { - rtc::CritScope lock(&cpu_measurement_lock_); - cpu_time_ -= rtc::GetThreadCpuTimeNanos(); - } - - double GetCpuUsagePercent() { - rtc::CritScope lock(&cpu_measurement_lock_); - return static_cast<double>(cpu_time_) / wallclock_time_ * 100.0; - } - - test::LayerFilteringTransport* const transport_; - PacketReceiver* receiver_; - - private: - struct FrameComparison { - FrameComparison() - : dropped(false), - input_time_ms(0), - send_time_ms(0), - recv_time_ms(0), - render_time_ms(0), - encoded_frame_size(0) {} - - FrameComparison(const VideoFrame& reference, - const VideoFrame& render, - bool dropped, - int64_t input_time_ms, - int64_t send_time_ms, - int64_t recv_time_ms, - int64_t render_time_ms, - size_t encoded_frame_size) - : reference(reference), - render(render), - dropped(dropped), - input_time_ms(input_time_ms), - send_time_ms(send_time_ms), - recv_time_ms(recv_time_ms), - render_time_ms(render_time_ms), - encoded_frame_size(encoded_frame_size) {} - - FrameComparison(bool dropped, - int64_t input_time_ms, - int64_t send_time_ms, - int64_t recv_time_ms, - int64_t render_time_ms, - size_t encoded_frame_size) - : dropped(dropped), - input_time_ms(input_time_ms), - send_time_ms(send_time_ms), - recv_time_ms(recv_time_ms), - render_time_ms(render_time_ms), - encoded_frame_size(encoded_frame_size) {} - - rtc::Optional<VideoFrame> reference; - rtc::Optional<VideoFrame> render; - bool dropped; - int64_t input_time_ms; - int64_t send_time_ms; - int64_t recv_time_ms; - int64_t render_time_ms; - size_t encoded_frame_size; - }; - - struct Sample { - Sample(int dropped, - int64_t input_time_ms, - int64_t send_time_ms, - int64_t recv_time_ms, - int64_t render_time_ms, - size_t encoded_frame_size, - double psnr, - double ssim) - : dropped(dropped), - input_time_ms(input_time_ms), - send_time_ms(send_time_ms), - recv_time_ms(recv_time_ms), - render_time_ms(render_time_ms), - encoded_frame_size(encoded_frame_size), - psnr(psnr), - ssim(ssim) {} - - int dropped; - int64_t input_time_ms; - int64_t send_time_ms; - int64_t recv_time_ms; - int64_t render_time_ms; - size_t encoded_frame_size; - double psnr; - double ssim; - }; - - // This class receives the send-side OnEncodeTiming and is provided to not - // conflict with the receiver-side pre_decode_callback. - class OnEncodeTimingProxy : public EncodedFrameObserver { - public: - explicit OnEncodeTimingProxy(VideoAnalyzer* parent) : parent_(parent) {} - - void EncodedFrameCallback(const EncodedFrame& frame) override { - parent_->PostEncodeFrameCallback(frame); - } - - private: - VideoAnalyzer* const parent_; - }; - - // This class receives the send-side OnFrame callback and is provided to not - // conflict with the receiver-side renderer callback. - class PreEncodeProxy : public rtc::VideoSinkInterface<VideoFrame> { - public: - explicit PreEncodeProxy(VideoAnalyzer* parent) : parent_(parent) {} - - void OnFrame(const VideoFrame& video_frame) override { - parent_->PreEncodeOnFrame(video_frame); - } - - private: - VideoAnalyzer* const parent_; - }; - - bool IsInSelectedSpatialAndTemporalLayer(const uint8_t* packet, - size_t length, - const RTPHeader& header) { - if (header.payloadType != test::CallTest::kPayloadTypeVP9 && - header.payloadType != test::CallTest::kPayloadTypeVP8) { - return true; - } else { - // Get VP8 and VP9 specific header to check layers indexes. - const uint8_t* payload = packet + header.headerLength; - const size_t payload_length = length - header.headerLength; - const size_t payload_data_length = payload_length - header.paddingLength; - const bool is_vp8 = header.payloadType == test::CallTest::kPayloadTypeVP8; - std::unique_ptr<RtpDepacketizer> depacketizer( - RtpDepacketizer::Create(is_vp8 ? kRtpVideoVp8 : kRtpVideoVp9)); - RtpDepacketizer::ParsedPayload parsed_payload; - bool result = - depacketizer->Parse(&parsed_payload, payload, payload_data_length); - RTC_DCHECK(result); - const int temporal_idx = static_cast<int>( - is_vp8 ? parsed_payload.type.Video.codecHeader.VP8.temporalIdx - : parsed_payload.type.Video.codecHeader.VP9.temporal_idx); - const int spatial_idx = static_cast<int>( - is_vp8 ? kNoSpatialIdx - : parsed_payload.type.Video.codecHeader.VP9.spatial_idx); - return (selected_tl_ < 0 || temporal_idx == kNoTemporalIdx || - temporal_idx <= selected_tl_) && - (selected_sl_ < 0 || spatial_idx == kNoSpatialIdx || - spatial_idx <= selected_sl_); - } - } - - void AddFrameComparison(const VideoFrame& reference, - const VideoFrame& render, - bool dropped, - int64_t render_time_ms) - RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_) { - int64_t reference_timestamp = wrap_handler_.Unwrap(reference.timestamp()); - int64_t send_time_ms = send_times_[reference_timestamp]; - send_times_.erase(reference_timestamp); - int64_t recv_time_ms = recv_times_[reference_timestamp]; - recv_times_.erase(reference_timestamp); - - // TODO(ivica): Make this work for > 2 streams. - auto it = encoded_frame_sizes_.find(reference_timestamp); - if (it == encoded_frame_sizes_.end()) - it = encoded_frame_sizes_.find(reference_timestamp - 1); - size_t encoded_size = it == encoded_frame_sizes_.end() ? 0 : it->second; - if (it != encoded_frame_sizes_.end()) - encoded_frame_sizes_.erase(it); - - rtc::CritScope crit(&comparison_lock_); - if (comparisons_.size() < kMaxComparisons) { - comparisons_.push_back(FrameComparison(reference, render, dropped, - reference.ntp_time_ms(), - send_time_ms, recv_time_ms, - render_time_ms, encoded_size)); - } else { - comparisons_.push_back(FrameComparison(dropped, - reference.ntp_time_ms(), - send_time_ms, recv_time_ms, - render_time_ms, encoded_size)); - } - comparison_available_event_.Set(); - } - - static void PollStatsThread(void* obj) { - static_cast<VideoAnalyzer*>(obj)->PollStats(); - } - - void PollStats() { - while (!done_.Wait(kSendStatsPollingIntervalMs)) { - rtc::CritScope crit(&comparison_lock_); - - Call::Stats call_stats = call_->GetStats(); - send_bandwidth_bps_.AddSample(call_stats.send_bandwidth_bps); - - VideoSendStream::Stats send_stats = send_stream_->GetStats(); - // It's not certain that we yet have estimates for any of these stats. - // Check that they are positive before mixing them in. - if (send_stats.encode_frame_rate > 0) - encode_frame_rate_.AddSample(send_stats.encode_frame_rate); - if (send_stats.avg_encode_time_ms > 0) - encode_time_ms_.AddSample(send_stats.avg_encode_time_ms); - if (send_stats.encode_usage_percent > 0) - encode_usage_percent_.AddSample(send_stats.encode_usage_percent); - if (send_stats.media_bitrate_bps > 0) - media_bitrate_bps_.AddSample(send_stats.media_bitrate_bps); - size_t fec_bytes = 0; - for (auto kv : send_stats.substreams) { - fec_bytes += kv.second.rtp_stats.fec.payload_bytes + - kv.second.rtp_stats.fec.padding_bytes; - } - fec_bitrate_bps_.AddSample((fec_bytes - last_fec_bytes_) * 8); - last_fec_bytes_ = fec_bytes; - - if (receive_stream_ != nullptr) { - VideoReceiveStream::Stats receive_stats = receive_stream_->GetStats(); - if (receive_stats.decode_ms > 0) - decode_time_ms_.AddSample(receive_stats.decode_ms); - if (receive_stats.max_decode_ms > 0) - decode_time_max_ms_.AddSample(receive_stats.max_decode_ms); - } - - memory_usage_.AddSample(rtc::GetProcessResidentSizeBytes()); - } - } - - static bool FrameComparisonThread(void* obj) { - return static_cast<VideoAnalyzer*>(obj)->CompareFrames(); - } - - bool CompareFrames() { - if (AllFramesRecorded()) - return false; - - FrameComparison comparison; - - if (!PopComparison(&comparison)) { - // Wait until new comparison task is available, or test is done. - // If done, wake up remaining threads waiting. - comparison_available_event_.Wait(1000); - if (AllFramesRecorded()) { - comparison_available_event_.Set(); - return false; - } - return true; // Try again. - } - - StartExcludingCpuThreadTime(); - - PerformFrameComparison(comparison); - - StopExcludingCpuThreadTime(); - - if (FrameProcessed()) { - PrintResults(); - if (graph_data_output_file_) - PrintSamplesToFile(); - done_.Set(); - comparison_available_event_.Set(); - return false; - } - - return true; - } - - bool PopComparison(FrameComparison* comparison) { - rtc::CritScope crit(&comparison_lock_); - // If AllFramesRecorded() is true, it means we have already popped - // frames_to_process_ frames from comparisons_, so there is no more work - // for this thread to be done. frames_processed_ might still be lower if - // all comparisons are not done, but those frames are currently being - // worked on by other threads. - if (comparisons_.empty() || AllFramesRecorded()) - return false; - - *comparison = comparisons_.front(); - comparisons_.pop_front(); - - FrameRecorded(); - return true; - } - - // Increment counter for number of frames received for comparison. - void FrameRecorded() { - rtc::CritScope crit(&comparison_lock_); - ++frames_recorded_; - } - - // Returns true if all frames to be compared have been taken from the queue. - bool AllFramesRecorded() { - rtc::CritScope crit(&comparison_lock_); - assert(frames_recorded_ <= frames_to_process_); - return frames_recorded_ == frames_to_process_; - } - - // Increase count of number of frames processed. Returns true if this was the - // last frame to be processed. - bool FrameProcessed() { - rtc::CritScope crit(&comparison_lock_); - ++frames_processed_; - assert(frames_processed_ <= frames_to_process_); - return frames_processed_ == frames_to_process_; - } - - void PrintResults() { - StopMeasuringCpuProcessTime(); - rtc::CritScope crit(&comparison_lock_); - // Record the time from the last freeze until the last rendered frame to - // ensure we cover the full timespan of the session. Otherwise the metric - // would penalize an early freeze followed by no freezes until the end. - time_between_freezes_.AddSample(last_render_time_ - last_unfreeze_time_ms_); - PrintResult("psnr", psnr_, " dB"); - PrintResult("ssim", ssim_, " score"); - PrintResult("sender_time", sender_time_, " ms"); - PrintResult("receiver_time", receiver_time_, " ms"); - PrintResult("network_time", network_time_, " ms"); - PrintResult("total_delay_incl_network", end_to_end_, " ms"); - PrintResult("time_between_rendered_frames", rendered_delta_, " ms"); - PrintResult("encode_frame_rate", encode_frame_rate_, " fps"); - PrintResult("encode_time", encode_time_ms_, " ms"); - PrintResult("media_bitrate", media_bitrate_bps_, " bps"); - PrintResult("fec_bitrate", fec_bitrate_bps_, " bps"); - PrintResult("send_bandwidth", send_bandwidth_bps_, " bps"); - PrintResult("time_between_freezes", time_between_freezes_, " ms"); - - if (worst_frame_) { - test::PrintResult("min_psnr", "", test_label_.c_str(), worst_frame_->psnr, - "dB", false); - } - - if (receive_stream_ != nullptr) { - PrintResult("decode_time", decode_time_ms_, " ms"); - } - - test::PrintResult("dropped_frames", "", test_label_.c_str(), - dropped_frames_, "frames", false); - test::PrintResult("cpu_usage", "", test_label_.c_str(), - GetCpuUsagePercent(), "%", false); - -#if defined(WEBRTC_WIN) - // On Linux and Mac in Resident Set some unused pages may be counted. - // Therefore this metric will depend on order in which tests are run and - // will be flaky. - PrintResult("memory_usage", memory_usage_, " bytes"); -#endif - - // Saving only the worst frame for manual analysis. Intention here is to - // only detect video corruptions and not to track picture quality. Thus, - // jpeg is used here. - if (FLAG_save_worst_frame && worst_frame_) { - std::string output_dir; - test::GetTestArtifactsDir(&output_dir); - std::string output_path = - rtc::Pathname(output_dir, test_label_ + ".jpg").pathname(); - RTC_LOG(LS_INFO) << "Saving worst frame to " << output_path; - test::JpegFrameWriter frame_writer(output_path); - RTC_CHECK(frame_writer.WriteFrame(worst_frame_->frame, - 100 /*best quality*/)); - } - - // Disable quality check for quick test, as quality checks may fail - // because too few samples were collected. - if (!is_quick_test_enabled_) { - EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_); - EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_); - } - } - - void PerformFrameComparison(const FrameComparison& comparison) { - // Perform expensive psnr and ssim calculations while not holding lock. - double psnr = -1.0; - double ssim = -1.0; - if (comparison.reference && !comparison.dropped) { - psnr = I420PSNR(&*comparison.reference, &*comparison.render); - ssim = I420SSIM(&*comparison.reference, &*comparison.render); - } - - rtc::CritScope crit(&comparison_lock_); - - if (psnr >= 0.0 && (!worst_frame_ || worst_frame_->psnr > psnr)) { - worst_frame_.emplace(FrameWithPsnr{psnr, *comparison.render}); - } - - if (graph_data_output_file_) { - samples_.push_back(Sample( - comparison.dropped, comparison.input_time_ms, comparison.send_time_ms, - comparison.recv_time_ms, comparison.render_time_ms, - comparison.encoded_frame_size, psnr, ssim)); - } - if (psnr >= 0.0) - psnr_.AddSample(psnr); - if (ssim >= 0.0) - ssim_.AddSample(ssim); - - if (comparison.dropped) { - ++dropped_frames_; - return; - } - if (last_unfreeze_time_ms_ == 0) - last_unfreeze_time_ms_ = comparison.render_time_ms; - if (last_render_time_ != 0) { - const int64_t render_delta_ms = - comparison.render_time_ms - last_render_time_; - rendered_delta_.AddSample(render_delta_ms); - if (last_render_delta_ms_ != 0 && - render_delta_ms - last_render_delta_ms_ > 150) { - time_between_freezes_.AddSample(last_render_time_ - - last_unfreeze_time_ms_); - last_unfreeze_time_ms_ = comparison.render_time_ms; - } - last_render_delta_ms_ = render_delta_ms; - } - last_render_time_ = comparison.render_time_ms; - - sender_time_.AddSample(comparison.send_time_ms - comparison.input_time_ms); - if (comparison.recv_time_ms > 0) { - // If recv_time_ms == 0, this frame consisted of a packets which were all - // lost in the transport. Since we were able to render the frame, however, - // the dropped packets were recovered by FlexFEC. The FlexFEC recovery - // happens internally in Call, and we can therefore here not know which - // FEC packets that protected the lost media packets. Consequently, we - // were not able to record a meaningful recv_time_ms. We therefore skip - // this sample. - // - // The reasoning above does not hold for ULPFEC and RTX, as for those - // strategies the timestamp of the received packets is set to the - // timestamp of the protected/retransmitted media packet. I.e., then - // recv_time_ms != 0, even though the media packets were lost. - receiver_time_.AddSample(comparison.render_time_ms - - comparison.recv_time_ms); - network_time_.AddSample(comparison.recv_time_ms - - comparison.send_time_ms); - } - end_to_end_.AddSample(comparison.render_time_ms - comparison.input_time_ms); - encoded_frame_size_.AddSample(comparison.encoded_frame_size); - } - - void PrintResult(const char* result_type, - test::Statistics stats, - const char* unit) { - test::PrintResultMeanAndError(result_type, "", test_label_.c_str(), - stats.Mean(), stats.StandardDeviation(), unit, - false); - } - - void PrintSamplesToFile(void) { - FILE* out = graph_data_output_file_; - rtc::CritScope crit(&comparison_lock_); - std::sort(samples_.begin(), samples_.end(), - [](const Sample& A, const Sample& B) -> bool { - return A.input_time_ms < B.input_time_ms; - }); - - fprintf(out, "%s\n", graph_title_.c_str()); - fprintf(out, "%" PRIuS "\n", samples_.size()); - fprintf(out, - "dropped " - "input_time_ms " - "send_time_ms " - "recv_time_ms " - "render_time_ms " - "encoded_frame_size " - "psnr " - "ssim " - "encode_time_ms\n"); - for (const Sample& sample : samples_) { - fprintf(out, "%d %" PRId64 " %" PRId64 " %" PRId64 " %" PRId64 " %" PRIuS - " %lf %lf\n", - sample.dropped, sample.input_time_ms, sample.send_time_ms, - sample.recv_time_ms, sample.render_time_ms, - sample.encoded_frame_size, sample.psnr, sample.ssim); - } - } - - double GetAverageMediaBitrateBps() { - if (last_sending_time_ == first_sending_time_) { - return 0; - } else { - return static_cast<double>(total_media_bytes_) * 8 / - (last_sending_time_ - first_sending_time_) * - rtc::kNumMillisecsPerSec; - } - } - - // Implements VideoSinkInterface to receive captured frames from a - // FrameGeneratorCapturer. Implements VideoSourceInterface to be able to act - // as a source to VideoSendStream. - // It forwards all input frames to the VideoAnalyzer for later comparison and - // forwards the captured frames to the VideoSendStream. - class CapturedFrameForwarder : public rtc::VideoSinkInterface<VideoFrame>, - public rtc::VideoSourceInterface<VideoFrame> { - public: - explicit CapturedFrameForwarder(VideoAnalyzer* analyzer, Clock* clock) - : analyzer_(analyzer), - send_stream_input_(nullptr), - video_capturer_(nullptr), - clock_(clock) {} - - void SetSource(test::VideoCapturer* video_capturer) { - video_capturer_ = video_capturer; - } - - private: - void OnFrame(const VideoFrame& video_frame) override { - VideoFrame copy = video_frame; - // Frames from the capturer does not have a rtp timestamp. - // Create one so it can be used for comparison. - RTC_DCHECK_EQ(0, video_frame.timestamp()); - if (video_frame.ntp_time_ms() == 0) - copy.set_ntp_time_ms(clock_->CurrentNtpInMilliseconds()); - copy.set_timestamp(copy.ntp_time_ms() * 90); - analyzer_->AddCapturedFrameForComparison(copy); - rtc::CritScope lock(&crit_); - if (send_stream_input_) - send_stream_input_->OnFrame(copy); - } - - // Called when |send_stream_.SetSource()| is called. - void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink, - const rtc::VideoSinkWants& wants) override { - { - rtc::CritScope lock(&crit_); - RTC_DCHECK(!send_stream_input_ || send_stream_input_ == sink); - send_stream_input_ = sink; - } - if (video_capturer_) { - video_capturer_->AddOrUpdateSink(this, wants); - } - } - - // Called by |send_stream_| when |send_stream_.SetSource()| is called. - void RemoveSink(rtc::VideoSinkInterface<VideoFrame>* sink) override { - rtc::CritScope lock(&crit_); - RTC_DCHECK(sink == send_stream_input_); - send_stream_input_ = nullptr; - } - - VideoAnalyzer* const analyzer_; - rtc::CriticalSection crit_; - rtc::VideoSinkInterface<VideoFrame>* send_stream_input_ - RTC_GUARDED_BY(crit_); - test::VideoCapturer* video_capturer_; - Clock* clock_; - }; - - void AddCapturedFrameForComparison(const VideoFrame& video_frame) { - rtc::CritScope lock(&crit_); - frames_.push_back(video_frame); - } - - Call* call_; - VideoSendStream* send_stream_; - VideoReceiveStream* receive_stream_; - CapturedFrameForwarder captured_frame_forwarder_; - const std::string test_label_; - FILE* const graph_data_output_file_; - const std::string graph_title_; - const uint32_t ssrc_to_analyze_; - const uint32_t rtx_ssrc_to_analyze_; - const size_t selected_stream_; - const int selected_sl_; - const int selected_tl_; - PreEncodeProxy pre_encode_proxy_; - OnEncodeTimingProxy encode_timing_proxy_; - std::vector<Sample> samples_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics sender_time_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics receiver_time_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics network_time_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics psnr_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics ssim_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics end_to_end_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics rendered_delta_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics encoded_frame_size_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics encode_frame_rate_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics encode_time_ms_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics encode_usage_percent_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics decode_time_ms_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics decode_time_max_ms_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics media_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics fec_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics send_bandwidth_bps_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics memory_usage_ RTC_GUARDED_BY(comparison_lock_); - test::Statistics time_between_freezes_ RTC_GUARDED_BY(comparison_lock_); - - struct FrameWithPsnr { - double psnr; - VideoFrame frame; - }; - - // Rendered frame with worst PSNR is saved for further analysis. - rtc::Optional<FrameWithPsnr> worst_frame_ RTC_GUARDED_BY(comparison_lock_); - - size_t last_fec_bytes_; - - const int frames_to_process_; - int frames_recorded_; - int frames_processed_; - int dropped_frames_; - int dropped_frames_before_first_encode_; - int dropped_frames_before_rendering_; - int64_t last_render_time_; - int64_t last_render_delta_ms_; - int64_t last_unfreeze_time_ms_; - uint32_t rtp_timestamp_delta_; - int64_t total_media_bytes_; - int64_t first_sending_time_; - int64_t last_sending_time_; - - int64_t cpu_time_ RTC_GUARDED_BY(cpu_measurement_lock_); - int64_t wallclock_time_ RTC_GUARDED_BY(cpu_measurement_lock_); - rtc::CriticalSection cpu_measurement_lock_; - - rtc::CriticalSection crit_; - std::deque<VideoFrame> frames_ RTC_GUARDED_BY(crit_); - rtc::Optional<VideoFrame> last_rendered_frame_ RTC_GUARDED_BY(crit_); - rtc::TimestampWrapAroundHandler wrap_handler_ RTC_GUARDED_BY(crit_); - std::map<int64_t, int64_t> send_times_ RTC_GUARDED_BY(crit_); - std::map<int64_t, int64_t> recv_times_ RTC_GUARDED_BY(crit_); - std::map<int64_t, size_t> encoded_frame_sizes_ RTC_GUARDED_BY(crit_); - rtc::Optional<uint32_t> first_encoded_timestamp_ RTC_GUARDED_BY(crit_); - rtc::Optional<uint32_t> first_sent_timestamp_ RTC_GUARDED_BY(crit_); - const double avg_psnr_threshold_; - const double avg_ssim_threshold_; - bool is_quick_test_enabled_; - - rtc::CriticalSection comparison_lock_; - std::vector<rtc::PlatformThread*> comparison_thread_pool_; - rtc::PlatformThread stats_polling_thread_; - rtc::Event comparison_available_event_; - std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(comparison_lock_); - rtc::Event done_; - - std::unique_ptr<test::RtpFileWriter> rtp_file_writer_; - Clock* const clock_; - const int64_t start_ms_; -}; - -// Not used by these tests. -std::vector<SdpVideoFormat> -VideoQualityTest::TestVideoEncoderFactory::GetSupportedFormats() const { - RTC_NOTREACHED(); - return {}; -} - -VideoEncoderFactory::CodecInfo -VideoQualityTest::TestVideoEncoderFactory::QueryVideoEncoder( - const SdpVideoFormat& format) const { - CodecInfo codec_info; - codec_info.is_hardware_accelerated = false; - codec_info.has_internal_source = false; - return codec_info; -} - -std::unique_ptr<VideoEncoder> -VideoQualityTest::TestVideoEncoderFactory::CreateVideoEncoder( +std::unique_ptr<VideoEncoder> VideoQualityTest::CreateVideoEncoder( const SdpVideoFormat& format) { if (format.name == "VP8") { - return rtc::MakeUnique<VP8EncoderSimulcastProxy>( - &internal_encoder_factory_); + return absl::make_unique<VP8EncoderSimulcastProxy>( + &internal_encoder_factory_, format); } else if (format.name == "multiplex") { - return rtc::MakeUnique<MultiplexEncoderAdapter>( + return absl::make_unique<MultiplexEncoderAdapter>( &internal_encoder_factory_, SdpVideoFormat(cricket::kVp9CodecName)); - } else { - return internal_encoder_factory_.CreateVideoEncoder(format); } + return internal_encoder_factory_.CreateVideoEncoder(format); } -VideoQualityTest::VideoQualityTest() - : clock_(Clock::GetRealTimeClock()), receive_logs_(0), send_logs_(0) { +VideoQualityTest::VideoQualityTest( + std::unique_ptr<FecControllerFactoryInterface> fec_controller_factory) + : clock_(Clock::GetRealTimeClock()), + video_encoder_factory_([this](const SdpVideoFormat& format) { + return this->CreateVideoEncoder(format); + }), + receive_logs_(0), + send_logs_(0) { payload_type_map_ = test::CallTest::payload_type_map_; RTC_DCHECK(payload_type_map_.find(kPayloadTypeH264) == payload_type_map_.end()); @@ -1117,20 +97,16 @@ VideoQualityTest::VideoQualityTest() payload_type_map_[kPayloadTypeH264] = webrtc::MediaType::VIDEO; payload_type_map_[kPayloadTypeVP8] = webrtc::MediaType::VIDEO; payload_type_map_[kPayloadTypeVP9] = webrtc::MediaType::VIDEO; -} -VideoQualityTest::VideoQualityTest( - std::unique_ptr<FecControllerFactoryInterface> fec_controller_factory) - : VideoQualityTest() { fec_controller_factory_ = std::move(fec_controller_factory); } VideoQualityTest::Params::Params() : call({false, BitrateConstraints(), 0}), video{{false, 640, 480, 30, 50, 800, 800, false, "VP8", 1, -1, 0, false, - false, ""}, + false, false, ""}, {false, 640, 480, 30, 50, 800, 800, false, "VP8", 1, -1, 0, false, - false, ""}}, + false, false, ""}}, audio({false, false, false}), screenshare{{false, false, 10, 0}, {false, false, 10, 0}}, analyzer({"", 0.0, 0.0, 0, "", ""}), @@ -1384,9 +360,6 @@ void VideoQualityTest::FillScalabilitySettings( void VideoQualityTest::SetupVideo(Transport* send_transport, Transport* recv_transport) { size_t total_streams_used = 0; - size_t num_flexfec_streams = params_.video[0].flexfec ? 1 : 0; - CreateAudioAndFecSendConfigs(0, num_flexfec_streams, send_transport); - CreateMatchingAudioAndFecConfigs(recv_transport); video_receive_configs_.clear(); video_send_configs_.clear(); video_encoder_configs_.clear(); @@ -1400,8 +373,9 @@ void VideoQualityTest::SetupVideo(Transport* send_transport, video_encoder_configs_.push_back(VideoEncoderConfig()); num_video_substreams = params_.ss[video_idx].streams.size(); RTC_CHECK_GT(num_video_substreams, 0); - CreateVideoSendConfig(&video_send_configs_[video_idx], num_video_substreams, - total_streams_used, send_transport); + for (size_t i = 0; i < num_video_substreams; ++i) + video_send_configs_[video_idx].rtp.ssrcs.push_back( + kVideoSendSsrcs[total_streams_used + i]); int payload_type; if (params_.video[video_idx].codec == "H264") { @@ -1446,6 +420,9 @@ void VideoQualityTest::SetupVideo(Transport* send_transport, video_encoder_configs_[video_idx].video_format.name = params_.video[video_idx].codec; + video_encoder_configs_[video_idx].video_format.parameters = + params_.video[video_idx].sdp_params; + video_encoder_configs_[video_idx].codec_type = PayloadStringToCodecType(params_.video[video_idx].codec); @@ -1479,40 +456,14 @@ void VideoQualityTest::SetupVideo(Transport* send_transport, video_encoder_configs_[video_idx].spatial_layers = params_.ss[video_idx].spatial_layers; - - std::vector<VideoReceiveStream::Config> new_receive_configs = - CreateMatchingVideoReceiveConfigs(video_send_configs_[video_idx], - recv_transport); - decode_all_receive_streams = params_.ss[video_idx].selected_stream == params_.ss[video_idx].streams.size(); - - for (size_t i = 0; i < num_video_substreams; ++i) { - new_receive_configs[i].rtp.nack.rtp_history_ms = kNackRtpHistoryMs; - new_receive_configs[i].rtp.rtx_ssrc = - kSendRtxSsrcs[i + total_streams_used]; - new_receive_configs[i] - .rtp.rtx_associated_payload_types[kSendRtxPayloadType] = payload_type; - new_receive_configs[i].rtp.transport_cc = params_.call.send_side_bwe; - new_receive_configs[i].rtp.remb = !params_.call.send_side_bwe; - // Enable RTT calculation so NTP time estimator will work. - new_receive_configs[i].rtp.rtcp_xr.receiver_reference_time_report = true; - // Force fake decoders on non-selected simulcast streams. - if (!decode_all_receive_streams && - i != params_.ss[video_idx].selected_stream) { - VideoReceiveStream::Decoder decoder; - decoder.decoder = new test::FakeDecoder(); - decoder.payload_type = video_send_configs_[video_idx].rtp.payload_type; - decoder.payload_name = video_send_configs_[video_idx].rtp.payload_name; - new_receive_configs[i].decoders.clear(); - allocated_decoders_.emplace_back(decoder.decoder); - new_receive_configs[i].decoders.push_back(decoder); - } - } - - for (VideoReceiveStream::Config& config : new_receive_configs) { - video_receive_configs_.push_back(config.Copy()); - } + absl::optional<int> decode_sub_stream; + if (!decode_all_receive_streams) + decode_sub_stream = params_.ss[video_idx].selected_stream; + CreateMatchingVideoReceiveConfigs( + video_send_configs_[video_idx], recv_transport, + params_.call.send_side_bwe, decode_sub_stream, true, kNackRtpHistoryMs); if (params_.screenshare[video_idx].enabled) { // Fill out codec settings. @@ -1553,77 +504,57 @@ void VideoQualityTest::SetupVideo(Transport* send_transport, video_encoder_configs_[video_idx].encoder_specific_settings = new rtc::RefCountedObject< VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings); + } else if (params_.video[video_idx].automatic_scaling) { + if (params_.video[video_idx].codec == "VP8") { + VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings(); + vp8_settings.automaticResizeOn = true; + video_encoder_configs_[video_idx].encoder_specific_settings = + new rtc::RefCountedObject< + VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8_settings); + } else if (params_.video[video_idx].codec == "VP9") { + VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings(); + vp9_settings.automaticResizeOn = true; + video_encoder_configs_[video_idx].encoder_specific_settings = + new rtc::RefCountedObject< + VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings); + } else { + RTC_NOTREACHED() << "Automatic scaling not supported for codec " + << params_.video[video_idx].codec << ", stream " + << video_idx; + } } total_streams_used += num_video_substreams; } // FEC supported only for single video stream mode yet. if (params_.video[0].flexfec) { - video_send_configs_[0].rtp.flexfec.payload_type = kFlexfecPayloadType; - video_send_configs_[0].rtp.flexfec.ssrc = kFlexfecSendSsrc; if (decode_all_receive_streams) { - for (uint32_t media_ssrc : video_send_configs_[0].rtp.ssrcs) { - video_send_configs_[0].rtp.flexfec.protected_media_ssrcs.push_back( - media_ssrc); - } + SetSendFecConfig(GetVideoSendConfig()->rtp.ssrcs); } else { - video_send_configs_[0].rtp.flexfec.protected_media_ssrcs = { - kVideoSendSsrcs[params_.ss[0].selected_stream]}; + SetSendFecConfig({kVideoSendSsrcs[params_.ss[0].selected_stream]}); } - // The matching receive config is _not_ created by - // CreateMatchingReceiveConfigs, since VideoQualityTest is not a BaseTest. - // Set up the receive config manually instead. - FlexfecReceiveStream::Config flexfec_receive_config(recv_transport); - flexfec_receive_config.payload_type = - video_send_configs_[0].rtp.flexfec.payload_type; - flexfec_receive_config.remote_ssrc = - video_send_configs_[0].rtp.flexfec.ssrc; - flexfec_receive_config.protected_media_ssrcs = - video_send_configs_[0].rtp.flexfec.protected_media_ssrcs; - flexfec_receive_config.local_ssrc = kReceiverLocalVideoSsrc; - flexfec_receive_config.transport_cc = params_.call.send_side_bwe; + CreateMatchingFecConfig(recv_transport, *GetVideoSendConfig()); + GetFlexFecConfig()->transport_cc = params_.call.send_side_bwe; if (params_.call.send_side_bwe) { - flexfec_receive_config.rtp_header_extensions.push_back( + GetFlexFecConfig()->rtp_header_extensions.push_back( RtpExtension(RtpExtension::kTransportSequenceNumberUri, test::kTransportSequenceNumberExtensionId)); } else { - flexfec_receive_config.rtp_header_extensions.push_back(RtpExtension( + GetFlexFecConfig()->rtp_header_extensions.push_back(RtpExtension( RtpExtension::kAbsSendTimeUri, test::kAbsSendTimeExtensionId)); } - flexfec_receive_configs_.push_back(flexfec_receive_config); - if (num_video_substreams > 0) { - video_receive_configs_[0].rtp.protected_by_flexfec = true; - } } if (params_.video[0].ulpfec) { - video_send_configs_[0].rtp.ulpfec.red_payload_type = kRedPayloadType; - video_send_configs_[0].rtp.ulpfec.ulpfec_payload_type = kUlpfecPayloadType; - video_send_configs_[0].rtp.ulpfec.red_rtx_payload_type = kRtxRedPayloadType; - + SetSendUlpFecConfig(GetVideoSendConfig()); if (decode_all_receive_streams) { - for (auto it = video_receive_configs_.begin(); - it != video_receive_configs_.end(); ++it) { - it->rtp.red_payload_type = - video_send_configs_[0].rtp.ulpfec.red_payload_type; - it->rtp.ulpfec_payload_type = - video_send_configs_[0].rtp.ulpfec.ulpfec_payload_type; - it->rtp.rtx_associated_payload_types - [video_send_configs_[0].rtp.ulpfec.red_rtx_payload_type] = - video_send_configs_[0].rtp.ulpfec.red_payload_type; + for (auto& receive_config : video_receive_configs_) { + SetReceiveUlpFecConfig(&receive_config); } } else { - video_receive_configs_[params_.ss[0].selected_stream] - .rtp.red_payload_type = - video_send_configs_[0].rtp.ulpfec.red_payload_type; - video_receive_configs_[params_.ss[0].selected_stream] - .rtp.ulpfec_payload_type = - video_send_configs_[0].rtp.ulpfec.ulpfec_payload_type; - video_receive_configs_[params_.ss[0].selected_stream] - .rtp.rtx_associated_payload_types - [video_send_configs_[0].rtp.ulpfec.red_rtx_payload_type] = - video_send_configs_[0].rtp.ulpfec.red_payload_type; + SetReceiveUlpFecConfig( + &video_receive_configs_[params_.ss[0].selected_stream]); } } } @@ -1673,39 +604,19 @@ void VideoQualityTest::SetupThumbnails(Transport* send_transport, } thumbnail_encoder_config.spatial_layers = params_.ss[0].spatial_layers; - VideoReceiveStream::Config thumbnail_receive_config(send_transport); - thumbnail_receive_config.rtp.remb = false; - thumbnail_receive_config.rtp.transport_cc = true; - thumbnail_receive_config.rtp.local_ssrc = kReceiverLocalVideoSsrc; - for (const RtpExtension& extension : thumbnail_send_config.rtp.extensions) - thumbnail_receive_config.rtp.extensions.push_back(extension); - thumbnail_receive_config.renderer = &fake_renderer_; - - VideoReceiveStream::Decoder decoder = - test::CreateMatchingDecoder(thumbnail_send_config); - allocated_decoders_.push_back( - std::unique_ptr<VideoDecoder>(decoder.decoder)); - thumbnail_receive_config.decoders.clear(); - thumbnail_receive_config.decoders.push_back(decoder); - thumbnail_receive_config.rtp.remote_ssrc = - thumbnail_send_config.rtp.ssrcs[0]; - - thumbnail_receive_config.rtp.nack.rtp_history_ms = kNackRtpHistoryMs; - thumbnail_receive_config.rtp.rtx_ssrc = kThumbnailRtxSsrcStart + i; - thumbnail_receive_config.rtp - .rtx_associated_payload_types[kSendRtxPayloadType] = kPayloadTypeVP8; - thumbnail_receive_config.rtp.transport_cc = params_.call.send_side_bwe; - thumbnail_receive_config.rtp.remb = !params_.call.send_side_bwe; - thumbnail_encoder_configs_.push_back(thumbnail_encoder_config.Copy()); thumbnail_send_configs_.push_back(thumbnail_send_config.Copy()); - thumbnail_receive_configs_.push_back(thumbnail_receive_config.Copy()); - } - for (int i = 0; i < params_.call.num_thumbnails; ++i) { + AddMatchingVideoReceiveConfigs( + &thumbnail_receive_configs_, thumbnail_send_config, send_transport, + params_.call.send_side_bwe, absl::nullopt, false, kNackRtpHistoryMs); + } + for (size_t i = 0; i < thumbnail_send_configs_.size(); ++i) { thumbnail_send_streams_.push_back(receiver_call_->CreateVideoSendStream( thumbnail_send_configs_[i].Copy(), thumbnail_encoder_configs_[i].Copy())); + } + for (size_t i = 0; i < thumbnail_receive_configs_.size(); ++i) { thumbnail_receive_streams_.push_back(sender_call_->CreateVideoReceiveStream( thumbnail_receive_configs_[i].Copy())); } @@ -1733,7 +644,7 @@ void VideoQualityTest::SetupThumbnailCapturers(size_t num_thumbnail_streams) { for (size_t i = 0; i < num_thumbnail_streams; ++i) { thumbnail_capturers_.emplace_back(test::FrameGeneratorCapturer::Create( static_cast<int>(thumbnail.width), static_cast<int>(thumbnail.height), - rtc::nullopt, rtc::nullopt, thumbnail.max_framerate, clock_)); + absl::nullopt, absl::nullopt, thumbnail.max_framerate, clock_)); RTC_DCHECK(thumbnail_capturers_.back()); } } @@ -1785,6 +696,8 @@ std::unique_ptr<test::FrameGenerator> VideoQualityTest::CreateFrameGenerator( } void VideoQualityTest::CreateCapturers() { + RTC_DCHECK(video_sources_.empty()); + RTC_DCHECK(video_capturers_.empty()); video_capturers_.resize(num_video_streams_); for (size_t video_idx = 0; video_idx < num_video_streams_; ++video_idx) { if (params_.screenshare[video_idx].enabled) { @@ -1799,13 +712,19 @@ void VideoQualityTest::CreateCapturers() { if (params_.video[video_idx].clip_name == "Generator") { video_capturers_[video_idx].reset(test::FrameGeneratorCapturer::Create( static_cast<int>(params_.video[video_idx].width), - static_cast<int>(params_.video[video_idx].height), rtc::nullopt, - rtc::nullopt, params_.video[video_idx].fps, clock_)); + static_cast<int>(params_.video[video_idx].height), absl::nullopt, + absl::nullopt, params_.video[video_idx].fps, clock_)); } else if (params_.video[video_idx].clip_name == "GeneratorI420A") { video_capturers_[video_idx].reset(test::FrameGeneratorCapturer::Create( static_cast<int>(params_.video[video_idx].width), static_cast<int>(params_.video[video_idx].height), - test::FrameGenerator::OutputType::I420A, rtc::nullopt, + test::FrameGenerator::OutputType::I420A, absl::nullopt, + params_.video[video_idx].fps, clock_)); + } else if (params_.video[video_idx].clip_name == "GeneratorI010") { + video_capturers_[video_idx].reset(test::FrameGeneratorCapturer::Create( + static_cast<int>(params_.video[video_idx].width), + static_cast<int>(params_.video[video_idx].height), + test::FrameGenerator::OutputType::I010, absl::nullopt, params_.video[video_idx].fps, clock_)); } else if (params_.video[video_idx].clip_name.empty()) { video_capturers_[video_idx].reset(test::VcmCapturer::Create( @@ -1818,7 +737,7 @@ void VideoQualityTest::CreateCapturers() { test::FrameGeneratorCapturer::Create( static_cast<int>(params_.video[video_idx].width), static_cast<int>(params_.video[video_idx].height), - rtc::nullopt, rtc::nullopt, params_.video[video_idx].fps, + absl::nullopt, absl::nullopt, params_.video[video_idx].fps, clock_)); } } else { @@ -1834,12 +753,13 @@ void VideoQualityTest::CreateCapturers() { } } RTC_DCHECK(video_capturers_[video_idx].get()); + video_sources_.push_back(video_capturers_[video_idx].get()); } } std::unique_ptr<test::LayerFilteringTransport> VideoQualityTest::CreateSendTransport() { - return rtc::MakeUnique<test::LayerFilteringTransport>( + return absl::make_unique<test::LayerFilteringTransport>( &task_queue_, params_.pipe, sender_call_.get(), kPayloadTypeVP8, kPayloadTypeVP9, params_.video[0].selected_tl, params_.ss[0].selected_sl, payload_type_map_, kVideoSendSsrcs[0], @@ -1849,61 +769,10 @@ VideoQualityTest::CreateSendTransport() { std::unique_ptr<test::DirectTransport> VideoQualityTest::CreateReceiveTransport() { - return rtc::MakeUnique<test::DirectTransport>( + return absl::make_unique<test::DirectTransport>( &task_queue_, params_.pipe, receiver_call_.get(), payload_type_map_); } -void VideoQualityTest::CreateVideoStreams() { - RTC_DCHECK(video_send_streams_.empty()); - RTC_DCHECK(video_receive_streams_.empty()); - RTC_DCHECK_EQ(video_send_configs_.size(), num_video_streams_); - - // We currently only support testing external fec controllers with a single - // VideoSendStream. - if (fec_controller_factory_.get()) { - RTC_DCHECK_LE(video_send_configs_.size(), 1); - } - - // TODO(http://crbug/818127): - // Remove this workaround when ALR is not screenshare-specific. - std::list<size_t> streams_creation_order; - for (size_t i = 0; i < video_send_configs_.size(); ++i) { - // If dual streams are created, add the screenshare stream last. - if (video_encoder_configs_[i].content_type == - VideoEncoderConfig::ContentType::kScreen) { - streams_creation_order.push_back(i); - } else { - streams_creation_order.push_front(i); - } - } - - video_send_streams_.resize(video_send_configs_.size(), nullptr); - - for (size_t i : streams_creation_order) { - if (fec_controller_factory_.get()) { - video_send_streams_[i] = sender_call_->CreateVideoSendStream( - video_send_configs_[i].Copy(), video_encoder_configs_[i].Copy(), - fec_controller_factory_->CreateFecController()); - } else { - video_send_streams_[i] = sender_call_->CreateVideoSendStream( - video_send_configs_[i].Copy(), video_encoder_configs_[i].Copy()); - } - } - for (size_t i = 0; i < video_receive_configs_.size(); ++i) { - video_receive_streams_.push_back(receiver_call_->CreateVideoReceiveStream( - video_receive_configs_[i].Copy())); - } - - AssociateFlexfecStreamsWithVideoStreams(); -} - -void VideoQualityTest::DestroyStreams() { - CallTest::DestroyStreams(); - - for (VideoSendStream* video_send_stream : video_send_streams_) - sender_call_->DestroyVideoSendStream(video_send_stream); -} - void VideoQualityTest::RunWithAnalyzer(const Params& params) { rtc::LogMessage::SetLogToStderr(params.logging.logs); num_video_streams_ = params.call.dual_video ? 2 : 1; @@ -1931,11 +800,11 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { send_event_log_ = RtcEventLog::Create(RtcEventLog::EncodingType::Legacy); recv_event_log_ = RtcEventLog::Create(RtcEventLog::EncodingType::Legacy); std::unique_ptr<RtcEventLogOutputFile> send_output( - rtc::MakeUnique<RtcEventLogOutputFile>( + absl::make_unique<RtcEventLogOutputFile>( params.logging.rtc_event_log_name + "_send", RtcEventLog::kUnlimitedOutput)); std::unique_ptr<RtcEventLogOutputFile> recv_output( - rtc::MakeUnique<RtcEventLogOutputFile>( + absl::make_unique<RtcEventLogOutputFile>( params.logging.rtc_event_log_name + "_recv", RtcEventLog::kUnlimitedOutput)); bool event_log_started = @@ -1965,7 +834,7 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { if (graph_title.empty()) graph_title = VideoQualityTest::GenerateGraphTitle(); bool is_quick_test_enabled = field_trial::IsEnabled("WebRTC-QuickPerfTest"); - analyzer = rtc::MakeUnique<VideoAnalyzer>( + analyzer = absl::make_unique<VideoAnalyzer>( send_transport.get(), params_.analyzer.test_label, params_.analyzer.avg_psnr_threshold, params_.analyzer.avg_ssim_threshold, is_quick_test_enabled @@ -1988,10 +857,9 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { SetupThumbnails(analyzer.get(), recv_transport.get()); video_receive_configs_[params_.ss[0].selected_stream].renderer = analyzer.get(); - video_send_configs_[0].pre_encode_callback = analyzer->pre_encode_proxy(); - RTC_DCHECK(!video_send_configs_[0].post_encode_callback); - video_send_configs_[0].post_encode_callback = - analyzer->encode_timing_proxy(); + GetVideoSendConfig()->pre_encode_callback = analyzer->pre_encode_proxy(); + RTC_DCHECK(!GetVideoSendConfig()->post_encode_callback); + GetVideoSendConfig()->post_encode_callback = analyzer.get(); CreateFlexfecStreams(); CreateVideoStreams(); @@ -1999,8 +867,8 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { if (video_receive_streams_.size() == 1) analyzer->SetReceiveStream(video_receive_streams_[0]); - video_send_streams_[0]->SetSource(analyzer->OutputInterface(), - degradation_preference_); + GetVideoSendStream()->SetSource(analyzer->OutputInterface(), + degradation_preference_); SetupThumbnailCapturers(params_.call.num_thumbnails); for (size_t i = 0; i < thumbnail_send_streams_.size(); ++i) { thumbnail_send_streams_[i]->SetSource(thumbnail_capturers_[i].get(), @@ -2016,24 +884,18 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { video_capturers_[video_idx].get(), degradation_preference_); } - StartEncodedFrameLogs(video_send_streams_[0]); + StartEncodedFrameLogs(GetVideoSendStream()); StartEncodedFrameLogs( video_receive_streams_[params_.ss[0].selected_stream]); - for (VideoSendStream* video_send_stream : video_send_streams_) - video_send_stream->Start(); + StartVideoStreams(); for (VideoSendStream* thumbnail_send_stream : thumbnail_send_streams_) thumbnail_send_stream->Start(); - for (VideoReceiveStream* receive_stream : video_receive_streams_) - receive_stream->Start(); for (VideoReceiveStream* thumbnail_receive_stream : thumbnail_receive_streams_) thumbnail_receive_stream->Start(); analyzer->StartMeasuringCpuProcessTime(); - - for (size_t video_idx = 0; video_idx < num_video_streams_; ++video_idx) { - video_capturers_[video_idx]->Start(); - } + StartVideoCapture(); for (std::unique_ptr<test::VideoCapturer>& video_caputurer : thumbnail_capturers_) { video_caputurer->Start(); @@ -2042,8 +904,6 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { analyzer->Wait(); - event_log_->StopLogging(); - task_queue_.SendTask([&]() { for (std::unique_ptr<test::VideoCapturer>& video_caputurer : thumbnail_capturers_) @@ -2075,42 +935,37 @@ void VideoQualityTest::RunWithAnalyzer(const Params& params) { }); } -void VideoQualityTest::SetupAudio(Transport* transport, - AudioReceiveStream** audio_receive_stream) { - audio_send_config_ = AudioSendStream::Config(transport); - audio_send_config_.rtp.ssrc = kAudioSendSsrc; +void VideoQualityTest::SetupAudio(Transport* transport) { + AudioSendStream::Config audio_send_config(transport); + audio_send_config.rtp.ssrc = kAudioSendSsrc; // Add extension to enable audio send side BWE, and allow audio bit rate // adaptation. - audio_send_config_.rtp.extensions.clear(); + audio_send_config.rtp.extensions.clear(); + audio_send_config.send_codec_spec = AudioSendStream::Config::SendCodecSpec( + kAudioSendPayloadType, + {"OPUS", + 48000, + 2, + {{"usedtx", (params_.audio.dtx ? "1" : "0")}, {"stereo", "1"}}}); + if (params_.call.send_side_bwe) { - audio_send_config_.rtp.extensions.push_back( + audio_send_config.rtp.extensions.push_back( webrtc::RtpExtension(webrtc::RtpExtension::kTransportSequenceNumberUri, test::kTransportSequenceNumberExtensionId)); - audio_send_config_.min_bitrate_bps = kOpusMinBitrateBps; - audio_send_config_.max_bitrate_bps = kOpusBitrateFbBps; + audio_send_config.min_bitrate_bps = kOpusMinBitrateBps; + audio_send_config.max_bitrate_bps = kOpusBitrateFbBps; + audio_send_config.send_codec_spec->transport_cc_enabled = true; } - audio_send_config_.send_codec_spec = AudioSendStream::Config::SendCodecSpec( - kAudioSendPayloadType, - {"OPUS", 48000, 2, - {{"usedtx", (params_.audio.dtx ? "1" : "0")}, - {"stereo", "1"}}}); - audio_send_config_.encoder_factory = audio_encoder_factory_; - audio_send_stream_ = sender_call_->CreateAudioSendStream(audio_send_config_); - - AudioReceiveStream::Config audio_config; - audio_config.rtp.local_ssrc = kReceiverLocalAudioSsrc; - audio_config.rtcp_send_transport = transport; - audio_config.rtp.remote_ssrc = audio_send_config_.rtp.ssrc; - audio_config.rtp.transport_cc = params_.call.send_side_bwe; - audio_config.rtp.extensions = audio_send_config_.rtp.extensions; - audio_config.decoder_factory = audio_decoder_factory_; - audio_config.decoder_map = {{kAudioSendPayloadType, {"OPUS", 48000, 2}}}; + audio_send_config.encoder_factory = audio_encoder_factory_; + SetAudioConfig(audio_send_config); + + const char* sync_group = nullptr; if (params_.video[0].enabled && params_.audio.sync_video) - audio_config.sync_group = kSyncGroup; + sync_group = kSyncGroup; - *audio_receive_stream = - receiver_call_->CreateAudioReceiveStream(audio_config); + CreateMatchingAudioConfigs(transport, sync_group); + CreateAudioStreams(); } void VideoQualityTest::RunWithRenderers(const Params& params) { @@ -2120,7 +975,7 @@ void VideoQualityTest::RunWithRenderers(const Params& params) { std::unique_ptr<test::DirectTransport> recv_transport; std::unique_ptr<test::VideoRenderer> local_preview; std::vector<std::unique_ptr<test::VideoRenderer>> loopback_renderers; - AudioReceiveStream* audio_receive_stream = nullptr; + RtcEventLogNullImpl null_event_log; task_queue_.SendTask([&]() { params_ = params; @@ -2128,7 +983,7 @@ void VideoQualityTest::RunWithRenderers(const Params& params) { // TODO(ivica): Remove bitrate_config and use the default Call::Config(), to // match the full stack tests. - Call::Config call_config(event_log_.get()); + Call::Config call_config(&null_event_log); call_config.bitrate_config = params_.call.call_bitrate_config; rtc::scoped_refptr<TestAudioDeviceModule> fake_audio_device = @@ -2166,7 +1021,7 @@ void VideoQualityTest::RunWithRenderers(const Params& params) { "Local Preview", params_.video[0].width, params_.video[0].height)); SetupVideo(send_transport.get(), recv_transport.get()); - video_send_configs_[0].pre_encode_callback = local_preview.get(); + GetVideoSendConfig()->pre_encode_callback = local_preview.get(); size_t num_streams_processed = 0; for (size_t video_idx = 0; video_idx < num_video_streams_; ++video_idx) { @@ -2206,58 +1061,24 @@ void VideoQualityTest::RunWithRenderers(const Params& params) { CreateVideoStreams(); CreateCapturers(); - for (size_t video_idx = 0; video_idx < num_video_streams_; ++video_idx) { - video_send_streams_[video_idx]->SetSource( - video_capturers_[video_idx].get(), degradation_preference_); - } + ConnectVideoSourcesToStreams(); } if (params_.audio.enabled) { - SetupAudio(send_transport.get(), &audio_receive_stream); + SetupAudio(send_transport.get()); } for (VideoReceiveStream* receive_stream : video_receive_streams_) StartEncodedFrameLogs(receive_stream); - StartEncodedFrameLogs(video_send_streams_[0]); - - // Start sending and receiving video. - if (params_.video[0].enabled) { - for (VideoReceiveStream* video_receive_stream : video_receive_streams_) - video_receive_stream->Start(); - for (VideoSendStream* video_send_stream : video_send_streams_) - video_send_stream->Start(); - for (auto& video_capturer : video_capturers_) - video_capturer->Start(); - } - - if (params_.audio.enabled) { - // Start receiving audio. - audio_receive_stream->Start(); - - // Start sending audio. - audio_send_stream_->Start(); - } + StartEncodedFrameLogs(GetVideoSendStream()); + Start(); }); test::PressEnterToContinue(); task_queue_.SendTask([&]() { - if (params_.audio.enabled) { - // Stop sending audio. - audio_send_stream_->Stop(); - - // Stop receiving audio. - audio_receive_stream->Stop(); - } - - // Stop receiving and sending video. - if (params_.video[0].enabled) { - for (auto& video_capturer : video_capturers_) - video_capturer->Stop(); - for (VideoSendStream* video_send_stream : video_send_streams_) - video_send_stream->Stop(); - DestroyStreams(); - } + Stop(); + DestroyStreams(); video_capturers_.clear(); send_transport.reset(); |