blob: d435917f1f328e21e72efba77de4b6ba1918642b [file] [log] [blame]
// Copyright 2014 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "media/cast/sender/video_sender.h"
#include <algorithm>
#include <cmath>
#include <cstring>
#include <utility>
#include "base/bind.h"
#include "base/logging.h"
#include "base/trace_event/trace_event.h"
#include "media/cast/net/cast_transport_config.h"
#include "media/cast/sender/performance_metrics_overlay.h"
#include "media/cast/sender/video_encoder.h"
#include "starboard/types.h"
namespace cobalt {
namespace media {
namespace cast {
namespace {
// The following two constants are used to adjust the target
// playout delay (when allowed). They were calculated using
// a combination of cast_benchmark runs and manual testing.
//
// This is how many round trips we think we need on the network.
const int kRoundTripsNeeded = 4;
// This is an estimate of all the the constant time needed independent of
// network quality (e.g., additional time that accounts for encode and decode
// time).
const int kConstantTimeMs = 75;
// The target maximum utilization of the encoder and network resources. This is
// used to attenuate the actual measured utilization values in order to provide
// "breathing room" (i.e., to ensure there will be sufficient CPU and bandwidth
// available to handle the occasional more-complex frames).
const int kTargetUtilizationPercentage = 75;
// This is the minimum duration in milliseconds that the sender sends key frame
// request to the encoder on receiving Pli messages. This is used to prevent
// sending multiple requests while the sender is waiting for an encoded key
// frame or receiving multiple Pli messages in a short period.
const int64_t kMinKeyFrameRequestOnPliIntervalMs = 500;
// Extract capture begin/end timestamps from |video_frame|'s metadata and log
// it.
void LogVideoCaptureTimestamps(CastEnvironment* cast_environment,
const media::VideoFrame& video_frame,
RtpTimeTicks rtp_timestamp) {
std::unique_ptr<FrameEvent> capture_begin_event(new FrameEvent());
capture_begin_event->type = FRAME_CAPTURE_BEGIN;
capture_begin_event->media_type = VIDEO_EVENT;
capture_begin_event->rtp_timestamp = rtp_timestamp;
std::unique_ptr<FrameEvent> capture_end_event(new FrameEvent());
capture_end_event->type = FRAME_CAPTURE_END;
capture_end_event->media_type = VIDEO_EVENT;
capture_end_event->rtp_timestamp = rtp_timestamp;
capture_end_event->width = video_frame.visible_rect().width();
capture_end_event->height = video_frame.visible_rect().height();
if (!video_frame.metadata()->GetTimeTicks(
media::VideoFrameMetadata::CAPTURE_BEGIN_TIME,
&capture_begin_event->timestamp) ||
!video_frame.metadata()->GetTimeTicks(
media::VideoFrameMetadata::CAPTURE_END_TIME,
&capture_end_event->timestamp)) {
// The frame capture timestamps were not provided by the video capture
// source. Simply log the events as happening right now.
capture_begin_event->timestamp = capture_end_event->timestamp =
cast_environment->Clock()->NowTicks();
}
cast_environment->logger()->DispatchFrameEvent(
std::move(capture_begin_event));
cast_environment->logger()->DispatchFrameEvent(std::move(capture_end_event));
}
} // namespace
// Note, we use a fixed bitrate value when external video encoder is used.
// Some hardware encoder shows bad behavior if we set the bitrate too
// frequently, e.g. quality drop, not abiding by target bitrate, etc.
// See details: crbug.com/392086.
VideoSender::VideoSender(
scoped_refptr<CastEnvironment> cast_environment,
const FrameSenderConfig& video_config,
const StatusChangeCallback& status_change_cb,
const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb,
CastTransport* const transport_sender,
const PlayoutDelayChangeCB& playout_delay_change_cb)
: FrameSender(
cast_environment, transport_sender, video_config,
video_config.use_external_encoder
? NewFixedCongestionControl(
(video_config.min_bitrate + video_config.max_bitrate) / 2)
: NewAdaptiveCongestionControl(
cast_environment->Clock(), video_config.max_bitrate,
video_config.min_bitrate, video_config.max_frame_rate)),
frames_in_encoder_(0),
last_bitrate_(0),
playout_delay_change_cb_(playout_delay_change_cb),
low_latency_mode_(false),
last_reported_encoder_utilization_(-1.0),
last_reported_lossy_utilization_(-1.0),
weak_factory_(this) {
video_encoder_ =
VideoEncoder::Create(cast_environment_, video_config, status_change_cb,
create_vea_cb, create_video_encode_mem_cb);
if (!video_encoder_) {
cast_environment_->PostTask(
CastEnvironment::MAIN, FROM_HERE,
base::Bind(status_change_cb, STATUS_UNSUPPORTED_CODEC));
}
}
VideoSender::~VideoSender() {}
void VideoSender::InsertRawVideoFrame(
const scoped_refptr<media::VideoFrame>& video_frame,
const base::TimeTicks& reference_time) {
DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
if (!video_encoder_) {
NOTREACHED();
return;
}
const RtpTimeTicks rtp_timestamp =
RtpTimeTicks::FromTimeDelta(video_frame->timestamp(), kVideoFrequency);
LogVideoCaptureTimestamps(cast_environment_.get(), *video_frame,
rtp_timestamp);
// Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
TRACE_EVENT_INSTANT2("cast_perf_test", "InsertRawVideoFrame",
TRACE_EVENT_SCOPE_THREAD, "timestamp",
reference_time.ToInternalValue(), "rtp_timestamp",
rtp_timestamp.lower_32_bits());
bool low_latency_mode;
if (video_frame->metadata()->GetBoolean(
VideoFrameMetadata::INTERACTIVE_CONTENT, &low_latency_mode)) {
if (low_latency_mode && !low_latency_mode_) {
VLOG(1) << "Interactive mode playout time " << min_playout_delay_;
playout_delay_change_cb_.Run(min_playout_delay_);
}
low_latency_mode_ = low_latency_mode;
}
// Drop the frame if either its RTP or reference timestamp is not an increase
// over the last frame's. This protects: 1) the duration calculations that
// assume timestamps are monotonically non-decreasing, and 2) assumptions made
// deeper in the implementation where each frame's RTP timestamp needs to be
// unique.
if (!last_enqueued_frame_reference_time_.is_null() &&
(rtp_timestamp <= last_enqueued_frame_rtp_timestamp_ ||
reference_time <= last_enqueued_frame_reference_time_)) {
VLOG(1) << "Dropping video frame: RTP or reference time did not increase.";
TRACE_EVENT_INSTANT2("cast.stream", "Video Frame Drop",
TRACE_EVENT_SCOPE_THREAD, "rtp_timestamp",
rtp_timestamp.lower_32_bits(), "reason",
"time did not increase");
return;
}
// Request a key frame when a Pli message was received, and it has been passed
// long enough from the last time sending key frame request on receiving a Pli
// message.
if (picture_lost_at_receiver_) {
const int64_t min_attemp_interval_ms =
std::max(kMinKeyFrameRequestOnPliIntervalMs,
6 * target_playout_delay_.InMilliseconds());
if (last_time_attempted_to_resolve_pli_.is_null() ||
((reference_time - last_time_attempted_to_resolve_pli_)
.InMilliseconds() > min_attemp_interval_ms)) {
video_encoder_->GenerateKeyFrame();
last_time_attempted_to_resolve_pli_ = reference_time;
}
}
// Two video frames are needed to compute the exact media duration added by
// the next frame. If there are no frames in the encoder, compute a guess
// based on the configured |max_frame_rate_|. Any error introduced by this
// guess will be eliminated when |duration_in_encoder_| is updated in
// OnEncodedVideoFrame().
const base::TimeDelta duration_added_by_next_frame =
frames_in_encoder_ > 0
? reference_time - last_enqueued_frame_reference_time_
: base::TimeDelta::FromSecondsD(1.0 / max_frame_rate_);
if (ShouldDropNextFrame(duration_added_by_next_frame)) {
base::TimeDelta new_target_delay =
std::min(current_round_trip_time_ * kRoundTripsNeeded +
base::TimeDelta::FromMilliseconds(kConstantTimeMs),
max_playout_delay_);
// In case of low latency mode, we prefer frame drops over increasing
// playout time.
if (!low_latency_mode_ && new_target_delay > target_playout_delay_) {
// In case we detect user is no longer in a low latency mode and there is
// a need to drop a frame, we ensure the playout delay is at-least the
// the starting value for playing animated content.
// This is intended to minimize freeze when moving from an interactive
// session to watching animating content while being limited by end-to-end
// delay.
VLOG(1) << "Ensure playout time is at least " << animated_playout_delay_;
if (new_target_delay < animated_playout_delay_)
new_target_delay = animated_playout_delay_;
VLOG(1) << "New target delay: " << new_target_delay.InMilliseconds();
playout_delay_change_cb_.Run(new_target_delay);
}
// Some encoder implementations have a frame window for analysis. Since we
// are dropping this frame, unless we instruct the encoder to flush all the
// frames that have been enqueued for encoding, frames_in_encoder_ and
// last_enqueued_frame_reference_time_ will never be updated and we will
// drop every subsequent frame for the rest of the session.
video_encoder_->EmitFrames();
TRACE_EVENT_INSTANT2("cast.stream", "Video Frame Drop",
TRACE_EVENT_SCOPE_THREAD, "rtp_timestamp",
rtp_timestamp.lower_32_bits(), "reason",
"too much in flight");
return;
}
if (video_frame->visible_rect().IsEmpty()) {
VLOG(1) << "Rejecting empty video frame.";
return;
}
const int bitrate = congestion_control_->GetBitrate(
reference_time + target_playout_delay_, target_playout_delay_);
if (bitrate != last_bitrate_) {
video_encoder_->SetBitRate(bitrate);
last_bitrate_ = bitrate;
}
TRACE_COUNTER_ID1("cast.stream", "Video Target Bitrate", this, bitrate);
MaybeRenderPerformanceMetricsOverlay(
GetTargetPlayoutDelay(), low_latency_mode_, bitrate,
frames_in_encoder_ + 1, last_reported_encoder_utilization_,
last_reported_lossy_utilization_, video_frame.get());
if (video_encoder_->EncodeVideoFrame(
video_frame, reference_time,
base::Bind(&VideoSender::OnEncodedVideoFrame,
weak_factory_.GetWeakPtr(), video_frame, bitrate))) {
TRACE_EVENT_ASYNC_BEGIN1("cast.stream", "Video Encode", video_frame.get(),
"rtp_timestamp", rtp_timestamp.lower_32_bits());
frames_in_encoder_++;
duration_in_encoder_ += duration_added_by_next_frame;
last_enqueued_frame_rtp_timestamp_ = rtp_timestamp;
last_enqueued_frame_reference_time_ = reference_time;
} else {
VLOG(1) << "Encoder rejected a frame. Skipping...";
TRACE_EVENT_INSTANT1("cast.stream", "Video Encode Reject",
TRACE_EVENT_SCOPE_THREAD, "rtp_timestamp",
rtp_timestamp.lower_32_bits());
}
}
std::unique_ptr<VideoFrameFactory> VideoSender::CreateVideoFrameFactory() {
return video_encoder_ ? video_encoder_->CreateVideoFrameFactory() : nullptr;
}
int VideoSender::GetNumberOfFramesInEncoder() const {
return frames_in_encoder_;
}
base::TimeDelta VideoSender::GetInFlightMediaDuration() const {
if (GetUnacknowledgedFrameCount() > 0) {
const FrameId oldest_unacked_frame_id = latest_acked_frame_id_ + 1;
return last_enqueued_frame_reference_time_ -
GetRecordedReferenceTime(oldest_unacked_frame_id);
} else {
return duration_in_encoder_;
}
}
void VideoSender::OnEncodedVideoFrame(
const scoped_refptr<media::VideoFrame>& video_frame, int encoder_bitrate,
std::unique_ptr<SenderEncodedFrame> encoded_frame) {
DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
frames_in_encoder_--;
DCHECK_GE(frames_in_encoder_, 0);
// Encoding was exited with errors.
if (!encoded_frame) return;
duration_in_encoder_ =
last_enqueued_frame_reference_time_ - encoded_frame->reference_time;
last_reported_encoder_utilization_ = encoded_frame->encoder_utilization;
last_reported_lossy_utilization_ = encoded_frame->lossy_utilization;
TRACE_EVENT_ASYNC_END2("cast.stream", "Video Encode", video_frame.get(),
"encoder_utilization",
last_reported_encoder_utilization_,
"lossy_utilization", last_reported_lossy_utilization_);
// Report the resource utilization for processing this frame. Take the
// greater of the two utilization values and attenuate them such that the
// target utilization is reported as the maximum sustainable amount.
const double attenuated_utilization =
std::max(last_reported_encoder_utilization_,
last_reported_lossy_utilization_) /
(kTargetUtilizationPercentage / 100.0);
if (attenuated_utilization >= 0.0) {
// Key frames are artificially capped to 1.0 because their actual
// utilization is atypical compared to the other frames in the stream, and
// this can misguide the producer of the input video frames.
video_frame->metadata()->SetDouble(
media::VideoFrameMetadata::RESOURCE_UTILIZATION,
encoded_frame->dependency == EncodedFrame::KEY
? std::min(1.0, attenuated_utilization)
: attenuated_utilization);
}
SendEncodedFrame(encoder_bitrate, std::move(encoded_frame));
}
} // namespace cast
} // namespace media
} // namespace cobalt