blob: d95e6e5765c96a0b0987acfa321384159038439a [file] [log] [blame]
// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "media/muxers/webm_muxer.h"
#include <algorithm>
#include <memory>
#include "base/bind.h"
#include "base/check.h"
#include "base/logging.h"
#include "base/sequence_checker.h"
#include "base/time/time.h"
#include "base/time/time_override.h"
#include "media/base/audio_parameters.h"
#include "media/base/limits.h"
#include "media/base/video_frame.h"
#include "media/formats/common/opus_constants.h"
namespace media {
namespace {
namespace av1 {
// CodecPrivate for AV1. See
// https://github.com/ietf-wg-cellar/matroska-specification/blob/master/codec/av1.md.
constexpr int high_bitdepth = 0;
constexpr int twelve_bit = 0;
constexpr int monochrome = 0;
constexpr int initial_presentation_delay_present = 0;
constexpr int initial_presentation_delay_minus_one = 0;
constexpr int chroma_sample_position = 0;
constexpr int seq_profile = 0; // Main
constexpr int seq_level_idx_0 = 9; // level 4.1 ~1920x1080@60fps
constexpr int seq_tier_0 = 0;
constexpr int chroma_subsampling_x = 1;
constexpr int chroma_subsampling_y = 1;
constexpr uint8_t codec_private[4] = {
255, //
(seq_profile << 5) | seq_level_idx_0, //
(seq_tier_0 << 7) | //
(high_bitdepth << 6) | //
(twelve_bit << 5) | //
(monochrome << 4) | //
(chroma_subsampling_x << 3) | //
(chroma_subsampling_y << 2) | //
chroma_sample_position, //
(initial_presentation_delay_present << 4) | //
initial_presentation_delay_minus_one //
};
} // namespace av1
// Force new clusters at a maximum rate of 10 Hz.
constexpr base::TimeDelta kMinimumForcedClusterDuration =
base::Milliseconds(100);
void WriteOpusHeader(const media::AudioParameters& params, uint8_t* header) {
// See https://wiki.xiph.org/OggOpus#ID_Header.
// Set magic signature.
std::string label = "OpusHead";
memcpy(header + OPUS_EXTRADATA_LABEL_OFFSET, label.c_str(), label.size());
// Set Opus version.
header[OPUS_EXTRADATA_VERSION_OFFSET] = 1;
// Set channel count.
DCHECK_LE(params.channels(), 2);
header[OPUS_EXTRADATA_CHANNELS_OFFSET] = params.channels();
// Set pre-skip
uint16_t skip = 0;
memcpy(header + OPUS_EXTRADATA_SKIP_SAMPLES_OFFSET, &skip, sizeof(uint16_t));
// Set original input sample rate in Hz.
uint32_t sample_rate = params.sample_rate();
memcpy(header + OPUS_EXTRADATA_SAMPLE_RATE_OFFSET, &sample_rate,
sizeof(uint32_t));
// Set output gain in dB.
uint16_t gain = 0;
memcpy(header + OPUS_EXTRADATA_GAIN_OFFSET, &gain, 2);
header[OPUS_EXTRADATA_CHANNEL_MAPPING_OFFSET] = 0;
}
static double GetFrameRate(const WebmMuxer::VideoParameters& params) {
const double kZeroFrameRate = 0.0;
const double kDefaultFrameRate = 30.0;
double frame_rate = params.frame_rate;
if (frame_rate <= kZeroFrameRate ||
frame_rate > media::limits::kMaxFramesPerSecond) {
frame_rate = kDefaultFrameRate;
}
return frame_rate;
}
static const char kH264CodecId[] = "V_MPEG4/ISO/AVC";
static const char kPcmCodecId[] = "A_PCM/FLOAT/IEEE";
static const char* MkvCodeIcForMediaVideoCodecId(VideoCodec video_codec) {
switch (video_codec) {
case VideoCodec::kVP8:
return mkvmuxer::Tracks::kVp8CodecId;
case VideoCodec::kVP9:
return mkvmuxer::Tracks::kVp9CodecId;
case VideoCodec::kAV1:
return mkvmuxer::Tracks::kAv1CodecId;
case VideoCodec::kH264:
return kH264CodecId;
default:
NOTREACHED() << "Unsupported codec " << GetCodecName(video_codec);
return "";
}
}
absl::optional<mkvmuxer::Colour> ColorFromColorSpace(
const gfx::ColorSpace& color) {
using mkvmuxer::Colour;
using MatrixID = gfx::ColorSpace::MatrixID;
using RangeID = gfx::ColorSpace::RangeID;
using TransferID = gfx::ColorSpace::TransferID;
using PrimaryID = gfx::ColorSpace::PrimaryID;
Colour colour;
int matrix_coefficients;
switch (color.GetMatrixID()) {
case MatrixID::BT709:
matrix_coefficients = Colour::kBt709;
break;
case MatrixID::BT2020_NCL:
matrix_coefficients = Colour::kBt2020NonConstantLuminance;
break;
default:
return absl::nullopt;
}
colour.set_matrix_coefficients(matrix_coefficients);
int range;
switch (color.GetRangeID()) {
case RangeID::LIMITED:
range = Colour::kBroadcastRange;
break;
case RangeID::FULL:
range = Colour::kFullRange;
break;
default:
return absl::nullopt;
}
colour.set_range(range);
int transfer_characteristics;
switch (color.GetTransferID()) {
case TransferID::BT709:
transfer_characteristics = Colour::kIturBt709Tc;
break;
case TransferID::IEC61966_2_1:
transfer_characteristics = Colour::kIec6196621;
break;
case TransferID::SMPTEST2084:
transfer_characteristics = Colour::kSmpteSt2084;
break;
default:
return absl::nullopt;
}
colour.set_transfer_characteristics(transfer_characteristics);
int primaries;
switch (color.GetPrimaryID()) {
case PrimaryID::BT709:
primaries = Colour::kIturBt709P;
break;
case PrimaryID::BT2020:
primaries = Colour::kIturBt2020;
break;
default:
return absl::nullopt;
}
colour.set_primaries(primaries);
return colour;
}
} // anonymous namespace
// -----------------------------------------------------------------------------
// WebmMuxer::Delegate:
WebmMuxer::Delegate::Delegate() {
// Creation can be done on a different sequence than main activities.
DETACH_FROM_SEQUENCE(sequence_checker_);
}
WebmMuxer::Delegate::~Delegate() = default;
mkvmuxer::int32 WebmMuxer::Delegate::Write(const void* buf,
mkvmuxer::uint32 len) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DVLOG(2) << __func__ << " len " << len;
DCHECK(buf);
last_data_output_timestamp_ = base::TimeTicks::Now();
const auto result = DoWrite(buf, len);
position_ += len;
return result;
}
// -----------------------------------------------------------------------------
// WebmMuxer::VideoParameters:
WebmMuxer::VideoParameters::VideoParameters(
scoped_refptr<media::VideoFrame> frame)
: visible_rect_size(frame->visible_rect().size()),
frame_rate(frame->metadata().frame_rate.value_or(0.0)),
codec(VideoCodec::kUnknown),
color_space(frame->ColorSpace()) {}
WebmMuxer::VideoParameters::VideoParameters(
gfx::Size visible_rect_size,
double frame_rate,
VideoCodec codec,
absl::optional<gfx::ColorSpace> color_space)
: visible_rect_size(visible_rect_size),
frame_rate(frame_rate),
codec(codec),
color_space(color_space) {}
WebmMuxer::VideoParameters::VideoParameters(const VideoParameters&) = default;
WebmMuxer::VideoParameters::~VideoParameters() = default;
// -----------------------------------------------------------------------------
// WebmMuxer:
WebmMuxer::WebmMuxer(AudioCodec audio_codec,
bool has_video,
bool has_audio,
std::unique_ptr<Delegate> delegate)
: audio_codec_(audio_codec),
video_codec_(VideoCodec::kUnknown),
video_track_index_(0),
audio_track_index_(0),
has_video_(has_video),
has_audio_(has_audio),
delegate_(std::move(delegate)),
force_one_libwebm_error_(false) {
DCHECK(has_video_ || has_audio_);
DCHECK(delegate_);
DCHECK(audio_codec == AudioCodec::kOpus || audio_codec == AudioCodec::kPCM)
<< " Unsupported audio codec: " << GetCodecName(audio_codec);
delegate_->InitSegment(&segment_);
mkvmuxer::SegmentInfo* const info = segment_.GetSegmentInfo();
info->set_writing_app("Chrome");
info->set_muxing_app("Chrome");
// Creation can be done on a different sequence than main activities.
DETACH_FROM_SEQUENCE(sequence_checker_);
}
WebmMuxer::~WebmMuxer() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
Flush();
}
void WebmMuxer::SetMaximumDurationToForceDataOutput(base::TimeDelta interval) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
max_data_output_interval_ = std::max(interval, kMinimumForcedClusterDuration);
}
bool WebmMuxer::OnEncodedVideo(const VideoParameters& params,
std::string encoded_data,
std::string encoded_alpha,
base::TimeTicks timestamp,
bool is_key_frame) {
DVLOG(2) << __func__ << " - " << encoded_data.size() << "B";
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK(params.codec == VideoCodec::kVP8 || params.codec == VideoCodec::kVP9 ||
params.codec == VideoCodec::kH264 || params.codec == VideoCodec::kAV1)
<< " Unsupported video codec: " << GetCodecName(params.codec);
DCHECK(video_codec_ == VideoCodec::kUnknown || video_codec_ == params.codec)
<< "Unsupported: codec switched, to: " << GetCodecName(params.codec);
if (encoded_data.size() == 0u) {
DLOG(WARNING) << __func__ << ": zero size encoded frame, skipping";
// Some encoders give sporadic zero-size data, see https://crbug.com/716451.
return true;
}
if (!video_track_index_) {
// |track_index_|, cannot be zero (!), initialize WebmMuxer in that case.
// http://www.matroska.org/technical/specs/index.html#Tracks
video_codec_ = params.codec;
AddVideoTrack(params.visible_rect_size, GetFrameRate(params),
params.color_space);
if (first_frame_timestamp_video_.is_null()) {
// Compensate for time in pause spent before the first frame.
first_frame_timestamp_video_ = timestamp - total_time_in_pause_;
last_frame_timestamp_video_ = first_frame_timestamp_video_;
}
// Add codec private for AV1.
if (params.codec == VideoCodec::kAV1 &&
!segment_.GetTrackByNumber(video_track_index_)
->SetCodecPrivate(av1::codec_private, sizeof(av1::codec_private)))
LOG(ERROR) << __func__ << " failed to set CodecPrivate for AV1.";
}
// TODO(ajose): Support multiple tracks: http://crbug.com/528523
if (has_audio_ && !audio_track_index_) {
DVLOG(1) << __func__ << ": delaying until audio track ready.";
if (is_key_frame) // Upon Key frame reception, empty the encoded queue.
video_frames_.clear();
}
const base::TimeTicks recorded_timestamp =
UpdateLastTimestampMonotonically(timestamp, &last_frame_timestamp_video_);
video_frames_.push_back(EncodedFrame{
std::move(encoded_data), std::move(encoded_alpha),
recorded_timestamp - first_frame_timestamp_video_, is_key_frame});
return PartiallyFlushQueues();
}
bool WebmMuxer::OnEncodedAudio(const media::AudioParameters& params,
std::string encoded_data,
base::TimeTicks timestamp) {
DVLOG(2) << __func__ << " - " << encoded_data.size() << "B";
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
MaybeForceNewCluster();
if (!audio_track_index_) {
AddAudioTrack(params);
if (first_frame_timestamp_audio_.is_null()) {
// Compensate for time in pause spent before the first frame.
first_frame_timestamp_audio_ = timestamp - total_time_in_pause_;
last_frame_timestamp_audio_ = first_frame_timestamp_audio_;
}
}
const base::TimeTicks recorded_timestamp =
UpdateLastTimestampMonotonically(timestamp, &last_frame_timestamp_audio_);
audio_frames_.push_back(
EncodedFrame{encoded_data, std::string(),
recorded_timestamp - first_frame_timestamp_audio_,
/*is_keyframe=*/true});
return PartiallyFlushQueues();
}
void WebmMuxer::SetLiveAndEnabled(bool track_live_and_enabled, bool is_video) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
bool& written_track_live_and_enabled =
is_video ? video_track_live_and_enabled_ : audio_track_live_and_enabled_;
if (written_track_live_and_enabled != track_live_and_enabled) {
DVLOG(1) << __func__ << (is_video ? " video " : " audio ")
<< "track live-and-enabled changed to " << track_live_and_enabled;
}
written_track_live_and_enabled = track_live_and_enabled;
}
void WebmMuxer::Pause() {
DVLOG(1) << __func__;
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (!elapsed_time_in_pause_)
elapsed_time_in_pause_ = std::make_unique<base::ElapsedTimer>();
}
void WebmMuxer::Resume() {
DVLOG(1) << __func__;
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (elapsed_time_in_pause_) {
total_time_in_pause_ += elapsed_time_in_pause_->Elapsed();
elapsed_time_in_pause_.reset();
}
}
bool WebmMuxer::Flush() {
// Depending on the |delegate_|, it can be either non-seekable (i.e. a live
// stream), or seekable (file mode). So calling |segment_.Finalize()| here is
// needed.
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
FlushQueues();
return segment_.Finalize();
}
void WebmMuxer::AddVideoTrack(
const gfx::Size& frame_size,
double frame_rate,
const absl::optional<gfx::ColorSpace>& color_space) {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK_EQ(0u, video_track_index_)
<< "WebmMuxer can only be initialized once.";
video_track_index_ =
segment_.AddVideoTrack(frame_size.width(), frame_size.height(), 0);
if (video_track_index_ <= 0) { // See https://crbug.com/616391.
NOTREACHED() << "Error adding video track";
return;
}
mkvmuxer::VideoTrack* const video_track =
reinterpret_cast<mkvmuxer::VideoTrack*>(
segment_.GetTrackByNumber(video_track_index_));
if (color_space) {
auto colour = ColorFromColorSpace(*color_space);
if (colour)
video_track->SetColour(*colour);
}
DCHECK(video_track);
video_track->set_codec_id(MkvCodeIcForMediaVideoCodecId(video_codec_));
DCHECK_EQ(0ull, video_track->crop_right());
DCHECK_EQ(0ull, video_track->crop_left());
DCHECK_EQ(0ull, video_track->crop_top());
DCHECK_EQ(0ull, video_track->crop_bottom());
DCHECK_EQ(0.0f, video_track->frame_rate());
// Segment's timestamps should be in milliseconds, DCHECK it. See
// http://www.webmproject.org/docs/container/#muxer-guidelines
DCHECK_EQ(1000000ull, segment_.GetSegmentInfo()->timecode_scale());
// Set alpha channel parameters for only VPX (crbug.com/711825).
if (video_codec_ == VideoCodec::kH264)
return;
video_track->SetAlphaMode(mkvmuxer::VideoTrack::kAlpha);
// Alpha channel, if present, is stored in a BlockAdditional next to the
// associated opaque Block, see
// https://matroska.org/technical/specs/index.html#BlockAdditional.
// This follows Method 1 for VP8 encoding of A-channel described on
// http://wiki.webmproject.org/alpha-channel.
video_track->set_max_block_additional_id(1);
}
void WebmMuxer::AddAudioTrack(const media::AudioParameters& params) {
DVLOG(1) << __func__ << " " << params.AsHumanReadableString();
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
DCHECK_EQ(0u, audio_track_index_)
<< "WebmMuxer audio can only be initialised once.";
audio_track_index_ =
segment_.AddAudioTrack(params.sample_rate(), params.channels(), 0);
if (audio_track_index_ <= 0) { // See https://crbug.com/616391.
NOTREACHED() << "Error adding audio track";
return;
}
mkvmuxer::AudioTrack* const audio_track =
reinterpret_cast<mkvmuxer::AudioTrack*>(
segment_.GetTrackByNumber(audio_track_index_));
DCHECK(audio_track);
DCHECK_EQ(params.sample_rate(), audio_track->sample_rate());
DCHECK_EQ(params.channels(), static_cast<int>(audio_track->channels()));
DCHECK_LE(params.channels(), 2)
<< "Only 1 or 2 channels supported, requested " << params.channels();
// Audio data is always pcm_f32le.
audio_track->set_bit_depth(32u);
if (audio_codec_ == AudioCodec::kOpus) {
audio_track->set_codec_id(mkvmuxer::Tracks::kOpusCodecId);
uint8_t opus_header[OPUS_EXTRADATA_SIZE];
WriteOpusHeader(params, opus_header);
if (!audio_track->SetCodecPrivate(opus_header, OPUS_EXTRADATA_SIZE))
LOG(ERROR) << __func__ << ": failed to set opus header.";
// Segment's timestamps should be in milliseconds, DCHECK it. See
// http://www.webmproject.org/docs/container/#muxer-guidelines
DCHECK_EQ(1000000ull, segment_.GetSegmentInfo()->timecode_scale());
} else if (audio_codec_ == AudioCodec::kPCM) {
audio_track->set_codec_id(kPcmCodecId);
}
}
void WebmMuxer::FlushQueues() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
while ((!video_frames_.empty() || !audio_frames_.empty()) &&
FlushNextFrame()) {
}
}
bool WebmMuxer::PartiallyFlushQueues() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
// Punt writing until all tracks have been created.
if ((has_audio_ && !audio_track_index_) ||
(has_video_ && !video_track_index_)) {
return true;
}
bool result = true;
// We strictly sort by timestamp unless a track is not live-and-enabled. In
// that case we relax this and allow drainage of the live-and-enabled leg.
while ((!has_video_ || !video_frames_.empty() ||
!video_track_live_and_enabled_) &&
(!has_audio_ || !audio_frames_.empty() ||
!audio_track_live_and_enabled_) &&
result) {
if (video_frames_.empty() && audio_frames_.empty())
return true;
result = FlushNextFrame();
}
return result;
}
bool WebmMuxer::FlushNextFrame() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
base::TimeDelta min_timestamp = base::TimeDelta::Max();
base::circular_deque<EncodedFrame>* queue = &video_frames_;
uint8_t track_index = video_track_index_;
if (!video_frames_.empty())
min_timestamp = video_frames_.front().relative_timestamp;
if (!audio_frames_.empty() &&
audio_frames_.front().relative_timestamp < min_timestamp) {
queue = &audio_frames_;
track_index = audio_track_index_;
}
EncodedFrame frame = std::move(queue->front());
queue->pop_front();
// The logic tracking live-and-enabled that temporarily relaxes the strict
// timestamp sorting allows for draining a track's queue completely in the
// presence of the other track being muted. When the muted track becomes
// live-and-enabled again the sorting recommences. However, tracks get encoded
// data before live-and-enabled transitions to true. This can lead to us
// emitting non-monotonic timestamps to the muxer, which results in an error
// return. Fix this by enforcing monotonicity by rewriting timestamps.
base::TimeDelta relative_timestamp = frame.relative_timestamp;
DLOG_IF(WARNING, relative_timestamp < last_timestamp_written_)
<< "Enforced a monotonically increasing timestamp. Last written "
<< last_timestamp_written_ << " new " << relative_timestamp;
relative_timestamp = std::max(relative_timestamp, last_timestamp_written_);
last_timestamp_written_ = relative_timestamp;
auto recorded_timestamp = relative_timestamp.InMicroseconds() *
base::Time::kNanosecondsPerMicrosecond;
if (force_one_libwebm_error_) {
DVLOG(1) << "Forcing a libwebm error";
force_one_libwebm_error_ = false;
return false;
}
DCHECK(frame.data.data());
bool result =
frame.alpha_data.empty()
? segment_.AddFrame(
reinterpret_cast<const uint8_t*>(frame.data.data()),
frame.data.size(), track_index, recorded_timestamp,
frame.is_keyframe)
: segment_.AddFrameWithAdditional(
reinterpret_cast<const uint8_t*>(frame.data.data()),
frame.data.size(),
reinterpret_cast<const uint8_t*>(frame.alpha_data.data()),
frame.alpha_data.size(), 1 /* add_id */, track_index,
recorded_timestamp, frame.is_keyframe);
return result;
}
base::TimeTicks WebmMuxer::UpdateLastTimestampMonotonically(
base::TimeTicks timestamp,
base::TimeTicks* last_timestamp) {
base::TimeTicks compensated_timestamp = timestamp - total_time_in_pause_;
// In theory, time increases monotonically. In practice, it does not.
// See http://crbug/618407.
DLOG_IF(WARNING, compensated_timestamp < *last_timestamp)
<< "Encountered a non-monotonically increasing timestamp. Was: "
<< *last_timestamp << ", compensated: " << compensated_timestamp
<< ", uncompensated: " << timestamp;
*last_timestamp = std::max(*last_timestamp, compensated_timestamp);
return *last_timestamp;
}
void WebmMuxer::MaybeForceNewCluster() {
DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
if (!has_video_ || max_data_output_interval_.is_zero() ||
delegate_->last_data_output_timestamp().is_null()) {
return;
}
if (base::TimeTicks::Now() - delegate_->last_data_output_timestamp() >=
max_data_output_interval_) {
segment_.ForceNewClusterOnNextFrame();
}
}
} // namespace media