| // Copyright 2016 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "media/filters/android/media_codec_audio_decoder.h" |
| |
| #include <cmath> |
| #include <memory> |
| |
| #include "base/android/build_info.h" |
| #include "base/bind.h" |
| #include "base/callback_helpers.h" |
| #include "base/logging.h" |
| #include "base/single_thread_task_runner.h" |
| #include "base/threading/thread_task_runner_handle.h" |
| #include "media/base/android/media_codec_bridge_impl.h" |
| #include "media/base/android/media_codec_util.h" |
| #include "media/base/audio_timestamp_helper.h" |
| #include "media/base/bind_to_current_loop.h" |
| #include "media/base/status.h" |
| #include "media/base/timestamp_constants.h" |
| #include "media/formats/ac3/ac3_util.h" |
| |
| namespace media { |
| |
| MediaCodecAudioDecoder::MediaCodecAudioDecoder( |
| scoped_refptr<base::SingleThreadTaskRunner> task_runner) |
| : task_runner_(task_runner), |
| state_(STATE_UNINITIALIZED), |
| is_passthrough_(false), |
| sample_format_(kSampleFormatS16), |
| channel_count_(0), |
| channel_layout_(CHANNEL_LAYOUT_NONE), |
| sample_rate_(0), |
| media_crypto_context_(nullptr), |
| pool_(new AudioBufferMemoryPool()) { |
| DVLOG(1) << __func__; |
| } |
| |
| MediaCodecAudioDecoder::~MediaCodecAudioDecoder() { |
| DVLOG(1) << __func__; |
| |
| codec_loop_.reset(); |
| |
| // Cancel previously registered callback (if any). |
| if (media_crypto_context_) |
| media_crypto_context_->SetMediaCryptoReadyCB(base::NullCallback()); |
| |
| ClearInputQueue(DecodeStatus::ABORTED); |
| } |
| |
| AudioDecoderType MediaCodecAudioDecoder::GetDecoderType() const { |
| return AudioDecoderType::kMediaCodec; |
| } |
| |
| void MediaCodecAudioDecoder::Initialize(const AudioDecoderConfig& config, |
| CdmContext* cdm_context, |
| InitCB init_cb, |
| const OutputCB& output_cb, |
| const WaitingCB& waiting_cb) { |
| DVLOG(1) << __func__ << ": " << config.AsHumanReadableString(); |
| DCHECK_NE(state_, STATE_WAITING_FOR_MEDIA_CRYPTO); |
| DCHECK(output_cb); |
| DCHECK(waiting_cb); |
| |
| // Initialization and reinitialization should not be called during pending |
| // decode. |
| DCHECK(input_queue_.empty()); |
| ClearInputQueue(DecodeStatus::ABORTED); |
| |
| is_passthrough_ = MediaCodecUtil::IsPassthroughAudioFormat(config.codec()); |
| sample_format_ = kSampleFormatS16; |
| |
| if (config.codec() == AudioCodec::kAC3) |
| sample_format_ = kSampleFormatAc3; |
| else if (config.codec() == AudioCodec::kEAC3) |
| sample_format_ = kSampleFormatEac3; |
| else if (config.codec() == AudioCodec::kMpegHAudio) |
| sample_format_ = kSampleFormatMpegHAudio; |
| |
| if (state_ == STATE_ERROR) { |
| DVLOG(1) << "Decoder is in error state."; |
| BindToCurrentLoop(std::move(init_cb)) |
| .Run(StatusCode::kDecoderFailedInitialization); |
| return; |
| } |
| |
| // We can support only the codecs that MediaCodecBridge can decode. |
| // TODO(xhwang): Get this list from MediaCodecBridge or just rely on |
| // attempting to create one to determine whether the codec is supported. |
| const bool is_codec_supported = config.codec() == AudioCodec::kVorbis || |
| config.codec() == AudioCodec::kFLAC || |
| config.codec() == AudioCodec::kAAC || |
| config.codec() == AudioCodec::kOpus || |
| is_passthrough_; |
| if (!is_codec_supported) { |
| DVLOG(1) << "Unsuported codec " << GetCodecName(config.codec()); |
| BindToCurrentLoop(std::move(init_cb)) |
| .Run(StatusCode::kDecoderUnsupportedCodec); |
| return; |
| } |
| |
| config_ = config; |
| |
| // TODO(xhwang): Check whether BindToCurrentLoop is needed here. |
| output_cb_ = BindToCurrentLoop(output_cb); |
| waiting_cb_ = BindToCurrentLoop(waiting_cb); |
| |
| SetInitialConfiguration(); |
| |
| if (config_.is_encrypted() && !media_crypto_) { |
| if (!cdm_context || !cdm_context->GetMediaCryptoContext()) { |
| LOG(ERROR) << "The stream is encrypted but there is no CdmContext or " |
| "MediaCryptoContext is not supported"; |
| SetState(STATE_ERROR); |
| BindToCurrentLoop(std::move(init_cb)) |
| .Run(StatusCode::kDecoderMissingCdmForEncryptedContent); |
| return; |
| } |
| |
| // Postpone initialization after MediaCrypto is available. |
| // SetCdm uses init_cb in a method that's already bound to the current loop. |
| SetState(STATE_WAITING_FOR_MEDIA_CRYPTO); |
| SetCdm(cdm_context, std::move(init_cb)); |
| return; |
| } |
| |
| if (!CreateMediaCodecLoop()) { |
| BindToCurrentLoop(std::move(init_cb)) |
| .Run(StatusCode::kDecoderFailedInitialization); |
| return; |
| } |
| |
| SetState(STATE_READY); |
| BindToCurrentLoop(std::move(init_cb)).Run(OkStatus()); |
| } |
| |
| bool MediaCodecAudioDecoder::CreateMediaCodecLoop() { |
| DVLOG(1) << __func__ << ": config:" << config_.AsHumanReadableString(); |
| |
| codec_loop_.reset(); |
| const base::android::JavaRef<jobject>& media_crypto = |
| media_crypto_ ? *media_crypto_ : nullptr; |
| std::unique_ptr<MediaCodecBridge> audio_codec_bridge( |
| MediaCodecBridgeImpl::CreateAudioDecoder( |
| config_, media_crypto, |
| // Use the asynchronous API if we're on Marshallow or higher. |
| base::android::BuildInfo::GetInstance()->sdk_int() >= |
| base::android::SDK_VERSION_MARSHMALLOW |
| ? BindToCurrentLoop(base::BindRepeating( |
| &MediaCodecAudioDecoder::PumpMediaCodecLoop, |
| weak_factory_.GetWeakPtr())) |
| : base::RepeatingClosure())); |
| if (!audio_codec_bridge) { |
| DLOG(ERROR) << __func__ << " failed: cannot create MediaCodecBridge"; |
| return false; |
| } |
| |
| codec_loop_ = std::make_unique<MediaCodecLoop>( |
| base::android::BuildInfo::GetInstance()->sdk_int(), this, |
| std::move(audio_codec_bridge), |
| scoped_refptr<base::SingleThreadTaskRunner>()); |
| |
| return true; |
| } |
| |
| void MediaCodecAudioDecoder::Decode(scoped_refptr<DecoderBuffer> buffer, |
| DecodeCB decode_cb) { |
| DecodeCB bound_decode_cb = BindToCurrentLoop(std::move(decode_cb)); |
| |
| if (!buffer->end_of_stream() && buffer->timestamp() == kNoTimestamp) { |
| DVLOG(2) << __func__ << " " << buffer->AsHumanReadableString() |
| << ": no timestamp, skipping this buffer"; |
| std::move(bound_decode_cb).Run(DecodeStatus::DECODE_ERROR); |
| return; |
| } |
| |
| // Note that we transition to STATE_ERROR if |codec_loop_| does. |
| if (state_ == STATE_ERROR) { |
| // We get here if an error happens in DequeueOutput() or Reset(). |
| DVLOG(2) << __func__ << " " << buffer->AsHumanReadableString() |
| << ": Error state, returning decode error for all buffers"; |
| ClearInputQueue(DecodeStatus::DECODE_ERROR); |
| std::move(bound_decode_cb).Run(DecodeStatus::DECODE_ERROR); |
| return; |
| } |
| |
| DCHECK(codec_loop_); |
| |
| DVLOG(3) << __func__ << " " << buffer->AsHumanReadableString(); |
| |
| DCHECK_EQ(state_, STATE_READY) << " unexpected state " << AsString(state_); |
| |
| // AudioDecoder requires that "Only one decode may be in flight at any given |
| // time". |
| DCHECK(input_queue_.empty()); |
| |
| input_queue_.push_back( |
| std::make_pair(std::move(buffer), std::move(bound_decode_cb))); |
| |
| codec_loop_->ExpectWork(); |
| } |
| |
| void MediaCodecAudioDecoder::Reset(base::OnceClosure closure) { |
| DVLOG(2) << __func__; |
| |
| ClearInputQueue(DecodeStatus::ABORTED); |
| |
| // Flush if we can, otherwise completely recreate and reconfigure the codec. |
| bool success = codec_loop_->TryFlush(); |
| |
| // If the flush failed, then we have to re-create the codec. |
| if (!success) |
| success = CreateMediaCodecLoop(); |
| |
| timestamp_helper_->SetBaseTimestamp(kNoTimestamp); |
| |
| SetState(success ? STATE_READY : STATE_ERROR); |
| |
| task_runner_->PostTask(FROM_HERE, std::move(closure)); |
| } |
| |
| bool MediaCodecAudioDecoder::NeedsBitstreamConversion() const { |
| // An AAC stream needs to be converted as ADTS stream. |
| DCHECK_NE(config_.codec(), AudioCodec::kUnknown); |
| return config_.codec() == AudioCodec::kAAC; |
| } |
| |
| void MediaCodecAudioDecoder::SetCdm(CdmContext* cdm_context, InitCB init_cb) { |
| DVLOG(1) << __func__; |
| DCHECK(cdm_context) << "No CDM provided"; |
| DCHECK(cdm_context->GetMediaCryptoContext()); |
| |
| media_crypto_context_ = cdm_context->GetMediaCryptoContext(); |
| |
| // CdmContext will always post the registered callback back to this thread. |
| event_cb_registration_ = cdm_context->RegisterEventCB(base::BindRepeating( |
| &MediaCodecAudioDecoder::OnCdmContextEvent, weak_factory_.GetWeakPtr())); |
| |
| // The callback will be posted back to this thread via BindToCurrentLoop. |
| media_crypto_context_->SetMediaCryptoReadyCB(media::BindToCurrentLoop( |
| base::BindOnce(&MediaCodecAudioDecoder::OnMediaCryptoReady, |
| weak_factory_.GetWeakPtr(), std::move(init_cb)))); |
| } |
| |
| void MediaCodecAudioDecoder::OnCdmContextEvent(CdmContext::Event event) { |
| DVLOG(1) << __func__; |
| |
| if (event != CdmContext::Event::kHasAdditionalUsableKey) |
| return; |
| |
| // We don't register |codec_loop_| directly with the DRM bridge, since it's |
| // subject to replacement. |
| if (codec_loop_) |
| codec_loop_->OnKeyAdded(); |
| } |
| |
| void MediaCodecAudioDecoder::OnMediaCryptoReady( |
| InitCB init_cb, |
| JavaObjectPtr media_crypto, |
| bool /*requires_secure_video_codec*/) { |
| DVLOG(1) << __func__; |
| |
| DCHECK(state_ == STATE_WAITING_FOR_MEDIA_CRYPTO); |
| DCHECK(media_crypto); |
| |
| if (media_crypto->is_null()) { |
| LOG(ERROR) << "MediaCrypto is not available, can't play encrypted stream."; |
| SetState(STATE_UNINITIALIZED); |
| std::move(init_cb).Run(StatusCode::kDecoderMissingCdmForEncryptedContent); |
| return; |
| } |
| |
| media_crypto_ = std::move(media_crypto); |
| |
| // We assume this is a part of the initialization process, thus MediaCodec |
| // is not created yet. |
| DCHECK(!codec_loop_); |
| |
| // After receiving |media_crypto_| we can configure MediaCodec. |
| if (!CreateMediaCodecLoop()) { |
| SetState(STATE_UNINITIALIZED); |
| std::move(init_cb).Run(StatusCode::kDecoderFailedInitialization); |
| return; |
| } |
| |
| SetState(STATE_READY); |
| std::move(init_cb).Run(OkStatus()); |
| } |
| |
| bool MediaCodecAudioDecoder::IsAnyInputPending() const { |
| if (state_ != STATE_READY) |
| return false; |
| |
| return !input_queue_.empty(); |
| } |
| |
| MediaCodecLoop::InputData MediaCodecAudioDecoder::ProvideInputData() { |
| DVLOG(3) << __func__; |
| |
| const DecoderBuffer* decoder_buffer = input_queue_.front().first.get(); |
| |
| MediaCodecLoop::InputData input_data; |
| if (decoder_buffer->end_of_stream()) { |
| input_data.is_eos = true; |
| } else { |
| input_data.memory = static_cast<const uint8_t*>(decoder_buffer->data()); |
| input_data.length = decoder_buffer->data_size(); |
| const DecryptConfig* decrypt_config = decoder_buffer->decrypt_config(); |
| if (decrypt_config) { |
| input_data.key_id = decrypt_config->key_id(); |
| input_data.iv = decrypt_config->iv(); |
| input_data.subsamples = decrypt_config->subsamples(); |
| input_data.encryption_scheme = decrypt_config->encryption_scheme(); |
| input_data.encryption_pattern = decrypt_config->encryption_pattern(); |
| } |
| input_data.presentation_time = decoder_buffer->timestamp(); |
| } |
| |
| // We do not pop |input_queue_| here. MediaCodecLoop may refer to data that |
| // it owns until OnInputDataQueued is called. |
| |
| return input_data; |
| } |
| |
| void MediaCodecAudioDecoder::OnInputDataQueued(bool success) { |
| // If this is an EOS buffer, then wait to call back until we are notified that |
| // it has been processed via OnDecodedEos(). If the EOS was not queued |
| // successfully, then we do want to signal error now since there is no queued |
| // EOS to process later. |
| if (input_queue_.front().first->end_of_stream() && success) |
| return; |
| |
| std::move(input_queue_.front().second) |
| .Run(success ? DecodeStatus::OK : DecodeStatus::DECODE_ERROR); |
| input_queue_.pop_front(); |
| } |
| |
| void MediaCodecAudioDecoder::ClearInputQueue(DecodeStatus decode_status) { |
| DVLOG(2) << __func__; |
| |
| for (auto& entry : input_queue_) |
| std::move(entry.second).Run(decode_status); |
| |
| input_queue_.clear(); |
| } |
| |
| void MediaCodecAudioDecoder::SetState(State new_state) { |
| DVLOG(3) << __func__ << ": " << AsString(state_) << "->" |
| << AsString(new_state); |
| state_ = new_state; |
| } |
| |
| void MediaCodecAudioDecoder::OnCodecLoopError() { |
| // If the codec transitions into the error state, then so should we. |
| SetState(STATE_ERROR); |
| ClearInputQueue(DecodeStatus::DECODE_ERROR); |
| } |
| |
| bool MediaCodecAudioDecoder::OnDecodedEos( |
| const MediaCodecLoop::OutputBuffer& out) { |
| DVLOG(2) << __func__ << " pts:" << out.pts; |
| |
| // Rarely, we seem to get multiple EOSes or, possibly, unsolicited ones from |
| // MediaCodec. Just transition to the error state. |
| // https://crbug.com/818866 |
| if (!input_queue_.size() || !input_queue_.front().first->end_of_stream()) { |
| LOG(WARNING) << "MCAD received unexpected eos"; |
| return false; |
| } |
| |
| // If we've transitioned into the error state, then we don't really know what |
| // to do. If we transitioned because of OnCodecError, then all of our |
| // buffers have been returned anyway. Otherwise, it's unclear. Note that |
| // MCL does not call us back after OnCodecError(), since it stops decoding. |
| // So, we shouldn't be in that state. So, just DCHECK here. |
| DCHECK_NE(state_, STATE_ERROR); |
| |
| std::move(input_queue_.front()).second.Run(DecodeStatus::OK); |
| input_queue_.pop_front(); |
| |
| return true; |
| } |
| |
| bool MediaCodecAudioDecoder::OnDecodedFrame( |
| const MediaCodecLoop::OutputBuffer& out) { |
| DVLOG(3) << __func__ << " pts:" << out.pts; |
| |
| DCHECK_NE(out.size, 0U); |
| DCHECK_NE(out.index, MediaCodecLoop::kInvalidBufferIndex); |
| DCHECK(codec_loop_); |
| MediaCodecBridge* media_codec = codec_loop_->GetCodec(); |
| DCHECK(media_codec); |
| |
| // For proper |frame_count| calculation we need to use the actual number |
| // of channels which can be different from |config_| value. |
| DCHECK_GT(channel_count_, 0); |
| |
| size_t frame_count = 1; |
| scoped_refptr<AudioBuffer> audio_buffer; |
| |
| if (is_passthrough_) { |
| audio_buffer = AudioBuffer::CreateBitstreamBuffer( |
| sample_format_, channel_layout_, channel_count_, sample_rate_, |
| frame_count, out.size, pool_); |
| |
| MediaCodecStatus status = media_codec->CopyFromOutputBuffer( |
| out.index, out.offset, audio_buffer->channel_data()[0], out.size); |
| |
| if (status != MEDIA_CODEC_OK) { |
| media_codec->ReleaseOutputBuffer(out.index, false); |
| return false; |
| } |
| |
| if (config_.codec() == AudioCodec::kAC3) { |
| frame_count = Ac3Util::ParseTotalAc3SampleCount( |
| audio_buffer->channel_data()[0], out.size); |
| } else if (config_.codec() == AudioCodec::kEAC3) { |
| frame_count = Ac3Util::ParseTotalEac3SampleCount( |
| audio_buffer->channel_data()[0], out.size); |
| } else { |
| NOTREACHED() << "Unsupported passthrough format."; |
| } |
| |
| // Create AudioOutput buffer based on current parameters. |
| audio_buffer = AudioBuffer::CreateBitstreamBuffer( |
| sample_format_, channel_layout_, channel_count_, sample_rate_, |
| frame_count, out.size, pool_); |
| } else { |
| // Android MediaCodec can only output 16bit PCM audio. |
| const int bytes_per_frame = sizeof(uint16_t) * channel_count_; |
| frame_count = out.size / bytes_per_frame; |
| |
| // Create AudioOutput buffer based on current parameters. |
| audio_buffer = AudioBuffer::CreateBuffer(sample_format_, channel_layout_, |
| channel_count_, sample_rate_, |
| frame_count, pool_); |
| } |
| |
| // Copy data into AudioBuffer. |
| CHECK_LE(out.size, audio_buffer->data_size()); |
| |
| MediaCodecStatus status = media_codec->CopyFromOutputBuffer( |
| out.index, out.offset, audio_buffer->channel_data()[0], out.size); |
| |
| // Release MediaCodec output buffer. |
| media_codec->ReleaseOutputBuffer(out.index, false); |
| |
| if (status != MEDIA_CODEC_OK) |
| return false; |
| |
| // Calculate and set buffer timestamp. |
| |
| const bool first_buffer = timestamp_helper_->base_timestamp() == kNoTimestamp; |
| if (first_buffer) { |
| // Clamp the base timestamp to zero. |
| timestamp_helper_->SetBaseTimestamp(std::max(base::TimeDelta(), out.pts)); |
| } |
| |
| audio_buffer->set_timestamp(timestamp_helper_->GetTimestamp()); |
| timestamp_helper_->AddFrames(frame_count); |
| |
| // Call the |output_cb_|. |
| output_cb_.Run(audio_buffer); |
| |
| return true; |
| } |
| |
| void MediaCodecAudioDecoder::OnWaiting(WaitingReason reason) { |
| DVLOG(2) << __func__; |
| waiting_cb_.Run(reason); |
| } |
| |
| bool MediaCodecAudioDecoder::OnOutputFormatChanged() { |
| DVLOG(2) << __func__; |
| MediaCodecBridge* media_codec = codec_loop_->GetCodec(); |
| |
| // Note that if we return false to transition |codec_loop_| to the error |
| // state, then we'll also transition to the error state when it notifies us. |
| |
| int new_sampling_rate = 0; |
| MediaCodecStatus status = |
| media_codec->GetOutputSamplingRate(&new_sampling_rate); |
| if (status != MEDIA_CODEC_OK) { |
| DLOG(ERROR) << "GetOutputSamplingRate failed."; |
| return false; |
| } |
| if (new_sampling_rate != sample_rate_) { |
| DVLOG(1) << __func__ << ": detected sample rate change: " << sample_rate_ |
| << " -> " << new_sampling_rate; |
| |
| sample_rate_ = new_sampling_rate; |
| const base::TimeDelta base_timestamp = |
| timestamp_helper_->base_timestamp() == kNoTimestamp |
| ? kNoTimestamp |
| : timestamp_helper_->GetTimestamp(); |
| timestamp_helper_ = std::make_unique<AudioTimestampHelper>(sample_rate_); |
| if (base_timestamp != kNoTimestamp) |
| timestamp_helper_->SetBaseTimestamp(base_timestamp); |
| } |
| |
| int new_channel_count = 0; |
| status = media_codec->GetOutputChannelCount(&new_channel_count); |
| if (status != MEDIA_CODEC_OK || !new_channel_count) { |
| DLOG(ERROR) << "GetOutputChannelCount failed."; |
| return false; |
| } |
| |
| if (new_channel_count != channel_count_) { |
| DVLOG(1) << __func__ |
| << ": detected channel count change: " << channel_count_ << " -> " |
| << new_channel_count; |
| channel_count_ = new_channel_count; |
| channel_layout_ = GuessChannelLayout(channel_count_); |
| } |
| |
| return true; |
| } |
| |
| void MediaCodecAudioDecoder::SetInitialConfiguration() { |
| // Guess the channel count from |config_| in case OnOutputFormatChanged |
| // that delivers the true count is not called before the first data arrives. |
| // It seems upon certain input errors a codec may substitute silence and |
| // not call OnOutputFormatChanged in this case. |
| channel_layout_ = config_.channel_layout(); |
| channel_count_ = ChannelLayoutToChannelCount(channel_layout_); |
| |
| sample_rate_ = config_.samples_per_second(); |
| timestamp_helper_ = std::make_unique<AudioTimestampHelper>(sample_rate_); |
| } |
| |
| void MediaCodecAudioDecoder::PumpMediaCodecLoop() { |
| codec_loop_->ExpectWork(); |
| } |
| |
| #undef RETURN_STRING |
| #define RETURN_STRING(x) \ |
| case x: \ |
| return #x; |
| |
| // static |
| const char* MediaCodecAudioDecoder::AsString(State state) { |
| switch (state) { |
| RETURN_STRING(STATE_UNINITIALIZED); |
| RETURN_STRING(STATE_WAITING_FOR_MEDIA_CRYPTO); |
| RETURN_STRING(STATE_READY); |
| RETURN_STRING(STATE_ERROR); |
| } |
| NOTREACHED() << "Unknown state " << state; |
| return nullptr; |
| } |
| |
| #undef RETURN_STRING |
| |
| } // namespace media |