| // Copyright 2013 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "media/gpu/android/android_video_encode_accelerator.h" |
| |
| #include <memory> |
| #include <set> |
| #include <tuple> |
| |
| #include "base/bind.h" |
| #include "base/location.h" |
| #include "base/logging.h" |
| #include "base/metrics/histogram_macros.h" |
| #include "base/single_thread_task_runner.h" |
| #include "base/threading/thread_task_runner_handle.h" |
| #include "gpu/command_buffer/service/gles2_cmd_decoder.h" |
| #include "gpu/ipc/service/gpu_channel.h" |
| #include "media/base/android/media_codec_util.h" |
| #include "media/base/bitstream_buffer.h" |
| #include "media/base/limits.h" |
| #include "media/base/unaligned_shared_memory.h" |
| #include "media/video/picture.h" |
| #include "third_party/libyuv/include/libyuv/convert_from.h" |
| #include "ui/gl/android/scoped_java_surface.h" |
| #include "ui/gl/gl_bindings.h" |
| |
| namespace media { |
| |
| // Limit default max video codec size for Android to avoid |
| // HW codec initialization failure for resolution higher than 720p. |
| // Default values are from Libjingle "jsepsessiondescription.cc". |
| const int kMaxEncodeFrameWidth = 1280; |
| const int kMaxEncodeFrameHeight = 720; |
| const int kMaxFramerateNumerator = 30; |
| const int kMaxFramerateDenominator = 1; |
| |
| enum PixelFormat { |
| // Subset of MediaCodecInfo.CodecCapabilities. |
| COLOR_FORMAT_YUV420_PLANAR = 19, |
| COLOR_FORMAT_YUV420_SEMIPLANAR = 21, |
| }; |
| |
| // Helper macros for dealing with failure. If |result| evaluates false, emit |
| // |log| to DLOG(ERROR), register |error| with the client, and return. |
| #define RETURN_ON_FAILURE(result, log, error) \ |
| do { \ |
| if (!(result)) { \ |
| DLOG(ERROR) << log; \ |
| if (!error_occurred_) { \ |
| client_ptr_factory_->GetWeakPtr()->NotifyError(error); \ |
| error_occurred_ = true; \ |
| } \ |
| return; \ |
| } \ |
| } while (0) |
| |
| // Because MediaCodec is thread-hostile (must be poked on a single thread) and |
| // has no callback mechanism (b/11990118), we must drive it by polling for |
| // complete frames (and available input buffers, when the codec is fully |
| // saturated). This function defines the polling delay. The value used is an |
| // arbitrary choice that trades off CPU utilization (spinning) against latency. |
| // Mirrors android_video_decode_accelerator.cc::DecodePollDelay(). |
| static inline const base::TimeDelta EncodePollDelay() { |
| // An alternative to this polling scheme could be to dedicate a new thread |
| // (instead of using the ChildThread) to run the MediaCodec, and make that |
| // thread use the timeout-based flavor of MediaCodec's dequeue methods when it |
| // believes the codec should complete "soon" (e.g. waiting for an input |
| // buffer, or waiting for a picture when it knows enough complete input |
| // pictures have been fed to saturate any internal buffering). This is |
| // speculative and it's unclear that this would be a win (nor that there's a |
| // reasonably device-agnostic way to fill in the "believes" above). |
| return base::Milliseconds(10); |
| } |
| |
| static inline const base::TimeDelta NoWaitTimeOut() { |
| return base::Microseconds(0); |
| } |
| |
| static bool GetSupportedColorFormatForMime(const std::string& mime, |
| PixelFormat* pixel_format) { |
| if (mime.empty()) |
| return false; |
| |
| std::set<int> formats = MediaCodecUtil::GetEncoderColorFormats(mime); |
| if (formats.count(COLOR_FORMAT_YUV420_SEMIPLANAR) > 0) |
| *pixel_format = COLOR_FORMAT_YUV420_SEMIPLANAR; |
| else if (formats.count(COLOR_FORMAT_YUV420_PLANAR) > 0) |
| *pixel_format = COLOR_FORMAT_YUV420_PLANAR; |
| else |
| return false; |
| |
| return true; |
| } |
| |
| AndroidVideoEncodeAccelerator::AndroidVideoEncodeAccelerator() |
| : num_buffers_at_codec_(0), last_set_bitrate_(0), error_occurred_(false) {} |
| |
| AndroidVideoEncodeAccelerator::~AndroidVideoEncodeAccelerator() { |
| DCHECK(thread_checker_.CalledOnValidThread()); |
| } |
| |
| VideoEncodeAccelerator::SupportedProfiles |
| AndroidVideoEncodeAccelerator::GetSupportedProfiles() { |
| SupportedProfiles profiles; |
| |
| const struct { |
| const VideoCodec codec; |
| const VideoCodecProfile profile; |
| } kSupportedCodecs[] = {{VideoCodec::kVP8, VP8PROFILE_ANY}, |
| {VideoCodec::kH264, H264PROFILE_BASELINE}}; |
| |
| for (const auto& supported_codec : kSupportedCodecs) { |
| if (supported_codec.codec == VideoCodec::kVP8 && |
| !MediaCodecUtil::IsVp8EncoderAvailable()) { |
| continue; |
| } |
| |
| if (supported_codec.codec == VideoCodec::kH264 && |
| !MediaCodecUtil::IsH264EncoderAvailable()) { |
| continue; |
| } |
| |
| if (MediaCodecUtil::IsKnownUnaccelerated(supported_codec.codec, |
| MediaCodecDirection::ENCODER)) { |
| continue; |
| } |
| |
| SupportedProfile profile; |
| profile.profile = supported_codec.profile; |
| // It would be nice if MediaCodec exposes the maximum capabilities of |
| // the encoder. Hard-code some reasonable defaults as workaround. |
| profile.max_resolution.SetSize(kMaxEncodeFrameWidth, kMaxEncodeFrameHeight); |
| profile.max_framerate_numerator = kMaxFramerateNumerator; |
| profile.max_framerate_denominator = kMaxFramerateDenominator; |
| profiles.push_back(profile); |
| } |
| return profiles; |
| } |
| |
| bool AndroidVideoEncodeAccelerator::Initialize(const Config& config, |
| Client* client) { |
| DVLOG(3) << __func__ << " " << config.AsHumanReadableString(); |
| DCHECK(!media_codec_); |
| DCHECK(thread_checker_.CalledOnValidThread()); |
| DCHECK(client); |
| |
| client_ptr_factory_ = std::make_unique<base::WeakPtrFactory<Client>>(client); |
| |
| if (config.input_format != PIXEL_FORMAT_I420) { |
| DLOG(ERROR) << "Unexpected combo: " << config.input_format << ", " |
| << GetProfileName(config.output_profile); |
| return false; |
| } |
| |
| std::string mime_type; |
| VideoCodec codec; |
| // The client should be prepared to feed at least this many frames into the |
| // encoder before being returned any output frames, since the encoder may |
| // need to hold onto some subset of inputs as reference pictures. |
| uint32_t frame_input_count; |
| uint32_t i_frame_interval; |
| if (config.output_profile == VP8PROFILE_ANY) { |
| codec = VideoCodec::kVP8; |
| mime_type = "video/x-vnd.on2.vp8"; |
| frame_input_count = 1; |
| i_frame_interval = IFRAME_INTERVAL_VPX; |
| } else if (config.output_profile == H264PROFILE_BASELINE || |
| config.output_profile == H264PROFILE_MAIN) { |
| codec = VideoCodec::kH264; |
| mime_type = "video/avc"; |
| frame_input_count = 30; |
| i_frame_interval = IFRAME_INTERVAL_H264; |
| } else { |
| return false; |
| } |
| |
| // Non 16x16 aligned resolutions don't work with MediaCodec unfortunately, see |
| // https://crbug.com/1084702 for details. |
| if (config.input_visible_size.width() % 16 != 0 || |
| config.input_visible_size.height() % 16 != 0) { |
| DLOG(ERROR) << "MediaCodec is only tested with resolutions " |
| "that are 16x16 aligned."; |
| return false; |
| } |
| |
| frame_size_ = config.input_visible_size; |
| last_set_bitrate_ = config.bitrate.target(); |
| |
| // Only consider using MediaCodec if it's likely backed by hardware. |
| if (MediaCodecUtil::IsKnownUnaccelerated(codec, |
| MediaCodecDirection::ENCODER)) { |
| DLOG(ERROR) << "No HW support"; |
| return false; |
| } |
| |
| PixelFormat pixel_format = COLOR_FORMAT_YUV420_SEMIPLANAR; |
| if (!GetSupportedColorFormatForMime(mime_type, &pixel_format)) { |
| DLOG(ERROR) << "No color format support."; |
| return false; |
| } |
| media_codec_ = MediaCodecBridgeImpl::CreateVideoEncoder( |
| codec, config.input_visible_size, config.bitrate.target(), |
| INITIAL_FRAMERATE, i_frame_interval, pixel_format); |
| |
| if (!media_codec_) { |
| DLOG(ERROR) << "Failed to create/start the codec: " |
| << config.input_visible_size.ToString(); |
| return false; |
| } |
| |
| // Conservative upper bound for output buffer size: decoded size + 2KB. |
| const size_t output_buffer_capacity = |
| VideoFrame::AllocationSize(config.input_format, |
| config.input_visible_size) + |
| 2048; |
| base::ThreadTaskRunnerHandle::Get()->PostTask( |
| FROM_HERE, |
| base::BindOnce(&VideoEncodeAccelerator::Client::RequireBitstreamBuffers, |
| client_ptr_factory_->GetWeakPtr(), frame_input_count, |
| config.input_visible_size, output_buffer_capacity)); |
| return true; |
| } |
| |
| void AndroidVideoEncodeAccelerator::MaybeStartIOTimer() { |
| if (!io_timer_.IsRunning() && |
| (num_buffers_at_codec_ > 0 || !pending_frames_.empty())) { |
| io_timer_.Start(FROM_HERE, EncodePollDelay(), this, |
| &AndroidVideoEncodeAccelerator::DoIOTask); |
| } |
| } |
| |
| void AndroidVideoEncodeAccelerator::MaybeStopIOTimer() { |
| if (io_timer_.IsRunning() && |
| (num_buffers_at_codec_ == 0 && pending_frames_.empty())) { |
| io_timer_.Stop(); |
| } |
| } |
| |
| void AndroidVideoEncodeAccelerator::Encode(scoped_refptr<VideoFrame> frame, |
| bool force_keyframe) { |
| DVLOG(3) << __PRETTY_FUNCTION__ << ": " << force_keyframe; |
| DCHECK(thread_checker_.CalledOnValidThread()); |
| RETURN_ON_FAILURE(frame->format() == PIXEL_FORMAT_I420, "Unexpected format", |
| kInvalidArgumentError); |
| RETURN_ON_FAILURE(frame->visible_rect().size() == frame_size_, |
| "Unexpected resolution", kInvalidArgumentError); |
| // MediaCodec doesn't have a way to specify stride for non-Packed formats, so |
| // we insist on being called with packed frames and no cropping :( |
| RETURN_ON_FAILURE(frame->row_bytes(VideoFrame::kYPlane) == |
| frame->stride(VideoFrame::kYPlane) && |
| frame->row_bytes(VideoFrame::kUPlane) == |
| frame->stride(VideoFrame::kUPlane) && |
| frame->row_bytes(VideoFrame::kVPlane) == |
| frame->stride(VideoFrame::kVPlane) && |
| frame->coded_size() == frame->visible_rect().size(), |
| "Non-packed frame, or visible_rect != coded_size", |
| kInvalidArgumentError); |
| |
| pending_frames_.push( |
| std::make_tuple(std::move(frame), force_keyframe, base::Time::Now())); |
| DoIOTask(); |
| } |
| |
| void AndroidVideoEncodeAccelerator::UseOutputBitstreamBuffer( |
| BitstreamBuffer buffer) { |
| DVLOG(3) << __PRETTY_FUNCTION__ << ": bitstream_buffer_id=" << buffer.id(); |
| DCHECK(thread_checker_.CalledOnValidThread()); |
| available_bitstream_buffers_.push_back(std::move(buffer)); |
| DoIOTask(); |
| } |
| |
| void AndroidVideoEncodeAccelerator::RequestEncodingParametersChange( |
| const Bitrate& bitrate, |
| uint32_t framerate) { |
| // If this is changed to use variable bitrate encoding, change the mode check |
| // to check that the mode matches the current mode. |
| RETURN_ON_FAILURE(bitrate.mode() == Bitrate::Mode::kConstant, |
| "Unexpected bitrate mode", kInvalidArgumentError); |
| DVLOG(3) << __PRETTY_FUNCTION__ << ": bitrate: " << bitrate.ToString() |
| << ", framerate: " << framerate; |
| DCHECK(thread_checker_.CalledOnValidThread()); |
| if (bitrate.target() != last_set_bitrate_) { |
| last_set_bitrate_ = bitrate.target(); |
| media_codec_->SetVideoBitrate(bitrate.target(), framerate); |
| } |
| // Note: Android's MediaCodec doesn't allow mid-stream adjustments to |
| // framerate, so we ignore that here. This is OK because Android only uses |
| // the framerate value from MediaFormat during configure() as a proxy for |
| // bitrate, and we set that explicitly. |
| } |
| |
| void AndroidVideoEncodeAccelerator::Destroy() { |
| DVLOG(3) << __PRETTY_FUNCTION__; |
| DCHECK(thread_checker_.CalledOnValidThread()); |
| client_ptr_factory_.reset(); |
| if (media_codec_) { |
| if (io_timer_.IsRunning()) |
| io_timer_.Stop(); |
| media_codec_->Stop(); |
| } |
| delete this; |
| } |
| |
| void AndroidVideoEncodeAccelerator::DoIOTask() { |
| QueueInput(); |
| DequeueOutput(); |
| MaybeStartIOTimer(); |
| MaybeStopIOTimer(); |
| } |
| |
| void AndroidVideoEncodeAccelerator::QueueInput() { |
| if (error_occurred_ || pending_frames_.empty()) |
| return; |
| |
| int input_buf_index = 0; |
| MediaCodecStatus status = |
| media_codec_->DequeueInputBuffer(NoWaitTimeOut(), &input_buf_index); |
| if (status != MEDIA_CODEC_OK) { |
| DCHECK(status == MEDIA_CODEC_TRY_AGAIN_LATER || |
| status == MEDIA_CODEC_ERROR); |
| RETURN_ON_FAILURE(status != MEDIA_CODEC_ERROR, "MediaCodec error", |
| kPlatformFailureError); |
| return; |
| } |
| |
| const PendingFrames::value_type& input = pending_frames_.front(); |
| bool is_key_frame = std::get<1>(input); |
| if (is_key_frame) { |
| // Ideally MediaCodec would honor BUFFER_FLAG_SYNC_FRAME so we could |
| // indicate this in the QueueInputBuffer() call below and guarantee _this_ |
| // frame be encoded as a key frame, but sadly that flag is ignored. |
| // Instead, we request a key frame "soon". |
| media_codec_->RequestKeyFrameSoon(); |
| } |
| scoped_refptr<VideoFrame> frame = std::get<0>(input); |
| |
| uint8_t* buffer = nullptr; |
| size_t capacity = 0; |
| status = media_codec_->GetInputBuffer(input_buf_index, &buffer, &capacity); |
| RETURN_ON_FAILURE(status == MEDIA_CODEC_OK, "GetInputBuffer failed.", |
| kPlatformFailureError); |
| |
| size_t queued_size = |
| VideoFrame::AllocationSize(PIXEL_FORMAT_I420, frame->coded_size()); |
| RETURN_ON_FAILURE(capacity >= queued_size, |
| "Failed to get input buffer: " << input_buf_index, |
| kPlatformFailureError); |
| |
| uint8_t* dst_y = buffer; |
| int dst_stride_y = frame->stride(VideoFrame::kYPlane); |
| uint8_t* dst_uv = buffer + frame->stride(VideoFrame::kYPlane) * |
| frame->rows(VideoFrame::kYPlane); |
| int dst_stride_uv = frame->stride(VideoFrame::kUPlane) * 2; |
| // Why NV12? Because COLOR_FORMAT_YUV420_SEMIPLANAR. See comment at other |
| // mention of that constant. |
| bool converted = !libyuv::I420ToNV12( |
| frame->data(VideoFrame::kYPlane), frame->stride(VideoFrame::kYPlane), |
| frame->data(VideoFrame::kUPlane), frame->stride(VideoFrame::kUPlane), |
| frame->data(VideoFrame::kVPlane), frame->stride(VideoFrame::kVPlane), |
| dst_y, dst_stride_y, dst_uv, dst_stride_uv, frame->coded_size().width(), |
| frame->coded_size().height()); |
| RETURN_ON_FAILURE(converted, "Failed to I420ToNV12!", kPlatformFailureError); |
| |
| // MediaCodec encoder assumes the presentation timestamps to be monotonically |
| // increasing at initialized framerate. But in Chromium, the video capture |
| // may be paused for a while or drop some frames, so the timestamp in input |
| // frames won't be continious. Here we cache the timestamps of input frames, |
| // mapping to the generated |presentation_timestamp_|, and will read them out |
| // after encoding. Then encoder can work happily always and we can preserve |
| // the timestamps in captured frames for other purpose. |
| presentation_timestamp_ += base::Microseconds( |
| base::Time::kMicrosecondsPerSecond / INITIAL_FRAMERATE); |
| DCHECK(frame_timestamp_map_.find(presentation_timestamp_) == |
| frame_timestamp_map_.end()); |
| frame_timestamp_map_[presentation_timestamp_] = frame->timestamp(); |
| |
| status = media_codec_->QueueInputBuffer(input_buf_index, nullptr, queued_size, |
| presentation_timestamp_); |
| UMA_HISTOGRAM_TIMES("Media.AVDA.InputQueueTime", |
| base::Time::Now() - std::get<2>(input)); |
| RETURN_ON_FAILURE(status == MEDIA_CODEC_OK, |
| "Failed to QueueInputBuffer: " << status, |
| kPlatformFailureError); |
| ++num_buffers_at_codec_; |
| DCHECK(static_cast<int32_t>(frame_timestamp_map_.size()) == |
| num_buffers_at_codec_); |
| |
| pending_frames_.pop(); |
| } |
| |
| void AndroidVideoEncodeAccelerator::DequeueOutput() { |
| if (error_occurred_ || available_bitstream_buffers_.empty() || |
| num_buffers_at_codec_ == 0) { |
| return; |
| } |
| |
| int32_t buf_index = 0; |
| size_t offset = 0; |
| size_t size = 0; |
| bool key_frame = false; |
| |
| base::TimeDelta presentaion_timestamp; |
| MediaCodecStatus status = media_codec_->DequeueOutputBuffer( |
| NoWaitTimeOut(), &buf_index, &offset, &size, &presentaion_timestamp, |
| nullptr, &key_frame); |
| switch (status) { |
| case MEDIA_CODEC_TRY_AGAIN_LATER: |
| return; |
| |
| case MEDIA_CODEC_ERROR: |
| RETURN_ON_FAILURE(false, "Codec error", kPlatformFailureError); |
| // Unreachable because of previous statement, but included for clarity. |
| return; |
| |
| case MEDIA_CODEC_OUTPUT_FORMAT_CHANGED: |
| return; |
| |
| case MEDIA_CODEC_OUTPUT_BUFFERS_CHANGED: |
| return; |
| |
| case MEDIA_CODEC_OK: |
| DCHECK_GE(buf_index, 0); |
| break; |
| |
| default: |
| NOTREACHED(); |
| break; |
| } |
| |
| const auto it = frame_timestamp_map_.find(presentaion_timestamp); |
| DCHECK(it != frame_timestamp_map_.end()); |
| const base::TimeDelta frame_timestamp = it->second; |
| frame_timestamp_map_.erase(it); |
| |
| BitstreamBuffer bitstream_buffer = |
| std::move(available_bitstream_buffers_.back()); |
| available_bitstream_buffers_.pop_back(); |
| auto shm = std::make_unique<UnalignedSharedMemory>( |
| bitstream_buffer.TakeRegion(), bitstream_buffer.size(), false); |
| RETURN_ON_FAILURE( |
| shm->MapAt(bitstream_buffer.offset(), bitstream_buffer.size()), |
| "Failed to map SHM", kPlatformFailureError); |
| RETURN_ON_FAILURE( |
| size <= bitstream_buffer.size(), |
| "Encoded buffer too large: " << size << ">" << bitstream_buffer.size(), |
| kPlatformFailureError); |
| |
| status = media_codec_->CopyFromOutputBuffer(buf_index, offset, shm->memory(), |
| size); |
| RETURN_ON_FAILURE(status == MEDIA_CODEC_OK, "CopyFromOutputBuffer failed", |
| kPlatformFailureError); |
| media_codec_->ReleaseOutputBuffer(buf_index, false); |
| --num_buffers_at_codec_; |
| |
| base::ThreadTaskRunnerHandle::Get()->PostTask( |
| FROM_HERE, |
| base::BindOnce( |
| &VideoEncodeAccelerator::Client::BitstreamBufferReady, |
| client_ptr_factory_->GetWeakPtr(), bitstream_buffer.id(), |
| BitstreamBufferMetadata(size, key_frame, frame_timestamp))); |
| } |
| |
| } // namespace media |
