media/gpu/test/video_player/video_decoder_client.cc

// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "media/gpu/test/video_player/video_decoder_client.h"

#include <string>
#include <utility>

#include "base/bind.h"
#include "base/memory/ptr_util.h"
#include "base/threading/thread_task_runner_handle.h"
#include "build/build_config.h"
#include "media/base/media_util.h"
#include "media/base/waiting.h"
#include "media/gpu/macros.h"
#include "media/gpu/test/video.h"
#include "media/gpu/test/video_player/frame_renderer.h"
#include "media/gpu/test/video_player/test_vda_video_decoder.h"
#include "media/gpu/test/video_test_helpers.h"
#include "media/media_buildflags.h"
#include "testing/gtest/include/gtest/gtest.h"

#if BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
#include "media/gpu/chromeos/platform_video_frame_pool.h"
#include "media/gpu/chromeos/video_decoder_pipeline.h"
#include "media/gpu/chromeos/video_frame_converter.h"
#endif  // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)

namespace media {
namespace test {

namespace {
// Callbacks can be called from any thread, but WeakPtrs are not thread-safe.
// This helper thunk wraps a WeakPtr into an 'Optional' value, so the WeakPtr is
// only dereferenced after rescheduling the task on the specified task runner.
template <typename F, typename... Args>
void CallbackThunk(
    absl::optional<base::WeakPtr<VideoDecoderClient>> decoder_client,
    scoped_refptr<base::SequencedTaskRunner> task_runner,
    F f,
    Args... args) {
  DCHECK(decoder_client);
  task_runner->PostTask(FROM_HERE, base::BindOnce(f, *decoder_client, args...));
}
}  // namespace

VideoDecoderClient::VideoDecoderClient(
    const VideoPlayer::EventCallback& event_cb,
    gpu::GpuMemoryBufferFactory* gpu_memory_buffer_factory,
    std::unique_ptr<FrameRenderer> renderer,
    std::vector<std::unique_ptr<VideoFrameProcessor>> frame_processors,
    const VideoDecoderClientConfig& config)
    : event_cb_(event_cb),
      frame_renderer_(std::move(renderer)),
      frame_processors_(std::move(frame_processors)),
      decoder_client_config_(config),
      decoder_client_thread_("VDAClientDecoderThread"),
      decoder_client_state_(VideoDecoderClientState::kUninitialized),
      gpu_memory_buffer_factory_(gpu_memory_buffer_factory) {
  DETACH_FROM_SEQUENCE(decoder_client_sequence_checker_);

  weak_this_ = weak_this_factory_.GetWeakPtr();
}

VideoDecoderClient::~VideoDecoderClient() {
  DCHECK_CALLED_ON_VALID_SEQUENCE(video_player_sequence_checker_);

  DestroyDecoder();

  // Wait until the renderer and frame processors are done before destroying
  // them. This needs to be done after destroying the decoder so no new frames
  // will be queued while waiting.
  WaitForRenderer();
  WaitForFrameProcessors();
  frame_renderer_ = nullptr;
  frame_processors_.clear();

  decoder_client_thread_.Stop();
}

// static
std::unique_ptr<VideoDecoderClient> VideoDecoderClient::Create(
    const VideoPlayer::EventCallback& event_cb,
    gpu::GpuMemoryBufferFactory* gpu_memory_buffer_factory,
    std::unique_ptr<FrameRenderer> frame_renderer,
    std::vector<std::unique_ptr<VideoFrameProcessor>> frame_processors,
    const VideoDecoderClientConfig& config) {
  auto decoder_client = base::WrapUnique(new VideoDecoderClient(
      event_cb, gpu_memory_buffer_factory, std::move(frame_renderer),
      std::move(frame_processors), config));
  if (!decoder_client->CreateDecoder()) {
    return nullptr;
  }
  return decoder_client;
}

bool VideoDecoderClient::CreateDecoder() {
  DCHECK_CALLED_ON_VALID_SEQUENCE(video_player_sequence_checker_);
  DCHECK(!decoder_client_thread_.IsRunning());
  DCHECK(event_cb_ && frame_renderer_);

  if (!decoder_client_thread_.Start()) {
    VLOGF(1) << "Failed to start decoder thread";
    return false;
  }

  bool success = false;
  base::WaitableEvent done;
  decoder_client_thread_.task_runner()->PostTask(
      FROM_HERE, base::BindOnce(&VideoDecoderClient::CreateDecoderTask,
                                weak_this_, &success, &done));
  done.Wait();
  return success;
}

void VideoDecoderClient::DestroyDecoder() {
  DCHECK_CALLED_ON_VALID_SEQUENCE(video_player_sequence_checker_);

  if (!decoder_client_thread_.IsRunning()) {
    return;
  }

  base::WaitableEvent done;
  decoder_client_thread_.task_runner()->PostTask(
      FROM_HERE, base::BindOnce(&VideoDecoderClient::DestroyDecoderTask,
                                weak_this_, &done));
  done.Wait();
}

bool VideoDecoderClient::WaitForFrameProcessors() {
  bool success = true;
  for (auto& frame_processor : frame_processors_)
    success &= frame_processor->WaitUntilDone();
  return success;
}

void VideoDecoderClient::WaitForRenderer() {
  ASSERT_TRUE(frame_renderer_);
  frame_renderer_->WaitUntilRenderingDone();
}

FrameRenderer* VideoDecoderClient::GetFrameRenderer() const {
  return frame_renderer_.get();
}

void VideoDecoderClient::Initialize(const Video* video) {
  DCHECK_CALLED_ON_VALID_SEQUENCE(video_player_sequence_checker_);
  DCHECK(video);

  base::WaitableEvent done;
  decoder_client_thread_.task_runner()->PostTask(
      FROM_HERE, base::BindOnce(&VideoDecoderClient::InitializeDecoderTask,
                                weak_this_, video, &done));
  done.Wait();
}

void VideoDecoderClient::Play() {
  DCHECK_CALLED_ON_VALID_SEQUENCE(video_player_sequence_checker_);

  decoder_client_thread_.task_runner()->PostTask(
      FROM_HERE, base::BindOnce(&VideoDecoderClient::PlayTask, weak_this_));
}

void VideoDecoderClient::Flush() {
  DCHECK_CALLED_ON_VALID_SEQUENCE(video_player_sequence_checker_);

  decoder_client_thread_.task_runner()->PostTask(
      FROM_HERE, base::BindOnce(&VideoDecoderClient::FlushTask, weak_this_));
}

void VideoDecoderClient::Reset() {
  DCHECK_CALLED_ON_VALID_SEQUENCE(video_player_sequence_checker_);

  decoder_client_thread_.task_runner()->PostTask(
      FROM_HERE, base::BindOnce(&VideoDecoderClient::ResetTask, weak_this_));
}

void VideoDecoderClient::CreateDecoderTask(bool* success,
                                           base::WaitableEvent* done) {
  DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
  DCHECK_EQ(decoder_client_state_, VideoDecoderClientState::kUninitialized);
  ASSERT_TRUE(!decoder_) << "Can't create decoder: already created";

  switch (decoder_client_config_.implementation) {
    case DecoderImplementation::kVD:
#if BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
      decoder_ = VideoDecoderPipeline::Create(
          base::ThreadTaskRunnerHandle::Get(),
          std::make_unique<PlatformVideoFramePool>(gpu_memory_buffer_factory_),
          std::make_unique<VideoFrameConverter>(),
          std::make_unique<NullMediaLog>());
#endif  // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
      break;
    case DecoderImplementation::kVDA:
    case DecoderImplementation::kVDVDA:
      // The video decoder client expects decoders to use the VD interface. We
      // can use the TestVDAVideoDecoder wrapper here to test VDA-based video
      // decoders.
      decoder_ = std::make_unique<TestVDAVideoDecoder>(
          decoder_client_config_.implementation ==
              DecoderImplementation::kVDVDA,
          gfx::ColorSpace(), frame_renderer_.get(), gpu_memory_buffer_factory_);
      break;
  }

  *success = (decoder_ != nullptr);
  done->Signal();
}

void VideoDecoderClient::InitializeDecoderTask(const Video* video,
                                               base::WaitableEvent* done) {
  DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
  DCHECK(decoder_client_state_ == VideoDecoderClientState::kUninitialized ||
         decoder_client_state_ == VideoDecoderClientState::kIdle);
  ASSERT_TRUE(decoder_) << "Can't initialize decoder: not created yet";
  ASSERT_TRUE(video);

  video_ = video;
  encoded_data_helper_ =
      std::make_unique<EncodedDataHelper>(video_->Data(), video_->Profile());

  // (Re-)initialize the decoder.
  VideoDecoderConfig config(
      video_->Codec(), video_->Profile(),
      VideoDecoderConfig::AlphaMode::kIsOpaque, VideoColorSpace(),
      kNoTransformation, video_->Resolution(), gfx::Rect(video_->Resolution()),
      video_->Resolution(), std::vector<uint8_t>(0), EncryptionScheme());

  VideoDecoder::InitCB init_cb = base::BindOnce(
      CallbackThunk<decltype(&VideoDecoderClient::DecoderInitializedTask),
                    Status>,
      weak_this_, decoder_client_thread_.task_runner(),
      &VideoDecoderClient::DecoderInitializedTask);
  VideoDecoder::OutputCB output_cb = base::BindRepeating(
      CallbackThunk<decltype(&VideoDecoderClient::FrameReadyTask),
                    scoped_refptr<VideoFrame>>,
      weak_this_, decoder_client_thread_.task_runner(),
      &VideoDecoderClient::FrameReadyTask);

  decoder_->Initialize(config, false, nullptr, std::move(init_cb), output_cb,
                       WaitingCB());

  DCHECK_LE(decoder_client_config_.max_outstanding_decode_requests,
            static_cast<size_t>(decoder_->GetMaxDecodeRequests()));

  done->Signal();
}

void VideoDecoderClient::DestroyDecoderTask(base::WaitableEvent* done) {
  DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
  DCHECK_EQ(0u, num_outstanding_decode_requests_);
  DVLOGF(4);

  // Invalidate all scheduled tasks.
  weak_this_factory_.InvalidateWeakPtrs();

  // Destroy the decoder. This will destroy all video frames.
  if (decoder_) {
    decoder_.reset();
  }

  decoder_client_state_ = VideoDecoderClientState::kUninitialized;
  done->Signal();
}

void VideoDecoderClient::PlayTask() {
  DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
  DVLOGF(4);

  // This method should only be called when the decoder client is idle. If
  // called e.g. while flushing, the behavior is undefined.
  ASSERT_EQ(decoder_client_state_, VideoDecoderClientState::kIdle);

  // Start decoding the first fragments. While in the decoding state new
  // fragments will automatically be fed to the decoder, when the decoder
  // notifies us it reached the end of a bitstream buffer.
  decoder_client_state_ = VideoDecoderClientState::kDecoding;
  for (size_t i = 0; i < decoder_client_config_.max_outstanding_decode_requests;
       ++i) {
    DecodeNextFragmentTask();
  }
}

void VideoDecoderClient::DecodeNextFragmentTask() {
  DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
  DVLOGF(4);

  // Stop decoding fragments if we're no longer in the decoding state.
  if (decoder_client_state_ != VideoDecoderClientState::kDecoding)
    return;

  // Flush immediately when we reached the end of the stream. This changes the
  // state to kFlushing so further decode tasks will be aborted.
  if (encoded_data_helper_->ReachEndOfStream()) {
    FlushTask();
    return;
  }

  scoped_refptr<DecoderBuffer> bitstream_buffer =
      encoded_data_helper_->GetNextBuffer();
  if (!bitstream_buffer) {
    LOG(ERROR) << "Failed to get next video stream data";
    return;
  }
  bitstream_buffer->set_timestamp(base::TimeTicks::Now().since_origin());
  bool has_config_info = media::test::EncodedDataHelper::HasConfigInfo(
      bitstream_buffer->data(), bitstream_buffer->data_size(),
      video_->Profile());

  VideoDecoder::DecodeCB decode_cb = base::BindOnce(
      CallbackThunk<decltype(&VideoDecoderClient::DecodeDoneTask),
                    media::Status>,
      weak_this_, decoder_client_thread_.task_runner(),
      &VideoDecoderClient::DecodeDoneTask);
  decoder_->Decode(std::move(bitstream_buffer), std::move(decode_cb));

  num_outstanding_decode_requests_++;

  // Throw event when we encounter a config info in a H.264/HEVC stream.
  if (has_config_info)
    FireEvent(VideoPlayerEvent::kConfigInfo);
}

void VideoDecoderClient::FlushTask() {
  DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
  DVLOGF(4);

  // Changing the state to flushing will abort any pending decodes.
  decoder_client_state_ = VideoDecoderClientState::kFlushing;

  VideoDecoder::DecodeCB flush_done_cb =
      base::BindOnce(CallbackThunk<decltype(&VideoDecoderClient::FlushDoneTask),
                                   media::Status>,
                     weak_this_, decoder_client_thread_.task_runner(),
                     &VideoDecoderClient::FlushDoneTask);
  decoder_->Decode(DecoderBuffer::CreateEOSBuffer(), std::move(flush_done_cb));

  FireEvent(VideoPlayerEvent::kFlushing);
}

void VideoDecoderClient::ResetTask() {
  DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
  DVLOGF(4);

  // Changing the state to resetting will abort any pending decodes.
  decoder_client_state_ = VideoDecoderClientState::kResetting;
  // TODO(dstaessens@) Allow resetting to any point in the stream.
  encoded_data_helper_->Rewind();

  decoder_->Reset(base::BindOnce(
      CallbackThunk<decltype(&VideoDecoderClient::ResetDoneTask)>, weak_this_,
      decoder_client_thread_.task_runner(),
      &VideoDecoderClient::ResetDoneTask));
  FireEvent(VideoPlayerEvent::kResetting);
}

void VideoDecoderClient::DecoderInitializedTask(Status status) {
  DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
  DCHECK(decoder_client_state_ == VideoDecoderClientState::kUninitialized ||
         decoder_client_state_ == VideoDecoderClientState::kIdle);
  ASSERT_TRUE(status.is_ok()) << "Initializing decoder failed";

  decoder_client_state_ = VideoDecoderClientState::kIdle;
  FireEvent(VideoPlayerEvent::kInitialized);
}

void VideoDecoderClient::DecodeDoneTask(media::Status status) {
  DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
  DCHECK_NE(VideoDecoderClientState::kIdle, decoder_client_state_);
  ASSERT_TRUE(status.code() != media::StatusCode::kAborted ||
              decoder_client_state_ == VideoDecoderClientState::kResetting);
  DVLOGF(4);

  num_outstanding_decode_requests_--;

  // Queue the next fragment to be decoded.
  decoder_client_thread_.task_runner()->PostTask(
      FROM_HERE,
      base::BindOnce(&VideoDecoderClient::DecodeNextFragmentTask, weak_this_));
}

void VideoDecoderClient::FrameReadyTask(scoped_refptr<VideoFrame> video_frame) {
  DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
  DCHECK(video_frame->metadata().power_efficient);

  frame_renderer_->RenderFrame(video_frame);

  for (auto& frame_processor : frame_processors_)
    frame_processor->ProcessVideoFrame(video_frame, current_frame_index_);

  // Notify the test a frame has been decoded. We should only do this after
  // scheduling the frame to be processed, so calling WaitForFrameProcessors()
  // after receiving this event will always guarantee the frame to be processed.
  FireEvent(VideoPlayerEvent::kFrameDecoded);

  current_frame_index_++;
}

void VideoDecoderClient::FlushDoneTask(media::Status status) {
  DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
  DCHECK_EQ(0u, num_outstanding_decode_requests_);

  // Send an EOS frame to the renderer, so it can reset any internal state it
  // might keep in preparation of the next stream of video frames.
  frame_renderer_->RenderFrame(VideoFrame::CreateEOSFrame());
  decoder_client_state_ = VideoDecoderClientState::kIdle;
  FireEvent(VideoPlayerEvent::kFlushDone);
}

void VideoDecoderClient::ResetDoneTask() {
  DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);
  DCHECK_EQ(0u, num_outstanding_decode_requests_);

  // We finished resetting to a different point in the stream, so we should
  // update the frame index. Currently only resetting to the start of the stream
  // is supported, so we can set the frame index to zero here.
  current_frame_index_ = 0;

  frame_renderer_->RenderFrame(VideoFrame::CreateEOSFrame());
  decoder_client_state_ = VideoDecoderClientState::kIdle;
  FireEvent(VideoPlayerEvent::kResetDone);
}

void VideoDecoderClient::FireEvent(VideoPlayerEvent event) {
  DCHECK_CALLED_ON_VALID_SEQUENCE(decoder_client_sequence_checker_);

  bool continue_decoding = event_cb_.Run(event);
  if (!continue_decoding) {
    // Changing the state to idle will abort any pending decodes.
    decoder_client_state_ = VideoDecoderClientState::kIdle;
  }
}

}  // namespace test
}  // namespace media