blob: fc72bd53d7df94c8ae023104b0feb707c7747ca9 [file] [log] [blame]
// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "media/gpu/test/image_processor/image_processor_client.h"
#include <functional>
#include <utility>
#include "base/bind.h"
#include "base/callback_helpers.h"
#include "base/logging.h"
#include "base/memory/ptr_util.h"
#include "base/synchronization/waitable_event.h"
#include "build/build_config.h"
#include "build/chromeos_buildflags.h"
#include "media/base/bind_to_current_loop.h"
#include "media/base/video_frame.h"
#include "media/base/video_frame_layout.h"
#include "media/gpu/chromeos/fourcc.h"
#include "media/gpu/chromeos/image_processor_factory.h"
#include "media/gpu/test/image.h"
#include "media/gpu/test/video_frame_helpers.h"
#include "media/media_buildflags.h"
#include "testing/gtest/include/gtest/gtest.h"
#include "ui/gfx/geometry/rect.h"
#if BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
#include "media/gpu/chromeos/platform_video_frame_utils.h"
#endif // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
namespace media {
namespace test {
namespace {
#define ASSERT_TRUE_OR_RETURN_NULLPTR(predicate) \
do { \
if (!(predicate)) { \
ADD_FAILURE(); \
return nullptr; \
} \
} while (0)
absl::optional<VideoFrameLayout> CreateLayout(
const ImageProcessor::PortConfig& config) {
const VideoPixelFormat pixel_format = config.fourcc.ToVideoPixelFormat();
if (config.planes.empty())
return absl::nullopt;
if (config.fourcc.IsMultiPlanar()) {
return VideoFrameLayout::CreateWithPlanes(pixel_format, config.size,
config.planes);
} else {
return VideoFrameLayout::CreateMultiPlanar(pixel_format, config.size,
config.planes);
}
}
} // namespace
// static
std::unique_ptr<ImageProcessorClient> ImageProcessorClient::Create(
const ImageProcessor::PortConfig& input_config,
const ImageProcessor::PortConfig& output_config,
size_t num_buffers,
VideoRotation relative_rotation,
std::vector<std::unique_ptr<VideoFrameProcessor>> frame_processors) {
auto ip_client =
base::WrapUnique(new ImageProcessorClient(std::move(frame_processors)));
if (!ip_client->CreateImageProcessor(input_config, output_config, num_buffers,
relative_rotation)) {
LOG(ERROR) << "Failed to create ImageProcessor";
return nullptr;
}
return ip_client;
}
ImageProcessorClient::ImageProcessorClient(
std::vector<std::unique_ptr<VideoFrameProcessor>> frame_processors)
: gpu_memory_buffer_factory_(
gpu::GpuMemoryBufferFactory::CreateNativeType(nullptr)),
frame_processors_(std::move(frame_processors)),
image_processor_client_thread_("ImageProcessorClientThread"),
output_cv_(&output_lock_),
num_processed_frames_(0),
image_processor_error_count_(0) {
CHECK(image_processor_client_thread_.Start());
DETACH_FROM_THREAD(image_processor_client_thread_checker_);
}
ImageProcessorClient::~ImageProcessorClient() {
DCHECK_CALLED_ON_VALID_THREAD(test_main_thread_checker_);
CHECK(image_processor_client_thread_.IsRunning());
// Destroys |image_processor_| on |image_processor_client_thread_|.
image_processor_client_thread_.task_runner()->DeleteSoon(
FROM_HERE, image_processor_.release());
image_processor_client_thread_.Stop();
}
bool ImageProcessorClient::CreateImageProcessor(
const ImageProcessor::PortConfig& input_config,
const ImageProcessor::PortConfig& output_config,
size_t num_buffers,
VideoRotation relative_rotation) {
DCHECK_CALLED_ON_VALID_THREAD(test_main_thread_checker_);
base::WaitableEvent done(base::WaitableEvent::ResetPolicy::AUTOMATIC,
base::WaitableEvent::InitialState::NOT_SIGNALED);
// base::Unretained(this) and std::cref() are safe here because |this|,
// |input_config| and |output_config| must outlive because this task is
// blocking.
image_processor_client_thread_.task_runner()->PostTask(
FROM_HERE, base::BindOnce(&ImageProcessorClient::CreateImageProcessorTask,
base::Unretained(this), std::cref(input_config),
std::cref(output_config), num_buffers,
relative_rotation, &done));
done.Wait();
if (!image_processor_) {
LOG(ERROR) << "Failed to create ImageProcessor";
return false;
}
return true;
}
void ImageProcessorClient::CreateImageProcessorTask(
const ImageProcessor::PortConfig& input_config,
const ImageProcessor::PortConfig& output_config,
size_t num_buffers,
VideoRotation relative_rotation,
base::WaitableEvent* done) {
DCHECK_CALLED_ON_VALID_THREAD(image_processor_client_thread_checker_);
// base::Unretained(this) for ErrorCB is safe here because the callback is
// executed on |image_processor_client_thread_| which is owned by this class.
image_processor_ = ImageProcessorFactory::Create(
input_config, output_config, {ImageProcessor::OutputMode::IMPORT},
num_buffers, relative_rotation,
image_processor_client_thread_.task_runner(),
base::BindRepeating(&ImageProcessorClient::NotifyError,
base::Unretained(this)));
done->Signal();
}
scoped_refptr<VideoFrame> ImageProcessorClient::CreateInputFrame(
const Image& input_image) const {
DCHECK_CALLED_ON_VALID_THREAD(test_main_thread_checker_);
ASSERT_TRUE_OR_RETURN_NULLPTR(image_processor_);
ASSERT_TRUE_OR_RETURN_NULLPTR(input_image.IsLoaded());
const ImageProcessor::PortConfig& input_config =
image_processor_->input_config();
const VideoFrame::StorageType input_storage_type =
input_config.storage_type();
absl::optional<VideoFrameLayout> input_layout = CreateLayout(input_config);
ASSERT_TRUE_OR_RETURN_NULLPTR(input_layout);
if (VideoFrame::IsStorageTypeMappable(input_storage_type)) {
return CloneVideoFrame(gpu_memory_buffer_factory_.get(),
CreateVideoFrameFromImage(input_image).get(),
*input_layout, VideoFrame::STORAGE_OWNED_MEMORY);
} else {
#if BUILDFLAG(IS_CHROMEOS_ASH)
ASSERT_TRUE_OR_RETURN_NULLPTR(
input_storage_type == VideoFrame::STORAGE_DMABUFS ||
input_storage_type == VideoFrame::STORAGE_GPU_MEMORY_BUFFER);
// NV12 is the only format that can be allocated with
// gfx::BufferUsage::VEA_READ_CAMERA_AND_CPU_READ_WRITE. So
// gfx::BufferUsage::GPU_READ_CPU_READ_WRITE is specified for other formats.
gfx::BufferUsage dst_buffer_usage =
(PIXEL_FORMAT_NV12 == input_image.PixelFormat())
? gfx::BufferUsage::VEA_READ_CAMERA_AND_CPU_READ_WRITE
: gfx::BufferUsage::GPU_READ_CPU_READ_WRITE;
return CloneVideoFrame(gpu_memory_buffer_factory_.get(),
CreateVideoFrameFromImage(input_image).get(),
*input_layout, input_storage_type, dst_buffer_usage);
#else
return nullptr;
#endif
}
}
scoped_refptr<VideoFrame> ImageProcessorClient::CreateOutputFrame(
const Image& output_image) const {
DCHECK_CALLED_ON_VALID_THREAD(test_main_thread_checker_);
ASSERT_TRUE_OR_RETURN_NULLPTR(output_image.IsMetadataLoaded());
ASSERT_TRUE_OR_RETURN_NULLPTR(image_processor_);
const ImageProcessor::PortConfig& output_config =
image_processor_->output_config();
const VideoFrame::StorageType output_storage_type =
output_config.storage_type();
absl::optional<VideoFrameLayout> output_layout = CreateLayout(output_config);
ASSERT_TRUE_OR_RETURN_NULLPTR(output_layout);
if (VideoFrame::IsStorageTypeMappable(output_storage_type)) {
return VideoFrame::CreateFrameWithLayout(
*output_layout, gfx::Rect(output_image.Size()), output_image.Size(),
base::TimeDelta(), false /* zero_initialize_memory*/);
}
#if BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
ASSERT_TRUE_OR_RETURN_NULLPTR(
output_storage_type == VideoFrame::STORAGE_DMABUFS ||
output_storage_type == VideoFrame::STORAGE_GPU_MEMORY_BUFFER);
scoped_refptr<VideoFrame> output_frame = CreatePlatformVideoFrame(
gpu_memory_buffer_factory_.get(), output_layout->format(),
output_layout->coded_size(), gfx::Rect(output_image.Size()),
output_image.Size(), base::TimeDelta(),
gfx::BufferUsage::GPU_READ_CPU_READ_WRITE);
if (output_storage_type == VideoFrame::STORAGE_GPU_MEMORY_BUFFER) {
output_frame = CreateGpuMemoryBufferVideoFrame(
gpu_memory_buffer_factory_.get(), output_frame.get(),
gfx::BufferUsage::GPU_READ_CPU_READ_WRITE);
}
return output_frame;
#else
return nullptr;
#endif // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION)
}
void ImageProcessorClient::FrameReady(size_t frame_index,
scoped_refptr<VideoFrame> frame) {
DCHECK_CALLED_ON_VALID_THREAD(image_processor_client_thread_checker_);
base::AutoLock auto_lock_(output_lock_);
// VideoFrame should be processed in FIFO order.
EXPECT_EQ(frame_index, num_processed_frames_);
for (auto& processor : frame_processors_)
processor->ProcessVideoFrame(frame, frame_index);
num_processed_frames_++;
output_cv_.Signal();
}
bool ImageProcessorClient::WaitUntilNumImageProcessed(
size_t num_processed,
base::TimeDelta max_wait) {
base::TimeDelta time_waiting;
// NOTE: Acquire lock here does not matter, because
// base::ConditionVariable::TimedWait() unlocks output_lock_ at the start and
// locks again at the end.
base::AutoLock auto_lock_(output_lock_);
while (time_waiting < max_wait) {
if (num_processed_frames_ >= num_processed)
return true;
const base::TimeTicks start_time = base::TimeTicks::Now();
output_cv_.TimedWait(max_wait);
time_waiting += base::TimeTicks::Now() - start_time;
}
return false;
}
bool ImageProcessorClient::WaitForFrameProcessors() {
bool success = true;
for (auto& processor : frame_processors_)
success &= processor->WaitUntilDone();
return success;
}
size_t ImageProcessorClient::GetNumOfProcessedImages() const {
base::AutoLock auto_lock_(output_lock_);
return num_processed_frames_;
}
size_t ImageProcessorClient::GetErrorCount() const {
base::AutoLock auto_lock_(output_lock_);
return image_processor_error_count_;
}
void ImageProcessorClient::NotifyError() {
DCHECK_CALLED_ON_VALID_THREAD(image_processor_client_thread_checker_);
base::AutoLock auto_lock_(output_lock_);
image_processor_error_count_++;
}
void ImageProcessorClient::Process(const Image& input_image,
const Image& output_image) {
DCHECK_CALLED_ON_VALID_THREAD(test_main_thread_checker_);
auto input_frame = CreateInputFrame(input_image);
ASSERT_TRUE(input_frame);
auto output_frame = CreateOutputFrame(output_image);
ASSERT_TRUE(output_frame);
image_processor_client_thread_.task_runner()->PostTask(
FROM_HERE,
base::BindOnce(&ImageProcessorClient::ProcessTask, base::Unretained(this),
std::move(input_frame), std::move(output_frame)));
}
void ImageProcessorClient::ProcessTask(scoped_refptr<VideoFrame> input_frame,
scoped_refptr<VideoFrame> output_frame) {
DCHECK_CALLED_ON_VALID_THREAD(image_processor_client_thread_checker_);
// base::Unretained(this) and std::cref() for FrameReadyCB is safe here
// because the callback is executed on |image_processor_client_thread_| which
// is owned by this class.
image_processor_->Process(std::move(input_frame), std::move(output_frame),
BindToCurrentLoop(base::BindOnce(
&ImageProcessorClient::FrameReady,
base::Unretained(this), next_frame_index_)));
next_frame_index_++;
}
} // namespace test
} // namespace media