| // Copyright 2019 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "media/gpu/test/image_processor/image_processor_client.h" |
| |
| #include <functional> |
| #include <utility> |
| |
| #include "base/bind.h" |
| |
| #include "base/callback_helpers.h" |
| #include "base/logging.h" |
| #include "base/memory/ptr_util.h" |
| #include "base/synchronization/waitable_event.h" |
| #include "build/build_config.h" |
| #include "build/chromeos_buildflags.h" |
| #include "media/base/bind_to_current_loop.h" |
| #include "media/base/video_frame.h" |
| #include "media/base/video_frame_layout.h" |
| #include "media/gpu/chromeos/fourcc.h" |
| #include "media/gpu/chromeos/image_processor_factory.h" |
| #include "media/gpu/test/image.h" |
| #include "media/gpu/test/video_frame_helpers.h" |
| #include "media/media_buildflags.h" |
| #include "testing/gtest/include/gtest/gtest.h" |
| #include "ui/gfx/geometry/rect.h" |
| |
| #if BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION) |
| #include "media/gpu/chromeos/platform_video_frame_utils.h" |
| #endif // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION) |
| |
| namespace media { |
| namespace test { |
| namespace { |
| |
| #define ASSERT_TRUE_OR_RETURN_NULLPTR(predicate) \ |
| do { \ |
| if (!(predicate)) { \ |
| ADD_FAILURE(); \ |
| return nullptr; \ |
| } \ |
| } while (0) |
| |
| absl::optional<VideoFrameLayout> CreateLayout( |
| const ImageProcessor::PortConfig& config) { |
| const VideoPixelFormat pixel_format = config.fourcc.ToVideoPixelFormat(); |
| if (config.planes.empty()) |
| return absl::nullopt; |
| |
| if (config.fourcc.IsMultiPlanar()) { |
| return VideoFrameLayout::CreateWithPlanes(pixel_format, config.size, |
| config.planes); |
| } else { |
| return VideoFrameLayout::CreateMultiPlanar(pixel_format, config.size, |
| config.planes); |
| } |
| } |
| } // namespace |
| |
| // static |
| std::unique_ptr<ImageProcessorClient> ImageProcessorClient::Create( |
| const ImageProcessor::PortConfig& input_config, |
| const ImageProcessor::PortConfig& output_config, |
| size_t num_buffers, |
| VideoRotation relative_rotation, |
| std::vector<std::unique_ptr<VideoFrameProcessor>> frame_processors) { |
| auto ip_client = |
| base::WrapUnique(new ImageProcessorClient(std::move(frame_processors))); |
| if (!ip_client->CreateImageProcessor(input_config, output_config, num_buffers, |
| relative_rotation)) { |
| LOG(ERROR) << "Failed to create ImageProcessor"; |
| return nullptr; |
| } |
| return ip_client; |
| } |
| |
| ImageProcessorClient::ImageProcessorClient( |
| std::vector<std::unique_ptr<VideoFrameProcessor>> frame_processors) |
| : gpu_memory_buffer_factory_( |
| gpu::GpuMemoryBufferFactory::CreateNativeType(nullptr)), |
| frame_processors_(std::move(frame_processors)), |
| image_processor_client_thread_("ImageProcessorClientThread"), |
| output_cv_(&output_lock_), |
| num_processed_frames_(0), |
| image_processor_error_count_(0) { |
| CHECK(image_processor_client_thread_.Start()); |
| DETACH_FROM_THREAD(image_processor_client_thread_checker_); |
| } |
| |
| ImageProcessorClient::~ImageProcessorClient() { |
| DCHECK_CALLED_ON_VALID_THREAD(test_main_thread_checker_); |
| CHECK(image_processor_client_thread_.IsRunning()); |
| // Destroys |image_processor_| on |image_processor_client_thread_|. |
| image_processor_client_thread_.task_runner()->DeleteSoon( |
| FROM_HERE, image_processor_.release()); |
| image_processor_client_thread_.Stop(); |
| } |
| |
| bool ImageProcessorClient::CreateImageProcessor( |
| const ImageProcessor::PortConfig& input_config, |
| const ImageProcessor::PortConfig& output_config, |
| size_t num_buffers, |
| VideoRotation relative_rotation) { |
| DCHECK_CALLED_ON_VALID_THREAD(test_main_thread_checker_); |
| base::WaitableEvent done(base::WaitableEvent::ResetPolicy::AUTOMATIC, |
| base::WaitableEvent::InitialState::NOT_SIGNALED); |
| // base::Unretained(this) and std::cref() are safe here because |this|, |
| // |input_config| and |output_config| must outlive because this task is |
| // blocking. |
| image_processor_client_thread_.task_runner()->PostTask( |
| FROM_HERE, base::BindOnce(&ImageProcessorClient::CreateImageProcessorTask, |
| base::Unretained(this), std::cref(input_config), |
| std::cref(output_config), num_buffers, |
| relative_rotation, &done)); |
| done.Wait(); |
| if (!image_processor_) { |
| LOG(ERROR) << "Failed to create ImageProcessor"; |
| return false; |
| } |
| return true; |
| } |
| |
| void ImageProcessorClient::CreateImageProcessorTask( |
| const ImageProcessor::PortConfig& input_config, |
| const ImageProcessor::PortConfig& output_config, |
| size_t num_buffers, |
| VideoRotation relative_rotation, |
| base::WaitableEvent* done) { |
| DCHECK_CALLED_ON_VALID_THREAD(image_processor_client_thread_checker_); |
| // base::Unretained(this) for ErrorCB is safe here because the callback is |
| // executed on |image_processor_client_thread_| which is owned by this class. |
| image_processor_ = ImageProcessorFactory::Create( |
| input_config, output_config, {ImageProcessor::OutputMode::IMPORT}, |
| num_buffers, relative_rotation, |
| image_processor_client_thread_.task_runner(), |
| base::BindRepeating(&ImageProcessorClient::NotifyError, |
| base::Unretained(this))); |
| done->Signal(); |
| } |
| |
| scoped_refptr<VideoFrame> ImageProcessorClient::CreateInputFrame( |
| const Image& input_image) const { |
| DCHECK_CALLED_ON_VALID_THREAD(test_main_thread_checker_); |
| ASSERT_TRUE_OR_RETURN_NULLPTR(image_processor_); |
| ASSERT_TRUE_OR_RETURN_NULLPTR(input_image.IsLoaded()); |
| |
| const ImageProcessor::PortConfig& input_config = |
| image_processor_->input_config(); |
| const VideoFrame::StorageType input_storage_type = |
| input_config.storage_type(); |
| absl::optional<VideoFrameLayout> input_layout = CreateLayout(input_config); |
| ASSERT_TRUE_OR_RETURN_NULLPTR(input_layout); |
| |
| if (VideoFrame::IsStorageTypeMappable(input_storage_type)) { |
| return CloneVideoFrame(gpu_memory_buffer_factory_.get(), |
| CreateVideoFrameFromImage(input_image).get(), |
| *input_layout, VideoFrame::STORAGE_OWNED_MEMORY); |
| } else { |
| #if BUILDFLAG(IS_CHROMEOS_ASH) |
| ASSERT_TRUE_OR_RETURN_NULLPTR( |
| input_storage_type == VideoFrame::STORAGE_DMABUFS || |
| input_storage_type == VideoFrame::STORAGE_GPU_MEMORY_BUFFER); |
| // NV12 is the only format that can be allocated with |
| // gfx::BufferUsage::VEA_READ_CAMERA_AND_CPU_READ_WRITE. So |
| // gfx::BufferUsage::GPU_READ_CPU_READ_WRITE is specified for other formats. |
| gfx::BufferUsage dst_buffer_usage = |
| (PIXEL_FORMAT_NV12 == input_image.PixelFormat()) |
| ? gfx::BufferUsage::VEA_READ_CAMERA_AND_CPU_READ_WRITE |
| : gfx::BufferUsage::GPU_READ_CPU_READ_WRITE; |
| return CloneVideoFrame(gpu_memory_buffer_factory_.get(), |
| CreateVideoFrameFromImage(input_image).get(), |
| *input_layout, input_storage_type, dst_buffer_usage); |
| #else |
| return nullptr; |
| #endif |
| } |
| } |
| |
| scoped_refptr<VideoFrame> ImageProcessorClient::CreateOutputFrame( |
| const Image& output_image) const { |
| DCHECK_CALLED_ON_VALID_THREAD(test_main_thread_checker_); |
| ASSERT_TRUE_OR_RETURN_NULLPTR(output_image.IsMetadataLoaded()); |
| ASSERT_TRUE_OR_RETURN_NULLPTR(image_processor_); |
| |
| const ImageProcessor::PortConfig& output_config = |
| image_processor_->output_config(); |
| const VideoFrame::StorageType output_storage_type = |
| output_config.storage_type(); |
| absl::optional<VideoFrameLayout> output_layout = CreateLayout(output_config); |
| ASSERT_TRUE_OR_RETURN_NULLPTR(output_layout); |
| if (VideoFrame::IsStorageTypeMappable(output_storage_type)) { |
| return VideoFrame::CreateFrameWithLayout( |
| *output_layout, gfx::Rect(output_image.Size()), output_image.Size(), |
| base::TimeDelta(), false /* zero_initialize_memory*/); |
| } |
| |
| #if BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION) |
| ASSERT_TRUE_OR_RETURN_NULLPTR( |
| output_storage_type == VideoFrame::STORAGE_DMABUFS || |
| output_storage_type == VideoFrame::STORAGE_GPU_MEMORY_BUFFER); |
| scoped_refptr<VideoFrame> output_frame = CreatePlatformVideoFrame( |
| gpu_memory_buffer_factory_.get(), output_layout->format(), |
| output_layout->coded_size(), gfx::Rect(output_image.Size()), |
| output_image.Size(), base::TimeDelta(), |
| gfx::BufferUsage::GPU_READ_CPU_READ_WRITE); |
| |
| if (output_storage_type == VideoFrame::STORAGE_GPU_MEMORY_BUFFER) { |
| output_frame = CreateGpuMemoryBufferVideoFrame( |
| gpu_memory_buffer_factory_.get(), output_frame.get(), |
| gfx::BufferUsage::GPU_READ_CPU_READ_WRITE); |
| } |
| return output_frame; |
| #else |
| return nullptr; |
| #endif // BUILDFLAG(USE_CHROMEOS_MEDIA_ACCELERATION) |
| } |
| |
| void ImageProcessorClient::FrameReady(size_t frame_index, |
| scoped_refptr<VideoFrame> frame) { |
| DCHECK_CALLED_ON_VALID_THREAD(image_processor_client_thread_checker_); |
| |
| base::AutoLock auto_lock_(output_lock_); |
| // VideoFrame should be processed in FIFO order. |
| EXPECT_EQ(frame_index, num_processed_frames_); |
| for (auto& processor : frame_processors_) |
| processor->ProcessVideoFrame(frame, frame_index); |
| num_processed_frames_++; |
| output_cv_.Signal(); |
| } |
| |
| bool ImageProcessorClient::WaitUntilNumImageProcessed( |
| size_t num_processed, |
| base::TimeDelta max_wait) { |
| base::TimeDelta time_waiting; |
| // NOTE: Acquire lock here does not matter, because |
| // base::ConditionVariable::TimedWait() unlocks output_lock_ at the start and |
| // locks again at the end. |
| base::AutoLock auto_lock_(output_lock_); |
| while (time_waiting < max_wait) { |
| if (num_processed_frames_ >= num_processed) |
| return true; |
| |
| const base::TimeTicks start_time = base::TimeTicks::Now(); |
| output_cv_.TimedWait(max_wait); |
| time_waiting += base::TimeTicks::Now() - start_time; |
| } |
| |
| return false; |
| } |
| |
| bool ImageProcessorClient::WaitForFrameProcessors() { |
| bool success = true; |
| for (auto& processor : frame_processors_) |
| success &= processor->WaitUntilDone(); |
| |
| return success; |
| } |
| |
| size_t ImageProcessorClient::GetNumOfProcessedImages() const { |
| base::AutoLock auto_lock_(output_lock_); |
| return num_processed_frames_; |
| } |
| |
| size_t ImageProcessorClient::GetErrorCount() const { |
| base::AutoLock auto_lock_(output_lock_); |
| return image_processor_error_count_; |
| } |
| |
| void ImageProcessorClient::NotifyError() { |
| DCHECK_CALLED_ON_VALID_THREAD(image_processor_client_thread_checker_); |
| base::AutoLock auto_lock_(output_lock_); |
| image_processor_error_count_++; |
| } |
| |
| void ImageProcessorClient::Process(const Image& input_image, |
| const Image& output_image) { |
| DCHECK_CALLED_ON_VALID_THREAD(test_main_thread_checker_); |
| auto input_frame = CreateInputFrame(input_image); |
| ASSERT_TRUE(input_frame); |
| auto output_frame = CreateOutputFrame(output_image); |
| ASSERT_TRUE(output_frame); |
| image_processor_client_thread_.task_runner()->PostTask( |
| FROM_HERE, |
| base::BindOnce(&ImageProcessorClient::ProcessTask, base::Unretained(this), |
| std::move(input_frame), std::move(output_frame))); |
| } |
| |
| void ImageProcessorClient::ProcessTask(scoped_refptr<VideoFrame> input_frame, |
| scoped_refptr<VideoFrame> output_frame) { |
| DCHECK_CALLED_ON_VALID_THREAD(image_processor_client_thread_checker_); |
| // base::Unretained(this) and std::cref() for FrameReadyCB is safe here |
| // because the callback is executed on |image_processor_client_thread_| which |
| // is owned by this class. |
| image_processor_->Process(std::move(input_frame), std::move(output_frame), |
| BindToCurrentLoop(base::BindOnce( |
| &ImageProcessorClient::FrameReady, |
| base::Unretained(this), next_frame_index_))); |
| next_frame_index_++; |
| } |
| |
| } // namespace test |
| } // namespace media |