| // Copyright 2019 The Chromium Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include "media/gpu/test/video_player/frame_renderer_thumbnail.h" |
| |
| #include <utility> |
| |
| #include "base/containers/contains.h" |
| #include "base/files/file_util.h" |
| #include "base/memory/ptr_util.h" |
| #include "base/threading/thread_task_runner_handle.h" |
| #include "build/build_config.h" |
| #include "media/gpu/test/video_test_helpers.h" |
| #include "testing/gtest/include/gtest/gtest.h" |
| #include "ui/gfx/codec/png_codec.h" |
| #include "ui/gl/gl_bindings.h" |
| #include "ui/gl/gl_context.h" |
| #include "ui/gl/gl_surface_egl.h" |
| #include "ui/gl/init/gl_factory.h" |
| |
| namespace media { |
| namespace test { |
| |
| namespace { |
| |
| // Size of the large image to which the thumbnails will be rendered. |
| constexpr gfx::Size kThumbnailsPageSize(1600, 1200); |
| // Size of the individual thumbnails that will be rendered. |
| constexpr gfx::Size kThumbnailSize(160, 120); |
| |
| // Default filename used to store the thumbnails image. |
| constexpr const base::FilePath::CharType* kThumbnailFilename = |
| FILE_PATH_LITERAL("thumbnail.png"); |
| |
| // Vertex shader used to render thumbnails. |
| constexpr char kVertexShader[] = |
| "varying vec2 interp_tc;\n" |
| "attribute vec4 in_pos;\n" |
| "attribute vec2 in_tc;\n" |
| "uniform bool tex_flip; void main() {\n" |
| " if (tex_flip)\n" |
| " interp_tc = vec2(in_tc.x, 1.0 - in_tc.y);\n" |
| " else\n" |
| " interp_tc = in_tc;\n" |
| " gl_Position = in_pos;\n" |
| "}\n"; |
| |
| // Fragment shader used to render thumbnails. |
| #if !defined(OS_WIN) |
| constexpr char kFragmentShader[] = |
| "#extension GL_OES_EGL_image_external : enable\n" |
| "precision mediump float;\n" |
| "varying vec2 interp_tc;\n" |
| "uniform sampler2D tex;\n" |
| "#ifdef GL_OES_EGL_image_external\n" |
| "uniform samplerExternalOES tex_external;\n" |
| "#endif\n" |
| "void main() {\n" |
| " vec4 color = texture2D(tex, interp_tc);\n" |
| "#ifdef GL_OES_EGL_image_external\n" |
| " color += texture2D(tex_external, interp_tc);\n" |
| "#endif\n" |
| " gl_FragColor = color;\n" |
| "}\n"; |
| #else |
| constexpr char kFragmentShader[] = |
| "#ifdef GL_ES\n" |
| "precision mediump float;\n" |
| "#endif\n" |
| "varying vec2 interp_tc;\n" |
| "uniform sampler2D tex;\n" |
| "void main() {\n" |
| " gl_FragColor = texture2D(tex, interp_tc);\n" |
| "}\n"; |
| #endif |
| |
| GLuint CreateTexture(GLenum texture_target, const gfx::Size& size) { |
| GLuint texture_id; |
| glGenTextures(1, &texture_id); |
| glBindTexture(texture_target, texture_id); |
| if (texture_target == GL_TEXTURE_2D) { |
| glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, size.width(), size.height(), 0, |
| GL_RGBA, GL_UNSIGNED_BYTE, nullptr); |
| } |
| |
| glTexParameteri(texture_target, GL_TEXTURE_MIN_FILTER, GL_LINEAR); |
| glTexParameteri(texture_target, GL_TEXTURE_MAG_FILTER, GL_LINEAR); |
| // OpenGLES2.0.25 section 3.8.2 requires CLAMP_TO_EDGE for NPOT textures. |
| glTexParameteri(texture_target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); |
| glTexParameteri(texture_target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); |
| |
| CHECK_EQ(static_cast<int>(glGetError()), GL_NO_ERROR); |
| return texture_id; |
| } |
| |
| void DeleteTexture(uint32_t texture_id) { |
| glDeleteTextures(1, &texture_id); |
| CHECK_EQ(static_cast<int>(glGetError()), GL_NO_ERROR); |
| } |
| |
| void RenderTexture(uint32_t texture_target, uint32_t texture_id) { |
| // The ExternalOES sampler is bound to GL_TEXTURE1 and the Texture2D sampler |
| // is bound to GL_TEXTURE0. |
| if (texture_target == GL_TEXTURE_2D) { |
| glActiveTexture(GL_TEXTURE0 + 0); |
| } else if (texture_target == GL_TEXTURE_EXTERNAL_OES) { |
| glActiveTexture(GL_TEXTURE0 + 1); |
| } |
| glBindTexture(texture_target, texture_id); |
| glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); |
| glBindTexture(texture_target, 0); |
| |
| CHECK_EQ(static_cast<int>(glGetError()), GL_NO_ERROR); |
| } |
| |
| void CreateShader(GLuint program, GLenum type, const char* source, int size) { |
| GLuint shader = glCreateShader(type); |
| glShaderSource(shader, 1, &source, &size); |
| glCompileShader(shader); |
| int result = GL_FALSE; |
| glGetShaderiv(shader, GL_COMPILE_STATUS, &result); |
| if (!result) { |
| char log[4096]; |
| glGetShaderInfoLog(shader, base::size(log), nullptr, log); |
| LOG(FATAL) << log; |
| } |
| glAttachShader(program, shader); |
| glDeleteShader(shader); |
| CHECK_EQ(static_cast<int>(glGetError()), GL_NO_ERROR); |
| } |
| |
| void GLSetViewPort(const gfx::Rect& area) { |
| glViewport(area.x(), area.y(), area.width(), area.height()); |
| glScissor(area.x(), area.y(), area.width(), area.height()); |
| } |
| |
| // Helper function to convert from RGBA to RGB. Returns false if any alpha |
| // channel is not 0xff, otherwise true. |
| bool ConvertRGBAToRGB(const std::vector<unsigned char>& rgba, |
| std::vector<unsigned char>* rgb) { |
| size_t num_pixels = rgba.size() / 4; |
| rgb->resize(num_pixels * 3); |
| // Drop the alpha channel, but check as we go that it is all 0xff. |
| bool solid = true; |
| for (size_t i = 0; i < num_pixels; i++) { |
| (*rgb)[3 * i] = rgba[4 * i]; |
| (*rgb)[3 * i + 1] = rgba[4 * i + 1]; |
| (*rgb)[3 * i + 2] = rgba[4 * i + 2]; |
| solid = solid && (rgba[4 * i + 3] == 0xff); |
| } |
| return solid; |
| } |
| |
| } // namespace |
| |
| bool FrameRendererThumbnail::gl_initialized_ = false; |
| |
| FrameRendererThumbnail::FrameRendererThumbnail( |
| const std::vector<std::string>& thumbnail_checksums, |
| const base::FilePath& output_folder) |
| : thumbnail_checksums_(thumbnail_checksums), output_folder_(output_folder) { |
| DETACH_FROM_SEQUENCE(renderer_sequence_checker_); |
| } |
| |
| FrameRendererThumbnail::~FrameRendererThumbnail() { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); |
| |
| if (renderer_task_runner_) { |
| base::WaitableEvent done; |
| renderer_task_runner_->PostTask( |
| FROM_HERE, base::BindOnce(&FrameRendererThumbnail::DestroyTask, |
| base::Unretained(this), &done)); |
| done.Wait(); |
| } |
| } |
| |
| // static |
| std::unique_ptr<FrameRendererThumbnail> FrameRendererThumbnail::Create( |
| const std::vector<std::string> thumbnail_checksums, |
| const base::FilePath& output_folder) { |
| auto frame_renderer = base::WrapUnique( |
| new FrameRendererThumbnail(thumbnail_checksums, output_folder)); |
| frame_renderer->Initialize(); |
| return frame_renderer; |
| } |
| |
| bool FrameRendererThumbnail::AcquireGLContext() { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(renderer_sequence_checker_); |
| |
| return gl_context_->MakeCurrent(gl_surface_.get()); |
| } |
| |
| gl::GLContext* FrameRendererThumbnail::GetGLContext() { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(renderer_sequence_checker_); |
| |
| return gl_context_.get(); |
| } |
| |
| void FrameRendererThumbnail::RenderFrame( |
| scoped_refptr<VideoFrame> video_frame) { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(renderer_sequence_checker_); |
| |
| if (video_frame->metadata().end_of_stream) |
| return; |
| |
| if (!renderer_task_runner_) |
| renderer_task_runner_ = base::ThreadTaskRunnerHandle::Get(); |
| |
| if (thumbnails_texture_id_ == 0u) |
| InitializeThumbnailImageTask(); |
| |
| if (video_frame->visible_rect().size().IsEmpty()) { |
| // This occurs in bitstream buffer in webrtc scenario. |
| DLOG(WARNING) << "Skipping rendering, because visible_rect is empty"; |
| return; |
| } |
| |
| // Find the texture associated with the video frame's mailbox. |
| const gpu::MailboxHolder& mailbox_holder = video_frame->mailbox_holder(0); |
| const gpu::Mailbox& mailbox = mailbox_holder.mailbox; |
| auto it = mailbox_texture_map_.find(mailbox); |
| ASSERT_NE(it, mailbox_texture_map_.end()); |
| |
| RenderThumbnailTask(mailbox_holder.texture_target, it->second); |
| } |
| |
| void FrameRendererThumbnail::WaitUntilRenderingDone() {} |
| |
| scoped_refptr<VideoFrame> FrameRendererThumbnail::CreateVideoFrame( |
| VideoPixelFormat pixel_format, |
| const gfx::Size& texture_size, |
| uint32_t texture_target, |
| uint32_t* texture_id) { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(renderer_sequence_checker_); |
| |
| // Make the GL context current in the case it's not currently yet. |
| AcquireGLContext(); |
| |
| // Create a mailbox. |
| gpu::Mailbox mailbox = gpu::Mailbox::Generate(); |
| gpu::MailboxHolder mailbox_holders[media::VideoFrame::kMaxPlanes]; |
| mailbox_holders[0] = |
| gpu::MailboxHolder(mailbox, gpu::SyncToken(), texture_target); |
| |
| // Create a new video frame associated with the mailbox. |
| base::OnceCallback<void(const gpu::SyncToken&)> mailbox_holder_release_cb = |
| base::BindOnce(&FrameRendererThumbnail::DeleteTextureTask, |
| base::Unretained(this), mailbox); |
| scoped_refptr<VideoFrame> frame = VideoFrame::WrapNativeTextures( |
| pixel_format, mailbox_holders, std::move(mailbox_holder_release_cb), |
| texture_size, gfx::Rect(texture_size), texture_size, base::TimeDelta()); |
| |
| // Create a texture and associate it with the mailbox. |
| *texture_id = CreateTexture(texture_target, texture_size); |
| |
| mailbox_texture_map_.insert(std::make_pair(mailbox, *texture_id)); |
| |
| return frame; |
| } |
| |
| bool FrameRendererThumbnail::ValidateThumbnail() { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); |
| |
| if (!renderer_task_runner_) |
| return false; |
| |
| bool success = false; |
| base::WaitableEvent done; |
| renderer_task_runner_->PostTask( |
| FROM_HERE, base::BindOnce(&FrameRendererThumbnail::ValidateThumbnailTask, |
| base::Unretained(this), &success, &done)); |
| done.Wait(); |
| |
| return success; |
| } |
| |
| void FrameRendererThumbnail::SaveThumbnailTask() { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(renderer_sequence_checker_); |
| |
| // Create the directory tree if it doesn't exist yet. |
| if (!DirectoryExists(output_folder_)) |
| base::CreateDirectory(output_folder_); |
| |
| const std::vector<uint8_t> rgba = ConvertThumbnailToRGBATask(); |
| |
| // Convert raw RGBA into PNG for export. |
| std::vector<unsigned char> png; |
| gfx::PNGCodec::Encode(&rgba[0], gfx::PNGCodec::FORMAT_RGBA, |
| kThumbnailsPageSize, kThumbnailsPageSize.width() * 4, |
| true, std::vector<gfx::PNGCodec::Comment>(), &png); |
| |
| base::FilePath filepath = |
| base::MakeAbsoluteFilePath(output_folder_).Append(kThumbnailFilename); |
| LOG(INFO) << "Saving thumbnails image to " << filepath; |
| |
| base::File thumbnail_file( |
| filepath, base::File::FLAG_CREATE_ALWAYS | base::File::FLAG_WRITE); |
| int num_bytes = |
| thumbnail_file.Write(0u, reinterpret_cast<char*>(&png[0]), png.size()); |
| ASSERT_NE(-1, num_bytes); |
| EXPECT_EQ(static_cast<size_t>(num_bytes), png.size()); |
| } |
| |
| void FrameRendererThumbnail::Initialize() { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_); |
| |
| // Initialize GL rendering and create GL context. |
| if (!gl_initialized_) { |
| if (!gl::init::InitializeGLOneOff()) |
| LOG(FATAL) << "Could not initialize GL"; |
| gl_initialized_ = true; |
| } |
| gl_surface_ = gl::init::CreateOffscreenGLSurface(gfx::Size()); |
| gl_context_ = gl::init::CreateGLContext(nullptr, gl_surface_.get(), |
| gl::GLContextAttribs()); |
| } |
| |
| void FrameRendererThumbnail::DestroyTask(base::WaitableEvent* done) { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(renderer_sequence_checker_); |
| DCHECK(mailbox_texture_map_.empty()); |
| |
| DestroyThumbnailImageTask(); |
| |
| // Release the |gl_context_| so it can be destroyed on the same thread it was |
| // created on. Otherwise random crashes might occur as not all resources are |
| // freed correctly. |
| gl_context_->ReleaseCurrent(gl_surface_.get()); |
| |
| done->Signal(); |
| } |
| |
| void FrameRendererThumbnail::InitializeThumbnailImageTask() { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(renderer_sequence_checker_); |
| |
| GLint max_texture_size; |
| glGetIntegerv(GL_MAX_TEXTURE_SIZE, &max_texture_size); |
| CHECK_GE(max_texture_size, kThumbnailsPageSize.width()); |
| CHECK_GE(max_texture_size, kThumbnailsPageSize.height()); |
| |
| thumbnails_fbo_size_ = kThumbnailsPageSize; |
| thumbnail_size_ = kThumbnailSize; |
| |
| glGenFramebuffersEXT(1, &thumbnails_fbo_id_); |
| glGenTextures(1, &thumbnails_texture_id_); |
| glBindTexture(GL_TEXTURE_2D, thumbnails_texture_id_); |
| glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, thumbnails_fbo_size_.width(), |
| thumbnails_fbo_size_.height(), 0, GL_RGB, |
| GL_UNSIGNED_SHORT_5_6_5, nullptr); |
| glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); |
| glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); |
| glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); |
| glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); |
| glBindTexture(GL_TEXTURE_2D, 0); |
| |
| glBindFramebufferEXT(GL_FRAMEBUFFER, thumbnails_fbo_id_); |
| glFramebufferTexture2DEXT(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, |
| thumbnails_texture_id_, 0); |
| |
| GLenum fb_status = glCheckFramebufferStatusEXT(GL_FRAMEBUFFER); |
| CHECK(fb_status == GL_FRAMEBUFFER_COMPLETE) << fb_status; |
| glClearColor(0.0f, 0.0f, 0.0f, 1.0f); |
| glClear(GL_COLOR_BUFFER_BIT); |
| glBindFramebufferEXT(GL_FRAMEBUFFER, |
| gl_surface_->GetBackingFramebufferObject()); |
| |
| // These vertices and texture coords map (0,0) in the texture to the bottom |
| // left of the viewport. Since we get the video frames with the the top left |
| // at (0,0) we need to flip the texture y coordinate in the vertex shader for |
| // this to be rendered the right way up. In the case of thumbnail rendering we |
| // use the same vertex shader to render the FBO to the screen, where we do not |
| // want this flipping. Vertices are 2 floats for position and 2 floats for |
| // texcoord each. |
| const float kVertices[] = { |
| -1, 1, 0, 1, // Vertex 0 |
| -1, -1, 0, 0, // Vertex 1 |
| 1, 1, 1, 1, // Vertex 2 |
| 1, -1, 1, 0, // Vertex 3 |
| }; |
| const GLvoid* kVertexPositionOffset = 0; |
| const GLvoid* kVertexTexcoordOffset = |
| reinterpret_cast<GLvoid*>(sizeof(float) * 2); |
| const GLsizei kVertexStride = sizeof(float) * 4; |
| |
| glGenBuffersARB(1, &vertex_buffer_); |
| glBindBuffer(GL_ARRAY_BUFFER, vertex_buffer_); |
| glBufferData(GL_ARRAY_BUFFER, sizeof(kVertices), kVertices, GL_STATIC_DRAW); |
| |
| program_ = glCreateProgram(); |
| CreateShader(program_, GL_VERTEX_SHADER, kVertexShader, |
| base::size(kVertexShader)); |
| CreateShader(program_, GL_FRAGMENT_SHADER, kFragmentShader, |
| base::size(kFragmentShader)); |
| glLinkProgram(program_); |
| GLint result = GL_FALSE; |
| glGetProgramiv(program_, GL_LINK_STATUS, &result); |
| if (!result) { |
| constexpr GLsizei kLogBufferSize = 4096; |
| char log[kLogBufferSize]; |
| glGetShaderInfoLog(program_, kLogBufferSize, nullptr, log); |
| LOG(FATAL) << log; |
| } |
| glUseProgram(program_); |
| glDeleteProgram(program_); |
| |
| glUniform1i(glGetUniformLocation(program_, "tex_flip"), 0); |
| glUniform1i(glGetUniformLocation(program_, "tex"), 0); |
| GLint tex_external = glGetUniformLocation(program_, "tex_external"); |
| if (tex_external != -1) { |
| glUniform1i(tex_external, 1); |
| } |
| GLint pos_location = glGetAttribLocation(program_, "in_pos"); |
| glEnableVertexAttribArray(pos_location); |
| glVertexAttribPointer(pos_location, 2, GL_FLOAT, GL_FALSE, kVertexStride, |
| kVertexPositionOffset); |
| GLint tc_location = glGetAttribLocation(program_, "in_tc"); |
| glEnableVertexAttribArray(tc_location); |
| glVertexAttribPointer(tc_location, 2, GL_FLOAT, GL_FALSE, kVertexStride, |
| kVertexTexcoordOffset); |
| |
| // Unbind the vertex buffer |
| glBindBuffer(GL_ARRAY_BUFFER, 0); |
| } |
| |
| void FrameRendererThumbnail::DestroyThumbnailImageTask() { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(renderer_sequence_checker_); |
| |
| glDeleteTextures(1, &thumbnails_texture_id_); |
| glDeleteFramebuffersEXT(1, &thumbnails_fbo_id_); |
| glDeleteBuffersARB(1, &vertex_buffer_); |
| |
| thumbnails_texture_id_ = 0u; |
| thumbnails_fbo_id_ = 0u; |
| vertex_buffer_ = 0u; |
| } |
| |
| void FrameRendererThumbnail::RenderThumbnailTask(uint32_t texture_target, |
| uint32_t texture_id) { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(renderer_sequence_checker_); |
| |
| const int width = thumbnail_size_.width(); |
| const int height = thumbnail_size_.height(); |
| const int thumbnails_in_row = thumbnails_fbo_size_.width() / width; |
| const int thumbnails_in_column = thumbnails_fbo_size_.height() / height; |
| const int row = (frame_count_ / thumbnails_in_row) % thumbnails_in_column; |
| const int col = frame_count_ % thumbnails_in_row; |
| gfx::Rect area(col * width, row * height, width, height); |
| |
| glUniform1i(glGetUniformLocation(program_, "tex_flip"), 0); |
| glBindFramebufferEXT(GL_FRAMEBUFFER, thumbnails_fbo_id_); |
| GLSetViewPort(area); |
| RenderTexture(texture_target, texture_id); |
| glBindFramebufferEXT(GL_FRAMEBUFFER, |
| gl_surface_->GetBackingFramebufferObject()); |
| // We need to flush the GL commands before returning the thumbnail texture to |
| // the decoder. |
| glFlush(); |
| |
| ++frame_count_; |
| } |
| |
| const std::vector<uint8_t> |
| FrameRendererThumbnail::ConvertThumbnailToRGBATask() { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(renderer_sequence_checker_); |
| |
| std::vector<uint8_t> rgba; |
| const size_t num_pixels = thumbnails_fbo_size_.GetArea(); |
| rgba.resize(num_pixels * 4); |
| glBindFramebufferEXT(GL_FRAMEBUFFER, thumbnails_fbo_id_); |
| glPixelStorei(GL_PACK_ALIGNMENT, 1); |
| // We can only count on GL_RGBA/GL_UNSIGNED_BYTE support. |
| glReadPixels(0, 0, thumbnails_fbo_size_.width(), |
| thumbnails_fbo_size_.height(), GL_RGBA, GL_UNSIGNED_BYTE, |
| &(rgba)[0]); |
| glBindFramebufferEXT(GL_FRAMEBUFFER, |
| gl_surface_->GetBackingFramebufferObject()); |
| |
| return rgba; |
| } |
| |
| void FrameRendererThumbnail::ValidateThumbnailTask(bool* success, |
| base::WaitableEvent* done) { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(renderer_sequence_checker_); |
| |
| const std::vector<uint8_t> rgba = ConvertThumbnailToRGBATask(); |
| |
| // Convert the thumbnail from RGBA to RGB. |
| std::vector<uint8_t> rgb; |
| EXPECT_EQ(ConvertRGBAToRGB(rgba, &rgb), true) |
| << "RGBA frame has incorrect alpha"; |
| |
| // Calculate the thumbnail's checksum and compare it to golden values. |
| std::string md5_string = base::MD5String( |
| base::StringPiece(reinterpret_cast<char*>(&rgb[0]), rgb.size())); |
| *success = base::Contains(thumbnail_checksums_, md5_string); |
| |
| // If validation failed, write the thumbnail image to disk. |
| if (!(*success)) |
| SaveThumbnailTask(); |
| |
| done->Signal(); |
| } |
| |
| void FrameRendererThumbnail::DeleteTextureTask(const gpu::Mailbox& mailbox, |
| const gpu::SyncToken&) { |
| DCHECK_CALLED_ON_VALID_SEQUENCE(renderer_sequence_checker_); |
| |
| auto it = mailbox_texture_map_.find(mailbox); |
| ASSERT_NE(it, mailbox_texture_map_.end()); |
| uint32_t texture_id = it->second; |
| mailbox_texture_map_.erase(mailbox); |
| |
| DeleteTexture(texture_id); |
| } |
| |
| } // namespace test |
| } // namespace media |