blob: feff4f746bc995089cafd5893bf437de75ee68eb [file] [log] [blame]
// Copyright 2015 The Cobalt Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "cobalt/loader/image/jpeg_image_decoder.h"
#include <algorithm>
#include "base/logging.h"
#include "base/trace_event/trace_event.h"
#include "cobalt/base/console_log.h"
#include "nb/memory_scope.h"
#include "third_party/libjpeg/jpegint.h"
namespace cobalt {
namespace loader {
namespace image {
namespace {
const JDIMENSION kInvalidHeight = 0xFFFFFF;
const JDIMENSION kDctScaleSize = 8;
JDIMENSION AlignUp(JDIMENSION value, JDIMENSION alignment) {
return (value + alignment - 1) / alignment * alignment;
}
bool CanDecodeIntoYUV(const jpeg_decompress_struct& decompress_info) {
auto comp_infos = decompress_info.cur_comp_info;
// Only images encoded to YCbCr in three planes are supported.
if (decompress_info.jpeg_color_space != JCS_YCbCr ||
decompress_info.comps_in_scan != 3) {
return false;
}
// Ensure that it is either YUV 420 or YUV 444.
bool is_yuv_420 =
comp_infos[0]->h_samp_factor == 2 && comp_infos[0]->v_samp_factor == 2 &&
comp_infos[1]->h_samp_factor == 1 && comp_infos[1]->v_samp_factor == 1 &&
comp_infos[2]->h_samp_factor == 1 && comp_infos[2]->v_samp_factor == 1 &&
decompress_info.max_h_samp_factor == 2 &&
decompress_info.max_v_samp_factor == 2;
bool is_yuv_444 =
comp_infos[0]->h_samp_factor == 1 && comp_infos[0]->v_samp_factor == 1 &&
comp_infos[1]->h_samp_factor == 1 && comp_infos[1]->v_samp_factor == 1 &&
comp_infos[2]->h_samp_factor == 1 && comp_infos[2]->v_samp_factor == 1 &&
decompress_info.max_h_samp_factor == 1 &&
decompress_info.max_v_samp_factor == 1;
if (!is_yuv_420 && !is_yuv_444) {
return false;
}
// The dimension of the image may not be a multiple of the sample factors,
// this can happen when the sample factors are not 1. In such case the
// mapping of the u/v plane won't be even, because the mapping of the last
// vertical line on the u/v plane will be different than the other vertical
// lines. Our renderer cannot handle this properly so we return false in this
// case.
if (decompress_info.image_width % decompress_info.max_h_samp_factor != 0 ||
decompress_info.image_height % decompress_info.max_v_samp_factor != 0) {
return false;
}
// Ensure that the DCT block size is expected.
return comp_infos[0]->DCT_scaled_size == kDctScaleSize &&
comp_infos[1]->DCT_scaled_size == kDctScaleSize &&
comp_infos[2]->DCT_scaled_size == kDctScaleSize;
}
void ErrorManagerExit(j_common_ptr common_ptr) {
// Returns the control to the setjmp point. The buffer which is filled by a
// previous call to setjmp that contains information to store the environment
// to that point.
jmp_buf* buffer = static_cast<jmp_buf*>(common_ptr->client_data);
longjmp(*buffer, 1);
}
void SourceManagerInitSource(j_decompress_ptr decompress_ptr) {
// no-op.
}
boolean SourceManagerFillInputBuffer(j_decompress_ptr decompress_ptr) {
// Normally, this function is called when we need to read more of the encoded
// buffer into memory and a return false indicates that we have no data to
// supply yet, but in our case, the encoded buffer is always in memory, so
// this should never get called unless there is an error, in which case we
// return false to indicate that.
return false;
}
boolean SourceManagerResyncToRestart(j_decompress_ptr decompress_ptr,
int desired) {
return false;
}
void SourceManagerTermSource(j_decompress_ptr decompress_ptr) {
// no-op.
}
void SourceManagerSkipInputData(j_decompress_ptr decompress_ptr,
long num_bytes) { // NOLINT(runtime/int)
// Since all our data that is required to be decoded should be in the buffer
// already, trying to skip beyond it means that there is some kind of error or
// corrupt input data. We acknowledge these error cases by setting
// |bytes_in_buffer| to 0 which will result in a call to
// SourceManagerFillInputBuffer() which will return false to indicate that an
// error has occurred.
size_t bytes_to_skip = std::min(decompress_ptr->src->bytes_in_buffer,
static_cast<size_t>(num_bytes));
decompress_ptr->src->bytes_in_buffer -= bytes_to_skip;
decompress_ptr->src->next_input_byte += bytes_to_skip;
}
} // namespace
JPEGImageDecoder::JPEGImageDecoder(
render_tree::ResourceProvider* resource_provider,
const base::DebuggerHooks& debugger_hooks,
bool allow_image_decoding_to_multi_plane)
: ImageDataDecoder(resource_provider, debugger_hooks),
allow_image_decoding_to_multi_plane_(
allow_image_decoding_to_multi_plane) {
TRACE_EVENT0("cobalt::loader::image", "JPEGImageDecoder::JPEGImageDecoder()");
TRACK_MEMORY_SCOPE("Rendering");
memset(&info_, 0, sizeof(info_));
memset(&source_manager_, 0, sizeof(source_manager_));
info_.err = jpeg_std_error(&error_manager_);
error_manager_.error_exit = &ErrorManagerExit;
// Allocate and initialize JPEG decompression object.
jpeg_create_decompress(&info_);
info_.src = &source_manager_;
// Set up callback functions.
// Even some of them are no-op, we have to set them up for jpeg lib.
source_manager_.init_source = SourceManagerInitSource;
source_manager_.fill_input_buffer = SourceManagerFillInputBuffer;
source_manager_.skip_input_data = SourceManagerSkipInputData;
source_manager_.resync_to_restart = SourceManagerResyncToRestart;
source_manager_.term_source = SourceManagerTermSource;
}
JPEGImageDecoder::~JPEGImageDecoder() {
// Deallocate a JPEG decompression object.
jpeg_destroy_decompress(&info_);
}
size_t JPEGImageDecoder::DecodeChunkInternal(const uint8* data,
size_t input_byte) {
TRACK_MEMORY_SCOPE("Rendering");
TRACE_EVENT0("cobalt::loader::image",
"JPEGImageDecoder::DecodeChunkInternal()");
// |client_data| is available for use by application.
jmp_buf jump_buffer;
info_.client_data = &jump_buffer;
// int setjmp(jmp_buf env) saves the current environment (ths program state),
// at some point of program execution, into a platform-specific data
// structure (jmp_buf) that can be used at some later point of program
// execution by longjmp to restore the program state to that saved by setjmp
// into jmp_buf. This process can be imagined to be a "jump" back to the point
// of program execution where setjmp saved the environment. The return value
// from setjmp indicates whether control reached that point normally or from a
// call to longjmp. If the return is from a direct invocation, setjmp returns
// 0. If the return is from a call to longjmp, setjmp returns a nonzero value.
MSVC_PUSH_DISABLE_WARNING(4611);
// warning C4611: interaction between '_setjmp' and C++ object destruction is
// non-portable.
if (setjmp(jump_buffer)) {
// image data is empty.
DLOG(WARNING) << "Decoder encounters an error.";
set_state(kError);
return 0;
}
MSVC_POP_WARNING();
// Next byte to read from buffer.
info_.src->next_input_byte = reinterpret_cast<const JOCTET*>(data);
// Number of bytes remaining in buffer.
info_.src->bytes_in_buffer = input_byte;
if (state() == kWaitingForHeader) {
if (!ReadHeader()) {
// Data is not enough for decoding the header.
return 0;
}
if (!StartDecompress()) {
return 0;
}
set_state(kReadLines);
}
if (state() == kReadLines) {
if (info_.buffered_image) { // Progressive JPEG.
if (!DecodeProgressiveJPEG()) {
// The size of undecoded bytes is info_.src->bytes_in_buffer.
return input_byte - info_.src->bytes_in_buffer;
}
} else if (!ReadLines()) { // Baseline sequential JPEG.
// The size of undecoded bytes is info_.src->bytes_in_buffer.
return input_byte - info_.src->bytes_in_buffer;
}
if (!jpeg_finish_decompress(&info_)) {
// In this case, we did read all the rows, so we don't really have to
// treat this as an error.
DLOG(WARNING) << "Data source requests suspension of the decompressor.";
}
set_state(kDone);
}
return input_byte;
}
scoped_refptr<Image> JPEGImageDecoder::FinishInternal() {
if (state() != kDone) {
decoded_image_data_.reset();
raw_image_memory_.reset();
return NULL;
}
SB_DCHECK(output_format_ != kOutputFormatInvalid);
if (output_format_ == kOutputFormatRGBA ||
output_format_ == kOutputFormatBGRA) {
SB_DCHECK(decoded_image_data_);
return CreateStaticImage(std::move(decoded_image_data_));
}
SB_DCHECK(output_format_ == kOutputFormatYUV);
SB_DCHECK(raw_image_memory_);
render_tree::MultiPlaneImageDataDescriptor descriptor(
render_tree::kMultiPlaneImageFormatYUV3PlaneBT601FullRange);
auto uv_plane_width = y_plane_width_ / h_sample_factor_;
auto uv_plane_height = y_plane_height_ / v_sample_factor_;
auto y_plane_size = y_plane_width_ * y_plane_height_;
auto uv_plane_size = uv_plane_width * uv_plane_height;
math::Size plane_size(info_.image_width, info_.image_height);
descriptor.AddPlane(
0, render_tree::ImageDataDescriptor(
plane_size, render_tree::kPixelFormatY8,
render_tree::kAlphaFormatPremultiplied, y_plane_width_));
plane_size.SetSize(plane_size.width() / h_sample_factor_,
plane_size.height() / v_sample_factor_);
descriptor.AddPlane(y_plane_size, render_tree::ImageDataDescriptor(
plane_size, render_tree::kPixelFormatU8,
render_tree::kAlphaFormatPremultiplied,
uv_plane_width));
descriptor.AddPlane(
y_plane_size + uv_plane_size,
render_tree::ImageDataDescriptor(plane_size, render_tree::kPixelFormatV8,
render_tree::kAlphaFormatPremultiplied,
uv_plane_width));
auto image = resource_provider()->CreateMultiPlaneImageFromRawMemory(
std::move(raw_image_memory_), descriptor);
SB_DCHECK(image);
return new StaticImage(image);
}
bool JPEGImageDecoder::ReadHeader() {
TRACK_MEMORY_SCOPE("Rendering");
TRACE_EVENT0("cobalt::loader::image", "JPEGImageDecoder::ReadHeader()");
if (jpeg_read_header(&info_, true) == JPEG_SUSPENDED) {
// Since |jpeg_read_header| doesn't have enough data, go back to the state
// before reading the header.
info_.global_state = DSTATE_START;
return false;
}
if (allow_image_decoding_to_multi_plane_ && CanDecodeIntoYUV(info_)) {
output_format_ = kOutputFormatYUV;
} else if (pixel_format() == render_tree::kPixelFormatRGBA8) {
output_format_ = kOutputFormatRGBA;
} else if (pixel_format() == render_tree::kPixelFormatBGRA8) {
output_format_ = kOutputFormatBGRA;
} else {
NOTREACHED() << "Unsupported pixel format: " << pixel_format();
}
if (output_format_ == kOutputFormatYUV) {
info_.out_color_space = JCS_YCbCr;
// Enable raw data output to avoid any copying.
info_.raw_data_out = TRUE;
auto y_info = info_.cur_comp_info[0];
h_sample_factor_ = y_info->h_samp_factor;
v_sample_factor_ = y_info->v_samp_factor;
y_plane_width_ = AlignUp(y_info->width_in_blocks,
static_cast<JDIMENSION>(y_info->h_samp_factor)) *
kDctScaleSize;
y_plane_height_ = AlignUp(y_info->height_in_blocks,
static_cast<JDIMENSION>(y_info->v_samp_factor)) *
kDctScaleSize;
// Raw read mode requires that the output data is aligned to dct block size.
auto aligned_size = math::Size(static_cast<int>(y_plane_width_),
static_cast<int>(y_plane_height_));
auto y_plane_size_in_bytes = aligned_size.width() * aligned_size.height();
auto uv_plane_size_in_bytes =
y_plane_size_in_bytes / h_sample_factor_ / v_sample_factor_;
raw_image_memory_ = resource_provider()->AllocateRawImageMemory(
y_plane_size_in_bytes + uv_plane_size_in_bytes * 2, sizeof(void*));
return raw_image_memory_ != NULL;
}
// TODO: switch libjpeg version to support JCS_RGBA_8888 output.
info_.out_color_space = JCS_RGB;
decoded_image_data_ =
AllocateImageData(math::Size(static_cast<int>(info_.image_width),
static_cast<int>(info_.image_height)),
false);
return decoded_image_data_ != NULL;
}
bool JPEGImageDecoder::StartDecompress() {
TRACK_MEMORY_SCOPE("Rendering");
TRACE_EVENT0("cobalt::loader::image", "JPEGImageDecoder::StartDecompress()");
// jpeg_has_multiple_scans() returns TRUE if the incoming image file has more
// than one scan.
info_.buffered_image = jpeg_has_multiple_scans(&info_);
// Compute output image dimensions
jpeg_calc_output_dimensions(&info_);
if (!jpeg_start_decompress(&info_)) {
LOG(WARNING) << "Start decompressor failed.";
set_state(kError);
return false;
}
return true;
}
// A baseline sequential JPEG is stored as one top-to-bottom scan of the image.
// Progressive JPEG divides the file into a series of scans. It is starting with
// a very low quality image, and then following scans gradually improve the
// quality.
// TODO: support displaying the low resolution image while decoding the
// progressive JPEG.
bool JPEGImageDecoder::DecodeProgressiveJPEG() {
TRACK_MEMORY_SCOPE("Rendering");
TRACE_EVENT0("cobalt::loader::image",
"JPEGImageDecoder::DecodeProgressiveJPEG()");
int status;
do {
// |jpeg_consume_input| decodes the input data as it arrives.
status = jpeg_consume_input(&info_);
} while (status == JPEG_REACHED_SOS || status == JPEG_ROW_COMPLETED ||
status == JPEG_SCAN_COMPLETED);
while (info_.output_scanline == kInvalidHeight ||
info_.output_scanline <= info_.output_height) {
if (info_.output_scanline == 0) {
// Initialize for an output pass in buffered-image mode.
// The |input_scan_number| indicates the scan of the image to be
// processed.
if (!jpeg_start_output(&info_, info_.input_scan_number)) {
// Decompression is suspended.
return false;
}
}
if (info_.output_scanline == kInvalidHeight) {
// Recover from the previous set flag.
info_.output_scanline = 0;
}
if (!ReadLines()) {
if (info_.output_scanline == 0) {
// Data is not enough for one line scan, so flag the |output_scanline|
// to make sure that we don't call |jpeg_start_output| multiple times
// for the same scan.
info_.output_scanline = kInvalidHeight;
}
return false;
}
if (info_.output_scanline >= info_.output_height) {
// Finish up after an output pass in buffered-image mode.
if (!jpeg_finish_output(&info_)) {
// Decompression is suspended due to
// input_scan_number <= output_scan_number and EOI is not reached.
// The suspension will be recovered when more data are coming.
return false;
}
// The output scan number is the notional scan being processed by the
// output side. The decompressor will not allow output scan number to get
// ahead of input scan number.
DCHECK_GE(info_.input_scan_number, info_.output_scan_number);
// This scan pass is done, so reset the output scanline.
info_.output_scanline = 0;
if (info_.input_scan_number == info_.output_scan_number) {
// No more scan needed at this point.
break;
}
}
}
// |jpeg_input_complete| tests for the end of image.
if (!jpeg_input_complete(&info_)) {
return false;
}
return true;
}
// Responsible for swizzeling and alpha-premultiplying a row of pixels.
template <int r, int g, int b, int a>
void FillRow(int width, uint8* dest, JSAMPLE* source) {
for (int x = 0; x < width; ++x, source += 3, dest += 4) {
dest[r] = source[0];
dest[g] = source[1];
dest[b] = source[2];
dest[a] = 0xFF;
}
}
bool JPEGImageDecoder::ReadYUVLines() {
DCHECK(output_format_ == kOutputFormatYUV);
TRACK_MEMORY_SCOPE("Rendering");
TRACE_EVENT0("cobalt::loader::image", "JPEGImageDecoder::ReadYUVLines()");
while (info_.output_scanline < info_.output_height) {
JSAMPROW y[kDctScaleSize * 2], u[kDctScaleSize], v[kDctScaleSize];
JSAMPARRAY planes[3] = {y, u, v};
auto offset = info_.output_scanline * y_plane_width_;
auto y_plane_size = y_plane_width_ * y_plane_height_;
auto uv_plane_size = y_plane_size / (h_sample_factor_ * v_sample_factor_);
uint8* y_plane_addr = raw_image_memory_->GetMemory() + offset;
uint8* u_plane_addr = raw_image_memory_->GetMemory() + y_plane_size +
offset / (h_sample_factor_ * v_sample_factor_);
uint8* v_plane_addr = raw_image_memory_->GetMemory() + y_plane_size +
uv_plane_size +
offset / (h_sample_factor_ * v_sample_factor_);
for (JDIMENSION i = 0; i < kDctScaleSize * v_sample_factor_; ++i) {
y[i] = y_plane_addr + y_plane_width_ * i;
}
for (JDIMENSION i = 0; i < kDctScaleSize; ++i) {
u[i] = u_plane_addr + y_plane_width_ / h_sample_factor_ * i;
v[i] = v_plane_addr + y_plane_width_ / h_sample_factor_ * i;
}
auto size =
jpeg_read_raw_data(&info_, planes, kDctScaleSize * v_sample_factor_);
if (size != kDctScaleSize * v_sample_factor_) {
return false;
}
}
return true;
}
bool JPEGImageDecoder::ReadRgbaOrGbraLines() {
DCHECK(output_format_ == kOutputFormatRGBA ||
output_format_ == kOutputFormatBGRA);
TRACK_MEMORY_SCOPE("Rendering");
TRACE_EVENT0("cobalt::loader::image",
"JPEGImageDecoder::ReadRgbaOrGbraLines()");
// Creation of 2-D sample arrays which is for one row.
// See the comments in jmemmgr.c.
JSAMPARRAY buffer = (*info_.mem->alloc_sarray)(
(j_common_ptr)&info_, JPOOL_IMAGE, info_.output_width * 4, 1);
while (info_.output_scanline < info_.output_height) {
// |info_.output_scanline| would be increased 1 after reading one row.
int row_index = static_cast<int>(info_.output_scanline);
// jpeg_read_scanlines() returns up to the maximum number of scanlines of
// decompressed image data. This may be less than the number requested in
// cases such as bottom of image, data source suspension, and operating
// modes that emit multiple scanlines at a time. Image data should be
// returned in top-to-bottom scanline order.
// TODO: Investigate the performance improvements by processing
// multiple pixel rows. It may have performance advantage to use values
// larger than 1. For example, JPEG images often use 4:2:0 downsampling, and
// in that case libjpeg needs to make an additional copy of half the image
// pixels(see merged_2v_upsample()).
if (jpeg_read_scanlines(&info_, buffer, 1) != 1) {
return false;
}
// Write the decoded row pixels to image data.
uint8* pixel_data =
decoded_image_data_->GetMemory() +
decoded_image_data_->GetDescriptor().pitch_in_bytes * row_index;
JSAMPLE* sample_buffer = *buffer;
switch (output_format_) {
case kOutputFormatRGBA: {
FillRow<0, 1, 2, 3>(static_cast<int>(info_.output_width), pixel_data,
sample_buffer);
} break;
case kOutputFormatBGRA: {
FillRow<2, 1, 0, 3>(static_cast<int>(info_.output_width), pixel_data,
sample_buffer);
} break;
case kOutputFormatInvalid:
case kOutputFormatYUV: {
NOTREACHED();
} break;
}
}
return true;
}
bool JPEGImageDecoder::ReadLines() {
return output_format_ == kOutputFormatYUV ? ReadYUVLines()
: ReadRgbaOrGbraLines();
}
} // namespace image
} // namespace loader
} // namespace cobalt